Add JavaScript source map symbolication

docs/docs/source-maps.md

@@ -0,0 +1,57 @@
+---
+title: "JavaScript Source Maps"
+---
+
+# JavaScript Source Maps
+
+Exceptionless uses source maps to turn minified JavaScript stack frames into the original file names, line and column numbers, and function names. Symbolication happens before an event is assigned to a stack, so readable function names also improve stack grouping.
+
+## Automatic discovery
+
+No domain allowlist or project setup is required for public source maps. When an error contains an absolute HTTPS JavaScript URL, Exceptionless checks the generated file's `SourceMap` or `X-SourceMap` response header and its `sourceMappingURL` comment. If neither is present, it also checks the conventional `<generated-file>.map` URL.
+
+Downloaded maps are validated and cached in project-scoped file storage. Automatically downloaded maps are revalidated after one hour so a stable generated-file URL cannot retain a map from an older deployment indefinitely. If refresh fails, Exceptionless leaves the generated frame unchanged instead of risking a misleading stack trace from the stale map.
+
+Exceptionless only makes anonymous HTTPS requests on the standard port to public network addresses. Redirects and every resolved address are revalidated. New automatic discoveries use plan-aware limits per client key, project, and organization; outbound requests are also limited per destination, IP address, and cluster. Free plans allow five new discoveries per client key and ten per project or organization in each 15-minute window by default. Paid plans have higher limits. Failed URLs are cached for 15 minutes, duplicate discoveries share the same in-flight work, and refreshes use a separate smaller outbound budget. If any limit is reached, Exceptionless leaves the generated frame unchanged without rejecting or delaying the event.
+
+Downloads also have time, redirect, size, and local and cluster-wide concurrency limits. Parsed maps use a bounded in-memory cache. Manual and deployment uploads do not consume automatic-discovery quotas. Self-hosted installations can tune these safeguards under the `SourceMaps` configuration section, including `AutoDownloadRateLimitPeriodMinutes`, `MaximumAutoDiscoveriesPerFreeClientKey`, `MaximumAutoDiscoveriesPerProject`, `MaximumAutoDownloadRequestsPerDestination`, `MaximumAutoRefreshRequestsGlobally`, `AutoDownloadRefreshIntervalMinutes`, `ParsedSourceMapCacheLifetimeMinutes`, and `MaximumParsedSourceMapCacheSize`.
+
+## Uploading a source map
+
+Upload a map when it is private or is not deployed next to the generated JavaScript:
+
+1. Open the project and select **Source Maps** under **Project Settings**.
+2. Enter the exact absolute URL that appears in the generated stack frame, including any path or query string used to identify the build.
+3. Select the corresponding source map and upload it.
+
+Uploading another map for the same generated file URL replaces the previous map. Uploaded and automatically discovered maps appear together on the Source Maps page and can be deleted there.
+
+### Uploading during deployment
+
+For build automation, create a project-scoped token that has only the `source-maps:write` scope. Set `EXCEPTIONLESS_SERVER_URL` to `https://be.exceptionless.io` for the hosted service or to the root URL of your self-hosted installation. Create the token once with a user-scoped token, then store the returned `id` as a protected CI/CD secret:
+
+```shell
+curl --fail-with-body --request POST \
+  "${EXCEPTIONLESS_SERVER_URL}/api/v2/projects/${EXCEPTIONLESS_PROJECT_ID}/tokens" \
+  --header "Authorization: Bearer ${EXCEPTIONLESS_USER_TOKEN}" \
+  --header "Content-Type: application/json" \
+  --data "{\"organization_id\":\"${EXCEPTIONLESS_ORGANIZATION_ID}\",\"project_id\":\"${EXCEPTIONLESS_PROJECT_ID}\",\"scopes\":[\"source-maps:write\"],\"notes\":\"CI source map uploads\"}"
+```
+
+Upload each map after the generated JavaScript has been deployed. The `generated_file_url` must be the exact absolute URL that will appear in stack frames:
+
+```shell
+curl --fail-with-body --request POST \
+  "${EXCEPTIONLESS_SERVER_URL}/api/v2/projects/${EXCEPTIONLESS_PROJECT_ID}/source-maps" \
+  --header "Authorization: Bearer ${EXCEPTIONLESS_SOURCE_MAP_TOKEN}" \
+  --form "generated_file_url=https://cdn.example.com/assets/app.a1b2c3.js" \
+  --form "file=@dist/assets/app.a1b2c3.js.map;type=application/json"
+```
+
+The upload token is accepted only for its assigned project and cannot read events, manage the project, list source maps, or use ordinary user APIs. Do not use a normal client API key for source map uploads because client keys are commonly embedded in distributed applications.
+
+Source maps must use the version 3 flat-map format. Indexed source maps with a `sections` property and authenticated automatic downloads are planned follow-up capabilities; private maps can be uploaded in the meantime.
+
+## Deployment guidance
+
+Generate source maps as part of the same build that produces the minified JavaScript. Content-hashed generated file names are preferred because the generated URL then identifies a specific build. You can publish the `.map` file for zero-configuration discovery or keep it private and upload it to Exceptionless during deployment.

src/Exceptionless.Core/Authorization/AuthorizationRoles.cs

@@ -12,12 +12,14 @@ public static class AuthorizationRoles
     public const string ProjectsReadPolicy = nameof(ProjectsReadPolicy);
     public const string StacksReadPolicy = nameof(StacksReadPolicy);
     public const string StacksWritePolicy = nameof(StacksWritePolicy);
+    public const string SourceMapsWritePolicy = nameof(SourceMapsWritePolicy);
     public const string EventsReadPolicy = nameof(EventsReadPolicy);
     public const string McpRead = "mcp:read";
     public const string ProjectsRead = "projects:read";
     public const string StacksRead = "stacks:read";
     public const string StacksWrite = "stacks:write";
+    public const string SourceMapsWrite = "source-maps:write";
     public const string EventsRead = "events:read";
     public const string OfflineAccess = "offline_access";
-    public static readonly ISet<string> AllScopes = new HashSet<string>([Client, User, GlobalAdmin]);
+    public static readonly ISet<string> AllScopes = new HashSet<string>([Client, User, GlobalAdmin, SourceMapsWrite]);
 }

src/Exceptionless.Core/Bootstrapper.cs

@@ -25,6 +25,7 @@
 using Exceptionless.Core.Seed;
 using Exceptionless.Core.Serialization;
 using Exceptionless.Core.Services;
+using Exceptionless.Core.Services.SourceMaps;
 using Exceptionless.Core.Utility;
 using Exceptionless.Core.Validation;
 using Foundatio.Caching;
@@ -193,6 +194,19 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
                 AllowAutoRedirect = false,
                 ConnectCallback = ConnectToPublicAddressAsync
             });
+        services.AddSingleton<SourceMapRequestThrottle>();
+        services.AddHttpClient(SourceMapService.HttpClientName)
+            .ConfigurePrimaryHttpMessageHandler(serviceProvider => new SocketsHttpHandler
+            {
+                ActivityHeadersPropagator = null,
+                AllowAutoRedirect = false,
+                AutomaticDecompression = DecompressionMethods.All,
+                ConnectCallback = serviceProvider.GetRequiredService<SourceMapRequestThrottle>().ConnectToPublicAddressAsync,
+                PreAuthenticate = false,
+                UseCookies = false,
+                UseProxy = false
+            });
+        services.AddSingleton<SourceMapService>();
         services.AddSingleton<OAuthService>();
         services.AddSingleton<UsageService>();
         services.AddSingleton<SlackService>();
@@ -223,7 +237,7 @@ private static async ValueTask<Stream> ConnectToPublicAddressAsync(SocketsHttpCo
             }
         }
 
-        throw new HttpRequestException($"OAuth client metadata host '{context.DnsEndPoint.Host}' did not resolve to a reachable public address.", lastException);
+        throw new HttpRequestException($"Host '{context.DnsEndPoint.Host}' did not resolve to a reachable public address.", lastException);
     }
 
     public static void LogConfiguration(IServiceProvider serviceProvider, AppOptions appOptions, ILogger logger)

src/Exceptionless.Core/Configuration/AppOptions.cs

@@ -80,6 +80,7 @@ public class AppOptions
     public StripeOptions StripeOptions { get; internal set; } = null!;
     public AuthOptions AuthOptions { get; internal set; } = null!;
     public OAuthServerOptions OAuthServerOptions { get; internal set; } = null!;
+    public SourceMapOptions SourceMapOptions { get; internal set; } = null!;
 
     public static AppOptions ReadFromConfiguration(IConfiguration config)
     {
@@ -133,6 +134,7 @@ public static AppOptions ReadFromConfiguration(IConfiguration config)
         options.StripeOptions = StripeOptions.ReadFromConfiguration(config);
         options.AuthOptions = AuthOptions.ReadFromConfiguration(config);
         options.OAuthServerOptions = OAuthServerOptions.ReadFromConfiguration(config);
+        options.SourceMapOptions = SourceMapOptions.ReadFromConfiguration(config);
 
         return options;
     }

src/Exceptionless.Core/Configuration/SourceMapOptions.cs

@@ -0,0 +1,74 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Exceptionless.Core.Configuration;
+
+public sealed class SourceMapOptions
+{
+    public bool EnableAutoDownload { get; internal set; }
+    public int RequestTimeoutMilliseconds { get; internal set; }
+    public int MaximumGeneratedFileSize { get; internal set; }
+    public int MaximumSourceMapSize { get; internal set; }
+    public int MaximumMappingSegments { get; internal set; }
+    public int MaximumRedirects { get; internal set; }
+    public int MaximumConcurrentDownloads { get; internal set; }
+    public int MaximumConcurrentDownloadsGlobally { get; internal set; }
+    public int AutoDownloadRateLimitPeriodMinutes { get; internal set; }
+    public int MaximumAutoDiscoveriesPerFreeClientKey { get; internal set; }
+    public int MaximumAutoDiscoveriesPerClientKey { get; internal set; }
+    public int MaximumAutoDiscoveriesPerFreeProject { get; internal set; }
+    public int MaximumAutoDiscoveriesPerProject { get; internal set; }
+    public int MaximumAutoDiscoveriesPerFreeOrganization { get; internal set; }
+    public int MaximumAutoDiscoveriesPerOrganization { get; internal set; }
+    public int MaximumAutoDownloadRequestsPerDestination { get; internal set; }
+    public int MaximumAutoDownloadConnectionsPerIpAddress { get; internal set; }
+    public int MaximumAutoDownloadRequestsGlobally { get; internal set; }
+    public int MaximumAutoRefreshRequestsPerDestination { get; internal set; }
+    public int MaximumAutoRefreshRequestsGlobally { get; internal set; }
+    public int MaximumFramesPerError { get; internal set; }
+    public int MaximumProcessingTimeMilliseconds { get; internal set; }
+    public int AutoDownloadRefreshIntervalMinutes { get; internal set; }
+    public int ParsedSourceMapCacheLifetimeMinutes { get; internal set; }
+    public long MaximumParsedSourceMapCacheSize { get; internal set; }
+
+    public TimeSpan RequestTimeout => TimeSpan.FromMilliseconds(RequestTimeoutMilliseconds);
+    public TimeSpan MaximumProcessingTime => TimeSpan.FromMilliseconds(MaximumProcessingTimeMilliseconds);
+    public TimeSpan AutoDownloadRefreshInterval => TimeSpan.FromMinutes(AutoDownloadRefreshIntervalMinutes);
+    public TimeSpan ParsedSourceMapCacheLifetime => TimeSpan.FromMinutes(ParsedSourceMapCacheLifetimeMinutes);
+    public TimeSpan AutoDownloadRateLimitPeriod => TimeSpan.FromMinutes(AutoDownloadRateLimitPeriodMinutes);
+
+    public static SourceMapOptions ReadFromConfiguration(IConfiguration configuration)
+    {
+        var section = configuration.GetSection("SourceMaps");
+        return new SourceMapOptions
+        {
+            EnableAutoDownload = section.GetValue(nameof(EnableAutoDownload), true),
+            RequestTimeoutMilliseconds = ReadPositive(section, nameof(RequestTimeoutMilliseconds), 3000),
+            MaximumGeneratedFileSize = ReadPositive(section, nameof(MaximumGeneratedFileSize), 5 * 1024 * 1024),
+            MaximumSourceMapSize = ReadPositive(section, nameof(MaximumSourceMapSize), 20 * 1024 * 1024),
+            MaximumMappingSegments = ReadPositive(section, nameof(MaximumMappingSegments), 1_000_000),
+            MaximumRedirects = Math.Max(0, section.GetValue(nameof(MaximumRedirects), 3)),
+            MaximumConcurrentDownloads = ReadPositive(section, nameof(MaximumConcurrentDownloads), 4),
+            MaximumConcurrentDownloadsGlobally = ReadPositive(section, nameof(MaximumConcurrentDownloadsGlobally), 16),
+            AutoDownloadRateLimitPeriodMinutes = ReadPositive(section, nameof(AutoDownloadRateLimitPeriodMinutes), 15),
+            MaximumAutoDiscoveriesPerFreeClientKey = Math.Max(0, section.GetValue(nameof(MaximumAutoDiscoveriesPerFreeClientKey), 5)),
+            MaximumAutoDiscoveriesPerClientKey = Math.Max(0, section.GetValue(nameof(MaximumAutoDiscoveriesPerClientKey), 25)),
+            MaximumAutoDiscoveriesPerFreeProject = Math.Max(0, section.GetValue(nameof(MaximumAutoDiscoveriesPerFreeProject), 10)),
+            MaximumAutoDiscoveriesPerProject = Math.Max(0, section.GetValue(nameof(MaximumAutoDiscoveriesPerProject), 50)),
+            MaximumAutoDiscoveriesPerFreeOrganization = Math.Max(0, section.GetValue(nameof(MaximumAutoDiscoveriesPerFreeOrganization), 10)),
+            MaximumAutoDiscoveriesPerOrganization = Math.Max(0, section.GetValue(nameof(MaximumAutoDiscoveriesPerOrganization), 100)),
+            MaximumAutoDownloadRequestsPerDestination = Math.Max(0, section.GetValue(nameof(MaximumAutoDownloadRequestsPerDestination), 100)),
+            MaximumAutoDownloadConnectionsPerIpAddress = Math.Max(0, section.GetValue(nameof(MaximumAutoDownloadConnectionsPerIpAddress), 200)),
+            MaximumAutoDownloadRequestsGlobally = Math.Max(0, section.GetValue(nameof(MaximumAutoDownloadRequestsGlobally), 1000)),
+            MaximumAutoRefreshRequestsPerDestination = Math.Max(0, section.GetValue(nameof(MaximumAutoRefreshRequestsPerDestination), 20)),
+            MaximumAutoRefreshRequestsGlobally = Math.Max(0, section.GetValue(nameof(MaximumAutoRefreshRequestsGlobally), 200)),
+            MaximumFramesPerError = ReadPositive(section, nameof(MaximumFramesPerError), 100),
+            MaximumProcessingTimeMilliseconds = ReadPositive(section, nameof(MaximumProcessingTimeMilliseconds), 5000),
+            AutoDownloadRefreshIntervalMinutes = ReadPositive(section, nameof(AutoDownloadRefreshIntervalMinutes), 60),
+            ParsedSourceMapCacheLifetimeMinutes = ReadPositive(section, nameof(ParsedSourceMapCacheLifetimeMinutes), 5),
+            MaximumParsedSourceMapCacheSize = Math.Max(1, section.GetValue(nameof(MaximumParsedSourceMapCacheSize), 100L * 1024 * 1024))
+        };
+    }
+
+    private static int ReadPositive(IConfiguration section, string name, int defaultValue)
+        => Math.Max(1, section.GetValue(name, defaultValue));
+}

src/Exceptionless.Core/Exceptionless.Core.csproj

@@ -31,6 +31,7 @@
     <PackageReference Include="MiniValidation" Version="0.10.0" />
     <PackageReference Include="Handlebars.Net" Version="2.1.6" />
     <PackageReference Include="McSherry.SemanticVersioning" Version="1.5.0" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.10" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.10" />
     <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="10.0.10" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="10.0.10" />

src/Exceptionless.Core/Jobs/CleanupDataJob.cs

@@ -286,6 +286,8 @@ private async Task RemoveOrganizationAsync(Organization organization, JobContext
         await RenewLockAsync(context);
         long removedStacks = await _stackRepository.RemoveAllByOrganizationIdAsync(organization.Id);
 
+        await RemoveOrganizationProjectFilesAsync(organization.Id, context);
+
         await RenewLockAsync(context);
         long removedProjects = await _projectRepository.RemoveAllByOrganizationIdAsync(organization.Id);
 
@@ -316,6 +318,23 @@ private async Task RemoveSyntheticUserAsync(User user)
     private Task RemoveOrganizationFilesAsync(Organization organization, JobContext context)
         => RemoveFilesAsync(OrganizationStoragePaths.GetProfileImagesPath(organization.Id), context.CancellationToken);
 
+    private async Task RemoveOrganizationProjectFilesAsync(string organizationId, JobContext context)
+    {
+        var projects = await _projectRepository.GetByOrganizationIdAsync(
+            organizationId,
+            options => options.Include(project => project.Id).SoftDeleteMode(SoftDeleteQueryMode.All).SearchAfterPaging().PageLimit(100));
+
+        while (projects.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
+        {
+            foreach (var project in projects.Documents)
+                await _fileStorage.DeleteFilesAsync($"source-maps/{project.Id}/*", context.CancellationToken);
+
+            await RenewLockAsync(context);
+            if (!await projects.NextPageAsync())
+                break;
+        }
+    }
+
     private async Task RemoveFilesAsync(string path, CancellationToken cancellationToken)
     {
         string searchPattern = $"{path}/*";
@@ -341,6 +360,7 @@ private async Task RemoveProjectsAsync(Project project, JobContext context)
         await RenewLockAsync(context);
         long removedStacks = await _stackRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id);
 
+        await _fileStorage.DeleteFilesAsync($"source-maps/{project.Id}/*", context.CancellationToken);
         await _projectRepository.RemoveAsync(project);
         _logger.RemoveProjectComplete(project.Name, project.Id, removedStacks, removedEvents);
     }

src/Exceptionless.Core/Jobs/EventPostsJob.cs

@@ -312,6 +312,7 @@ await _eventPostService.EnqueueAsync(new EventPost(false)
                 {
                     ApiVersion = ep.ApiVersion,
                     CharSet = ep.CharSet,
+                    ClientKeyHash = ep.ClientKeyHash,
                     ContentEncoding = null,
                     IpAddress = ep.IpAddress,
                     MediaType = ep.MediaType,

src/Exceptionless.Core/Models/Queues/EventPostInfo.cs

@@ -10,6 +10,7 @@ public record EventPostInfo
     public string? UserAgent { get; init; }
     public string? ContentEncoding { get; init; }
     public string? IpAddress { get; init; }
+    public string? ClientKeyHash { get; init; }
 }
 
 public record EventPost : EventPostInfo

src/Exceptionless.Core/Plugins/EventProcessor/Default/15_SourceMapPlugin.cs

@@ -0,0 +1,44 @@
+using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Billing;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Pipeline;
+using Exceptionless.Core.Services.SourceMaps;
+using Foundatio.Serializer;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Plugins.EventProcessor.Default;
+
+[Priority(15)]
+public sealed class SourceMapPlugin : EventProcessorPluginBase
+{
+    private readonly SourceMapService _sourceMapService;
+    private readonly ITextSerializer _serializer;
+    private readonly BillingPlans _billingPlans;
+
+    public SourceMapPlugin(SourceMapService sourceMapService, ITextSerializer serializer, BillingPlans billingPlans, AppOptions options, ILoggerFactory loggerFactory)
+        : base(options, loggerFactory)
+    {
+        _sourceMapService = sourceMapService;
+        _serializer = serializer;
+        _billingPlans = billingPlans;
+        ContinueOnError = true;
+    }
+
+    public override async Task EventProcessingAsync(EventContext context)
+    {
+        if (!context.Event.IsError())
+            return;
+
+        var error = context.Event.GetError(_serializer, _logger);
+        if (error is null)
+            return;
+
+        var request = new SourceMapRequest(
+            context.Organization.Id,
+            context.Project.Id,
+            context.EventPostInfo?.ClientKeyHash,
+            String.Equals(context.Organization.PlanId, _billingPlans.FreePlan.Id, StringComparison.OrdinalIgnoreCase));
+        if (await _sourceMapService.SymbolicateAsync(request, error))
+            context.Event.SetError(error);
+    }
+}