From 6550a8d700eb674b284caa3532bee41e7f8b69b6 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 14 Jul 2026 17:29:41 -0500 Subject: [PATCH 01/15] Add JavaScript source map symbolication --- docs/docs/source-maps.md | 29 + src/Exceptionless.Core/Bootstrapper.cs | 13 +- .../Configuration/AppOptions.cs | 2 + .../Configuration/SourceMapOptions.cs | 38 ++ src/Exceptionless.Core/Jobs/CleanupDataJob.cs | 1 + .../Default/15_SourceMapPlugin.cs | 36 ++ .../Services/SourceMaps/SourceMapArtifact.cs | 12 + .../Services/SourceMaps/SourceMapDocument.cs | 175 ++++++ .../Services/SourceMaps/SourceMapService.cs | 513 ++++++++++++++++++ .../src/lib/features/projects/api.svelte.ts | 65 ++- .../src/lib/features/projects/models.ts | 10 + .../src/lib/features/projects/schemas.ts | 8 +- .../project/[projectId]/routes.svelte.ts | 7 + .../[projectId]/source-maps/+page.svelte | 234 ++++++++ .../Controllers/SourceMapController.cs | 103 ++++ .../RequestBodyContentOperationTransformer.cs | 47 +- .../Controllers/Data/controller-manifest.json | 47 ++ .../Controllers/Data/openapi.json | 191 ++++++- .../Controllers/OpenApiControllerTests.cs | 8 + .../Controllers/SourceMapControllerTests.cs | 87 +++ .../Jobs/CleanupDataJobTests.cs | 4 + .../SourceMaps/SourceMapDocumentTests.cs | 77 +++ .../SourceMaps/SourceMapServiceTests.cs | 158 ++++++ 23 files changed, 1835 insertions(+), 30 deletions(-) create mode 100644 docs/docs/source-maps.md create mode 100644 src/Exceptionless.Core/Configuration/SourceMapOptions.cs create mode 100644 src/Exceptionless.Core/Plugins/EventProcessor/Default/15_SourceMapPlugin.cs create mode 100644 src/Exceptionless.Core/Services/SourceMaps/SourceMapArtifact.cs create mode 100644 src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs create mode 100644 src/Exceptionless.Core/Services/SourceMaps/SourceMapService.cs create mode 100644 src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/source-maps/+page.svelte create mode 100644 src/Exceptionless.Web/Controllers/SourceMapController.cs create mode 100644 tests/Exceptionless.Tests/Controllers/SourceMapControllerTests.cs create mode 100644 tests/Exceptionless.Tests/Services/SourceMaps/SourceMapDocumentTests.cs create mode 100644 tests/Exceptionless.Tests/Services/SourceMaps/SourceMapServiceTests.cs diff --git a/docs/docs/source-maps.md b/docs/docs/source-maps.md new file mode 100644 index 0000000000..f967d41d8d --- /dev/null +++ b/docs/docs/source-maps.md @@ -0,0 +1,29 @@ +--- +title: "JavaScript Source Maps" +--- + +# JavaScript Source Maps + +Exceptionless uses source maps to turn minified JavaScript stack frames into the original file names, line and column numbers, and function names. Symbolication happens before an event is assigned to a stack, so readable function names also improve stack grouping. + +## Automatic discovery + +No domain allowlist or project setup is required for public source maps. When an error contains an absolute HTTPS JavaScript URL, Exceptionless checks the generated file's `SourceMap` or `X-SourceMap` response header and its `sourceMappingURL` comment. If neither is present, it also checks the conventional `.map` URL. + +Downloaded maps are validated and cached in project-scoped file storage. Exceptionless only makes anonymous HTTPS requests to public network addresses. Redirects are revalidated, and downloads have time, redirect, size, concurrency, and per-project rate limits. Self-hosted installations can tune these safeguards under the `SourceMaps` configuration section. + +## Uploading a source map + +Upload a map when it is private or is not deployed next to the generated JavaScript: + +1. Open the project and select **Source Maps** under **Project Settings**. +2. Enter the exact absolute URL that appears in the generated stack frame, including any path or query string used to identify the build. +3. Select the corresponding source map and upload it. + +Uploading another map for the same generated file URL replaces the previous map. Uploaded and automatically discovered maps appear together on the Source Maps page and can be deleted there. + +Source maps must use the version 3 flat-map format. Indexed source maps with a `sections` property and authenticated automatic downloads are planned follow-up capabilities; private maps can be uploaded in the meantime. + +## Deployment guidance + +Generate source maps as part of the same build that produces the minified JavaScript. Content-hashed generated file names are preferred because the generated URL then identifies a specific build. You can publish the `.map` file for zero-configuration discovery or keep it private and upload it to Exceptionless during deployment. diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index efcfae9efb..3d37048d57 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -25,6 +25,7 @@ using Exceptionless.Core.Seed; using Exceptionless.Core.Serialization; using Exceptionless.Core.Services; +using Exceptionless.Core.Services.SourceMaps; using Exceptionless.Core.Utility; using Exceptionless.Core.Validation; using Foundatio.Caching; @@ -193,6 +194,16 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO AllowAutoRedirect = false, ConnectCallback = ConnectToPublicAddressAsync }); + services.AddHttpClient(SourceMapService.HttpClientName) + .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler + { + AllowAutoRedirect = false, + AutomaticDecompression = DecompressionMethods.All, + ConnectCallback = ConnectToPublicAddressAsync, + UseCookies = false, + UseProxy = false + }); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -223,7 +234,7 @@ private static async ValueTask ConnectToPublicAddressAsync(SocketsHttpCo } } - throw new HttpRequestException($"OAuth client metadata host '{context.DnsEndPoint.Host}' did not resolve to a reachable public address.", lastException); + throw new HttpRequestException($"Host '{context.DnsEndPoint.Host}' did not resolve to a reachable public address.", lastException); } public static void LogConfiguration(IServiceProvider serviceProvider, AppOptions appOptions, ILogger logger) diff --git a/src/Exceptionless.Core/Configuration/AppOptions.cs b/src/Exceptionless.Core/Configuration/AppOptions.cs index ff6217829a..99da0ce9d3 100644 --- a/src/Exceptionless.Core/Configuration/AppOptions.cs +++ b/src/Exceptionless.Core/Configuration/AppOptions.cs @@ -80,6 +80,7 @@ public class AppOptions public StripeOptions StripeOptions { get; internal set; } = null!; public AuthOptions AuthOptions { get; internal set; } = null!; public OAuthServerOptions OAuthServerOptions { get; internal set; } = null!; + public SourceMapOptions SourceMapOptions { get; internal set; } = null!; public static AppOptions ReadFromConfiguration(IConfiguration config) { @@ -133,6 +134,7 @@ public static AppOptions ReadFromConfiguration(IConfiguration config) options.StripeOptions = StripeOptions.ReadFromConfiguration(config); options.AuthOptions = AuthOptions.ReadFromConfiguration(config); options.OAuthServerOptions = OAuthServerOptions.ReadFromConfiguration(config); + options.SourceMapOptions = SourceMapOptions.ReadFromConfiguration(config); return options; } diff --git a/src/Exceptionless.Core/Configuration/SourceMapOptions.cs b/src/Exceptionless.Core/Configuration/SourceMapOptions.cs new file mode 100644 index 0000000000..b6aba0aab2 --- /dev/null +++ b/src/Exceptionless.Core/Configuration/SourceMapOptions.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.Configuration; + +namespace Exceptionless.Core.Configuration; + +public sealed class SourceMapOptions +{ + public bool EnableAutoDownload { get; internal set; } + public int RequestTimeoutMilliseconds { get; internal set; } + public int MaximumGeneratedFileSize { get; internal set; } + public int MaximumSourceMapSize { get; internal set; } + public int MaximumMappingSegments { get; internal set; } + public int MaximumRedirects { get; internal set; } + public int MaximumConcurrentDownloads { get; internal set; } + public int MaximumAutoDownloadsPerProjectPerHour { get; internal set; } + public int MaximumFramesPerError { get; internal set; } + public int MaximumProcessingTimeMilliseconds { get; internal set; } + + public TimeSpan RequestTimeout => TimeSpan.FromMilliseconds(RequestTimeoutMilliseconds); + public TimeSpan MaximumProcessingTime => TimeSpan.FromMilliseconds(MaximumProcessingTimeMilliseconds); + + public static SourceMapOptions ReadFromConfiguration(IConfiguration configuration) + { + var section = configuration.GetSection("SourceMaps"); + return new SourceMapOptions + { + EnableAutoDownload = section.GetValue(nameof(EnableAutoDownload), true), + RequestTimeoutMilliseconds = section.GetValue(nameof(RequestTimeoutMilliseconds), 3000), + MaximumGeneratedFileSize = section.GetValue(nameof(MaximumGeneratedFileSize), 5 * 1024 * 1024), + MaximumSourceMapSize = section.GetValue(nameof(MaximumSourceMapSize), 20 * 1024 * 1024), + MaximumMappingSegments = section.GetValue(nameof(MaximumMappingSegments), 1_000_000), + MaximumRedirects = section.GetValue(nameof(MaximumRedirects), 3), + MaximumConcurrentDownloads = section.GetValue(nameof(MaximumConcurrentDownloads), 4), + MaximumAutoDownloadsPerProjectPerHour = section.GetValue(nameof(MaximumAutoDownloadsPerProjectPerHour), 100), + MaximumFramesPerError = section.GetValue(nameof(MaximumFramesPerError), 100), + MaximumProcessingTimeMilliseconds = section.GetValue(nameof(MaximumProcessingTimeMilliseconds), 5000) + }; + } +} diff --git a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs index f212c67cf4..14ee393433 100644 --- a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs @@ -341,6 +341,7 @@ private async Task RemoveProjectsAsync(Project project, JobContext context) await RenewLockAsync(context); long removedStacks = await _stackRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id); + await _fileStorage.DeleteFilesAsync($"source-maps/{project.Id}/*", context.CancellationToken); await _projectRepository.RemoveAsync(project); _logger.RemoveProjectComplete(project.Name, project.Id, removedStacks, removedEvents); } diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/15_SourceMapPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/15_SourceMapPlugin.cs new file mode 100644 index 0000000000..cd5481ffc9 --- /dev/null +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/15_SourceMapPlugin.cs @@ -0,0 +1,36 @@ +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Pipeline; +using Exceptionless.Core.Services.SourceMaps; +using Foundatio.Serializer; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Plugins.EventProcessor.Default; + +[Priority(15)] +public sealed class SourceMapPlugin : EventProcessorPluginBase +{ + private readonly SourceMapService _sourceMapService; + private readonly ITextSerializer _serializer; + + public SourceMapPlugin(SourceMapService sourceMapService, ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) + : base(options, loggerFactory) + { + _sourceMapService = sourceMapService; + _serializer = serializer; + ContinueOnError = true; + } + + public override async Task EventProcessingAsync(EventContext context) + { + if (!context.Event.IsError()) + return; + + var error = context.Event.GetError(_serializer, _logger); + if (error is null) + return; + + if (await _sourceMapService.SymbolicateAsync(context.Project.Id, error)) + context.Event.SetError(error); + } +} diff --git a/src/Exceptionless.Core/Services/SourceMaps/SourceMapArtifact.cs b/src/Exceptionless.Core/Services/SourceMaps/SourceMapArtifact.cs new file mode 100644 index 0000000000..365c6a6fd3 --- /dev/null +++ b/src/Exceptionless.Core/Services/SourceMaps/SourceMapArtifact.cs @@ -0,0 +1,12 @@ +namespace Exceptionless.Core.Services.SourceMaps; + +public sealed record SourceMapArtifact +{ + public required string Id { get; init; } + public required string GeneratedFileUrl { get; init; } + public string? SourceMapUrl { get; init; } + public string? FileName { get; init; } + public required long Size { get; init; } + public required bool IsAutoDownloaded { get; init; } + public required DateTime CreatedUtc { get; init; } +} diff --git a/src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs b/src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs new file mode 100644 index 0000000000..d2f2d53c69 --- /dev/null +++ b/src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs @@ -0,0 +1,175 @@ +using System.Text.Json; + +namespace Exceptionless.Core.Services.SourceMaps; + +public sealed class SourceMapDocument +{ + private const string Base64Characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + private readonly IReadOnlyList> _lines; + private readonly string[] _names; + private readonly string[] _sources; + + private SourceMapDocument(string? sourceRoot, string[] sources, string[] names, IReadOnlyList> lines) + { + SourceRoot = sourceRoot; + _sources = sources; + _names = names; + _lines = lines; + } + + public string? SourceRoot { get; } + + public static SourceMapDocument Parse(byte[] sourceMap, int maximumSegments = 1_000_000) + { + using var document = JsonDocument.Parse(sourceMap); + var root = document.RootElement; + + if (!root.TryGetProperty("version", out var version) || version.GetInt32() != 3) + throw new JsonException("Only source map version 3 is supported."); + + if (root.TryGetProperty("sections", out _)) + throw new JsonException("Indexed source maps are not supported."); + + string mappings = root.GetProperty("mappings").GetString() ?? throw new JsonException("The source map mappings are required."); + string[] sources = ReadStringArray(root, "sources"); + string[] names = root.TryGetProperty("names", out _) ? ReadStringArray(root, "names") : []; + string? sourceRoot = root.TryGetProperty("sourceRoot", out var sourceRootElement) ? sourceRootElement.GetString() : null; + + return new SourceMapDocument(sourceRoot, sources, names, DecodeMappings(mappings, sources.Length, names.Length, maximumSegments)); + } + + public SourceMapLocation? FindOriginalLocation(int generatedLine, int generatedColumn) + { + if (generatedLine < 0 || generatedLine >= _lines.Count || generatedColumn < 0) + return null; + + var segments = _lines[generatedLine]; + MappingSegment? match = null; + foreach (var segment in segments) + { + if (segment.GeneratedColumn > generatedColumn) + break; + + match = segment; + } + + if (match?.SourceIndex is not int sourceIndex || match.OriginalLine is not int originalLine || match.OriginalColumn is not int originalColumn) + return null; + + string? name = match.NameIndex is int nameIndex ? _names[nameIndex] : null; + return new SourceMapLocation(CombineSource(SourceRoot, _sources[sourceIndex]), originalLine, originalColumn, name); + } + + private static string[] ReadStringArray(JsonElement root, string propertyName) + { + if (!root.TryGetProperty(propertyName, out var element) || element.ValueKind != JsonValueKind.Array) + throw new JsonException($"The source map {propertyName} array is required."); + + return element.EnumerateArray() + .Select(value => value.GetString() ?? throw new JsonException($"The source map {propertyName} array contains a null value.")) + .ToArray(); + } + + private static IReadOnlyList> DecodeMappings(string mappings, int sourceCount, int nameCount, int maximumSegments) + { + if (maximumSegments < 1) + throw new ArgumentOutOfRangeException(nameof(maximumSegments)); + + var lines = new List>(); + int sourceIndex = 0; + int originalLine = 0; + int originalColumn = 0; + int nameIndex = 0; + int segmentCount = 0; + + foreach (string encodedLine in mappings.Split(';')) + { + int generatedColumn = 0; + var line = new List(); + foreach (string encodedSegment in encodedLine.Split(',', StringSplitOptions.RemoveEmptyEntries)) + { + if (++segmentCount > maximumSegments) + throw new JsonException("The source map contains too many mapping segments."); + + int index = 0; + generatedColumn += DecodeValue(encodedSegment, ref index); + if (generatedColumn < 0) + throw new JsonException("A source map generated column cannot be negative."); + + if (index == encodedSegment.Length) + { + line.Add(new MappingSegment(generatedColumn, null, null, null, null)); + continue; + } + + sourceIndex += DecodeValue(encodedSegment, ref index); + originalLine += DecodeValue(encodedSegment, ref index); + originalColumn += DecodeValue(encodedSegment, ref index); + if (sourceIndex < 0 || sourceIndex >= sourceCount || originalLine < 0 || originalColumn < 0) + throw new JsonException("A source map mapping points outside its source data."); + + int? segmentNameIndex = null; + if (index < encodedSegment.Length) + { + nameIndex += DecodeValue(encodedSegment, ref index); + if (nameIndex < 0 || nameIndex >= nameCount) + throw new JsonException("A source map mapping points outside its names array."); + + segmentNameIndex = nameIndex; + } + + if (index != encodedSegment.Length) + throw new JsonException("A source map segment has an invalid field count."); + + line.Add(new MappingSegment(generatedColumn, sourceIndex, originalLine, originalColumn, segmentNameIndex)); + } + + lines.Add(line); + } + + return lines; + } + + private static int DecodeValue(string segment, ref int index) + { + long result = 0; + int shift = 0; + bool hasContinuation; + + do + { + if (index >= segment.Length) + throw new JsonException("A source map VLQ value is incomplete."); + + int digit = Base64Characters.IndexOf(segment[index++]); + if (digit < 0) + throw new JsonException("A source map VLQ value contains an invalid character."); + + hasContinuation = (digit & 32) != 0; + digit &= 31; + result += (long)digit << shift; + shift += 5; + if (shift > 35 || result > (long)Int32.MaxValue * 2 + 1) + throw new JsonException("A source map VLQ value is too large."); + } while (hasContinuation); + + bool isNegative = (result & 1) == 1; + result >>= 1; + return isNegative ? -(int)result : (int)result; + } + + private static string CombineSource(string? sourceRoot, string source) + { + if (String.IsNullOrWhiteSpace(sourceRoot) || Uri.TryCreate(source, UriKind.Absolute, out _)) + return source; + + if (Uri.TryCreate(sourceRoot, UriKind.Absolute, out var sourceRootUri) && Uri.TryCreate(sourceRootUri, source, out var combinedUri)) + return combinedUri.ToString(); + + return $"{sourceRoot.TrimEnd('/')}/{source.TrimStart('/')}"; + } + + private sealed record MappingSegment(int GeneratedColumn, int? SourceIndex, int? OriginalLine, int? OriginalColumn, int? NameIndex); +} + +public sealed record SourceMapLocation(string Source, int Line, int Column, string? Name); diff --git a/src/Exceptionless.Core/Services/SourceMaps/SourceMapService.cs b/src/Exceptionless.Core/Services/SourceMaps/SourceMapService.cs new file mode 100644 index 0000000000..3db4fa4a3f --- /dev/null +++ b/src/Exceptionless.Core/Services/SourceMaps/SourceMapService.cs @@ -0,0 +1,513 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using Exceptionless.Core.Configuration; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.Data; +using Foundatio.Caching; +using Foundatio.Storage; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services.SourceMaps; + +public sealed class SourceMapService +{ + public const string HttpClientName = "SourceMaps"; + public const int MaximumUploadRequestSize = 21 * 1024 * 1024; + private const string SourceMapDataKey = "@source_map"; + private static readonly TimeSpan FailureCacheLifetime = TimeSpan.FromMinutes(15); + private readonly SemaphoreSlim _downloadSemaphore; + private readonly ConcurrentDictionary>> _sourceMaps = new(StringComparer.Ordinal); + private readonly IHttpClientFactory _httpClientFactory; + private readonly IFileStorage _storage; + private readonly ICacheClient _cache; + private readonly JsonSerializerOptions _serializerOptions; + private readonly SourceMapOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public SourceMapService( + IHttpClientFactory httpClientFactory, + IFileStorage storage, + ICacheClient cache, + JsonSerializerOptions serializerOptions, + AppOptions options, + TimeProvider timeProvider, + ILogger logger) + { + _httpClientFactory = httpClientFactory; + _storage = storage; + _cache = cache; + _serializerOptions = serializerOptions; + _options = options.SourceMapOptions; + _downloadSemaphore = new SemaphoreSlim(Math.Max(1, _options.MaximumConcurrentDownloads)); + _timeProvider = timeProvider; + _logger = logger; + } + + public async Task SaveUploadedAsync(string projectId, string generatedFileUrl, string? fileName, Stream stream, CancellationToken cancellationToken = default) + { + if (!TryNormalizeGeneratedFileUrl(generatedFileUrl, requireHttps: false, out var generatedFileUri)) + throw new ArgumentException("The generated file URL must be an absolute HTTP or HTTPS URL without credentials or a fragment.", nameof(generatedFileUrl)); + + byte[] sourceMap = await ReadLimitedAsync(stream, _options.MaximumSourceMapSize, cancellationToken); + _ = SourceMapDocument.Parse(sourceMap, _options.MaximumMappingSegments); + + string normalizedUrl = generatedFileUri.AbsoluteUri; + var artifact = new SourceMapArtifact + { + Id = GetArtifactId(normalizedUrl), + GeneratedFileUrl = normalizedUrl, + FileName = String.IsNullOrWhiteSpace(fileName) ? null : Path.GetFileName(fileName), + Size = sourceMap.LongLength, + IsAutoDownloaded = false, + CreatedUtc = _timeProvider.GetUtcNow().UtcDateTime + }; + + await SaveArtifactAsync(projectId, artifact, sourceMap, cancellationToken); + await ClearCachesAsync(projectId, normalizedUrl); + return artifact; + } + + public async Task> GetArtifactsAsync(string projectId, CancellationToken cancellationToken = default) + { + string pattern = $"source-maps/{projectId}/*.json"; + var files = await _storage.GetFileListAsync(pattern, cancellationToken: cancellationToken); + var artifacts = new List(files.Count); + foreach (var file in files) + { + try + { + await using var stream = await _storage.GetFileStreamAsync(file.Path, StreamMode.Read, cancellationToken); + if (stream is null) + continue; + + var artifact = await JsonSerializer.DeserializeAsync(stream, _serializerOptions, cancellationToken); + if (artifact is not null) + artifacts.Add(artifact); + } + catch (Exception ex) when (ex is JsonException or IOException) + { + _logger.LogWarning(ex, "Unable to read source map metadata {SourceMapMetadataPath}.", file.Path); + } + } + + return artifacts.OrderByDescending(artifact => artifact.CreatedUtc).ToArray(); + } + + public async Task DeleteArtifactAsync(string projectId, string artifactId, CancellationToken cancellationToken = default) + { + if (!IsArtifactId(artifactId)) + return false; + + string metadataPath = GetMetadataPath(projectId, artifactId); + SourceMapArtifact? artifact = await ReadArtifactMetadataAsync(metadataPath, cancellationToken); + bool mapDeleted = await _storage.DeleteFileAsync(GetMapPath(projectId, artifactId), cancellationToken); + bool metadataDeleted = await _storage.DeleteFileAsync(metadataPath, cancellationToken); + if (artifact is not null) + await ClearCachesAsync(projectId, artifact.GeneratedFileUrl); + + return mapDeleted || metadataDeleted; + } + + public async Task DeleteAllArtifactsAsync(string projectId, CancellationToken cancellationToken = default) + { + await _storage.DeleteFilesAsync($"source-maps/{projectId}/*", cancellationToken); + foreach (string key in _sourceMaps.Keys.Where(key => key.StartsWith(projectId + ':', StringComparison.Ordinal))) + _sourceMaps.TryRemove(key, out _); + } + + public async Task SymbolicateAsync(string projectId, InnerError? error, CancellationToken cancellationToken = default) + { + bool changed = false; + int framesProcessed = 0; + using var processingCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + processingCancellationTokenSource.CancelAfter(_options.MaximumProcessingTime); + try + { + while (error is not null) + { + if (error.StackTrace is not null) + { + foreach (var frame in error.StackTrace) + { + if (++framesProcessed > _options.MaximumFramesPerError) + return changed; + + if (await SymbolicateFrameAsync(projectId, frame, processingCancellationTokenSource.Token)) + changed = true; + } + } + + error = error.Inner; + } + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + _logger.LogDebug("Source map processing exceeded its time budget for project {ProjectId}.", projectId); + } + + return changed; + } + + private async Task SymbolicateFrameAsync(string projectId, StackFrame frame, CancellationToken cancellationToken) + { + if (frame.Data?.ContainsKey(SourceMapDataKey) == true || frame.LineNumber is null || frame.LineNumber < 1 || String.IsNullOrWhiteSpace(frame.FileName)) + return false; + + if (!TryNormalizeGeneratedFileUrl(frame.FileName, requireHttps: false, out var generatedFileUri)) + return false; + + var resolved = await GetSourceMapAsync(projectId, generatedFileUri, cancellationToken); + if (resolved is null) + return false; + + int generatedColumn = frame.Column.GetValueOrDefault(1); + if (generatedColumn > 0) + generatedColumn--; + + var original = resolved.Document.FindOriginalLocation(frame.LineNumber.Value - 1, generatedColumn); + if (original is null) + return false; + + frame.Data ??= new DataDictionary(); + frame.Data[SourceMapDataKey] = new DataDictionary + { + ["generated_file_name"] = frame.FileName, + ["generated_line_number"] = frame.LineNumber, + ["generated_column"] = frame.Column, + ["source_map_id"] = resolved.Artifact.Id + }; + frame.FileName = original.Source; + frame.LineNumber = original.Line + 1; + frame.Column = original.Column + 1; + if (!String.IsNullOrWhiteSpace(original.Name)) + frame.Name = original.Name; + + return true; + } + + private async Task GetSourceMapAsync(string projectId, Uri generatedFileUri, CancellationToken cancellationToken) + { + string normalizedUrl = generatedFileUri.AbsoluteUri; + string cacheKey = GetMemoryCacheKey(projectId, normalizedUrl); + var lazy = _sourceMaps.GetOrAdd(cacheKey, _ => new Lazy>( + () => LoadSourceMapAsync(projectId, generatedFileUri), LazyThreadSafetyMode.ExecutionAndPublication)); + + try + { + var result = await lazy.Value.WaitAsync(cancellationToken); + if (result is null) + _sourceMaps.TryRemove(cacheKey, out _); + + return result; + } + catch + { + _sourceMaps.TryRemove(cacheKey, out _); + throw; + } + } + + private async Task LoadSourceMapAsync(string projectId, Uri generatedFileUri) + { + string generatedFileUrl = generatedFileUri.AbsoluteUri; + string artifactId = GetArtifactId(generatedFileUrl); + string mapPath = GetMapPath(projectId, artifactId); + var artifact = await ReadArtifactMetadataAsync(GetMetadataPath(projectId, artifactId), CancellationToken.None); + if (artifact is not null && await _storage.ExistsAsync(mapPath)) + { + byte[] storedSourceMap = await ReadStorageFileAsync(mapPath, _options.MaximumSourceMapSize, CancellationToken.None); + return new ResolvedSourceMap(artifact, SourceMapDocument.Parse(storedSourceMap, _options.MaximumMappingSegments)); + } + + if (!_options.EnableAutoDownload || !String.Equals(generatedFileUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + return null; + + string failureCacheKey = GetFailureCacheKey(projectId, generatedFileUrl); + if ((await _cache.GetAsync(failureCacheKey)).HasValue) + return null; + + try + { + using var timeoutCancellationTokenSource = new CancellationTokenSource(_options.RequestTimeout); + if (!await TryReserveAutoDownloadAsync(projectId)) + return await CacheFailureAsync(failureCacheKey); + + await _downloadSemaphore.WaitAsync(timeoutCancellationTokenSource.Token); + DownloadedSourceMap? downloaded; + try + { + downloaded = await DownloadSourceMapAsync(generatedFileUri, timeoutCancellationTokenSource.Token); + if (downloaded is null) + return await CacheFailureAsync(failureCacheKey); + } + finally + { + _downloadSemaphore.Release(); + } + + var downloadedArtifact = new SourceMapArtifact + { + Id = artifactId, + GeneratedFileUrl = generatedFileUrl, + SourceMapUrl = downloaded.SourceMapUrl, + FileName = GetDownloadedFileName(downloaded.SourceMapUrl), + Size = downloaded.Content.LongLength, + IsAutoDownloaded = true, + CreatedUtc = _timeProvider.GetUtcNow().UtcDateTime + }; + var document = SourceMapDocument.Parse(downloaded.Content, _options.MaximumMappingSegments); + await SaveArtifactAsync(projectId, downloadedArtifact, downloaded.Content, CancellationToken.None); + return new ResolvedSourceMap(downloadedArtifact, document); + } + catch (OperationCanceledException ex) + { + _logger.LogDebug(ex, "Timed out downloading a source map for {GeneratedFileUrl}.", generatedFileUrl); + return await CacheFailureAsync(failureCacheKey); + } + catch (Exception ex) when (ex is HttpRequestException or IOException or JsonException or InvalidOperationException or FormatException) + { + _logger.LogWarning(ex, "Unable to download a source map for {GeneratedFileUrl}.", generatedFileUrl); + return await CacheFailureAsync(failureCacheKey); + } + } + + private async Task DownloadSourceMapAsync(Uri generatedFileUri, CancellationToken cancellationToken) + { + using var generatedResponse = await SendAsync(generatedFileUri, _options.MaximumGeneratedFileSize, request => request.Headers.Range = new RangeHeaderValue(null, 64 * 1024), cancellationToken); + if (!generatedResponse.Response.IsSuccessStatusCode) + return null; + + string? sourceMapReference = GetSourceMapHeader(generatedResponse.Response); + if (String.IsNullOrWhiteSpace(sourceMapReference)) + { + byte[] generatedContent = await ReadLimitedAsync(await generatedResponse.Response.Content.ReadAsStreamAsync(cancellationToken), _options.MaximumGeneratedFileSize, cancellationToken); + sourceMapReference = FindSourceMapReference(Encoding.UTF8.GetString(generatedContent)); + } + + if (String.IsNullOrWhiteSpace(sourceMapReference)) + { + var fallbackUriBuilder = new UriBuilder(generatedResponse.Uri) { Path = generatedResponse.Uri.AbsolutePath + ".map" }; + return await DownloadSourceMapContentAsync(fallbackUriBuilder.Uri, cancellationToken); + } + + if (sourceMapReference.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) + return new DownloadedSourceMap(DecodeDataUri(sourceMapReference, _options.MaximumSourceMapSize), null); + + if (!Uri.TryCreate(generatedResponse.Uri, sourceMapReference, out var sourceMapUri) || !IsAutoDownloadUri(sourceMapUri)) + return null; + + return await DownloadSourceMapContentAsync(sourceMapUri, cancellationToken); + } + + private async Task DownloadSourceMapContentAsync(Uri sourceMapUri, CancellationToken cancellationToken) + { + using var result = await SendAsync(sourceMapUri, _options.MaximumSourceMapSize, null, cancellationToken); + if (!result.Response.IsSuccessStatusCode) + return null; + + byte[] content = await ReadLimitedAsync(await result.Response.Content.ReadAsStreamAsync(cancellationToken), _options.MaximumSourceMapSize, cancellationToken); + return new DownloadedSourceMap(content, result.Uri.AbsoluteUri); + } + + private async Task SendAsync(Uri uri, int maximumBytes, Action? configureRequest, CancellationToken cancellationToken) + { + Uri currentUri = uri; + for (int redirectCount = 0; ; redirectCount++) + { + if (!IsAutoDownloadUri(currentUri)) + throw new InvalidOperationException("Source map auto-download only supports public HTTPS URLs."); + + using var request = new HttpRequestMessage(HttpMethod.Get, currentUri); + request.Headers.AcceptEncoding.ParseAdd("gzip, deflate, br"); + configureRequest?.Invoke(request); + + var client = _httpClientFactory.CreateClient(HttpClientName); + var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + if (response.Content.Headers.ContentLength > maximumBytes) + { + response.Dispose(); + throw new InvalidOperationException("The downloaded file exceeded the configured maximum size."); + } + + if (!IsRedirect(response.StatusCode)) + return new HttpDownloadResult(currentUri, response); + + if (redirectCount >= _options.MaximumRedirects || response.Headers.Location is null) + { + response.Dispose(); + throw new InvalidOperationException("The source map download exceeded the allowed redirects."); + } + + Uri redirectUri = response.Headers.Location.IsAbsoluteUri ? response.Headers.Location : new Uri(currentUri, response.Headers.Location); + response.Dispose(); + currentUri = redirectUri; + } + } + + private async Task SaveArtifactAsync(string projectId, SourceMapArtifact artifact, byte[] sourceMap, CancellationToken cancellationToken) + { + string mapPath = GetMapPath(projectId, artifact.Id); + string metadataPath = GetMetadataPath(projectId, artifact.Id); + await using var mapStream = new MemoryStream(sourceMap, writable: false); + if (!await _storage.SaveFileAsync(mapPath, mapStream, cancellationToken)) + throw new IOException("Unable to save the source map."); + + try + { + byte[] metadata = JsonSerializer.SerializeToUtf8Bytes(artifact, _serializerOptions); + await using var metadataStream = new MemoryStream(metadata, writable: false); + if (!await _storage.SaveFileAsync(metadataPath, metadataStream, cancellationToken)) + throw new IOException("Unable to save the source map metadata."); + } + catch + { + await _storage.DeleteFileAsync(mapPath, CancellationToken.None); + throw; + } + } + + private async Task ReadArtifactMetadataAsync(string path, CancellationToken cancellationToken) + { + if (!await _storage.ExistsAsync(path)) + return null; + + await using var stream = await _storage.GetFileStreamAsync(path, StreamMode.Read, cancellationToken); + return stream is null ? null : await JsonSerializer.DeserializeAsync(stream, _serializerOptions, cancellationToken); + } + + private async Task ReadStorageFileAsync(string path, int maximumBytes, CancellationToken cancellationToken) + { + await using var stream = await _storage.GetFileStreamAsync(path, StreamMode.Read, cancellationToken) + ?? throw new FileNotFoundException("The source map file was not found.", path); + return await ReadLimitedAsync(stream, maximumBytes, cancellationToken); + } + + private Task ClearCachesAsync(string projectId, string generatedFileUrl) + { + _sourceMaps.TryRemove(GetMemoryCacheKey(projectId, generatedFileUrl), out _); + return _cache.RemoveAsync(GetFailureCacheKey(projectId, generatedFileUrl)); + } + + private async Task CacheFailureAsync(string failureCacheKey) + { + await _cache.SetAsync(failureCacheKey, true, FailureCacheLifetime); + return null; + } + + private async Task TryReserveAutoDownloadAsync(string projectId) + { + string hour = _timeProvider.GetUtcNow().ToString("yyyyMMddHH"); + long count = await _cache.IncrementAsync($"source-maps:downloads:{projectId}:{hour}", 1, TimeSpan.FromHours(2)); + if (count <= _options.MaximumAutoDownloadsPerProjectPerHour) + return true; + + if (count == _options.MaximumAutoDownloadsPerProjectPerHour + 1) + _logger.LogWarning("Source map auto-download limit reached for project {ProjectId}.", projectId); + return false; + } + + public static bool TryNormalizeGeneratedFileUrl(string? value, bool requireHttps, out Uri uri) + { + uri = null!; + if (String.IsNullOrWhiteSpace(value) || !Uri.TryCreate(value.Trim(), UriKind.Absolute, out var parsedUri)) + return false; + + bool validScheme = String.Equals(parsedUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase) + || (!requireHttps && String.Equals(parsedUri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)); + if (!validScheme || !String.IsNullOrEmpty(parsedUri.UserInfo) || !String.IsNullOrEmpty(parsedUri.Fragment)) + return false; + + uri = parsedUri; + return true; + } + + private static bool IsAutoDownloadUri(Uri uri) => TryNormalizeGeneratedFileUrl(uri.AbsoluteUri, requireHttps: true, out _); + + private static bool IsRedirect(HttpStatusCode statusCode) => statusCode is HttpStatusCode.MovedPermanently + or HttpStatusCode.Redirect + or HttpStatusCode.RedirectMethod + or HttpStatusCode.TemporaryRedirect + or HttpStatusCode.PermanentRedirect; + + private static string? GetSourceMapHeader(HttpResponseMessage response) + { + if (response.Headers.TryGetValues("SourceMap", out var sourceMapValues)) + return sourceMapValues.FirstOrDefault(); + if (response.Headers.TryGetValues("X-SourceMap", out var legacySourceMapValues)) + return legacySourceMapValues.FirstOrDefault(); + return null; + } + + private static string? FindSourceMapReference(string generatedContent) + { + const string marker = "sourceMappingURL="; + int markerIndex = generatedContent.LastIndexOf(marker, StringComparison.Ordinal); + if (markerIndex < 0) + return null; + + int start = markerIndex + marker.Length; + int end = generatedContent.IndexOfAny(['\r', '\n'], start); + string value = generatedContent[start..(end < 0 ? generatedContent.Length : end)].Trim(); + if (value.EndsWith("*/", StringComparison.Ordinal)) + value = value[..^2].Trim(); + return value; + } + + private static byte[] DecodeDataUri(string value, int maximumBytes) + { + int commaIndex = value.IndexOf(','); + if (commaIndex < 0) + throw new FormatException("The inline source map data URI is invalid."); + + string metadata = value[..commaIndex]; + string data = value[(commaIndex + 1)..]; + byte[] decoded = metadata.Contains(";base64", StringComparison.OrdinalIgnoreCase) + ? Convert.FromBase64String(data) + : Encoding.UTF8.GetBytes(Uri.UnescapeDataString(data)); + if (decoded.Length > maximumBytes) + throw new InvalidOperationException("The inline source map exceeded the configured maximum size."); + return decoded; + } + + private static async Task ReadLimitedAsync(Stream stream, int maximumBytes, CancellationToken cancellationToken) + { + var memoryStream = new MemoryStream(Math.Min(maximumBytes, 64 * 1024)); + byte[] buffer = new byte[81920]; + int read; + while ((read = await stream.ReadAsync(buffer, cancellationToken)) > 0) + { + if (memoryStream.Length + read > maximumBytes) + throw new InvalidOperationException("The file exceeded the configured maximum size."); + await memoryStream.WriteAsync(buffer.AsMemory(0, read), cancellationToken); + } + + return memoryStream.ToArray(); + } + + private static bool IsArtifactId(string value) => value.Length == 64 && value.All(Uri.IsHexDigit); + private static string GetArtifactId(string generatedFileUrl) => generatedFileUrl.ToSHA256(); + private static string GetMapPath(string projectId, string artifactId) => $"source-maps/{projectId}/{artifactId}.map"; + private static string GetMetadataPath(string projectId, string artifactId) => $"source-maps/{projectId}/{artifactId}.json"; + private static string GetMemoryCacheKey(string projectId, string generatedFileUrl) => $"{projectId}:{generatedFileUrl}"; + private static string GetFailureCacheKey(string projectId, string generatedFileUrl) => $"source-maps:failure:{projectId}:{generatedFileUrl.ToSHA256()}"; + + private static string? GetDownloadedFileName(string? sourceMapUrl) + { + if (sourceMapUrl is null || !Uri.TryCreate(sourceMapUrl, UriKind.Absolute, out var uri)) + return null; + return Path.GetFileName(uri.AbsolutePath); + } + + private sealed record DownloadedSourceMap(byte[] Content, string? SourceMapUrl); + private sealed record ResolvedSourceMap(SourceMapArtifact Artifact, SourceMapDocument Document); + + private sealed record HttpDownloadResult(Uri Uri, HttpResponseMessage Response) : IDisposable + { + public void Dispose() => Response.Dispose(); + } +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts index 189193d7cd..85a0120b24 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts @@ -1,8 +1,9 @@ -import type { ClientConfiguration, NewProject, NotificationSettings, UpdateProject, ViewProject } from '$features/projects/models'; +import type { ClientConfiguration, NewProject, NotificationSettings, SourceMapArtifact, UpdateProject, ViewProject } from '$features/projects/models'; import type { StringValueFromBody, WorkInProgressResult } from '$features/shared/models'; import type { WebSocketMessageValue } from '$features/websockets/models'; import { accessToken } from '$features/auth/index.svelte'; +import { fetchApiJson } from '$features/shared/api/api.svelte'; import { type FetchClientResponse, type ProblemDetails, useFetchClient } from '@exceptionless/fetchclient'; import { createMutation, createQuery, QueryClient, useQueryClient } from '@tanstack/svelte-query'; @@ -44,6 +45,7 @@ export const queryKeys = { putIntegrationNotificationSettings: (id: string | undefined, integration: string) => [...queryKeys.id(id), integration, 'put-notification-settings'] as const, resetData: (id: string | undefined) => [...queryKeys.id(id), 'reset-data'] as const, + sourceMaps: (id: string | undefined) => [...queryKeys.id(id), 'source-maps'] as const, type: ['Project'] as const, userNotificationSettings: (id: string | undefined, userId: string | undefined) => [...queryKeys.id(id), userId, 'notification-settings'] as const }; @@ -188,6 +190,17 @@ export interface ResetDataRequest { }; } +export interface SourceMapRequest { + route: { + id: string | undefined; + }; +} + +export interface SourceMapUpload { + file: File; + generated_file_url: string; +} + export interface UpdateProjectRequest { route: { id: string; @@ -283,6 +296,23 @@ export function deleteSlack(request: DeleteSlackRequest) { })); } +export function deleteSourceMapMutation(request: SourceMapRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => !!accessToken.current && !!request.route.id, + mutationFn: async (sourceMapId: string) => { + const client = useFetchClient(); + const response = await client.delete(`projects/${request.route.id}/source-maps/${sourceMapId}`); + return response.ok; + }, + mutationKey: [...queryKeys.sourceMaps(request.route.id), 'delete'], + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: queryKeys.sourceMaps(request.route.id) }); + } + })); +} + export function generateSampleData(request: GenerateSampleDataRequest) { return createMutation(() => ({ enabled: () => !!accessToken.current && !!request.route.id, @@ -412,6 +442,18 @@ export function getProjectUserNotificationSettings(request: GetProjectUserNotifi })); } +export function getSourceMapsQuery(request: SourceMapRequest) { + return createQuery(() => ({ + enabled: () => !!accessToken.current && !!request.route.id, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const client = useFetchClient(); + const response = await client.getJSON(`projects/${request.route.id}/source-maps`, { signal }); + return response.data!; + }, + queryKey: queryKeys.sourceMaps(request.route.id) + })); +} + export function postProject() { const queryClient = useQueryClient(); @@ -530,6 +572,27 @@ export function postSlack(request: PostSlackRequest) { })); } +export function postSourceMapMutation(request: SourceMapRequest) { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => !!accessToken.current && !!request.route.id, + mutationFn: async (sourceMap: SourceMapUpload) => { + const data = new FormData(); + data.append('generated_file_url', sourceMap.generated_file_url); + data.append('file', sourceMap.file); + return await fetchApiJson(`projects/${request.route.id}/source-maps`, { + body: data, + method: 'POST' + }); + }, + mutationKey: [...queryKeys.sourceMaps(request.route.id), 'post'], + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: queryKeys.sourceMaps(request.route.id) }); + } + })); +} + export function putProjectIntegrationNotificationSettings(request: PutProjectIntegrationNotificationSettingsRequest) { const queryClient = useQueryClient(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/models.ts index a055dfb212..b9c8b05831 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/models.ts @@ -4,3 +4,13 @@ export interface ClientConfigurationSetting { key: string; value: string; } + +export interface SourceMapArtifact { + created_utc: string; + file_name?: string; + generated_file_url: string; + id: string; + is_auto_downloaded: boolean; + size: number; + source_map_url?: string; +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/schemas.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/schemas.ts index e495e7dd4f..8c9b789bf3 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/schemas.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/schemas.ts @@ -5,7 +5,7 @@ import { type NotificationSettingsFormData, NotificationSettingsSchema } from '$generated/schemas'; -import { type infer as Infer, object, string } from 'zod'; +import { custom, type infer as Infer, object, string } from 'zod'; export { type NewProjectFormData, NewProjectSchema, type NotificationSettingsFormData, NotificationSettingsSchema }; @@ -17,3 +17,9 @@ export type ClientConfigurationSettingFormData = Infer; + +export const SourceMapUploadSchema = object({ + file: custom((value) => typeof File !== 'undefined' && value instanceof File, 'Source map file is required'), + generated_file_url: string().url('Enter the absolute URL of the generated JavaScript file') +}); +export type SourceMapUploadFormData = Infer; diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/routes.svelte.ts b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/routes.svelte.ts index 5a9d9a2978..ae7325d656 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/routes.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/routes.svelte.ts @@ -2,6 +2,7 @@ import { resolve } from '$app/paths'; import { page } from '$app/state'; import Usage from '@lucide/svelte/icons/bar-chart'; import ClientConfig from '@lucide/svelte/icons/braces'; +import SourceMaps from '@lucide/svelte/icons/file-code-2'; import ApiKey from '@lucide/svelte/icons/key'; import Stacks from '@lucide/svelte/icons/layers'; import Integration from '@lucide/svelte/icons/plug-2'; @@ -39,6 +40,12 @@ export function routes(): NavigationItem[] { icon: ClientConfig, title: 'Configuration' }, + { + group: 'Project Settings', + href: resolve('/(app)/project/[projectId]/source-maps', { projectId: page.params.projectId }), + icon: SourceMaps, + title: 'Source Maps' + }, { group: 'Project Settings', href: resolve('/(app)/project/[projectId]/integrations', { projectId: page.params.projectId }), diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/source-maps/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/source-maps/+page.svelte new file mode 100644 index 0000000000..4e9d0866fe --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/source-maps/+page.svelte @@ -0,0 +1,234 @@ + + +
+
+

Source Map Discovery

+ + Exceptionless automatically downloads publicly available source maps over HTTPS using the generated file's SourceMap header or sourceMappingURL. + Upload a map only when it is private or not published with the generated JavaScript. + +
+ +
+

Upload Source Map

+
{ + event.preventDefault(); + event.stopPropagation(); + form.handleSubmit(); + }} + > + state.errors}> + {#snippet children(errors)} +
+ {/snippet} + + + + {#snippet children(field)} + + Generated JavaScript URL + field.handleChange(event.currentTarget.value)} + aria-invalid={ariaInvalid(field)} + /> + + + {/snippet} + + + + {#snippet children(field)} + + Source map + field.handleChange(event.currentTarget.files?.[0] ?? null)} + aria-invalid={ariaInvalid(field)} + /> + + + {/snippet} + + + state.isSubmitting}> + {#snippet children(isSubmitting)} + + {/snippet} + +
+
+ +
+

Available Source Maps

+
+ + + + Generated file + Source + Size + Added + Actions + + + + {#if sourceMapsQuery.isLoading} + + {:else if !sourceMapsQuery.data?.length} + No source maps have been discovered or uploaded yet. + {:else} + {#each sourceMapsQuery.data as sourceMap (sourceMap.id)} + + {sourceMap.generated_file_url} + + + {sourceMap.is_auto_downloaded ? 'Automatic' : 'Uploaded'} + + {#if sourceMap.file_name}
{sourceMap.file_name}
{/if} + + + + + + + + {/each} + {/if} + + +
+
+
+ + + + + Delete Source Map + + Delete the source map for {sourceMapToDelete?.generated_file_url}? Future events will use + automatic discovery or retain their generated stack frames. + + + + Cancel + Delete Source Map + + + diff --git a/src/Exceptionless.Web/Controllers/SourceMapController.cs b/src/Exceptionless.Web/Controllers/SourceMapController.cs new file mode 100644 index 0000000000..1d2defa569 --- /dev/null +++ b/src/Exceptionless.Web/Controllers/SourceMapController.cs @@ -0,0 +1,103 @@ +using System.Text.Json; +using Exceptionless.Core.Authorization; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services.SourceMaps; +using Exceptionless.Web.Utility.OpenApi; +using Foundatio.Repositories; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace Exceptionless.Web.Controllers; + +[Route(API_PREFIX + "/projects/{projectId:objectid}/source-maps")] +[Authorize(Policy = AuthorizationRoles.UserPolicy)] +public sealed class SourceMapController : ExceptionlessApiController +{ + private readonly IProjectRepository _projectRepository; + private readonly SourceMapService _sourceMapService; + + public SourceMapController(IProjectRepository projectRepository, SourceMapService sourceMapService, TimeProvider timeProvider) + : base(timeProvider) + { + _projectRepository = projectRepository; + _sourceMapService = sourceMapService; + } + + /// + /// Get source maps for a project. + /// + [HttpGet] + public async Task>> GetAsync(string projectId, CancellationToken cancellationToken = default) + { + if (!await CanAccessProjectAsync(projectId)) + return NotFound(); + + return Ok(await _sourceMapService.GetArtifactsAsync(projectId, cancellationToken)); + } + + /// + /// Upload a source map for a generated JavaScript file. + /// + /// The identifier of the project. + /// The exact absolute URL of the generated JavaScript file. + /// The source map file. + /// The cancellation token. + [HttpPost] + [Consumes("multipart/form-data")] + [MultipartFileUpload("file", "generated_file_url", FileDescription = "The source map file to upload.")] + [RequestSizeLimit(SourceMapService.MaximumUploadRequestSize)] + [RequestFormLimits(MultipartBodyLengthLimit = SourceMapService.MaximumUploadRequestSize)] + [ProducesResponseType(StatusCodes.Status201Created)] + public async Task> PostAsync( + string projectId, + [FromForm(Name = "generated_file_url")] string? generatedFileUrl, + [FromForm] IFormFile? file, + CancellationToken cancellationToken = default) + { + if (!await CanAccessProjectAsync(projectId)) + return NotFound(); + + if (String.IsNullOrWhiteSpace(generatedFileUrl)) + ModelState.AddModelError("generated_file_url", "The generated file URL is required."); + if (file is null || file.Length == 0) + ModelState.AddModelError("file", "A source map file is required."); + if (!ModelState.IsValid) + return ValidationProblem(ModelState); + + try + { + await using var stream = file!.OpenReadStream(); + var artifact = await _sourceMapService.SaveUploadedAsync(projectId, generatedFileUrl!, file.FileName, stream, cancellationToken); + return StatusCode(StatusCodes.Status201Created, artifact); + } + catch (ArgumentException ex) + { + ModelState.AddModelError("generated_file_url", ex.Message); + } + catch (Exception ex) when (ex is JsonException or InvalidOperationException) + { + ModelState.AddModelError("file", ex.Message); + } + + return ValidationProblem(ModelState); + } + + /// + /// Delete a source map from a project. + /// + [HttpDelete("{sourceMapId}")] + [ProducesResponseType(StatusCodes.Status204NoContent)] + public async Task DeleteAsync(string projectId, string sourceMapId, CancellationToken cancellationToken = default) + { + if (!await CanAccessProjectAsync(projectId)) + return NotFound(); + + return await _sourceMapService.DeleteArtifactAsync(projectId, sourceMapId, cancellationToken) ? NoContent() : NotFound(); + } + + private async Task CanAccessProjectAsync(string projectId) + { + var project = await _projectRepository.GetByIdAsync(projectId, options => options.Cache()); + return project is not null && CanAccessOrganization(project.OrganizationId); + } +} diff --git a/src/Exceptionless.Web/Utility/OpenApi/RequestBodyContentOperationTransformer.cs b/src/Exceptionless.Web/Utility/OpenApi/RequestBodyContentOperationTransformer.cs index 15db2876ba..3e784210b8 100644 --- a/src/Exceptionless.Web/Utility/OpenApi/RequestBodyContentOperationTransformer.cs +++ b/src/Exceptionless.Web/Utility/OpenApi/RequestBodyContentOperationTransformer.cs @@ -37,20 +37,7 @@ public Task TransformAsync(OpenApiOperation operation, OpenApiOperationTransform { ["multipart/form-data"] = new() { - Schema = new OpenApiSchema - { - Type = JsonSchemaType.Object, - Required = new HashSet { multipartFileUploadAttribute.FileParameterName }, - Properties = new Dictionary - { - [multipartFileUploadAttribute.FileParameterName] = new OpenApiSchema - { - Type = JsonSchemaType.String, - Format = "binary", - Description = "The image file to upload." - } - } - } + Schema = CreateMultipartSchema(multipartFileUploadAttribute) } } }; @@ -82,6 +69,33 @@ public Task TransformAsync(OpenApiOperation operation, OpenApiOperationTransform return Task.CompletedTask; } + + private static OpenApiSchema CreateMultipartSchema(MultipartFileUploadAttribute attribute) + { + var required = new HashSet { attribute.FileParameterName }; + var properties = new Dictionary + { + [attribute.FileParameterName] = new OpenApiSchema + { + Type = JsonSchemaType.String, + Format = "binary", + Description = attribute.FileDescription + } + }; + + foreach (string parameterName in attribute.RequiredStringParameterNames) + { + required.Add(parameterName); + properties[parameterName] = new OpenApiSchema { Type = JsonSchemaType.String }; + } + + return new OpenApiSchema + { + Type = JsonSchemaType.Object, + Required = required, + Properties = properties + }; + } } /// @@ -99,9 +113,12 @@ public class RequestBodyContentAttribute : Attribute public class MultipartFileUploadAttribute : Attribute { public string FileParameterName { get; } + public string FileDescription { get; init; } = "The image file to upload."; + public IReadOnlyCollection RequiredStringParameterNames { get; } - public MultipartFileUploadAttribute(string fileParameterName = "file") + public MultipartFileUploadAttribute(string fileParameterName = "file", params string[] requiredStringParameterNames) { FileParameterName = fileParameterName; + RequiredStringParameterNames = requiredStringParameterNames; } } diff --git a/tests/Exceptionless.Tests/Controllers/Data/controller-manifest.json b/tests/Exceptionless.Tests/Controllers/Data/controller-manifest.json index 5bc93034e3..4f10ade6cc 100644 --- a/tests/Exceptionless.Tests/Controllers/Data/controller-manifest.json +++ b/tests/Exceptionless.Tests/Controllers/Data/controller-manifest.json @@ -2304,6 +2304,53 @@ ], "ExcludeFromDescription": false }, + { + "Controller": "SourceMapController", + "Action": "GetAsync", + "HttpMethod": "GET", + "Route": "/api/v2/projects/{projectId:objectid}/source-maps", + "Authorization": [ + "Authorize(Policy=UserPolicy)" + ], + "Consumes": [], + "Produces": [ + "application/json", + "application/problem\u002Bjson" + ], + "ExcludeFromDescription": false + }, + { + "Controller": "SourceMapController", + "Action": "PostAsync", + "HttpMethod": "POST", + "Route": "/api/v2/projects/{projectId:objectid}/source-maps", + "Authorization": [ + "Authorize(Policy=UserPolicy)" + ], + "Consumes": [ + "multipart/form-data" + ], + "Produces": [ + "application/json", + "application/problem\u002Bjson" + ], + "ExcludeFromDescription": false + }, + { + "Controller": "SourceMapController", + "Action": "DeleteAsync", + "HttpMethod": "DELETE", + "Route": "/api/v2/projects/{projectId:objectid}/source-maps/{sourceMapId}", + "Authorization": [ + "Authorize(Policy=UserPolicy)" + ], + "Consumes": [], + "Produces": [ + "application/json", + "application/problem\u002Bjson" + ], + "ExcludeFromDescription": false + }, { "Controller": "StackController", "Action": "GetByProjectAsync", diff --git a/tests/Exceptionless.Tests/Controllers/Data/openapi.json b/tests/Exceptionless.Tests/Controllers/Data/openapi.json index 57f7ade478..3179aec7fc 100644 --- a/tests/Exceptionless.Tests/Controllers/Data/openapi.json +++ b/tests/Exceptionless.Tests/Controllers/Data/openapi.json @@ -696,7 +696,7 @@ "Token" ], "summary": "Create for organization", - "description": "This is a helper action that makes it easier to create a token for a specific organization.\r\nYou may also specify a scope when creating a token. There are three valid scopes: client, user and admin.", + "description": "This is a helper action that makes it easier to create a token for a specific organization.\nYou may also specify a scope when creating a token. There are three valid scopes: client, user and admin.", "parameters": [ { "name": "organizationId", @@ -820,7 +820,7 @@ "Token" ], "summary": "Create for project", - "description": "This is a helper action that makes it easier to create a token for a specific project.\r\nYou may also specify a scope when creating a token. There are three valid scopes: client, user and admin.", + "description": "This is a helper action that makes it easier to create a token for a specific project.\nYou may also specify a scope when creating a token. There are three valid scopes: client, user and admin.", "parameters": [ { "name": "projectId", @@ -1327,7 +1327,7 @@ "Auth" ], "summary": "Login", - "description": "Log in with your email address and password to generate a token scoped with your users roles.\r\n\r\n```{ \"email\": \"noreply@exceptionless.io\", \"password\": \"exceptionless\" }```\r\n\r\nThis token can then be used to access the api. You can use this token in the header (bearer authentication)\r\nor append it onto the query string: ?access_token=MY_TOKEN\r\n\r\nPlease note that you can also use this token on the documentation site by placing it in the\r\nheaders api_key input box.", + "description": "Log in with your email address and password to generate a token scoped with your users roles.\n\n```{ \"email\": \"noreply@exceptionless.io\", \"password\": \"exceptionless\" }```\n\nThis token can then be used to access the api. You can use this token in the header (bearer authentication)\nor append it onto the query string: ?access_token=MY_TOKEN\n\nPlease note that you can also use this token on the documentation site by placing it in the\nheaders api_key input box.", "requestBody": { "content": { "application/json": { @@ -2202,7 +2202,7 @@ "Event" ], "summary": "Submit event by POST", - "description": " You can create an event by posting any uncompressed or compressed (gzip or deflate) string or json object. If we know how to handle it\r\n we will create a new event. If none of the JSON properties match the event object then we will create a new event and place your JSON\r\n object into the events data collection.\r\n\r\n You can also post a multi-line string. We automatically split strings by the \\n character and create a new log event for every line.\r\n\r\n Simple event:\r\n ```{ \"message\": \"Exceptionless is amazing!\" }```\r\n\r\n Simple log event with user identity:\r\n ```{\r\n \"type\": \"log\",\r\n \"message\": \"Exceptionless is amazing!\",\r\n \"date\":\"2030-01-01T12:00:00.0000000-05:00\",\r\n \"@user\":{ \"identity\":\"123456789\", \"name\": \"Test User\" }\r\n}```\r\n\r\n Multiple events from string content:\r\n ```Exceptionless is amazing!\r\nExceptionless is really amazing!```\r\n\r\n Simple error:\r\n ```{\r\n \"type\": \"error\",\r\n \"date\":\"2030-01-01T12:00:00.0000000-05:00\",\r\n \"@simple_error\": {\r\n \"message\": \"Simple Exception\",\r\n \"type\": \"System.Exception\",\r\n \"stack_trace\": \" at Client.Tests.ExceptionlessClientTests.CanSubmitSimpleException() in ExceptionlessClientTests.cs:line 77\"\r\n }\r\n}```", + "description": " You can create an event by posting any uncompressed or compressed (gzip or deflate) string or json object. If we know how to handle it\n we will create a new event. If none of the JSON properties match the event object then we will create a new event and place your JSON\n object into the events data collection.\n\n You can also post a multi-line string. We automatically split strings by the \\n character and create a new log event for every line.\n\n Simple event:\n ```{ \"message\": \"Exceptionless is amazing!\" }```\n\n Simple log event with user identity:\n ```{\n \"type\": \"log\",\n \"message\": \"Exceptionless is amazing!\",\n \"date\":\"2030-01-01T12:00:00.0000000-05:00\",\n \"@user\":{ \"identity\":\"123456789\", \"name\": \"Test User\" }\n}```\n\n Multiple events from string content:\n ```Exceptionless is amazing!\nExceptionless is really amazing!```\n\n Simple error:\n ```{\n \"type\": \"error\",\n \"date\":\"2030-01-01T12:00:00.0000000-05:00\",\n \"@simple_error\": {\n \"message\": \"Simple Exception\",\n \"type\": \"System.Exception\",\n \"stack_trace\": \" at Client.Tests.ExceptionlessClientTests.CanSubmitSimpleException() in ExceptionlessClientTests.cs:line 77\"\n }\n}```", "parameters": [ { "name": "userAgent", @@ -2519,7 +2519,7 @@ "Event" ], "summary": "Submit event by POST for a specific project", - "description": " You can create an event by posting any uncompressed or compressed (gzip or deflate) string or json object. If we know how to handle it\r\n we will create a new event. If none of the JSON properties match the event object then we will create a new event and place your JSON\r\n object into the events data collection.\r\n\r\n You can also post a multi-line string. We automatically split strings by the \\n character and create a new log event for every line.\r\n\r\n Simple event:\r\n ```{ \"message\": \"Exceptionless is amazing!\" }```\r\n\r\n Simple log event with user identity:\r\n ```{\r\n \"type\": \"log\",\r\n \"message\": \"Exceptionless is amazing!\",\r\n \"date\":\"2030-01-01T12:00:00.0000000-05:00\",\r\n \"@user\":{ \"identity\":\"123456789\", \"name\": \"Test User\" }\r\n}```\r\n\r\n Multiple events from string content:\r\n ```Exceptionless is amazing!\r\nExceptionless is really amazing!```\r\n\r\n Simple error:\r\n ```{\r\n \"type\": \"error\",\r\n \"date\":\"2030-01-01T12:00:00.0000000-05:00\",\r\n \"@simple_error\": {\r\n \"message\": \"Simple Exception\",\r\n \"type\": \"System.Exception\",\r\n \"stack_trace\": \" at Client.Tests.ExceptionlessClientTests.CanSubmitSimpleException() in ExceptionlessClientTests.cs:line 77\"\r\n }\r\n}```", + "description": " You can create an event by posting any uncompressed or compressed (gzip or deflate) string or json object. If we know how to handle it\n we will create a new event. If none of the JSON properties match the event object then we will create a new event and place your JSON\n object into the events data collection.\n\n You can also post a multi-line string. We automatically split strings by the \\n character and create a new log event for every line.\n\n Simple event:\n ```{ \"message\": \"Exceptionless is amazing!\" }```\n\n Simple log event with user identity:\n ```{\n \"type\": \"log\",\n \"message\": \"Exceptionless is amazing!\",\n \"date\":\"2030-01-01T12:00:00.0000000-05:00\",\n \"@user\":{ \"identity\":\"123456789\", \"name\": \"Test User\" }\n}```\n\n Multiple events from string content:\n ```Exceptionless is amazing!\nExceptionless is really amazing!```\n\n Simple error:\n ```{\n \"type\": \"error\",\n \"date\":\"2030-01-01T12:00:00.0000000-05:00\",\n \"@simple_error\": {\n \"message\": \"Simple Exception\",\n \"type\": \"System.Exception\",\n \"stack_trace\": \" at Client.Tests.ExceptionlessClientTests.CanSubmitSimpleException() in ExceptionlessClientTests.cs:line 77\"\n }\n}```", "parameters": [ { "name": "projectId", @@ -3931,7 +3931,7 @@ "Event" ], "summary": "Submit event by GET", - "description": "You can submit an event using an HTTP GET and query string parameters. Any unknown query string parameters will be added to the extended data of the event.\r\n\r\nFeature usage named build with a duration of 10:\r\n```/events/submit?access_token=YOUR_API_KEY&type=usage&source=build&value=10```\r\n\r\nLog with message, geo and extended data\r\n```/events/submit?access_token=YOUR_API_KEY&type=log&message=Hello World&source=server01&geo=32.85,-96.9613&randomproperty=true```", + "description": "You can submit an event using an HTTP GET and query string parameters. Any unknown query string parameters will be added to the extended data of the event.\n\nFeature usage named build with a duration of 10:\n```/events/submit?access_token=YOUR_API_KEY&type=usage&source=build&value=10```\n\nLog with message, geo and extended data\n```/events/submit?access_token=YOUR_API_KEY&type=log&message=Hello World&source=server01&geo=32.85,-96.9613&randomproperty=true```", "parameters": [ { "name": "type", @@ -4065,7 +4065,7 @@ "Event" ], "summary": "Submit event type by GET", - "description": "You can submit an event using an HTTP GET and query string parameters.\r\n\r\nFeature usage event named build with a value of 10:\r\n```/events/submit/usage?access_token=YOUR_API_KEY&source=build&value=10```\r\n\r\nLog event with message, geo and extended data\r\n```/events/submit/log?access_token=YOUR_API_KEY&message=Hello World&source=server01&geo=32.85,-96.9613&randomproperty=true```", + "description": "You can submit an event using an HTTP GET and query string parameters.\n\nFeature usage event named build with a value of 10:\n```/events/submit/usage?access_token=YOUR_API_KEY&source=build&value=10```\n\nLog event with message, geo and extended data\n```/events/submit/log?access_token=YOUR_API_KEY&message=Hello World&source=server01&geo=32.85,-96.9613&randomproperty=true```", "parameters": [ { "name": "type", @@ -4201,7 +4201,7 @@ "Event" ], "summary": "Submit event type by GET for a specific project", - "description": "You can submit an event using an HTTP GET and query string parameters.\r\n\r\nFeature usage named build with a duration of 10:\r\n```/projects/{projectId}/events/submit?access_token=YOUR_API_KEY&type=usage&source=build&value=10```\r\n\r\nLog with message, geo and extended data\r\n```/projects/{projectId}/events/submit?access_token=YOUR_API_KEY&type=log&message=Hello World&source=server01&geo=32.85,-96.9613&randomproperty=true```", + "description": "You can submit an event using an HTTP GET and query string parameters.\n\nFeature usage named build with a duration of 10:\n```/projects/{projectId}/events/submit?access_token=YOUR_API_KEY&type=usage&source=build&value=10```\n\nLog with message, geo and extended data\n```/projects/{projectId}/events/submit?access_token=YOUR_API_KEY&type=log&message=Hello World&source=server01&geo=32.85,-96.9613&randomproperty=true```", "parameters": [ { "name": "projectId", @@ -4337,7 +4337,7 @@ "Event" ], "summary": "Submit event type by GET for a specific project", - "description": "You can submit an event using an HTTP GET and query string parameters.\r\n\r\nFeature usage named build with a duration of 10:\r\n```/projects/{projectId}/events/submit?access_token=YOUR_API_KEY&type=usage&source=build&value=10```\r\n\r\nLog with message, geo and extended data\r\n```/projects/{projectId}/events/submit?access_token=YOUR_API_KEY&type=log&message=Hello World&source=server01&geo=32.85,-96.9613&randomproperty=true```", + "description": "You can submit an event using an HTTP GET and query string parameters.\n\nFeature usage named build with a duration of 10:\n```/projects/{projectId}/events/submit?access_token=YOUR_API_KEY&type=usage&source=build&value=10```\n\nLog with message, geo and extended data\n```/projects/{projectId}/events/submit?access_token=YOUR_API_KEY&type=log&message=Hello World&source=server01&geo=32.85,-96.9613&randomproperty=true```", "parameters": [ { "name": "projectId", @@ -5585,7 +5585,7 @@ "Organization" ], "summary": "Change plan", - "description": "Upgrades or downgrades the organization's plan.\r\nAccepts parameters via JSON body (preferred) or query string (legacy).", + "description": "Upgrades or downgrades the organization's plan.\nAccepts parameters via JSON body (preferred) or query string (legacy).", "parameters": [ { "name": "id", @@ -7275,6 +7275,129 @@ } } }, + "/api/v2/projects/{projectId}/source-maps": { + "get": { + "tags": [ + "SourceMap" + ], + "summary": "Get source maps for a project.", + "parameters": [ + { + "name": "projectId", + "in": "path", + "required": true, + "schema": { + "pattern": "^[a-zA-Z\\d]{24,36}$", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SourceMapArtifact" + } + } + } + } + } + } + }, + "post": { + "tags": [ + "SourceMap" + ], + "summary": "Upload a source map for a generated JavaScript file.", + "parameters": [ + { + "name": "projectId", + "in": "path", + "description": "The identifier of the project.", + "required": true, + "schema": { + "pattern": "^[a-zA-Z\\d]{24,36}$", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "required": [ + "file", + "generated_file_url" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "description": "The source map file to upload.", + "format": "binary" + }, + "generated_file_url": { + "type": "string" + } + } + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "Created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SourceMapArtifact" + } + } + } + } + } + } + }, + "/api/v2/projects/{projectId}/source-maps/{sourceMapId}": { + "delete": { + "tags": [ + "SourceMap" + ], + "summary": "Delete a source map from a project.", + "parameters": [ + { + "name": "projectId", + "in": "path", + "required": true, + "schema": { + "pattern": "^[a-zA-Z\\d]{24,36}$", + "type": "string" + } + }, + { + "name": "sourceMapId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "No Content", + "content": { + "application/json": { } + } + } + } + } + }, "/api/v2/stacks/{id}": { "get": { "tags": [ @@ -9604,7 +9727,7 @@ "null", "string" ], - "description": "The event type (ie. error, log message, feature usage). Check KnownTypes for standard event types.\r\nNullable in transit; the pipeline infers a default before save. Validated as required on repository save." + "description": "The event type (ie. error, log message, feature usage). Check KnownTypes for standard event types.\nNullable in transit; the pipeline infers a default before save. Validated as required on repository save." }, "source": { "maxLength": 2000, @@ -9846,6 +9969,47 @@ } } }, + "SourceMapArtifact": { + "required": [ + "id", + "generated_file_url", + "size", + "is_auto_downloaded", + "created_utc" + ], + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "generated_file_url": { + "type": "string" + }, + "source_map_url": { + "type": [ + "null", + "string" + ] + }, + "file_name": { + "type": [ + "null", + "string" + ] + }, + "size": { + "type": "integer", + "format": "int64" + }, + "is_auto_downloaded": { + "type": "boolean" + }, + "created_utc": { + "type": "string", + "format": "date-time" + } + } + }, "Stack": { "required": [ "organization_id", @@ -11292,6 +11456,9 @@ { "name": "Project" }, + { + "name": "SourceMap" + }, { "name": "Stack" }, @@ -11299,4 +11466,4 @@ "name": "User" } ] -} +} \ No newline at end of file diff --git a/tests/Exceptionless.Tests/Controllers/OpenApiControllerTests.cs b/tests/Exceptionless.Tests/Controllers/OpenApiControllerTests.cs index 09ee908ed1..2aef157979 100644 --- a/tests/Exceptionless.Tests/Controllers/OpenApiControllerTests.cs +++ b/tests/Exceptionless.Tests/Controllers/OpenApiControllerTests.cs @@ -62,6 +62,14 @@ public async Task GetOpenApiJson_ContainsExpectedRoutesOperationsAndResponses() Assert.True(projectsPost.TryGetProperty("requestBody", out _)); AssertResponseCodes(projectsPost, "201"); + Assert.True(paths.TryGetProperty("/api/v2/projects/{projectId}/source-maps", out var sourceMapsPath)); + Assert.True(sourceMapsPath.TryGetProperty("post", out var sourceMapsPost)); + var sourceMapForm = sourceMapsPost.GetProperty("requestBody").GetProperty("content").GetProperty("multipart/form-data").GetProperty("schema"); + Assert.True(sourceMapForm.GetProperty("properties").TryGetProperty("generated_file_url", out _)); + Assert.Contains(sourceMapForm.GetProperty("required").EnumerateArray(), item => item.GetString() == "generated_file_url"); + AssertResponseCodes(sourceMapsPost, "201"); + AssertResponseCodes(paths.GetProperty("/api/v2/projects/{projectId}/source-maps/{sourceMapId}").GetProperty("delete"), "204"); + Assert.True(paths.TryGetProperty("/api/v2/events/by-ref/{referenceId}/user-description", out var userDescriptionPath)); Assert.True(userDescriptionPath.TryGetProperty("post", out var userDescriptionPost)); Assert.True(userDescriptionPost.TryGetProperty("requestBody", out _)); diff --git a/tests/Exceptionless.Tests/Controllers/SourceMapControllerTests.cs b/tests/Exceptionless.Tests/Controllers/SourceMapControllerTests.cs new file mode 100644 index 0000000000..9661f69ce6 --- /dev/null +++ b/tests/Exceptionless.Tests/Controllers/SourceMapControllerTests.cs @@ -0,0 +1,87 @@ +using System.Net; +using System.Text; +using Exceptionless.Core.Services.SourceMaps; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Xunit; + +namespace Exceptionless.Tests.Controllers; + +public sealed class SourceMapControllerTests : IntegrationTestsBase +{ + private const string GeneratedFileUrl = "https://cdn.example.com/assets/app.min.js"; + private static readonly byte[] SourceMap = Encoding.UTF8.GetBytes("""{"version":3,"sources":["src/app.ts"],"names":["meaningfulFunction"],"mappings":"AAAAA"}"""); + + public SourceMapControllerTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + await GetService().CreateDataAsync(); + } + + [Fact] + public async Task PostAsync_WithValidSourceMap_CanListAndDeleteArtifact() + { + using var content = CreateSourceMapContent(SourceMap); + + var uploaded = await SendRequestAsAsync(request => request + .Post() + .AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "source-maps") + .Content(content) + .StatusCodeShouldBeCreated()); + + Assert.NotNull(uploaded); + Assert.Equal(GeneratedFileUrl, uploaded.GeneratedFileUrl); + Assert.False(uploaded.IsAutoDownloaded); + + var artifacts = await SendRequestAsAsync>(request => request + .AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "source-maps") + .StatusCodeShouldBeOk()); + Assert.NotNull(artifacts); + Assert.Single(artifacts); + + await SendRequestAsync(request => request + .Delete() + .AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "source-maps", uploaded.Id) + .StatusCodeShouldBeNoContent()); + } + + [Fact] + public async Task PostAsync_WithInvalidSourceMap_ReturnsUnprocessableEntity() + { + using var content = CreateSourceMapContent(Encoding.UTF8.GetBytes("not a source map")); + + await SendRequestAsync(request => request + .Post() + .AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "source-maps") + .Content(content) + .StatusCodeShouldBeUnprocessableEntity()); + } + + [Fact] + public Task GetAsync_ProjectOutsideUserOrganization_ReturnsNotFound() + { + return SendRequestAsync(request => request + .AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.FREE_PROJECT_ID, "source-maps") + .StatusCodeShouldBeNotFound()); + } + + private static MultipartFormDataContent CreateSourceMapContent(byte[] sourceMap) + { + var fileContent = new ByteArrayContent(sourceMap); + fileContent.Headers.ContentType = new("application/json"); + + var content = new MultipartFormDataContent(); + content.Add(new StringContent(GeneratedFileUrl), "generated_file_url"); + content.Add(fileContent, "file", "app.min.js.map"); + return content; + } +} diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs index c7300641a2..14e37b44ae 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs @@ -248,6 +248,9 @@ public async Task CanCleanupSoftDeletedProject() var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); var persistentEvent = await _eventRepository.AddAsync(_eventData.GenerateEvent(organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + string sourceMapPath = $"source-maps/{project.Id}/app.map"; + await using (var sourceMap = new MemoryStream([0x7B, 0x7D])) + await _fileStorage.SaveFileAsync(sourceMapPath, sourceMap, TestCancellationToken); await _job.RunAsync(TestCancellationToken); @@ -255,6 +258,7 @@ public async Task CanCleanupSoftDeletedProject() Assert.Null(await _projectRepository.GetByIdAsync(project.Id, o => o.IncludeSoftDeletes())); Assert.Null(await _stackRepository.GetByIdAsync(stack.Id, o => o.IncludeSoftDeletes())); Assert.Null(await _eventRepository.GetByIdAsync(persistentEvent.Id, o => o.IncludeSoftDeletes())); + Assert.False(await _fileStorage.ExistsAsync(sourceMapPath)); } [Fact] diff --git a/tests/Exceptionless.Tests/Services/SourceMaps/SourceMapDocumentTests.cs b/tests/Exceptionless.Tests/Services/SourceMaps/SourceMapDocumentTests.cs new file mode 100644 index 0000000000..3a43dfe0c0 --- /dev/null +++ b/tests/Exceptionless.Tests/Services/SourceMaps/SourceMapDocumentTests.cs @@ -0,0 +1,77 @@ +using System.Text; +using System.Text.Json; +using Exceptionless.Core.Services.SourceMaps; +using Xunit; + +namespace Exceptionless.Tests.Services.SourceMaps; + +public sealed class SourceMapDocumentTests +{ + [Fact] + public void FindOriginalLocation_MappedSegment_ReturnsOriginalSourceAndName() + { + byte[] sourceMap = Encoding.UTF8.GetBytes("""{"version":3,"sources":["src/app.ts"],"names":["meaningfulFunction"],"mappings":"AAAAA"}"""); + var document = SourceMapDocument.Parse(sourceMap); + + var location = document.FindOriginalLocation(0, 12); + + Assert.NotNull(location); + Assert.Equal("src/app.ts", location.Source); + Assert.Equal(0, location.Line); + Assert.Equal(0, location.Column); + Assert.Equal("meaningfulFunction", location.Name); + } + + [Fact] + public void FindOriginalLocation_WithSourceRoot_CombinesRelativeSource() + { + byte[] sourceMap = Encoding.UTF8.GetBytes("""{"version":3,"sourceRoot":"https://cdn.example.com/source/","sources":["app.ts"],"names":[],"mappings":"AAAA"}"""); + var document = SourceMapDocument.Parse(sourceMap); + + var location = document.FindOriginalLocation(0, 0); + + Assert.NotNull(location); + Assert.Equal("https://cdn.example.com/source/app.ts", location.Source); + } + + [Fact] + public void FindOriginalLocation_AfterUnmappedSegment_ReturnsNull() + { + byte[] sourceMap = Encoding.UTF8.GetBytes("""{"version":3,"sources":["src/app.ts"],"names":[],"mappings":"AAAA,U"}"""); + var document = SourceMapDocument.Parse(sourceMap); + + var location = document.FindOriginalLocation(0, 12); + + Assert.Null(location); + } + + [Fact] + public void Parse_IndexedSourceMap_ThrowsJsonException() + { + byte[] sourceMap = Encoding.UTF8.GetBytes("""{"version":3,"sections":[],"sources":[],"names":[],"mappings":""}"""); + + var exception = Assert.Throws(() => SourceMapDocument.Parse(sourceMap)); + + Assert.Contains("Indexed source maps", exception.Message); + } + + [Fact] + public void Parse_OversizedVlqValue_ThrowsJsonException() + { + byte[] sourceMap = Encoding.UTF8.GetBytes("""{"version":3,"sources":[],"names":[],"mappings":"gggggggA"}"""); + + var exception = Assert.Throws(() => SourceMapDocument.Parse(sourceMap)); + + Assert.Contains("too large", exception.Message); + } + + [Fact] + public void Parse_TooManyMappingSegments_ThrowsJsonException() + { + byte[] sourceMap = Encoding.UTF8.GetBytes("""{"version":3,"sources":["app.ts"],"names":[],"mappings":"AAAA,CAAC"}"""); + + var exception = Assert.Throws(() => SourceMapDocument.Parse(sourceMap, maximumSegments: 1)); + + Assert.Contains("too many mapping segments", exception.Message); + } +} diff --git a/tests/Exceptionless.Tests/Services/SourceMaps/SourceMapServiceTests.cs b/tests/Exceptionless.Tests/Services/SourceMaps/SourceMapServiceTests.cs new file mode 100644 index 0000000000..818fb48870 --- /dev/null +++ b/tests/Exceptionless.Tests/Services/SourceMaps/SourceMapServiceTests.cs @@ -0,0 +1,158 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using Exceptionless.Core; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.Data; +using Exceptionless.Core.Plugins.EventProcessor; +using Exceptionless.Core.Plugins.EventProcessor.Default; +using Exceptionless.Core.Services.SourceMaps; +using Foundatio.Caching; +using Foundatio.Serializer; +using Foundatio.Storage; +using Microsoft.Extensions.Logging; +using Xunit; + +namespace Exceptionless.Tests.Services.SourceMaps; + +public sealed class SourceMapServiceTests : TestWithServices +{ + private const string ProjectId = "507f1f77bcf86cd799439011"; + private const string GeneratedFileUrl = "https://cdn.example.com/assets/app.min.js"; + private static readonly byte[] SourceMap = Encoding.UTF8.GetBytes("""{"version":3,"sources":["src/app.ts"],"names":["meaningfulFunction"],"mappings":"AAAAA"}"""); + + public SourceMapServiceTests(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SymbolicateAsync_WithUploadedSourceMap_RewritesFrameAndPreservesGeneratedLocation() + { + var service = GetService(); + await using var sourceMapStream = new MemoryStream(SourceMap); + var artifact = await service.SaveUploadedAsync(ProjectId, GeneratedFileUrl, "app.min.js.map", sourceMapStream, TestContext.Current.CancellationToken); + var error = CreateError(); + + bool changed = await service.SymbolicateAsync(ProjectId, error, TestContext.Current.CancellationToken); + + Assert.True(changed); + var frame = Assert.Single(error.StackTrace!); + Assert.Equal("meaningfulFunction", frame.Name); + Assert.Equal("src/app.ts", frame.FileName); + Assert.Equal(1, frame.LineNumber); + Assert.Equal(1, frame.Column); + var generatedLocation = Assert.IsType(frame.Data!["@source_map"]); + Assert.Equal(GeneratedFileUrl, generatedLocation["generated_file_name"]); + Assert.Equal(artifact.Id, generatedLocation["source_map_id"]); + } + + [Fact] + public async Task GetArtifactsAsync_AfterUploadAndDelete_ReflectsStoredArtifact() + { + var service = GetService(); + await using var sourceMapStream = new MemoryStream(SourceMap); + var uploaded = await service.SaveUploadedAsync(ProjectId, GeneratedFileUrl, "app.min.js.map", sourceMapStream, TestContext.Current.CancellationToken); + + var artifacts = await service.GetArtifactsAsync(ProjectId, TestContext.Current.CancellationToken); + bool deleted = await service.DeleteArtifactAsync(ProjectId, uploaded.Id, TestContext.Current.CancellationToken); + + var artifact = Assert.Single(artifacts); + Assert.Equal(GeneratedFileUrl, artifact.GeneratedFileUrl); + Assert.False(artifact.IsAutoDownloaded); + Assert.True(deleted); + Assert.Empty(await service.GetArtifactsAsync(ProjectId, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task SymbolicateAsync_WithPublicSourceMapHeader_DownloadsAndStoresSourceMap() + { + var requestedUris = new List(); + var handler = new DelegateHandler(request => + { + requestedUris.Add(request.RequestUri!); + if (request.RequestUri == new Uri(GeneratedFileUrl)) + { + var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("minified") }; + response.Headers.TryAddWithoutValidation("SourceMap", "app.min.js.map"); + return response; + } + + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(SourceMap) }; + }); + using var httpClient = new HttpClient(handler); + var service = new SourceMapService( + new TestHttpClientFactory(httpClient), + GetService(), + GetService(), + GetService(), + GetService(), + GetService(), + GetService>()); + var error = CreateError(); + + bool changed = await service.SymbolicateAsync(ProjectId, error, TestContext.Current.CancellationToken); + + Assert.True(changed); + Assert.Equal([new Uri(GeneratedFileUrl), new Uri("https://cdn.example.com/assets/app.min.js.map")], requestedUris); + var artifact = Assert.Single(await service.GetArtifactsAsync(ProjectId, TestContext.Current.CancellationToken)); + Assert.True(artifact.IsAutoDownloaded); + Assert.Equal("https://cdn.example.com/assets/app.min.js.map", artifact.SourceMapUrl); + } + + [Fact] + public async Task EventProcessingAsync_WithUploadedSourceMap_UsesOriginalNameForStackSignature() + { + var service = GetService(); + await using var sourceMapStream = new MemoryStream(SourceMap); + await service.SaveUploadedAsync(ProjectId, GeneratedFileUrl, "app.min.js.map", sourceMapStream, TestContext.Current.CancellationToken); + var serializer = GetService(); + var options = GetService(); + var loggerFactory = GetService(); + var sourceMapPlugin = new SourceMapPlugin(service, serializer, options, loggerFactory); + var errorPlugin = new ErrorPlugin(serializer, options, loggerFactory); + var persistentEvent = new PersistentEvent { Type = Event.KnownTypes.Error }; + persistentEvent.SetError(CreateError()); + var context = new EventContext( + persistentEvent, + new Organization { Id = "507f1f77bcf86cd799439012" }, + new Project { Id = ProjectId, OrganizationId = "507f1f77bcf86cd799439012" }); + + await sourceMapPlugin.EventProcessingAsync(context); + await errorPlugin.EventProcessingAsync(context); + + Assert.Equal("meaningfulFunction()", context.StackSignatureData["Method"]); + Assert.True(sourceMapPlugin.HandleError(new IOException("Unavailable"), context)); + } + + private static Error CreateError() + { + return new Error + { + Message = "Test error", + Type = "TypeError", + StackTrace = + [ + new StackFrame + { + Name = "a", + FileName = GeneratedFileUrl, + LineNumber = 1, + Column = 1 + } + ] + }; + } + + private sealed class TestHttpClientFactory(HttpClient httpClient) : IHttpClientFactory + { + public HttpClient CreateClient(string name) => httpClient; + } + + private sealed class DelegateHandler(Func handler) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return Task.FromResult(handler(request)); + } + } +} From 5e7dd25ce757b5ce174df587ea055d9f5621e506 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 14 Jul 2026 18:13:27 -0500 Subject: [PATCH 02/15] Harden source map symbolication lifecycle --- docs/docs/source-maps.md | 4 +- .../Configuration/SourceMapOptions.cs | 29 +- .../Exceptionless.Core.csproj | 1 + .../Services/SourceMaps/SourceMapContent.cs | 20 + .../Services/SourceMaps/SourceMapDocument.cs | 22 +- .../SourceMaps/SourceMapDownloader.cs | 161 +++++++ .../Services/SourceMaps/SourceMapService.cs | 441 ++++++++---------- .../Services/SourceMaps/SourceMapStorage.cs | 132 ++++++ .../[projectId]/source-maps/+page.svelte | 11 +- .../SourceMaps/SourceMapServiceTests.cs | 252 ++++++++++ 10 files changed, 805 insertions(+), 268 deletions(-) create mode 100644 src/Exceptionless.Core/Services/SourceMaps/SourceMapContent.cs create mode 100644 src/Exceptionless.Core/Services/SourceMaps/SourceMapDownloader.cs create mode 100644 src/Exceptionless.Core/Services/SourceMaps/SourceMapStorage.cs diff --git a/docs/docs/source-maps.md b/docs/docs/source-maps.md index f967d41d8d..4dcd5c20cf 100644 --- a/docs/docs/source-maps.md +++ b/docs/docs/source-maps.md @@ -10,7 +10,9 @@ Exceptionless uses source maps to turn minified JavaScript stack frames into the No domain allowlist or project setup is required for public source maps. When an error contains an absolute HTTPS JavaScript URL, Exceptionless checks the generated file's `SourceMap` or `X-SourceMap` response header and its `sourceMappingURL` comment. If neither is present, it also checks the conventional `.map` URL. -Downloaded maps are validated and cached in project-scoped file storage. Exceptionless only makes anonymous HTTPS requests to public network addresses. Redirects are revalidated, and downloads have time, redirect, size, concurrency, and per-project rate limits. Self-hosted installations can tune these safeguards under the `SourceMaps` configuration section. +Downloaded maps are validated and cached in project-scoped file storage. Automatically downloaded maps are revalidated after one hour so a stable generated-file URL cannot retain a map from an older deployment indefinitely. If refresh fails, Exceptionless leaves the generated frame unchanged instead of risking a misleading stack trace from the stale map. + +Exceptionless only makes anonymous HTTPS requests to public network addresses. Redirects are revalidated, and downloads have time, redirect, size, concurrency, and per-project rate limits. Parsed maps use a bounded in-memory cache. Self-hosted installations can tune these safeguards under the `SourceMaps` configuration section, including `AutoDownloadRefreshIntervalMinutes`, `ParsedSourceMapCacheLifetimeMinutes`, and `MaximumParsedSourceMapCacheSize`. ## Uploading a source map diff --git a/src/Exceptionless.Core/Configuration/SourceMapOptions.cs b/src/Exceptionless.Core/Configuration/SourceMapOptions.cs index b6aba0aab2..7017bc603c 100644 --- a/src/Exceptionless.Core/Configuration/SourceMapOptions.cs +++ b/src/Exceptionless.Core/Configuration/SourceMapOptions.cs @@ -14,9 +14,14 @@ public sealed class SourceMapOptions public int MaximumAutoDownloadsPerProjectPerHour { get; internal set; } public int MaximumFramesPerError { get; internal set; } public int MaximumProcessingTimeMilliseconds { get; internal set; } + public int AutoDownloadRefreshIntervalMinutes { get; internal set; } + public int ParsedSourceMapCacheLifetimeMinutes { get; internal set; } + public long MaximumParsedSourceMapCacheSize { get; internal set; } public TimeSpan RequestTimeout => TimeSpan.FromMilliseconds(RequestTimeoutMilliseconds); public TimeSpan MaximumProcessingTime => TimeSpan.FromMilliseconds(MaximumProcessingTimeMilliseconds); + public TimeSpan AutoDownloadRefreshInterval => TimeSpan.FromMinutes(AutoDownloadRefreshIntervalMinutes); + public TimeSpan ParsedSourceMapCacheLifetime => TimeSpan.FromMinutes(ParsedSourceMapCacheLifetimeMinutes); public static SourceMapOptions ReadFromConfiguration(IConfiguration configuration) { @@ -24,15 +29,21 @@ public static SourceMapOptions ReadFromConfiguration(IConfiguration configuratio return new SourceMapOptions { EnableAutoDownload = section.GetValue(nameof(EnableAutoDownload), true), - RequestTimeoutMilliseconds = section.GetValue(nameof(RequestTimeoutMilliseconds), 3000), - MaximumGeneratedFileSize = section.GetValue(nameof(MaximumGeneratedFileSize), 5 * 1024 * 1024), - MaximumSourceMapSize = section.GetValue(nameof(MaximumSourceMapSize), 20 * 1024 * 1024), - MaximumMappingSegments = section.GetValue(nameof(MaximumMappingSegments), 1_000_000), - MaximumRedirects = section.GetValue(nameof(MaximumRedirects), 3), - MaximumConcurrentDownloads = section.GetValue(nameof(MaximumConcurrentDownloads), 4), - MaximumAutoDownloadsPerProjectPerHour = section.GetValue(nameof(MaximumAutoDownloadsPerProjectPerHour), 100), - MaximumFramesPerError = section.GetValue(nameof(MaximumFramesPerError), 100), - MaximumProcessingTimeMilliseconds = section.GetValue(nameof(MaximumProcessingTimeMilliseconds), 5000) + RequestTimeoutMilliseconds = ReadPositive(section, nameof(RequestTimeoutMilliseconds), 3000), + MaximumGeneratedFileSize = ReadPositive(section, nameof(MaximumGeneratedFileSize), 5 * 1024 * 1024), + MaximumSourceMapSize = ReadPositive(section, nameof(MaximumSourceMapSize), 20 * 1024 * 1024), + MaximumMappingSegments = ReadPositive(section, nameof(MaximumMappingSegments), 1_000_000), + MaximumRedirects = Math.Max(0, section.GetValue(nameof(MaximumRedirects), 3)), + MaximumConcurrentDownloads = ReadPositive(section, nameof(MaximumConcurrentDownloads), 4), + MaximumAutoDownloadsPerProjectPerHour = Math.Max(0, section.GetValue(nameof(MaximumAutoDownloadsPerProjectPerHour), 100)), + MaximumFramesPerError = ReadPositive(section, nameof(MaximumFramesPerError), 100), + MaximumProcessingTimeMilliseconds = ReadPositive(section, nameof(MaximumProcessingTimeMilliseconds), 5000), + AutoDownloadRefreshIntervalMinutes = ReadPositive(section, nameof(AutoDownloadRefreshIntervalMinutes), 60), + ParsedSourceMapCacheLifetimeMinutes = ReadPositive(section, nameof(ParsedSourceMapCacheLifetimeMinutes), 5), + MaximumParsedSourceMapCacheSize = Math.Max(1, section.GetValue(nameof(MaximumParsedSourceMapCacheSize), 100L * 1024 * 1024)) }; } + + private static int ReadPositive(IConfiguration section, string name, int defaultValue) + => Math.Max(1, section.GetValue(name, defaultValue)); } diff --git a/src/Exceptionless.Core/Exceptionless.Core.csproj b/src/Exceptionless.Core/Exceptionless.Core.csproj index 64605e0c70..712f31f78e 100644 --- a/src/Exceptionless.Core/Exceptionless.Core.csproj +++ b/src/Exceptionless.Core/Exceptionless.Core.csproj @@ -31,6 +31,7 @@ + diff --git a/src/Exceptionless.Core/Services/SourceMaps/SourceMapContent.cs b/src/Exceptionless.Core/Services/SourceMaps/SourceMapContent.cs new file mode 100644 index 0000000000..59306e9d18 --- /dev/null +++ b/src/Exceptionless.Core/Services/SourceMaps/SourceMapContent.cs @@ -0,0 +1,20 @@ +namespace Exceptionless.Core.Services.SourceMaps; + +internal static class SourceMapContent +{ + public static async Task ReadLimitedAsync(Stream stream, int maximumBytes, CancellationToken cancellationToken) + { + using var memoryStream = new MemoryStream(Math.Min(maximumBytes, 64 * 1024)); + byte[] buffer = new byte[81920]; + int read; + while ((read = await stream.ReadAsync(buffer, cancellationToken)) > 0) + { + if (memoryStream.Length + read > maximumBytes) + throw new InvalidOperationException("The file exceeded the configured maximum size."); + + await memoryStream.WriteAsync(buffer.AsMemory(0, read), cancellationToken); + } + + return memoryStream.ToArray(); + } +} diff --git a/src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs b/src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs index d2f2d53c69..5d648aebb6 100644 --- a/src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs +++ b/src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs @@ -9,15 +9,22 @@ public sealed class SourceMapDocument private readonly string[] _names; private readonly string[] _sources; - private SourceMapDocument(string? sourceRoot, string[] sources, string[] names, IReadOnlyList> lines) + private SourceMapDocument( + string? sourceRoot, + string[] sources, + string[] names, + IReadOnlyList> lines, + long estimatedMemorySize) { SourceRoot = sourceRoot; _sources = sources; _names = names; _lines = lines; + EstimatedMemorySize = estimatedMemorySize; } public string? SourceRoot { get; } + internal long EstimatedMemorySize { get; } public static SourceMapDocument Parse(byte[] sourceMap, int maximumSegments = 1_000_000) { @@ -35,7 +42,9 @@ public static SourceMapDocument Parse(byte[] sourceMap, int maximumSegments = 1_ string[] names = root.TryGetProperty("names", out _) ? ReadStringArray(root, "names") : []; string? sourceRoot = root.TryGetProperty("sourceRoot", out var sourceRootElement) ? sourceRootElement.GetString() : null; - return new SourceMapDocument(sourceRoot, sources, names, DecodeMappings(mappings, sources.Length, names.Length, maximumSegments)); + var lines = DecodeMappings(mappings, sources.Length, names.Length, maximumSegments, out int segmentCount); + long estimatedMemorySize = sourceMap.LongLength + (segmentCount * 64L) + (lines.Count * 64L); + return new SourceMapDocument(sourceRoot, sources, names, lines, estimatedMemorySize); } public SourceMapLocation? FindOriginalLocation(int generatedLine, int generatedColumn) @@ -70,7 +79,12 @@ private static string[] ReadStringArray(JsonElement root, string propertyName) .ToArray(); } - private static IReadOnlyList> DecodeMappings(string mappings, int sourceCount, int nameCount, int maximumSegments) + private static IReadOnlyList> DecodeMappings( + string mappings, + int sourceCount, + int nameCount, + int maximumSegments, + out int segmentCount) { if (maximumSegments < 1) throw new ArgumentOutOfRangeException(nameof(maximumSegments)); @@ -80,7 +94,7 @@ private static IReadOnlyList> DecodeMappings(strin int originalLine = 0; int originalColumn = 0; int nameIndex = 0; - int segmentCount = 0; + segmentCount = 0; foreach (string encodedLine in mappings.Split(';')) { diff --git a/src/Exceptionless.Core/Services/SourceMaps/SourceMapDownloader.cs b/src/Exceptionless.Core/Services/SourceMaps/SourceMapDownloader.cs new file mode 100644 index 0000000000..de82d5d3be --- /dev/null +++ b/src/Exceptionless.Core/Services/SourceMaps/SourceMapDownloader.cs @@ -0,0 +1,161 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using Exceptionless.Core.Configuration; + +namespace Exceptionless.Core.Services.SourceMaps; + +internal sealed class SourceMapDownloader +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly SourceMapOptions _options; + + public SourceMapDownloader(IHttpClientFactory httpClientFactory, AppOptions options) + { + _httpClientFactory = httpClientFactory; + _options = options.SourceMapOptions; + } + + public async Task DownloadAsync(Uri generatedFileUri, CancellationToken cancellationToken) + { + using var generatedResponse = await SendAsync( + generatedFileUri, + _options.MaximumGeneratedFileSize, + request => request.Headers.Range = new RangeHeaderValue(null, 64 * 1024), + cancellationToken); + if (!generatedResponse.Response.IsSuccessStatusCode) + return null; + + string? sourceMapReference = GetSourceMapHeader(generatedResponse.Response); + if (String.IsNullOrWhiteSpace(sourceMapReference)) + { + byte[] generatedContent = await SourceMapContent.ReadLimitedAsync( + await generatedResponse.Response.Content.ReadAsStreamAsync(cancellationToken), + _options.MaximumGeneratedFileSize, + cancellationToken); + sourceMapReference = FindSourceMapReference(Encoding.UTF8.GetString(generatedContent)); + } + + if (String.IsNullOrWhiteSpace(sourceMapReference)) + { + var fallbackUriBuilder = new UriBuilder(generatedResponse.Uri) { Path = generatedResponse.Uri.AbsolutePath + ".map" }; + return await DownloadContentAsync(fallbackUriBuilder.Uri, cancellationToken); + } + + if (sourceMapReference.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) + return new DownloadedSourceMap(DecodeDataUri(sourceMapReference, _options.MaximumSourceMapSize), null); + + if (!Uri.TryCreate(generatedResponse.Uri, sourceMapReference, out var sourceMapUri) || !IsAutoDownloadUri(sourceMapUri)) + return null; + + return await DownloadContentAsync(sourceMapUri, cancellationToken); + } + + private async Task DownloadContentAsync(Uri sourceMapUri, CancellationToken cancellationToken) + { + using var result = await SendAsync(sourceMapUri, _options.MaximumSourceMapSize, null, cancellationToken); + if (!result.Response.IsSuccessStatusCode) + return null; + + byte[] content = await SourceMapContent.ReadLimitedAsync( + await result.Response.Content.ReadAsStreamAsync(cancellationToken), + _options.MaximumSourceMapSize, + cancellationToken); + return new DownloadedSourceMap(content, result.Uri.AbsoluteUri); + } + + private async Task SendAsync( + Uri uri, + int maximumBytes, + Action? configureRequest, + CancellationToken cancellationToken) + { + Uri currentUri = uri; + for (int redirectCount = 0; ; redirectCount++) + { + if (!IsAutoDownloadUri(currentUri)) + throw new InvalidOperationException("Source map auto-download only supports public HTTPS URLs."); + + using var request = new HttpRequestMessage(HttpMethod.Get, currentUri); + request.Headers.AcceptEncoding.ParseAdd("gzip, deflate, br"); + configureRequest?.Invoke(request); + + var client = _httpClientFactory.CreateClient(SourceMapService.HttpClientName); + var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + if (response.Content.Headers.ContentLength > maximumBytes) + { + response.Dispose(); + throw new InvalidOperationException("The downloaded file exceeded the configured maximum size."); + } + + if (!IsRedirect(response.StatusCode)) + return new HttpDownloadResult(currentUri, response); + + if (redirectCount >= _options.MaximumRedirects || response.Headers.Location is null) + { + response.Dispose(); + throw new InvalidOperationException("The source map download exceeded the allowed redirects."); + } + + Uri redirectUri = response.Headers.Location.IsAbsoluteUri ? response.Headers.Location : new Uri(currentUri, response.Headers.Location); + response.Dispose(); + currentUri = redirectUri; + } + } + + private static bool IsAutoDownloadUri(Uri uri) + => SourceMapService.TryNormalizeGeneratedFileUrl(uri.AbsoluteUri, requireHttps: true, out _); + + private static bool IsRedirect(HttpStatusCode statusCode) => statusCode is HttpStatusCode.MovedPermanently + or HttpStatusCode.Redirect + or HttpStatusCode.RedirectMethod + or HttpStatusCode.TemporaryRedirect + or HttpStatusCode.PermanentRedirect; + + private static string? GetSourceMapHeader(HttpResponseMessage response) + { + if (response.Headers.TryGetValues("SourceMap", out var sourceMapValues)) + return sourceMapValues.FirstOrDefault(); + if (response.Headers.TryGetValues("X-SourceMap", out var legacySourceMapValues)) + return legacySourceMapValues.FirstOrDefault(); + return null; + } + + private static string? FindSourceMapReference(string generatedContent) + { + const string marker = "sourceMappingURL="; + int markerIndex = generatedContent.LastIndexOf(marker, StringComparison.Ordinal); + if (markerIndex < 0) + return null; + + int start = markerIndex + marker.Length; + int end = generatedContent.IndexOfAny(['\r', '\n'], start); + string value = generatedContent[start..(end < 0 ? generatedContent.Length : end)].Trim(); + if (value.EndsWith("*/", StringComparison.Ordinal)) + value = value[..^2].Trim(); + return value; + } + + private static byte[] DecodeDataUri(string value, int maximumBytes) + { + int commaIndex = value.IndexOf(','); + if (commaIndex < 0) + throw new FormatException("The inline source map data URI is invalid."); + + string metadata = value[..commaIndex]; + string data = value[(commaIndex + 1)..]; + byte[] decoded = metadata.Contains(";base64", StringComparison.OrdinalIgnoreCase) + ? Convert.FromBase64String(data) + : Encoding.UTF8.GetBytes(Uri.UnescapeDataString(data)); + if (decoded.Length > maximumBytes) + throw new InvalidOperationException("The inline source map exceeded the configured maximum size."); + return decoded; + } + + internal sealed record DownloadedSourceMap(byte[] Content, string? SourceMapUrl); + + private sealed record HttpDownloadResult(Uri Uri, HttpResponseMessage Response) : IDisposable + { + public void Dispose() => Response.Dispose(); + } +} diff --git a/src/Exceptionless.Core/Services/SourceMaps/SourceMapService.cs b/src/Exceptionless.Core/Services/SourceMaps/SourceMapService.cs index 3db4fa4a3f..740175f0f2 100644 --- a/src/Exceptionless.Core/Services/SourceMaps/SourceMapService.cs +++ b/src/Exceptionless.Core/Services/SourceMaps/SourceMapService.cs @@ -1,30 +1,31 @@ using System.Collections.Concurrent; -using System.Net; -using System.Net.Http.Headers; -using System.Text; using System.Text.Json; using Exceptionless.Core.Configuration; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Foundatio.Caching; +using Foundatio.Lock; using Foundatio.Storage; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Services.SourceMaps; -public sealed class SourceMapService +public sealed class SourceMapService : IDisposable { public const string HttpClientName = "SourceMaps"; public const int MaximumUploadRequestSize = 21 * 1024 * 1024; private const string SourceMapDataKey = "@source_map"; private static readonly TimeSpan FailureCacheLifetime = TimeSpan.FromMinutes(15); private readonly SemaphoreSlim _downloadSemaphore; - private readonly ConcurrentDictionary>> _sourceMaps = new(StringComparer.Ordinal); - private readonly IHttpClientFactory _httpClientFactory; - private readonly IFileStorage _storage; + private readonly ConcurrentDictionary>> _inflightSourceMaps = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _parsedSourceMapEntries = new(StringComparer.Ordinal); + private readonly MemoryCache _parsedSourceMaps; + private readonly SourceMapDownloader _downloader; + private readonly SourceMapStorage _storage; private readonly ICacheClient _cache; - private readonly JsonSerializerOptions _serializerOptions; + private readonly ILockProvider _lockProvider; private readonly SourceMapOptions _options; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; @@ -33,27 +34,34 @@ public SourceMapService( IHttpClientFactory httpClientFactory, IFileStorage storage, ICacheClient cache, + ILockProvider lockProvider, JsonSerializerOptions serializerOptions, AppOptions options, TimeProvider timeProvider, ILogger logger) { - _httpClientFactory = httpClientFactory; - _storage = storage; + _downloader = new SourceMapDownloader(httpClientFactory, options); + _storage = new SourceMapStorage(storage, serializerOptions, logger); _cache = cache; - _serializerOptions = serializerOptions; + _lockProvider = lockProvider; _options = options.SourceMapOptions; - _downloadSemaphore = new SemaphoreSlim(Math.Max(1, _options.MaximumConcurrentDownloads)); + _downloadSemaphore = new SemaphoreSlim(_options.MaximumConcurrentDownloads); + _parsedSourceMaps = new MemoryCache(new MemoryCacheOptions { SizeLimit = _options.MaximumParsedSourceMapCacheSize }); _timeProvider = timeProvider; _logger = logger; } - public async Task SaveUploadedAsync(string projectId, string generatedFileUrl, string? fileName, Stream stream, CancellationToken cancellationToken = default) + public async Task SaveUploadedAsync( + string projectId, + string generatedFileUrl, + string? fileName, + Stream stream, + CancellationToken cancellationToken = default) { if (!TryNormalizeGeneratedFileUrl(generatedFileUrl, requireHttps: false, out var generatedFileUri)) throw new ArgumentException("The generated file URL must be an absolute HTTP or HTTPS URL without credentials or a fragment.", nameof(generatedFileUrl)); - byte[] sourceMap = await ReadLimitedAsync(stream, _options.MaximumSourceMapSize, cancellationToken); + byte[] sourceMap = await SourceMapContent.ReadLimitedAsync(stream, _options.MaximumSourceMapSize, cancellationToken); _ = SourceMapDocument.Parse(sourceMap, _options.MaximumMappingSegments); string normalizedUrl = generatedFileUri.AbsoluteUri; @@ -67,57 +75,51 @@ public async Task SaveUploadedAsync(string projectId, string CreatedUtc = _timeProvider.GetUtcNow().UtcDateTime }; - await SaveArtifactAsync(projectId, artifact, sourceMap, cancellationToken); - await ClearCachesAsync(projectId, normalizedUrl); - return artifact; - } - - public async Task> GetArtifactsAsync(string projectId, CancellationToken cancellationToken = default) - { - string pattern = $"source-maps/{projectId}/*.json"; - var files = await _storage.GetFileListAsync(pattern, cancellationToken: cancellationToken); - var artifacts = new List(files.Count); - foreach (var file in files) + string cacheKey = GetMemoryCacheKey(projectId, normalizedUrl); + await WaitForInflightSourceMapAsync(cacheKey, cancellationToken); + await using var artifactLock = await _lockProvider.TryAcquireAsync(GetArtifactLockKey(projectId, artifact.Id), TimeSpan.FromSeconds(30), cancellationToken); + if (artifactLock is null) { - try - { - await using var stream = await _storage.GetFileStreamAsync(file.Path, StreamMode.Read, cancellationToken); - if (stream is null) - continue; - - var artifact = await JsonSerializer.DeserializeAsync(stream, _serializerOptions, cancellationToken); - if (artifact is not null) - artifacts.Add(artifact); - } - catch (Exception ex) when (ex is JsonException or IOException) - { - _logger.LogWarning(ex, "Unable to read source map metadata {SourceMapMetadataPath}.", file.Path); - } + cancellationToken.ThrowIfCancellationRequested(); + throw new IOException("Unable to acquire the source map storage lock."); } - return artifacts.OrderByDescending(artifact => artifact.CreatedUtc).ToArray(); + await _storage.SaveAsync(projectId, artifact, sourceMap, cancellationToken); + await ClearCachesAsync(projectId, normalizedUrl); + return artifact; } + public Task> GetArtifactsAsync(string projectId, CancellationToken cancellationToken = default) + => _storage.GetArtifactsAsync(projectId, cancellationToken); + public async Task DeleteArtifactAsync(string projectId, string artifactId, CancellationToken cancellationToken = default) { if (!IsArtifactId(artifactId)) return false; - string metadataPath = GetMetadataPath(projectId, artifactId); - SourceMapArtifact? artifact = await ReadArtifactMetadataAsync(metadataPath, cancellationToken); - bool mapDeleted = await _storage.DeleteFileAsync(GetMapPath(projectId, artifactId), cancellationToken); - bool metadataDeleted = await _storage.DeleteFileAsync(metadataPath, cancellationToken); - if (artifact is not null) - await ClearCachesAsync(projectId, artifact.GeneratedFileUrl); + await WaitForInflightArtifactAsync(projectId, artifactId, cancellationToken); + await using var artifactLock = await _lockProvider.TryAcquireAsync(GetArtifactLockKey(projectId, artifactId), TimeSpan.FromSeconds(30), cancellationToken); + if (artifactLock is null) + { + cancellationToken.ThrowIfCancellationRequested(); + return false; + } + + var artifact = await _storage.DeleteAsync(projectId, artifactId, cancellationToken); + if (artifact is null) + return false; - return mapDeleted || metadataDeleted; + await ClearCachesAsync(projectId, artifact.GeneratedFileUrl); + return true; } public async Task DeleteAllArtifactsAsync(string projectId, CancellationToken cancellationToken = default) { - await _storage.DeleteFilesAsync($"source-maps/{projectId}/*", cancellationToken); - foreach (string key in _sourceMaps.Keys.Where(key => key.StartsWith(projectId + ':', StringComparison.Ordinal))) - _sourceMaps.TryRemove(key, out _); + await _storage.DeleteAllAsync(projectId, cancellationToken); + foreach (string key in _parsedSourceMapEntries.Keys.Where(key => key.StartsWith(projectId + ':', StringComparison.Ordinal))) + _parsedSourceMaps.Remove(key); + foreach (string key in _inflightSourceMaps.Keys.Where(key => key.StartsWith(projectId + ':', StringComparison.Ordinal))) + _inflightSourceMaps.TryRemove(key, out _); } public async Task SymbolicateAsync(string projectId, InnerError? error, CancellationToken cancellationToken = default) @@ -153,6 +155,12 @@ public async Task SymbolicateAsync(string projectId, InnerError? error, Ca return changed; } + public void Dispose() + { + _parsedSourceMaps.Dispose(); + _downloadSemaphore.Dispose(); + } + private async Task SymbolicateFrameAsync(string projectId, StackFrame frame, CancellationToken cancellationToken) { if (frame.Data?.ContainsKey(SourceMapDataKey) == true || frame.LineNumber is null || frame.LineNumber < 1 || String.IsNullOrWhiteSpace(frame.FileName)) @@ -192,37 +200,123 @@ private async Task SymbolicateFrameAsync(string projectId, StackFrame fram private async Task GetSourceMapAsync(string projectId, Uri generatedFileUri, CancellationToken cancellationToken) { - string normalizedUrl = generatedFileUri.AbsoluteUri; - string cacheKey = GetMemoryCacheKey(projectId, normalizedUrl); - var lazy = _sourceMaps.GetOrAdd(cacheKey, _ => new Lazy>( - () => LoadSourceMapAsync(projectId, generatedFileUri), LazyThreadSafetyMode.ExecutionAndPublication)); + string cacheKey = GetMemoryCacheKey(projectId, generatedFileUri.AbsoluteUri); + if (_parsedSourceMaps.TryGetValue(cacheKey, out ResolvedSourceMap? cached)) + return cached; + + var lazy = _inflightSourceMaps.GetOrAdd(cacheKey, _ => new Lazy>( + () => LoadAndCacheSourceMapAsync(projectId, generatedFileUri, cacheKey), + LazyThreadSafetyMode.ExecutionAndPublication)); + Task loadTask = lazy.Value; try { - var result = await lazy.Value.WaitAsync(cancellationToken); - if (result is null) - _sourceMaps.TryRemove(cacheKey, out _); + return await loadTask.WaitAsync(cancellationToken); + } + finally + { + if (loadTask.IsCompleted) + RemoveInflightSourceMap(cacheKey, lazy); + else + _ = RemoveInflightSourceMapWhenCompleteAsync(cacheKey, lazy, loadTask); + } + } + + private async Task LoadAndCacheSourceMapAsync(string projectId, Uri generatedFileUri, string cacheKey) + { + var resolved = await LoadSourceMapAsync(projectId, generatedFileUri); + if (resolved is not null && resolved.Document.EstimatedMemorySize <= _options.MaximumParsedSourceMapCacheSize) + { + _parsedSourceMapEntries[cacheKey] = resolved; + var cacheOptions = new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = _options.ParsedSourceMapCacheLifetime, + Size = Math.Max(1, resolved.Document.EstimatedMemorySize) + }.RegisterPostEvictionCallback( + static (key, _, _, state) => + { + if (key is string evictedCacheKey && state is ParsedSourceMapCacheRegistration registration) + registration.Service.RemoveParsedSourceMapEntry(evictedCacheKey, registration.SourceMap); + }, + new ParsedSourceMapCacheRegistration(this, resolved)); + _parsedSourceMaps.Set(cacheKey, resolved, cacheOptions); + } + + return resolved; + } - return result; + private async Task RemoveInflightSourceMapWhenCompleteAsync( + string cacheKey, + Lazy> lazy, + Task loadTask) + { + try + { + await loadTask; } catch { - _sourceMaps.TryRemove(cacheKey, out _); + // The caller observes the load failure; this continuation only releases the in-flight entry. + } + + RemoveInflightSourceMap(cacheKey, lazy); + } + + private void RemoveInflightSourceMap(string cacheKey, Lazy> lazy) + { + ICollection>>> entries = _inflightSourceMaps; + entries.Remove(new KeyValuePair>>(cacheKey, lazy)); + } + + private void RemoveParsedSourceMapEntry(string cacheKey, ResolvedSourceMap sourceMap) + { + ICollection> entries = _parsedSourceMapEntries; + entries.Remove(new KeyValuePair(cacheKey, sourceMap)); + } + + private async Task WaitForInflightSourceMapAsync(string cacheKey, CancellationToken cancellationToken) + { + if (_inflightSourceMaps.TryGetValue(cacheKey, out var sourceMap)) + await WaitForInflightSourceMapAsync(sourceMap, cancellationToken); + } + + private async Task WaitForInflightArtifactAsync(string projectId, string artifactId, CancellationToken cancellationToken) + { + string cacheKeyPrefix = projectId + ':'; + var sourceMaps = _inflightSourceMaps + .Where(entry => entry.Key.StartsWith(cacheKeyPrefix, StringComparison.Ordinal) + && GetArtifactId(entry.Key[cacheKeyPrefix.Length..]) == artifactId) + .Select(entry => entry.Value) + .ToArray(); + + foreach (var sourceMap in sourceMaps) + await WaitForInflightSourceMapAsync(sourceMap, cancellationToken); + } + + private static async Task WaitForInflightSourceMapAsync(Lazy> sourceMap, CancellationToken cancellationToken) + { + try + { + await sourceMap.Value.WaitAsync(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { throw; } + catch + { + // A failed lookup must not prevent an explicit upload or delete from repairing the artifact. + } } private async Task LoadSourceMapAsync(string projectId, Uri generatedFileUri) { string generatedFileUrl = generatedFileUri.AbsoluteUri; string artifactId = GetArtifactId(generatedFileUrl); - string mapPath = GetMapPath(projectId, artifactId); - var artifact = await ReadArtifactMetadataAsync(GetMetadataPath(projectId, artifactId), CancellationToken.None); - if (artifact is not null && await _storage.ExistsAsync(mapPath)) - { - byte[] storedSourceMap = await ReadStorageFileAsync(mapPath, _options.MaximumSourceMapSize, CancellationToken.None); - return new ResolvedSourceMap(artifact, SourceMapDocument.Parse(storedSourceMap, _options.MaximumMappingSegments)); - } + var stored = await _storage.GetAsync(projectId, artifactId, _options.MaximumSourceMapSize, CancellationToken.None); + bool refreshStoredMap = ShouldRefresh(stored, generatedFileUri); + if (stored is not null && !refreshStoredMap) + return Resolve(stored.Artifact, stored.Content); if (!_options.EnableAutoDownload || !String.Equals(generatedFileUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) return null; @@ -234,14 +328,25 @@ private async Task SymbolicateFrameAsync(string projectId, StackFrame fram try { using var timeoutCancellationTokenSource = new CancellationTokenSource(_options.RequestTimeout); + await using var artifactLock = await _lockProvider.TryAcquireAsync( + GetArtifactLockKey(projectId, artifactId), + TimeSpan.FromSeconds(30), + timeoutCancellationTokenSource.Token); + if (artifactLock is null) + return await CacheFailureAsync(failureCacheKey); + + stored = await _storage.GetAsync(projectId, artifactId, _options.MaximumSourceMapSize, CancellationToken.None); + if (stored is not null && !ShouldRefresh(stored, generatedFileUri)) + return Resolve(stored.Artifact, stored.Content); + if (!await TryReserveAutoDownloadAsync(projectId)) return await CacheFailureAsync(failureCacheKey); await _downloadSemaphore.WaitAsync(timeoutCancellationTokenSource.Token); - DownloadedSourceMap? downloaded; + SourceMapDownloader.DownloadedSourceMap? downloaded; try { - downloaded = await DownloadSourceMapAsync(generatedFileUri, timeoutCancellationTokenSource.Token); + downloaded = await _downloader.DownloadAsync(generatedFileUri, timeoutCancellationTokenSource.Token); if (downloaded is null) return await CacheFailureAsync(failureCacheKey); } @@ -250,7 +355,7 @@ private async Task SymbolicateFrameAsync(string projectId, StackFrame fram _downloadSemaphore.Release(); } - var downloadedArtifact = new SourceMapArtifact + var artifact = new SourceMapArtifact { Id = artifactId, GeneratedFileUrl = generatedFileUrl, @@ -260,9 +365,9 @@ private async Task SymbolicateFrameAsync(string projectId, StackFrame fram IsAutoDownloaded = true, CreatedUtc = _timeProvider.GetUtcNow().UtcDateTime }; - var document = SourceMapDocument.Parse(downloaded.Content, _options.MaximumMappingSegments); - await SaveArtifactAsync(projectId, downloadedArtifact, downloaded.Content, CancellationToken.None); - return new ResolvedSourceMap(downloadedArtifact, document); + var resolved = Resolve(artifact, downloaded.Content); + await _storage.SaveAsync(projectId, artifact, downloaded.Content, CancellationToken.None); + return resolved; } catch (OperationCanceledException ex) { @@ -276,120 +381,19 @@ private async Task SymbolicateFrameAsync(string projectId, StackFrame fram } } - private async Task DownloadSourceMapAsync(Uri generatedFileUri, CancellationToken cancellationToken) - { - using var generatedResponse = await SendAsync(generatedFileUri, _options.MaximumGeneratedFileSize, request => request.Headers.Range = new RangeHeaderValue(null, 64 * 1024), cancellationToken); - if (!generatedResponse.Response.IsSuccessStatusCode) - return null; - - string? sourceMapReference = GetSourceMapHeader(generatedResponse.Response); - if (String.IsNullOrWhiteSpace(sourceMapReference)) - { - byte[] generatedContent = await ReadLimitedAsync(await generatedResponse.Response.Content.ReadAsStreamAsync(cancellationToken), _options.MaximumGeneratedFileSize, cancellationToken); - sourceMapReference = FindSourceMapReference(Encoding.UTF8.GetString(generatedContent)); - } - - if (String.IsNullOrWhiteSpace(sourceMapReference)) - { - var fallbackUriBuilder = new UriBuilder(generatedResponse.Uri) { Path = generatedResponse.Uri.AbsolutePath + ".map" }; - return await DownloadSourceMapContentAsync(fallbackUriBuilder.Uri, cancellationToken); - } - - if (sourceMapReference.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) - return new DownloadedSourceMap(DecodeDataUri(sourceMapReference, _options.MaximumSourceMapSize), null); - - if (!Uri.TryCreate(generatedResponse.Uri, sourceMapReference, out var sourceMapUri) || !IsAutoDownloadUri(sourceMapUri)) - return null; - - return await DownloadSourceMapContentAsync(sourceMapUri, cancellationToken); - } - - private async Task DownloadSourceMapContentAsync(Uri sourceMapUri, CancellationToken cancellationToken) - { - using var result = await SendAsync(sourceMapUri, _options.MaximumSourceMapSize, null, cancellationToken); - if (!result.Response.IsSuccessStatusCode) - return null; - - byte[] content = await ReadLimitedAsync(await result.Response.Content.ReadAsStreamAsync(cancellationToken), _options.MaximumSourceMapSize, cancellationToken); - return new DownloadedSourceMap(content, result.Uri.AbsoluteUri); - } - - private async Task SendAsync(Uri uri, int maximumBytes, Action? configureRequest, CancellationToken cancellationToken) - { - Uri currentUri = uri; - for (int redirectCount = 0; ; redirectCount++) - { - if (!IsAutoDownloadUri(currentUri)) - throw new InvalidOperationException("Source map auto-download only supports public HTTPS URLs."); - - using var request = new HttpRequestMessage(HttpMethod.Get, currentUri); - request.Headers.AcceptEncoding.ParseAdd("gzip, deflate, br"); - configureRequest?.Invoke(request); - - var client = _httpClientFactory.CreateClient(HttpClientName); - var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); - if (response.Content.Headers.ContentLength > maximumBytes) - { - response.Dispose(); - throw new InvalidOperationException("The downloaded file exceeded the configured maximum size."); - } - - if (!IsRedirect(response.StatusCode)) - return new HttpDownloadResult(currentUri, response); - - if (redirectCount >= _options.MaximumRedirects || response.Headers.Location is null) - { - response.Dispose(); - throw new InvalidOperationException("The source map download exceeded the allowed redirects."); - } - - Uri redirectUri = response.Headers.Location.IsAbsoluteUri ? response.Headers.Location : new Uri(currentUri, response.Headers.Location); - response.Dispose(); - currentUri = redirectUri; - } - } - - private async Task SaveArtifactAsync(string projectId, SourceMapArtifact artifact, byte[] sourceMap, CancellationToken cancellationToken) - { - string mapPath = GetMapPath(projectId, artifact.Id); - string metadataPath = GetMetadataPath(projectId, artifact.Id); - await using var mapStream = new MemoryStream(sourceMap, writable: false); - if (!await _storage.SaveFileAsync(mapPath, mapStream, cancellationToken)) - throw new IOException("Unable to save the source map."); - - try - { - byte[] metadata = JsonSerializer.SerializeToUtf8Bytes(artifact, _serializerOptions); - await using var metadataStream = new MemoryStream(metadata, writable: false); - if (!await _storage.SaveFileAsync(metadataPath, metadataStream, cancellationToken)) - throw new IOException("Unable to save the source map metadata."); - } - catch - { - await _storage.DeleteFileAsync(mapPath, CancellationToken.None); - throw; - } - } - - private async Task ReadArtifactMetadataAsync(string path, CancellationToken cancellationToken) - { - if (!await _storage.ExistsAsync(path)) - return null; - - await using var stream = await _storage.GetFileStreamAsync(path, StreamMode.Read, cancellationToken); - return stream is null ? null : await JsonSerializer.DeserializeAsync(stream, _serializerOptions, cancellationToken); - } + private ResolvedSourceMap Resolve(SourceMapArtifact artifact, byte[] content) + => new(artifact, SourceMapDocument.Parse(content, _options.MaximumMappingSegments)); - private async Task ReadStorageFileAsync(string path, int maximumBytes, CancellationToken cancellationToken) - { - await using var stream = await _storage.GetFileStreamAsync(path, StreamMode.Read, cancellationToken) - ?? throw new FileNotFoundException("The source map file was not found.", path); - return await ReadLimitedAsync(stream, maximumBytes, cancellationToken); - } + private bool ShouldRefresh(SourceMapStorage.StoredSourceMap? stored, Uri generatedFileUri) + => stored is not null + && stored.Artifact.IsAutoDownloaded + && _options.EnableAutoDownload + && String.Equals(generatedFileUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase) + && _timeProvider.GetUtcNow().UtcDateTime - stored.Artifact.CreatedUtc >= _options.AutoDownloadRefreshInterval; private Task ClearCachesAsync(string projectId, string generatedFileUrl) { - _sourceMaps.TryRemove(GetMemoryCacheKey(projectId, generatedFileUrl), out _); + _parsedSourceMaps.Remove(GetMemoryCacheKey(projectId, generatedFileUrl)); return _cache.RemoveAsync(GetFailureCacheKey(projectId, generatedFileUrl)); } @@ -426,73 +430,9 @@ public static bool TryNormalizeGeneratedFileUrl(string? value, bool requireHttps return true; } - private static bool IsAutoDownloadUri(Uri uri) => TryNormalizeGeneratedFileUrl(uri.AbsoluteUri, requireHttps: true, out _); - - private static bool IsRedirect(HttpStatusCode statusCode) => statusCode is HttpStatusCode.MovedPermanently - or HttpStatusCode.Redirect - or HttpStatusCode.RedirectMethod - or HttpStatusCode.TemporaryRedirect - or HttpStatusCode.PermanentRedirect; - - private static string? GetSourceMapHeader(HttpResponseMessage response) - { - if (response.Headers.TryGetValues("SourceMap", out var sourceMapValues)) - return sourceMapValues.FirstOrDefault(); - if (response.Headers.TryGetValues("X-SourceMap", out var legacySourceMapValues)) - return legacySourceMapValues.FirstOrDefault(); - return null; - } - - private static string? FindSourceMapReference(string generatedContent) - { - const string marker = "sourceMappingURL="; - int markerIndex = generatedContent.LastIndexOf(marker, StringComparison.Ordinal); - if (markerIndex < 0) - return null; - - int start = markerIndex + marker.Length; - int end = generatedContent.IndexOfAny(['\r', '\n'], start); - string value = generatedContent[start..(end < 0 ? generatedContent.Length : end)].Trim(); - if (value.EndsWith("*/", StringComparison.Ordinal)) - value = value[..^2].Trim(); - return value; - } - - private static byte[] DecodeDataUri(string value, int maximumBytes) - { - int commaIndex = value.IndexOf(','); - if (commaIndex < 0) - throw new FormatException("The inline source map data URI is invalid."); - - string metadata = value[..commaIndex]; - string data = value[(commaIndex + 1)..]; - byte[] decoded = metadata.Contains(";base64", StringComparison.OrdinalIgnoreCase) - ? Convert.FromBase64String(data) - : Encoding.UTF8.GetBytes(Uri.UnescapeDataString(data)); - if (decoded.Length > maximumBytes) - throw new InvalidOperationException("The inline source map exceeded the configured maximum size."); - return decoded; - } - - private static async Task ReadLimitedAsync(Stream stream, int maximumBytes, CancellationToken cancellationToken) - { - var memoryStream = new MemoryStream(Math.Min(maximumBytes, 64 * 1024)); - byte[] buffer = new byte[81920]; - int read; - while ((read = await stream.ReadAsync(buffer, cancellationToken)) > 0) - { - if (memoryStream.Length + read > maximumBytes) - throw new InvalidOperationException("The file exceeded the configured maximum size."); - await memoryStream.WriteAsync(buffer.AsMemory(0, read), cancellationToken); - } - - return memoryStream.ToArray(); - } - private static bool IsArtifactId(string value) => value.Length == 64 && value.All(Uri.IsHexDigit); private static string GetArtifactId(string generatedFileUrl) => generatedFileUrl.ToSHA256(); - private static string GetMapPath(string projectId, string artifactId) => $"source-maps/{projectId}/{artifactId}.map"; - private static string GetMetadataPath(string projectId, string artifactId) => $"source-maps/{projectId}/{artifactId}.json"; + private static string GetArtifactLockKey(string projectId, string artifactId) => $"source-maps:artifact:{projectId}:{artifactId}"; private static string GetMemoryCacheKey(string projectId, string generatedFileUrl) => $"{projectId}:{generatedFileUrl}"; private static string GetFailureCacheKey(string projectId, string generatedFileUrl) => $"source-maps:failure:{projectId}:{generatedFileUrl.ToSHA256()}"; @@ -503,11 +443,6 @@ private static async Task ReadLimitedAsync(Stream stream, int maximumByt return Path.GetFileName(uri.AbsolutePath); } - private sealed record DownloadedSourceMap(byte[] Content, string? SourceMapUrl); private sealed record ResolvedSourceMap(SourceMapArtifact Artifact, SourceMapDocument Document); - - private sealed record HttpDownloadResult(Uri Uri, HttpResponseMessage Response) : IDisposable - { - public void Dispose() => Response.Dispose(); - } + private sealed record ParsedSourceMapCacheRegistration(SourceMapService Service, ResolvedSourceMap SourceMap); } diff --git a/src/Exceptionless.Core/Services/SourceMaps/SourceMapStorage.cs b/src/Exceptionless.Core/Services/SourceMaps/SourceMapStorage.cs new file mode 100644 index 0000000000..42fee614d4 --- /dev/null +++ b/src/Exceptionless.Core/Services/SourceMaps/SourceMapStorage.cs @@ -0,0 +1,132 @@ +using System.Security.Cryptography; +using System.Text.Json; +using Foundatio.Storage; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services.SourceMaps; + +internal sealed class SourceMapStorage +{ + private const string RootPath = "source-maps"; + private readonly IFileStorage _storage; + private readonly JsonSerializerOptions _serializerOptions; + private readonly ILogger _logger; + + public SourceMapStorage(IFileStorage storage, JsonSerializerOptions serializerOptions, ILogger logger) + { + _storage = storage; + _serializerOptions = serializerOptions; + _logger = logger; + } + + public async Task SaveAsync(string projectId, SourceMapArtifact artifact, byte[] sourceMap, CancellationToken cancellationToken) + { + string metadataPath = GetMetadataPath(projectId, artifact.Id); + var previousMetadata = await ReadMetadataAsync(metadataPath, cancellationToken); + string mapFileName = $"{artifact.Id}-{Convert.ToHexString(SHA256.HashData(sourceMap)).ToLowerInvariant()}.map"; + string mapPath = GetMapPath(projectId, mapFileName); + + await using (var mapStream = new MemoryStream(sourceMap, writable: false)) + { + if (!await _storage.SaveFileAsync(mapPath, mapStream, cancellationToken)) + throw new IOException("Unable to save the source map."); + } + + try + { + var metadata = new StoredSourceMapMetadata(artifact, mapFileName); + byte[] metadataBytes = JsonSerializer.SerializeToUtf8Bytes(metadata, _serializerOptions); + await using var metadataStream = new MemoryStream(metadataBytes, writable: false); + if (!await _storage.SaveFileAsync(metadataPath, metadataStream, cancellationToken)) + throw new IOException("Unable to save the source map metadata."); + } + catch + { + if (previousMetadata?.MapFileName != mapFileName) + await _storage.DeleteFileAsync(mapPath, CancellationToken.None); + throw; + } + + if (previousMetadata is not null && previousMetadata.MapFileName != mapFileName) + await _storage.DeleteFileAsync(GetMapPath(projectId, previousMetadata.MapFileName), CancellationToken.None); + } + + public async Task GetAsync(string projectId, string artifactId, int maximumBytes, CancellationToken cancellationToken) + { + var metadata = await ReadMetadataAsync(GetMetadataPath(projectId, artifactId), cancellationToken); + if (metadata is null || !IsValidMapFileName(artifactId, metadata.MapFileName)) + return null; + + string mapPath = GetMapPath(projectId, metadata.MapFileName); + if (!await _storage.ExistsAsync(mapPath)) + return null; + + await using var stream = await _storage.GetFileStreamAsync(mapPath, StreamMode.Read, cancellationToken); + if (stream is null) + return null; + + byte[] content = await SourceMapContent.ReadLimitedAsync(stream, maximumBytes, cancellationToken); + return new StoredSourceMap(metadata.Artifact, content); + } + + public async Task> GetArtifactsAsync(string projectId, CancellationToken cancellationToken) + { + var files = await _storage.GetFileListAsync($"{RootPath}/{projectId}/*.json", cancellationToken: cancellationToken); + var artifacts = new List(files.Count); + foreach (var file in files) + { + try + { + var metadata = await ReadMetadataAsync(file.Path, cancellationToken); + if (metadata is not null) + artifacts.Add(metadata.Artifact); + } + catch (Exception ex) when (ex is JsonException or IOException) + { + _logger.LogWarning(ex, "Unable to read source map metadata {SourceMapMetadataPath}.", file.Path); + } + } + + return artifacts.OrderByDescending(artifact => artifact.CreatedUtc).ToArray(); + } + + public async Task DeleteAsync(string projectId, string artifactId, CancellationToken cancellationToken) + { + string metadataPath = GetMetadataPath(projectId, artifactId); + var metadata = await ReadMetadataAsync(metadataPath, cancellationToken); + bool mapDeleted = false; + if (metadata is not null && IsValidMapFileName(artifactId, metadata.MapFileName)) + mapDeleted = await _storage.DeleteFileAsync(GetMapPath(projectId, metadata.MapFileName), cancellationToken); + else + await _storage.DeleteFilesAsync($"{RootPath}/{projectId}/{artifactId}-*.map", cancellationToken); + + bool metadataDeleted = await _storage.DeleteFileAsync(metadataPath, cancellationToken); + return mapDeleted || metadataDeleted ? metadata?.Artifact : null; + } + + public Task DeleteAllAsync(string projectId, CancellationToken cancellationToken) + => _storage.DeleteFilesAsync($"{RootPath}/{projectId}/*", cancellationToken); + + private async Task ReadMetadataAsync(string path, CancellationToken cancellationToken) + { + if (!await _storage.ExistsAsync(path)) + return null; + + await using var stream = await _storage.GetFileStreamAsync(path, StreamMode.Read, cancellationToken); + return stream is null + ? null + : await JsonSerializer.DeserializeAsync(stream, _serializerOptions, cancellationToken); + } + + private static bool IsValidMapFileName(string artifactId, string mapFileName) + => mapFileName.Length == artifactId.Length + 1 + 64 + 4 + && mapFileName.StartsWith(artifactId + '-', StringComparison.Ordinal) + && mapFileName.EndsWith(".map", StringComparison.Ordinal) + && mapFileName.AsSpan(artifactId.Length + 1, 64).ToArray().All(character => Uri.IsHexDigit(character)); + + private static string GetMetadataPath(string projectId, string artifactId) => $"{RootPath}/{projectId}/{artifactId}.json"; + private static string GetMapPath(string projectId, string mapFileName) => $"{RootPath}/{projectId}/{mapFileName}"; + + internal sealed record StoredSourceMap(SourceMapArtifact Artifact, byte[] Content); + private sealed record StoredSourceMapMetadata(SourceMapArtifact Artifact, string MapFileName); +} diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/source-maps/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/source-maps/+page.svelte index 4e9d0866fe..f454005f5a 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/source-maps/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/source-maps/+page.svelte @@ -73,7 +73,12 @@ } try { - await deleteSourceMap.mutateAsync(sourceMapToDelete.id); + const deleted = await deleteSourceMap.mutateAsync(sourceMapToDelete.id); + if (!deleted) { + toast.error('The source map no longer exists.'); + return; + } + toast.success('Source map deleted.'); showDeleteDialog = false; sourceMapToDelete = undefined; @@ -179,6 +184,10 @@ {#if sourceMapsQuery.isLoading} + {:else if sourceMapsQuery.isError} + {:else if !sourceMapsQuery.data?.length} (), GetService(), + GetService(), GetService(), GetService(), GetService(), @@ -99,6 +104,232 @@ public async Task SymbolicateAsync_WithPublicSourceMapHeader_DownloadsAndStoresS Assert.Equal("https://cdn.example.com/assets/app.min.js.map", artifact.SourceMapUrl); } + [Fact] + public async Task SymbolicateAsync_WithExpiredAutoDownloadedSourceMap_RefreshesStoredMap() + { + int sourceMapDownloadCount = 0; + var handler = new DelegateHandler(request => + { + if (request.RequestUri == new Uri(GeneratedFileUrl)) + { + var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("minified") }; + response.Headers.TryAddWithoutValidation("SourceMap", "app.min.js.map"); + return response; + } + + sourceMapDownloadCount++; + byte[] content = sourceMapDownloadCount == 1 ? SourceMap : UpdatedSourceMap; + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(content) }; + }); + using var httpClient = new HttpClient(handler); + using (var initialService = CreateService(httpClient)) + { + var initialError = CreateError(); + Assert.True(await initialService.SymbolicateAsync(ProjectId, initialError, TestContext.Current.CancellationToken)); + Assert.Equal("meaningfulFunction", Assert.Single(initialError.StackTrace!).Name); + } + + TimeProvider.Advance(TimeSpan.FromMinutes(61)); + + using var refreshedService = CreateService(httpClient); + var refreshedError = CreateError(); + Assert.True(await refreshedService.SymbolicateAsync(ProjectId, refreshedError, TestContext.Current.CancellationToken)); + var refreshedFrame = Assert.Single(refreshedError.StackTrace!); + Assert.Equal("updatedFunction", refreshedFrame.Name); + Assert.Equal("src/updated.ts", refreshedFrame.FileName); + Assert.Equal(2, sourceMapDownloadCount); + } + + [Fact] + public async Task SymbolicateAsync_WhenExpiredSourceMapCannotRefresh_DoesNotUseStaleMap() + { + int sourceMapDownloadCount = 0; + var handler = new DelegateHandler(request => + { + if (request.RequestUri == new Uri(GeneratedFileUrl)) + { + var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("minified") }; + response.Headers.TryAddWithoutValidation("SourceMap", "app.min.js.map"); + return response; + } + + sourceMapDownloadCount++; + return sourceMapDownloadCount == 1 + ? new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(SourceMap) } + : new HttpResponseMessage(HttpStatusCode.ServiceUnavailable); + }); + using var httpClient = new HttpClient(handler); + using (var initialService = CreateService(httpClient)) + Assert.True(await initialService.SymbolicateAsync(ProjectId, CreateError(), TestContext.Current.CancellationToken)); + + TimeProvider.Advance(TimeSpan.FromMinutes(61)); + + using var refreshedService = CreateService(httpClient); + var error = CreateError(); + Assert.False(await refreshedService.SymbolicateAsync(ProjectId, error, TestContext.Current.CancellationToken)); + var frame = Assert.Single(error.StackTrace!); + Assert.Equal("a", frame.Name); + Assert.Equal(GeneratedFileUrl, frame.FileName); + Assert.Equal(2, sourceMapDownloadCount); + } + + [Fact] + public async Task SaveUploadedAsync_WhenReplacingMap_UsesNewMapAndRemovesOldContent() + { + var service = GetService(); + await using (var sourceMapStream = new MemoryStream(SourceMap)) + await service.SaveUploadedAsync(ProjectId, GeneratedFileUrl, "app.min.js.map", sourceMapStream, TestContext.Current.CancellationToken); + await using (var sourceMapStream = new MemoryStream(UpdatedSourceMap)) + await service.SaveUploadedAsync(ProjectId, GeneratedFileUrl, "app.min.js.map", sourceMapStream, TestContext.Current.CancellationToken); + + var error = CreateError(); + Assert.True(await service.SymbolicateAsync(ProjectId, error, TestContext.Current.CancellationToken)); + Assert.Equal("updatedFunction", Assert.Single(error.StackTrace!).Name); + + var mapFiles = await GetService().GetFileListAsync($"source-maps/{ProjectId}/*.map", cancellationToken: TestContext.Current.CancellationToken); + Assert.Single(mapFiles); + } + + [Fact] + public async Task SymbolicateAsync_WhenMapExceedsParsedCacheLimit_DoesNotRetainMapInMemory() + { + var options = GetService(); + options.SourceMapOptions.MaximumParsedSourceMapCacheSize = 1; + int requestCount = 0; + var handler = new DelegateHandler(request => + { + requestCount++; + if (request.RequestUri == new Uri(GeneratedFileUrl)) + { + var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("minified") }; + response.Headers.TryAddWithoutValidation("SourceMap", "app.min.js.map"); + return response; + } + + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(SourceMap) }; + }); + using var httpClient = new HttpClient(handler); + using var service = CreateService(httpClient); + Assert.True(await service.SymbolicateAsync(ProjectId, CreateError(), TestContext.Current.CancellationToken)); + + await GetService().DeleteFilesAsync($"source-maps/{ProjectId}/*", TestContext.Current.CancellationToken); + + Assert.True(await service.SymbolicateAsync(ProjectId, CreateError(), TestContext.Current.CancellationToken)); + Assert.Equal(4, requestCount); + } + + [Fact] + public async Task SymbolicateAsync_WhenFirstCallerCancels_KeepsSharedDownloadForSecondCaller() + { + int requestCount = 0; + var mapRequestStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseMapRequest = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var handler = new AsyncDelegateHandler(async request => + { + requestCount++; + if (request.RequestUri == new Uri(GeneratedFileUrl)) + { + var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("minified") }; + response.Headers.TryAddWithoutValidation("SourceMap", "app.min.js.map"); + return response; + } + + mapRequestStarted.TrySetResult(); + await releaseMapRequest.Task; + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(SourceMap) }; + }); + using var httpClient = new HttpClient(handler); + using var service = CreateService(httpClient); + using var cancellationTokenSource = new CancellationTokenSource(); + Task canceledCaller = service.SymbolicateAsync(ProjectId, CreateError(), cancellationTokenSource.Token); + await mapRequestStarted.Task; + + cancellationTokenSource.Cancel(); + await Assert.ThrowsAnyAsync(() => canceledCaller); + Task secondCaller = service.SymbolicateAsync(ProjectId, CreateError(), TestContext.Current.CancellationToken); + releaseMapRequest.TrySetResult(); + + Assert.True(await secondCaller); + Assert.Equal(2, requestCount); + } + + [Fact] + public async Task SaveUploadedAsync_DuringAutomaticDownload_UploadRemainsAuthoritative() + { + var mapRequestStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseMapRequest = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var handler = new AsyncDelegateHandler(async request => + { + if (request.RequestUri == new Uri(GeneratedFileUrl)) + { + var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("minified") }; + response.Headers.TryAddWithoutValidation("SourceMap", "app.min.js.map"); + return response; + } + + mapRequestStarted.TrySetResult(); + await releaseMapRequest.Task; + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(SourceMap) }; + }); + using var httpClient = new HttpClient(handler); + using var service = CreateService(httpClient); + Task automaticDownload = service.SymbolicateAsync(ProjectId, CreateError(), TestContext.Current.CancellationToken); + await mapRequestStarted.Task; + + await using var uploadedStream = new MemoryStream(UpdatedSourceMap); + Task upload = service.SaveUploadedAsync( + ProjectId, + GeneratedFileUrl, + "app.min.js.map", + uploadedStream, + TestContext.Current.CancellationToken); + releaseMapRequest.TrySetResult(); + + Assert.True(await automaticDownload); + var uploaded = await upload; + Assert.False(uploaded.IsAutoDownloaded); + var error = CreateError(); + Assert.True(await service.SymbolicateAsync(ProjectId, error, TestContext.Current.CancellationToken)); + Assert.Equal("updatedFunction", Assert.Single(error.StackTrace!).Name); + Assert.False(Assert.Single(await service.GetArtifactsAsync(ProjectId, TestContext.Current.CancellationToken)).IsAutoDownloaded); + } + + [Fact] + public void ReadFromConfiguration_WithUnsafeLimits_NormalizesLimits() + { + var values = new Dictionary + { + ["SourceMaps:RequestTimeoutMilliseconds"] = "0", + ["SourceMaps:MaximumGeneratedFileSize"] = "0", + ["SourceMaps:MaximumSourceMapSize"] = "-1", + ["SourceMaps:MaximumMappingSegments"] = "0", + ["SourceMaps:MaximumRedirects"] = "-1", + ["SourceMaps:MaximumConcurrentDownloads"] = "0", + ["SourceMaps:MaximumAutoDownloadsPerProjectPerHour"] = "-1", + ["SourceMaps:MaximumFramesPerError"] = "0", + ["SourceMaps:MaximumProcessingTimeMilliseconds"] = "0", + ["SourceMaps:AutoDownloadRefreshIntervalMinutes"] = "0", + ["SourceMaps:ParsedSourceMapCacheLifetimeMinutes"] = "0", + ["SourceMaps:MaximumParsedSourceMapCacheSize"] = "0" + }; + var configuration = new ConfigurationBuilder().AddInMemoryCollection(values).Build(); + + var options = SourceMapOptions.ReadFromConfiguration(configuration); + + Assert.Equal(1, options.RequestTimeoutMilliseconds); + Assert.Equal(1, options.MaximumGeneratedFileSize); + Assert.Equal(1, options.MaximumSourceMapSize); + Assert.Equal(1, options.MaximumMappingSegments); + Assert.Equal(0, options.MaximumRedirects); + Assert.Equal(1, options.MaximumConcurrentDownloads); + Assert.Equal(0, options.MaximumAutoDownloadsPerProjectPerHour); + Assert.Equal(1, options.MaximumFramesPerError); + Assert.Equal(1, options.MaximumProcessingTimeMilliseconds); + Assert.Equal(1, options.AutoDownloadRefreshIntervalMinutes); + Assert.Equal(1, options.ParsedSourceMapCacheLifetimeMinutes); + Assert.Equal(1, options.MaximumParsedSourceMapCacheSize); + } + [Fact] public async Task EventProcessingAsync_WithUploadedSourceMap_UsesOriginalNameForStackSignature() { @@ -143,6 +374,19 @@ private static Error CreateError() }; } + private SourceMapService CreateService(HttpClient httpClient) + { + return new SourceMapService( + new TestHttpClientFactory(httpClient), + GetService(), + GetService(), + GetService(), + GetService(), + GetService(), + GetService(), + GetService>()); + } + private sealed class TestHttpClientFactory(HttpClient httpClient) : IHttpClientFactory { public HttpClient CreateClient(string name) => httpClient; @@ -155,4 +399,12 @@ protected override Task SendAsync(HttpRequestMessage reques return Task.FromResult(handler(request)); } } + + private sealed class AsyncDelegateHandler(Func> handler) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return handler(request); + } + } } From 3a42b1f6e74173fc873a2a14bdd5e291c586273f Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 14 Jul 2026 18:52:53 -0500 Subject: [PATCH 03/15] Add project-scoped source map upload tokens --- docs/docs/source-maps.md | 24 ++++ .../Authorization/AuthorizationRoles.cs | 4 +- src/Exceptionless.Core/Jobs/CleanupDataJob.cs | 19 +++ .../Services/SourceMaps/SourceMapDocument.cs | 5 +- .../Controllers/SourceMapController.cs | 14 +- .../Controllers/TokenController.cs | 6 + src/Exceptionless.Web/Startup.cs | 1 + .../Controllers/Data/controller-manifest.json | 2 +- .../Controllers/SourceMapControllerTests.cs | 126 ++++++++++++++++++ .../Controllers/TokenControllerTests.cs | 40 ++++++ .../Jobs/CleanupDataJobTests.cs | 4 + .../SourceMaps/SourceMapDocumentTests.cs | 10 ++ tests/http/source-maps.http | 64 +++++++++ 13 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 tests/http/source-maps.http diff --git a/docs/docs/source-maps.md b/docs/docs/source-maps.md index 4dcd5c20cf..402652322c 100644 --- a/docs/docs/source-maps.md +++ b/docs/docs/source-maps.md @@ -24,6 +24,30 @@ Upload a map when it is private or is not deployed next to the generated JavaScr Uploading another map for the same generated file URL replaces the previous map. Uploaded and automatically discovered maps appear together on the Source Maps page and can be deleted there. +### Uploading during deployment + +For build automation, create a project-scoped token that has only the `source-maps:write` scope. Set `EXCEPTIONLESS_SERVER_URL` to `https://be.exceptionless.io` for the hosted service or to the root URL of your self-hosted installation. Create the token once with a user-scoped token, then store the returned `id` as a protected CI/CD secret: + +```shell +curl --fail-with-body --request POST \ + "${EXCEPTIONLESS_SERVER_URL}/api/v2/projects/${EXCEPTIONLESS_PROJECT_ID}/tokens" \ + --header "Authorization: Bearer ${EXCEPTIONLESS_USER_TOKEN}" \ + --header "Content-Type: application/json" \ + --data "{\"organization_id\":\"${EXCEPTIONLESS_ORGANIZATION_ID}\",\"project_id\":\"${EXCEPTIONLESS_PROJECT_ID}\",\"scopes\":[\"source-maps:write\"],\"notes\":\"CI source map uploads\"}" +``` + +Upload each map after the generated JavaScript has been deployed. The `generated_file_url` must be the exact absolute URL that will appear in stack frames: + +```shell +curl --fail-with-body --request POST \ + "${EXCEPTIONLESS_SERVER_URL}/api/v2/projects/${EXCEPTIONLESS_PROJECT_ID}/source-maps" \ + --header "Authorization: Bearer ${EXCEPTIONLESS_SOURCE_MAP_TOKEN}" \ + --form "generated_file_url=https://cdn.example.com/assets/app.a1b2c3.js" \ + --form "file=@dist/assets/app.a1b2c3.js.map;type=application/json" +``` + +The upload token is accepted only for its assigned project and cannot read events, manage the project, list source maps, or use ordinary user APIs. Do not use a normal client API key for source map uploads because client keys are commonly embedded in distributed applications. + Source maps must use the version 3 flat-map format. Indexed source maps with a `sections` property and authenticated automatic downloads are planned follow-up capabilities; private maps can be uploaded in the meantime. ## Deployment guidance diff --git a/src/Exceptionless.Core/Authorization/AuthorizationRoles.cs b/src/Exceptionless.Core/Authorization/AuthorizationRoles.cs index 546b06e864..0ac1565c2e 100644 --- a/src/Exceptionless.Core/Authorization/AuthorizationRoles.cs +++ b/src/Exceptionless.Core/Authorization/AuthorizationRoles.cs @@ -12,12 +12,14 @@ public static class AuthorizationRoles public const string ProjectsReadPolicy = nameof(ProjectsReadPolicy); public const string StacksReadPolicy = nameof(StacksReadPolicy); public const string StacksWritePolicy = nameof(StacksWritePolicy); + public const string SourceMapsWritePolicy = nameof(SourceMapsWritePolicy); public const string EventsReadPolicy = nameof(EventsReadPolicy); public const string McpRead = "mcp:read"; public const string ProjectsRead = "projects:read"; public const string StacksRead = "stacks:read"; public const string StacksWrite = "stacks:write"; + public const string SourceMapsWrite = "source-maps:write"; public const string EventsRead = "events:read"; public const string OfflineAccess = "offline_access"; - public static readonly ISet AllScopes = new HashSet([Client, User, GlobalAdmin]); + public static readonly ISet AllScopes = new HashSet([Client, User, GlobalAdmin, SourceMapsWrite]); } diff --git a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs index 14ee393433..5e42acc0ec 100644 --- a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs @@ -286,6 +286,8 @@ private async Task RemoveOrganizationAsync(Organization organization, JobContext await RenewLockAsync(context); long removedStacks = await _stackRepository.RemoveAllByOrganizationIdAsync(organization.Id); + await RemoveOrganizationProjectFilesAsync(organization.Id, context); + await RenewLockAsync(context); long removedProjects = await _projectRepository.RemoveAllByOrganizationIdAsync(organization.Id); @@ -316,6 +318,23 @@ private async Task RemoveSyntheticUserAsync(User user) private Task RemoveOrganizationFilesAsync(Organization organization, JobContext context) => RemoveFilesAsync(OrganizationStoragePaths.GetProfileImagesPath(organization.Id), context.CancellationToken); + private async Task RemoveOrganizationProjectFilesAsync(string organizationId, JobContext context) + { + var projects = await _projectRepository.GetByOrganizationIdAsync( + organizationId, + options => options.Include(project => project.Id).SearchAfterPaging().PageLimit(100)); + + while (projects.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) + { + foreach (var project in projects.Documents) + await _fileStorage.DeleteFilesAsync($"source-maps/{project.Id}/*", context.CancellationToken); + + await RenewLockAsync(context); + if (!await projects.NextPageAsync()) + break; + } + } + private async Task RemoveFilesAsync(string path, CancellationToken cancellationToken) { string searchPattern = $"{path}/*"; diff --git a/src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs b/src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs index 5d648aebb6..7e14cf9528 100644 --- a/src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs +++ b/src/Exceptionless.Core/Services/SourceMaps/SourceMapDocument.cs @@ -37,7 +37,10 @@ public static SourceMapDocument Parse(byte[] sourceMap, int maximumSegments = 1_ if (root.TryGetProperty("sections", out _)) throw new JsonException("Indexed source maps are not supported."); - string mappings = root.GetProperty("mappings").GetString() ?? throw new JsonException("The source map mappings are required."); + if (!root.TryGetProperty("mappings", out var mappingsElement) || mappingsElement.ValueKind != JsonValueKind.String) + throw new JsonException("The source map mappings are required."); + + string mappings = mappingsElement.GetString() ?? throw new JsonException("The source map mappings are required."); string[] sources = ReadStringArray(root, "sources"); string[] names = root.TryGetProperty("names", out _) ? ReadStringArray(root, "names") : []; string? sourceRoot = root.TryGetProperty("sourceRoot", out var sourceRootElement) ? sourceRootElement.GetString() : null; diff --git a/src/Exceptionless.Web/Controllers/SourceMapController.cs b/src/Exceptionless.Web/Controllers/SourceMapController.cs index 1d2defa569..918168b82b 100644 --- a/src/Exceptionless.Web/Controllers/SourceMapController.cs +++ b/src/Exceptionless.Web/Controllers/SourceMapController.cs @@ -2,6 +2,7 @@ using Exceptionless.Core.Authorization; using Exceptionless.Core.Repositories; using Exceptionless.Core.Services.SourceMaps; +using Exceptionless.Web.Extensions; using Exceptionless.Web.Utility.OpenApi; using Foundatio.Repositories; using Microsoft.AspNetCore.Authorization; @@ -10,7 +11,6 @@ namespace Exceptionless.Web.Controllers; [Route(API_PREFIX + "/projects/{projectId:objectid}/source-maps")] -[Authorize(Policy = AuthorizationRoles.UserPolicy)] public sealed class SourceMapController : ExceptionlessApiController { private readonly IProjectRepository _projectRepository; @@ -27,6 +27,7 @@ public SourceMapController(IProjectRepository projectRepository, SourceMapServic /// Get source maps for a project. /// [HttpGet] + [Authorize(Policy = AuthorizationRoles.UserPolicy)] public async Task>> GetAsync(string projectId, CancellationToken cancellationToken = default) { if (!await CanAccessProjectAsync(projectId)) @@ -43,6 +44,7 @@ public async Task>> GetAsync /// The source map file. /// The cancellation token. [HttpPost] + [Authorize(Policy = AuthorizationRoles.SourceMapsWritePolicy)] [Consumes("multipart/form-data")] [MultipartFileUpload("file", "generated_file_url", FileDescription = "The source map file to upload.")] [RequestSizeLimit(SourceMapService.MaximumUploadRequestSize)] @@ -86,6 +88,7 @@ public async Task> PostAsync( /// Delete a source map from a project. /// [HttpDelete("{sourceMapId}")] + [Authorize(Policy = AuthorizationRoles.UserPolicy)] [ProducesResponseType(StatusCodes.Status204NoContent)] public async Task DeleteAsync(string projectId, string sourceMapId, CancellationToken cancellationToken = default) { @@ -98,6 +101,13 @@ public async Task DeleteAsync(string projectId, string sourceMapI private async Task CanAccessProjectAsync(string projectId) { var project = await _projectRepository.GetByIdAsync(projectId, options => options.Cache()); - return project is not null && CanAccessOrganization(project.OrganizationId); + if (project is null || !CanAccessOrganization(project.OrganizationId)) + return false; + + string? tokenProjectId = Request.GetProjectId(); + if (User.IsInRole(AuthorizationRoles.SourceMapsWrite) && !User.IsInRole(AuthorizationRoles.User)) + return String.Equals(tokenProjectId, projectId, StringComparison.Ordinal); + + return String.IsNullOrEmpty(tokenProjectId) || String.Equals(tokenProjectId, projectId, StringComparison.Ordinal); } } diff --git a/src/Exceptionless.Web/Controllers/TokenController.cs b/src/Exceptionless.Web/Controllers/TokenController.cs index d5bc931d0d..4cc4495c51 100644 --- a/src/Exceptionless.Web/Controllers/TokenController.cs +++ b/src/Exceptionless.Web/Controllers/TokenController.cs @@ -296,6 +296,12 @@ protected override async Task CanAddAsync(Token value) if (value.Scopes.Count == 0) value.Scopes.Add(AuthorizationRoles.Client); + if (value.Scopes.Contains(AuthorizationRoles.SourceMapsWrite) && String.IsNullOrEmpty(value.ProjectId)) + { + ModelState.AddModelError(m => m.ProjectId, "The source-maps:write scope requires a project-scoped token."); + return PermissionResult.DenyWithValidationProblem(); + } + if (value.Scopes.Contains(AuthorizationRoles.Client) && !hasUserRole) { ModelState.AddModelError(m => m.Scopes, "Invalid token scope requested."); diff --git a/src/Exceptionless.Web/Startup.cs b/src/Exceptionless.Web/Startup.cs index e856f08b37..ea1a176633 100644 --- a/src/Exceptionless.Web/Startup.cs +++ b/src/Exceptionless.Web/Startup.cs @@ -92,6 +92,7 @@ public void ConfigureServices(IServiceCollection services) o.AddPolicy(AuthorizationRoles.ProjectsReadPolicy, policy => policy.RequireAssertion(context => context.User.IsInRole(AuthorizationRoles.User) || context.User.IsInRole(AuthorizationRoles.ProjectsRead))); o.AddPolicy(AuthorizationRoles.StacksReadPolicy, policy => policy.RequireAssertion(context => context.User.IsInRole(AuthorizationRoles.User) || context.User.IsInRole(AuthorizationRoles.StacksRead))); o.AddPolicy(AuthorizationRoles.StacksWritePolicy, policy => policy.RequireAssertion(context => context.User.IsInRole(AuthorizationRoles.User) || context.User.IsInRole(AuthorizationRoles.StacksWrite))); + o.AddPolicy(AuthorizationRoles.SourceMapsWritePolicy, policy => policy.RequireAssertion(context => context.User.IsInRole(AuthorizationRoles.User) || context.User.IsInRole(AuthorizationRoles.SourceMapsWrite))); o.AddPolicy(AuthorizationRoles.EventsReadPolicy, policy => policy.RequireAssertion(context => context.User.IsInRole(AuthorizationRoles.User) || context.User.IsInRole(AuthorizationRoles.EventsRead))); }); diff --git a/tests/Exceptionless.Tests/Controllers/Data/controller-manifest.json b/tests/Exceptionless.Tests/Controllers/Data/controller-manifest.json index 4f10ade6cc..d9d101339e 100644 --- a/tests/Exceptionless.Tests/Controllers/Data/controller-manifest.json +++ b/tests/Exceptionless.Tests/Controllers/Data/controller-manifest.json @@ -2325,7 +2325,7 @@ "HttpMethod": "POST", "Route": "/api/v2/projects/{projectId:objectid}/source-maps", "Authorization": [ - "Authorize(Policy=UserPolicy)" + "Authorize(Policy=SourceMapsWritePolicy)" ], "Consumes": [ "multipart/form-data" diff --git a/tests/Exceptionless.Tests/Controllers/SourceMapControllerTests.cs b/tests/Exceptionless.Tests/Controllers/SourceMapControllerTests.cs index 9661f69ce6..c0d202457c 100644 --- a/tests/Exceptionless.Tests/Controllers/SourceMapControllerTests.cs +++ b/tests/Exceptionless.Tests/Controllers/SourceMapControllerTests.cs @@ -1,8 +1,16 @@ using System.Net; using System.Text; +using Exceptionless.Core.Authorization; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; using Exceptionless.Core.Services.SourceMaps; using Exceptionless.Core.Utility; using Exceptionless.Tests.Extensions; +using Exceptionless.Tests.Utility; +using Exceptionless.Web.Models; +using FluentRest; +using Foundatio.Repositories; using Xunit; namespace Exceptionless.Tests.Controllers; @@ -65,6 +73,104 @@ await SendRequestAsync(request => request .StatusCodeShouldBeUnprocessableEntity()); } + [Fact] + public async Task PostAsync_WithMissingMappings_ReturnsUnprocessableEntity() + { + using var content = CreateSourceMapContent(Encoding.UTF8.GetBytes("""{"version":3,"sources":[],"names":[]}""")); + + await SendRequestAsync(request => request + .Post() + .AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "source-maps") + .Content(content) + .StatusCodeShouldBeUnprocessableEntity()); + } + + [Fact] + public async Task PostAsync_WithProjectSourceMapsWriteToken_CreatesArtifact() + { + var token = await CreateSourceMapUploadTokenAsync(SampleDataService.TEST_PROJECT_ID); + using var content = CreateSourceMapContent(SourceMap); + + var uploaded = await SendRequestAsAsync(request => request + .Post() + .BearerToken(token.Id) + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "source-maps") + .Content(content) + .StatusCodeShouldBeCreated()); + + Assert.NotNull(uploaded); + Assert.Equal(GeneratedFileUrl, uploaded.GeneratedFileUrl); + } + + [Fact] + public async Task PostAsync_WithClientToken_ReturnsForbidden() + { + using var content = CreateSourceMapContent(SourceMap); + + await SendRequestAsync(request => request + .Post() + .AsTestOrganizationClientUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "source-maps") + .Content(content) + .StatusCodeShouldBeForbidden()); + } + + [Fact] + public async Task PostAsync_WithSourceMapsWriteTokenForDifferentProject_ReturnsNotFound() + { + var otherProject = await GetService().AddAsync(new Project + { + Name = "Other project in the same organization", + OrganizationId = SampleDataService.TEST_ORG_ID, + NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks + }, options => options.ImmediateConsistency()); + var token = await CreateSourceMapUploadTokenAsync(SampleDataService.TEST_PROJECT_ID); + using var content = CreateSourceMapContent(SourceMap); + + await SendRequestAsync(request => request + .Post() + .BearerToken(token.Id) + .AppendPaths("projects", otherProject.Id, "source-maps") + .Content(content) + .StatusCodeShouldBeNotFound()); + } + + [Fact] + public async Task PostAsync_WithOrganizationScopedSourceMapsWriteToken_ReturnsNotFound() + { + var utcNow = TimeProvider.GetUtcNow().UtcDateTime; + var token = await GetService().AddAsync(new Token + { + Id = StringExtensions.GetNewToken(), + OrganizationId = SampleDataService.TEST_ORG_ID, + Scopes = [AuthorizationRoles.SourceMapsWrite], + Type = TokenType.Access, + CreatedBy = TestConstants.UserId, + CreatedUtc = utcNow, + UpdatedUtc = utcNow + }, options => options.ImmediateConsistency()); + using var content = CreateSourceMapContent(SourceMap); + + await SendRequestAsync(request => request + .Post() + .BearerToken(token.Id) + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "source-maps") + .Content(content) + .StatusCodeShouldBeNotFound()); + } + + [Fact] + public async Task GetAsync_WithSourceMapsWriteToken_ReturnsForbidden() + { + var token = await CreateSourceMapUploadTokenAsync(SampleDataService.TEST_PROJECT_ID); + + await SendRequestAsync(request => request + .BearerToken(token.Id) + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "source-maps") + .StatusCodeShouldBeForbidden()); + } + [Fact] public Task GetAsync_ProjectOutsideUserOrganization_ReturnsNotFound() { @@ -84,4 +190,24 @@ private static MultipartFormDataContent CreateSourceMapContent(byte[] sourceMap) content.Add(fileContent, "file", "app.min.js.map"); return content; } + + private async Task CreateSourceMapUploadTokenAsync(string projectId) + { + var token = await SendRequestAsAsync(request => request + .Post() + .AsTestOrganizationUser() + .AppendPaths("projects", projectId, "tokens") + .Content(new NewToken + { + OrganizationId = SampleDataService.TEST_ORG_ID, + ProjectId = projectId, + Scopes = [AuthorizationRoles.SourceMapsWrite], + Notes = "Source map deployment" + }) + .StatusCodeShouldBeCreated()); + + Assert.NotNull(token); + await RefreshDataAsync(); + return token; + } } diff --git a/tests/Exceptionless.Tests/Controllers/TokenControllerTests.cs b/tests/Exceptionless.Tests/Controllers/TokenControllerTests.cs index b20f0e369b..4b4c88dad1 100644 --- a/tests/Exceptionless.Tests/Controllers/TokenControllerTests.cs +++ b/tests/Exceptionless.Tests/Controllers/TokenControllerTests.cs @@ -432,6 +432,46 @@ public async Task PostByProjectAsync_WithValidProject_CreatesProjectScopedToken( Assert.Equal(SampleDataService.TEST_PROJECT_ID, token.ProjectId); } + [Fact] + public async Task PostByProjectAsync_WithSourceMapsWriteScope_CreatesProjectScopedToken() + { + var viewToken = await SendRequestAsAsync(request => request + .Post() + .AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "tokens") + .Content(new NewToken + { + OrganizationId = SampleDataService.TEST_ORG_ID, + ProjectId = SampleDataService.TEST_PROJECT_ID, + Scopes = [AuthorizationRoles.SourceMapsWrite], + Notes = "Source map deployment" + }) + .StatusCodeShouldBeCreated()); + + Assert.NotNull(viewToken); + Assert.Equal(SampleDataService.TEST_ORG_ID, viewToken.OrganizationId); + Assert.Equal(SampleDataService.TEST_PROJECT_ID, viewToken.ProjectId); + Assert.Equal([AuthorizationRoles.SourceMapsWrite], viewToken.Scopes); + } + + [Fact] + public async Task PostByOrganizationAsync_WithSourceMapsWriteScope_ReturnsValidationError() + { + var problemDetails = await SendRequestAsAsync(request => request + .Post() + .AsTestOrganizationUser() + .AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "tokens") + .Content(new NewToken + { + OrganizationId = SampleDataService.TEST_ORG_ID, + Scopes = [AuthorizationRoles.SourceMapsWrite] + }) + .StatusCodeShouldBeBadRequest()); + + Assert.NotNull(problemDetails); + Assert.Contains(problemDetails.Errors, error => error.Key.Equals("project_id", StringComparison.OrdinalIgnoreCase)); + } + [Fact] public async Task PostAsync_WithProjectFromUnauthorizedOrganization_ReturnsValidationError() { diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs index 14e37b44ae..051acd3998 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs @@ -148,6 +148,9 @@ public async Task CanCleanupSoftDeletedOrganization() string iconPath = OrganizationStoragePaths.GetProfileImagePath(organization.Id, "icon.png"); using var stream = new MemoryStream([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]); await _fileStorage.SaveFileAsync(iconPath, stream, TestCancellationToken); + string sourceMapPath = $"source-maps/{project.Id}/app.map"; + await using (var sourceMap = new MemoryStream([0x7B, 0x7D])) + await _fileStorage.SaveFileAsync(sourceMapPath, sourceMap, TestCancellationToken); await _job.RunAsync(TestCancellationToken); @@ -156,6 +159,7 @@ public async Task CanCleanupSoftDeletedOrganization() Assert.Null(await _stackRepository.GetByIdAsync(stack.Id, o => o.IncludeSoftDeletes())); Assert.Null(await _eventRepository.GetByIdAsync(persistentEvent.Id, o => o.IncludeSoftDeletes())); Assert.False(await _fileStorage.ExistsAsync(iconPath)); + Assert.False(await _fileStorage.ExistsAsync(sourceMapPath)); } [Fact] diff --git a/tests/Exceptionless.Tests/Services/SourceMaps/SourceMapDocumentTests.cs b/tests/Exceptionless.Tests/Services/SourceMaps/SourceMapDocumentTests.cs index 3a43dfe0c0..2516d65b02 100644 --- a/tests/Exceptionless.Tests/Services/SourceMaps/SourceMapDocumentTests.cs +++ b/tests/Exceptionless.Tests/Services/SourceMaps/SourceMapDocumentTests.cs @@ -55,6 +55,16 @@ public void Parse_IndexedSourceMap_ThrowsJsonException() Assert.Contains("Indexed source maps", exception.Message); } + [Fact] + public void Parse_MissingMappings_ThrowsJsonException() + { + byte[] sourceMap = Encoding.UTF8.GetBytes("""{"version":3,"sources":[],"names":[]}"""); + + var exception = Assert.Throws(() => SourceMapDocument.Parse(sourceMap)); + + Assert.Contains("mappings", exception.Message); + } + [Fact] public void Parse_OversizedVlqValue_ThrowsJsonException() { diff --git a/tests/http/source-maps.http b/tests/http/source-maps.http new file mode 100644 index 0000000000..80d14bafd8 --- /dev/null +++ b/tests/http/source-maps.http @@ -0,0 +1,64 @@ +@apiUrl = http://localhost:7110/api/v2 +@email = admin@exceptionless.test +@password = tester +@organizationId = 537650f3b77efe23a47914f3 +@projectId = 537650f3b77efe23a47914f4 +@generatedFileUrl = https://cdn.example.com/assets/app.min.js +@sourceMapFile = ./app.min.js.map + +### Login to test account +# @name login +POST {{apiUrl}}/auth/login +Content-Type: application/json + +{ + "email": "{{email}}", + "password": "{{password}}" +} + +### + +@userToken = {{login.response.body.$.token}} + +### Create a project-scoped source-map upload token +# @name sourceMapToken +POST {{apiUrl}}/projects/{{projectId}}/tokens +Authorization: Bearer {{userToken}} +Content-Type: application/json + +{ + "organization_id": "{{organizationId}}", + "project_id": "{{projectId}}", + "scopes": ["source-maps:write"], + "notes": "HTTP sample source map uploads" +} + +### + +@uploadToken = {{sourceMapToken.response.body.$.id}} + +### Upload source map +# Replace {{sourceMapFile}} with a local version 3 source map. +# @name uploadedSourceMap +POST {{apiUrl}}/projects/{{projectId}}/source-maps +Authorization: Bearer {{uploadToken}} +Content-Type: multipart/form-data; boundary=source-map-boundary + +--source-map-boundary +Content-Disposition: form-data; name="generated_file_url" + +{{generatedFileUrl}} +--source-map-boundary +Content-Disposition: form-data; name="file"; filename="app.min.js.map" +Content-Type: application/json + +< {{sourceMapFile}} +--source-map-boundary-- + +### List source maps +GET {{apiUrl}}/projects/{{projectId}}/source-maps +Authorization: Bearer {{userToken}} + +### Delete source map +DELETE {{apiUrl}}/projects/{{projectId}}/source-maps/{{uploadedSourceMap.response.body.$.id}} +Authorization: Bearer {{userToken}} From 5bc2414e48eb8bd1cf08edaebf7e04664c4c7489 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 14 Jul 2026 19:06:54 -0500 Subject: [PATCH 04/15] Add source map token creation to the UI --- .../dialogs/disable-token-dialog.svelte | 4 +- .../dialogs/enable-token-dialog.svelte | 4 +- .../dialogs/remove-token-dialog.svelte | 4 +- .../dialogs/update-token-notes-dialog.svelte | 2 +- .../tokens/components/table/options.svelte.ts | 13 +++- .../table/token-actions-cell.svelte | 12 ++-- .../components/table/token-scopes-cell.svelte | 15 +++++ .../lib/features/tokens/project-token.test.ts | 27 ++++++++ .../src/lib/features/tokens/project-token.ts | 12 ++++ .../project/[projectId]/api-keys/+page.svelte | 61 +++++++++++++++---- .../[projectId]/source-maps/+page.svelte | 7 ++- 11 files changed, 133 insertions(+), 28 deletions(-) create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/token-scopes-cell.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/tokens/project-token.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/tokens/project-token.ts diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/disable-token-dialog.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/disable-token-dialog.svelte index 9201f757d4..b53df29ac8 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/disable-token-dialog.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/disable-token-dialog.svelte @@ -20,14 +20,14 @@ - Disable API Key + Disable Token Are you sure you want to disable "{id}" {#if notes}({notes}){/if}? Cancel - Disable API Key + Disable Token diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/enable-token-dialog.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/enable-token-dialog.svelte index a0679bad08..10d581c7bd 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/enable-token-dialog.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/enable-token-dialog.svelte @@ -19,14 +19,14 @@ - Enable API Key + Enable Token Are you sure you want to enable "{id}" {#if notes}({notes}){/if}? Cancel - Enable API Key + Enable Token diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/remove-token-dialog.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/remove-token-dialog.svelte index 9382fd1d5f..49041755bf 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/remove-token-dialog.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/remove-token-dialog.svelte @@ -20,14 +20,14 @@ - Delete API Key + Delete Token Are you sure you want to delete "{id}" {#if notes}({notes}){/if}? Cancel - Delete API Key + Delete Token diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/update-token-notes-dialog.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/update-token-notes-dialog.svelte index f375843862..5226ef59ef 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/update-token-notes-dialog.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/dialogs/update-token-notes-dialog.svelte @@ -57,7 +57,7 @@ }} > - API Key Notes + Token Notes state.errors}> diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/options.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/options.svelte.ts index 8142bc8593..fef24c4c75 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/options.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/options.svelte.ts @@ -5,6 +5,7 @@ import type { CreateQueryResult } from '@tanstack/svelte-query'; import { getSharedTableOptions } from '$features/shared/table.svelte'; import TokenActionsCell from '$features/tokens/components/table/token-actions-cell.svelte'; import TokenIdCell from '$features/tokens/components/table/token-id-cell.svelte'; +import TokenScopesCell from '$features/tokens/components/table/token-scopes-cell.svelte'; import { type ColumnDef, renderComponent, type StockFeatures } from '@tanstack/svelte-table'; import type { GetProjectTokensParams } from '../../api.svelte'; @@ -16,7 +17,17 @@ export function getColumns(): ColumnDef renderComponent(TokenIdCell, { token: info.row.original }), enableHiding: false, enableSorting: false, - header: 'API Key', + header: 'Token', + meta: { + class: 'w-45' + } + }, + { + accessorKey: 'scopes', + cell: (info) => renderComponent(TokenScopesCell, { scopes: info.getValue() }), + enableHiding: true, + enableSorting: false, + header: 'Scope', meta: { class: 'w-45' } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/token-actions-cell.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/token-actions-cell.svelte index 0281e538b0..3f6e22616e 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/token-actions-cell.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/token-actions-cell.svelte @@ -31,7 +31,7 @@ const clipboard = new UseClipboard(); async function copyToClipboard() { - await clipboard.copy(token.id); // Assuming token.apiKey holds the API key to copy + await clipboard.copy(token.id); if (clipboard.copied) { toast.success('Copy to clipboard succeeded'); } else { @@ -49,7 +49,7 @@ async function updateDisabled(is_disabled: boolean) { await updateToken.mutateAsync({ is_disabled, notes: token.notes }); - toast.success(`Successfully ${is_disabled ? 'disabled' : 'enabled'} API key`); + toast.success(`Successfully ${is_disabled ? 'disabled' : 'enabled'} token`); } function onEnableDisableClick() { @@ -75,7 +75,7 @@ async function remove() { await removeToken.mutateAsync(); - toast.success('Successfully deleted API key'); + toast.success('Successfully deleted token'); } @@ -91,7 +91,7 @@ - Copy Api Key + Copy Token (showUpdateNotesDialog = true)}> @@ -100,10 +100,10 @@ {#if token.is_disabled} - Enable API Key + Enable Token {:else} - Disable API Key + Disable Token {/if} (showRemoveTokenDialog = true)} disabled={removeToken.isPending}> diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/token-scopes-cell.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/token-scopes-cell.svelte new file mode 100644 index 0000000000..712c707746 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/components/table/token-scopes-cell.svelte @@ -0,0 +1,15 @@ + + +
+ {#each scopes as scope (scope)} + {scope} + {/each} +
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/project-token.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/project-token.test.ts new file mode 100644 index 0000000000..666b93b963 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/project-token.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; + +import { createProjectToken } from './project-token'; + +describe('createProjectToken', () => { + it('creates a project-scoped source map upload token', () => { + const token = createProjectToken('organization-id', 'project-id', 'source-maps:write'); + + expect(token).toEqual({ + notes: 'Source map deployment', + organization_id: 'organization-id', + project_id: 'project-id', + scopes: ['source-maps:write'] + }); + }); + + it('preserves normal client API key creation', () => { + const token = createProjectToken('organization-id', 'project-id', 'client'); + + expect(token).toEqual({ + notes: undefined, + organization_id: 'organization-id', + project_id: 'project-id', + scopes: ['client'] + }); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/project-token.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/project-token.ts new file mode 100644 index 0000000000..286abcbe1c --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tokens/project-token.ts @@ -0,0 +1,12 @@ +import type { NewToken } from '$features/tokens/models'; + +export type ProjectTokenScope = 'client' | 'source-maps:write'; + +export function createProjectToken(organizationId: string, projectId: string, scope: ProjectTokenScope): NewToken { + return { + notes: scope === 'source-maps:write' ? 'Source map deployment' : undefined, + organization_id: organizationId, + project_id: projectId, + scopes: [scope] + }; +} diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/api-keys/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/api-keys/+page.svelte index 4c69ffdaed..f25ff8cbed 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/api-keys/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/api-keys/+page.svelte @@ -1,16 +1,20 @@