From 66c8775589777746fde7067425a1bac5cbe7216a Mon Sep 17 00:00:00 2001 From: Chris Cavell Date: Mon, 21 Sep 2026 18:59:17 -0500 Subject: [PATCH] Bound the reconciliation query for audit records without a batch id ReconcileAsync OR-ed records without a mutation batch id into the bounded batch query, so every such record ever written was loaded on every pass of the hosted reconciliation loop. Those records are now read by a separate query, newest first, capped by the new MaximumMalformedRecordsPerRun option (default 1000, validated 1-10000). Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 3 ++ .../Auditing/ApplicationAuditReconciler.cs | 14 +++++++- ...ApplicationAuditReconciliationContracts.cs | 2 ++ ...ionAuditReconciliationServiceExtensions.cs | 2 ++ .../ApplicationAuditReconciliationTests.cs | 33 +++++++++++++++++-- 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41e7298..9c2ff85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ This project follows Semantic Versioning using the format `MAJOR.MINOR.PATCH`. and a full rollback instead of a last-writer-wins or partially applied run. * `byte[]` audit values no longer canonicalize to the literal `System.Byte[]`, which made every binary value hash and truncate to the same constant. +* Audit reconciliation no longer loads every audit record without a mutation + batch id on each pass. Those records are now read newest first and capped by + the new `MaximumMalformedRecordsPerRun` option (default 1000, range 1-10000). ## 2.10.0 - 2026-09-19 diff --git a/src/ProjectTemplate.Infrastructure/Data/Auditing/ApplicationAuditReconciler.cs b/src/ProjectTemplate.Infrastructure/Data/Auditing/ApplicationAuditReconciler.cs index a7e2221..2c11f9d 100644 --- a/src/ProjectTemplate.Infrastructure/Data/Auditing/ApplicationAuditReconciler.cs +++ b/src/ProjectTemplate.Infrastructure/Data/Auditing/ApplicationAuditReconciler.cs @@ -48,10 +48,22 @@ public async Task ReconcileAsync( List auditRecords = await _dbContext.AuditRecords .AsNoTracking() - .Where(record => batchIds.Contains(record.MutationBatchId) || record.MutationBatchId == string.Empty) + .Where(record => batchIds.Contains(record.MutationBatchId)) .ToListAsync(cancellationToken) .ConfigureAwait(false); + // Records without a batch id are not bounded by the batch selection above, so cap them separately; + // otherwise every such record ever written would be loaded on every reconciliation pass. + List malformedRecords = await _dbContext.AuditRecords + .AsNoTracking() + .Where(record => record.MutationBatchId == string.Empty) + .OrderByDescending(record => record.ModifiedOnUtc) + .ThenByDescending(record => record.Id) + .Take(_options.MaximumMalformedRecordsPerRun) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + auditRecords.AddRange(malformedRecords); + List completionEntries = await _dbContext .ApplicationAuditCompletionOutboxEntries .AsNoTracking() diff --git a/src/ProjectTemplate.Infrastructure/Data/Auditing/ApplicationAuditReconciliationContracts.cs b/src/ProjectTemplate.Infrastructure/Data/Auditing/ApplicationAuditReconciliationContracts.cs index 9596e39..88d3d5c 100644 --- a/src/ProjectTemplate.Infrastructure/Data/Auditing/ApplicationAuditReconciliationContracts.cs +++ b/src/ProjectTemplate.Infrastructure/Data/Auditing/ApplicationAuditReconciliationContracts.cs @@ -52,6 +52,8 @@ public sealed class ApplicationAuditReconciliationOptions public int MaximumBatchesPerRun { get; set; } = 1_000; + public int MaximumMalformedRecordsPerRun { get; set; } = 1_000; + public int HealthWarningFindingCount { get; set; } = 1; public int HealthUnhealthyFindingCount { get; set; } = 10; diff --git a/src/ProjectTemplate.Infrastructure/Data/Extensions/ApplicationAuditReconciliationServiceExtensions.cs b/src/ProjectTemplate.Infrastructure/Data/Extensions/ApplicationAuditReconciliationServiceExtensions.cs index 674cd23..abd8ccc 100644 --- a/src/ProjectTemplate.Infrastructure/Data/Extensions/ApplicationAuditReconciliationServiceExtensions.cs +++ b/src/ProjectTemplate.Infrastructure/Data/Extensions/ApplicationAuditReconciliationServiceExtensions.cs @@ -27,6 +27,8 @@ public static IServiceCollection AddApplicationAuditReconciliationCore( "The stale retry-ready threshold must be greater than zero.") .Validate(options => options.MaximumBatchesPerRun is > 0 and <= 10_000, "Maximum batches per reconciliation run must be between 1 and 10000.") + .Validate(options => options.MaximumMalformedRecordsPerRun is > 0 and <= 10_000, + "Maximum malformed audit records per reconciliation run must be between 1 and 10000.") .Validate(options => options.HealthWarningFindingCount >= 0, "The warning finding threshold must not be negative.") .Validate(options => options.HealthUnhealthyFindingCount >= options.HealthWarningFindingCount, diff --git a/tests/ProjectTemplate.Web.Tests/ApplicationAuditReconciliationTests.cs b/tests/ProjectTemplate.Web.Tests/ApplicationAuditReconciliationTests.cs index 8d53830..b22db96 100644 --- a/tests/ProjectTemplate.Web.Tests/ApplicationAuditReconciliationTests.cs +++ b/tests/ProjectTemplate.Web.Tests/ApplicationAuditReconciliationTests.cs @@ -379,6 +379,33 @@ public async Task RecordRemediationAsync_InvalidRequest_ThrowsBeforeWriting() await AssertFindingUnchangedAsync(database, finding); } + [Fact] + public async Task ReconcileAsync_RecordsWithoutBatchId_AreBoundedPerRunNewestFirst() + { + await using TestDatabase database = await TestDatabase.CreateAsync(maximumMalformedRecordsPerRun: 2); + var malformed = new List(); + for (int index = 0; index < 5; index++) + { + AuditRecord record = CreateAuditRecord(string.Empty); + record.ModifiedOnUtc = _now.AddMinutes(-10 - index); + malformed.Add(record); + } + + database.Context.AuditRecords.AddRange(malformed); + _ = await database.Context.SaveChangesAsync(TestContext.Current.CancellationToken); + + _ = await database.Reconciler.ReconcileAsync(TestContext.Current.CancellationToken); + + List findingKeys = await database.Context.ApplicationAuditReconciliationFindings + .AsNoTracking() + .Where(finding => finding.ReasonCode == ApplicationAuditReconciliationReasonCodes.MalformedCorrelation) + .Select(finding => finding.MutationBatchId) + .ToListAsync(TestContext.Current.CancellationToken); + Assert.Equal( + malformed.Take(2).Select(record => $"missing-{record.Id:N}").Order(StringComparer.Ordinal), + findingKeys.Order(StringComparer.Ordinal)); + } + [Fact] public async Task DisabledMode_DoesNotCreateFindings() { @@ -494,7 +521,8 @@ private TestDatabase( public static async Task CreateAsync( bool enabled = true, - IInterceptor? interceptor = null) + IInterceptor? interceptor = null, + int maximumMalformedRecordsPerRun = 1_000) { var connection = new SqliteConnection("Data Source=:memory:"); await connection.OpenAsync(TestContext.Current.CancellationToken); @@ -533,7 +561,8 @@ public static async Task CreateAsync( Enabled = enabled, CompletionGracePeriod = TimeSpan.Zero, StalePendingThreshold = TimeSpan.FromMinutes(15), - StaleRetryReadyThreshold = TimeSpan.FromMinutes(15) + StaleRetryReadyThreshold = TimeSpan.FromMinutes(15), + MaximumMalformedRecordsPerRun = maximumMalformedRecordsPerRun }), metrics, new FixedTimeProvider(_now));