From de937428da09ab7fb399f08fca929d67076f420d Mon Sep 17 00:00:00 2001 From: fabbrik <22822543+fabbrik@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:17:19 -0300 Subject: [PATCH] feat: finalize captured runs into durable experience Add ExperienceFinalizationService: one call that loads a completed captured run, evaluates its own closed verification round, checks host authorization and the host's storage decision, reflects, creates the record as Candidate, and commits its initial lifecycle event to Validated or Quarantined. Record, reflection and event IDs derive from the run, so a retry re-derives them and converges instead of duplicating. Required checks can now name the evaluator kind that may satisfy them, and Core and Storage.Postgres each expose a service-registration extension. The MAF adapter finalizes a fully captured run through the host's resolver. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 93 +- .../AgentExperience.Core.csproj | 5 +- ...perienceCoreServiceCollectionExtensions.cs | 70 ++ .../ExperienceFinalizationService.cs | 697 ++++++++++++ .../Finalization/FinalizationResults.cs | 150 +++ .../Finalization/FinalizeExperienceRequest.cs | 40 + .../Finalization/StorageDecision.cs | 30 + .../Verification/RequiredCheck.cs | 41 + .../Verification/VerificationAggregator.cs | 53 +- src/AgentExperience.Core/packages.lock.json | 11 +- .../CaptureScope.cs | 102 +- ...ExperienceCaptureAgentBuilderExtensions.cs | 17 +- .../ExperienceCaptureOptions.cs | 67 +- .../README.md | 56 +- .../packages.lock.json | 3 +- .../AgentExperience.Storage.Postgres.csproj | 3 + ...encePostgresServiceCollectionExtensions.cs | 57 + .../README.md | 29 +- .../packages.lock.json | 11 +- .../packages.lock.json | 26 +- .../AgentExperience.Core.Tests.csproj | 3 + .../CoreServiceRegistrationTests.cs | 110 ++ .../DefaultExperienceReflectorTests.cs | 11 +- .../DependencyBoundaryTests.cs | 29 +- .../ExperienceFinalizationServiceTests.cs | 993 ++++++++++++++++++ .../VerificationAggregatorTests.cs | 131 ++- .../packages.lock.json | 12 +- .../ExperienceFinalizationWiringTests.cs | 431 ++++++++ .../packages.lock.json | 3 +- ...ntExperience.Storage.Postgres.Tests.csproj | 13 + .../DependencyBoundaryTests.cs | 17 +- .../PostgresFinalizationTests.cs | 322 ++++++ .../PostgresServiceRegistrationTests.cs | 68 ++ .../packages.lock.json | 97 +- 34 files changed, 3698 insertions(+), 103 deletions(-) create mode 100644 src/AgentExperience.Core/DependencyInjection/AgentExperienceCoreServiceCollectionExtensions.cs create mode 100644 src/AgentExperience.Core/Finalization/ExperienceFinalizationService.cs create mode 100644 src/AgentExperience.Core/Finalization/FinalizationResults.cs create mode 100644 src/AgentExperience.Core/Finalization/FinalizeExperienceRequest.cs create mode 100644 src/AgentExperience.Core/Finalization/StorageDecision.cs create mode 100644 src/AgentExperience.Core/Verification/RequiredCheck.cs create mode 100644 src/AgentExperience.Storage.Postgres/DependencyInjection/AgentExperiencePostgresServiceCollectionExtensions.cs create mode 100644 tests/AgentExperience.Core.Tests/CoreServiceRegistrationTests.cs create mode 100644 tests/AgentExperience.Core.Tests/ExperienceFinalizationServiceTests.cs create mode 100644 tests/AgentExperience.MicrosoftAgentFramework.Tests/ExperienceFinalizationWiringTests.cs create mode 100644 tests/AgentExperience.Storage.Postgres.Tests/PostgresFinalizationTests.cs create mode 100644 tests/AgentExperience.Storage.Postgres.Tests/PostgresServiceRegistrationTests.cs diff --git a/README.md b/README.md index 0ef8ec8..38431d6 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ AgentExperience.NET captures what an AI agent actually tried, verifies whether it worked, and turns the result into an auditable lesson that future runs can reuse safely. It sits between [Microsoft Agent Framework](https://github.com/microsoft/agent-framework) (MAF) execution and durable storage, without replacing either. -> **Status: early development.** Epic 1 (capture and explain agent experience) is implemented and tested. Epic 2 has started: Experience Records can be stored in PostgreSQL and moved through their lifecycle with atomic, audited commits. Retrieval, injection, and governance are planned (see [Roadmap](#roadmap)). Nothing is published to NuGet yet, and APIs may change. +> **Status: early development.** Epic 1 (capture and explain agent experience) is implemented and tested. Epic 2 has started: a completed run can now be finalized into a durable Experience Record in PostgreSQL in one call, and moved through its lifecycle with atomic, audited commits. Retrieval, injection, and governance are planned (see [Roadmap](#roadmap)). Nothing is published to NuGet yet, and APIs may change. ## Why @@ -33,6 +33,8 @@ AgentExperience.NET records observable evidence (tool calls, results, errors, ve | PostgreSQL Experience Record store: create, get, and scoped query; host authorization checked before database access; exact scope matching in SQL | `AgentExperience.Storage.Postgres` | | Atomic audited lifecycle commits: the event and the record's projection in one transaction, idempotent by event ID, revision-checked, with append-only history | `AgentExperience.Core`, `AgentExperience.Storage.Postgres` | | Journaled schema migrations: embedded scripts applied once, one transaction per script, serialized across processes by an advisory lock | `AgentExperience.Storage.Postgres` | +| One finalization call: evaluate, gate on authorization and the host's storage decision, reflect, create the record as a `Candidate`, commit the initial event that promotes it — replay-safe and structured at every stage | `AgentExperience.Core` | +| Dependency-injection registration for each package, so a host wires capture, finalization, and storage without knowing concrete types | `AgentExperience.Core`, `AgentExperience.Storage.Postgres` | ## Quick look @@ -54,9 +56,88 @@ await agent.RunAsync("Triage ticket #4812", session); See the [adapter README](src/AgentExperience.MicrosoftAgentFramework/README.md) for options, supported agent types, and caveats. See the [PostgreSQL store README](src/AgentExperience.Storage.Postgres/README.md) for the trust boundary, the `ExperienceSchemaMigrator.MigrateAsync` startup call, and data semantics. +## Turning a run into a durable record + +A captured run becomes a durable, reusable Experience Record through one Core call. +`ExperienceFinalizationService.FinalizeAsync` runs six stages in order — load the captured snapshot, evaluate it, +check authorization and the host's storage decision, reflect on it, create the record, commit its initial lifecycle +event — and stops at the first stage that ends the call, always returning a structured result rather than throwing. +The two gates precede reflection on purpose: the reflector is the seam a host would plug a model into, so a run that +is about to be refused is never handed to it. + +```csharp +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Verification; + +var result = await finalization.FinalizeAsync( + new FinalizeExperienceRequest( + RunId: runId, + Authorization: authorization, // host-established; the run's scope must lie inside it + ClosedRound: new ClosedVerificationRound(roundId, "rev-7"), + RequiredChecks: [new RequiredCheck("unit-tests-pass", ExpectedKind: "TestResult")], + Evidence: evidence, // finalization filters and aggregates it itself + CurrentArtifactRevision: "rev-7", + StorageDecision: StorageDecision.Permit, // or StorageDecision.Deny("retention policy") + FinalizedAt: DateTimeOffset.UtcNow), + cancellationToken); + +if (result.IsDurable) +{ + logger.LogInformation("Experience {Id} is {Status} at revision {Revision}", + result.ExperienceId, result.Status, result.Revision); +} +else +{ + logger.LogWarning("Finalization ended at {Stage}: {Outcome} — {Reason}", + result.Stage, result.Outcome, result.Failure?.Reason); +} +``` + +| Outcome | When | What was written | +| --- | --- | --- | +| `Validated` | Verified, reflection succeeded, storage permitted | The record (reuse confidence 2/3, one supporting validation, no contradictions), created as `Candidate`, plus the initial event that moved it to `Validated` | +| `Quarantined` | Storage permitted, but verification did not pass or the reflector threw | The record, with **no** reflection, created as `Candidate`, plus the initial event that moved it to `Quarantined`. `Failure` names the stage that decided it | +| `AlreadyFinalized` | This run's record already exists *and* is already confirmed | Nothing. The result reports the stored record, status, and revision. (A record left unconfirmed by an earlier call is resumed instead: the retry commits its initial event and returns `Validated`/`Quarantined`.) | +| `StorageDenied` | The host's `StorageDecision` denied | Nothing at all, and no record ID is issued | +| `NotAuthorized` | The run's scope lies outside the authorization | Nothing; denied before any store call | +| `RunNotFound` / `RunNotFinished` | No such captured run, or it has no execution status | Nothing | +| `Failed` | A stage failed (for example the database was unavailable) | Never reported as durable. Any record already created stays a `Candidate`, which is never reusable, and the captured run stays available for a retry | + +Three properties make retrying safe. The record is *created* as a `Candidate` and its initial lifecycle event +performs the real transition, so a commit that never lands leaves nothing reusable behind. The record ID, the +reflection ID, and the initial event ID are all derived from the run ID, so a second call cannot create a second +record or a second initial confirmation. And the initial event's fields are a pure function of the stored record, so +a retry re-derives exactly the event the store already deduplicates on. + +Finalization never sanitizes — capture already rejected anything unsafe — and never decides storage or risk policy on +the host's behalf: `StorageDecision` travels in the request and Core simply obeys it. + +### Wiring it + +Each package registers its own services, so a host never names a concrete type: + +```csharp +using AgentExperience.Core.DependencyInjection; +using AgentExperience.Storage.Postgres.DependencyInjection; + +services.AddSingleton(NpgsqlDataSource.Create(connectionString)); +services.AddAgentExperiencePostgresStore(); // IExperienceRecordStore +services.AddAgentExperienceCore(sanitizationOptions, captureLimits); +// -> ISanitizer, IExperienceCaptureService, IExperienceReflector, +// ExperienceLifecycleService, ExperienceFinalizationService +``` + +`AgentExperience.Abstractions` stays BCL-only; only `Core` and the storage adapter take +`Microsoft.Extensions.DependencyInjection.Abstractions`, and every registration uses `TryAdd`, so a host's own +implementation wins. Call `ExperienceSchemaMigrator.MigrateAsync` once at startup before the store is used. + +The MAF adapter can drive finalization for you: set `FinalizationService` and `ResolveFinalization` on +`ExperienceCaptureOptions` and every successfully captured invocation is finalized right after it is completed. See +the [adapter README](src/AgentExperience.MicrosoftAgentFramework/README.md#finalizing-captured-runs). + ## Design principles -- **Hexagonal core.** `Abstractions` and `Core` depend only on the BCL and a redaction primitive. MAF, databases, models, and telemetry stay in adapters. Dependency-boundary tests enforce this in CI. +- **Hexagonal core.** `Abstractions` depends only on the BCL; `Core` adds a redaction primitive and the dependency-injection *abstractions* it needs to register its own services. MAF, databases, models, and telemetry stay in adapters. Dependency-boundary tests enforce this in CI. - **Failure-preserving capture.** Failed and cancelled runs are recorded through an outer lifecycle path, never only a success callback. - **Evidence before trust.** Verification is deterministic and bound to a host-closed round and artifact revision. A completion score is never mistaken for reuse confidence. - **Sanitize before anything is stored.** Unknown payload fields are dropped by default, and secrets are redacted from nested values. @@ -67,7 +148,7 @@ See the [adapter README](src/AgentExperience.MicrosoftAgentFramework/README.md) ``` src/ AgentExperience.Abstractions/ domain contracts and ports (BCL only) - AgentExperience.Core/ sanitization, capture, verification, reflection, lifecycle transitions + AgentExperience.Core/ sanitization, capture, verification, reflection, lifecycle transitions, finalization AgentExperience.MicrosoftAgentFramework/ MAF adapter (pinned Microsoft.Agents.AI 1.20.0) AgentExperience.Storage.Postgres/ PostgreSQL Experience Record store and schema migrator (pinned Npgsql 10.0.3, dbup-postgresql 7.0.1, dbup-core 6.1.1) tests/ @@ -90,16 +171,16 @@ dotnet build dotnet test ``` -Unit and MAF adapter tests run in memory, with no network, database, or model credentials. `AgentExperience.CompatibilityProof` and the `PostgresExperienceRecordStoreTests`, `PostgresLifecycleCommitTests`, and `ExperienceSchemaMigratorTests` in `AgentExperience.Storage.Postgres.Tests` start a PostgreSQL/pgvector container through Testcontainers, so they need Docker. If Testcontainers' Ryuk container fails to start under your local Docker setup, set `TESTCONTAINERS_RYUK_DISABLED=true`. To skip the container-backed tests: +Unit and MAF adapter tests run in memory, with no network, database, or model credentials. `AgentExperience.CompatibilityProof` and the `PostgresExperienceRecordStoreTests`, `PostgresLifecycleCommitTests`, `PostgresFinalizationTests`, and `ExperienceSchemaMigratorTests` in `AgentExperience.Storage.Postgres.Tests` start a PostgreSQL/pgvector container through Testcontainers, so they need Docker. If Testcontainers' Ryuk container fails to start under your local Docker setup, set `TESTCONTAINERS_RYUK_DISABLED=true`. To skip the container-backed tests: ```bash -dotnet test --filter "FullyQualifiedName!~CompatibilityProof&FullyQualifiedName!~PostgresExperienceRecordStoreTests&FullyQualifiedName!~PostgresLifecycleCommitTests&FullyQualifiedName!~ExperienceSchemaMigratorTests" +dotnet test --filter "FullyQualifiedName!~CompatibilityProof&FullyQualifiedName!~PostgresExperienceRecordStoreTests&FullyQualifiedName!~PostgresLifecycleCommitTests&FullyQualifiedName!~PostgresFinalizationTests&FullyQualifiedName!~ExperienceSchemaMigratorTests" ``` ## Roadmap 1. **Capture and explain agent experience** ✅ contracts, sanitization, capture, verification, reflection, MAF adapter -2. **Reuse relevant experience:** PostgreSQL persistence and atomic audited lifecycle commits (in place), hybrid text and vector retrieval, historical-reference injection into MAF +2. **Reuse relevant experience:** PostgreSQL persistence, atomic audited lifecycle commits, and one-call finalization of captured runs (in place), hybrid text and vector retrieval, historical-reference injection into MAF 3. **Govern experience safely:** sharing grants, the remaining lifecycle transitions, evidence-based confidence updates 4. **Operate and measure the learning loop:** OpenTelemetry instrumentation, an end-to-end demo, measured reuse against a baseline, data deletion and expiry diff --git a/src/AgentExperience.Core/AgentExperience.Core.csproj b/src/AgentExperience.Core/AgentExperience.Core.csproj index 117d9b5..20bae5a 100644 --- a/src/AgentExperience.Core/AgentExperience.Core.csproj +++ b/src/AgentExperience.Core/AgentExperience.Core.csproj @@ -1,11 +1,14 @@ - AgentExperience.NET's first production Core package: the default sanitization pipeline (per-Kind allowlists, secret-field classification, recursive traversal, fail-closed rejection) built on AgentExperience.Abstractions' ISanitizer port. No dependency on MAF, EF Core, PostgreSQL, model providers, or OpenTelemetry -- only the BCL, Abstractions, and Microsoft.Extensions.Compliance.Redaction (AD-1). + AgentExperience.NET's first production Core package: the default sanitization pipeline (per-Kind allowlists, secret-field classification, recursive traversal, fail-closed rejection) built on AgentExperience.Abstractions' ISanitizer port. No dependency on MAF, EF Core, PostgreSQL, model providers, or OpenTelemetry -- only the BCL, Abstractions, Microsoft.Extensions.Compliance.Redaction, and Microsoft.Extensions.DependencyInjection.Abstractions (AD-1). + + diff --git a/src/AgentExperience.Core/DependencyInjection/AgentExperienceCoreServiceCollectionExtensions.cs b/src/AgentExperience.Core/DependencyInjection/AgentExperienceCoreServiceCollectionExtensions.cs new file mode 100644 index 0000000..6a3cfc6 --- /dev/null +++ b/src/AgentExperience.Core/DependencyInjection/AgentExperienceCoreServiceCollectionExtensions.cs @@ -0,0 +1,70 @@ +using AgentExperience.Abstractions; +using AgentExperience.Core.Capture; +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Lifecycle; +using AgentExperience.Core.Reflections; +using AgentExperience.Core.Sanitization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace AgentExperience.Core.DependencyInjection; + +/// +/// Registers AgentExperience.NET's Core services in a . Core owns +/// its own registration so a host never has to know which concrete types implement which port; the +/// storage adapter registers its own in the same way (see +/// AddAgentExperiencePostgresStore), and AgentExperience.Abstractions stays BCL-only. +/// +public static class AgentExperienceCoreServiceCollectionExtensions +{ + /// + /// Registers the sanitizer, the in-memory capture service, the default reflector, the lifecycle + /// service, and the finalization service as singletons. + /// + /// + /// + /// Every registration uses TryAdd, so a host that has already registered its own + /// , , or + /// keeps it. + /// + /// + /// and both + /// need an , which Core does not implement: register a + /// storage adapter (for example AddAgentExperiencePostgresStore) as well, or resolving + /// them fails. + /// + /// + /// No sanitization policy or capture limit is invented here: both are host decisions with real + /// security and memory consequences, so both are required arguments. + /// + /// + /// The service collection to add to. + /// The per-Kind sanitization policy the default sanitizer applies. + /// The limits in-memory capture enforces. + /// , for chaining. + /// Any argument is . + public static IServiceCollection AddAgentExperienceCore( + this IServiceCollection services, + SanitizationOptions sanitizationOptions, + CaptureLimits captureLimits) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(sanitizationOptions); + ArgumentNullException.ThrowIfNull(captureLimits); + + // The arguments are captured by the factories rather than resolved back out of the container. + // Re-resolving them would let a SanitizationOptions or CaptureLimits the host registered earlier + // silently replace the caller's, so the sanitizer would run a policy nobody passed to it. + services.TryAddSingleton(sanitizationOptions); + services.TryAddSingleton(captureLimits); + services.TryAddSingleton(_ => new DefaultSanitizer(sanitizationOptions)); + services.TryAddSingleton(provider => new InMemoryExperienceCaptureService( + provider.GetRequiredService(), + captureLimits)); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + + return services; + } +} diff --git a/src/AgentExperience.Core/Finalization/ExperienceFinalizationService.cs b/src/AgentExperience.Core/Finalization/ExperienceFinalizationService.cs new file mode 100644 index 0000000..e1c4448 --- /dev/null +++ b/src/AgentExperience.Core/Finalization/ExperienceFinalizationService.cs @@ -0,0 +1,697 @@ +using System.Globalization; +using System.Security.Cryptography; +using AgentExperience.Abstractions; +using AgentExperience.Core.Capture; +using AgentExperience.Core.Lifecycle; +using AgentExperience.Core.Reflections; +using AgentExperience.Core.Verification; + +namespace AgentExperience.Core.Finalization; + +/// +/// The single Core call that turns a captured, completed run into a durable Experience Record. It +/// runs the stages of in order and stops at the first one that ends +/// the call, returning a structured naming that stage -- +/// never an exception for an expected condition, and never a durable success for a database failure. +/// +/// +/// +/// Stages. Load the captured snapshot, evaluate it against its own host-closed round, check the +/// host authorization and the host's storage decision, reflect on it, create the record, then commit +/// the record's initial lifecycle event. The two gates deliberately precede reflection: +/// is the documented seam for a model-backed reflector, so a run +/// the host is about to refuse is never handed to it. +/// +/// +/// Validated vs quarantined. A verified evaluation plus a successful reflection plus a +/// permitting storage decision produces a record with reuse +/// confidence 2/3, one supporting validation and no contradictions. A permitted record whose +/// verification is not , or whose reflection threw, is +/// instead, carrying no reflection at all and safe failure +/// metadata on the result. Reflection is not even attempted for an unverified run, so an unreflected +/// lesson can never reach a quarantined record. Confidence is never computed from evidence counts +/// (that is a later story), and risk is never decided on the host's behalf. +/// +/// +/// Candidate first. The record is created as , +/// and its initial lifecycle event performs the real transition to +/// or through +/// Core's transition table (both are allowed moves, and the store's prior-status guard applies). So a +/// commit that never lands leaves a -- never injectable -- +/// rather than a reusable record with no lifecycle history. The record's reuse-confidence inputs are +/// stamped at create time because a lifecycle commit updates only status, revision, and the updated +/// timestamp: the store persists Core's decision and derives no score of its own (2.4). The returned +/// record mirrors the projection the commit applied. +/// +/// +/// Replay. The record ID, the reflection ID, and the initial event ID are all derived from the +/// run ID, and the record's is the event's +/// , so finalizing the same run twice cannot create a second +/// record or a second initial confirmation. A second call finds the stored record and reports the +/// first call's outcome. When an earlier call created the record but its initial commit did not land +/// (the record is still at revision 0), a retry finishes that commit rather than starting over. +/// +/// +/// Sanitization. Finalization never sanitizes: capture already rejected anything unsafe before +/// storing an attempt, so the run's attempts are copied onto the record unchanged. +/// +/// +/// Failures. Every stage failure comes back as a structured +/// result naming the stage, including any exception a port +/// throws; the captured snapshot is never evicted, so the host can retry. The one exception is +/// cancellation: an from any stage -- the caller's +/// token or a port cancelling for its own reasons -- always propagates, so a cancelled call never +/// silently becomes a quarantined record. +/// +/// +public sealed class ExperienceFinalizationService +{ + /// The every initial event this service commits carries. + public const string ProducerIdentity = "AgentExperience.ExperienceFinalizationService/1.0.0"; + + /// + /// The reuse confidence a freshly validated record starts at: two thirds. It is an initial, + /// evidence-gated value, not a score computed from evidence counts. + /// + public const double InitialValidatedReuseConfidence = 2d / 3d; + + /// The status every Experience Record is created in, before its initial lifecycle event moves it. + public const ExperienceStatus CreatedStatus = ExperienceStatus.Candidate; + + /// + /// Fixed namespace for the derived identifiers below. Changing it would re-issue every record ID, + /// so it is a constant of this library, never configurable. + /// + private static readonly Guid DerivationNamespace = new("0b6a8a3f-1c2d-4f5e-9a70-3d1c9f2b8e41"); + + private const byte ExperienceIdTag = 1; + private const byte InitialEventIdTag = 2; + private const byte ReflectionIdTag = 3; + + private static readonly IReadOnlyList NoErrors = []; + + private readonly IExperienceCaptureService _captureService; + private readonly IExperienceReflector _reflector; + private readonly IExperienceRecordStore _store; + private readonly ExperienceLifecycleService _lifecycleService; + + /// Creates a finalization service over the capture snapshot, the reflector, the record store, and Core's lifecycle owner. + /// Where the completed run's sanitized snapshot is read from. + /// Turns the evaluated run into an auditable reflection. + /// The durable Experience Record store. + /// Core's lifecycle owner, which stamps and commits the initial event. + /// Any argument is . + public ExperienceFinalizationService( + IExperienceCaptureService captureService, + IExperienceReflector reflector, + IExperienceRecordStore store, + ExperienceLifecycleService lifecycleService) + { + ArgumentNullException.ThrowIfNull(captureService); + ArgumentNullException.ThrowIfNull(reflector); + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(lifecycleService); + + _captureService = captureService; + _reflector = reflector; + _store = store; + _lifecycleService = lifecycleService; + } + + /// The finalizing issues, derived from the run so a retry re-derives the same ID. + /// The captured run. + public static Guid ExperienceIdFor(Guid runId) => Derive(runId, ExperienceIdTag); + + /// The of the record's initial event, derived from the run so a retry cannot commit a second initial confirmation. + /// The captured run. + public static Guid InitialEventIdFor(Guid runId) => Derive(runId, InitialEventIdTag); + + /// The finalizing asks the reflector to stamp, derived from the run so a retry reflects under the same identity. + /// The captured run. + public static Guid ReflectionIdFor(Guid runId) => Derive(runId, ReflectionIdTag); + + /// + /// Finalizes one captured run, running every stage in order and stopping at the first one that + /// ends the call. + /// + /// The run to finalize, its verification inputs, the host authorization, and the host's storage decision. + /// Cancels the operation. Cancellation is not an expected condition and propagates. + /// A structured result naming the stage finalization ended at. + /// , or its , , , or , is . + /// is , is blank, or is unset. + /// was cancelled, or a port cancelled. + public async Task FinalizeAsync( + FinalizeExperienceRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(request.Authorization, $"{nameof(request)}.{nameof(request.Authorization)}"); + ArgumentNullException.ThrowIfNull(request.RequiredChecks, $"{nameof(request)}.{nameof(request.RequiredChecks)}"); + ArgumentNullException.ThrowIfNull(request.Evidence, $"{nameof(request)}.{nameof(request.Evidence)}"); + ArgumentNullException.ThrowIfNull(request.StorageDecision, $"{nameof(request)}.{nameof(request.StorageDecision)}"); + if (request.RunId == Guid.Empty) + { + throw new ArgumentException("RunId must not be an empty GUID.", nameof(request)); + } + + if (string.IsNullOrWhiteSpace(request.CurrentArtifactRevision)) + { + throw new ArgumentException("CurrentArtifactRevision must not be null, empty, or whitespace.", nameof(request)); + } + + if (request.FinalizedAt == default) + { + // An unset timestamp would be stamped onto the record and its initial event, and the store + // rejects an unset OccurredAt as Invalid -- so the record would be created and then every + // commit, including every retry, would be refused forever. + throw new ArgumentException("FinalizedAt must be set to when the host decided to finalize this run.", nameof(request)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + // Stage 1 -- Load. An unknown or unfinished run is an expected condition, not an exception. + ExperienceRun? run; + try + { + _ = _captureService.TryGetRun(request.RunId, out run); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return PortFailed(FinalizationStage.Load, ex, evaluation: null, record: null, reflection: null); + } + + if (run is null) + { + return Ended( + FinalizationOutcome.RunNotFound, + FinalizationStage.Load, + "No captured run exists for the requested run ID."); + } + + if (run.ExecutionStatus is null) + { + return Ended( + FinalizationOutcome.RunNotFinished, + FinalizationStage.Load, + "The captured run has no execution status, so it has not finished and cannot be finalized."); + } + + // The store keeps whole microseconds, so the record's CreatedAt -- which is also the initial + // event's OccurredAt -- is truncated here rather than by the database. That keeps a replay's + // re-derived event byte-for-byte identical to the stored one. + var finalizedAt = TruncateToMicroseconds(request.FinalizedAt); + + // Stage 2 -- Evaluate, against this run's own closed round and evidence. No caller-supplied + // evaluation is accepted, so an evaluation from another run cannot be substituted. + VerificationResult evaluation; + try + { + evaluation = VerificationAggregator.Aggregate( + request.Evidence, + request.RequiredChecks, + request.ClosedRound, + request.CurrentArtifactRevision, + finalizedAt, + cancellationToken); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return Ended( + FinalizationOutcome.Failed, + FinalizationStage.Evaluate, + "The run could not be evaluated.", + new FinalizationFailure( + FinalizationStage.Evaluate, + $"Aggregating verification threw {ex.GetType().FullName}; the verification inputs are malformed.", + NoErrors, + ex), + evaluation: null); + } + + // Stage 3 -- Authorize, then read the host's storage decision. Both are decided before any + // store call *and before the reflector is called*, so a refused run is never handed to the + // model-backed reflection seam and nothing at all is written. + if (!request.Authorization.Permits(run.Scope)) + { + return Ended( + FinalizationOutcome.NotAuthorized, + FinalizationStage.Authorize, + "The captured run's scope lies outside the host-established authorization; nothing was stored and the run was not reflected on.", + evaluation: evaluation); + } + + if (!request.StorageDecision.Permitted) + { + return Ended( + FinalizationOutcome.StorageDenied, + FinalizationStage.Authorize, + request.StorageDecision.Reason ?? "The host's storage decision did not permit persisting this run; nothing was stored and the run was not reflected on.", + evaluation: evaluation); + } + + // Stage 4 -- Reflect, but only on a verified run: a quarantined record must never carry an + // unreflected lesson, so an unverified run is not reflected on at all. + Reflection? reflection = null; + FinalizationFailure? failure = null; + + if (evaluation.Outcome.Status == TaskVerificationStatus.Verified) + { + try + { + reflection = await _reflector + .ReflectAsync( + new ReflectionRequest(run, evaluation, ReflectionIdFor(run.RunId), finalizedAt), + cancellationToken) + .ConfigureAwait(false); + + if (reflection is null) + { + failure = new FinalizationFailure( + FinalizationStage.Reflect, + "The reflector returned no reflection; the record is quarantined without an eligible lesson.", + NoErrors, + Exception: null); + } + } + catch (OperationCanceledException) + { + // Cancellation is never quietly turned into a quarantine, whoever cancelled and why. + throw; + } + catch (Exception ex) + { + // A reflector failure does not end finalization: the run is still worth keeping, just + // not as a validated lesson. The exception is caught, never rethrown. + reflection = null; + failure = new FinalizationFailure( + FinalizationStage.Reflect, + $"The reflector threw {ex.GetType().FullName}; the record is quarantined without an eligible lesson.", + NoErrors, + ex); + } + } + else + { + failure = new FinalizationFailure( + FinalizationStage.Evaluate, + $"Verification resolved to {evaluation.Outcome.Status} rather than {TaskVerificationStatus.Verified}; the record is quarantined without an eligible lesson.", + NoErrors, + Exception: null); + } + + // Stage 5 -- Create the record, as a Candidate. Attempts are copied unchanged: capture already + // rejected anything unsafe, and finalization never sanitizes. + var record = new ExperienceRecord( + ExperienceId: ExperienceIdFor(run.RunId), + SourceRunId: run.RunId, + Scope: run.Scope, + TaskId: run.TaskId, + TaskSummary: run.TaskDescription, + Attempts: run.Attempts, + Outcome: evaluation.Outcome, + CompletionScore: evaluation.CompletionScore, + Reflection: reflection, + Environment: run.Environment, + Provenance: run.Provenance, + Status: CreatedStatus, + ReuseConfidence: reflection is not null ? InitialValidatedReuseConfidence : 0d, + SupportingValidations: reflection is not null ? 1 : 0, + Contradictions: 0, + Revision: 0, + CreatedAt: finalizedAt, + UpdatedAt: finalizedAt); + + ExperienceRecordCreateResult created; + try + { + created = await _store.CreateAsync(request.Authorization, record, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return PortFailed(FinalizationStage.CreateRecord, ex, evaluation, record: null, reflection); + } + + switch (created.Outcome) + { + case ExperienceStoreOutcome.Created: + break; + + case ExperienceStoreOutcome.Conflict: + // This run has been finalized before (the ID is derived from it). Report the stored + // outcome rather than writing anything a second time. + return await ReplayAsync(request, run, record, evaluation, cancellationToken).ConfigureAwait(false); + + case ExperienceStoreOutcome.Denied: + return Ended( + FinalizationOutcome.NotAuthorized, + FinalizationStage.CreateRecord, + "The store refused the record's scope as outside the host-established authorization; nothing was stored.", + evaluation: evaluation); + + case ExperienceStoreOutcome.Invalid: + return Ended( + FinalizationOutcome.Failed, + FinalizationStage.CreateRecord, + "The store rejected the Experience Record as malformed; nothing was stored.", + new FinalizationFailure( + FinalizationStage.CreateRecord, + "The store reported the Experience Record invalid. See the validation errors.", + created.Errors, + Exception: null), + evaluation); + + default: + return Ended( + FinalizationOutcome.Failed, + FinalizationStage.CreateRecord, + $"The store returned '{created.Outcome}', which is not a create outcome.", + new FinalizationFailure( + FinalizationStage.CreateRecord, + $"The Experience Record store returned '{created.Outcome}' from CreateAsync.", + created.Errors, + Exception: null), + evaluation); + } + + // Stage 6 -- Commit the record's initial lifecycle event, which performs the real transition. + return await CommitInitialEventAsync(request, run, record, evaluation, failure, cancellationToken).ConfigureAwait(false); + } + + /// + /// Handles a create that conflicted because this run was already finalized. A record still at + /// revision 0 had its create land but not its initial commit, so the commit is finished here; + /// anything past revision 0 is fully finalized and is reported as it stands. + /// + private async Task ReplayAsync( + FinalizeExperienceRequest request, + ExperienceRun run, + ExperienceRecord attempted, + VerificationResult evaluation, + CancellationToken cancellationToken) + { + ExperienceRecordGetResult stored; + try + { + stored = await _store + .GetAsync(request.Authorization, run.Scope, attempted.ExperienceId, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return PortFailed(FinalizationStage.CreateRecord, ex, evaluation, record: null, attempted.Reflection); + } + + if (stored.Outcome != ExperienceStoreOutcome.Found || stored.Record is null) + { + // The derived ID is taken by a record this caller's scope cannot see. Nothing was written, + // and nothing about the other record is revealed. + return Ended( + FinalizationOutcome.Failed, + FinalizationStage.CreateRecord, + "An Experience Record with this run's derived ID already exists outside the requested scope; nothing was stored.", + new FinalizationFailure( + FinalizationStage.CreateRecord, + $"CreateAsync conflicted and the stored record is not readable in this scope ({stored.Outcome}).", + stored.Errors, + Exception: null), + evaluation); + } + + if (stored.Record.Revision > 0) + { + return AlreadyFinalized(stored.Record, evaluation); + } + + // The earlier call created the record but never confirmed it. Finish that same commit, from + // the stored record, so the event stays byte-for-byte what the first attempt would have sent. + // The failure is reconstructed from the stored record, so a quarantined resume still names the + // stage that decided it rather than reporting a reason-less quarantine. + return await CommitInitialEventAsync( + request, + run, + stored.Record, + evaluation, + FailureFor(stored.Record), + cancellationToken).ConfigureAwait(false); + } + + private async Task CommitInitialEventAsync( + FinalizeExperienceRequest request, + ExperienceRun run, + ExperienceRecord record, + VerificationResult evaluation, + FinalizationFailure? failure, + CancellationToken cancellationToken) + { + // Every field is a pure function of the stored record and the run, so a retry re-derives the + // identical event and the store deduplicates it instead of appending a second one. This is a + // real transition out of Candidate, so it goes through Core's transition table and the store's + // prior-status guard, not a null-prior self-transition. + var targetStatus = TargetStatusFor(record); + var transition = new CommitLifecycleTransitionRequest( + EventId: InitialEventIdFor(run.RunId), + ExperienceId: record.ExperienceId, + Scope: record.Scope, + PriorStatus: CreatedStatus, + CurrentStatus: targetStatus, + Reason: InitialEventReason(record), + Producer: ProducerIdentity, + OccurredAt: record.CreatedAt, + ExpectedRevision: 0); + + CommitLifecycleTransitionResult commit; + try + { + commit = await _lifecycleService.CommitAsync(request.Authorization, transition, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return PortFailed(FinalizationStage.CommitInitialEvent, ex, evaluation, record, record.Reflection); + } + + if (commit.Outcome != LifecycleTransitionOutcome.Committed) + { + // Someone else may have finalized this record between the read above and this commit. A + // moved revision is not a failure to report forever: re-read and converge on their result. + if (commit.Outcome is LifecycleTransitionOutcome.StaleRevision or LifecycleTransitionOutcome.Conflict + && await TryReadFinalizedAsync(request, record, cancellationToken).ConfigureAwait(false) is { } finalized) + { + return AlreadyFinalized(finalized, evaluation); + } + + // The record exists and is still a Candidate. Report it, and why it was going to be + // quarantined, so the host can reconcile rather than guess. + return new FinalizeExperienceResult( + FinalizationOutcome.Failed, + FinalizationStage.CommitInitialEvent, + record, + commit.Event, + record.Revision, + evaluation, + record.Reflection, + failure ?? new FinalizationFailure( + FinalizationStage.CommitInitialEvent, + $"Committing the record's initial lifecycle event returned {commit.Outcome}.{(commit.Reason is null ? string.Empty : " " + commit.Reason)}", + commit.Errors, + Exception: null), + $"The Experience Record's initial lifecycle event returned {commit.Outcome}, so the record is still a {record.Status} and finalization is not durable; the captured run is still available for a retry."); + } + + // Mirror the projection the store just applied, so the returned record is the record as it now + // stands rather than the pre-transition Candidate. + var committed = record with + { + Status = targetStatus, + Revision = commit.Revision, + UpdatedAt = transition.OccurredAt, + }; + + return new FinalizeExperienceResult( + targetStatus == ExperienceStatus.Validated ? FinalizationOutcome.Validated : FinalizationOutcome.Quarantined, + FinalizationStage.CommitInitialEvent, + committed, + commit.Event, + commit.Revision, + evaluation, + committed.Reflection, + failure, + Reason: null); + } + + /// + /// Re-reads the record after a commit the store refused, returning it only when its revision has + /// moved past 0 -- that is, when someone else committed the initial event first. A read that fails + /// or still shows revision 0 returns , and the caller reports the original + /// commit outcome. + /// + private async Task TryReadFinalizedAsync( + FinalizeExperienceRequest request, + ExperienceRecord record, + CancellationToken cancellationToken) + { + try + { + var reread = await _store + .GetAsync(request.Authorization, record.Scope, record.ExperienceId, cancellationToken) + .ConfigureAwait(false); + + return reread is { Outcome: ExperienceStoreOutcome.Found, Record.Revision: > 0 } ? reread.Record : null; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception) + { + // Reconciliation is best-effort; the caller still reports the commit outcome it saw. + return null; + } + } + + private static FinalizeExperienceResult AlreadyFinalized(ExperienceRecord stored, VerificationResult evaluation) => new( + FinalizationOutcome.AlreadyFinalized, + FinalizationStage.CommitInitialEvent, + stored, + Event: null, + stored.Revision, + evaluation, + stored.Reflection, + FailureFor(stored), + $"This run was already finalized as {stored.Status}; no second record and no second initial event were written."); + + /// + /// The status a created record's initial event moves it to: + /// only when the record carries a reflection, which it does only for a verified run whose + /// reflection succeeded. + /// + private static ExperienceStatus TargetStatusFor(ExperienceRecord record) => + record.Reflection is not null ? ExperienceStatus.Validated : ExperienceStatus.Quarantined; + + /// + /// Reconstructs the safe failure metadata for a record this call did not itself build -- a stored + /// record found by a replay. A quarantine always names the stage that decided it. + /// + private static FinalizationFailure? FailureFor(ExperienceRecord record) + { + if (record.Reflection is not null) + { + return null; + } + + return record.Outcome.Status == TaskVerificationStatus.Verified + ? new FinalizationFailure( + FinalizationStage.Reflect, + "The stored record carries no reflection although its verification passed; it is quarantined without an eligible lesson.", + NoErrors, + Exception: null) + : new FinalizationFailure( + FinalizationStage.Evaluate, + $"Verification resolved to {record.Outcome.Status} rather than {TaskVerificationStatus.Verified}; the record is quarantined without an eligible lesson.", + NoErrors, + Exception: null); + } + + /// + /// The initial event's auditable reason, derived only from the record so a retry -- which reads + /// the stored record rather than recomputing -- produces exactly the same text. + /// + private static string InitialEventReason(ExperienceRecord record) => string.Format( + CultureInfo.InvariantCulture, + "Initial finalization: verification {0}, completion score {1}, {2}.", + record.Outcome.Status, + record.CompletionScore.ToString("R", CultureInfo.InvariantCulture), + record.Reflection is null ? "no eligible lesson recorded" : "reflection recorded"); + + /// + /// Turns any non-cancellation exception a port threw into a structured failed stage, so "every + /// stage failure comes back as a structured result" holds for more than + /// . + /// + private static FinalizeExperienceResult PortFailed( + FinalizationStage stage, + Exception exception, + VerificationResult? evaluation, + ExperienceRecord? record, + Reflection? reflection) => new( + FinalizationOutcome.Failed, + stage, + record, + Event: null, + record?.Revision ?? 0, + evaluation, + reflection, + new FinalizationFailure( + stage, + $"The {stage} stage's port threw {exception.GetType().FullName}.", + NoErrors, + exception), + "A port failed, so finalization is not durable; the captured run is still available for a retry."); + + private static FinalizeExperienceResult Ended( + FinalizationOutcome outcome, + FinalizationStage stage, + string reason, + FinalizationFailure? failure = null, + VerificationResult? evaluation = null) => new( + outcome, + stage, + Record: null, + Event: null, + Revision: 0, + evaluation, + Reflection: null, + failure, + reason); + + /// + /// Truncates to whole microseconds in UTC, which is the precision PostgreSQL's timestamptz + /// keeps. Without this, a value read back from the store would differ from the one sent, and a + /// replay's re-derived lifecycle event would no longer be identical to the stored one. + /// + private static DateTimeOffset TruncateToMicroseconds(DateTimeOffset value) + { + var utc = value.UtcDateTime; + return new DateTimeOffset(utc.Ticks - (utc.Ticks % TimeSpan.TicksPerMicrosecond), TimeSpan.Zero); + } + + /// + /// Derives a stable identifier from a run ID and a per-purpose tag: SHA-256 over a fixed + /// namespace, the run ID, and the tag, stamped with the RFC 9562 custom version (8) and variant. + /// Same run in, same identifiers out -- which is what makes replaying finalization safe. + /// + private static Guid Derive(Guid runId, byte tag) + { + Span input = stackalloc byte[33]; + DerivationNamespace.TryWriteBytes(input[..16], bigEndian: true, out _); + runId.TryWriteBytes(input.Slice(16, 16), bigEndian: true, out _); + input[32] = tag; + + Span hash = stackalloc byte[32]; + SHA256.HashData(input, hash); + + var id = hash[..16]; + id[6] = (byte)((id[6] & 0x0F) | 0x80); + id[8] = (byte)((id[8] & 0x3F) | 0x80); + return new Guid(id, bigEndian: true); + } +} diff --git a/src/AgentExperience.Core/Finalization/FinalizationResults.cs b/src/AgentExperience.Core/Finalization/FinalizationResults.cs new file mode 100644 index 0000000..6268d76 --- /dev/null +++ b/src/AgentExperience.Core/Finalization/FinalizationResults.cs @@ -0,0 +1,150 @@ +using AgentExperience.Abstractions; +using AgentExperience.Core.Verification; + +namespace AgentExperience.Core.Finalization; + +/// +/// The ordered stages of . Every result +/// names the stage it ended at, so a host always knows how far finalization got. +/// +public enum FinalizationStage +{ + /// Reading the captured run's snapshot back from the capture service. + Load, + + /// Computing the run's verification result from the host-closed round and its evidence. + Evaluate, + + /// Checking the run's scope against the host authorization, then the host's storage decision. No store call has been made yet, and the reflector has not been called. + Authorize, + + /// Reflecting on the evaluated run. Skipped entirely when verification did not pass, or when either gate above refused. + Reflect, + + /// Creating the Experience Record through the store port. + CreateRecord, + + /// Committing the record's initial lifecycle event, atomically with its projection. + CommitInitialEvent, +} + +/// +/// The disposition one call reached. +/// +public enum FinalizationOutcome +{ + /// + /// The run verified, reflection succeeded, and the host permitted storage: the record was created + /// as and its initial lifecycle event moved it to + /// . + /// + Validated, + + /// + /// The host permitted storage but the run did not verify, or reflection failed: the record was + /// created as , carrying no reflection, and its initial + /// lifecycle event moved it to . Failure names why. + /// + Quarantined, + + /// + /// This run had already been finalized, by an earlier call or by a concurrent one. Nothing was + /// written: no second record and no second initial event. Record and Revision report + /// the stored outcome, Status is the status that call produced, and Failure names why + /// it was quarantined when it was. + /// + AlreadyFinalized, + + /// + /// The host's did not permit storage. Nothing was written, no + /// record ID was issued, and Reason carries the host's own content-free reason. + /// + StorageDenied, + + /// + /// The run's scope lies outside the host-established . Denied + /// before any store call; nothing was written. + /// + NotAuthorized, + + /// No captured run exists for the requested run ID. Nothing was written. + RunNotFound, + + /// + /// The run exists but has no , so it has not finished + /// and cannot be finalized. Nothing was written. + /// + RunNotFinished, + + /// + /// A stage failed. Stage and Failure name which and why. This is never a durable + /// success: the captured run stays available so the host can retry finalization. When the record + /// had already been created but its initial event was refused, Record carries that record -- + /// still a , so it is not reusable -- and Failure + /// carries either the commit's own refusal or the reason the record was going to be quarantined. + /// + Failed, +} + +/// +/// Why a finalization stage could not produce what it was asked for. Present on +/// , and also on +/// , where it is the safe failure metadata explaining +/// why the record carries no eligible lesson. +/// +/// The stage the failure belongs to. On a quarantined record this is the stage that decided it could not be validated, which is not necessarily the stage the call ended at. +/// A content-free, auditable explanation. Never echoes captured content, record payload, or private reasoning. +/// The store's validation errors when a store call reported the request malformed; otherwise empty. +/// The exception behind the failure, if any. Handed to the host for diagnostics only -- never persisted, and never recorded into the Experience Record. +public sealed record FinalizationFailure( + FinalizationStage Stage, + string Reason, + IReadOnlyList Errors, + Exception? Exception); + +/// +/// The result of one call. +/// +/// What happened. +/// The stage finalization ended at. +/// +/// The Experience Record, when one exists: the record this call created (with the projection its +/// initial event applied, when that committed), or the stored record a replay found. +/// whenever nothing was persisted. On +/// after a refused initial commit it is the created record, +/// still a at revision 0. +/// +/// The initial lifecycle event this call committed, or when none was committed by this call. +/// The record's revision after the initial event was committed (1), or the stored revision reported by a replay; 0 when the record is still an uncommitted or nothing was written. +/// The verification result finalization computed for this run, once the evaluate stage ran; otherwise . +/// The reflection stored on the record, or -- always for a quarantined record. +/// Why the call failed, or why a record was (or was going to be) quarantined; otherwise . +/// Optional, auditable, content-free explanation of the outcome. +public sealed record FinalizeExperienceResult( + FinalizationOutcome Outcome, + FinalizationStage Stage, + ExperienceRecord? Record, + LifecycleEvent? Event, + long Revision, + VerificationResult? Evaluation, + Reflection? Reflection, + FinalizationFailure? Failure, + string? Reason) +{ + /// The Experience Record's ID, when one exists. No ID is issued when nothing was persisted. + public Guid? ExperienceId => Record?.ExperienceId; + + /// The record's lifecycle status, when one exists. + public ExperienceStatus? Status => Record?.Status; + + /// + /// Whether an Experience Record for this run is durably stored and confirmed by its initial + /// lifecycle event -- true only for , + /// , and + /// . A database failure is never reported as + /// durable success. + /// + public bool IsDurable => Outcome is FinalizationOutcome.Validated + or FinalizationOutcome.Quarantined + or FinalizationOutcome.AlreadyFinalized; +} diff --git a/src/AgentExperience.Core/Finalization/FinalizeExperienceRequest.cs b/src/AgentExperience.Core/Finalization/FinalizeExperienceRequest.cs new file mode 100644 index 0000000..3c1da2d --- /dev/null +++ b/src/AgentExperience.Core/Finalization/FinalizeExperienceRequest.cs @@ -0,0 +1,40 @@ +using AgentExperience.Abstractions; +using AgentExperience.Core.Verification; + +namespace AgentExperience.Core.Finalization; + +/// +/// One request to turn a captured, completed run into a durable Experience Record, submitted to +/// . +/// +/// +/// +/// No evaluation is accepted here. The request carries the host-closed round, the artifact revision, +/// the declared required checks, and the evidence; finalization runs +/// itself against exactly those, so an evaluation +/// computed for some other run can never be substituted for this one's. +/// +/// +/// is caller-supplied rather than read from a clock so a retry of the same +/// run is the same request. The record ID, the reflection ID, and the initial lifecycle event ID are +/// all derived from , so a retry can never produce a second record or a second +/// initial confirmation (see ). +/// +/// +/// The captured run to finalize. Must not be ; the run must exist in the capture service and must already carry an . +/// What the host has established the caller may do. The run's own must lie within it or nothing is stored. +/// The verification round the host closed for this run, or when the host has closed none (which evaluates to ). +/// The task's declared required checks. An empty set can never verify. +/// All evidence available for this run, in the order it was produced. Aggregation filters it to and itself. +/// The artifact revision verification is being judged against. Must be non-blank. +/// The host's storage-policy decision. A decision that does not permit storage writes nothing at all. +/// When the host decided to finalize this run. Stamped onto the record and its initial lifecycle event (truncated to whole microseconds in UTC, which is the precision the store keeps). +public sealed record FinalizeExperienceRequest( + Guid RunId, + AuthorizationContext Authorization, + ClosedVerificationRound? ClosedRound, + IReadOnlyList RequiredChecks, + IReadOnlyList Evidence, + string CurrentArtifactRevision, + StorageDecision StorageDecision, + DateTimeOffset FinalizedAt); diff --git a/src/AgentExperience.Core/Finalization/StorageDecision.cs b/src/AgentExperience.Core/Finalization/StorageDecision.cs new file mode 100644 index 0000000..279bb57 --- /dev/null +++ b/src/AgentExperience.Core/Finalization/StorageDecision.cs @@ -0,0 +1,30 @@ +namespace AgentExperience.Core.Finalization; + +/// +/// The host's decision on whether a finalized run may be persisted at all, carried in a +/// . +/// +/// +/// +/// Storage policy belongs to the host, not to this library: only the host knows its retention rules, +/// its data-residency obligations, and its own risk appetite. Core therefore calls no policy port and +/// makes no policy decision of its own -- it reads this value and obeys it. A decision that does not +/// permit storage stops finalization before any store call, so nothing at all is written, whatever +/// the run's verification says. +/// +/// +/// is content-free: it is surfaced back to the host on the finalization result +/// and must never carry record payload, captured content, or private reasoning. +/// +/// +/// when the host permits this run to be persisted as an Experience Record. +/// Optional, auditable, content-free explanation of the decision (most usefully, why storage was denied). +public sealed record StorageDecision(bool Permitted, string? Reason = null) +{ + /// A decision that permits storage, with no reason attached. + public static StorageDecision Permit { get; } = new(Permitted: true); + + /// Creates a decision that denies storage. + /// A content-free explanation of why storage was denied. + public static StorageDecision Deny(string? reason = null) => new(Permitted: false, reason); +} diff --git a/src/AgentExperience.Core/Verification/RequiredCheck.cs b/src/AgentExperience.Core/Verification/RequiredCheck.cs new file mode 100644 index 0000000..ab2f6cb --- /dev/null +++ b/src/AgentExperience.Core/Verification/RequiredCheck.cs @@ -0,0 +1,41 @@ +namespace AgentExperience.Core.Verification; + +/// +/// One check a task declares as required for verification, and (optionally) the kind of evaluator +/// that is allowed to satisfy it. +/// +/// +/// +/// Matching a required check by alone lets any producer claim any check: a +/// human approval, for instance, could satisfy a check the task meant to be answered by a test run. +/// closes that gap. When it is non-, +/// counts a piece of +/// for this check only when the evidence's own +/// equals it (ordinal, case-sensitive); +/// evidence of any other kind is ignored entirely, exactly as if it had been recorded for a +/// different . A check whose evidence is all ignored has no evidence at all and +/// therefore resolves to -- a +/// mismatched evaluator can never turn a check into a pass. +/// +/// +/// A accepts evidence of any kind, which is the +/// behaviour this type replaced (checks were previously declared as bare CheckId strings). +/// +/// +/// The task-declared required check ID. Must be non-blank and unique within one aggregation. +/// +/// The that may satisfy this check (e.g. +/// "TestResult", "ToolExitCode", "HumanApproval"), or to +/// accept any kind. When supplied it must be non-blank. +/// +public sealed record RequiredCheck(string CheckId, string? ExpectedKind = null) +{ + /// + /// Whether is allowed to satisfy this check: always + /// when no is named, otherwise an exact + /// ordinal match. + /// + /// The candidate evidence's own Kind. + public bool Accepts(string? evidenceKind) => + ExpectedKind is null || string.Equals(ExpectedKind, evidenceKind, StringComparison.Ordinal); +} diff --git a/src/AgentExperience.Core/Verification/VerificationAggregator.cs b/src/AgentExperience.Core/Verification/VerificationAggregator.cs index ee1d535..ab95502 100644 --- a/src/AgentExperience.Core/Verification/VerificationAggregator.cs +++ b/src/AgentExperience.Core/Verification/VerificationAggregator.cs @@ -25,7 +25,9 @@ namespace AgentExperience.Core.Verification; /// /// /// Per-check resolution (the two AC4 conflict clauses, reconciled -- see this story's Design -/// Notes): for each required CheckId, gather only the selected evidence carrying it. No +/// Notes): for each , gather only the selected evidence carrying its +/// and a its +/// accepts. No /// evidence at all is a missing check (); any /// among it makes the check -- /// dominating even a recorded for the same check in the same @@ -65,26 +67,26 @@ public static class VerificationAggregator public const string RuleVersion = "1.0.0"; /// - /// Aggregates against , reading + /// Aggregates against , reading /// only the evidence in for /// -- see this type's remarks for the full selection, per-check, and overall-verdict rules. /// /// All evidence available to consider, in the order it was produced. Never filtered or reordered by the caller; this call does that filtering itself. A entry is a caller error and throws. - /// The task's declared required check IDs, which must be unique (a duplicate is a caller error and throws, rather than silently skewing the completion score). An empty set always yields . + /// The task's declared required checks, whose s must be unique (a duplicate is a caller error and throws, rather than silently skewing the completion score) and non-blank. A entry is a caller error and throws. An empty set always yields . /// The host-closed verification round and artifact revision to read from, or if the host has not closed a round yet. Never agent-suppliable -- only a host establishes this. /// The artifact's current revision. If it does not match 's own revision, verification is stale. /// When this aggregation is being performed. /// Checked cooperatively; a cancelled call throws rather than returning any . public static VerificationResult Aggregate( IReadOnlyList evidence, - IReadOnlyList requiredCheckIds, + IReadOnlyList requiredChecks, ClosedVerificationRound? closedRound, string currentArtifactRevision, DateTimeOffset evaluatedAt, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(evidence); - ArgumentNullException.ThrowIfNull(requiredCheckIds); + ArgumentNullException.ThrowIfNull(requiredChecks); ArgumentException.ThrowIfNullOrWhiteSpace(currentArtifactRevision); // Invalid input throws -- never silently dropped or tolerated into a fabricated result, per @@ -94,15 +96,30 @@ public static VerificationResult Aggregate( throw new ArgumentException("Evidence must not contain null entries.", nameof(evidence)); } - if (requiredCheckIds.Distinct(StringComparer.Ordinal).Count() != requiredCheckIds.Count) + if (requiredChecks.Any(c => c is null)) { - throw new ArgumentException("Required check IDs must be unique.", nameof(requiredCheckIds)); + throw new ArgumentException("Required checks must not contain null entries.", nameof(requiredChecks)); + } + + if (requiredChecks.Any(c => string.IsNullOrWhiteSpace(c.CheckId))) + { + throw new ArgumentException("Required check IDs must not be null, empty, or whitespace.", nameof(requiredChecks)); + } + + if (requiredChecks.Any(c => c.ExpectedKind is not null && string.IsNullOrWhiteSpace(c.ExpectedKind))) + { + throw new ArgumentException("A required check's ExpectedKind must be null or non-blank.", nameof(requiredChecks)); + } + + if (requiredChecks.Select(c => c.CheckId).Distinct(StringComparer.Ordinal).Count() != requiredChecks.Count) + { + throw new ArgumentException("Required check IDs must be unique.", nameof(requiredChecks)); } cancellationToken.ThrowIfCancellationRequested(); // Stale/unclosed short-circuit -- no round or revision selection from evidence or - // requiredCheckIds themselves is ever consulted here; only the host-supplied closedRound + // requiredChecks themselves is ever consulted here; only the host-supplied closedRound // decides. Nothing is examined further. if (closedRound is null) { @@ -116,7 +133,7 @@ public static VerificationResult Aggregate( evaluatedAt); } - if (requiredCheckIds.Count == 0) + if (requiredChecks.Count == 0) { return UnknownResult("No required checks were declared for this task; an empty required set can never be conclusively verified.", evaluatedAt); } @@ -132,11 +149,17 @@ public static VerificationResult Aggregate( var failedCheckIds = new List(); var unknownCheckIds = new List(); - foreach (var checkId in requiredCheckIds) + foreach (var requiredCheck in requiredChecks) { cancellationToken.ThrowIfCancellationRequested(); - var checkEvidence = selectedEvidence.Where(e => string.Equals(e.CheckId, checkId, StringComparison.Ordinal)).ToList(); + // A named ExpectedKind narrows the evidence for this check: evidence of any other kind is + // ignored outright, so a mismatched evaluator can never satisfy the check (it becomes a + // check with no evidence, i.e. Unknown). + var checkEvidence = selectedEvidence + .Where(e => string.Equals(e.CheckId, requiredCheck.CheckId, StringComparison.Ordinal) && requiredCheck.Accepts(e.Kind)) + .ToList(); + foreach (var e in checkEvidence) { contributingEvidenceIds.Add(e.EvidenceId); @@ -145,11 +168,11 @@ public static VerificationResult Aggregate( switch (ResolveCheck(checkEvidence)) { case CheckResult.Fail: - failedCheckIds.Add(checkId); + failedCheckIds.Add(requiredCheck.CheckId); break; case CheckResult.Unknown: - unknownCheckIds.Add(checkId); + unknownCheckIds.Add(requiredCheck.CheckId); break; case CheckResult.Pass: @@ -158,10 +181,10 @@ public static VerificationResult Aggregate( } } - var completionScore = (double)passingCheckCount / requiredCheckIds.Count; + var completionScore = (double)passingCheckCount / requiredChecks.Count; // The evidence backing the outcome, in the order it was produced (Outcome.Evidence's own - // contract) -- the original evidence list's own order, not the order requiredCheckIds + // contract) -- the original evidence list's own order, not the order requiredChecks // happened to name checks in. Drawn only from the already round/revision-scoped selection. var contributingEvidence = selectedEvidence.Where(e => contributingEvidenceIds.Contains(e.EvidenceId)).ToList(); diff --git a/src/AgentExperience.Core/packages.lock.json b/src/AgentExperience.Core/packages.lock.json index 4030d55..54ebb50 100644 --- a/src/AgentExperience.Core/packages.lock.json +++ b/src/AgentExperience.Core/packages.lock.json @@ -12,6 +12,12 @@ "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.11" } }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Direct", + "requested": "[10.0.11, 10.0.11]", + "resolved": "10.0.11", + "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" + }, "Microsoft.Extensions.Compliance.Abstractions": { "type": "Transitive", "resolved": "10.9.0", @@ -47,11 +53,6 @@ "Microsoft.Extensions.Configuration.Abstractions": "10.0.11" } }, - "Microsoft.Extensions.DependencyInjection.Abstractions": { - "type": "Transitive", - "resolved": "10.0.11", - "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" - }, "Microsoft.Extensions.ObjectPool": { "type": "Transitive", "resolved": "10.0.11", diff --git a/src/AgentExperience.MicrosoftAgentFramework/CaptureScope.cs b/src/AgentExperience.MicrosoftAgentFramework/CaptureScope.cs index 64a4eac..98d26df 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/CaptureScope.cs +++ b/src/AgentExperience.MicrosoftAgentFramework/CaptureScope.cs @@ -4,6 +4,7 @@ using System.Text.Json; using AgentExperience.Abstractions; using AgentExperience.Core.Capture; +using AgentExperience.Core.Finalization; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; @@ -210,7 +211,8 @@ internal void FailToolCall(PendingToolCall pending, Exception exception) => /// /// Finalizes the run exactly once: appends the invocation's single attempt (every buffered tool - /// call, ordered by start) and then completes the run. Bounded by + /// call, ordered by start), completes the run, and -- when the host configured one -- hands the + /// completed run to Core's finalization service to become a durable Experience Record. Bounded by /// with its own token, never the /// caller's. A second call is a no-op. Never throws. /// @@ -311,6 +313,104 @@ private async Task FinalizeCoreAsync(AppendAttemptRequest request, Guid completi if (problems.Count > 0) { ReportFailure(ExperienceCaptureFailureStage.Finalize, string.Join(" ", problems), firstException); + return; + } + + // Only a run whose attempt and completion both landed is worth turning into a durable record: + // finalizing a half-captured run would persist an incomplete history as if it were whole. + if (_options.FinalizationService is { } finalization && _options.ResolveFinalization is { } resolve) + { + await FinalizeExperienceAsync(finalization, resolve, cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Hands the completed run to Core's finalization service, through the host's own request + /// resolver. Like everything else on this type, nothing here throws into MAF or the caller: every + /// problem is reported to and swallowed, + /// and the captured run is left untouched so the host can retry finalization itself. + /// + private async Task FinalizeExperienceAsync( + ExperienceFinalizationService finalization, + Func resolve, + CancellationToken cancellationToken) + { + FinalizeExperienceRequest? request; + try + { + if (!_service.TryGetRun(RunId, out var run)) + { + ReportFailure(ExperienceCaptureFailureStage.Finalization, "The completed run could not be read back for finalization; it is not finalized.", null); + return; + } + + request = resolve(new ExperienceFinalizationContext(run)); + } + catch (Exception ex) + { + ReportFailure(ExperienceCaptureFailureStage.Finalization, $"ResolveFinalization threw {ex.GetType().FullName}; the run is not finalized.", ex); + return; + } + + // A null request is the host declining to finalize this particular run -- not a failure. + if (request is null) + { + return; + } + + // The resolver is host code and could hand back a request for some other captured run, which + // would finalize an unrelated run on this invocation's behalf. + if (request.RunId != RunId) + { + ReportFailure( + ExperienceCaptureFailureStage.Finalization, + "ResolveFinalization returned a request for a different run; the run is not finalized.", + null); + return; + } + + FinalizeExperienceResult result; + try + { + result = await finalization.FinalizeAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + ReportFailure(ExperienceCaptureFailureStage.Finalization, $"Finalizing the run threw {ex.GetType().FullName}.", ex); + return; + } + + if (result is null) + { + ReportFailure(ExperienceCaptureFailureStage.Finalization, "FinalizeAsync returned null.", null); + return; + } + + // Only a genuine defect goes to the failure channel. A host whose policy denies storage, or + // whose authorization refuses a scope, made that decision on purpose and should not get a + // failure callback per invocation -- OnRunFinalized already carries the whole result. + if (result.Outcome is FinalizationOutcome.Failed) + { + ReportFailure( + ExperienceCaptureFailureStage.Finalization, + $"Finalization ended at stage {result.Stage} with outcome {result.Outcome}; no Experience Record is durable for this run.", + result.Failure?.Exception); + } + + // Finalization may have overrun the timeout and already been reported as such; telling the + // host it finished after that would contradict the failure it already saw. + if (cancellationToken.IsCancellationRequested) + { + return; + } + + try + { + _options.OnRunFinalized?.Invoke(result); + } + catch + { + // The host's finalization callback must never affect the agent invocation. } } diff --git a/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureAgentBuilderExtensions.cs b/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureAgentBuilderExtensions.cs index cf67a3b..db5e5b8 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureAgentBuilderExtensions.cs +++ b/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureAgentBuilderExtensions.cs @@ -25,7 +25,11 @@ public static class ExperienceCaptureAgentBuilderExtensions /// executes and completes it exactly once on success, failure, cancellation, or a streaming /// consumer that stops reading early -- and then, when /// is , function - /// middleware that records each tool call into that run. + /// middleware that records each tool call into that run. When + /// and + /// are configured, each successfully + /// captured run is then handed to Core's finalization service to become a durable Experience + /// Record, inside the same timeout-bounded step. /// /// The agent builder. /// The capture service runs are recorded through. @@ -38,6 +42,7 @@ public static class ExperienceCaptureAgentBuilderExtensions /// (a ); throws otherwise. /// /// Any argument, or , , , or , is . + /// Exactly one of and is set. /// is not positive or exceeds the timer maximum. public static AIAgentBuilder UseExperienceCapture( this AIAgentBuilder builder, @@ -56,6 +61,16 @@ public static AIAgentBuilder UseExperienceCapture( throw new ArgumentOutOfRangeException(nameof(options), options.FinalizationTimeout, $"FinalizationTimeout must be positive and at most {uint.MaxValue - 1} milliseconds."); } + // Either half alone could only ever do nothing, silently -- and a resolver without a service is + // the easier mistake to make. Only the host can supply a run's required checks, evidence, + // authorization, and storage decision, so the two are configured together or not at all. + if (options.FinalizationService is null != (options.ResolveFinalization is null)) + { + throw new ArgumentException( + $"{nameof(ExperienceCaptureOptions.FinalizationService)} and {nameof(ExperienceCaptureOptions.ResolveFinalization)} must be set together, or neither set.", + nameof(options)); + } + var middleware = new ExperienceCaptureMiddleware(captureService, options); // The first Use call is the outermost layer: run middleware wraps function middleware. diff --git a/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureOptions.cs b/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureOptions.cs index d285f3e..afccc79 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureOptions.cs +++ b/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureOptions.cs @@ -1,5 +1,6 @@ using System.Runtime.InteropServices; using AgentExperience.Abstractions; +using AgentExperience.Core.Finalization; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; @@ -32,6 +33,16 @@ public sealed record ExperienceRunDescriptor( Scope Scope, string? TaskDescription = null); +/// +/// What the host sees when it is asked how a completed, captured run should be finalized into a +/// durable Experience Record. +/// +/// +/// The completed run's sanitized snapshot, read back from the capture service after the invocation's +/// attempt and completion were recorded. Its is set. +/// +public sealed record ExperienceFinalizationContext(ExperienceRun Run); + /// Where in the capture pipeline an happened. public enum ExperienceCaptureFailureStage { @@ -46,6 +57,15 @@ public enum ExperienceCaptureFailureStage /// Capturing a tool call's start, result, or error threw; that tool call is not recorded. ToolCall, + + /// + /// Finalizing the completed run into a durable Experience Record threw, was declined for a + /// foreign run ID, or failed a stage. A host decision (storage denied, or a scope outside the + /// authorization) is not reported here -- it is an expected outcome on + /// . The captured run is unchanged and still + /// available for the host to retry. + /// + Finalization, } /// @@ -89,9 +109,18 @@ public sealed class ExperienceCaptureOptions public bool CaptureToolCalls { get; init; } = true; /// - /// The upper bound on finalization (append attempt, then complete run) per run. Default 5 seconds. - /// Must be positive and at most - 1 milliseconds. Finalization never uses the caller's cancellation token. + /// The upper bound on the whole post-invocation step per run: append the attempt, complete the + /// run, and -- when is configured -- finalize it into a durable + /// Experience Record. Default 5 seconds. Must be positive and at most + /// - 1 milliseconds. It never uses the caller's cancellation token. /// + /// + /// With finalization configured this bound covers database round trips, not just in-memory + /// capture, so 5 seconds may be too tight for a slow or distant database. A timeout is reported + /// through and can leave the Experience Record created but not yet + /// confirmed -- a Candidate, which is never reusable. Finalizing that run again completes + /// the same commit. + /// public TimeSpan FinalizationTimeout { get; init; } = TimeSpan.FromSeconds(5); /// @@ -101,6 +130,40 @@ public sealed class ExperienceCaptureOptions /// public Action? OnCaptureFailure { get; init; } + /// + /// Optional. The Core service that turns each completed, captured run into a durable Experience + /// Record. Leave it to capture only -- the host can still finalize runs + /// itself, whenever it likes, from the capture service's snapshot. + /// + /// + /// Setting this requires too (and vice versa), because only the + /// host knows a run's required checks, its verification evidence, its authorization context, and + /// its storage policy. Finalization runs inside the same once-only, -bounded + /// step as capture finalization, after the run's attempt and completion were recorded, and only + /// when both of those succeeded. It never uses the caller's cancellation token and never changes + /// what the caller of the agent observes. + /// + public ExperienceFinalizationService? FinalizationService { get; init; } + + /// + /// Required when is set (and only valid then): builds the + /// finalize request for one completed run. Returning skips finalizing that + /// run. The request's RunId must be this invocation's run. If it throws, or returns a + /// request for another run, the run is not finalized and the failure is reported through + /// . + /// + public Func? ResolveFinalization { get; init; } + + /// + /// Optional. Receives every finalization result, durable or not -- including an expected + /// or + /// , which are host decisions rather than capture + /// failures and are therefore not reported through . Not called when + /// finalization already overran . Exceptions thrown by the + /// callback are swallowed. + /// + public Action? OnRunFinalized { get; init; } + /// The clock used for run, attempt, and tool-call timestamps, durations, and the finalization timeout. public TimeProvider TimeProvider { get; init; } = TimeProvider.System; diff --git a/src/AgentExperience.MicrosoftAgentFramework/README.md b/src/AgentExperience.MicrosoftAgentFramework/README.md index 56c4cc2..a0cdb57 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/README.md +++ b/src/AgentExperience.MicrosoftAgentFramework/README.md @@ -40,7 +40,8 @@ Call `UseExperienceCapture` first on the builder so capture is the outermost lay its result or exception, into the run for that invocation. Every recorded value goes through the `IExperienceCaptureService` you pass in, so its sanitizer and limits apply -unchanged. This adapter does not evaluate, reflect, or persist runs. +unchanged. This adapter never evaluates, reflects, or persists anything itself; when you configure finalization +(below) it hands the completed run to Core's `ExperienceFinalizationService`, which owns all of that. When the caller passes a session, the run ID is written to it under `ExperienceCaptureAgentBuilderExtensions.RunIdStateKey` (`"AgentExperience.RunId"`) as a `"D"`-formatted GUID @@ -54,11 +55,56 @@ because MAF forwards those to the model provider. | `ResolveRun` | required | Maps messages, session, and agent to a task ID, scope, and task description. If it throws or returns a null descriptor, task ID, or scope, the invocation runs uncaptured and the failure is reported. | | `Environment` | machine name + `RuntimeInformation` | Environment fingerprint recorded on every run. | | `CaptureToolCalls` | `true` | Registers function middleware. Requires a `ChatClientAgent`. | -| `FinalizationTimeout` | 5 s | Upper bound on appending the attempt and completing the run. Must be positive and at most `uint.MaxValue - 1` milliseconds. Finalization uses its own token, never the caller's. | -| `OnCaptureFailure` | none | Called when capture fails, at most once per failure stage (`ResolveRun`, `StartRun`, `Finalize`, `ToolCall`) per run. Exceptions it throws are swallowed. | +| `FinalizationTimeout` | 5 s | Upper bound on the whole post-invocation step: appending the attempt, completing the run, and — when `FinalizationService` is set — finalizing it into a durable Experience Record. With finalization configured this bounds database round trips, not just in-memory capture, so 5 s may be too tight. Must be positive and at most `uint.MaxValue - 1` milliseconds. It uses its own token, never the caller's. | +| `OnCaptureFailure` | none | Called when capture fails, at most once per failure stage per run. The stages are `ResolveRun`, `StartRun`, `Finalize` (recording the attempt and completing the run in memory), `ToolCall`, and `Finalization` (turning the completed run into a durable Experience Record). Exceptions it throws are swallowed. | +| `FinalizationService` | none | Core's `ExperienceFinalizationService`. When set, each successfully captured run is finalized into a durable Experience Record. Requires `ResolveFinalization`. | +| `ResolveFinalization` | none | Builds the `FinalizeExperienceRequest` for one completed run. Return `null` to skip finalizing that run. Required when `FinalizationService` is set. | +| `OnRunFinalized` | none | Receives every `FinalizeExperienceResult`, durable or not — including a host decision such as `StorageDenied`, which is not a capture failure. Not called once finalization has overrun `FinalizationTimeout`. Exceptions it throws are swallowed. | | `TimeProvider` | `TimeProvider.System` | Timestamps, durations, and the finalization timeout. | | `NewId` | `Guid.NewGuid` | Run, attempt, tool-call, and completion-event IDs. Must be thread-safe. | +## Finalizing captured runs + +Capture alone keeps the run in memory. To turn each invocation into a durable Experience Record, give the adapter +Core's finalization service and a resolver that supplies what only the host knows — the required checks, the +verification evidence and the round it was closed in, the authorization context, and the storage decision: + +```csharp +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Verification; + +var options = new ExperienceCaptureOptions +{ + ResolveRun = context => new ExperienceRunDescriptor("triage-ticket", hostScope), + + FinalizationService = finalization, // AgentExperience.Core.Finalization.ExperienceFinalizationService + ResolveFinalization = context => new FinalizeExperienceRequest( + RunId: context.Run.RunId, + Authorization: hostAuthorization, + ClosedRound: new ClosedVerificationRound(roundId, artifactRevision), + RequiredChecks: [new RequiredCheck("unit-tests-pass", ExpectedKind: "TestResult")], + Evidence: evidenceFor(context.Run), + CurrentArtifactRevision: artifactRevision, + StorageDecision: StorageDecision.Permit, + FinalizedAt: DateTimeOffset.UtcNow), + + OnRunFinalized = result => logger.LogInformation( + "Experience {Id} is {Status}", result.ExperienceId, result.Status), +}; +``` + +- **When it runs.** Immediately after the run's attempt and completion were both recorded, inside the same once-only + step and the same `FinalizationTimeout`. A run whose capture reported a problem is never finalized, so a + half-captured run is never persisted as if it were whole. +- **Opting out per run.** `ResolveFinalization` returning `null` skips that run, and is not a failure. +- **Failures.** A throwing resolver, a throwing `FinalizeAsync`, or a non-durable outcome (`StorageDenied`, + `NotAuthorized`, `Failed`, …) is reported through `OnCaptureFailure` with stage `Finalization` and never thrown. + The captured run is left untouched, so the host can retry finalization itself from the capture service. +- **Latency.** Finalization is a database round trip and is awaited inside `FinalizationTimeout`, so it adds + caller-visible latency. Leave `FinalizationService` unset and finalize out of band if that is not acceptable. +- **Setting `FinalizationService` without `ResolveFinalization` throws** at `UseExperienceCapture`, rather than + silently doing nothing. + ## Supported agent types | Agent | Run lifecycle | Tool calls | @@ -89,7 +135,9 @@ because MAF forwards those to the model provider. the agent would produce without capture. Capture does not wrap or re-execute tools, retry, or keep its own session store. - **Capture failures** are reported through `OnCaptureFailure` and never thrown. They include a resolver exception, a - non-success capture outcome, a capture exception, and a finalization timeout. Each failure stage is reported at most + non-success capture outcome, a capture exception, a finalization timeout, and a failed Experience Record + finalization. Note the two similarly named stages: `Finalize` is the in-memory capture step (append the attempt, + complete the run), while `Finalization` is turning that completed run into a durable Experience Record. Each failure stage is reported at most once per run, so an earlier tool-call or session-write failure never hides a later finalization failure. - **Finalization.** Completion is attempted even when appending the attempt fails. If finalization times out, the run may be left without an attempt or completion. diff --git a/src/AgentExperience.MicrosoftAgentFramework/packages.lock.json b/src/AgentExperience.MicrosoftAgentFramework/packages.lock.json index 89f118c..c850164 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/packages.lock.json +++ b/src/AgentExperience.MicrosoftAgentFramework/packages.lock.json @@ -187,7 +187,8 @@ "type": "Project", "dependencies": { "AgentExperience.Abstractions": "[1.0.0, )", - "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )" + "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]" } } } diff --git a/src/AgentExperience.Storage.Postgres/AgentExperience.Storage.Postgres.csproj b/src/AgentExperience.Storage.Postgres/AgentExperience.Storage.Postgres.csproj index f024a5d..0724e0f 100644 --- a/src/AgentExperience.Storage.Postgres/AgentExperience.Storage.Postgres.csproj +++ b/src/AgentExperience.Storage.Postgres/AgentExperience.Storage.Postgres.csproj @@ -15,6 +15,9 @@ in lockstep, to the dbup-core version the new dbup-postgresql depends on; DependencyBoundaryTests asserts this exact set of three PackageReferences. --> + + diff --git a/src/AgentExperience.Storage.Postgres/DependencyInjection/AgentExperiencePostgresServiceCollectionExtensions.cs b/src/AgentExperience.Storage.Postgres/DependencyInjection/AgentExperiencePostgresServiceCollectionExtensions.cs new file mode 100644 index 0000000..8463e56 --- /dev/null +++ b/src/AgentExperience.Storage.Postgres/DependencyInjection/AgentExperiencePostgresServiceCollectionExtensions.cs @@ -0,0 +1,57 @@ +using AgentExperience.Abstractions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Npgsql; + +namespace AgentExperience.Storage.Postgres.DependencyInjection; + +/// +/// Registers the PostgreSQL Experience Record store in a . The +/// adapter owns its own registration, exactly as Core owns AddAgentExperienceCore, so a host +/// wires the two together without either package knowing the other's concrete types. +/// +public static class AgentExperiencePostgresServiceCollectionExtensions +{ + /// + /// Registers as the singleton + /// , over an resolved from the + /// container. + /// + /// + /// The host owns the data source's lifetime and the store never disposes it. The schema is not + /// applied here: call + /// once + /// at startup. + /// + /// The service collection to add to. + /// , for chaining. + /// is . + public static IServiceCollection AddAgentExperiencePostgresStore(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(provider => + new PostgresExperienceRecordStore(provider.GetRequiredService())); + + return services; + } + + /// + /// Registers as the singleton + /// over , for a host that keeps + /// its data source outside the container. + /// + /// The service collection to add to. + /// The host-owned data source the store opens connections from. Never disposed by the store. + /// , for chaining. + /// Any argument is . + public static IServiceCollection AddAgentExperiencePostgresStore(this IServiceCollection services, NpgsqlDataSource dataSource) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(dataSource); + + services.TryAddSingleton(new PostgresExperienceRecordStore(dataSource)); + + return services; + } +} diff --git a/src/AgentExperience.Storage.Postgres/README.md b/src/AgentExperience.Storage.Postgres/README.md index 2227c5a..1c50383 100644 --- a/src/AgentExperience.Storage.Postgres/README.md +++ b/src/AgentExperience.Storage.Postgres/README.md @@ -3,9 +3,11 @@ Stores AgentExperience.NET Experience Records in PostgreSQL through the `IExperienceRecordStore` port, using plain Npgsql. -Pinned to `Npgsql` **10.0.3**, `dbup-postgresql` **7.0.1**, and `dbup-core` **6.1.1** (all exact). Integration tests -run against PostgreSQL 16 (`pgvector/pgvector:pg16`) through `Testcontainers.PostgreSql` 4.15.0. This package does not -use EF Core, Dapper, Pgvector, or the pgvector extension. +Pinned to `Npgsql` **10.0.3**, `dbup-postgresql` **7.0.1**, `dbup-core` **6.1.1**, and +`Microsoft.Extensions.DependencyInjection.Abstractions` **10.0.11** (all exact; the DI package is abstractions only — +no container, no hosting — and exists for this package's own registration extension). Integration tests run against +PostgreSQL 16 (`pgvector/pgvector:pg16`) through `Testcontainers.PostgreSql` 4.15.0. This package does not use EF +Core, Dapper, Pgvector, or the pgvector extension. ## Usage @@ -58,6 +60,27 @@ var history = await store.GetHistoryAsync(authorization, record.Scope, record.Ex The store never disposes the data source. The host owns it. +### Registering it + +```csharp +using AgentExperience.Core.DependencyInjection; +using AgentExperience.Storage.Postgres.DependencyInjection; + +services.AddSingleton(NpgsqlDataSource.Create(connectionString)); +services.AddAgentExperiencePostgresStore(); // or AddAgentExperiencePostgresStore(dataSource) + +// Core's own extension then supplies capture, reflection, lifecycle, and finalization over this store. +services.AddAgentExperienceCore(sanitizationOptions, captureLimits); +``` + +The registration is `TryAdd`-based, so a host that has already registered its own `IExperienceRecordStore` keeps it. +It does **not** apply the schema: call `ExperienceSchemaMigrator.MigrateAsync` once at startup (see +[Schema](#schema)). + +Records are normally written by Core's `ExperienceFinalizationService`, which creates the record and commits its +initial lifecycle event; `CreateAsync` and `CommitLifecycleEventAsync` stay available for hosts that orchestrate that +themselves. + ## Trusted host boundary - `AuthorizationContext` is the authority, and `Scope` only selects within it. The host must build the context from diff --git a/src/AgentExperience.Storage.Postgres/packages.lock.json b/src/AgentExperience.Storage.Postgres/packages.lock.json index b0e78c4..f474c1c 100644 --- a/src/AgentExperience.Storage.Postgres/packages.lock.json +++ b/src/AgentExperience.Storage.Postgres/packages.lock.json @@ -21,6 +21,12 @@ "dbup-core": "6.1.1" } }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Direct", + "requested": "[10.0.11, 10.0.11]", + "resolved": "10.0.11", + "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" + }, "Npgsql": { "type": "Direct", "requested": "[10.0.3, 10.0.3]", @@ -30,11 +36,6 @@ "Microsoft.Extensions.Logging.Abstractions": "10.0.0" } }, - "Microsoft.Extensions.DependencyInjection.Abstractions": { - "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "L3AdmZ1WOK4XXT5YFPEwyt0ep6l8lGIPs7F5OOBZc77Zqeo01Of7XXICy47628sdVl0v/owxYJTe86DTgFwKCA==" - }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", "resolved": "10.0.0", diff --git a/tests/AgentExperience.Abstractions.Tests/packages.lock.json b/tests/AgentExperience.Abstractions.Tests/packages.lock.json index 1597822..50c646d 100644 --- a/tests/AgentExperience.Abstractions.Tests/packages.lock.json +++ b/tests/AgentExperience.Abstractions.Tests/packages.lock.json @@ -34,19 +34,6 @@ "resolved": "17.14.1", "contentHash": "pmTrhfFIoplzFVbhVwUquT+77CbGH+h4/3mBpdmIlYtBi9nAB+kKI6dN3A/nV4DFi3wLLx/BlHIPK+MkbQ6Tpg==" }, - "Microsoft.Extensions.DependencyInjection.Abstractions": { - "type": "Transitive", - "resolved": "8.0.0", - "contentHash": "cjWrLkJXK0rs4zofsK4bSdg+jhDLTaxrkXu4gS6Y7MAlCvRyNNgwY/lJi5RDlQOnSZweHqoyvgvbdvQsRIW+hg==" - }, - "Microsoft.Extensions.Logging.Abstractions": { - "type": "Transitive", - "resolved": "8.0.0", - "contentHash": "arDBqTgFCyS0EvRV7O3MZturChstm50OJ0y9bDJvAcmEPJm0FFpFyjU/JLYyStNGGey081DvnQYlncNX5SJJGA==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0" - } - }, "Microsoft.TestPlatform.ObjectModel": { "type": "Transitive", "resolved": "17.14.1", @@ -66,14 +53,6 @@ "resolved": "13.0.3", "contentHash": "HrC5BXdl00IP9zeV+0Z848QWPAoCr9P3bDEZguI+gkLcBKAOxix/tLEAAHC+UvDNPv4a2d18lOReHMOagPa+zQ==" }, - "Npgsql": { - "type": "Transitive", - "resolved": "8.0.3", - "contentHash": "6WEmzsQJCZAlUG1pThKg/RmeF6V+I0DmBBBE/8YzpRtEzhyZzKcK7ulMANDm5CkxrALBEC8H+5plxHWtIL7xnA==", - "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "8.0.0" - } - }, "xunit.abstractions": { "type": "Transitive", "resolved": "2.0.3", @@ -115,10 +94,7 @@ } }, "agentexperience.abstractions": { - "type": "Project", - "dependencies": { - "Npgsql": "[8.0.3, )" - } + "type": "Project" } } } diff --git a/tests/AgentExperience.Core.Tests/AgentExperience.Core.Tests.csproj b/tests/AgentExperience.Core.Tests/AgentExperience.Core.Tests.csproj index ab259fa..e85d756 100644 --- a/tests/AgentExperience.Core.Tests/AgentExperience.Core.Tests.csproj +++ b/tests/AgentExperience.Core.Tests/AgentExperience.Core.Tests.csproj @@ -11,6 +11,9 @@ + + diff --git a/tests/AgentExperience.Core.Tests/CoreServiceRegistrationTests.cs b/tests/AgentExperience.Core.Tests/CoreServiceRegistrationTests.cs new file mode 100644 index 0000000..60c9cbf --- /dev/null +++ b/tests/AgentExperience.Core.Tests/CoreServiceRegistrationTests.cs @@ -0,0 +1,110 @@ +using AgentExperience.Core.DependencyInjection; +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Lifecycle; +using Microsoft.Extensions.DependencyInjection; + +namespace AgentExperience.Core.Tests; + +/// +/// Resolves what +/// registers out of a real container, so deleting a registration fails here rather than only at a +/// host's startup. Also pins the two properties a host depends on: the caller's own arguments are the +/// ones that reach the services, and a host implementation registered first still wins. +/// +public class CoreServiceRegistrationTests +{ + private static readonly SanitizationOptions CallerOptions = new(new Dictionary(StringComparer.Ordinal) + { + ["ToolResult"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "value" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 2, + MaxFieldCount: 5, + MaxValueLength: 1_000, + MaxFieldNameLength: 100), + }); + + private static readonly CaptureLimits CallerLimits = new(10, 10, 1_000, 1_000); + + [Fact] + public void AddAgentExperienceCore_registers_every_service_the_finalization_path_needs() + { + var services = new ServiceCollection(); + services.AddSingleton(new StubStore()); // the storage adapter's job + services.AddAgentExperienceCore(CallerOptions, CallerLimits); + + using var provider = services.BuildServiceProvider(); + + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + + // Singletons, so capture's in-memory snapshots survive between resolutions. + Assert.Same(provider.GetRequiredService(), provider.GetRequiredService()); + Assert.Same(provider.GetRequiredService(), provider.GetRequiredService()); + } + + [Fact] + public async Task The_sanitizer_applies_the_policy_the_caller_passed_even_when_the_host_registered_another() + { + // A host-registered SanitizationOptions must not silently replace the policy the caller handed + // to AddAgentExperienceCore: the sanitizer would then enforce a policy nobody passed to it. + var services = new ServiceCollection(); + services.AddSingleton(SanitizationOptions.Empty); // registered first, so TryAdd keeps it + services.AddAgentExperienceCore(CallerOptions, CallerLimits); + + using var provider = services.BuildServiceProvider(); + + var sanitized = await provider.GetRequiredService().SanitizeAsync( + new RawPayload("ToolResult", new Dictionary { ["value"] = "ok" })); + + Assert.Equal(SanitizationDecision.Allowed, sanitized.Decision); + } + + [Fact] + public void A_host_implementation_registered_first_wins() + { + var hostSanitizer = new DefaultSanitizer(CallerOptions); + var hostReflector = new DefaultExperienceReflector(); + + var services = new ServiceCollection(); + services.AddSingleton(new StubStore()); + services.AddSingleton(hostSanitizer); + services.AddSingleton(hostReflector); + services.AddAgentExperienceCore(CallerOptions, CallerLimits); + + using var provider = services.BuildServiceProvider(); + + Assert.Same(hostSanitizer, provider.GetRequiredService()); + Assert.Same(hostReflector, provider.GetRequiredService()); + } + + [Fact] + public void Null_arguments_throw() + { + Assert.Throws(() => ((IServiceCollection)null!).AddAgentExperienceCore(CallerOptions, CallerLimits)); + Assert.Throws(() => new ServiceCollection().AddAgentExperienceCore(null!, CallerLimits)); + Assert.Throws(() => new ServiceCollection().AddAgentExperienceCore(CallerOptions, null!)); + } + + /// Stands in for a storage adapter's registration; finalization never calls it here. + private sealed class StubStore : IExperienceRecordStore + { + public Task CreateAsync(AuthorizationContext authorization, ExperienceRecord record, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public Task GetAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public Task QueryAsync(AuthorizationContext authorization, ExperienceRecordQuery query, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public Task CommitLifecycleEventAsync(AuthorizationContext authorization, Scope scope, LifecycleEvent lifecycleEvent, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public Task GetHistoryAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) => + throw new NotSupportedException(); + } +} diff --git a/tests/AgentExperience.Core.Tests/DefaultExperienceReflectorTests.cs b/tests/AgentExperience.Core.Tests/DefaultExperienceReflectorTests.cs index 7f10d95..a32c1d6 100644 --- a/tests/AgentExperience.Core.Tests/DefaultExperienceReflectorTests.cs +++ b/tests/AgentExperience.Core.Tests/DefaultExperienceReflectorTests.cs @@ -62,6 +62,9 @@ public static ExperienceRun Run( StartedAt: BaseTime, EndedAt: executionStatus is null ? null : BaseTime.AddMinutes(1)); + /// A required check that accepts evidence of any kind unless a kind is named. + public static RequiredCheck Check(string checkId, string? expectedKind = null) => new(checkId, expectedKind); + public static Evidence Evidence(int index, string checkId, CheckResult result) => new( EvidenceId: Guid.Parse($"aaaaaaaa-0000-0000-0000-{index:D12}"), @@ -77,18 +80,18 @@ public static Evidence Evidence(int index, string checkId, CheckResult result) = public static VerificationResult Verified() => VerificationAggregator.Aggregate( [Evidence(1, "build", CheckResult.Pass), Evidence(2, "tests", CheckResult.Pass)], - ["build", "tests"], Round, Round.ArtifactRevision, BaseTime); + [Check("build"), Check("tests")], Round, Round.ArtifactRevision, BaseTime); public static VerificationResult Failed() => VerificationAggregator.Aggregate( [Evidence(1, "build", CheckResult.Fail), Evidence(2, "tests", CheckResult.Pass)], - ["build", "tests"], Round, Round.ArtifactRevision, BaseTime); + [Check("build"), Check("tests")], Round, Round.ArtifactRevision, BaseTime); /// "build" passes, "tests" has no evidence: Unknown with completion score 0.5. public static VerificationResult Unknown() => VerificationAggregator.Aggregate( [Evidence(1, "build", CheckResult.Pass)], - ["build", "tests"], Round, Round.ArtifactRevision, BaseTime); + [Check("build"), Check("tests")], Round, Round.ArtifactRevision, BaseTime); /// Attempt 0 errors, attempt 1 completes. public static IReadOnlyList RepairAttempts() => @@ -447,7 +450,7 @@ public async Task Tool_calls_are_described_in_sequence_number_order_regardless_o [Fact] public async Task Unknown_with_a_zero_completion_score_has_no_partial_score_warning() { - var evaluation = VerificationAggregator.Aggregate([], ["build"], null, "rev-1", ReflectionFixtures.BaseTime); + var evaluation = VerificationAggregator.Aggregate([], [ReflectionFixtures.Check("build")], null, "rev-1", ReflectionFixtures.BaseTime); Assert.Equal(TaskVerificationStatus.Unknown, evaluation.Outcome.Status); Assert.Equal(0.0, evaluation.CompletionScore); diff --git a/tests/AgentExperience.Core.Tests/DependencyBoundaryTests.cs b/tests/AgentExperience.Core.Tests/DependencyBoundaryTests.cs index acb7904..66bf767 100644 --- a/tests/AgentExperience.Core.Tests/DependencyBoundaryTests.cs +++ b/tests/AgentExperience.Core.Tests/DependencyBoundaryTests.cs @@ -7,8 +7,11 @@ namespace AgentExperience.Core.Tests; /// /// Proves AgentExperience.Core has no dependency on MAF, EF Core, Npgsql, DbUp, OpenTelemetry, /// or a model-provider package (AD-1) -- its only allowed dependencies are -/// AgentExperience.Abstractions and Microsoft.Extensions.Compliance.Redaction (plus -/// that package's own transitive Microsoft.Extensions.* configuration/DI/options graph). +/// AgentExperience.Abstractions, Microsoft.Extensions.Compliance.Redaction, and +/// Microsoft.Extensions.DependencyInjection.Abstractions (plus the redaction package's own +/// transitive Microsoft.Extensions.* configuration/DI/options graph). The DI package is +/// abstractions only -- no container, no hosting -- and exists so Core can ship its own +/// AddAgentExperienceCore registration extension without a host guessing concrete types. /// Mirrors AgentExperience.Abstractions.Tests/DependencyBoundaryTests.cs. This runs in CI on /// every push/PR so the boundary cannot silently regress as later stories/adapters are added to /// the solution. @@ -79,6 +82,28 @@ public void AgentExperience_Core_csproj_declares_no_forbidden_PackageReference() } } + [Fact] + public void AgentExperience_Core_csproj_declares_exactly_the_allowed_PackageReferences() + { + // The forbidden-substring checks above cannot catch a newly added package that is merely + // unwanted rather than forbidden. Pinning the whole declared set makes every future addition + // a deliberate, reviewed change to this list. + var declared = XDocument.Load(GetCoreCsprojPath()) + .Descendants("PackageReference") + .Select(element => $"{element.Attribute("Include")?.Value} {element.Attribute("Version")?.Value}") + .Order(StringComparer.Ordinal) + .ToList(); + + // The DI abstractions pin is exact, matching AgentExperience.Storage.Postgres, so the two + // packages can never resolve different versions of the same dependency. + Assert.Equal( + [ + "Microsoft.Extensions.Compliance.Redaction 10.9.0", + "Microsoft.Extensions.DependencyInjection.Abstractions [10.0.11]", + ], + declared); + } + private static string GetCoreCsprojPath([CallerFilePath] string testSourceFilePath = "") { var testsProjectDirectory = Path.GetDirectoryName(testSourceFilePath)!; diff --git a/tests/AgentExperience.Core.Tests/ExperienceFinalizationServiceTests.cs b/tests/AgentExperience.Core.Tests/ExperienceFinalizationServiceTests.cs new file mode 100644 index 0000000..32d2d44 --- /dev/null +++ b/tests/AgentExperience.Core.Tests/ExperienceFinalizationServiceTests.cs @@ -0,0 +1,993 @@ +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Lifecycle; + +namespace AgentExperience.Core.Tests; + +/// +/// Covers every row of Story 2.5's I/O and edge-case matrix against fakes: the happy path, an +/// unverified run, a throwing reflector, a host storage denial, a run beyond the caller's authority, +/// an unknown run, an unfinished run, a replay, and a store failure in each of the two stages that +/// touch the database. Capture and the lifecycle service are the real implementations; only the +/// record store and (where a failure is being forced) the reflector are doubles. +/// +public class ExperienceFinalizationServiceTests +{ + private const string ArtifactRevision = "rev-1"; + + private static readonly DateTimeOffset Now = new(2026, 9, 18, 10, 0, 0, TimeSpan.Zero); + private static readonly Scope TestScope = new("tenant-1", "app-1", "project-1"); + private static readonly AuthorizationContext Authorization = new("tenant-1", "host-principal", ["experience:write"], Now); + private static readonly ClosedVerificationRound Round = new(Guid.Parse("11111111-1111-1111-1111-111111111111"), ArtifactRevision); + + private static readonly SanitizationOptions PermissiveOptions = new(new Dictionary(StringComparer.Ordinal) + { + ["ToolArguments"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "query" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 3, + MaxFieldCount: 10, + MaxValueLength: 1_000, + MaxFieldNameLength: 100), + ["ToolResult"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "value" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 2, + MaxFieldCount: 5, + MaxValueLength: 1_000, + MaxFieldNameLength: 100), + }); + + private static Evidence PassingEvidence(string checkId = "tests", CheckResult result = CheckResult.Pass) => new( + EvidenceId: Guid.NewGuid(), + VerificationRoundId: Round.RoundId, + ArtifactRevision: ArtifactRevision, + CheckId: checkId, + Kind: "TestResult", + Result: result, + Producer: "ci", + Detail: null, + CapturedAt: Now); + + // --------------------------------------------------------------------------------------------- + // Happy path + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task A_verified_run_a_successful_reflection_and_a_permitting_decision_produce_a_Validated_record() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, result.Stage); + Assert.True(result.IsDurable); + Assert.Null(result.Failure); + + var record = Assert.IsType(result.Record); + Assert.Equal(ExperienceStatus.Validated, record.Status); + Assert.Equal(2d / 3d, record.ReuseConfidence); + Assert.Equal(1, record.SupportingValidations); + Assert.Equal(0, record.Contradictions); + Assert.Equal(TaskVerificationStatus.Verified, record.Outcome.Status); + Assert.NotNull(record.Reflection); + Assert.Equal(harness.RunId, record.SourceRunId); + + // One record, at revision 1, with exactly one lifecycle event. + Assert.Equal(1, result.Revision); + + // The record is *created* as a Candidate; the initial event performs the real transition, so a + // commit that never lands can only ever leave a Candidate behind. + Assert.Equal(ExperienceStatus.Candidate, Assert.Single(harness.Store.Creates).Status); + + var committed = Assert.Single(harness.Store.Commits); + Assert.Equal(ExperienceStatus.Candidate, committed.PriorStatus); + Assert.Equal(ExperienceStatus.Validated, committed.CurrentStatus); + Assert.Equal(0, committed.ExpectedRevision); + Assert.Equal(ExperienceFinalizationService.ProducerIdentity, committed.Producer); + Assert.Equal(1, harness.Store.RevisionOf(record.ExperienceId)); + Assert.Equal(ExperienceStatus.Validated, harness.Store.StatusOf(record.ExperienceId)); + } + + [Fact] + public async Task The_record_copies_the_captured_attempts_unchanged() + { + var harness = await Harness.WithCompletedRunAsync(); + var captured = harness.CapturedRun(); + + var result = await harness.FinalizeAsync(); + + // Finalization never sanitizes: capture already rejected anything unsafe. + Assert.Equal(captured.Attempts, result.Record!.Attempts); + Assert.Equal(captured.TaskId, result.Record.TaskId); + Assert.Equal(captured.TaskDescription, result.Record.TaskSummary); + Assert.Same(captured.Environment, result.Record.Environment); + Assert.Same(captured.Provenance, result.Record.Provenance); + Assert.Same(captured.Scope, result.Record.Scope); + } + + // --------------------------------------------------------------------------------------------- + // Unverified + // --------------------------------------------------------------------------------------------- + + [Theory] + [InlineData(CheckResult.Fail, TaskVerificationStatus.Failed)] + [InlineData(CheckResult.Unknown, TaskVerificationStatus.Unknown)] + public async Task An_unverified_run_is_quarantined_with_failure_metadata_and_no_reflection( + CheckResult evidenceResult, + TaskVerificationStatus expectedStatus) + { + var reflector = new CountingReflector(); + var harness = await Harness.WithCompletedRunAsync(reflector: reflector); + + var result = await harness.FinalizeAsync(evidence: [PassingEvidence(result: evidenceResult)]); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(ExperienceStatus.Quarantined, result.Record!.Status); + Assert.Null(result.Record.Reflection); + Assert.Equal(0d, result.Record.ReuseConfidence); + Assert.Equal(0, result.Record.SupportingValidations); + Assert.Equal(expectedStatus, result.Record.Outcome.Status); + + // Safe failure metadata: which stage decided it, in content-free prose. + var failure = Assert.IsType(result.Failure); + Assert.Equal(FinalizationStage.Evaluate, failure.Stage); + Assert.Contains(expectedStatus.ToString(), failure.Reason, StringComparison.Ordinal); + + // An unverified run is never reflected on, so no unreflected lesson can reach the record. + Assert.Equal(0, reflector.Calls); + Assert.Equal(ExperienceStatus.Quarantined, Assert.Single(harness.Store.Commits).CurrentStatus); + } + + [Fact] + public async Task A_run_with_no_closed_round_is_quarantined_rather_than_validated() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(noRound: true); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Equal(TaskVerificationStatus.Unknown, result.Evaluation!.Outcome.Status); + } + + // --------------------------------------------------------------------------------------------- + // Reflection fails + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task A_throwing_reflector_quarantines_the_record_names_the_stage_and_is_not_rethrown() + { + var harness = await Harness.WithCompletedRunAsync(reflector: new ThrowingReflector()); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Equal(ExperienceStatus.Quarantined, result.Record!.Status); + Assert.Null(result.Record.Reflection); + + var failure = Assert.IsType(result.Failure); + Assert.Equal(FinalizationStage.Reflect, failure.Stage); + Assert.IsType(failure.Exception); + + // The record is still committed, and the verification it was judged against is preserved. + Assert.Equal(TaskVerificationStatus.Verified, result.Record.Outcome.Status); + Assert.Single(harness.Store.Commits); + Assert.Equal(1, result.Revision); + } + + [Fact] + public async Task A_reflector_that_returns_no_reflection_quarantines_the_record_and_names_the_stage() + { + // Returning null is not the same failure mode as throwing, and it is the one a lenient custom + // reflector is most likely to produce. + var harness = await Harness.WithCompletedRunAsync(reflector: new NullReturningReflector()); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Equal(ExperienceStatus.Quarantined, result.Record!.Status); + Assert.Null(result.Record.Reflection); + Assert.Equal(FinalizationStage.Reflect, result.Failure!.Stage); + Assert.Null(result.Failure.Exception); + } + + // --------------------------------------------------------------------------------------------- + // Storage denied + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task A_host_decision_that_denies_storage_writes_nothing_and_returns_a_structured_denial() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(decision: StorageDecision.Deny("retention policy")); + + Assert.Equal(FinalizationOutcome.StorageDenied, result.Outcome); + Assert.Equal(FinalizationStage.Authorize, result.Stage); + Assert.False(result.IsDurable); + Assert.Null(result.Record); + Assert.Null(result.ExperienceId); // no record ID is issued + Assert.Null(result.Event); + Assert.Equal("retention policy", result.Reason); + Assert.Empty(harness.Store.Creates); + Assert.Empty(harness.Store.Commits); + } + + [Fact] + public async Task Storage_is_denied_whatever_the_verification_says() + { + var harness = await Harness.WithCompletedRunAsync(); + + var denied = await harness.FinalizeAsync( + evidence: [PassingEvidence(result: CheckResult.Fail)], + decision: StorageDecision.Deny()); + + Assert.Equal(FinalizationOutcome.StorageDenied, denied.Outcome); + Assert.Empty(harness.Store.Creates); + } + + // --------------------------------------------------------------------------------------------- + // Beyond authority + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task A_run_outside_the_authorization_is_denied_before_any_store_call() + { + var harness = await Harness.WithCompletedRunAsync(); + var otherTenant = new AuthorizationContext("tenant-2", "host-principal", ["experience:write"], Now); + + var result = await harness.FinalizeAsync(authorization: otherTenant); + + Assert.Equal(FinalizationOutcome.NotAuthorized, result.Outcome); + Assert.Equal(FinalizationStage.Authorize, result.Stage); + Assert.Null(result.Record); + Assert.Empty(harness.Store.Creates); + Assert.Empty(harness.Store.Commits); + } + + [Fact] + public async Task A_refused_run_is_never_handed_to_the_reflector() + { + // IExperienceReflector is the documented seam for a model-backed reflector, so both gates run + // before it: a run the host is about to refuse never has its content handed over. + var denied = new CountingReflector(); + var harnessDenied = await Harness.WithCompletedRunAsync(reflector: denied); + await harnessDenied.FinalizeAsync(decision: StorageDecision.Deny("retention policy")); + Assert.Equal(0, denied.Calls); + + var unauthorized = new CountingReflector(); + var harnessUnauthorized = await Harness.WithCompletedRunAsync(reflector: unauthorized); + await harnessUnauthorized.FinalizeAsync( + authorization: new AuthorizationContext("tenant-2", "host-principal", ["experience:write"], Now)); + Assert.Equal(0, unauthorized.Calls); + } + + // --------------------------------------------------------------------------------------------- + // Unknown and unfinished runs + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task An_unknown_run_is_a_structured_not_found_result() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(runId: Guid.NewGuid()); + + Assert.Equal(FinalizationOutcome.RunNotFound, result.Outcome); + Assert.Equal(FinalizationStage.Load, result.Stage); + Assert.Null(result.Evaluation); + Assert.Empty(harness.Store.Creates); + } + + [Fact] + public async Task A_run_with_no_execution_status_is_a_structured_invalid_result_and_writes_nothing() + { + var harness = Harness.Create(); + var runId = harness.StartRun(); + await harness.AppendAttemptAsync(runId); + // Deliberately not completed. + + var result = await harness.FinalizeAsync(runId: runId); + + Assert.Equal(FinalizationOutcome.RunNotFinished, result.Outcome); + Assert.Equal(FinalizationStage.Load, result.Stage); + Assert.Empty(harness.Store.Creates); + Assert.Empty(harness.Store.Commits); + } + + // --------------------------------------------------------------------------------------------- + // Replay + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task Finalizing_the_same_run_twice_creates_one_record_at_revision_one_with_one_event() + { + var harness = await Harness.WithCompletedRunAsync(); + + var first = await harness.FinalizeAsync(); + var second = await harness.FinalizeAsync(finalizedAt: Now.AddMinutes(5)); // a later retry + + Assert.Equal(FinalizationOutcome.Validated, first.Outcome); + Assert.Equal(FinalizationOutcome.AlreadyFinalized, second.Outcome); + + // The second call reports the first call's outcome... + Assert.Equal(first.ExperienceId, second.ExperienceId); + Assert.Equal(ExperienceStatus.Validated, second.Status); + Assert.Equal(1, second.Revision); + Assert.True(second.IsDurable); + + // ...and writes nothing: no second record, no second initial event. + Assert.Single(harness.Store.Creates); + Assert.Single(harness.Store.Commits); + Assert.Equal(1, harness.Store.RevisionOf(first.ExperienceId!.Value)); + } + + [Fact] + public async Task A_quarantined_replay_still_names_the_stage_that_quarantined_it() + { + var harness = await Harness.WithCompletedRunAsync(); + var unverified = new[] { PassingEvidence(result: CheckResult.Fail) }; + + var first = await harness.FinalizeAsync(evidence: unverified); + Assert.Equal(FinalizationOutcome.Quarantined, first.Outcome); + + var second = await harness.FinalizeAsync(evidence: unverified, finalizedAt: Now.AddMinutes(5)); + + Assert.Equal(FinalizationOutcome.AlreadyFinalized, second.Outcome); + Assert.Equal(ExperienceStatus.Quarantined, second.Status); + + // A quarantine always names the stage that decided it, replayed or not. + Assert.Equal(FinalizationStage.Evaluate, second.Failure!.Stage); + } + + [Fact] + public async Task A_resumed_commit_on_a_quarantined_record_names_the_stage_that_quarantined_it() + { + var harness = await Harness.WithCompletedRunAsync(); + var unverified = new[] { PassingEvidence(result: CheckResult.Fail) }; + harness.Store.ThrowOnCommit = () => new ExperienceStoreException("commit unavailable"); + await harness.FinalizeAsync(evidence: unverified); + + harness.Store.ThrowOnCommit = null; + var resumed = await harness.FinalizeAsync(evidence: unverified, finalizedAt: Now.AddMinutes(5)); + + Assert.Equal(FinalizationOutcome.Quarantined, resumed.Outcome); + Assert.Equal(FinalizationStage.Evaluate, resumed.Failure!.Stage); + Assert.Single(harness.Store.Commits); + } + + [Fact] + public async Task The_record_and_initial_event_ids_derive_from_the_run() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), result.ExperienceId); + Assert.Equal(ExperienceFinalizationService.InitialEventIdFor(harness.RunId), result.Event!.EventId); + Assert.Equal(ExperienceFinalizationService.ReflectionIdFor(harness.RunId), result.Record!.Reflection!.ReflectionId); + + // Derivation is stable across calls and distinct per purpose and per run. + Assert.Equal(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), ExperienceFinalizationService.ExperienceIdFor(harness.RunId)); + Assert.NotEqual(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), ExperienceFinalizationService.InitialEventIdFor(harness.RunId)); + Assert.NotEqual(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), ExperienceFinalizationService.ExperienceIdFor(Guid.NewGuid())); + Assert.NotEqual(Guid.Empty, ExperienceFinalizationService.ExperienceIdFor(harness.RunId)); + } + + [Fact] + public async Task A_retry_after_a_failed_initial_commit_finishes_that_commit_rather_than_starting_over() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.ThrowOnCommit = () => new ExperienceStoreException("commit unavailable"); + + var failed = await harness.FinalizeAsync(); + Assert.Equal(FinalizationOutcome.Failed, failed.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, failed.Stage); + + // The record exists but is still a Candidate, so nothing can reuse it in the meantime. + Assert.Equal(ExperienceStatus.Candidate, harness.Store.StatusOf(failed.ExperienceId!.Value)); + + harness.Store.ThrowOnCommit = null; + var retried = await harness.FinalizeAsync(finalizedAt: Now.AddMinutes(5)); + + Assert.Equal(FinalizationOutcome.Validated, retried.Outcome); + Assert.Equal(1, retried.Revision); + Assert.Single(harness.Store.Creates); // still exactly one record + Assert.Single(harness.Store.Commits); // and exactly one initial event + } + + [Fact] + public async Task A_derived_record_id_taken_in_another_scope_is_a_failure_not_a_silent_success() + { + var harness = await Harness.WithCompletedRunAsync(); + var foreignScope = new Scope("tenant-9", "app-1", "project-1"); + harness.Store.Seed(TestRecord(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), foreignScope)); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, result.Stage); + Assert.False(result.IsDurable); + Assert.Empty(harness.Store.Commits); + } + + // --------------------------------------------------------------------------------------------- + // Store failures + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task A_store_failure_while_creating_is_a_failed_stage_and_the_run_stays_retrievable() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.ThrowOnCreate = () => new ExperienceStoreException("database unavailable"); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, result.Stage); + Assert.False(result.IsDurable); + Assert.Null(result.Record); + Assert.IsType(result.Failure!.Exception); + Assert.Empty(harness.Store.Commits); + + // The captured snapshot is still there for the host to retry against. + Assert.True(harness.Capture.TryGetRun(harness.RunId, out _)); + } + + [Fact] + public async Task A_store_failure_while_committing_is_never_reported_as_durable_success() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.ThrowOnCommit = () => new ExperienceStoreException("database unavailable"); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, result.Stage); + Assert.False(result.IsDurable); + Assert.Equal(0, result.Revision); + Assert.IsType(result.Failure!.Exception); + Assert.True(harness.Capture.TryGetRun(harness.RunId, out _)); + + // The record that now exists is reported, so the host can reconcile it rather than guess -- and + // it is still a Candidate, so nothing can reuse it. + Assert.Equal(ExperienceStatus.Candidate, result.Record!.Status); + Assert.Equal(ExperienceStatus.Candidate, harness.Store.StatusOf(result.ExperienceId!.Value)); + } + + [Fact] + public async Task A_store_that_reports_the_record_invalid_is_a_failed_stage_carrying_its_errors() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.CreateResult = new ExperienceRecordCreateResult( + ExperienceStoreOutcome.Invalid, + [new StoreValidationError("TaskId", "must not be empty or whitespace.")]); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, result.Stage); + Assert.Equal("TaskId", Assert.Single(result.Failure!.Errors).Path); + Assert.Empty(harness.Store.Commits); + } + + [Fact] + public async Task A_store_that_denies_the_create_is_reported_as_not_authorized() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.CreateResult = new ExperienceRecordCreateResult(ExperienceStoreOutcome.Denied, []); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.NotAuthorized, result.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, result.Stage); + Assert.Empty(harness.Store.Commits); + } + + [Fact] + public async Task A_lifecycle_commit_that_does_not_commit_is_a_failed_stage() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.CommitResult = new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.StaleRevision, 4, null, []); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, result.Stage); + Assert.Contains("StaleRevision", result.Failure!.Reason, StringComparison.Ordinal); + Assert.Equal(ExperienceStatus.Candidate, result.Record!.Status); + } + + [Fact] + public async Task A_refused_commit_on_a_quarantined_record_still_names_the_stage_that_quarantined_it() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.CommitResult = new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Invalid, 0, null, []); + + var result = await harness.FinalizeAsync(evidence: [PassingEvidence(result: CheckResult.Fail)]); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(ExperienceStatus.Candidate, result.Record!.Status); + + // The reason the record was going to be quarantined is not dropped in favour of the commit's + // own refusal -- the host still learns why no lesson was recorded. + Assert.Equal(FinalizationStage.Evaluate, result.Failure!.Stage); + } + + [Fact] + public async Task A_record_finalized_concurrently_converges_on_AlreadyFinalized_rather_than_failing_forever() + { + var harness = await Harness.WithCompletedRunAsync(); + + // The create lands, then someone else commits the initial event before this call's own commit, + // so the store reports a stale revision against a record that is now durably finalized. + harness.Store.BeforeCommit = store => + { + store.BeforeCommit = null; + store.ForceFinalize(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), ExperienceStatus.Validated); + }; + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.AlreadyFinalized, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(ExperienceStatus.Validated, result.Status); + Assert.Equal(1, result.Revision); + } + + [Fact] + public async Task An_exception_that_is_not_an_ExperienceStoreException_is_still_a_structured_failed_stage() + { + // "Every stage failure comes back as a structured result" is not limited to the store's own + // documented exception type. + var creating = await Harness.WithCompletedRunAsync(); + creating.Store.ThrowOnCreate = () => new ObjectDisposedException("data source"); + var createResult = await creating.FinalizeAsync(); + Assert.Equal(FinalizationOutcome.Failed, createResult.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, createResult.Stage); + Assert.IsType(createResult.Failure!.Exception); + + var committing = await Harness.WithCompletedRunAsync(); + committing.Store.ThrowOnCommit = () => new ObjectDisposedException("data source"); + var commitResult = await committing.FinalizeAsync(); + Assert.Equal(FinalizationOutcome.Failed, commitResult.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, commitResult.Stage); + Assert.IsType(commitResult.Failure!.Exception); + + var loading = Harness.Create(captureService: new ThrowingCaptureService()); + var loadResult = await loading.Service.FinalizeAsync(loading.Request(runId: Guid.NewGuid()), CancellationToken.None); + Assert.Equal(FinalizationOutcome.Failed, loadResult.Outcome); + Assert.Equal(FinalizationStage.Load, loadResult.Stage); + Assert.IsType(loadResult.Failure!.Exception); + } + + // --------------------------------------------------------------------------------------------- + // Evaluation ownership, arguments, cancellation + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task Malformed_verification_inputs_end_the_evaluate_stage_rather_than_throwing() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(requiredChecks: [new RequiredCheck("tests"), new RequiredCheck("tests")]); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.Evaluate, result.Stage); + Assert.IsType(result.Failure!.Exception); + Assert.Empty(harness.Store.Creates); + } + + [Fact] + public async Task Evidence_of_a_kind_the_required_check_does_not_expect_never_validates_the_record() + { + var harness = await Harness.WithCompletedRunAsync(); + var approval = PassingEvidence() with { Kind = "HumanApproval" }; + + var result = await harness.FinalizeAsync( + requiredChecks: [new RequiredCheck("tests", "TestResult")], + evidence: [approval]); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Equal(TaskVerificationStatus.Unknown, result.Record!.Outcome.Status); + } + + [Fact] + public async Task Evidence_from_another_round_never_finalizes_this_run_as_validated() + { + var harness = await Harness.WithCompletedRunAsync(); + var otherRound = PassingEvidence() with { VerificationRoundId = Guid.NewGuid() }; + + var result = await harness.FinalizeAsync(evidence: [otherRound]); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + } + + [Fact] + public async Task A_round_the_host_closed_for_another_artifact_revision_is_stale_and_never_validates() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(closedRound: new ClosedVerificationRound(Round.RoundId, "rev-2")); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Equal(TaskVerificationStatus.Unknown, result.Evaluation!.Outcome.Status); + } + + [Fact] + public async Task Null_arguments_throw_ArgumentNullException() + { + var harness = await Harness.WithCompletedRunAsync(); + + Assert.Throws(() => new ExperienceFinalizationService(null!, new DefaultExperienceReflector(), harness.Store, new ExperienceLifecycleService(harness.Store))); + Assert.Throws(() => new ExperienceFinalizationService(harness.Capture, null!, harness.Store, new ExperienceLifecycleService(harness.Store))); + Assert.Throws(() => new ExperienceFinalizationService(harness.Capture, new DefaultExperienceReflector(), null!, new ExperienceLifecycleService(harness.Store))); + Assert.Throws(() => new ExperienceFinalizationService(harness.Capture, new DefaultExperienceReflector(), harness.Store, null!)); + + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(null!, CancellationToken.None)); + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { Authorization = null! }, CancellationToken.None)); + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { RequiredChecks = null! }, CancellationToken.None)); + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { Evidence = null! }, CancellationToken.None)); + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { StorageDecision = null! }, CancellationToken.None)); + } + + [Fact] + public async Task A_malformed_request_is_rejected_before_anything_is_stored() + { + var harness = await Harness.WithCompletedRunAsync(); + + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { RunId = Guid.Empty }, CancellationToken.None)); + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { CurrentArtifactRevision = " " }, CancellationToken.None)); + + // An unset FinalizedAt would create a record whose every commit -- including every retry -- the + // store then rejects as Invalid forever, because an unset OccurredAt is not a valid event. + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { FinalizedAt = default }, CancellationToken.None)); + + Assert.Empty(harness.Store.Creates); + } + + [Fact] + public async Task Cancellation_propagates_rather_than_becoming_a_structured_result() + { + var harness = await Harness.WithCompletedRunAsync(); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => harness.Service.FinalizeAsync(harness.Request(), cts.Token)); + Assert.Empty(harness.Store.Creates); + } + + [Fact] + public async Task A_reflector_that_cancels_for_its_own_reasons_propagates_rather_than_quarantining_silently() + { + // One rule for every stage: an OperationCanceledException always propagates, whoever raised it. + var harness = await Harness.WithCompletedRunAsync(reflector: new CancellingReflector()); + + await Assert.ThrowsAnyAsync(() => harness.Service.FinalizeAsync(harness.Request(), CancellationToken.None)); + Assert.Empty(harness.Store.Creates); + } + + // --------------------------------------------------------------------------------------------- + // Fixtures + // --------------------------------------------------------------------------------------------- + + private static ExperienceRecord TestRecord(Guid experienceId, Scope scope) => new( + ExperienceId: experienceId, + SourceRunId: Guid.NewGuid(), + Scope: scope, + TaskId: "task-1", + TaskSummary: null, + Attempts: [], + Outcome: new Outcome(TaskVerificationStatus.Unknown, [], null, Now), + CompletionScore: 0, + Reflection: null, + Environment: new EnvironmentFingerprint("host", "10.0.0", "linux-x64", null, new Dictionary()), + Provenance: new Provenance("tests", null, Now, null), + Status: ExperienceStatus.Candidate, + ReuseConfidence: 0, + SupportingValidations: 0, + Contradictions: 0, + Revision: 0, + CreatedAt: Now, + UpdatedAt: Now); + + /// Real capture plus a real lifecycle service over an in-memory fake store. + private sealed class Harness + { + public required InMemoryExperienceCaptureService Capture { get; init; } + + public required FakeStore Store { get; init; } + + public required ExperienceFinalizationService Service { get; init; } + + public Guid RunId { get; private set; } + + public static Harness Create(IExperienceReflector? reflector = null, IExperienceCaptureService? captureService = null) + { + var capture = new InMemoryExperienceCaptureService( + new DefaultSanitizer(PermissiveOptions), + new CaptureLimits(50, 50, 10_000, 10_000)); + var store = new FakeStore(); + + return new Harness + { + Capture = capture, + Store = store, + Service = new ExperienceFinalizationService( + captureService ?? capture, + reflector ?? new DefaultExperienceReflector(), + store, + new ExperienceLifecycleService(store)), + }; + } + + public static async Task WithCompletedRunAsync(IExperienceReflector? reflector = null) + { + var harness = Create(reflector); + harness.RunId = harness.StartRun(); + await harness.AppendAttemptAsync(harness.RunId); + var completed = await harness.Capture.CompleteRunAsync(harness.RunId, Guid.NewGuid(), RunExecutionStatus.Completed, Now.AddMinutes(1)); + Assert.Equal(CompleteRunOutcome.Recorded, completed.Outcome); + return harness; + } + + public Guid StartRun() + { + var runId = Guid.NewGuid(); + RunId = runId; + var started = Capture.StartRun( + runId, + taskId: "task-1", + taskDescription: "a test task", + scope: TestScope, + environment: new EnvironmentFingerprint("host-1", "net10.0", "test-os", null, new Dictionary()), + provenance: new Provenance("unit-tests", "1.0.0", Now, null), + startedAt: Now); + Assert.Equal(StartRunOutcome.Started, started.Outcome); + return runId; + } + + public async Task AppendAttemptAsync(Guid runId) + { + var appended = await Capture.AppendAttemptAsync( + runId, + new AppendAttemptRequest(Guid.NewGuid(), Now, TimeSpan.FromSeconds(1), [], "done", null)); + Assert.Equal(AppendAttemptOutcome.Recorded, appended.Outcome); + } + + public ExperienceRun CapturedRun() + { + Assert.True(Capture.TryGetRun(RunId, out var run)); + return run; + } + + public FinalizeExperienceRequest Request( + Guid? runId = null, + AuthorizationContext? authorization = null, + ClosedVerificationRound? closedRound = null, + bool noRound = false, + IReadOnlyList? requiredChecks = null, + IReadOnlyList? evidence = null, + StorageDecision? decision = null, + DateTimeOffset? finalizedAt = null) => new( + RunId: runId ?? RunId, + Authorization: authorization ?? Authorization, + ClosedRound: noRound ? null : closedRound ?? Round, + RequiredChecks: requiredChecks ?? [new RequiredCheck("tests")], + Evidence: evidence ?? [PassingEvidence()], + CurrentArtifactRevision: ArtifactRevision, + StorageDecision: decision ?? StorageDecision.Permit, + FinalizedAt: finalizedAt ?? Now.AddMinutes(2)); + + public Task FinalizeAsync( + Guid? runId = null, + AuthorizationContext? authorization = null, + ClosedVerificationRound? closedRound = null, + bool noRound = false, + IReadOnlyList? requiredChecks = null, + IReadOnlyList? evidence = null, + StorageDecision? decision = null, + DateTimeOffset? finalizedAt = null) => + Service.FinalizeAsync( + Request(runId, authorization, closedRound, noRound, requiredChecks, evidence, decision, finalizedAt), + CancellationToken.None); + } + + private sealed class CountingReflector : IExperienceReflector + { + private readonly DefaultExperienceReflector _inner = new(); + + public int Calls { get; private set; } + + public Task ReflectAsync(ReflectionRequest request, CancellationToken cancellationToken = default) + { + Calls++; + return _inner.ReflectAsync(request, cancellationToken); + } + } + + private sealed class ThrowingReflector : IExperienceReflector + { + public Task ReflectAsync(ReflectionRequest request, CancellationToken cancellationToken = default) => + throw new InvalidOperationException("the reflector template failed"); + } + + /// A lenient custom reflector that declines rather than throwing. + private sealed class NullReturningReflector : IExperienceReflector + { + public Task ReflectAsync(ReflectionRequest request, CancellationToken cancellationToken = default) => + Task.FromResult(null!); + } + + /// A reflector that cancels on a token of its own, not the caller's. + private sealed class CancellingReflector : IExperienceReflector + { + public Task ReflectAsync(ReflectionRequest request, CancellationToken cancellationToken = default) => + throw new OperationCanceledException("the reflector's own budget expired"); + } + + /// A capture service whose snapshot read fails with something other than a store exception. + private sealed class ThrowingCaptureService : IExperienceCaptureService + { + public StartRunResult StartRun(Guid runId, string taskId, string? taskDescription, Scope scope, EnvironmentFingerprint environment, Provenance provenance, DateTimeOffset startedAt) => + throw new InvalidOperationException("Finalization must not start runs."); + + public Task AppendAttemptAsync(Guid runId, AppendAttemptRequest request, CancellationToken cancellationToken = default) => + throw new InvalidOperationException("Finalization must not append attempts."); + + public Task CompleteRunAsync(Guid runId, Guid completionEventId, RunExecutionStatus executionStatus, DateTimeOffset endedAt, CancellationToken cancellationToken = default) => + throw new InvalidOperationException("Finalization must not complete runs."); + + public bool TryGetRun(Guid runId, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out ExperienceRun? run) => + throw new InvalidOperationException("the capture snapshot store is disposed"); + } + + /// + /// A minimal in-memory : create-only inserts, scoped reads, + /// and event-ID-idempotent, revision-checked lifecycle commits -- just enough of the port's real + /// contract that replay and failure behaviour are exercised rather than assumed. Query and history + /// are out of this story's scope and fail loudly if finalization ever calls them. + /// + private sealed class FakeStore : IExperienceRecordStore + { + private readonly Dictionary _records = []; + private readonly Dictionary _events = []; + + public List Creates { get; } = []; + + public List Commits { get; } = []; + + public Func? ThrowOnCreate { get; set; } + + public Func? ThrowOnCommit { get; set; } + + public ExperienceRecordCreateResult? CreateResult { get; set; } + + public ExperienceLifecycleCommitResult? CommitResult { get; set; } + + /// Runs just before a commit is applied, so a test can simulate a concurrent writer. + public Action? BeforeCommit { get; set; } + + public void Seed(ExperienceRecord record) => _records[record.ExperienceId] = record; + + public long RevisionOf(Guid experienceId) => _records[experienceId].Revision; + + public ExperienceStatus StatusOf(Guid experienceId) => _records[experienceId].Status; + + /// Applies someone else's initial commit to a stored record, exactly as a racing caller would. + public void ForceFinalize(Guid experienceId, ExperienceStatus status) + { + var record = _records[experienceId]; + _records[experienceId] = record with { Status = status, Revision = record.Revision + 1 }; + _events[Guid.NewGuid()] = (Event(experienceId, record.Status, status, record.Revision), record.Revision + 1); + } + + private static LifecycleEvent Event(Guid experienceId, ExperienceStatus? prior, ExperienceStatus current, long expectedRevision) => + new(Guid.NewGuid(), experienceId, prior, current, "concurrent finalization", "another host", Now, expectedRevision); + + public Task CreateAsync(AuthorizationContext authorization, ExperienceRecord record, CancellationToken cancellationToken) + { + Assert.NotNull(authorization); + cancellationToken.ThrowIfCancellationRequested(); + + if (ThrowOnCreate is not null) + { + throw ThrowOnCreate(); + } + + if (CreateResult is not null) + { + return Task.FromResult(CreateResult); + } + + if (_records.ContainsKey(record.ExperienceId)) + { + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Conflict, [])); + } + + if (!authorization.Permits(record.Scope)) + { + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Denied, [])); + } + + _records[record.ExperienceId] = record; + Creates.Add(record); + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Created, [])); + } + + public Task GetAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) + { + Assert.NotNull(authorization); + cancellationToken.ThrowIfCancellationRequested(); + + // A record in another scope is indistinguishable from a missing one. + return Task.FromResult(_records.TryGetValue(experienceId, out var record) && record.Scope == scope + ? new ExperienceRecordGetResult(ExperienceStoreOutcome.Found, record, []) + : new ExperienceRecordGetResult(ExperienceStoreOutcome.NotFound, null, [])); + } + + public Task CommitLifecycleEventAsync( + AuthorizationContext authorization, + Scope scope, + LifecycleEvent lifecycleEvent, + CancellationToken cancellationToken) + { + Assert.NotNull(authorization); + cancellationToken.ThrowIfCancellationRequested(); + + if (ThrowOnCommit is not null) + { + throw ThrowOnCommit(); + } + + BeforeCommit?.Invoke(this); + + if (CommitResult is not null) + { + return Task.FromResult(CommitResult); + } + + if (_events.TryGetValue(lifecycleEvent.EventId, out var stored)) + { + // Identical replay reports the original commit and writes nothing; any differing field conflicts. + return Task.FromResult(stored.Event == lifecycleEvent + ? new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Committed, stored.AppliedRevision, null, []) + : new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Conflict, 0, null, [])); + } + + if (!_records.TryGetValue(lifecycleEvent.ExperienceRecordId, out var record) || record.Scope != scope) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.NotFound, 0, null, [])); + } + + if (record.Revision != lifecycleEvent.ExpectedRevision) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.StaleRevision, record.Revision, null, [])); + } + + if (lifecycleEvent.PriorStatus is { } prior && record.Status != prior) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.StatusMismatch, record.Revision, record.Status, [])); + } + + var applied = lifecycleEvent.ExpectedRevision + 1; + _records[record.ExperienceId] = record with + { + Status = lifecycleEvent.CurrentStatus, + Revision = applied, + UpdatedAt = lifecycleEvent.OccurredAt, + }; + _events[lifecycleEvent.EventId] = (lifecycleEvent, applied); + Commits.Add(lifecycleEvent); + + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Committed, applied, null, [])); + } + + public Task QueryAsync(AuthorizationContext authorization, ExperienceRecordQuery query, CancellationToken cancellationToken) => + throw new InvalidOperationException("Finalization must not query records."); + + public Task GetHistoryAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) => + throw new InvalidOperationException("Finalization must not read history."); + } +} diff --git a/tests/AgentExperience.Core.Tests/VerificationAggregatorTests.cs b/tests/AgentExperience.Core.Tests/VerificationAggregatorTests.cs index 75c9302..42f41b3 100644 --- a/tests/AgentExperience.Core.Tests/VerificationAggregatorTests.cs +++ b/tests/AgentExperience.Core.Tests/VerificationAggregatorTests.cs @@ -14,6 +14,9 @@ namespace AgentExperience.Core.Tests; /// public class VerificationAggregatorTests { + /// A required check with no expected kind, i.e. one any evidence kind may satisfy. + private static RequiredCheck Check(string checkId, string? expectedKind = null) => new(checkId, expectedKind); + private static Evidence MakeEvidence(Guid roundId, string artifactRevision, string checkId, CheckResult result, string producer = "evaluator") => new( EvidenceId: Guid.NewGuid(), @@ -36,7 +39,7 @@ public void AC2_all_required_checks_passing_in_the_closed_round_yields_Verified( MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass), }; - var result = VerificationAggregator.Aggregate(evidence, ["build", "tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Verified, result.Outcome.Status); Assert.Equal(1.0, result.CompletionScore); @@ -54,7 +57,7 @@ public void AC2_one_required_check_failing_yields_Failed_regardless_of_other_pas MakeEvidence(round.RoundId, round.ArtifactRevision, "lint", CheckResult.Pass), }; - var result = VerificationAggregator.Aggregate(evidence, ["build", "tests", "lint"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests"), Check("lint")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Failed, result.Outcome.Status); Assert.Contains("build", result.Outcome.Reason); @@ -67,7 +70,7 @@ public void AC2_a_required_check_with_no_evidence_at_all_yields_Unknown() var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); var evidence = new[] { MakeEvidence(round.RoundId, round.ArtifactRevision, "build", CheckResult.Pass) }; - var result = VerificationAggregator.Aggregate(evidence, ["build", "tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Equal(0.5, result.CompletionScore); @@ -82,7 +85,7 @@ public void AC1_evidence_for_a_CheckId_outside_the_required_set_never_contribute var required = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass); var unrelated = MakeEvidence(round.RoundId, round.ArtifactRevision, "lint", CheckResult.Fail); - var result = VerificationAggregator.Aggregate([required, unrelated], ["tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([required, unrelated], [Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Verified, result.Outcome.Status); Assert.Same(required, Assert.Single(result.Outcome.Evidence)); @@ -95,7 +98,7 @@ public void Outcome_evidence_keeps_production_order_even_when_required_checks_ar var producedFirst = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass); var producedSecond = MakeEvidence(round.RoundId, round.ArtifactRevision, "build", CheckResult.Pass); - var result = VerificationAggregator.Aggregate([producedFirst, producedSecond], ["build", "tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([producedFirst, producedSecond], [Check("build"), Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal([producedFirst, producedSecond], result.Outcome.Evidence); } @@ -106,7 +109,7 @@ public void An_evidence_Result_outside_the_defined_CheckResult_values_resolves_t var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); var evidence = new[] { MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", (CheckResult)999) }; - var result = VerificationAggregator.Aggregate(evidence, ["tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Equal(0.0, result.CompletionScore); @@ -119,7 +122,7 @@ public void Duplicate_required_check_ids_throw_rather_than_skewing_the_completio var evidence = new[] { MakeEvidence(round.RoundId, round.ArtifactRevision, "build", CheckResult.Pass) }; Assert.Throws(() => - VerificationAggregator.Aggregate(evidence, ["build", "build", "tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + VerificationAggregator.Aggregate(evidence, [Check("build"), Check("build"), Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); } [Fact] @@ -129,7 +132,7 @@ public void A_null_evidence_entry_throws_rather_than_being_silently_dropped() var evidence = new[] { MakeEvidence(round.RoundId, round.ArtifactRevision, "build", CheckResult.Pass), null! }; Assert.Throws(() => - VerificationAggregator.Aggregate(evidence, ["build"], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + VerificationAggregator.Aggregate(evidence, [Check("build")], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); } [Fact] @@ -142,7 +145,7 @@ public void AC2_a_required_check_whose_only_evidence_is_Unknown_yields_overall_U MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Unknown), }; - var result = VerificationAggregator.Aggregate(evidence, ["build", "tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); } @@ -172,7 +175,7 @@ public void AC3_a_later_closed_round_can_verify_success_after_an_earlier_round_f var laterPass = MakeEvidence(laterRoundId, artifactRevision, "tests", CheckResult.Pass); var allEvidence = new[] { earlierFailure, laterPass }; - var result = VerificationAggregator.Aggregate(allEvidence, ["tests"], closedRound, artifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(allEvidence, [Check("tests")], closedRound, artifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Verified, result.Outcome.Status); // The earlier round's failing evidence never contributes to the outcome... @@ -193,7 +196,7 @@ public void AC3_AC4_evidence_from_a_different_artifact_revision_never_contribute // contribute even though VerificationRoundId matches. var staleRevisionEvidence = MakeEvidence(roundId, "rev-1", "tests", CheckResult.Pass); - var result = VerificationAggregator.Aggregate([staleRevisionEvidence], ["tests"], closedRound, "rev-2", DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([staleRevisionEvidence], [Check("tests")], closedRound, "rev-2", DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Empty(result.Outcome.Evidence); @@ -207,7 +210,7 @@ public void AC3_evidence_from_a_non_selected_round_id_never_contributes_even_und var otherRoundEvidence = MakeEvidence(Guid.NewGuid(), artifactRevision, "tests", CheckResult.Pass); - var result = VerificationAggregator.Aggregate([otherRoundEvidence], ["tests"], closedRound, artifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([otherRoundEvidence], [Check("tests")], closedRound, artifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Empty(result.Outcome.Evidence); @@ -216,7 +219,7 @@ public void AC3_evidence_from_a_non_selected_round_id_never_contributes_even_und [Fact] public void AC4_no_closed_round_yields_Unknown() { - var result = VerificationAggregator.Aggregate([], ["tests"], null, "rev-1", DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([], [Check("tests")], null, "rev-1", DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Equal(0.0, result.CompletionScore); @@ -229,7 +232,7 @@ public void AC4_a_closed_round_for_a_different_artifact_revision_than_current_is var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); var evidence = new[] { MakeEvidence(round.RoundId, "rev-2", "tests", CheckResult.Pass) }; - var result = VerificationAggregator.Aggregate(evidence, ["tests"], round, "rev-2", DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("tests")], round, "rev-2", DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); } @@ -244,7 +247,7 @@ public void AC4_mixed_Pass_and_Fail_evidence_for_the_same_required_check_resolve MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Fail), }; - var result = VerificationAggregator.Aggregate(evidence, ["tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Failed, result.Outcome.Status); // Both pieces of conflicting evidence are retained for audit, not discarded. @@ -262,7 +265,7 @@ public void AC5_completion_score_is_the_fraction_of_required_checks_that_conclus // "lint" has no evidence at all -> Unknown, so overall stays Unknown even though 2/3 passed. }; - var result = VerificationAggregator.Aggregate(evidence, ["build", "tests", "lint"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests"), Check("lint")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); // completion score alone never grants verification Assert.Equal(2.0 / 3.0, result.CompletionScore, precision: 10); @@ -297,12 +300,102 @@ public void AC6_an_evaluators_own_internal_failure_is_caught_upstream_and_aggreg Assert.Equal(CheckResult.Unknown, produced.Result); Assert.False(string.IsNullOrWhiteSpace(produced.Detail)); - var result = VerificationAggregator.Aggregate([produced], ["workflow-completes"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([produced], [Check("workflow-completes")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Same(produced, Assert.Single(result.Outcome.Evidence)); } + [Fact] + public void A_required_check_naming_an_ExpectedKind_ignores_evidence_of_any_other_kind() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + + // A human approval claiming the check the task declared must be answered by a test run. Under + // CheckId-only matching this would have verified the task; the expected kind stops it. + var wrongKind = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass) with { Kind = "HumanApproval" }; + + var result = VerificationAggregator.Aggregate( + [wrongKind], [Check("tests", "TestResult")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + + Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); + Assert.Equal(0.0, result.CompletionScore); + Assert.Empty(result.Outcome.Evidence); + } + + [Fact] + public void A_required_check_naming_an_ExpectedKind_is_satisfied_by_matching_evidence() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + var matching = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass); // Kind "TestResult" + + var result = VerificationAggregator.Aggregate( + [matching], [Check("tests", "TestResult")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + + Assert.Equal(TaskVerificationStatus.Verified, result.Outcome.Status); + Assert.Same(matching, Assert.Single(result.Outcome.Evidence)); + } + + [Fact] + public void An_ExpectedKind_match_is_ordinal_and_case_sensitive() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + var wrongCase = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass) with { Kind = "testresult" }; + + var result = VerificationAggregator.Aggregate( + [wrongCase], [Check("tests", "TestResult")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + + Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); + } + + [Fact] + public void A_null_ExpectedKind_accepts_evidence_of_any_kind() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + var approval = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass) with { Kind = "HumanApproval" }; + + var result = VerificationAggregator.Aggregate( + [approval], [Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + + Assert.Equal(TaskVerificationStatus.Verified, result.Outcome.Status); + } + + [Fact] + public void Evidence_of_the_wrong_kind_cannot_hide_a_Fail_recorded_by_the_expected_kind() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + var failingTest = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Fail); + var approvalPass = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass) with { Kind = "HumanApproval" }; + + var result = VerificationAggregator.Aggregate( + [failingTest, approvalPass], [Check("tests", "TestResult")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + + Assert.Equal(TaskVerificationStatus.Failed, result.Outcome.Status); + Assert.Same(failingTest, Assert.Single(result.Outcome.Evidence)); + } + + [Fact] + public void A_malformed_required_check_throws_rather_than_being_tolerated() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + + Assert.Throws(() => + VerificationAggregator.Aggregate([], [null!], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + Assert.Throws(() => + VerificationAggregator.Aggregate([], [Check(" ")], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + Assert.Throws(() => + VerificationAggregator.Aggregate([], [Check("tests", " ")], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + } + + [Fact] + public void Duplicate_check_ids_throw_even_when_their_expected_kinds_differ() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + + Assert.Throws(() => VerificationAggregator.Aggregate( + [], [Check("tests", "TestResult"), Check("tests", "HumanApproval")], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + } + [Fact] public void AC6_cancellation_propagates_as_an_exception_rather_than_returning_any_VerificationResult() { @@ -311,7 +404,7 @@ public void AC6_cancellation_propagates_as_an_exception_rather_than_returning_an cts.Cancel(); Assert.Throws(() => - VerificationAggregator.Aggregate([], ["tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow, cts.Token)); + VerificationAggregator.Aggregate([], [Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow, cts.Token)); } [Fact] @@ -323,6 +416,6 @@ public void AC6_cancellation_requested_mid_aggregation_still_propagates_rather_t cts.Cancel(); Assert.Throws(() => - VerificationAggregator.Aggregate(evidence, ["build", "tests", "lint"], round, round.ArtifactRevision, DateTimeOffset.UtcNow, cts.Token)); + VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests"), Check("lint")], round, round.ArtifactRevision, DateTimeOffset.UtcNow, cts.Token)); } } diff --git a/tests/AgentExperience.Core.Tests/packages.lock.json b/tests/AgentExperience.Core.Tests/packages.lock.json index fb14393..6d950d6 100644 --- a/tests/AgentExperience.Core.Tests/packages.lock.json +++ b/tests/AgentExperience.Core.Tests/packages.lock.json @@ -12,6 +12,15 @@ "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.11" } }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Direct", + "requested": "[10.0.11, )", + "resolved": "10.0.11", + "contentHash": "PSmotV19c7E3lKed++uYo1kSiXFI+uTl37CBSrhq+CfLC3FCHjG7R91+xPnNehQfHS1b0Tzo/CCLPWH3qaEheg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11" + } + }, "Microsoft.NET.Test.Sdk": { "type": "Direct", "requested": "[17.14.1, )", @@ -181,7 +190,8 @@ "type": "Project", "dependencies": { "AgentExperience.Abstractions": "[1.0.0, )", - "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )" + "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]" } } } diff --git a/tests/AgentExperience.MicrosoftAgentFramework.Tests/ExperienceFinalizationWiringTests.cs b/tests/AgentExperience.MicrosoftAgentFramework.Tests/ExperienceFinalizationWiringTests.cs new file mode 100644 index 0000000..b9ddbec --- /dev/null +++ b/tests/AgentExperience.MicrosoftAgentFramework.Tests/ExperienceFinalizationWiringTests.cs @@ -0,0 +1,431 @@ +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Lifecycle; +using AgentExperience.Core.Reflections; +using AgentExperience.Core.Verification; +using AgentExperience.MicrosoftAgentFramework; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace AgentExperience.MicrosoftAgentFramework.Tests; + +/// +/// Story 2.5: the MAF adapter's completed run reaches Core's finalization service. The agent runs +/// for real against the scripted fake model; capture, reflection, and lifecycle are the real +/// implementations, and only the record store is in-memory. Finalization never changes what the +/// caller of the agent observes, and a finalization problem is a reported capture failure, never an +/// exception. +/// +public class ExperienceFinalizationWiringTests +{ + private const string ArtifactRevision = "rev-1"; + + private static readonly Scope TestScope = new("tenant-1", "app-1", "project-1"); + private static readonly AuthorizationContext Authorization = new("tenant-1", "host", ["experience:write"], DateTimeOffset.UnixEpoch); + private static readonly ClosedVerificationRound Round = new(Guid.Parse("22222222-2222-2222-2222-222222222222"), ArtifactRevision); + + private static readonly SanitizationOptions Sanitization = new(new Dictionary(StringComparer.Ordinal) + { + ["ToolResult"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "value" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 2, + MaxFieldCount: 5, + MaxValueLength: 10_000, + MaxFieldNameLength: 100), + }); + + private static Evidence PassingEvidence(CheckResult result = CheckResult.Pass) => new( + EvidenceId: Guid.NewGuid(), + VerificationRoundId: Round.RoundId, + ArtifactRevision: ArtifactRevision, + CheckId: "tests", + Kind: "TestResult", + Result: result, + Producer: "ci", + Detail: null, + CapturedAt: DateTimeOffset.UnixEpoch); + + [Fact] + public async Task A_completed_run_is_finalized_into_a_durable_Validated_record() + { + var harness = new Harness(); + var agent = harness.Capture(new ScriptedChatClient()); + + var response = await agent.RunAsync("task-finalize"); + + Assert.Equal("Hello, world", response.Text); + + var result = Assert.Single(harness.Finalized); + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(1, result.Revision); + Assert.Empty(harness.Failures); + + // The record is the one this invocation's run produced. + var runId = Assert.Single(harness.Service.StartedRunIds); + Assert.Equal(ExperienceFinalizationService.ExperienceIdFor(runId), result.ExperienceId); + Assert.Equal(runId, result.Record!.SourceRunId); + Assert.Equal(ExperienceStatus.Validated, harness.Store.StatusOf(result.ExperienceId!.Value)); + } + + [Fact] + public async Task A_resolver_that_returns_null_skips_finalizing_that_run_without_reporting_a_failure() + { + var harness = new Harness { Resolve = _ => null }; + + await harness.Capture(new ScriptedChatClient()).RunAsync("task-skip"); + + Assert.Empty(harness.Finalized); + Assert.Empty(harness.Failures); + Assert.Empty(harness.Store.Records); + } + + [Fact] + public async Task A_host_storage_denial_is_an_outcome_not_a_capture_failure() + { + var harness = new Harness { Decision = StorageDecision.Deny("host policy") }; + + var response = await harness.Capture(new ScriptedChatClient()).RunAsync("task-denied"); + + // The agent's own result is untouched. + Assert.Equal("Hello, world", response.Text); + + var result = Assert.Single(harness.Finalized); + Assert.Equal(FinalizationOutcome.StorageDenied, result.Outcome); + Assert.Empty(harness.Store.Records); + + // The host decided this on purpose. Reporting it through the capture-failure channel would give + // a host whose policy denies most runs one "failure" per invocation, mixed in with real defects. + Assert.Empty(harness.Failures); + } + + [Fact] + public async Task A_finalization_that_actually_failed_is_reported_as_a_capture_failure_and_never_thrown() + { + var harness = new Harness(); + harness.Store.ThrowOnCreate = true; + + var response = await harness.Capture(new ScriptedChatClient()).RunAsync("task-store-down"); + + Assert.Equal("Hello, world", response.Text); + + var result = Assert.Single(harness.Finalized); + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + + var failure = Assert.Single(harness.Failures); + Assert.Equal(ExperienceCaptureFailureStage.Finalization, failure.Stage); + Assert.Contains("Failed", failure.Reason, StringComparison.Ordinal); + } + + [Fact] + public async Task A_resolver_that_returns_a_request_for_another_run_is_refused() + { + var foreignRunId = Guid.NewGuid(); + var harness = new Harness { Resolve = _ => ForeignRequest(foreignRunId) }; + + await harness.Capture(new ScriptedChatClient()).RunAsync("task-foreign-run"); + + // An unrelated captured run must never be finalized on this invocation's behalf. + Assert.Empty(harness.Finalized); + Assert.Empty(harness.Store.Records); + var failure = Assert.Single(harness.Failures); + Assert.Equal(ExperienceCaptureFailureStage.Finalization, failure.Stage); + Assert.Contains("different run", failure.Reason, StringComparison.Ordinal); + } + + [Fact] + public async Task Exceptions_thrown_by_the_finalized_callback_are_swallowed() + { + var harness = new Harness { ThrowFromOnRunFinalized = true }; + + var response = await harness.Capture(new ScriptedChatClient()).RunAsync("task-callback-throws"); + + Assert.Equal("Hello, world", response.Text); + Assert.Equal(FinalizationOutcome.Validated, Assert.Single(harness.Finalized).Outcome); + Assert.Empty(harness.Failures); + } + + [Fact] + public async Task A_throwing_resolver_reports_a_capture_failure_and_leaves_the_invocation_alone() + { + var harness = new Harness { Resolve = _ => throw new InvalidOperationException("resolver failed") }; + + var response = await harness.Capture(new ScriptedChatClient()).RunAsync("task-resolver-throws"); + + Assert.Equal("Hello, world", response.Text); + Assert.Empty(harness.Finalized); + var failure = Assert.Single(harness.Failures); + Assert.Equal(ExperienceCaptureFailureStage.Finalization, failure.Stage); + Assert.IsType(failure.Exception); + } + + [Fact] + public async Task A_run_whose_capture_failed_is_never_finalized() + { + var harness = new Harness(); + harness.Service.ForcedCompleteOutcome = CompleteRunOutcome.Conflict; + + await harness.Capture(new ScriptedChatClient()).RunAsync("task-capture-failed"); + + // A half-captured run must not be persisted as if it were whole. + Assert.Empty(harness.Finalized); + Assert.Empty(harness.Store.Records); + Assert.Equal(ExperienceCaptureFailureStage.Finalize, Assert.Single(harness.Failures).Stage); + } + + [Fact] + public void Half_configured_finalization_is_rejected_at_wiring_time_in_both_directions() + { + var harness = new Harness(); + + // A service with no resolver can never build a request... + var serviceOnly = new ExperienceCaptureOptions + { + ResolveRun = _ => new ExperienceRunDescriptor("task", TestScope), + FinalizationService = harness.Finalization, + ResolveFinalization = null, + }; + + // ...and a resolver with no service -- the easier mistake -- would silently never finalize. + var resolverOnly = new ExperienceCaptureOptions + { + ResolveRun = _ => new ExperienceRunDescriptor("task", TestScope), + FinalizationService = null, + ResolveFinalization = _ => null, + }; + + foreach (var options in new[] { serviceOnly, resolverOnly }) + { + var exception = Assert.Throws(() => + new ScriptedAgent().AsBuilder().UseExperienceCapture(harness.Service, options).Build()); + + Assert.Contains(nameof(ExperienceCaptureOptions.ResolveFinalization), exception.Message, StringComparison.Ordinal); + Assert.Contains(nameof(ExperienceCaptureOptions.FinalizationService), exception.Message, StringComparison.Ordinal); + } + } + + [Fact] + public async Task A_failed_invocation_is_still_finalized_and_quarantined() + { + var harness = new Harness { Evidence = () => [PassingEvidence(CheckResult.Fail)] }; + var client = new ScriptedChatClient { Throw = true }; + + await Assert.ThrowsAsync(() => harness.Capture(client).RunAsync("task-failed")); + + var result = Assert.Single(harness.Finalized); + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Null(result.Record!.Reflection); + Assert.Equal(ExperienceStatus.Quarantined, harness.Store.StatusOf(result.ExperienceId!.Value)); + } + + /// A well-formed request that simply names some other captured run. + private static FinalizeExperienceRequest ForeignRequest(Guid runId) => new( + RunId: runId, + Authorization: Authorization, + ClosedRound: Round, + RequiredChecks: [new RequiredCheck("tests", "TestResult")], + Evidence: [PassingEvidence()], + CurrentArtifactRevision: ArtifactRevision, + StorageDecision: StorageDecision.Permit, + FinalizedAt: DateTimeOffset.UnixEpoch.AddDays(1)); + + private sealed class Harness + { + private readonly List _failures = []; + private readonly List _finalized = []; + + public Harness() + { + Service = new RecordingCaptureService(new InMemoryExperienceCaptureService( + new DefaultSanitizer(Sanitization), + new CaptureLimits(10, 50, 10_000, 10_000))); + Store = new InMemoryRecordStore(); + Finalization = new ExperienceFinalizationService( + Service, + new DefaultExperienceReflector(), + Store, + new ExperienceLifecycleService(Store)); + } + + public RecordingCaptureService Service { get; } + + public InMemoryRecordStore Store { get; } + + public ExperienceFinalizationService Finalization { get; } + + public StorageDecision Decision { get; init; } = StorageDecision.Permit; + + public Func> Evidence { get; init; } = () => [PassingEvidence()]; + + public Func? Resolve { get; init; } + + public bool ThrowFromOnRunFinalized { get; init; } + + public FinalizeExperienceRequest RequestFor(Guid runId) => new( + RunId: runId, + Authorization: Authorization, + ClosedRound: Round, + RequiredChecks: [new RequiredCheck("tests", "TestResult")], + Evidence: Evidence(), + CurrentArtifactRevision: ArtifactRevision, + StorageDecision: Decision, + FinalizedAt: DateTimeOffset.UnixEpoch.AddDays(1)); + + public IReadOnlyList Failures + { + get + { + lock (_failures) + { + return _failures.ToList(); + } + } + } + + public IReadOnlyList Finalized + { + get + { + lock (_finalized) + { + return _finalized.ToList(); + } + } + } + + public AIAgent Capture(ScriptedChatClient client) + { + var inner = new ChatClientAgent(client, new ChatClientAgentOptions()); + return inner.AsBuilder().UseExperienceCapture(Service, Options()).Build(); + } + + private ExperienceCaptureOptions Options() => new() + { + ResolveRun = context => new ExperienceRunDescriptor(context.Messages.Last().Text, TestScope), + CaptureToolCalls = false, + FinalizationService = Finalization, + ResolveFinalization = Resolve ?? (context => RequestFor(context.Run.RunId)), + OnRunFinalized = result => + { + lock (_finalized) + { + _finalized.Add(result); + } + + if (ThrowFromOnRunFinalized) + { + throw new InvalidOperationException("host finalization callback failure"); + } + }, + OnCaptureFailure = failure => + { + lock (_failures) + { + _failures.Add(failure); + } + }, + }; + } + + /// + /// A minimal in-memory : enough of the port's contract for the + /// adapter wiring to be exercised end to end without a database. Query and history are not part of + /// finalization and fail loudly if they are ever called. + /// + private sealed class InMemoryRecordStore : IExperienceRecordStore + { + private readonly Dictionary _records = []; + private readonly HashSet _events = []; + + public IReadOnlyDictionary Records + { + get + { + lock (_records) + { + return _records.ToDictionary(); + } + } + } + + public ExperienceStatus StatusOf(Guid experienceId) + { + lock (_records) + { + return _records[experienceId].Status; + } + } + + public bool ThrowOnCreate { get; set; } + + public Task CreateAsync(AuthorizationContext authorization, ExperienceRecord record, CancellationToken cancellationToken) + { + if (ThrowOnCreate) + { + throw new ExperienceStoreException("database unavailable"); + } + + lock (_records) + { + if (_records.ContainsKey(record.ExperienceId)) + { + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Conflict, [])); + } + + if (!authorization.Permits(record.Scope)) + { + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Denied, [])); + } + + _records[record.ExperienceId] = record; + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Created, [])); + } + } + + public Task GetAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) + { + lock (_records) + { + return Task.FromResult(_records.TryGetValue(experienceId, out var record) && record.Scope == scope + ? new ExperienceRecordGetResult(ExperienceStoreOutcome.Found, record, []) + : new ExperienceRecordGetResult(ExperienceStoreOutcome.NotFound, null, [])); + } + } + + public Task CommitLifecycleEventAsync( + AuthorizationContext authorization, + Scope scope, + LifecycleEvent lifecycleEvent, + CancellationToken cancellationToken) + { + lock (_records) + { + if (!_records.TryGetValue(lifecycleEvent.ExperienceRecordId, out var record) || record.Scope != scope) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.NotFound, 0, null, [])); + } + + if (!_events.Add(lifecycleEvent.EventId)) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Committed, record.Revision, null, [])); + } + + var applied = lifecycleEvent.ExpectedRevision + 1; + _records[record.ExperienceId] = record with + { + Status = lifecycleEvent.CurrentStatus, + Revision = applied, + UpdatedAt = lifecycleEvent.OccurredAt, + }; + + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Committed, applied, null, [])); + } + } + + public Task QueryAsync(AuthorizationContext authorization, ExperienceRecordQuery query, CancellationToken cancellationToken) => + throw new InvalidOperationException("Finalization must not query records."); + + public Task GetHistoryAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) => + throw new InvalidOperationException("Finalization must not read history."); + } +} diff --git a/tests/AgentExperience.MicrosoftAgentFramework.Tests/packages.lock.json b/tests/AgentExperience.MicrosoftAgentFramework.Tests/packages.lock.json index afa1bb7..f68aae2 100644 --- a/tests/AgentExperience.MicrosoftAgentFramework.Tests/packages.lock.json +++ b/tests/AgentExperience.MicrosoftAgentFramework.Tests/packages.lock.json @@ -277,7 +277,8 @@ "type": "Project", "dependencies": { "AgentExperience.Abstractions": "[1.0.0, )", - "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )" + "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]" } }, "agentexperience.microsoftagentframework": { diff --git a/tests/AgentExperience.Storage.Postgres.Tests/AgentExperience.Storage.Postgres.Tests.csproj b/tests/AgentExperience.Storage.Postgres.Tests/AgentExperience.Storage.Postgres.Tests.csproj index 465eb1a..466c280 100644 --- a/tests/AgentExperience.Storage.Postgres.Tests/AgentExperience.Storage.Postgres.Tests.csproj +++ b/tests/AgentExperience.Storage.Postgres.Tests/AgentExperience.Storage.Postgres.Tests.csproj @@ -11,6 +11,9 @@ + + @@ -18,6 +21,12 @@ + + + + + + @@ -32,6 +41,10 @@ + + diff --git a/tests/AgentExperience.Storage.Postgres.Tests/DependencyBoundaryTests.cs b/tests/AgentExperience.Storage.Postgres.Tests/DependencyBoundaryTests.cs index 8623590..28481a6 100644 --- a/tests/AgentExperience.Storage.Postgres.Tests/DependencyBoundaryTests.cs +++ b/tests/AgentExperience.Storage.Postgres.Tests/DependencyBoundaryTests.cs @@ -4,9 +4,11 @@ namespace AgentExperience.Storage.Postgres.Tests; /// -/// Proves AgentExperience.Storage.Postgres uses plain Npgsql plus DbUp for schema migrations and -/// nothing else: no MAF, EF Core, Dapper, Pgvector, or model-provider dependency, in either its compiled -/// references or its csproj. +/// Proves AgentExperience.Storage.Postgres uses plain Npgsql, DbUp for schema migrations, and +/// the dependency-injection abstractions its own AddAgentExperiencePostgresStore +/// extension needs -- and nothing else: no MAF, EF Core, Dapper, Pgvector, or model-provider +/// dependency, in either its compiled references or its csproj. The DI package is abstractions only +/// (no container, no hosting), so the adapter still imposes no composition root on a host. /// public class DependencyBoundaryTests { @@ -42,7 +44,7 @@ public void Storage_Postgres_does_not_reference_a_forbidden_assembly() } [Fact] - public void Storage_Postgres_csproj_declares_only_the_exact_Npgsql_and_DbUp_pins() + public void Storage_Postgres_csproj_declares_only_the_exact_Npgsql_DbUp_and_DI_abstractions_pins() { var csprojPath = GetCsprojPath(); Assert.True(File.Exists(csprojPath), $"Could not locate AgentExperience.Storage.Postgres.csproj at '{csprojPath}'."); @@ -54,7 +56,12 @@ public void Storage_Postgres_csproj_declares_only_the_exact_Npgsql_and_DbUp_pins .ToList(); Assert.Equal( - ["Npgsql [10.0.3]", "dbup-core [6.1.1]", "dbup-postgresql [7.0.1]"], + [ + "Microsoft.Extensions.DependencyInjection.Abstractions [10.0.11]", + "Npgsql [10.0.3]", + "dbup-core [6.1.1]", + "dbup-postgresql [7.0.1]", + ], packages); } diff --git a/tests/AgentExperience.Storage.Postgres.Tests/PostgresFinalizationTests.cs b/tests/AgentExperience.Storage.Postgres.Tests/PostgresFinalizationTests.cs new file mode 100644 index 0000000..99655e6 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Tests/PostgresFinalizationTests.cs @@ -0,0 +1,322 @@ +using static AgentExperience.Storage.Postgres.Tests.TestRecords; + +namespace AgentExperience.Storage.Postgres.Tests; + +/// +/// Story 2.5 end to end against a real PostgreSQL 16 container: capture a run, finalize it through +/// Core's , and read the durable record and its lifecycle +/// history back through the real store. Nothing here is faked below the service under test -- the +/// sanitizer, the capture service, the reflector, the lifecycle service, and the PostgreSQL store are +/// all the shipping implementations. Each test uses its own random tenant, so tests sharing the +/// container never see each other's rows. +/// +[Collection(PostgresCollection.Name)] +public sealed class PostgresFinalizationTests +{ + private const string ArtifactRevision = "rev-1"; + + private static readonly ClosedVerificationRound Round = new(Guid.Parse("33333333-3333-3333-3333-333333333333"), ArtifactRevision); + + private static readonly SanitizationOptions Sanitization = new(new Dictionary(StringComparer.Ordinal) + { + ["ToolArguments"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "query" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 3, + MaxFieldCount: 10, + MaxValueLength: 10_000, + MaxFieldNameLength: 100), + ["ToolResult"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "value" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 2, + MaxFieldCount: 5, + MaxValueLength: 10_000, + MaxFieldNameLength: 100), + }); + + private readonly PostgresExperienceRecordStore _store; + private readonly InMemoryExperienceCaptureService _capture; + private readonly ExperienceFinalizationService _finalization; + + public PostgresFinalizationTests(PostgresFixture fixture) + { + _store = new PostgresExperienceRecordStore(fixture.DataSource); + _capture = new InMemoryExperienceCaptureService( + new DefaultSanitizer(Sanitization), + new CaptureLimits(MaxAttemptsPerRun: 10, MaxToolCallsPerAttempt: 50, MaxResultLength: 10_000, MaxErrorLength: 10_000)); + _finalization = new ExperienceFinalizationService( + _capture, + new DefaultExperienceReflector(), + _store, + new ExperienceLifecycleService(_store)); + } + + [Fact] + public async Task A_captured_run_finalizes_into_a_durable_record_readable_with_its_history() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + var result = await _finalization.FinalizeAsync(Request(runId, auth), CancellationToken.None); + + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(1, result.Revision); + + // Read the record back through the real store. + var read = await _store.GetAsync(auth, scope, result.ExperienceId!.Value, CancellationToken.None); + Assert.Equal(ExperienceStoreOutcome.Found, read.Outcome); + var stored = read.Record!; + + Assert.Equal(ExperienceStatus.Validated, stored.Status); + Assert.Equal(1, stored.Revision); + Assert.Equal(runId, stored.SourceRunId); + Assert.Equal("task-1", stored.TaskId); + Assert.Equal(2d / 3d, stored.ReuseConfidence, precision: 12); + Assert.Equal(1, stored.SupportingValidations); + Assert.Equal(0, stored.Contradictions); + Assert.Equal(TaskVerificationStatus.Verified, stored.Outcome.Status); + Assert.Equal(1.0, stored.CompletionScore); + + // The reflection round-tripped whole, and is traceable to the evidence it was derived from. + var reflection = Assert.IsType(stored.Reflection); + Assert.Equal(runId, reflection.ExperienceRunId); + Assert.Equal(ExperienceFinalizationService.ReflectionIdFor(runId), reflection.ReflectionId); + Assert.Equal(TaskVerificationStatus.Verified, reflection.VerificationStatus); + Assert.Equal(Assert.Single(stored.Outcome.Evidence).EvidenceId, Assert.Single(reflection.EvidenceIds)); + + // The captured attempt and its tool call came through unchanged. + var attempt = Assert.Single(stored.Attempts); + Assert.Equal("done", attempt.Result); + Assert.Equal("search", Assert.Single(attempt.ToolCalls).ToolName); + + // And so did the lifecycle history: exactly one initial event. + var history = await _store.GetHistoryAsync(auth, scope, stored.ExperienceId, CancellationToken.None); + Assert.Equal(ExperienceStoreOutcome.Found, history.Outcome); + Assert.Equal(1, history.Revision); + var initial = Assert.Single(history.Events); + Assert.Equal(ExperienceFinalizationService.InitialEventIdFor(runId), initial.EventId); + Assert.Equal(ExperienceStatus.Candidate, initial.PriorStatus); // the record was created as a Candidate + Assert.Equal(ExperienceStatus.Validated, initial.CurrentStatus); + Assert.Equal(0, initial.ExpectedRevision); + Assert.Equal(ExperienceFinalizationService.ProducerIdentity, initial.Producer); + } + + [Fact] + public async Task Finalizing_the_same_run_twice_leaves_one_record_at_revision_one_with_one_event() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + var first = await _finalization.FinalizeAsync(Request(runId, auth), CancellationToken.None); + var second = await _finalization.FinalizeAsync( + Request(runId, auth) with { FinalizedAt = DateTimeOffset.UtcNow.AddMinutes(10) }, + CancellationToken.None); + + Assert.Equal(FinalizationOutcome.Validated, first.Outcome); + Assert.Equal(FinalizationOutcome.AlreadyFinalized, second.Outcome); + Assert.Equal(first.ExperienceId, second.ExperienceId); + Assert.Equal(ExperienceStatus.Validated, second.Status); + Assert.Equal(1, second.Revision); + + var records = await _store.QueryAsync(auth, new ExperienceRecordQuery(scope), CancellationToken.None); + Assert.Single(records.Records); + + var history = await _store.GetHistoryAsync(auth, scope, first.ExperienceId!.Value, CancellationToken.None); + Assert.Equal(1, history.Revision); + Assert.Single(history.Events); + } + + [Fact] + public async Task A_denied_storage_decision_leaves_no_record_and_no_event_for_the_run() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + var result = await _finalization.FinalizeAsync( + Request(runId, auth) with { StorageDecision = StorageDecision.Deny("host retention policy") }, + CancellationToken.None); + + Assert.Equal(FinalizationOutcome.StorageDenied, result.Outcome); + Assert.Null(result.ExperienceId); + Assert.Equal("host retention policy", result.Reason); + + var records = await _store.QueryAsync(auth, new ExperienceRecordQuery(scope), CancellationToken.None); + Assert.Empty(records.Records); + + var history = await _store.GetHistoryAsync(auth, scope, ExperienceFinalizationService.ExperienceIdFor(runId), CancellationToken.None); + Assert.Equal(ExperienceStoreOutcome.NotFound, history.Outcome); + } + + [Fact] + public async Task An_unverified_run_finalizes_into_a_quarantined_record_with_no_reflection() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + var result = await _finalization.FinalizeAsync( + Request(runId, auth) with { Evidence = [Evidence(CheckResult.Fail)] }, + CancellationToken.None); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + + var stored = (await _store.GetAsync(auth, scope, result.ExperienceId!.Value, CancellationToken.None)).Record!; + Assert.Equal(ExperienceStatus.Quarantined, stored.Status); + Assert.Null(stored.Reflection); + Assert.Equal(0d, stored.ReuseConfidence); + Assert.Equal(TaskVerificationStatus.Failed, stored.Outcome.Status); + Assert.Equal(ExperienceStatus.Quarantined, Assert.Single((await _store.GetHistoryAsync(auth, scope, stored.ExperienceId, CancellationToken.None)).Events).CurrentStatus); + } + + [Fact] + public async Task A_store_failure_during_finalization_is_never_reported_as_durable_success() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + await using var unreachable = Unreachable(); + var offline = new ExperienceFinalizationService( + _capture, + new DefaultExperienceReflector(), + new PostgresExperienceRecordStore(unreachable), + new ExperienceLifecycleService(new PostgresExperienceRecordStore(unreachable))); + + var result = await offline.FinalizeAsync(Request(runId, auth), CancellationToken.None); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, result.Stage); + Assert.False(result.IsDurable); + Assert.IsType(result.Failure!.Exception); + + // The captured snapshot is still available for the host to retry with -- and the retry, against + // a reachable database, succeeds. + Assert.True(_capture.TryGetRun(runId, out _)); + var retried = await _finalization.FinalizeAsync(Request(runId, auth), CancellationToken.None); + Assert.Equal(FinalizationOutcome.Validated, retried.Outcome); + } + + [Fact] + public async Task A_record_created_but_never_confirmed_stays_a_Candidate_and_a_retry_completes_its_commit() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + // The create lands against the real database, but the commit cannot: the lifecycle service is + // pointed at an unreachable one. + await using var unreachable = Unreachable(); + var halfway = new ExperienceFinalizationService( + _capture, + new DefaultExperienceReflector(), + _store, + new ExperienceLifecycleService(new PostgresExperienceRecordStore(unreachable))); + + var interrupted = await halfway.FinalizeAsync(Request(runId, auth), CancellationToken.None); + Assert.Equal(FinalizationOutcome.Failed, interrupted.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, interrupted.Stage); + Assert.False(interrupted.IsDurable); + + // What is stored is a Candidate at revision 0 with no history -- never a reusable record. + var experienceId = ExperienceFinalizationService.ExperienceIdFor(runId); + var unconfirmed = (await _store.GetAsync(auth, scope, experienceId, CancellationToken.None)).Record!; + Assert.Equal(ExperienceStatus.Candidate, unconfirmed.Status); + Assert.Equal(0, unconfirmed.Revision); + Assert.Empty((await _store.GetHistoryAsync(auth, scope, experienceId, CancellationToken.None)).Events); + + // The retry re-derives the very same initial event -- including its OccurredAt, which has been + // through PostgreSQL's microsecond truncation on the way back out -- and finishes that commit. + var retried = await _finalization.FinalizeAsync( + Request(runId, auth) with { FinalizedAt = ColumnTime.AddMinutes(30) }, + CancellationToken.None); + + Assert.Equal(FinalizationOutcome.Validated, retried.Outcome); + Assert.Equal(1, retried.Revision); + + var confirmed = (await _store.GetAsync(auth, scope, experienceId, CancellationToken.None)).Record!; + Assert.Equal(ExperienceStatus.Validated, confirmed.Status); + Assert.Equal(1, confirmed.Revision); + Assert.Equal(unconfirmed.CreatedAt, confirmed.CreatedAt); // the first call's timestamp, not the retry's + + var only = Assert.Single((await _store.GetHistoryAsync(auth, scope, experienceId, CancellationToken.None)).Events); + Assert.Equal(ExperienceFinalizationService.InitialEventIdFor(runId), only.EventId); + Assert.Equal(ExperienceStatus.Candidate, only.PriorStatus); + Assert.Equal(ExperienceStatus.Validated, only.CurrentStatus); + Assert.Equal(unconfirmed.CreatedAt, only.OccurredAt); + + // And finalizing once more is now the plain already-finalized replay. + var again = await _finalization.FinalizeAsync(Request(runId, auth), CancellationToken.None); + Assert.Equal(FinalizationOutcome.AlreadyFinalized, again.Outcome); + Assert.Single((await _store.GetHistoryAsync(auth, scope, experienceId, CancellationToken.None)).Events); + } + + private static Evidence Evidence(CheckResult result) => new( + EvidenceId: Guid.NewGuid(), + VerificationRoundId: Round.RoundId, + ArtifactRevision: ArtifactRevision, + CheckId: "unit-tests-pass", + Kind: "TestResult", + Result: result, + Producer: "ci", + Detail: "42 of 42 passed", + CapturedAt: PayloadTime); + + private static FinalizeExperienceRequest Request(Guid runId, AuthorizationContext auth) => new( + RunId: runId, + Authorization: auth, + ClosedRound: Round, + RequiredChecks: [new RequiredCheck("unit-tests-pass", "TestResult")], + Evidence: [Evidence(CheckResult.Pass)], + CurrentArtifactRevision: ArtifactRevision, + StorageDecision: StorageDecision.Permit, + FinalizedAt: ColumnTime); + + private async Task CaptureRunAsync(Scope scope) + { + var runId = Guid.NewGuid(); + var started = _capture.StartRun( + runId, + taskId: "task-1", + taskDescription: "resolve the ticket", + scope: scope, + environment: new EnvironmentFingerprint("worker-01", "net10.0", "linux-x64", "1.2.3", new Dictionary { ["region"] = "us-east" }), + provenance: new Provenance("integration-tests", "1.0.0", PayloadTime, "trace-1"), + startedAt: PayloadTime); + Assert.Equal(StartRunOutcome.Started, started.Outcome); + + var appended = await _capture.AppendAttemptAsync(runId, new AppendAttemptRequest( + AttemptId: Guid.NewGuid(), + StartedAt: PayloadTime, + Duration: TimeSpan.FromSeconds(2), + ToolCalls: + [ + new RawToolCall( + ToolCallId: Guid.NewGuid(), + ToolName: "search", + Arguments: new Dictionary { ["query"] = "refund policy" }, + StartedAt: PayloadTime, + Duration: TimeSpan.FromMilliseconds(120), + Result: "3 documents", + Error: null), + ], + Result: "done", + Error: null)); + Assert.Equal(AppendAttemptOutcome.Recorded, appended.Outcome); + + var completed = await _capture.CompleteRunAsync(runId, Guid.NewGuid(), RunExecutionStatus.Completed, PayloadTime.AddMinutes(1)); + Assert.Equal(CompleteRunOutcome.Recorded, completed.Outcome); + + return runId; + } +} diff --git a/tests/AgentExperience.Storage.Postgres.Tests/PostgresServiceRegistrationTests.cs b/tests/AgentExperience.Storage.Postgres.Tests/PostgresServiceRegistrationTests.cs new file mode 100644 index 0000000..ff4d605 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Tests/PostgresServiceRegistrationTests.cs @@ -0,0 +1,68 @@ +using AgentExperience.Storage.Postgres.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; +using Npgsql; + +namespace AgentExperience.Storage.Postgres.Tests; + +/// +/// Resolves what +/// +/// registers out of a real container, so deleting the registration fails here rather than only at a +/// host's startup. No database is touched: registering a store does not open a connection. +/// +public class PostgresServiceRegistrationTests +{ + [Fact] + public void The_store_is_resolved_from_a_data_source_in_the_container() + { + using var dataSource = TestRecords.Unreachable(); + + var services = new ServiceCollection(); + services.AddSingleton(dataSource); + services.AddAgentExperiencePostgresStore(); + + using var provider = services.BuildServiceProvider(); + + var store = provider.GetRequiredService(); + Assert.IsType(store); + Assert.Same(store, provider.GetRequiredService()); // singleton + } + + [Fact] + public void The_overload_taking_a_data_source_needs_nothing_else_in_the_container() + { + using var dataSource = TestRecords.Unreachable(); + + var services = new ServiceCollection(); + services.AddAgentExperiencePostgresStore(dataSource); + + using var provider = services.BuildServiceProvider(); + + Assert.IsType(provider.GetRequiredService()); + } + + [Fact] + public void A_host_store_registered_first_wins() + { + using var dataSource = TestRecords.Unreachable(); + var hostStore = new PostgresExperienceRecordStore(dataSource); + + var services = new ServiceCollection(); + services.AddSingleton(hostStore); + services.AddAgentExperiencePostgresStore(dataSource); + + using var provider = services.BuildServiceProvider(); + + Assert.Same(hostStore, provider.GetRequiredService()); + } + + [Fact] + public void Null_arguments_throw() + { + using var dataSource = TestRecords.Unreachable(); + + Assert.Throws(() => ((IServiceCollection)null!).AddAgentExperiencePostgresStore()); + Assert.Throws(() => ((IServiceCollection)null!).AddAgentExperiencePostgresStore(dataSource)); + Assert.Throws(() => new ServiceCollection().AddAgentExperiencePostgresStore((NpgsqlDataSource)null!)); + } +} diff --git a/tests/AgentExperience.Storage.Postgres.Tests/packages.lock.json b/tests/AgentExperience.Storage.Postgres.Tests/packages.lock.json index 414661c..ea1283f 100644 --- a/tests/AgentExperience.Storage.Postgres.Tests/packages.lock.json +++ b/tests/AgentExperience.Storage.Postgres.Tests/packages.lock.json @@ -2,6 +2,15 @@ "version": 1, "dependencies": { "net10.0": { + "Microsoft.Extensions.DependencyInjection": { + "type": "Direct", + "requested": "[10.0.11, )", + "resolved": "10.0.11", + "contentHash": "PSmotV19c7E3lKed++uYo1kSiXFI+uTl37CBSrhq+CfLC3FCHjG7R91+xPnNehQfHS1b0Tzo/CCLPWH3qaEheg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11" + } + }, "Microsoft.NET.Test.Sdk": { "type": "Direct", "requested": "[17.14.1, )", @@ -126,10 +135,54 @@ "resolved": "17.14.1", "contentHash": "pmTrhfFIoplzFVbhVwUquT+77CbGH+h4/3mBpdmIlYtBi9nAB+kKI6dN3A/nV4DFi3wLLx/BlHIPK+MkbQ6Tpg==" }, + "Microsoft.Extensions.Compliance.Abstractions": { + "type": "Transitive", + "resolved": "10.9.0", + "contentHash": "tuSqNuiJxlln43sZ8c1EDA4WXit1eX4foGadylXso3DnMVc+DtKfaNEwvHuiFXfPsEUZ6Z3GnF0Bfk9vvOsE4Q==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.ObjectPool": "10.0.11" + } + }, + "Microsoft.Extensions.Compliance.Redaction": { + "type": "Transitive", + "resolved": "10.9.0", + "contentHash": "2P0WFFq9WAyhOAZqb0FjTeKW86yL4M2vymSGyuBu5XEBWwDCiEI+BoR4TuAFtyFurDRZs5wG3JtysUy8Svlnmw==", + "dependencies": { + "Microsoft.Extensions.Compliance.Abstractions": "10.9.0", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "wlhRqZW8LcJPa+vk2oLAc/REXDItHtkFQdf/QcXYGZbZOO13izcsKY1pCvuFQYwUiZD+hwSZwsKASjqT+BNaVg==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "fVi053xdpda9Em7vSkmgVxO/PtgC2m78ekReKWsgcyskqY0U82Bz/MONwxpGzI0hElYKJfw+fupqMVeKW3fSaA==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration.Binder": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "rFn8RuszZn3qquPVkDytMUlPc2+rXl9MCoygwc1XmAgC5vg5/oXJ8hkOosOrLoBLsqdTy4lFwP6iQdPS9uSYOA==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11" + } + }, "Microsoft.Extensions.DependencyInjection.Abstractions": { "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "L3AdmZ1WOK4XXT5YFPEwyt0ep6l8lGIPs7F5OOBZc77Zqeo01Of7XXICy47628sdVl0v/owxYJTe86DTgFwKCA==" + "resolved": "10.0.11", + "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", @@ -139,6 +192,37 @@ "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0" } }, + "Microsoft.Extensions.ObjectPool": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "p76ztQFROBOlHgdV1vXfmTjRyu073Av7ZlsiLR93ka6+nzkLCeV2ONXq0DO/BGf71REqGW29Uy/20fzQHAjB7Q==" + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "eY1GAKcTfD2maP27J84X9IovT3yjHJ2dVDzPmDg6/XqYvt3jMzJhtfQCLjG9pVsZGAd+8DQ2QrjaDcs2+VQLGw==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Options.ConfigurationExtensions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "syEhXQ/sEaSBFaqzlp9gDGHX/nk6gkQkh1sIUpBO1mlBj3Phu1rmb4ML1uCiyPW9N6Kxfxv3y5FGObC+bV01Qw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.Configuration.Binder": "10.0.11", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Options": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "SXcz+kF+4Oo9b1+55zntpJFYfwb1jw66ioxptyNOOTDc8g2FHnBFWjZpsWfCvZIhzr0x+4e2trVTs4OKwQfBtw==" + }, "Microsoft.TestPlatform.ObjectModel": { "type": "Transitive", "resolved": "17.14.1", @@ -235,10 +319,19 @@ "agentexperience.abstractions": { "type": "Project" }, + "agentexperience.core": { + "type": "Project", + "dependencies": { + "AgentExperience.Abstractions": "[1.0.0, )", + "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]" + } + }, "agentexperience.storage.postgres": { "type": "Project", "dependencies": { "AgentExperience.Abstractions": "[1.0.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]", "Npgsql": "[10.0.3, 10.0.3]", "dbup-core": "[6.1.1, 6.1.1]", "dbup-postgresql": "[7.0.1, 7.0.1]"