diff --git a/AgentExperience.NET.sln b/AgentExperience.NET.sln index 2829344..d39b855 100644 --- a/AgentExperience.NET.sln +++ b/AgentExperience.NET.sln @@ -25,6 +25,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.Storage.Pos EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.Storage.Postgres.Tests", "tests\AgentExperience.Storage.Postgres.Tests\AgentExperience.Storage.Postgres.Tests.csproj", "{86C76642-C4EB-49A8-9E4C-A885644EE49F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.Storage.Postgres.Vectors", "src\AgentExperience.Storage.Postgres.Vectors\AgentExperience.Storage.Postgres.Vectors.csproj", "{9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.Storage.Postgres.Vectors.Tests", "tests\AgentExperience.Storage.Postgres.Vectors.Tests\AgentExperience.Storage.Postgres.Vectors.Tests.csproj", "{6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -143,6 +147,30 @@ Global {86C76642-C4EB-49A8-9E4C-A885644EE49F}.Release|x64.Build.0 = Release|Any CPU {86C76642-C4EB-49A8-9E4C-A885644EE49F}.Release|x86.ActiveCfg = Release|Any CPU {86C76642-C4EB-49A8-9E4C-A885644EE49F}.Release|x86.Build.0 = Release|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Debug|x64.ActiveCfg = Debug|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Debug|x64.Build.0 = Debug|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Debug|x86.ActiveCfg = Debug|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Debug|x86.Build.0 = Debug|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Release|Any CPU.Build.0 = Release|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Release|x64.ActiveCfg = Release|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Release|x64.Build.0 = Release|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Release|x86.ActiveCfg = Release|Any CPU + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E}.Release|x86.Build.0 = Release|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Debug|x64.ActiveCfg = Debug|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Debug|x64.Build.0 = Debug|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Debug|x86.ActiveCfg = Debug|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Debug|x86.Build.0 = Debug|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Release|Any CPU.Build.0 = Release|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Release|x64.ActiveCfg = Release|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Release|x64.Build.0 = Release|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Release|x86.ActiveCfg = Release|Any CPU + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -157,5 +185,7 @@ Global {9B771769-47C4-4EA1-8808-3E4728899A04} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {DFBB8572-3FF3-43F5-8A16-2ADB91C0E25E} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {86C76642-C4EB-49A8-9E4C-A885644EE49F} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {9E836167-D0C5-49CC-B1A7-1F364DAEDA9E} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A} = {0AB3BF05-4346-4AA6-1389-037BE0695223} EndGlobalSection EndGlobal diff --git a/README.md b/README.md index 0ef8ec8..ae26ff5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ AgentExperience.NET captures what an AI agent actually tried, verifies whether it worked, and turns the result into an auditable lesson that future runs can reuse safely. It sits between [Microsoft Agent Framework](https://github.com/microsoft/agent-framework) (MAF) execution and durable storage, without replacing either. -> **Status: early development.** Epic 1 (capture and explain agent experience) is implemented and tested. Epic 2 has started: Experience Records can be stored in PostgreSQL and moved through their lifecycle with atomic, audited commits. Retrieval, injection, and governance are planned (see [Roadmap](#roadmap)). Nothing is published to NuGet yet, and APIs may change. +> **Status: early development.** Epic 1 (capture and explain agent experience) is implemented and tested. Epic 2 has started: a completed run can now be finalized into a durable Experience Record in PostgreSQL in one call, moved through its lifecycle with atomic, audited commits, indexed as an embedding after the fact, and retrieved by task text *and* by meaning with bounded, explainable ranking. Injection and governance are planned (see [Roadmap](#roadmap)). Nothing is published to NuGet yet, and APIs may change. ## Why @@ -33,6 +33,11 @@ AgentExperience.NET records observable evidence (tool calls, results, errors, ve | PostgreSQL Experience Record store: create, get, and scoped query; host authorization checked before database access; exact scope matching in SQL | `AgentExperience.Storage.Postgres` | | Atomic audited lifecycle commits: the event and the record's projection in one transaction, idempotent by event ID, revision-checked, with append-only history | `AgentExperience.Core`, `AgentExperience.Storage.Postgres` | | Journaled schema migrations: embedded scripts applied once, one transaction per script, serialized across processes by an advisory lock | `AgentExperience.Storage.Postgres` | +| One finalization call: evaluate, gate on authorization and the host's storage decision, reflect, create the record as a `Candidate`, commit the initial event that promotes it — replay-safe and structured at every stage | `AgentExperience.Core` | +| Text retrieval of applicable experience: eligibility decided before ranking, every ranking component and effective weight exposed, bounded by a timeout that is never an exception | `AgentExperience.Core`, `AgentExperience.Storage.Postgres` | +| Embedding ingestion after the canonical commit: only the sanitized retrieval summary is embedded, writes are conditional on the live revision, and every provider failure leaves the record committed and retryable | `AgentExperience.Core`, `AgentExperience.Storage.Postgres.Vectors` | +| Hybrid retrieval: a bounded vector channel merged with the text one under the same eligibility, timeout, and ceiling, with an explicit, flagged text-only fallback whenever the vector channel cannot be trusted | `AgentExperience.Core`, `AgentExperience.Storage.Postgres.Vectors` | +| Dependency-injection registration for each package, so a host wires capture, finalization, storage, indexing, and retrieval without knowing concrete types | `AgentExperience.Core`, `AgentExperience.Storage.Postgres`, `AgentExperience.Storage.Postgres.Vectors` | ## Quick look @@ -54,27 +59,344 @@ await agent.RunAsync("Triage ticket #4812", session); See the [adapter README](src/AgentExperience.MicrosoftAgentFramework/README.md) for options, supported agent types, and caveats. See the [PostgreSQL store README](src/AgentExperience.Storage.Postgres/README.md) for the trust boundary, the `ExperienceSchemaMigrator.MigrateAsync` startup call, and data semantics. +## Turning a run into a durable record + +A captured run becomes a durable, reusable Experience Record through one Core call. +`ExperienceFinalizationService.FinalizeAsync` runs six stages in order — load the captured snapshot, evaluate it, +check authorization and the host's storage decision, reflect on it, create the record, commit its initial lifecycle +event — and stops at the first stage that ends the call, always returning a structured result rather than throwing. +The two gates precede reflection on purpose: the reflector is the seam a host would plug a model into, so a run that +is about to be refused is never handed to it. + +```csharp +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Verification; + +var result = await finalization.FinalizeAsync( + new FinalizeExperienceRequest( + RunId: runId, + Authorization: authorization, // host-established; the run's scope must lie inside it + ClosedRound: new ClosedVerificationRound(roundId, "rev-7"), + RequiredChecks: [new RequiredCheck("unit-tests-pass", ExpectedKind: "TestResult")], + Evidence: evidence, // finalization filters and aggregates it itself + CurrentArtifactRevision: "rev-7", + StorageDecision: StorageDecision.Permit, // or StorageDecision.Deny("retention policy") + FinalizedAt: DateTimeOffset.UtcNow), + cancellationToken); + +if (result.IsDurable) +{ + logger.LogInformation("Experience {Id} is {Status} at revision {Revision}", + result.ExperienceId, result.Status, result.Revision); +} +else +{ + logger.LogWarning("Finalization ended at {Stage}: {Outcome} — {Reason}", + result.Stage, result.Outcome, result.Failure?.Reason); +} +``` + +| Outcome | When | What was written | +| --- | --- | --- | +| `Validated` | Verified, reflection succeeded, storage permitted | The record (reuse confidence 2/3, one supporting validation, no contradictions), created as `Candidate`, plus the initial event that moved it to `Validated` | +| `Quarantined` | Storage permitted, but verification did not pass or the reflector threw | The record, with **no** reflection, created as `Candidate`, plus the initial event that moved it to `Quarantined`. `Failure` names the stage that decided it | +| `AlreadyFinalized` | This run's record already exists *and* is already confirmed | Nothing. The result reports the stored record, status, and revision. (A record left unconfirmed by an earlier call is resumed instead: the retry commits its initial event and returns `Validated`/`Quarantined`.) | +| `StorageDenied` | The host's `StorageDecision` denied | Nothing at all, and no record ID is issued | +| `NotAuthorized` | The run's scope lies outside the authorization | Nothing; denied before any store call | +| `RunNotFound` / `RunNotFinished` | No such captured run, or it has no execution status | Nothing | +| `Failed` | A stage failed (for example the database was unavailable) | Never reported as durable. Any record already created stays a `Candidate`, which is never reusable, and the captured run stays available for a retry | + +Three properties make retrying safe. The record is *created* as a `Candidate` and its initial lifecycle event +performs the real transition, so a commit that never lands leaves nothing reusable behind. The record ID, the +reflection ID, and the initial event ID are all derived from the run ID, so a second call cannot create a second +record or a second initial confirmation. And the initial event's fields are a pure function of the stored record, so +a retry re-derives exactly the event the store already deduplicates on. + +Finalization never sanitizes — capture already rejected anything unsafe — and never decides storage or risk policy on +the host's behalf: `StorageDecision` travels in the request and Core simply obeys it. + +If an indexing hook is registered, one more thing happens *after* those six stages: the committed record is embedded +and its vector stored. That step is outside the canonical write and can never change the outcome above — see +[Indexing experience for semantic reuse](#indexing-experience-for-semantic-reuse). + +## Indexing experience for semantic reuse + +A record that is committed is already reusable: it is text-searchable the moment it lands. Indexing gives it a +second way to be found — by meaning — and it is **derived data** throughout. Nothing about the canonical write +depends on an embedding provider being up. + +If an `ExperienceIndexingService` is registered, finalization embeds each record it commits, right after the commit: + +```csharp +var result = await finalization.FinalizeAsync(request, cancellationToken); + +if (result.Indexing is { IsIndexed: false } indexing) +{ + // Never a reason to treat the record as anything less than durable. + logger.LogWarning("Experience {Id} is {Status} but not indexed ({Outcome}, retryable: {Retryable}): {Reason}", + result.ExperienceId, result.Status, indexing.Outcome, indexing.IsRetryable, indexing.Failure?.Reason); +} +``` + +**Only the sanitized retrieval summary is embedded** — the task ID, the sanitized task summary, and the reflection's +lesson, the same three fields the text index analyzes. Attempts, tool calls, evidence, provenance, and environment +metadata are never sent to a provider. The summary is read from the database at index time, not from a record the +caller happens to be holding, so what is embedded is what is really stored, at the revision it is really stored at. + +The two channels read the same *fields* but not necessarily the same *length*: the embedded summary is capped at +8,192 characters (`ExperienceRetrievalSummary.MaxLength`, so the hashed text and the text sent to a provider are +always identical), while `0003` analyzes the concatenation up to 100,000. A record whose summary and lesson together +run past 8 KB is therefore matched on more of its text by words than by meaning. Both caps are far past any +realistic summary. + +**Only records a search could actually return are embedded.** The indexing scan applies the same status filter and +confidence floor the vector search applies, and the post-commit hook checks the record before calling anything, so +a `Quarantined`, `Revoked`, `Superseded`, or `Candidate` record's summary and lesson never leave the database for a +third party — its vector could never be returned anyway. + +Each stored vector carries **model ID, dimension, content hash, and source revision**, kept entirely separate from +lifecycle state. None of them ever influences eligibility, status, or reuse confidence; they exist so a write can be +conditional, a re-index can be free, and a query vector is never compared with something it is not comparable with. + +| Outcome | When | What was written | +| --- | --- | --- | +| `Indexed` | The summary was embedded and stored | The vector and its descriptor | +| `Skipped` | This model already embedded exactly this text | Nothing — and **no provider call was made** | +| `Stale` | The record moved to a newer revision before the write landed | Nothing; the stored vector is unchanged. Retryable | +| `Missing` | The record no longer exists in this scope | Nothing, and **no row is created** — an in-flight write cannot resurrect a deleted record | +| `Ineligible` | The record's status or confidence means a search could never return it | Nothing, and **nothing was sent to a provider** | +| `ProviderFailed` | The provider threw, timed out, or returned a vector of the wrong width or with a non-finite component | Nothing. The record stays committed, durable, and text-searchable. Retryable | +| `IndexFailed` | The index itself failed or refused the write | Nothing. Retryable | +| `Denied` | The scope lies outside the authorization | Nothing was read, embedded, or written | + +**Re-indexing is explicit, scoped, and idempotent.** It never runs on its own: + +```csharp +var pass = await indexing.ReindexAsync( + authorization, + new ReindexExperienceRequest(scope, ExperienceIds: null, Limit: 100), // bounded; pass again to page + cancellationToken); + +logger.LogInformation("{Examined} examined, {Indexed} re-embedded, {Skipped} unchanged, {Failed} failed", + pass.Examined, pass.Indexed, pass.Skipped, pass.Failed); +``` + +A pass is **bounded and resumable**: records are considered in ascending `ExperienceId` order, and `pass.LastExaminedId` +is the cursor to hand to the next pass's `StartAfterId`. Keep going until it comes back `null`, which is how a scope +larger than one page is walked to the end. + +The content hash covers the model ID and the normalized summary, so a record whose vector already came from this +model and this text is skipped **before** any provider call — running a pass twice over unchanged records costs one +read and nothing else. Changing the model looks exactly like changing the text, which is the point: two models +produce incomparable vectors, so "same text" alone must never be enough to skip. + +**The approximate-nearest-neighbour index is created out of band**, because it needs a dimension no shipped +migration can know: + +```csharp +await ExperienceVectorIndexMaintenance.EnsureHnswIndexAsync(dataSource, dimension: 1536, cancellationToken); +``` + +It is optional — every search is correct without it, using an exact scan — it makes search *approximate*, and +building it locks the table for the duration, so run it from a maintenance path. See the +[vectors README](src/AgentExperience.Storage.Postgres.Vectors/README.md) for why the `embedding` column is an +unconstrained `vector` and the index is a partial one over `embedding::vector(n)`. + +## Retrieving applicable experience + +Finding experience that applies to a task is one Core call: `ExperienceRetrievalService.RetrieveAsync`. It asks the +storage adapter for scope-, status- and confidence-filtered text matches — and, when a vector channel is wired in, +for the same thing matched on meaning — decides the remaining eligibility itself, and ranks what survives, always +returning a structured result rather than throwing. + +```csharp +using AgentExperience.Core.Retrieval; + +var result = await retrieval.RetrieveAsync( + new RetrieveExperienceRequest( + Authorization: authorization, // host-established; the request scope must lie inside it + Scope: scope, // the exact scope to retrieve within, never widened + TaskText: "refund ticket stuck on a lock", + RequiredEnvironmentAttributes: new Dictionary { ["region"] = "us-east" }, + CorrelationId: traceId), + cancellationToken); + +if (result.TimedOut) +{ + logger.LogInformation("Retrieval timed out for {CorrelationId}; the agent runs without memory", result.CorrelationId); +} + +foreach (var ranked in result.Records) // highest score first, ties by ExperienceId ascending +{ + logger.LogDebug("{Id} scored {Score} from {Components}", + ranked.Record.ExperienceId, + ranked.Score, + string.Join(", ", ranked.Components.Select(c => $"{c.Kind}={c.Value}*{c.Weight}"))); +} +``` + +**Eligibility is decided before ranking, and nothing is scored before it is known to be reusable.** + +| Check | Where it runs | Effect | +| --- | --- | --- | +| Scope | SQL | Only records in the request's *exact* scope; a foreign scope reveals nothing | +| Status | SQL | Only `Validated` and `Reinforced`. `Candidate`, `Quarantined`, `Contested`, `Stale`, `Superseded`, and `Revoked` are never returned, whatever their text match | +| Reuse confidence | SQL | Below `RetrievalPolicy.MinimumConfidence` (default 0.5) is excluded | +| Text match | SQL | PostgreSQL full-text search over task ID, task summary, and reflection lesson (analyzed up to 100,000 characters) | +| Vector match | SQL | pgvector cosine distance over the embedding of those same three fields (embedded up to 8,192 characters), filtered to the query's own model and dimension | +| Expiry | Core | Last lifecycle activity older than `RetrievalPolicy.MaxAge` is excluded. `null` (the default) means no expiry | +| Environment | Core | Every required attribute must equal the record's `EnvironmentFingerprint.Metadata` entry; a missing key excludes the record. A request with no required attributes sets `EnvironmentUnrestricted` on the result | + +Scope, status, and the confidence floor are pushed into **both** channels as the same predicates, so neither can +return something the other would have filtered out. + +`result.Excluded` itemizes what the **Core** checks removed — expiry and environment — so "nothing matched" is +distinguishable from "something matched but was not reusable here". It is deliberately not a complete account of +everything filtered: scope, status, and the confidence floor are applied in SQL, so records they exclude never reach +Core and are never listed. That split is the point — a foreign-scope or revoked record must not be observable, even +as a count. + +**Two channels, one answer.** When an embedding index and an embedding generator are both registered, the task text +is also embedded and searched as a vector, concurrently with the text search and inside the same timeout. The two +candidate lists are then deduplicated by `ExperienceId`, and a record found by both keeps the **higher** of its two +normalized relevances. Ranking runs once over the merged list, with the same five weights as before: there is no +sixth axis and no "found by both" bonus. An embedding can only make a record a *candidate* — it never decides +eligibility, status, or confidence. + +**A vector channel that cannot be trusted produces an explicit text-only answer, never a failure.** The text +candidates still come back, and `result.TextOnly` is `true` with `result.VectorFallback.Reason` saying which: + +| `TextOnlyReason` | When | Vector comparison attempted? | +| --- | --- | --- | +| `NotConfigured` | No embedding index or no generator is registered — a supported, text-only deployment | No channel exists | +| `ProviderUnavailable` | The provider threw, cancelled for its own reasons (a client-side request timeout), or returned a query vector of the wrong width or with a non-finite component | No — caught before any query is issued | +| `ModelMismatch` | Every embedding stored in this scope came from a different model | No — excluded by the query's own predicate | +| `DimensionMismatch` | Every embedding stored in this scope is a different width | No — excluded by the query's own predicate | +| `VectorSearchFailed` | The vector search threw, was denied, or was refused as malformed | Attempted; nothing usable came back | + +`TextOnly` is never set merely because the vector channel matched nothing: "nothing was semantically similar" and +"the vector channel could not be trusted" are different claims, and only the second one is a reason to look at your +wiring. + +**There is a recall ceiling, and it is visible.** Each channel returns at most `RetrievalPolicy.CandidateLimit` +candidates (default 50), ordered by its own relevance, and ranking only ever sees those. So a record with a weaker +match but strong confidence, recency, or status is not ranked at all once that many stronger matches exist in both +channels: the weighting can only reorder what the ceiling let through. When *either* channel reaches its ceiling, +`result.Truncated` is `true` — the records beyond it are in no exclusion list either, because no eligibility check +ever looked at them. Raise `CandidateLimit` or narrow the task text when that matters. `request.Limit` may not +exceed `CandidateLimit`; a larger value is rejected rather than quietly capped. + +**Ranking is explainable.** Every returned record carries all five normalized components (each in 0–1) and the +effective weight applied to it, so the score is always reproducible from what the result holds. + +| Component | Default weight | Normalized as | +| --- | --- | --- | +| Relevance | 0.35 | `ts_rank_cd` of the text match, or `1 - cosine_distance / 2` of the vector match — whichever is higher for that record — normalized to 0–1 | +| Confidence | 0.25 | The record's `ReuseConfidence` | +| Recency | 0.15 | `2^(-age / RecencyHalfLife)`, half-life 30 days by default. *Age* is measured from `UpdatedAt` | +| Status | 0.15 | `Reinforced` 1.0, `Validated` 0.5 | +| Environment compatibility | 0.10 | 1.0 for a record that satisfied the request's required attributes — which every ranked record did, since a mismatch excludes it before ranking | + +Weights must be finite, non-negative, and sum to 1 (within `RankingWeights.SumTolerance`); anything else throws +`ArgumentOutOfRangeException` at construction, so an invalid weighting can never reach a retrieval call. Ties sort by +`ExperienceId` ascending and ordinal, so the ordering is total and stable, and a golden fixture pins the default +ordering together with every component value. + +**"Recency" and "expiry" mean last lifecycle activity, not when the lesson was learned.** Both read +`ExperienceRecord.UpdatedAt`, which every lifecycle commit bumps. A years-old lesson reinforced yesterday is one day +old by this measure: it scores as fully recent and never expires. That is deliberate — recent revalidation is +evidence the lesson still holds — but it is not a measure of how old the underlying knowledge is, and a policy that +needs one should not use `MaxAge` for it. + +**Bounded, and fail-closed.** The whole call is bounded by `RetrievalPolicy.Timeout` (default 500 ms, maximum one +day), measured with an injected `TimeProvider`. + +| Situation | Outcome | Records | +| --- | --- | --- | +| Ran inside the timeout | `Completed` | Every eligible record among the candidates considered, ranked and cut to the request's limit. Check `result.Truncated`: `true` means more matched than were considered | +| Exceeded the timeout | `TimedOut` (`result.TimedOut`), with the request's `CorrelationId` — never an exception | Empty | +| Request scope outside the authorization | `Denied` | Empty; **neither channel is issued a query, and nothing is embedded** | +| The **text** search failed, or a candidate from either channel could not be read, came back out of scope, or was returned twice | `Failed`, with `result.Failure` | Empty, never unfiltered | +| The **vector** channel failed, timed out on its own, or was incomparable | `Completed`, with `result.TextOnly` and `result.VectorFallback` | The text channel's eligible records, ranked | +| Caller cancelled | `OperationCanceledException`, unwrapped and distinct from the timeout | — | + +`result.Failure.Reason` is content-free and safe to log. `result.Failure.Exception`, when present, is whatever the +port threw — a driver message can quote SQL text or connection detail, so treat it as local diagnostics rather than +something to pass on. + +Retrieval returns ranked records and the evidence for their ranking. Building a labeled Historical Reference payload +and injecting it into an agent is a separate, later step, and retrieved content never becomes authority. + +## Wiring it all together + +Each package registers its own services, so a host never names a concrete type: + +```csharp +using AgentExperience.Core.DependencyInjection; +using AgentExperience.Storage.Postgres.DependencyInjection; +using AgentExperience.Storage.Postgres.Vectors.DependencyInjection; // optional: the vector channel + +services.AddSingleton(NpgsqlDataSource.Create(connectionString)); +services.AddAgentExperiencePostgresStore(); // IExperienceRecordStore +services.AddAgentExperiencePostgresCandidateSource(); // IExperienceCandidateSource +services.AddAgentExperiencePostgresEmbeddingIndex(); // IExperienceEmbeddingIndex +services.AddAgentExperienceEmbeddingGenerator(); // IExperienceEmbeddingGenerator, over a registered + // IEmbeddingGenerator> +services.AddAgentExperienceCore(sanitizationOptions, captureLimits); +// -> ISanitizer, IExperienceCaptureService, IExperienceReflector, +// ExperienceLifecycleService, ExperienceFinalizationService +services.AddAgentExperienceIndexing(); // ExperienceIndexingService, and finalization's + // post-commit hook, in either registration order +services.AddAgentExperienceRetrieval(); // ExperienceRetrievalService +// -> defaults to RetrievalPolicy.Default and RankingWeights.Default; pass your own to override +// -> hybrid, because an index *and* a generator are registered; text-only, and flagged, if either is missing +``` + +Schema comes in two calls, matching that split: + +```csharp +await ExperienceSchemaMigrator.MigrateAsync(dataSource, cancellationToken); // 0001-0003, always +await ExperienceVectorSchemaMigrator.MigrateAsync(dataSource, cancellationToken); // 0004, only with the vector channel +``` + +The vector registrations and the second migration are optional, and genuinely so: leave them out and everything +still works — finalization commits records with no indexing hook, and retrieval answers from text alone with +`TextOnly` set to `NotConfigured`. That is also why the embedding schema is not in the base adapter's script list: +`CREATE EXTENSION vector` needs a superuser, and a text-only deployment must never be made to run it for a feature +it has not enabled. + +`AgentExperience.Abstractions` stays BCL-only; only `Core` and the storage adapter take +`Microsoft.Extensions.DependencyInjection.Abstractions`, and every registration uses `TryAdd`, so a host's own +implementation wins. + +The MAF adapter can drive finalization for you: set `FinalizationService` and `ResolveFinalization` on +`ExperienceCaptureOptions` and every successfully captured invocation is finalized right after it is completed. See +the [adapter README](src/AgentExperience.MicrosoftAgentFramework/README.md#finalizing-captured-runs). + ## Design principles -- **Hexagonal core.** `Abstractions` and `Core` depend only on the BCL and a redaction primitive. MAF, databases, models, and telemetry stay in adapters. Dependency-boundary tests enforce this in CI. +- **Hexagonal core.** `Abstractions` depends only on the BCL; `Core` adds a redaction primitive and the dependency-injection *abstractions* it needs to register its own services. MAF, databases, models, and telemetry stay in adapters. Dependency-boundary tests enforce this in CI. - **Failure-preserving capture.** Failed and cancelled runs are recorded through an outer lifecycle path, never only a success callback. - **Evidence before trust.** Verification is deterministic and bound to a host-closed round and artifact revision. A completion score is never mistaken for reuse confidence. - **Sanitize before anything is stored.** Unknown payload fields are dropped by default, and secrets are redacted from nested values. -- **Reuse, don't rebuild.** MAF middleware and `Microsoft.Extensions.Compliance.Redaction` are used at the edges, and planned storage builds on existing pgvector connectors. Each integration was proven with executable compatibility tests before an adapter was built. +- **Reuse, don't rebuild.** MAF middleware and `Microsoft.Extensions.Compliance.Redaction` are used at the edges, and storage builds on Npgsql and pgvector rather than on a bespoke engine. Each integration was proven with executable compatibility tests before an adapter was built. +- **Derived data never blocks canonical data.** Embeddings are produced after the commit, through a replaceable provider port, and every failure leaves the record committed, text-searchable, and retryable. ## Repository layout ``` src/ AgentExperience.Abstractions/ domain contracts and ports (BCL only) - AgentExperience.Core/ sanitization, capture, verification, reflection, lifecycle transitions + AgentExperience.Core/ sanitization, capture, verification, reflection, lifecycle transitions, finalization, indexing, retrieval AgentExperience.MicrosoftAgentFramework/ MAF adapter (pinned Microsoft.Agents.AI 1.20.0) - AgentExperience.Storage.Postgres/ PostgreSQL Experience Record store and schema migrator (pinned Npgsql 10.0.3, dbup-postgresql 7.0.1, dbup-core 6.1.1) + AgentExperience.Storage.Postgres/ PostgreSQL Experience Record store, text search, and schema migrator (pinned Npgsql 10.0.3, dbup-postgresql 7.0.1, dbup-core 6.1.1) + AgentExperience.Storage.Postgres.Vectors/ pgvector embedding index, conditional writes, scoped re-index, and vector search (pinned Npgsql 10.0.3, Pgvector 0.3.2, Microsoft.Extensions.AI.Abstractions 10.9.0) tests/ AgentExperience.Abstractions.Tests/ contract and dependency-boundary tests - AgentExperience.Core.Tests/ sanitizer, capture, verification, reflection, lifecycle tests + AgentExperience.Core.Tests/ sanitizer, capture, verification, reflection, lifecycle, indexing, retrieval tests AgentExperience.MicrosoftAgentFramework.Tests/ real ChatClientAgent runs against a scripted fake model AgentExperience.Storage.Postgres.Tests/ store tests, mostly against a PostgreSQL container + AgentExperience.Storage.Postgres.Vectors.Tests/ embedding index and hybrid retrieval, against a pgvector container AgentExperience.CompatibilityProof/ executable proofs for MAF hooks, context providers, pgvector, redaction docs/ original production architecture research _sdlc/ product brief, PRD, architecture, epics, and specs @@ -90,16 +412,16 @@ dotnet build dotnet test ``` -Unit and MAF adapter tests run in memory, with no network, database, or model credentials. `AgentExperience.CompatibilityProof` and the `PostgresExperienceRecordStoreTests`, `PostgresLifecycleCommitTests`, and `ExperienceSchemaMigratorTests` in `AgentExperience.Storage.Postgres.Tests` start a PostgreSQL/pgvector container through Testcontainers, so they need Docker. If Testcontainers' Ryuk container fails to start under your local Docker setup, set `TESTCONTAINERS_RYUK_DISABLED=true`. To skip the container-backed tests: +Unit and MAF adapter tests run in memory, with no network, database, or model credentials. **No test anywhere needs model credentials**: every embedding in the test suite comes from a deterministic in-test generator. `AgentExperience.CompatibilityProof`, the `PostgresExperienceRecordStoreTests`, `PostgresExperienceCandidateSourceTests`, `PostgresLifecycleCommitTests`, `PostgresFinalizationTests`, and `ExperienceSchemaMigratorTests` in `AgentExperience.Storage.Postgres.Tests`, the `PlainPostgresMigrationTests` in the same project (a stock `postgres:16` image, proving the base schema needs nothing pgvector provides), and the `PostgresEmbeddingIndexTests` and `HybridRetrievalIntegrationTests` in `AgentExperience.Storage.Postgres.Vectors.Tests` start a PostgreSQL/pgvector container through Testcontainers, so they need Docker. If Testcontainers' Ryuk container fails to start under your local Docker setup, set `TESTCONTAINERS_RYUK_DISABLED=true`. To skip the container-backed tests: ```bash -dotnet test --filter "FullyQualifiedName!~CompatibilityProof&FullyQualifiedName!~PostgresExperienceRecordStoreTests&FullyQualifiedName!~PostgresLifecycleCommitTests&FullyQualifiedName!~ExperienceSchemaMigratorTests" +dotnet test --filter "FullyQualifiedName!~CompatibilityProof&FullyQualifiedName!~PostgresExperienceRecordStoreTests&FullyQualifiedName!~PostgresExperienceCandidateSourceTests&FullyQualifiedName!~PostgresLifecycleCommitTests&FullyQualifiedName!~PostgresFinalizationTests&FullyQualifiedName!~ExperienceSchemaMigratorTests&FullyQualifiedName!~PlainPostgresMigrationTests&FullyQualifiedName!~PostgresEmbeddingIndexTests&FullyQualifiedName!~HybridRetrievalIntegrationTests" ``` ## Roadmap 1. **Capture and explain agent experience** ✅ contracts, sanitization, capture, verification, reflection, MAF adapter -2. **Reuse relevant experience:** PostgreSQL persistence and atomic audited lifecycle commits (in place), hybrid text and vector retrieval, historical-reference injection into MAF +2. **Reuse relevant experience:** PostgreSQL persistence, atomic audited lifecycle commits, one-call finalization of captured runs, bounded text retrieval with explainable ranking, and revision-safe embedding ingestion with hybrid retrieval (in place), historical-reference injection into MAF 3. **Govern experience safely:** sharing grants, the remaining lifecycle transitions, evidence-based confidence updates 4. **Operate and measure the learning loop:** OpenTelemetry instrumentation, an end-to-end demo, measured reuse against a baseline, data deletion and expiry diff --git a/src/AgentExperience.Abstractions/ExperienceCandidateSource.cs b/src/AgentExperience.Abstractions/ExperienceCandidateSource.cs new file mode 100644 index 0000000..1387b4d --- /dev/null +++ b/src/AgentExperience.Abstractions/ExperienceCandidateSource.cs @@ -0,0 +1,102 @@ +namespace AgentExperience.Abstractions; + +/// +/// Port for finding Experience Records that could apply to a task, matched on task text. It is a +/// read-only search seam kept deliberately separate from : the +/// store persists and reads canonical records by identity or scope, while this port answers "which +/// stored records look relevant to this text?" and nothing else. +/// +/// +/// +/// The same trust boundary applies as to : every call takes a +/// host-established , a request scope outside it is +/// before any storage access, and scope matching is exact +/// (ordinal, case-sensitive, matches only ). Expected +/// conditions return typed results; infrastructure failures throw +/// ; caller cancellation surfaces as an unwrapped +/// . +/// +/// +/// An implementation decides nothing about eligibility beyond what the query asks for: it +/// applies the scope, the requested statuses, and the minimum confidence, matches the text, and +/// returns each match with a normalized relevance. Which statuses are eligible, whether a record has +/// expired, whether its environment is compatible, and how candidates are ranked are all Core's +/// decisions, made over what this port returns. +/// +/// +public interface IExperienceCandidateSource +{ + /// + /// Finds records within exactly whose indexed task + /// text matches , whose + /// is one of + /// , and whose + /// is at least + /// . At most + /// records are returned, the strongest text matches + /// first. + /// + /// What the host has established the caller may do. + /// The scoped search. Never treated as authority. + /// Cancels the operation. + /// (possibly with no candidates), , or . + Task SearchAsync( + AuthorizationContext authorization, + ExperienceCandidateQuery query, + CancellationToken cancellationToken); +} + +/// +/// A scoped, text-matched search for reusable Experience Records. +/// +/// The exact scope to search within. Never treated as authority. +/// The task text to match against. Must be non-blank and at most characters. +/// The statuses a record must be in to be returned. Must be non-empty and contain only defined values; the caller decides which statuses are eligible. +/// The smallest a record may have and still be returned, in [0, 1]. +/// Maximum number of candidates to return, from to . Defaults to . +public sealed record ExperienceCandidateQuery( + Scope Scope, + string TaskText, + IReadOnlyList EligibleStatuses, + double MinimumConfidence, + int Limit = ExperienceCandidateQuery.DefaultLimit) +{ + /// + /// The longest permitted . A task description is a sentence or a paragraph; + /// bounding it here keeps an accidental multi-megabyte payload a typed + /// rather than something the text-search parser chokes + /// on deep inside the database. + /// + public const int MaxTaskTextLength = 4096; + + /// The smallest permitted . + public const int MinLimit = 1; + + /// The largest permitted . + public const int MaxLimit = 200; + + /// The used when none is specified. + public const int DefaultLimit = 50; +} + +/// +/// One record a search matched, with how strongly its indexed text matched the query. +/// +/// The matching record, read back in full. +/// +/// How strongly the record's indexed text matched, normalized to [0, 1] by the implementation, where +/// 0 is no measurable match and 1 is the strongest the implementation can report. Comparable only +/// between candidates from the same search. +/// +public sealed record ExperienceCandidate(ExperienceRecord Record, double Relevance); + +/// +/// The result of . +/// +/// What happened. +/// The matching candidates, strongest match first, when is ; otherwise empty. +/// Every validation error when is ; otherwise empty. +public sealed record ExperienceCandidateSearchResult( + ExperienceStoreOutcome Outcome, + IReadOnlyList Candidates, + IReadOnlyList Errors); diff --git a/src/AgentExperience.Abstractions/ExperienceIndex.cs b/src/AgentExperience.Abstractions/ExperienceIndex.cs new file mode 100644 index 0000000..26ffd13 --- /dev/null +++ b/src/AgentExperience.Abstractions/ExperienceIndex.cs @@ -0,0 +1,487 @@ +using System.Security.Cryptography; +using System.Text; + +namespace AgentExperience.Abstractions; + +/// +/// The sanitized retrieval summary: the only text of an that is ever +/// handed to an embedding provider, and exactly the three fields the text index already analyzes -- +/// , , and the +/// . +/// +/// +/// +/// Attempts, tool calls, evidence, provenance, and environment metadata are deliberately excluded. +/// They are operational detail, they would flood a vector with identifiers and stack-trace-like +/// fragments, and -- unlike the three fields above -- they are not what a record is about. +/// Reading the same fields as the text index means the two retrieval channels answer over the same +/// claim about the record rather than over two different documents. They do not necessarily read the +/// same length: this is capped at , while the text index analyzes far +/// more, so a very long summary is matched on more of its text by words than by meaning. +/// +/// +/// The text is normalized before it is hashed or embedded: every run of whitespace collapses to one +/// space and the result is trimmed, so a record whose summary differs only in line endings or +/// indentation hashes the same and is never re-embedded for nothing. +/// +/// +public static class ExperienceRetrievalSummary +{ + /// + /// The longest summary that is ever embedded. Providers bound their input, and a very long + /// summary would be truncated by the provider anyway -- truncating here instead keeps the hashed + /// text and the embedded text identical, so the skip-on-unchanged-hash rule stays sound. + /// + public const int MaxLength = 8192; + + /// Builds the normalized retrieval summary for . + /// The record to summarize. Never mutated. + /// The normalized summary, at most characters. Never . + /// is . + public static string For(ExperienceRecord record) + { + ArgumentNullException.ThrowIfNull(record); + return For(record.TaskId, record.TaskSummary, record.Reflection?.Lesson); + } + + /// + /// Builds the normalized retrieval summary from the three fields directly, for a caller that + /// holds them without a materialized (a storage adapter reading + /// them out of a row, for example). + /// + /// The record's task identifier. + /// The record's sanitized task summary, if any. + /// The reflection's lesson, if the record carries a reflection. + /// The normalized summary, at most characters. Never . + public static string For(string? taskId, string? taskSummary, string? lesson) + { + var builder = new StringBuilder(capacity: 256); + Append(builder, taskId); + Append(builder, taskSummary); + Append(builder, lesson); + + Truncate(builder); + return builder.ToString(); + } + + /// + /// Cuts the summary to without ever splitting a character or leaving a + /// dangling separator. + /// + /// + /// A raw cut at a UTF-16 index can land between a surrogate pair, and a lone surrogate is not a + /// valid character: UTF-8 encoding silently replaces it with U+FFFD. The hash would then be taken + /// over one string and the provider handed another, which is exactly the disagreement the content + /// hash exists to rule out. Backing off one unit when the last kept unit is a high surrogate makes + /// the cut land on a character boundary; dropping a trailing space afterwards keeps the result + /// identical in shape to an untruncated summary. + /// + private static void Truncate(StringBuilder builder) + { + if (builder.Length <= MaxLength) + { + return; + } + + var length = MaxLength; + if (char.IsHighSurrogate(builder[length - 1])) + { + length--; + } + + while (length > 0 && builder[length - 1] == ' ') + { + length--; + } + + builder.Length = length; + } + + /// + /// Appends one field, collapsing every run of whitespace or control characters inside it + /// to a single space and separating it from what is already there by exactly one space. A + /// separator is only ever written immediately before a kept character, so the result never ends in + /// a space and a blank field contributes nothing at all. + /// + /// + /// Control characters are dropped rather than kept for a reason that is not cosmetic: + /// joins the model ID and the + /// summary with U+001F, and that separator is only unambiguous while neither part can + /// contain it. Stripping the whole C0/C1 range here makes that true of every summary, whatever a + /// task description happened to carry. + /// + private static void Append(StringBuilder builder, string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + var pendingSpace = builder.Length > 0; + foreach (var character in value) + { + if (char.IsWhiteSpace(character) || char.IsControl(character)) + { + pendingSpace = builder.Length > 0; + continue; + } + + if (pendingSpace) + { + builder.Append(' '); + pendingSpace = false; + } + + builder.Append(character); + } + } +} + +/// +/// What a stored embedding is, kept entirely separate from lifecycle state: which model +/// produced it, how wide the vector is, a hash of the exact text that was embedded, and the record +/// revision the text was read at. +/// +/// +/// +/// None of these four fields ever influences eligibility, status, or reuse confidence. They exist so +/// an index write can be made conditional (), so a re-index can tell +/// "nothing changed" from "the text changed" without calling a provider +/// (), and so a query vector is never compared against a vector it is not +/// comparable with ( and ). +/// +/// +/// The provider's identifier for the model that produced the vector. Compared ordinally; a different value makes two vectors incomparable. +/// How many components the vector has. Must be strictly positive, and must equal the stored vector's length. +/// A hash of the model ID and the normalized summary that was embedded. See . +/// The the summary was read at. An index write applies only while the record is still at exactly this revision. +public sealed record ExperienceEmbeddingDescriptor( + string ModelId, + int Dimension, + string ContentHash, + long SourceRevision) +{ + /// The longest permitted . + public const int MaxModelIdLength = 256; + + /// + /// The largest permitted . pgvector's own ceiling for an indexable vector + /// is well below this; bounding it here keeps an absurd dimension a typed validation error rather + /// than a multi-megabyte parameter sent to a database. + /// + public const int MaxDimension = 16000; + + /// + /// The content hash: lowercase hexadecimal SHA-256 over the model ID, a separator that cannot + /// occur in either part, and the normalized summary text. + /// + /// + /// The model ID is inside the hash on purpose. Two models embedding identical text produce + /// different, incomparable vectors, so "same text" alone must never be enough to skip a + /// re-embedding; including the model makes a model change look exactly like a text change. The + /// separator is U+001F (unit separator), which neither a model ID nor a normalized summary + /// can contain, so no concatenation of one pair can collide with another. + /// + /// The model that would produce the vector. + /// The normalized summary, as produces it. + /// 64 lowercase hexadecimal characters. + /// Either argument is . + public static string ComputeContentHash(string modelId, string summary) + { + ArgumentNullException.ThrowIfNull(modelId); + ArgumentNullException.ThrowIfNull(summary); + + // Written as an escape, never as the literal byte: every stored content hash depends on this + // one character surviving every editor, diff, and patch tool that ever touches this file, and + // an unprintable byte in source does not. + var bytes = Encoding.UTF8.GetBytes(string.Concat(modelId, "\u001F", summary)); + return Convert.ToHexStringLower(SHA256.HashData(bytes)); + } +} + +/// +/// Port for turning a record's sanitized retrieval summary into a vector. It is deliberately stated +/// in domain terms only -- a string in, floats out -- so neither +/// AgentExperience.Abstractions nor AgentExperience.Core ever takes a dependency on a +/// model-provider package. An adapter bridges this to whatever generator a host actually runs. +/// +/// +/// and must be known without calling the +/// provider: the content hash covers the model ID, so an unchanged summary under the same model has +/// to be recognizable before any provider call is made. +/// +public interface IExperienceEmbeddingGenerator +{ + /// + /// The identifier of the model this generator produces vectors with, stored on every embedding it + /// produces and compared ordinally at query time. Must be stable for the life of the generator. + /// + string ModelId { get; } + + /// + /// How many components returns. Must be strictly positive and stable + /// for the life of the generator; a vector of any other length is rejected by the caller. + /// + int Dimension { get; } + + /// Embeds one already-sanitized, already-normalized summary. + /// The text to embed. Never a raw payload: only output reaches here. + /// Cancels the operation. + /// A vector of exactly finite components. + Task> GenerateAsync(string text, CancellationToken cancellationToken); +} + +/// +/// Port for the derived embedding index: it stores one vector per Experience Record, lists what a +/// scoped re-index would have to look at, and answers a scoped vector search. It is deliberately +/// separate from , because an embedding is derived data -- the +/// canonical record is written, committed, and text-searchable whether or not any of this ever runs. +/// +/// +/// +/// The same trust boundary applies as to : every call takes a +/// host-established , a request scope outside it is +/// before any storage access, and scope matching is +/// exact (ordinal, case-sensitive, matches only ). +/// Expected conditions return typed results; infrastructure failures throw +/// ; caller cancellation surfaces as an unwrapped +/// . +/// +/// +/// An implementation decides nothing about eligibility or lifecycle. It never creates, +/// updates, or deletes an Experience Record, and it never writes a vector for a record that does not +/// currently exist at the revision the write names. +/// +/// +public interface IExperienceEmbeddingIndex +{ + /// + /// Stores one record's vector, but only while that record still exists, in exactly + /// , at exactly + /// . A record whose revision has moved + /// on is and the stored vector is left untouched; a + /// record that no longer exists is and no row is + /// created, so an in-flight write can never resurrect a deleted record. + /// + /// What the host has established the caller may do. + /// The vector and its descriptor, for one record in one scope. + /// Cancels the operation. + /// A result naming what happened; nothing is written unless it is . + Task WriteAsync( + AuthorizationContext authorization, + ExperienceIndexWrite write, + CancellationToken cancellationToken); + + /// + /// Lists what an index pass would have to consider within exactly + /// : each record's current revision, its normalized + /// retrieval summary, and the descriptor of the vector already stored for it, if any. This is the + /// read that lets a caller skip an unchanged record without ever calling a provider. + /// + /// What the host has established the caller may do. + /// Which records to list, and under which model to report stored descriptors. + /// Cancels the operation. + /// (possibly with no targets), , or . + Task ScanAsync( + AuthorizationContext authorization, + ExperienceIndexScan scan, + CancellationToken cancellationToken); + + /// + /// Finds records within exactly whose stored vector was + /// produced by at + /// 's dimension, nearest first, applying the same + /// status filter and confidence floor the text channel applies. + /// + /// + /// A vector from another model, or of another dimension, is never compared: it is excluded by the + /// query itself. When the scope holds embeddings but none of them are comparable, the result says + /// so ( or + /// ) rather than looking like an + /// empty match, so the caller can fall back to text alone and say why. + /// + /// What the host has established the caller may do. + /// The scoped vector search. Never treated as authority. + /// Cancels the operation. + /// A result naming what happened; candidates are present only on . + Task SearchAsync( + AuthorizationContext authorization, + ExperienceVectorQuery query, + CancellationToken cancellationToken); +} + +/// +/// One conditional index write: a record's vector, what it is, and the scope and revision it is only +/// valid for. +/// +/// The exact scope the record must lie in. Never treated as authority. +/// The record the vector describes. Must not be . +/// The model, dimension, content hash, and source revision the write is conditional on. +/// The vector itself. Must hold exactly finite components. +public sealed record ExperienceIndexWrite( + Scope Scope, + Guid ExperienceId, + ExperienceEmbeddingDescriptor Descriptor, + ReadOnlyMemory Vector); + +/// What a conditional index write ended as. +public enum ExperienceIndexOutcome +{ + /// The vector and its descriptor were stored, replacing any previous vector for that record. + Written, + + /// + /// The record exists in the requested scope but is no longer at the revision the write named, so + /// the write was rejected and the stored vector is unchanged. The result carries the record's + /// current revision. + /// + Stale, + + /// + /// No record with that ID exists within the requested scope -- it was deleted, or it is in another + /// scope. Nothing was written, and no row was created. + /// + Missing, + + /// The request scope lies outside the host-established authorization. No storage was accessed. + Denied, + + /// The request was malformed. See the result's validation errors. No storage was accessed. + Invalid, +} + +/// +/// The result of . +/// +/// What happened. +/// The record's revision as the index observed it, when is ; otherwise 0. +/// Every validation error when is ; otherwise empty. +public sealed record ExperienceIndexWriteResult( + ExperienceIndexOutcome Outcome, + long CurrentRevision, + IReadOnlyList Errors); + +/// +/// Which records an index pass should consider, within exactly one scope. +/// +/// The exact scope to scan. Never treated as authority. +/// The model whose stored descriptor to report for each record. A vector stored under another model is reported as it stands, so the caller can see the model changed. +/// +/// The statuses a record must be in to be listed. Non-empty, and only defined values. This is +/// deliberately the same filter a vector search applies: embedding a record whose vector could +/// never be returned would send its summary and lesson to an external provider for nothing. +/// +/// The smallest a record may have and still be listed, in [0, 1]. Again, the search's own floor. +/// Optional. Exactly which records to list; lists every record in the scope, bounded by . When supplied it must be non-empty and hold no empty GUID. +/// Maximum number of targets to return, from to . Defaults to . +/// +/// Optional keyset cursor: list only records whose sorts +/// strictly after this one. Targets always come back in ascending ID order, so passing the previous +/// page's walks a scope larger than +/// to the end. starts from the beginning. +/// +public sealed record ExperienceIndexScan( + Scope Scope, + string ModelId, + IReadOnlyList EligibleStatuses, + double MinimumConfidence, + IReadOnlyList? ExperienceIds = null, + int Limit = ExperienceIndexScan.DefaultLimit, + Guid? StartAfterId = null) +{ + /// The smallest permitted . + public const int MinLimit = 1; + + /// The largest permitted . A re-index is a batch job: it pages, rather than loading a whole scope at once. + public const int MaxLimit = 500; + + /// The used when none is specified. + public const int DefaultLimit = 100; +} + +/// +/// One record an index pass may have to (re-)embed, read at a single instant so its revision, its +/// summary, and its stored descriptor cannot disagree with each other. +/// +/// The record. +/// The record's revision at the moment it was read. An index write for this target is conditional on it. +/// The record's normalized retrieval summary, as defines it. +/// The descriptor of the vector already stored for this record, or when it has never been indexed. +public sealed record ExperienceIndexTarget( + Guid ExperienceId, + long SourceRevision, + string Summary, + ExperienceEmbeddingDescriptor? Stored); + +/// +/// The result of . +/// +/// What happened. +/// The records to consider, in ascending order, when is ; otherwise empty. +/// Every validation error when is ; otherwise empty. +/// +/// The last listed record's ID, to pass as the next scan's . +/// when nothing was listed, which is how a caller knows the scope is exhausted. +/// +public sealed record ExperienceIndexScanResult( + ExperienceStoreOutcome Outcome, + IReadOnlyList Targets, + IReadOnlyList Errors, + Guid? LastExaminedId = null); + +/// +/// A scoped nearest-neighbour search for reusable Experience Records. It mirrors +/// field for field, apart from matching on a vector instead of +/// on text, so both retrieval channels are filtered identically. +/// +/// The exact scope to search within. Never treated as authority. +/// The model the query vector was produced by. Only vectors stored under exactly this model are ever compared against it. +/// The query vector. Must be non-empty, hold only finite components, and be no wider than . +/// The statuses a record must be in to be returned. Must be non-empty and contain only defined values. +/// The smallest a record may have and still be returned, in [0, 1]. +/// Maximum number of candidates to return, from to . Defaults to . +public sealed record ExperienceVectorQuery( + Scope Scope, + string ModelId, + ReadOnlyMemory Vector, + IReadOnlyList EligibleStatuses, + double MinimumConfidence, + int Limit = ExperienceCandidateQuery.DefaultLimit); + +/// What a scoped vector search ended as. +public enum ExperienceVectorSearchOutcome +{ + /// The search ran. It may still have matched nothing, which is an answer, not a failure. + Found, + + /// + /// The scope holds embeddings, but every one of them was produced by a different model, so none + /// is comparable with the query vector. No comparison was attempted and no candidate is returned. + /// + ModelMismatch, + + /// + /// The scope holds embeddings from this model, but at a different dimension, so none is comparable + /// with the query vector. No comparison was attempted and no candidate is returned. + /// + DimensionMismatch, + + /// The request scope lies outside the host-established authorization. No storage was accessed. + Denied, + + /// The request was malformed. See the result's validation errors. No storage was accessed. + Invalid, +} + +/// +/// The result of . Candidates carry the same +/// normalized [0, 1] relevance always does, so the two retrieval +/// channels can be merged on one comparable measure. +/// +/// What happened. +/// The matching candidates, nearest first, when is ; otherwise empty. +/// Every validation error when is ; otherwise empty. +public sealed record ExperienceVectorSearchResult( + ExperienceVectorSearchOutcome Outcome, + IReadOnlyList Candidates, + IReadOnlyList Errors); diff --git a/src/AgentExperience.Core/AgentExperience.Core.csproj b/src/AgentExperience.Core/AgentExperience.Core.csproj index 117d9b5..20bae5a 100644 --- a/src/AgentExperience.Core/AgentExperience.Core.csproj +++ b/src/AgentExperience.Core/AgentExperience.Core.csproj @@ -1,11 +1,14 @@ - AgentExperience.NET's first production Core package: the default sanitization pipeline (per-Kind allowlists, secret-field classification, recursive traversal, fail-closed rejection) built on AgentExperience.Abstractions' ISanitizer port. No dependency on MAF, EF Core, PostgreSQL, model providers, or OpenTelemetry -- only the BCL, Abstractions, and Microsoft.Extensions.Compliance.Redaction (AD-1). + AgentExperience.NET's first production Core package: the default sanitization pipeline (per-Kind allowlists, secret-field classification, recursive traversal, fail-closed rejection) built on AgentExperience.Abstractions' ISanitizer port. No dependency on MAF, EF Core, PostgreSQL, model providers, or OpenTelemetry -- only the BCL, Abstractions, Microsoft.Extensions.Compliance.Redaction, and Microsoft.Extensions.DependencyInjection.Abstractions (AD-1). + + diff --git a/src/AgentExperience.Core/DependencyInjection/AgentExperienceCoreServiceCollectionExtensions.cs b/src/AgentExperience.Core/DependencyInjection/AgentExperienceCoreServiceCollectionExtensions.cs new file mode 100644 index 0000000..2845336 --- /dev/null +++ b/src/AgentExperience.Core/DependencyInjection/AgentExperienceCoreServiceCollectionExtensions.cs @@ -0,0 +1,180 @@ +using AgentExperience.Abstractions; +using AgentExperience.Core.Capture; +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Indexing; +using AgentExperience.Core.Lifecycle; +using AgentExperience.Core.Reflections; +using AgentExperience.Core.Retrieval; +using AgentExperience.Core.Sanitization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace AgentExperience.Core.DependencyInjection; + +/// +/// Registers AgentExperience.NET's Core services in a . Core owns +/// its own registration so a host never has to know which concrete types implement which port; the +/// storage adapter registers its own in the same way (see +/// AddAgentExperiencePostgresStore), and AgentExperience.Abstractions stays BCL-only. +/// +public static class AgentExperienceCoreServiceCollectionExtensions +{ + /// + /// Registers the sanitizer, the in-memory capture service, the default reflector, the lifecycle + /// service, and the finalization service as singletons. + /// + /// + /// + /// Every registration uses TryAdd, so a host that has already registered its own + /// , , or + /// keeps it. + /// + /// + /// and both + /// need an , which Core does not implement: register a + /// storage adapter (for example AddAgentExperiencePostgresStore) as well, or resolving + /// them fails. + /// + /// + /// No sanitization policy or capture limit is invented here: both are host decisions with real + /// security and memory consequences, so both are required arguments. + /// + /// + /// The service collection to add to. + /// The per-Kind sanitization policy the default sanitizer applies. + /// The limits in-memory capture enforces. + /// , for chaining. + /// Any argument is . + public static IServiceCollection AddAgentExperienceCore( + this IServiceCollection services, + SanitizationOptions sanitizationOptions, + CaptureLimits captureLimits) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(sanitizationOptions); + ArgumentNullException.ThrowIfNull(captureLimits); + + // The arguments are captured by the factories rather than resolved back out of the container. + // Re-resolving them would let a SanitizationOptions or CaptureLimits the host registered earlier + // silently replace the caller's, so the sanitizer would run a policy nobody passed to it. + services.TryAddSingleton(sanitizationOptions); + services.TryAddSingleton(captureLimits); + services.TryAddSingleton(_ => new DefaultSanitizer(sanitizationOptions)); + services.TryAddSingleton(provider => new InMemoryExperienceCaptureService( + provider.GetRequiredService(), + captureLimits)); + services.TryAddSingleton(); + services.TryAddSingleton(); + + // The indexing hook is resolved optionally, not required: a host that never registered + // AddAgentExperienceIndexing gets finalization with no hook at all, which is exactly the + // text-only deployment. Registering it later still works, because this factory runs when the + // finalization singleton is first resolved rather than now. + services.TryAddSingleton(provider => new ExperienceFinalizationService( + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetService())); + + return services; + } + + /// + /// Registers as a singleton, so a committed record can be + /// embedded and its vector stored. + /// + /// + /// + /// Registered separately from because it needs an + /// and an , + /// neither of which Core implements: register an adapter's index (for example + /// AddAgentExperiencePostgresEmbeddingIndex) and a generator as well, or resolving the + /// service fails. + /// + /// + /// Order does not matter. resolves this service optionally + /// and lazily, so calling this before or after it wires the post-commit indexing hook either way; + /// omitting it entirely leaves finalization with no hook, which is a supported deployment. + /// + /// + /// The service collection to add to. + /// , for chaining. + /// is . + public static IServiceCollection AddAgentExperienceIndexing(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + // The retrieval policy is resolved optionally and shared: its confidence floor is the same one + // the vector search applies, so a record the search would never return is never embedded. A + // host that never registered one gets RetrievalPolicy.Default, which is what retrieval would + // have used anyway. + services.TryAddSingleton(provider => new ExperienceIndexingService( + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetService())); + + return services; + } + + /// + /// Registers as a singleton, together with the + /// and it runs under. + /// + /// + /// + /// Retrieval is registered separately from because it needs + /// an , which Core does not implement: register a storage + /// adapter's search as well (for example AddAgentExperiencePostgresCandidateSource), or + /// resolving the service fails. + /// + /// + /// Unlike sanitization policy and capture limits, retrieval has documented defaults + /// ( and ), so both + /// arguments are optional. Passing an invalid policy or weighting is impossible: both throw at + /// construction, before this call. The the timeout, expiry, and recency + /// are measured with is unless the host registered its own + /// first. + /// + /// + /// Every registration uses TryAdd, so a host that registered its own policy, weights, + /// clock, or service keeps it. + /// + /// + /// The service collection to add to. + /// The retrieval bounds and thresholds. Defaults to . + /// The ranking weights. Defaults to . + /// , for chaining. + /// is . + public static IServiceCollection AddAgentExperienceRetrieval( + this IServiceCollection services, + RetrievalPolicy? policy = null, + RankingWeights? weights = null) + { + ArgumentNullException.ThrowIfNull(services); + + var effectivePolicy = policy ?? RetrievalPolicy.Default; + var effectiveWeights = weights ?? RankingWeights.Default; + + services.TryAddSingleton(effectivePolicy); + services.TryAddSingleton(effectiveWeights); + services.TryAddSingleton(TimeProvider.System); + + // The caller's own policy and weights are captured rather than resolved back out of the + // container, for the same reason the sanitizer's options are: a RetrievalPolicy the host + // registered earlier must not silently replace the one passed here. + // The vector channel is resolved optionally: both halves of it present means hybrid + // retrieval, and anything less means an explicitly flagged text-only result rather than a + // failure. A host adds it by registering an IExperienceEmbeddingIndex and an + // IExperienceEmbeddingGenerator, in any order relative to this call. + services.TryAddSingleton(provider => new ExperienceRetrievalService( + provider.GetRequiredService(), + effectivePolicy, + effectiveWeights, + provider.GetRequiredService(), + provider.GetService(), + provider.GetService())); + + return services; + } +} diff --git a/src/AgentExperience.Core/Finalization/ExperienceFinalizationService.cs b/src/AgentExperience.Core/Finalization/ExperienceFinalizationService.cs new file mode 100644 index 0000000..24b3ba4 --- /dev/null +++ b/src/AgentExperience.Core/Finalization/ExperienceFinalizationService.cs @@ -0,0 +1,852 @@ +using System.Globalization; +using System.Security.Cryptography; +using AgentExperience.Abstractions; +using AgentExperience.Core.Capture; +using AgentExperience.Core.Indexing; +using AgentExperience.Core.Lifecycle; +using AgentExperience.Core.Reflections; +using AgentExperience.Core.Verification; + +namespace AgentExperience.Core.Finalization; + +/// +/// The single Core call that turns a captured, completed run into a durable Experience Record. It +/// runs the stages of in order and stops at the first one that ends +/// the call, returning a structured naming that stage -- +/// never an exception for an expected condition, and never a durable success for a database failure. +/// +/// +/// +/// Stages. Load the captured snapshot, evaluate it against its own host-closed round, check the +/// host authorization and the host's storage decision, reflect on it, create the record, then commit +/// the record's initial lifecycle event. The two gates deliberately precede reflection: +/// is the documented seam for a model-backed reflector, so a run +/// the host is about to refuse is never handed to it. +/// +/// +/// Indexing, after the fact. When an is wired in, the +/// committed record's sanitized retrieval summary is embedded once the initial event has landed -- +/// outside the canonical write, and only for an event this call committed, so an +/// replay never re-embeds. It runs only for a +/// record a vector search could actually return +/// (), so a quarantined record's summary and lesson +/// are never sent to a provider, and it is bounded by , so a hung +/// provider cannot hold this call open after the record is durable. It is reported on +/// and can never change the outcome: a provider that +/// is down leaves the record committed, durable, text-searchable, and indexable by a later pass. +/// +/// +/// Validated vs quarantined. A verified evaluation plus a successful reflection plus a +/// permitting storage decision produces a record with reuse +/// confidence 2/3, one supporting validation and no contradictions. A permitted record whose +/// verification is not , or whose reflection threw, is +/// instead, carrying no reflection at all and safe failure +/// metadata on the result. Reflection is not even attempted for an unverified run, so an unreflected +/// lesson can never reach a quarantined record. Confidence is never computed from evidence counts +/// (that is a later story), and risk is never decided on the host's behalf. +/// +/// +/// Candidate first. The record is created as , +/// and its initial lifecycle event performs the real transition to +/// or through +/// Core's transition table (both are allowed moves, and the store's prior-status guard applies). So a +/// commit that never lands leaves a -- never injectable -- +/// rather than a reusable record with no lifecycle history. The record's reuse-confidence inputs are +/// stamped at create time because a lifecycle commit updates only status, revision, and the updated +/// timestamp: the store persists Core's decision and derives no score of its own (2.4). The returned +/// record mirrors the projection the commit applied. +/// +/// +/// Replay. The record ID, the reflection ID, and the initial event ID are all derived from the +/// run ID, and the record's is the event's +/// , so finalizing the same run twice cannot create a second +/// record or a second initial confirmation. A second call finds the stored record and reports the +/// first call's outcome. When an earlier call created the record but its initial commit did not land +/// (the record is still at revision 0), a retry finishes that commit rather than starting over. +/// +/// +/// Sanitization. Finalization never sanitizes: capture already rejected anything unsafe before +/// storing an attempt, so the run's attempts are copied onto the record unchanged. +/// +/// +/// Failures. Every stage failure comes back as a structured +/// result naming the stage, including any exception a port +/// throws; the captured snapshot is never evicted, so the host can retry. The one exception is +/// cancellation: an from any stage -- the caller's +/// token or a port cancelling for its own reasons -- always propagates, so a cancelled call never +/// silently becomes a quarantined record. The post-commit indexing hook is outside that rule, because +/// by the time it runs the record is already durable and throwing would deny a fact that is true. +/// +/// +public sealed class ExperienceFinalizationService +{ + /// The every initial event this service commits carries. + public const string ProducerIdentity = "AgentExperience.ExperienceFinalizationService/1.0.0"; + + /// + /// The reuse confidence a freshly validated record starts at: two thirds. It is an initial, + /// evidence-gated value, not a score computed from evidence counts. + /// + public const double InitialValidatedReuseConfidence = 2d / 3d; + + /// The status every Experience Record is created in, before its initial lifecycle event moves it. + public const ExperienceStatus CreatedStatus = ExperienceStatus.Candidate; + + /// + /// The default budget for the post-commit indexing hook, after which it is abandoned and reported + /// as retryable. The record is already durable when the hook starts, so this bounds nothing but the + /// caller's wait -- which is exactly what it exists for: a hung provider must not hold + /// open after the canonical work is done. + /// + public static readonly TimeSpan DefaultIndexingTimeout = TimeSpan.FromSeconds(10); + + /// + /// Fixed namespace for the derived identifiers below. Changing it would re-issue every record ID, + /// so it is a constant of this library, never configurable. + /// + private static readonly Guid DerivationNamespace = new("0b6a8a3f-1c2d-4f5e-9a70-3d1c9f2b8e41"); + + private const byte ExperienceIdTag = 1; + private const byte InitialEventIdTag = 2; + private const byte ReflectionIdTag = 3; + + private static readonly IReadOnlyList NoErrors = []; + + private readonly IExperienceCaptureService _captureService; + private readonly IExperienceReflector _reflector; + private readonly IExperienceRecordStore _store; + private readonly ExperienceLifecycleService _lifecycleService; + private readonly ExperienceIndexingService? _indexingService; + + /// Creates a finalization service over the capture snapshot, the reflector, the record store, and Core's lifecycle owner, with no indexing hook. + /// Where the completed run's sanitized snapshot is read from. + /// Turns the evaluated run into an auditable reflection. + /// The durable Experience Record store. + /// Core's lifecycle owner, which stamps and commits the initial event. + /// Any argument is . + public ExperienceFinalizationService( + IExperienceCaptureService captureService, + IExperienceReflector reflector, + IExperienceRecordStore store, + ExperienceLifecycleService lifecycleService) + : this(captureService, reflector, store, lifecycleService, indexingService: null) + { + } + + /// + /// Creates a finalization service with an optional post-commit indexing hook. + /// + /// + /// The hook runs only after an initial lifecycle event this call actually committed, never on a + /// replay of an already-finalized run, and it can never fail finalization: every outcome it + /// reaches, including a provider that throws and a cancellation, is reported on the result and + /// nothing more. See . + /// + /// Where the completed run's sanitized snapshot is read from. + /// Turns the evaluated run into an auditable reflection. + /// The durable Experience Record store. + /// Core's lifecycle owner, which stamps and commits the initial event. + /// Optional. Embeds the committed record's sanitized retrieval summary after the fact. + /// Optional. How long that hook may take before it is abandoned and reported as retryable. Must be strictly positive. Defaults to . + /// Any non-optional argument is . + /// is not strictly positive. + public ExperienceFinalizationService( + IExperienceCaptureService captureService, + IExperienceReflector reflector, + IExperienceRecordStore store, + ExperienceLifecycleService lifecycleService, + ExperienceIndexingService? indexingService, + TimeSpan? indexingTimeout = null) + { + ArgumentNullException.ThrowIfNull(captureService); + ArgumentNullException.ThrowIfNull(reflector); + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(lifecycleService); + + _captureService = captureService; + _reflector = reflector; + _store = store; + _lifecycleService = lifecycleService; + _indexingService = indexingService; + IndexingTimeout = indexingTimeout ?? DefaultIndexingTimeout; + + if (IndexingTimeout <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(indexingTimeout), + IndexingTimeout, + "The indexing budget must be strictly positive; an unbounded hook is what this exists to prevent."); + } + } + + /// The budget this service gives the post-commit indexing hook. + public TimeSpan IndexingTimeout { get; } + + /// The finalizing issues, derived from the run so a retry re-derives the same ID. + /// The captured run. + public static Guid ExperienceIdFor(Guid runId) => Derive(runId, ExperienceIdTag); + + /// The of the record's initial event, derived from the run so a retry cannot commit a second initial confirmation. + /// The captured run. + public static Guid InitialEventIdFor(Guid runId) => Derive(runId, InitialEventIdTag); + + /// The finalizing asks the reflector to stamp, derived from the run so a retry reflects under the same identity. + /// The captured run. + public static Guid ReflectionIdFor(Guid runId) => Derive(runId, ReflectionIdTag); + + /// + /// Finalizes one captured run, running every stage in order and stopping at the first one that + /// ends the call. + /// + /// The run to finalize, its verification inputs, the host authorization, and the host's storage decision. + /// Cancels the operation. Cancellation is not an expected condition and propagates. + /// A structured result naming the stage finalization ended at. + /// , or its , , , or , is . + /// is , is blank, or is unset. + /// was cancelled, or a port cancelled. + public async Task FinalizeAsync( + FinalizeExperienceRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(request.Authorization, $"{nameof(request)}.{nameof(request.Authorization)}"); + ArgumentNullException.ThrowIfNull(request.RequiredChecks, $"{nameof(request)}.{nameof(request.RequiredChecks)}"); + ArgumentNullException.ThrowIfNull(request.Evidence, $"{nameof(request)}.{nameof(request.Evidence)}"); + ArgumentNullException.ThrowIfNull(request.StorageDecision, $"{nameof(request)}.{nameof(request.StorageDecision)}"); + if (request.RunId == Guid.Empty) + { + throw new ArgumentException("RunId must not be an empty GUID.", nameof(request)); + } + + if (string.IsNullOrWhiteSpace(request.CurrentArtifactRevision)) + { + throw new ArgumentException("CurrentArtifactRevision must not be null, empty, or whitespace.", nameof(request)); + } + + if (request.FinalizedAt == default) + { + // An unset timestamp would be stamped onto the record and its initial event, and the store + // rejects an unset OccurredAt as Invalid -- so the record would be created and then every + // commit, including every retry, would be refused forever. + throw new ArgumentException("FinalizedAt must be set to when the host decided to finalize this run.", nameof(request)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + // Stage 1 -- Load. An unknown or unfinished run is an expected condition, not an exception. + ExperienceRun? run; + try + { + _ = _captureService.TryGetRun(request.RunId, out run); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return PortFailed(FinalizationStage.Load, ex, evaluation: null, record: null, reflection: null); + } + + if (run is null) + { + return Ended( + FinalizationOutcome.RunNotFound, + FinalizationStage.Load, + "No captured run exists for the requested run ID."); + } + + if (run.ExecutionStatus is null) + { + return Ended( + FinalizationOutcome.RunNotFinished, + FinalizationStage.Load, + "The captured run has no execution status, so it has not finished and cannot be finalized."); + } + + // The store keeps whole microseconds, so the record's CreatedAt -- which is also the initial + // event's OccurredAt -- is truncated here rather than by the database. That keeps a replay's + // re-derived event byte-for-byte identical to the stored one. + var finalizedAt = TruncateToMicroseconds(request.FinalizedAt); + + // Stage 2 -- Evaluate, against this run's own closed round and evidence. No caller-supplied + // evaluation is accepted, so an evaluation from another run cannot be substituted. + VerificationResult evaluation; + try + { + evaluation = VerificationAggregator.Aggregate( + request.Evidence, + request.RequiredChecks, + request.ClosedRound, + request.CurrentArtifactRevision, + finalizedAt, + cancellationToken); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return Ended( + FinalizationOutcome.Failed, + FinalizationStage.Evaluate, + "The run could not be evaluated.", + new FinalizationFailure( + FinalizationStage.Evaluate, + $"Aggregating verification threw {ex.GetType().FullName}; the verification inputs are malformed.", + NoErrors, + ex), + evaluation: null); + } + + // Stage 3 -- Authorize, then read the host's storage decision. Both are decided before any + // store call *and before the reflector is called*, so a refused run is never handed to the + // model-backed reflection seam and nothing at all is written. + if (!request.Authorization.Permits(run.Scope)) + { + return Ended( + FinalizationOutcome.NotAuthorized, + FinalizationStage.Authorize, + "The captured run's scope lies outside the host-established authorization; nothing was stored and the run was not reflected on.", + evaluation: evaluation); + } + + if (!request.StorageDecision.Permitted) + { + return Ended( + FinalizationOutcome.StorageDenied, + FinalizationStage.Authorize, + request.StorageDecision.Reason ?? "The host's storage decision did not permit persisting this run; nothing was stored and the run was not reflected on.", + evaluation: evaluation); + } + + // Stage 4 -- Reflect, but only on a verified run: a quarantined record must never carry an + // unreflected lesson, so an unverified run is not reflected on at all. + Reflection? reflection = null; + FinalizationFailure? failure = null; + + if (evaluation.Outcome.Status == TaskVerificationStatus.Verified) + { + try + { + reflection = await _reflector + .ReflectAsync( + new ReflectionRequest(run, evaluation, ReflectionIdFor(run.RunId), finalizedAt), + cancellationToken) + .ConfigureAwait(false); + + if (reflection is null) + { + failure = new FinalizationFailure( + FinalizationStage.Reflect, + "The reflector returned no reflection; the record is quarantined without an eligible lesson.", + NoErrors, + Exception: null); + } + } + catch (OperationCanceledException) + { + // Cancellation is never quietly turned into a quarantine, whoever cancelled and why. + throw; + } + catch (Exception ex) + { + // A reflector failure does not end finalization: the run is still worth keeping, just + // not as a validated lesson. The exception is caught, never rethrown. + reflection = null; + failure = new FinalizationFailure( + FinalizationStage.Reflect, + $"The reflector threw {ex.GetType().FullName}; the record is quarantined without an eligible lesson.", + NoErrors, + ex); + } + } + else + { + failure = new FinalizationFailure( + FinalizationStage.Evaluate, + $"Verification resolved to {evaluation.Outcome.Status} rather than {TaskVerificationStatus.Verified}; the record is quarantined without an eligible lesson.", + NoErrors, + Exception: null); + } + + // Stage 5 -- Create the record, as a Candidate. Attempts are copied unchanged: capture already + // rejected anything unsafe, and finalization never sanitizes. + var record = new ExperienceRecord( + ExperienceId: ExperienceIdFor(run.RunId), + SourceRunId: run.RunId, + Scope: run.Scope, + TaskId: run.TaskId, + TaskSummary: run.TaskDescription, + Attempts: run.Attempts, + Outcome: evaluation.Outcome, + CompletionScore: evaluation.CompletionScore, + Reflection: reflection, + Environment: run.Environment, + Provenance: run.Provenance, + Status: CreatedStatus, + ReuseConfidence: reflection is not null ? InitialValidatedReuseConfidence : 0d, + SupportingValidations: reflection is not null ? 1 : 0, + Contradictions: 0, + Revision: 0, + CreatedAt: finalizedAt, + UpdatedAt: finalizedAt); + + ExperienceRecordCreateResult created; + try + { + created = await _store.CreateAsync(request.Authorization, record, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return PortFailed(FinalizationStage.CreateRecord, ex, evaluation, record: null, reflection); + } + + switch (created.Outcome) + { + case ExperienceStoreOutcome.Created: + break; + + case ExperienceStoreOutcome.Conflict: + // This run has been finalized before (the ID is derived from it). Report the stored + // outcome rather than writing anything a second time. + return await ReplayAsync(request, run, record, evaluation, cancellationToken).ConfigureAwait(false); + + case ExperienceStoreOutcome.Denied: + return Ended( + FinalizationOutcome.NotAuthorized, + FinalizationStage.CreateRecord, + "The store refused the record's scope as outside the host-established authorization; nothing was stored.", + evaluation: evaluation); + + case ExperienceStoreOutcome.Invalid: + return Ended( + FinalizationOutcome.Failed, + FinalizationStage.CreateRecord, + "The store rejected the Experience Record as malformed; nothing was stored.", + new FinalizationFailure( + FinalizationStage.CreateRecord, + "The store reported the Experience Record invalid. See the validation errors.", + created.Errors, + Exception: null), + evaluation); + + default: + return Ended( + FinalizationOutcome.Failed, + FinalizationStage.CreateRecord, + $"The store returned '{created.Outcome}', which is not a create outcome.", + new FinalizationFailure( + FinalizationStage.CreateRecord, + $"The Experience Record store returned '{created.Outcome}' from CreateAsync.", + created.Errors, + Exception: null), + evaluation); + } + + // Stage 6 -- Commit the record's initial lifecycle event, which performs the real transition. + return await CommitInitialEventAsync(request, run, record, evaluation, failure, cancellationToken).ConfigureAwait(false); + } + + /// + /// Handles a create that conflicted because this run was already finalized. A record still at + /// revision 0 had its create land but not its initial commit, so the commit is finished here; + /// anything past revision 0 is fully finalized and is reported as it stands. + /// + private async Task ReplayAsync( + FinalizeExperienceRequest request, + ExperienceRun run, + ExperienceRecord attempted, + VerificationResult evaluation, + CancellationToken cancellationToken) + { + ExperienceRecordGetResult stored; + try + { + stored = await _store + .GetAsync(request.Authorization, run.Scope, attempted.ExperienceId, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return PortFailed(FinalizationStage.CreateRecord, ex, evaluation, record: null, attempted.Reflection); + } + + if (stored.Outcome != ExperienceStoreOutcome.Found || stored.Record is null) + { + // The derived ID is taken by a record this caller's scope cannot see. Nothing was written, + // and nothing about the other record is revealed. + return Ended( + FinalizationOutcome.Failed, + FinalizationStage.CreateRecord, + "An Experience Record with this run's derived ID already exists outside the requested scope; nothing was stored.", + new FinalizationFailure( + FinalizationStage.CreateRecord, + $"CreateAsync conflicted and the stored record is not readable in this scope ({stored.Outcome}).", + stored.Errors, + Exception: null), + evaluation); + } + + if (stored.Record.Revision > 0) + { + return AlreadyFinalized(stored.Record, evaluation); + } + + // The earlier call created the record but never confirmed it. Finish that same commit, from + // the stored record, so the event stays byte-for-byte what the first attempt would have sent. + // The failure is reconstructed from the stored record, so a quarantined resume still names the + // stage that decided it rather than reporting a reason-less quarantine. + return await CommitInitialEventAsync( + request, + run, + stored.Record, + evaluation, + FailureFor(stored.Record), + cancellationToken).ConfigureAwait(false); + } + + private async Task CommitInitialEventAsync( + FinalizeExperienceRequest request, + ExperienceRun run, + ExperienceRecord record, + VerificationResult evaluation, + FinalizationFailure? failure, + CancellationToken cancellationToken) + { + // Every field is a pure function of the stored record and the run, so a retry re-derives the + // identical event and the store deduplicates it instead of appending a second one. This is a + // real transition out of Candidate, so it goes through Core's transition table and the store's + // prior-status guard, not a null-prior self-transition. + var targetStatus = TargetStatusFor(record); + var transition = new CommitLifecycleTransitionRequest( + EventId: InitialEventIdFor(run.RunId), + ExperienceId: record.ExperienceId, + Scope: record.Scope, + PriorStatus: CreatedStatus, + CurrentStatus: targetStatus, + Reason: InitialEventReason(record), + Producer: ProducerIdentity, + OccurredAt: record.CreatedAt, + ExpectedRevision: 0); + + CommitLifecycleTransitionResult commit; + try + { + commit = await _lifecycleService.CommitAsync(request.Authorization, transition, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return PortFailed(FinalizationStage.CommitInitialEvent, ex, evaluation, record, record.Reflection); + } + + if (commit.Outcome != LifecycleTransitionOutcome.Committed) + { + // Someone else may have finalized this record between the read above and this commit. A + // moved revision is not a failure to report forever: re-read and converge on their result. + if (commit.Outcome is LifecycleTransitionOutcome.StaleRevision or LifecycleTransitionOutcome.Conflict + && await TryReadFinalizedAsync(request, record, cancellationToken).ConfigureAwait(false) is { } finalized) + { + return AlreadyFinalized(finalized, evaluation); + } + + // The record exists and is still a Candidate. Report it, and why it was going to be + // quarantined, so the host can reconcile rather than guess. + return new FinalizeExperienceResult( + FinalizationOutcome.Failed, + FinalizationStage.CommitInitialEvent, + record, + commit.Event, + record.Revision, + evaluation, + record.Reflection, + failure ?? new FinalizationFailure( + FinalizationStage.CommitInitialEvent, + $"Committing the record's initial lifecycle event returned {commit.Outcome}.{(commit.Reason is null ? string.Empty : " " + commit.Reason)}", + commit.Errors, + Exception: null), + $"The Experience Record's initial lifecycle event returned {commit.Outcome}, so the record is still a {record.Status} and finalization is not durable; the captured run is still available for a retry."); + } + + // Mirror the projection the store just applied, so the returned record is the record as it now + // stands rather than the pre-transition Candidate. + var committed = record with + { + Status = targetStatus, + Revision = commit.Revision, + UpdatedAt = transition.OccurredAt, + }; + + // Stage 7 -- Index, after the commit and outside it. The record is already durable at this + // point; nothing below can undo that, and nothing below is allowed to change this result's + // outcome. An AlreadyFinalized replay never reaches here, so a re-finalized run never + // re-embeds. + var indexing = await TryIndexAsync(request, committed, cancellationToken).ConfigureAwait(false); + + return new FinalizeExperienceResult( + targetStatus == ExperienceStatus.Validated ? FinalizationOutcome.Validated : FinalizationOutcome.Quarantined, + FinalizationStage.CommitInitialEvent, + committed, + commit.Event, + commit.Revision, + evaluation, + committed.Reflection, + failure, + Reason: null, + indexing); + } + + /// + /// Runs the optional indexing hook for a record whose initial event this call just committed, and + /// swallows everything it can do wrong. + /// + /// + /// + /// Embeddings are derived data: the canonical write must never depend on a provider being up. So + /// every failure here -- a throwing provider, an unreachable index, a stale or missing record -- + /// is turned into a reported, retryable and never rethrown + /// into finalization. + /// + /// + /// Even cancellation. This is the one place in the service where an + /// does not propagate, and deliberately: by the time this + /// runs the record is already committed and durable. Throwing would discard that fact and leave + /// the caller believing finalization did not happen, which is a worse lie than reporting a + /// cancelled index. + /// + /// + private async Task TryIndexAsync( + FinalizeExperienceRequest request, + ExperienceRecord committed, + CancellationToken cancellationToken) + { + if (_indexingService is null) + { + return null; + } + + if (!_indexingService.IsIndexable(committed.Status, committed.ReuseConfidence)) + { + // A quarantined (or otherwise ineligible) record's vector could never be returned by a + // search, so its task summary and reflection lesson are never handed to a provider. This is + // checked here, before any call, rather than discovered from an empty scan. + return new ExperienceIndexingResult( + ExperienceIndexingOutcome.Ineligible, + committed.ExperienceId, + Descriptor: null, + new ExperienceIndexingFailure( + $"The record is {committed.Status} with reuse confidence {committed.ReuseConfidence.ToString("R", CultureInfo.InvariantCulture)}, " + + "so a vector search could never return it; nothing was embedded and nothing left the database.", + NoErrors, + Exception: null)); + } + + // Bounded, and on its own budget. The record is already durable at this point, so a hung + // provider must not hold FinalizeAsync open: derived data never blocks canonical data, and that + // includes blocking the caller's thread after the canonical work is done. + using var indexing = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + indexing.CancelAfter(IndexingTimeout); + + try + { + return await _indexingService + .IndexAsync(request.Authorization, committed.Scope, committed.ExperienceId, indexing.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + return new ExperienceIndexingResult( + ExperienceIndexingOutcome.IndexFailed, + committed.ExperienceId, + Descriptor: null, + new ExperienceIndexingFailure( + cancellationToken.IsCancellationRequested + ? "Indexing was cancelled after the record was already committed; the record is durable and text-searchable, and can be indexed later." + : $"Indexing did not finish within {IndexingTimeout.TotalSeconds.ToString("R", CultureInfo.InvariantCulture)}s of the commit; " + + "the record is durable and text-searchable, and can be indexed later.", + NoErrors, + ex)); + } + catch (Exception ex) + { + return new ExperienceIndexingResult( + ExperienceIndexingOutcome.IndexFailed, + committed.ExperienceId, + Descriptor: null, + new ExperienceIndexingFailure( + $"The indexing hook threw {ex.GetType().FullName} after the record was already committed; the record is durable and text-searchable, and can be indexed later.", + NoErrors, + ex)); + } + } + + /// + /// Re-reads the record after a commit the store refused, returning it only when its revision has + /// moved past 0 -- that is, when someone else committed the initial event first. A read that fails + /// or still shows revision 0 returns , and the caller reports the original + /// commit outcome. + /// + private async Task TryReadFinalizedAsync( + FinalizeExperienceRequest request, + ExperienceRecord record, + CancellationToken cancellationToken) + { + try + { + var reread = await _store + .GetAsync(request.Authorization, record.Scope, record.ExperienceId, cancellationToken) + .ConfigureAwait(false); + + return reread is { Outcome: ExperienceStoreOutcome.Found, Record.Revision: > 0 } ? reread.Record : null; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception) + { + // Reconciliation is best-effort; the caller still reports the commit outcome it saw. + return null; + } + } + + private static FinalizeExperienceResult AlreadyFinalized(ExperienceRecord stored, VerificationResult evaluation) => new( + FinalizationOutcome.AlreadyFinalized, + FinalizationStage.CommitInitialEvent, + stored, + Event: null, + stored.Revision, + evaluation, + stored.Reflection, + FailureFor(stored), + $"This run was already finalized as {stored.Status}; no second record and no second initial event were written."); + + /// + /// The status a created record's initial event moves it to: + /// only when the record carries a reflection, which it does only for a verified run whose + /// reflection succeeded. + /// + private static ExperienceStatus TargetStatusFor(ExperienceRecord record) => + record.Reflection is not null ? ExperienceStatus.Validated : ExperienceStatus.Quarantined; + + /// + /// Reconstructs the safe failure metadata for a record this call did not itself build -- a stored + /// record found by a replay. A quarantine always names the stage that decided it. + /// + private static FinalizationFailure? FailureFor(ExperienceRecord record) + { + if (record.Reflection is not null) + { + return null; + } + + return record.Outcome.Status == TaskVerificationStatus.Verified + ? new FinalizationFailure( + FinalizationStage.Reflect, + "The stored record carries no reflection although its verification passed; it is quarantined without an eligible lesson.", + NoErrors, + Exception: null) + : new FinalizationFailure( + FinalizationStage.Evaluate, + $"Verification resolved to {record.Outcome.Status} rather than {TaskVerificationStatus.Verified}; the record is quarantined without an eligible lesson.", + NoErrors, + Exception: null); + } + + /// + /// The initial event's auditable reason, derived only from the record so a retry -- which reads + /// the stored record rather than recomputing -- produces exactly the same text. + /// + private static string InitialEventReason(ExperienceRecord record) => string.Format( + CultureInfo.InvariantCulture, + "Initial finalization: verification {0}, completion score {1}, {2}.", + record.Outcome.Status, + record.CompletionScore.ToString("R", CultureInfo.InvariantCulture), + record.Reflection is null ? "no eligible lesson recorded" : "reflection recorded"); + + /// + /// Turns any non-cancellation exception a port threw into a structured failed stage, so "every + /// stage failure comes back as a structured result" holds for more than + /// . + /// + private static FinalizeExperienceResult PortFailed( + FinalizationStage stage, + Exception exception, + VerificationResult? evaluation, + ExperienceRecord? record, + Reflection? reflection) => new( + FinalizationOutcome.Failed, + stage, + record, + Event: null, + record?.Revision ?? 0, + evaluation, + reflection, + new FinalizationFailure( + stage, + $"The {stage} stage's port threw {exception.GetType().FullName}.", + NoErrors, + exception), + "A port failed, so finalization is not durable; the captured run is still available for a retry."); + + private static FinalizeExperienceResult Ended( + FinalizationOutcome outcome, + FinalizationStage stage, + string reason, + FinalizationFailure? failure = null, + VerificationResult? evaluation = null) => new( + outcome, + stage, + Record: null, + Event: null, + Revision: 0, + evaluation, + Reflection: null, + failure, + reason); + + /// + /// Truncates to whole microseconds in UTC, which is the precision PostgreSQL's timestamptz + /// keeps. Without this, a value read back from the store would differ from the one sent, and a + /// replay's re-derived lifecycle event would no longer be identical to the stored one. + /// + private static DateTimeOffset TruncateToMicroseconds(DateTimeOffset value) + { + var utc = value.UtcDateTime; + return new DateTimeOffset(utc.Ticks - (utc.Ticks % TimeSpan.TicksPerMicrosecond), TimeSpan.Zero); + } + + /// + /// Derives a stable identifier from a run ID and a per-purpose tag: SHA-256 over a fixed + /// namespace, the run ID, and the tag, stamped with the RFC 9562 custom version (8) and variant. + /// Same run in, same identifiers out -- which is what makes replaying finalization safe. + /// + private static Guid Derive(Guid runId, byte tag) + { + Span input = stackalloc byte[33]; + DerivationNamespace.TryWriteBytes(input[..16], bigEndian: true, out _); + runId.TryWriteBytes(input.Slice(16, 16), bigEndian: true, out _); + input[32] = tag; + + Span hash = stackalloc byte[32]; + SHA256.HashData(input, hash); + + var id = hash[..16]; + id[6] = (byte)((id[6] & 0x0F) | 0x80); + id[8] = (byte)((id[8] & 0x3F) | 0x80); + return new Guid(id, bigEndian: true); + } +} diff --git a/src/AgentExperience.Core/Finalization/FinalizationResults.cs b/src/AgentExperience.Core/Finalization/FinalizationResults.cs new file mode 100644 index 0000000..328f63e --- /dev/null +++ b/src/AgentExperience.Core/Finalization/FinalizationResults.cs @@ -0,0 +1,160 @@ +using AgentExperience.Abstractions; +using AgentExperience.Core.Indexing; +using AgentExperience.Core.Verification; + +namespace AgentExperience.Core.Finalization; + +/// +/// The ordered stages of . Every result +/// names the stage it ended at, so a host always knows how far finalization got. +/// +public enum FinalizationStage +{ + /// Reading the captured run's snapshot back from the capture service. + Load, + + /// Computing the run's verification result from the host-closed round and its evidence. + Evaluate, + + /// Checking the run's scope against the host authorization, then the host's storage decision. No store call has been made yet, and the reflector has not been called. + Authorize, + + /// Reflecting on the evaluated run. Skipped entirely when verification did not pass, or when either gate above refused. + Reflect, + + /// Creating the Experience Record through the store port. + CreateRecord, + + /// Committing the record's initial lifecycle event, atomically with its projection. + CommitInitialEvent, +} + +/// +/// The disposition one call reached. +/// +public enum FinalizationOutcome +{ + /// + /// The run verified, reflection succeeded, and the host permitted storage: the record was created + /// as and its initial lifecycle event moved it to + /// . + /// + Validated, + + /// + /// The host permitted storage but the run did not verify, or reflection failed: the record was + /// created as , carrying no reflection, and its initial + /// lifecycle event moved it to . Failure names why. + /// + Quarantined, + + /// + /// This run had already been finalized, by an earlier call or by a concurrent one. Nothing was + /// written: no second record and no second initial event. Record and Revision report + /// the stored outcome, Status is the status that call produced, and Failure names why + /// it was quarantined when it was. + /// + AlreadyFinalized, + + /// + /// The host's did not permit storage. Nothing was written, no + /// record ID was issued, and Reason carries the host's own content-free reason. + /// + StorageDenied, + + /// + /// The run's scope lies outside the host-established . Denied + /// before any store call; nothing was written. + /// + NotAuthorized, + + /// No captured run exists for the requested run ID. Nothing was written. + RunNotFound, + + /// + /// The run exists but has no , so it has not finished + /// and cannot be finalized. Nothing was written. + /// + RunNotFinished, + + /// + /// A stage failed. Stage and Failure name which and why. This is never a durable + /// success: the captured run stays available so the host can retry finalization. When the record + /// had already been created but its initial event was refused, Record carries that record -- + /// still a , so it is not reusable -- and Failure + /// carries either the commit's own refusal or the reason the record was going to be quarantined. + /// + Failed, +} + +/// +/// Why a finalization stage could not produce what it was asked for. Present on +/// , and also on +/// , where it is the safe failure metadata explaining +/// why the record carries no eligible lesson. +/// +/// The stage the failure belongs to. On a quarantined record this is the stage that decided it could not be validated, which is not necessarily the stage the call ended at. +/// A content-free, auditable explanation. Never echoes captured content, record payload, or private reasoning. +/// The store's validation errors when a store call reported the request malformed; otherwise empty. +/// The exception behind the failure, if any. Handed to the host for diagnostics only -- never persisted, and never recorded into the Experience Record. +public sealed record FinalizationFailure( + FinalizationStage Stage, + string Reason, + IReadOnlyList Errors, + Exception? Exception); + +/// +/// The result of one call. +/// +/// What happened. +/// The stage finalization ended at. +/// +/// The Experience Record, when one exists: the record this call created (with the projection its +/// initial event applied, when that committed), or the stored record a replay found. +/// whenever nothing was persisted. On +/// after a refused initial commit it is the created record, +/// still a at revision 0. +/// +/// The initial lifecycle event this call committed, or when none was committed by this call. +/// The record's revision after the initial event was committed (1), or the stored revision reported by a replay; 0 when the record is still an uncommitted or nothing was written. +/// The verification result finalization computed for this run, once the evaluate stage ran; otherwise . +/// The reflection stored on the record, or -- always for a quarantined record. +/// Why the call failed, or why a record was (or was going to be) quarantined; otherwise . +/// Optional, auditable, content-free explanation of the outcome. +/// +/// What the optional post-commit indexing hook did, when one is wired in and this call actually +/// committed the record's initial event; otherwise . It is reported, never +/// acted on: an indexing failure here is always retryable and never changes +/// , , or anything about the stored record. A +/// value means no indexing was attempted -- because no hook is registered, or +/// because nothing was committed by this call (a replay of an already-finalized run never re-indexes). +/// +public sealed record FinalizeExperienceResult( + FinalizationOutcome Outcome, + FinalizationStage Stage, + ExperienceRecord? Record, + LifecycleEvent? Event, + long Revision, + VerificationResult? Evaluation, + Reflection? Reflection, + FinalizationFailure? Failure, + string? Reason, + ExperienceIndexingResult? Indexing = null) +{ + /// The Experience Record's ID, when one exists. No ID is issued when nothing was persisted. + public Guid? ExperienceId => Record?.ExperienceId; + + /// The record's lifecycle status, when one exists. + public ExperienceStatus? Status => Record?.Status; + + /// + /// Whether an Experience Record for this run is durably stored and confirmed by its initial + /// lifecycle event -- true only for , + /// , and + /// . A database failure is never reported as + /// durable success. + /// + public bool IsDurable => Outcome is FinalizationOutcome.Validated + or FinalizationOutcome.Quarantined + or FinalizationOutcome.AlreadyFinalized; +} diff --git a/src/AgentExperience.Core/Finalization/FinalizeExperienceRequest.cs b/src/AgentExperience.Core/Finalization/FinalizeExperienceRequest.cs new file mode 100644 index 0000000..3c1da2d --- /dev/null +++ b/src/AgentExperience.Core/Finalization/FinalizeExperienceRequest.cs @@ -0,0 +1,40 @@ +using AgentExperience.Abstractions; +using AgentExperience.Core.Verification; + +namespace AgentExperience.Core.Finalization; + +/// +/// One request to turn a captured, completed run into a durable Experience Record, submitted to +/// . +/// +/// +/// +/// No evaluation is accepted here. The request carries the host-closed round, the artifact revision, +/// the declared required checks, and the evidence; finalization runs +/// itself against exactly those, so an evaluation +/// computed for some other run can never be substituted for this one's. +/// +/// +/// is caller-supplied rather than read from a clock so a retry of the same +/// run is the same request. The record ID, the reflection ID, and the initial lifecycle event ID are +/// all derived from , so a retry can never produce a second record or a second +/// initial confirmation (see ). +/// +/// +/// The captured run to finalize. Must not be ; the run must exist in the capture service and must already carry an . +/// What the host has established the caller may do. The run's own must lie within it or nothing is stored. +/// The verification round the host closed for this run, or when the host has closed none (which evaluates to ). +/// The task's declared required checks. An empty set can never verify. +/// All evidence available for this run, in the order it was produced. Aggregation filters it to and itself. +/// The artifact revision verification is being judged against. Must be non-blank. +/// The host's storage-policy decision. A decision that does not permit storage writes nothing at all. +/// When the host decided to finalize this run. Stamped onto the record and its initial lifecycle event (truncated to whole microseconds in UTC, which is the precision the store keeps). +public sealed record FinalizeExperienceRequest( + Guid RunId, + AuthorizationContext Authorization, + ClosedVerificationRound? ClosedRound, + IReadOnlyList RequiredChecks, + IReadOnlyList Evidence, + string CurrentArtifactRevision, + StorageDecision StorageDecision, + DateTimeOffset FinalizedAt); diff --git a/src/AgentExperience.Core/Finalization/StorageDecision.cs b/src/AgentExperience.Core/Finalization/StorageDecision.cs new file mode 100644 index 0000000..279bb57 --- /dev/null +++ b/src/AgentExperience.Core/Finalization/StorageDecision.cs @@ -0,0 +1,30 @@ +namespace AgentExperience.Core.Finalization; + +/// +/// The host's decision on whether a finalized run may be persisted at all, carried in a +/// . +/// +/// +/// +/// Storage policy belongs to the host, not to this library: only the host knows its retention rules, +/// its data-residency obligations, and its own risk appetite. Core therefore calls no policy port and +/// makes no policy decision of its own -- it reads this value and obeys it. A decision that does not +/// permit storage stops finalization before any store call, so nothing at all is written, whatever +/// the run's verification says. +/// +/// +/// is content-free: it is surfaced back to the host on the finalization result +/// and must never carry record payload, captured content, or private reasoning. +/// +/// +/// when the host permits this run to be persisted as an Experience Record. +/// Optional, auditable, content-free explanation of the decision (most usefully, why storage was denied). +public sealed record StorageDecision(bool Permitted, string? Reason = null) +{ + /// A decision that permits storage, with no reason attached. + public static StorageDecision Permit { get; } = new(Permitted: true); + + /// Creates a decision that denies storage. + /// A content-free explanation of why storage was denied. + public static StorageDecision Deny(string? reason = null) => new(Permitted: false, reason); +} diff --git a/src/AgentExperience.Core/Indexing/ExperienceIndexingService.cs b/src/AgentExperience.Core/Indexing/ExperienceIndexingService.cs new file mode 100644 index 0000000..75086b8 --- /dev/null +++ b/src/AgentExperience.Core/Indexing/ExperienceIndexingService.cs @@ -0,0 +1,523 @@ +using AgentExperience.Abstractions; +using AgentExperience.Core.Retrieval; + +namespace AgentExperience.Core.Indexing; + +/// +/// The single Core call that gives a stored Experience Record a vector: it reads what the record's +/// sanitized retrieval summary currently is, embeds it through a replaceable provider port, and asks +/// the index to store the result conditionally on the record still being exactly where and what it +/// was read as. +/// +/// +/// +/// Embeddings are derived data, and this is why nothing here can break a record. Indexing runs +/// strictly after the canonical commit, never inside it. Every failure -- an unavailable provider, a +/// rejected write, an unreachable index -- comes back as a structured result, and the record stays +/// committed, durable, and text-searchable either way. Nothing this service does can change a +/// record's status, revision, confidence, or eligibility. +/// +/// +/// Only the sanitized retrieval summary is ever sent to a provider. That is the task ID, the +/// sanitized task summary, and the reflection's lesson -- exactly the three fields the text index +/// analyzes (). Attempts, tool calls, evidence, provenance, +/// and environment metadata never leave the database. The summary is read from the index's own scan +/// rather than from a record the caller happens to be holding, so the text that is embedded is the +/// text that is really stored, at the revision it is really stored at. +/// +/// +/// Unchanged means free. The content hash covers the model ID and the normalized summary, so a +/// record whose stored vector already came from this model and this text is reported +/// before any provider call. That is what +/// makes a re-index pass idempotent: running it twice over unchanged records costs two reads and +/// nothing else. +/// +/// +/// Writes are conditional, and this service never forces one. The descriptor it sends carries +/// the revision the summary was read at, and the index applies the write only while the record is +/// still at that revision. A record that moved is and a +/// record that is gone is -- never a retry loop that +/// eventually overwrites newer state, and never a row for a record that no longer exists. +/// +/// +/// Only records a search could return are ever embedded. Both the scan and the post-commit hook +/// apply the same status filter and confidence floor the vector search applies, so a +/// , , +/// , or record's +/// summary and lesson never leave the database for a third-party provider -- its vector could never be +/// returned anyway. +/// +/// +/// Only the caller's cancellation escapes. An observed +/// while the caller's own token is cancelled propagates, so a cancelled pass is never reported +/// as a completed one. A port that cancels for its own reasons does not: that is the ordinary shape of +/// a client-side request timeout, and one slow record must not abandon a whole pass, so it is reported +/// as like any other provider failure. +/// +/// +public sealed class ExperienceIndexingService +{ + private static readonly IReadOnlyList NoErrors = []; + + private static readonly IReadOnlyList NoRecords = []; + + private readonly IExperienceEmbeddingIndex _index; + private readonly IExperienceEmbeddingGenerator _generator; + + /// Creates an indexing service over the embedding index and the provider that produces vectors. + /// Where vectors are stored, listed, and searched. + /// The replaceable provider port. Its model ID and dimension are read once, here. + /// + /// Optional. The retrieval policy whose decides + /// which records are worth embedding -- deliberately the same object retrieval runs under, because + /// a record the search would never return must never be sent to a provider. Defaults to + /// . + /// + /// or is . + /// reports a blank or over-long model ID. + /// reports a dimension outside 1... + public ExperienceIndexingService( + IExperienceEmbeddingIndex index, + IExperienceEmbeddingGenerator generator, + RetrievalPolicy? policy = null) + { + ArgumentNullException.ThrowIfNull(index); + ArgumentNullException.ThrowIfNull(generator); + + // Read once, at construction: the content hash covers the model ID, so both have to be stable + // and known before any provider call. A generator that cannot say what it is fails here, + // at wiring time, rather than producing vectors nobody can decide the comparability of. + ModelId = generator.ModelId; + if (string.IsNullOrWhiteSpace(ModelId)) + { + throw new ArgumentException( + "The embedding generator must report a non-blank model ID: it is part of every stored embedding's " + + "content hash and decides which vectors are comparable.", + nameof(generator)); + } + + if (ModelId.Length > ExperienceEmbeddingDescriptor.MaxModelIdLength) + { + throw new ArgumentException( + $"The embedding generator's model ID must be at most {ExperienceEmbeddingDescriptor.MaxModelIdLength} characters.", + nameof(generator)); + } + + Dimension = generator.Dimension; + if (Dimension is < 1 or > ExperienceEmbeddingDescriptor.MaxDimension) + { + throw new ArgumentOutOfRangeException( + nameof(generator), + Dimension, + $"The embedding generator's dimension must be between 1 and {ExperienceEmbeddingDescriptor.MaxDimension}."); + } + + _index = index; + _generator = generator; + MinimumConfidence = (policy ?? RetrievalPolicy.Default).MinimumConfidence; + } + + /// + /// The only statuses whose records are ever embedded: exactly the ones a vector search can return + /// (). A + /// , , + /// , or record's + /// summary and lesson are never handed to a provider, because its vector could never be returned. + /// + public static IReadOnlyList IndexableStatuses => ExperienceRetrievalService.EligibleStatuses; + + /// The model every vector this service writes is stamped with, and compared under at query time. + public string ModelId { get; } + + /// The width of every vector this service writes. A provider that returns any other width is rejected. + public int Dimension { get; } + + /// The reuse-confidence floor below which a record is not worth embedding, because the search would not return it either. + public double MinimumConfidence { get; } + + /// + /// Whether a record in this state could ever be returned by a vector search, and is therefore worth + /// sending to an embedding provider. Callers that already hold a record -- the post-commit hook, + /// for one -- check this before any provider call rather than discovering it from an empty scan. + /// + /// The record's lifecycle status. + /// The record's reuse confidence. + /// when the record is worth embedding. + public bool IsIndexable(ExperienceStatus status, double reuseConfidence) => + IndexableStatuses.Contains(status) && reuseConfidence >= MinimumConfidence; + + /// + /// Indexes one record: reads its current summary and revision, skips it when this model already + /// embedded exactly that text, and otherwise embeds it and writes the vector conditionally. + /// + /// What the host has established the caller may do. + /// The exact scope the record must lie in. Never treated as authority. + /// The record to index. + /// Cancels the operation. Cancellation is not an expected condition and propagates unwrapped. + /// + /// A structured result; never an exception for an expected condition. A record the index does not + /// list is , which covers all three ways that can + /// happen: it was deleted, it is in another scope, or its status or confidence means a search could + /// never return it. A caller that already holds the record -- the post-commit hook -- distinguishes + /// the last case itself with before calling. + /// + /// or is . + /// is . + /// was cancelled, or a port cancelled. + public async Task IndexAsync( + AuthorizationContext authorization, + Scope scope, + Guid experienceId, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(authorization); + ArgumentNullException.ThrowIfNull(scope); + if (experienceId == Guid.Empty) + { + throw new ArgumentException("ExperienceId must not be an empty GUID.", nameof(experienceId)); + } + + var pass = await ReindexAsync( + authorization, + new ReindexExperienceRequest(scope, [experienceId], Limit: 1), + cancellationToken).ConfigureAwait(false); + + if (pass.Records.Count > 0) + { + return pass.Records[0]; + } + + return pass.Outcome switch + { + // The scan ran and listed nothing: the record is not in this scope, or no longer exists. + ExperienceReindexOutcome.Completed => new(ExperienceIndexingOutcome.Missing, experienceId, null, null), + ExperienceReindexOutcome.Denied => new(ExperienceIndexingOutcome.Denied, experienceId, null, pass.Failure), + _ => new(ExperienceIndexingOutcome.IndexFailed, experienceId, null, pass.Failure), + }; + } + + /// + /// Runs one scoped, explicit re-index pass: lists what the scope holds, and for each record either + /// skips it (this model already embedded exactly that text) or re-embeds and rewrites it. + /// Re-running the same pass over unchanged records calls no provider and writes nothing. + /// + /// What the host has established the caller may do. + /// The scope to re-index, optionally narrowed to specific records, and the bound on how many to consider. + /// Cancels the operation. Cancellation is not an expected condition and propagates unwrapped. + /// A structured result with the tally and one entry per considered record. + /// , , or its , is . + /// was cancelled, or a port cancelled. + public async Task ReindexAsync( + AuthorizationContext authorization, + ReindexExperienceRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(authorization); + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(request.Scope, $"{nameof(request)}.{nameof(request.Scope)}"); + + cancellationToken.ThrowIfCancellationRequested(); + + ExperienceIndexScanResult scan; + try + { + scan = await _index + .ScanAsync( + authorization, + new ExperienceIndexScan( + request.Scope, + ModelId, + IndexableStatuses, + MinimumConfidence, + request.ExperienceIds, + request.Limit, + request.StartAfterId), + cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return Ended( + ExperienceReindexOutcome.Failed, + new ExperienceIndexingFailure( + $"The embedding index threw {ex.GetType().FullName} while listing what to re-index.", + NoErrors, + ex)); + } + + if (scan is null) + { + return Ended( + ExperienceReindexOutcome.Failed, + new ExperienceIndexingFailure("The embedding index returned no scan result at all.", NoErrors, Exception: null)); + } + + switch (scan.Outcome) + { + case ExperienceStoreOutcome.Found: + break; + + case ExperienceStoreOutcome.Denied: + return Ended( + ExperienceReindexOutcome.Denied, + new ExperienceIndexingFailure( + "The requested scope lies outside the host-established authorization; nothing was read, embedded, or written.", + NoErrors, + Exception: null)); + + case ExperienceStoreOutcome.Invalid: + return Ended( + ExperienceReindexOutcome.Invalid, + new ExperienceIndexingFailure( + "The embedding index rejected the scan as malformed. See the validation errors.", + scan.Errors ?? NoErrors, + Exception: null)); + + default: + return Ended( + ExperienceReindexOutcome.Failed, + new ExperienceIndexingFailure( + $"The embedding index returned '{scan.Outcome}' rather than '{ExperienceStoreOutcome.Found}'.", + NoErrors, + Exception: null)); + } + + if (scan.Targets is null) + { + return Ended( + ExperienceReindexOutcome.Failed, + new ExperienceIndexingFailure("The embedding index reported a successful scan but returned no target list.", NoErrors, Exception: null)); + } + + var results = new List(scan.Targets.Count); + var indexed = 0; + var skipped = 0; + var rejected = 0; + var failed = 0; + + foreach (var target in scan.Targets) + { + if (target is null) + { + return Ended( + ExperienceReindexOutcome.Failed, + new ExperienceIndexingFailure("The embedding index returned a null scan target.", NoErrors, Exception: null)); + } + + var result = await IndexTargetAsync(authorization, request.Scope, target, cancellationToken).ConfigureAwait(false); + results.Add(result); + + switch (result.Outcome) + { + case ExperienceIndexingOutcome.Indexed: + indexed++; + break; + case ExperienceIndexingOutcome.Skipped: + skipped++; + break; + case ExperienceIndexingOutcome.Stale: + case ExperienceIndexingOutcome.Missing: + case ExperienceIndexingOutcome.Denied: + case ExperienceIndexingOutcome.Ineligible: + rejected++; + break; + default: + failed++; + break; + } + } + + return new( + ExperienceReindexOutcome.Completed, + results.Count, + indexed, + skipped, + rejected, + failed, + results, + Failure: null, + scan.LastExaminedId ?? (results.Count > 0 ? results[^1].ExperienceId : null)); + } + + /// + /// Indexes one already-listed record: decide whether anything changed, embed only if it did, then + /// write conditionally on the revision the target was read at. + /// + private async Task IndexTargetAsync( + AuthorizationContext authorization, + Scope scope, + ExperienceIndexTarget target, + CancellationToken cancellationToken) + { + // Only the caller's own cancellation ends a pass. A provider that cancels for its own reasons + // is the ordinary shape of a client-side timeout -- HttpClient raises its request timeout as a + // TaskCanceledException with the caller's token untouched -- and one slow record must not + // abandon a whole re-index with no per-record results at all. + bool CallerCancelled() => cancellationToken.IsCancellationRequested; + + var summary = target.Summary ?? string.Empty; + var contentHash = ExperienceEmbeddingDescriptor.ComputeContentHash(ModelId, summary); + + if (target.Stored is { } stored + && string.Equals(stored.ModelId, ModelId, StringComparison.Ordinal) + && string.Equals(stored.ContentHash, contentHash, StringComparison.Ordinal)) + { + // Same model, same text: the stored vector is exactly what this pass would have produced. + // No provider call, no write -- this is the whole point of storing the hash. + return new(ExperienceIndexingOutcome.Skipped, target.ExperienceId, stored, null); + } + + ReadOnlyMemory vector; + try + { + vector = await _generator.GenerateAsync(summary, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (CallerCancelled()) + { + // Only the caller's own cancellation ends the pass. + throw; + } + catch (Exception ex) + { + // Caught, never rethrown: the record stays committed and text-searchable, and this is + // reported as retryable rather than failing whatever called us. A provider-side timeout + // arrives here as an OperationCanceledException and is a provider failure like any other. + return new( + ExperienceIndexingOutcome.ProviderFailed, + target.ExperienceId, + target.Stored, + new ExperienceIndexingFailure( + $"The embedding provider threw {ex.GetType().FullName}; the record is unchanged and still indexable later.", + NoErrors, + ex)); + } + + if (vector.Length != Dimension) + { + return new( + ExperienceIndexingOutcome.ProviderFailed, + target.ExperienceId, + target.Stored, + new ExperienceIndexingFailure( + $"The embedding provider returned a {vector.Length}-component vector where {Dimension} were declared; " + + "a stored descriptor must never disagree with its own vector.", + NoErrors, + Exception: null)); + } + + if (!IsFinite(vector)) + { + // A NaN or an infinity is the right width and so would pass every later check, then be + // rejected by the database -- reported as retryable, which is a retry loop that never ends. + // It is a provider failure, and it is caught here, once. + return new( + ExperienceIndexingOutcome.ProviderFailed, + target.ExperienceId, + target.Stored, + new ExperienceIndexingFailure( + "The embedding provider returned a vector with a non-finite component; a stored vector must be " + + "entirely finite or every distance computed against it is meaningless.", + NoErrors, + Exception: null)); + } + + var descriptor = new ExperienceEmbeddingDescriptor(ModelId, Dimension, contentHash, target.SourceRevision); + + ExperienceIndexWriteResult write; + try + { + write = await _index + .WriteAsync(authorization, new ExperienceIndexWrite(scope, target.ExperienceId, descriptor, vector), cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (CallerCancelled()) + { + throw; + } + catch (Exception ex) + { + return new( + ExperienceIndexingOutcome.IndexFailed, + target.ExperienceId, + target.Stored, + new ExperienceIndexingFailure( + $"The embedding index threw {ex.GetType().FullName} while storing the vector; the record is unchanged.", + NoErrors, + ex)); + } + + return write?.Outcome switch + { + ExperienceIndexOutcome.Written => new(ExperienceIndexingOutcome.Indexed, target.ExperienceId, descriptor, null), + ExperienceIndexOutcome.Stale => new( + ExperienceIndexingOutcome.Stale, + target.ExperienceId, + target.Stored, + new ExperienceIndexingFailure( + // Two ways to lose: the record moved on, or another writer stored a vector from a + // newer revision first. Naming the same revision twice would describe neither. + write.CurrentRevision != target.SourceRevision + ? $"The record moved to revision {write.CurrentRevision} after its summary was read at revision {target.SourceRevision}; " + + "the write was rejected and the stored vector is unchanged." + : $"A concurrent write had already stored a vector for this record from revision {target.SourceRevision} or newer; " + + "this write was rejected and the stored vector is unchanged.", + NoErrors, + Exception: null)), + ExperienceIndexOutcome.Missing => new( + ExperienceIndexingOutcome.Missing, + target.ExperienceId, + null, + new ExperienceIndexingFailure( + "The record no longer exists within the requested scope; nothing was written and no row was created.", + NoErrors, + Exception: null)), + ExperienceIndexOutcome.Denied => new( + ExperienceIndexingOutcome.Denied, + target.ExperienceId, + target.Stored, + new ExperienceIndexingFailure( + "The embedding index refused the record's scope as outside the host-established authorization; nothing was written.", + NoErrors, + Exception: null)), + ExperienceIndexOutcome.Invalid => new( + ExperienceIndexingOutcome.IndexFailed, + target.ExperienceId, + target.Stored, + new ExperienceIndexingFailure( + "The embedding index rejected the write as malformed. See the validation errors.", + write.Errors ?? NoErrors, + Exception: null)), + _ => new( + ExperienceIndexingOutcome.IndexFailed, + target.ExperienceId, + target.Stored, + new ExperienceIndexingFailure( + write is null + ? "The embedding index returned no write result at all." + : $"The embedding index returned '{write.Outcome}', which is not a write outcome.", + NoErrors, + Exception: null)), + }; + } + + /// Whether every component is a real number. A NaN or an infinity poisons every distance computed against the vector. + private static bool IsFinite(ReadOnlyMemory vector) + { + foreach (var component in vector.Span) + { + if (!float.IsFinite(component)) + { + return false; + } + } + + return true; + } + + private static ExperienceReindexResult Ended(ExperienceReindexOutcome outcome, ExperienceIndexingFailure failure) => + new(outcome, 0, 0, 0, 0, 0, NoRecords, failure, LastExaminedId: null); +} diff --git a/src/AgentExperience.Core/Indexing/IndexingResults.cs b/src/AgentExperience.Core/Indexing/IndexingResults.cs new file mode 100644 index 0000000..ab6ce58 --- /dev/null +++ b/src/AgentExperience.Core/Indexing/IndexingResults.cs @@ -0,0 +1,167 @@ +using AgentExperience.Abstractions; + +namespace AgentExperience.Core.Indexing; + +/// One scoped, explicit re-index pass. +/// The exact scope to re-index within. Never treated as authority, and never widened. +/// +/// Optional. Exactly which records to consider; considers every record in the +/// scope, bounded by . Re-indexing is never implicit and never repository-wide: +/// a caller always names a scope, and may narrow it further to specific records. +/// +/// +/// The most records one pass may consider, from to +/// . A pass is a batch job that pages; it never loads a +/// whole scope at once. +/// +/// +/// Optional keyset cursor. Records are always considered in ascending +/// order, so passing the previous pass's +/// here walks a scope larger than +/// to the end; a pass that returns a +/// has reached it. +/// +public sealed record ReindexExperienceRequest( + Scope Scope, + IReadOnlyList? ExperienceIds = null, + int Limit = ExperienceIndexScan.DefaultLimit, + Guid? StartAfterId = null); + +/// What indexing one record ended as. +public enum ExperienceIndexingOutcome +{ + /// The record was embedded and its vector stored, replacing any previous vector for it. + Indexed, + + /// + /// The stored vector was already produced by this model from exactly this text, so nothing + /// changed: no provider was called and nothing was written. This is what makes re-indexing + /// idempotent and cheap. + /// + Skipped, + + /// + /// The record's revision moved between being read and the write landing, so the write was + /// rejected and the stored vector is unchanged. Re-running the pass picks up the new revision. + /// + Stale, + + /// + /// No such record exists within the requested scope -- it was deleted, or it is in another scope. + /// Nothing was written and no row was created. + /// + Missing, + + /// The request scope lies outside the host-established authorization. Nothing was read, embedded, or written. + Denied, + + /// + /// The record's status or reuse confidence means a vector search could never return it, so it was + /// not embedded at all. Nothing was written, and -- the point of the check -- its task summary and + /// reflection lesson never left the database for a third-party provider. + /// + Ineligible, + + /// + /// The embedding provider failed, timed out, or returned something unusable. The record is + /// untouched: still committed, still durable, still text-searchable -- and still indexable by a + /// later pass. + /// + ProviderFailed, + + /// + /// The index itself failed or refused the write. As with a provider failure, the record is + /// untouched and a later pass can try again. + /// + IndexFailed, +} + +/// +/// Why indexing could not do what it was asked. is safe to log or surface: it +/// never carries record content. is not held to that standard -- it +/// is whatever the port or the provider threw, and a driver or HTTP client message can quote SQL +/// text, parameter values, connection detail, or a request body. Treat it as local diagnostics only. +/// +/// A human-readable, content-free explanation. +/// The index's validation errors when it reported the write malformed; otherwise empty. +/// The original failure, when one was caught. Diagnostic only; may carry adapter detail. +public sealed record ExperienceIndexingFailure( + string Reason, + IReadOnlyList Errors, + Exception? Exception); + +/// +/// The result of indexing one record. It always says what happened; it is never an exception for an +/// expected condition, and never a silent success. +/// +/// What happened. +/// The record this result is about. +/// +/// The descriptor now stored for the record: the one just written +/// () or the one already there +/// (). whenever nothing is +/// known to be stored. +/// +/// Why the pass could not index this record; otherwise . +public sealed record ExperienceIndexingResult( + ExperienceIndexingOutcome Outcome, + Guid ExperienceId, + ExperienceEmbeddingDescriptor? Descriptor, + ExperienceIndexingFailure? Failure) +{ + /// Whether a vector for this record is now stored under the current model, whether this call wrote it or found it already there. + public bool IsIndexed => Outcome is ExperienceIndexingOutcome.Indexed or ExperienceIndexingOutcome.Skipped; + + /// + /// Whether running the same pass again could still succeed. A provider or index failure is + /// transient by nature, and a stale revision simply means the record moved on and should be read + /// again. A missing record and a denied scope are not: repeating them changes nothing. + /// + public bool IsRetryable => Outcome is ExperienceIndexingOutcome.ProviderFailed + or ExperienceIndexingOutcome.IndexFailed + or ExperienceIndexingOutcome.Stale; +} + +/// What a whole re-index pass ended as. +public enum ExperienceReindexOutcome +{ + /// The pass ran to the end. Individual records may still have been skipped, rejected, or failed; see the per-record results. + Completed, + + /// The request scope lies outside the host-established authorization. Nothing was read, embedded, or written. + Denied, + + /// The request was malformed, so the index refused to list anything. Nothing was read, embedded, or written. + Invalid, + + /// The pass could not list what to consider at all, so no record was examined. + Failed, +} + +/// +/// The result of one scoped re-index pass: the tally, plus a result for every record it considered. +/// +/// What the pass as a whole ended as. +/// How many records the pass considered. Bounded by the request's limit; reaching it means there may be more. +/// How many were embedded and written. +/// How many already had this model's vector for exactly this text, so no provider was called for them. +/// How many were rejected without being written -- stale or missing. Not failures: the index declined to overwrite state it does not own. +/// How many failed against the provider or the index. Each is retryable. +/// One result per considered record, in ascending order. +/// Why the pass itself could not run, when is or ; otherwise . +/// +/// The last record this pass considered, to pass as the next pass's +/// . means the pass +/// considered nothing, which is how a caller knows the scope is exhausted rather than looping over the +/// same first page forever. +/// +public sealed record ExperienceReindexResult( + ExperienceReindexOutcome Outcome, + int Examined, + int Indexed, + int Skipped, + int Rejected, + int Failed, + IReadOnlyList Records, + ExperienceIndexingFailure? Failure, + Guid? LastExaminedId = null); diff --git a/src/AgentExperience.Core/Retrieval/ExperienceRetrievalService.cs b/src/AgentExperience.Core/Retrieval/ExperienceRetrievalService.cs new file mode 100644 index 0000000..d47c1b5 --- /dev/null +++ b/src/AgentExperience.Core/Retrieval/ExperienceRetrievalService.cs @@ -0,0 +1,853 @@ +using AgentExperience.Abstractions; + +namespace AgentExperience.Core.Retrieval; + +/// +/// The single Core call that finds experience applicable to a task: it asks an +/// for scope-, status- and confidence-filtered candidates +/// matched on task text, optionally asks an for the same +/// thing matched on meaning, merges the two, decides the remaining eligibility itself, and ranks what +/// survives with weights and components it reports back in full. +/// +/// +/// +/// Filter, then rank. Nothing is ever scored before it is known to be reusable. The database +/// decides scope, status, and the confidence floor -- identically for both channels; Core decides +/// expiry and environment compatibility, because both depend on the clock and on the request rather +/// than on stored state alone. Only and +/// records are ever eligible -- a +/// , , +/// , , +/// , or record is +/// dropped whatever its text or vector match. +/// +/// +/// Two channels, one answer. When an embedding index and an embedding generator are both +/// wired in, the task text is also embedded and searched as a vector, inside the same timeout and +/// under the same candidate ceiling. The two candidate lists are then deduplicated by +/// , and a record found by both keeps the higher +/// of its two normalized relevances. Ranking then runs once, over the merged list, with the same five +/// weights as before: there is no sixth axis and no "found by both" bonus. +/// +/// +/// An embedding never decides anything. It can only make a record a candidate; eligibility, +/// status, and confidence are untouched by it. And when the vector channel cannot be trusted -- no +/// provider, a provider that failed or timed out on its own, or stored vectors from another model or +/// another dimension -- the result is an explicit text-only answer carrying VectorFallback with +/// the reason, and the text candidates still come back. No incompatible comparison is ever attempted, +/// and nothing the vector channel does can turn a good text answer into a failure: only cancellation +/// of the caller's own token ever escapes it. +/// +/// +/// The candidate ceiling is a real recall limit. Each channel returns at most +/// candidates, ordered by its own relevance, and ranking +/// only ever sees those. So a record with a weaker match but strong confidence, recency, or status is +/// not ranked at all once that many stronger matches exist in both channels -- the weighting can only +/// reorder what the ceiling let through. When either channel reached its ceiling the result says so +/// (Truncated); the records beyond it are not in the exclusion list either, because no +/// eligibility check ever looked at them. Raise the ceiling, or narrow the task text, when that +/// matters. +/// +/// +/// Bounded, and a timeout is not an error. The whole call is bounded by +/// , measured with the injected . +/// Exceeding it returns an empty result carrying and +/// the request's correlation ID -- never an exception -- so an agent whose memory is slow simply runs +/// without it. Caller cancellation is different in kind and propagates as an unwrapped +/// ; the timeout is reported before the inner token is +/// cancelled, so a token-honouring source cannot race a cancellation ahead of the timeout report. +/// +/// +/// Fail-closed. An authorization mismatch, a source failure inside the timeout, a candidate +/// that cannot be read, and a candidate returned outside the requested scope all produce an +/// empty result rather than an unfiltered one. Retrieval never answers with experience it +/// could not fully check. +/// +/// +/// This service does not build an injectable payload: it returns ranked records and the evidence for +/// their ranking, and what a host does with them is a separate decision. It also never +/// writes an embedding -- producing and storing them is +/// 's job, and happens after a +/// record is already committed. +/// +/// +public sealed class ExperienceRetrievalService +{ + /// + /// The only statuses a record may be in and still be returned. This is the eligibility rule, not + /// a default: a record in any other status is never injectable, whatever its text match or + /// confidence. + /// + public static IReadOnlyList EligibleStatuses { get; } = + [ExperienceStatus.Validated, ExperienceStatus.Reinforced]; + + /// The status component's value for a record. + public const double ValidatedStatusScore = 0.5; + + /// + /// The status component's value for a record: reuse was + /// observed to succeed again, which is the strongest evidence this axis can carry. + /// + public const double ReinforcedStatusScore = 1.0; + + /// + /// The environment component's value for a record that satisfied the request's required + /// attributes -- which, by construction, every ranked record did, since a mismatch excludes the + /// record before ranking. It is reported anyway, with its effective weight, so the score a host + /// sees always adds up from every documented axis. + /// + public const double CompatibleEnvironmentScore = 1.0; + + private static readonly IReadOnlyList NoRecords = []; + + private static readonly IReadOnlyList NoExclusions = []; + + /// + /// The text-only signal for a deployment that has no vector channel at all. It is a statement + /// about the wiring, not about this call, so it is a single shared instance. + /// + private static readonly VectorChannelFallback NotConfigured = new( + TextOnlyReason.NotConfigured, + "No embedding index or embedding generator is registered, so this retrieval has no vector channel.", + Exception: null); + + private readonly IExperienceCandidateSource _candidateSource; + private readonly RetrievalPolicy _policy; + private readonly RankingWeights _weights; + private readonly TimeProvider _timeProvider; + private readonly IExperienceEmbeddingIndex? _embeddingIndex; + private readonly IExperienceEmbeddingGenerator? _embeddingGenerator; + + /// + /// Creates a text-only retrieval service over a candidate source, its policy, its weights, and the + /// clock it measures with. Every result it produces is flagged + /// with + /// . + /// + /// Where scope-, status- and confidence-filtered text matches come from. + /// The timeout, confidence floor, expiry, recency half-life, and candidate bound. + /// The weights applied to each normalized ranking component. + /// The clock the timeout, expiry, and recency are measured with. + /// Any argument is . + public ExperienceRetrievalService( + IExperienceCandidateSource candidateSource, + RetrievalPolicy policy, + RankingWeights weights, + TimeProvider timeProvider) + : this(candidateSource, policy, weights, timeProvider, embeddingIndex: null, embeddingGenerator: null) + { + } + + /// + /// Creates a retrieval service with an optional vector channel alongside the text one. The vector + /// channel is active only when both and + /// are supplied: one without the other cannot produce a + /// comparison, so it is treated as no vector channel at all rather than as a failure on every + /// call. + /// + /// Where scope-, status- and confidence-filtered text matches come from. + /// The timeout, confidence floor, expiry, recency half-life, and candidate bound. Both channels run under it. + /// The weights applied to each normalized ranking component. Unchanged by hybrid retrieval: still five. + /// The clock the timeout, expiry, and recency are measured with. + /// Optional. Where scope-, status- and confidence-filtered vector matches come from. + /// Optional. What turns the request's task text into a query vector. + /// Any non-optional argument is . + public ExperienceRetrievalService( + IExperienceCandidateSource candidateSource, + RetrievalPolicy policy, + RankingWeights weights, + TimeProvider timeProvider, + IExperienceEmbeddingIndex? embeddingIndex, + IExperienceEmbeddingGenerator? embeddingGenerator) + { + ArgumentNullException.ThrowIfNull(candidateSource); + ArgumentNullException.ThrowIfNull(policy); + ArgumentNullException.ThrowIfNull(weights); + ArgumentNullException.ThrowIfNull(timeProvider); + + _candidateSource = candidateSource; + _policy = policy; + _weights = weights; + _timeProvider = timeProvider; + _embeddingIndex = embeddingIndex; + _embeddingGenerator = embeddingGenerator; + } + + /// The policy this service runs under. + public RetrievalPolicy Policy => _policy; + + /// The weights this service ranks with. + public RankingWeights Weights => _weights; + + /// + /// Whether this service has a vector channel at all. means every result + /// is text-only for ; means the + /// channel is wired, not that it will succeed on any given call. + /// + public bool HybridEnabled => _embeddingIndex is not null && _embeddingGenerator is not null; + + /// + /// Retrieves the experience that applies to , ranked, or an empty + /// result when it is denied, times out, or fails. + /// + /// The scope, task text, required environment attributes, and correlation ID to retrieve for. + /// Cancels the operation. Cancellation is not an expected condition and propagates unwrapped, distinct from the timeout fallback. + /// A result that always says what happened; it is empty unless . + /// , or its or , is . + /// is blank or longer than , or is not strictly positive or exceeds . + /// was cancelled. + public async Task RetrieveAsync( + RetrieveExperienceRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(request.Authorization, $"{nameof(request)}.{nameof(request.Authorization)}"); + ArgumentNullException.ThrowIfNull(request.Scope, $"{nameof(request)}.{nameof(request.Scope)}"); + + if (string.IsNullOrWhiteSpace(request.TaskText)) + { + throw new ArgumentException("TaskText must be set to the task text to match experience against.", nameof(request)); + } + + if (request.TaskText.Length > ExperienceCandidateQuery.MaxTaskTextLength) + { + throw new ArgumentException( + $"TaskText must be at most {ExperienceCandidateQuery.MaxTaskTextLength} characters.", + nameof(request)); + } + + if (request.Limit is <= 0) + { + throw new ArgumentException("Limit must be strictly positive when specified.", nameof(request)); + } + + if (request.Limit > _policy.CandidateLimit) + { + // Rejected rather than quietly capped: asking for more than the search will ever consider + // is a configuration mistake, and silently returning fewer would hide it. + throw new ArgumentException( + $"Limit must be at most the policy's candidate limit ({_policy.CandidateLimit}); a larger limit could never be satisfied.", + nameof(request)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var startedAt = _timeProvider.GetTimestamp(); + var unrestricted = request.RequiredEnvironmentAttributes is null or { Count: 0 }; + + // Authorization first, and fail-closed: a request scope outside the host's authorization ends + // here, so no search is issued at all and nothing about foreign scopes is observable. + if (!request.Authorization.Permits(request.Scope)) + { + // Denied before either channel is touched, so neither the database nor the embedding + // provider ever sees a request outside the host's authorization. + return Empty(RetrievalOutcome.Denied, request, unrestricted, startedAt, failure: null, EndedEarly()); + } + + // One more than the ceiling: the extra candidate is never ranked, it only distinguishes "exactly + // at the ceiling" from "more existed", which is what the result's Truncated flag reports. + var query = new ExperienceCandidateQuery( + request.Scope, + request.TaskText, + EligibleStatuses, + _policy.MinimumConfidence, + _policy.CandidateLimit + 1); + + // Cancelled only after a timeout has been reported (it carries no timer of its own), so a + // token-honouring source can never race a cancellation failure ahead of the timeout report. + var inner = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + + // Task.Run also bounds a source that blocks or throws synchronously. Both channels start here + // and run concurrently, so the one timeout below bounds the pair rather than each in turn. + var work = Task.Run(() => SearchChannelsAsync(request, query, inner.Token), CancellationToken.None); + + // Set once the abandoned-search path has taken ownership of disposing the token source; every + // other exit -- returned, thrown, or an unexpected failure from WaitAsync or Rank -- disposes it + // here, so a long-lived caller token never accumulates registrations. + var abandoned = false; + try + { + ChannelOutcome outcome; + try + { + outcome = await work.WaitAsync(_policy.Timeout, _timeProvider, cancellationToken).ConfigureAwait(false); + } + catch (TimeoutException) + { + // Report first, so a token-honouring source's cancellation cannot be reported in its place. + abandoned = true; + Abandon(work, inner); + return Empty(RetrievalOutcome.TimedOut, request, unrestricted, startedAt, failure: null, EndedEarly()); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // The caller cancelled. Distinct from the timeout in kind and in reporting: it propagates. + abandoned = true; + Abandon(work, inner); + throw; + } + catch (OperationCanceledException ex) + { + // Neither the caller nor the timeout: a channel cancelled for its own reasons. Fail-closed. + abandoned = true; + Abandon(work, inner); + return Empty( + RetrievalOutcome.Failed, + request, + unrestricted, + startedAt, + new RetrievalFailure("A retrieval channel cancelled the search for its own reasons.", ex), + EndedEarly()); + } + + if (outcome.Text.Failure is { } failure) + { + // The text channel is the one that can end the call: it is the channel every + // deployment has, and answering from vectors alone would be an unfiltered-by-text + // result the caller never asked for. + return Empty(RetrievalOutcome.Failed, request, unrestricted, startedAt, failure, outcome.Vector.Fallback); + } + + return Rank(request, outcome, unrestricted, startedAt); + } + finally + { + if (!abandoned) + { + inner.Dispose(); + } + } + } + + /// + /// Runs both channels concurrently under the caller's one bound, and never lets the vector + /// channel's trouble become the call's. Cancellation from either escapes, for the caller to + /// classify as a timeout, a caller cancellation, or a channel cancelling for its own reasons. + /// + private async Task SearchChannelsAsync( + RetrieveExperienceRequest request, + ExperienceCandidateQuery query, + CancellationToken cancellationToken) + { + var text = SearchAsync(request, query, cancellationToken); + var vector = VectorSearchAsync(request, cancellationToken); + + // Awaited together rather than in sequence: the policy's timeout bounds the pair, so running + // them one after the other would halve the budget each actually gets. + await Task.WhenAll(text, vector).ConfigureAwait(false); + return new ChannelOutcome(await text.ConfigureAwait(false), await vector.ConfigureAwait(false)); + } + + /// + /// Embeds the request's task text and searches the index with it, turning every expected condition + /// and every non-cancellation failure into an explicit text-only fallback rather than a failure. + /// The text channel's candidates are never lost to something that went wrong here. + /// + private async Task VectorSearchAsync( + RetrieveExperienceRequest request, + CancellationToken cancellationToken) + { + if (_embeddingIndex is null || _embeddingGenerator is null) + { + return VectorOutcome.FellBack(NotConfigured); + } + + // The vector channel may never take the text channel's answer down with it, and that includes + // cancellation it did not receive from the caller. An HttpClient request timeout surfaces as a + // TaskCanceledException with the caller's token untouched, so a merely slow embedding provider + // would otherwise turn every retrieval into a Failed result with no records at all. + bool CallerCancelled() => cancellationToken.IsCancellationRequested; + + string modelId; + int dimension; + ReadOnlyMemory vector; + try + { + modelId = _embeddingGenerator.ModelId; + dimension = _embeddingGenerator.Dimension; + vector = await _embeddingGenerator.GenerateAsync(request.TaskText, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (CallerCancelled()) + { + throw; + } + catch (Exception ex) + { + // Including a provider-side timeout, which arrives here as an OperationCanceledException + // the caller never asked for. + return VectorOutcome.FellBack(new VectorChannelFallback( + TextOnlyReason.ProviderUnavailable, + $"The embedding provider threw {ex.GetType().FullName}, so no query vector was produced.", + ex)); + } + + if (string.IsNullOrWhiteSpace(modelId)) + { + return VectorOutcome.FellBack(new VectorChannelFallback( + TextOnlyReason.ProviderUnavailable, + "The embedding provider reported no model ID, so no stored vector could be known to be comparable.", + Exception: null)); + } + + if (vector.Length == 0 || vector.Length != dimension) + { + // A vector that does not match the width the provider declared cannot be compared with + // anything stored under that declaration, so nothing is sent to the index at all. + return VectorOutcome.FellBack(new VectorChannelFallback( + TextOnlyReason.ProviderUnavailable, + $"The embedding provider returned a {vector.Length}-component query vector where {dimension} were declared.", + Exception: null)); + } + + if (!IsFinite(vector)) + { + // A non-finite component makes every distance computed against it meaningless, and pgvector + // would reject it at the server. Fall back here rather than spend a database round trip. + return VectorOutcome.FellBack(new VectorChannelFallback( + TextOnlyReason.ProviderUnavailable, + "The embedding provider returned a query vector with a non-finite component, so no comparison is possible.", + Exception: null)); + } + + ExperienceVectorSearchResult result; + try + { + result = await _embeddingIndex + .SearchAsync( + request.Authorization, + new ExperienceVectorQuery( + request.Scope, + modelId, + vector, + EligibleStatuses, + _policy.MinimumConfidence, + // The same ceiling-plus-one probe the text channel uses, so either channel + // reaching the ceiling is visible as truncation. + _policy.CandidateLimit + 1), + cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (CallerCancelled()) + { + throw; + } + catch (Exception ex) + { + return VectorOutcome.FellBack(new VectorChannelFallback( + TextOnlyReason.VectorSearchFailed, + $"The embedding index threw {ex.GetType().FullName} while searching stored vectors.", + ex)); + } + + if (result is null) + { + return VectorOutcome.FellBack(new VectorChannelFallback( + TextOnlyReason.VectorSearchFailed, + "The embedding index returned no result at all.", + Exception: null)); + } + + return result.Outcome switch + { + ExperienceVectorSearchOutcome.Found when result.Candidates is not null => + VectorOutcome.Succeeded(result.Candidates), + ExperienceVectorSearchOutcome.Found => VectorOutcome.FellBack(new VectorChannelFallback( + TextOnlyReason.VectorSearchFailed, + "The embedding index reported matches but returned no candidate list.", + Exception: null)), + ExperienceVectorSearchOutcome.ModelMismatch => VectorOutcome.FellBack(new VectorChannelFallback( + TextOnlyReason.ModelMismatch, + "Every embedding stored in this scope came from a different model than the query vector; no comparison was attempted.", + Exception: null)), + ExperienceVectorSearchOutcome.DimensionMismatch => VectorOutcome.FellBack(new VectorChannelFallback( + TextOnlyReason.DimensionMismatch, + "Every embedding stored in this scope is a different width than the query vector; no comparison was attempted.", + Exception: null)), + _ => VectorOutcome.FellBack(new VectorChannelFallback( + TextOnlyReason.VectorSearchFailed, + $"The embedding index returned '{result.Outcome}' rather than '{ExperienceVectorSearchOutcome.Found}'.", + Exception: null)), + }; + } + + /// + /// Runs the text search and turns every expected condition and every non-cancellation failure into + /// a . Cancellation alone escapes, for the caller to classify. + /// + private async Task SearchAsync( + RetrieveExperienceRequest request, + ExperienceCandidateQuery query, + CancellationToken cancellationToken) + { + ExperienceCandidateSearchResult result; + try + { + result = await _candidateSource.SearchAsync(request.Authorization, query, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (ExperienceStoreException ex) + { + return SearchOutcome.Failed(new RetrievalFailure("The candidate source failed to search stored experience.", ex)); + } + catch (Exception ex) + { + return SearchOutcome.Failed(new RetrievalFailure($"The candidate source threw {ex.GetType().FullName}.", ex)); + } + + if (result is null) + { + return SearchOutcome.Failed(new RetrievalFailure("The candidate source returned no result at all.", Exception: null)); + } + + if (result.Outcome != ExperienceStoreOutcome.Found) + { + // Denied or Invalid from the source is still a refusal to answer, not an empty answer. + return SearchOutcome.Failed(new RetrievalFailure( + $"The candidate source returned '{result.Outcome}' rather than '{ExperienceStoreOutcome.Found}'.", + Exception: null)); + } + + return result.Candidates is null + ? SearchOutcome.Failed(new RetrievalFailure("The candidate source reported matches but returned no candidate list.", Exception: null)) + : SearchOutcome.Succeeded(result.Candidates); + } + + /// + /// Merges the two channels, applies the eligibility checks Core owns, scores what survives, and + /// orders the result. Any candidate that cannot be fully checked -- unreadable, or outside the + /// requested scope -- makes the whole result empty rather than partly filtered. + /// + private ExperienceRetrievalResult Rank( + RetrieveExperienceRequest request, + ChannelOutcome channels, + bool unrestricted, + long startedAt) + { + var now = _timeProvider.GetUtcNow(); + var required = request.RequiredEnvironmentAttributes; + + // Each channel was asked for one candidate past the ceiling: its presence means more matched + // than were considered, and it is dropped rather than ranked, so the ceiling still holds for + // each channel separately. + var merged = new List(); + var positions = new Dictionary(); + var truncated = false; + + if (Absorb(request, channels.Text.Candidates, "candidate source", merged, positions, ref truncated) is { } textFailure) + { + return Empty(RetrievalOutcome.Failed, request, unrestricted, startedAt, textFailure, channels.Vector.Fallback); + } + + if (Absorb(request, channels.Vector.Candidates, "embedding index", merged, positions, ref truncated) is { } vectorFailure) + { + // A vector channel that answered with something unverifiable is treated exactly like a + // text one that did: fail-closed. Silently dropping it would mean returning a result + // built partly on an answer we just decided we could not check. + return Empty(RetrievalOutcome.Failed, request, unrestricted, startedAt, vectorFailure, channels.Vector.Fallback); + } + + var ranked = new List<(RankedExperience Ranked, string TieBreak)>(merged.Count); + var excluded = new List(); + + foreach (var candidate in merged) + { + var record = candidate.Record; + + if (!EligibleStatuses.Contains(record.Status)) + { + excluded.Add(new ExcludedExperience(record.ExperienceId, RetrievalExclusionReason.IneligibleStatus)); + continue; + } + + if (_policy.MaxAge is { } maxAge && now - record.UpdatedAt > maxAge) + { + excluded.Add(new ExcludedExperience(record.ExperienceId, RetrievalExclusionReason.Expired)); + continue; + } + + if (!unrestricted && !EnvironmentMatches(required!, record.Environment.Metadata)) + { + excluded.Add(new ExcludedExperience(record.ExperienceId, RetrievalExclusionReason.EnvironmentMismatch)); + continue; + } + + ranked.Add((Score(record, candidate.Relevance, now), record.ExperienceId.ToString("D"))); + } + + // Ties sort by ExperienceId ascending and ordinal, so the order is total and stable rather than + // whatever order the database happened to return equally-scored rows in. + ranked.Sort(static (left, right) => + { + var byScore = right.Ranked.Score.CompareTo(left.Ranked.Score); + return byScore != 0 ? byScore : string.CompareOrdinal(left.TieBreak, right.TieBreak); + }); + + var limit = request.Limit ?? _policy.CandidateLimit; + var records = ranked.Take(limit).Select(entry => entry.Ranked).ToArray(); + + return new ExperienceRetrievalResult( + RetrievalOutcome.Completed, + records, + excluded, + truncated, + unrestricted, + request.CorrelationId, + _timeProvider.GetElapsedTime(startedAt), + Failure: null, + channels.Vector.Fallback); + } + + /// + /// Validates one channel's candidates and folds them into the merged list. Returns the failure + /// that makes the whole result empty, or when the channel's answer was + /// entirely checkable. + /// + /// + /// + /// A record already contributed by the other channel is not added twice: it keeps whichever of the + /// two normalized relevances is higher, and its position in the merged list does not move. That + /// is the whole merge rule -- being found twice is not itself evidence of anything, so it earns no + /// bonus and adds no sixth ranking axis. + /// + /// + /// A duplicate within one channel is a different matter and is fail-closed: a channel + /// that returned the same record twice cannot be trusted for the rest of its answer either. + /// + /// + private RetrievalFailure? Absorb( + RetrieveExperienceRequest request, + IReadOnlyList candidates, + string channel, + List merged, + Dictionary positions, + ref bool truncated) + { + if (candidates.Count > _policy.CandidateLimit) + { + truncated = true; + } + + var considered = Math.Min(candidates.Count, _policy.CandidateLimit); + var seen = new HashSet(considered); + + for (var index = 0; index < considered; index++) + { + var candidate = candidates[index]; + if (candidate?.Record is not { } record || record.Environment?.Metadata is null || record.Scope is null) + { + return new RetrievalFailure( + $"A candidate from the {channel} could not be read, so the result would have been unfiltered.", + Exception: null); + } + + if (record.Scope != request.Scope) + { + // The channel answered outside the exact request scope. Nothing it returned can be + // trusted to be in scope, so none of it is returned. + return new RetrievalFailure($"A candidate was returned by the {channel} outside the requested scope.", Exception: null); + } + + if (!seen.Add(record.ExperienceId)) + { + // The same record twice would be scored twice and ordered arbitrarily against itself, + // so the ranking would no longer be total. + return new RetrievalFailure($"The {channel} returned the same record more than once.", Exception: null); + } + + if (positions.TryGetValue(record.ExperienceId, out var existing)) + { + // Only the relevance is merged, never the record: the two channels read the record at + // different instants, and adopting the other snapshot would let the expiry, status, and + // environment checks below be decided on the staler of the two. + var kept = merged[existing]; + if (Normalize(candidate.Relevance) > Normalize(kept.Relevance)) + { + merged[existing] = kept with { Relevance = candidate.Relevance }; + } + + continue; + } + + positions[record.ExperienceId] = merged.Count; + merged.Add(candidate); + } + + return null; + } + + /// + /// Every required attribute must be present on the record and equal ordinally. A missing key + /// excludes the record: an unstated environment is not a matching one. + /// + private static bool EnvironmentMatches(IReadOnlyDictionary required, IReadOnlyDictionary metadata) + { + foreach (var (key, value) in required) + { + if (!metadata.TryGetValue(key, out var stored) || !string.Equals(stored, value, StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + /// + /// Scores one eligible record. Every component is normalized to [0, 1] and reported with the + /// weight applied to it, so the total is always reproducible from what the result carries. + /// + private RankedExperience Score(ExperienceRecord record, double relevance, DateTimeOffset now) + { + RankingComponent[] components = + [ + new(RankingComponentKind.Relevance, Normalize(relevance), _weights.Relevance), + new(RankingComponentKind.Confidence, Normalize(record.ReuseConfidence), _weights.Confidence), + new(RankingComponentKind.Recency, Recency(record.UpdatedAt, now), _weights.Recency), + new(RankingComponentKind.Status, StatusScore(record.Status), _weights.Status), + new(RankingComponentKind.EnvironmentCompatibility, CompatibleEnvironmentScore, _weights.EnvironmentCompatibility), + ]; + + var score = 0d; + foreach (var component in components) + { + score += component.Contribution; + } + + return new RankedExperience(record, score, components); + } + + /// + /// Exponential decay with the policy's half-life: 1 for a record updated now (or, with clock skew, + /// in the future), 0.5 at one half-life, and always inside (0, 1]. + /// + private double Recency(DateTimeOffset updatedAt, DateTimeOffset now) + { + var age = now - updatedAt; + if (age <= TimeSpan.Zero) + { + return 1d; + } + + return Normalize(Math.Pow(2d, -age.TotalSeconds / _policy.RecencyHalfLife.TotalSeconds)); + } + + private static double StatusScore(ExperienceStatus status) => status switch + { + ExperienceStatus.Reinforced => ReinforcedStatusScore, + ExperienceStatus.Validated => ValidatedStatusScore, + // Unreachable: anything else was excluded before ranking. Scored 0 rather than assumed. + _ => 0d, + }; + + /// + /// Clamps a component into [0, 1]. A NaN is scored 0: left alone it would poison every comparison + /// against the record and make the ordering non-total. + /// + private static double Normalize(double value) => double.IsNaN(value) ? 0d : Math.Clamp(value, 0d, 1d); + + /// Whether every component is a real number. A NaN or an infinity makes every distance computed against the vector meaningless. + private static bool IsFinite(ReadOnlyMemory vector) + { + foreach (var component in vector.Span) + { + if (!float.IsFinite(component)) + { + return false; + } + } + + return true; + } + + /// + /// Hands an abandoned search off to run itself down in the background: cancel its token, then -- + /// once both the search and the cancellation have actually finished -- observe any exception it + /// faulted with and dispose the token source. + /// + /// + /// The cancellation deliberately does not run on the caller's thread. A cancellation callback can + /// be arbitrarily slow -- Npgsql's opens a new connection to the server to cancel the + /// running statement -- so cancelling inline would let the call overrun the very timeout it is in + /// the middle of reporting, exactly when the bound matters most. Disposal waits for both tasks, + /// because disposing earlier would tear the token out from under a search or a callback still + /// reading it. + /// + private static void Abandon(Task task, CancellationTokenSource source) + { + var cancelling = Task.Run( + async () => + { + try + { + await source.CancelAsync().ConfigureAwait(false); + } + catch (Exception) + { + // A throwing cancellation callback must never affect the retrieval call's own result. + } + }, + CancellationToken.None); + + _ = Task.WhenAll(task, cancelling).ContinueWith( + completed => + { + _ = completed.Exception; + _ = task.Exception; + source.Dispose(); + }, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } + + private ExperienceRetrievalResult Empty( + RetrievalOutcome outcome, + RetrieveExperienceRequest request, + bool unrestricted, + long startedAt, + RetrievalFailure? failure, + VectorChannelFallback? vectorFallback) => new( + outcome, + NoRecords, + NoExclusions, + // Nothing was ranked, so nothing was cut: a denied, timed-out, or failed result is empty for + // its own reason, never because a ceiling was reached. + Truncated: false, + unrestricted, + request.CorrelationId, + _timeProvider.GetElapsedTime(startedAt), + failure, + vectorFallback); + + /// + /// The vector signal for a call that ended before either channel could contribute: the standing + /// fact that this deployment has no vector channel, when that is so, and otherwise nothing -- the + /// outcome itself already says why the result is empty. + /// + private VectorChannelFallback? EndedEarly() => HybridEnabled ? null : NotConfigured; + + /// What the bounded text search produced: either candidates, or the failure that ended it. + private readonly record struct SearchOutcome(IReadOnlyList Candidates, RetrievalFailure? Failure) + { + public static SearchOutcome Succeeded(IReadOnlyList candidates) => new(candidates, null); + + public static SearchOutcome Failed(RetrievalFailure failure) => new([], failure); + } + + /// + /// What the bounded vector search produced. Unlike the text channel it has no failure: everything + /// that can go wrong here is a fallback, because the text channel's answer must survive it. + /// + private readonly record struct VectorOutcome(IReadOnlyList Candidates, VectorChannelFallback? Fallback) + { + public static VectorOutcome Succeeded(IReadOnlyList candidates) => new(candidates, null); + + public static VectorOutcome FellBack(VectorChannelFallback fallback) => new([], fallback); + } + + /// Both channels' answers, produced together inside the one timeout. + private readonly record struct ChannelOutcome(SearchOutcome Text, VectorOutcome Vector); +} diff --git a/src/AgentExperience.Core/Retrieval/RankingWeights.cs b/src/AgentExperience.Core/Retrieval/RankingWeights.cs new file mode 100644 index 0000000..7db3aa0 --- /dev/null +++ b/src/AgentExperience.Core/Retrieval/RankingWeights.cs @@ -0,0 +1,100 @@ +using System.Globalization; + +namespace AgentExperience.Core.Retrieval; + +/// +/// The weight each normalized ranking component carries in a retrieved record's total score. Every +/// weight must be a finite, non-negative number, and the five must sum to 1 within +/// ; anything else throws at +/// construction, so an invalid set can never reach a retrieval call. +/// +/// +/// +/// This follows 's convention of a validated +/// options record that refuses an invalid value outright, with one deliberate difference: the +/// properties are get-only rather than init. CaptureLimits re-validates in each +/// init accessor because each of its limits is independently valid or not. The sum-to-1 rule +/// here spans all five weights at once, and a with expression assigns them one at a time +/// after the copy constructor has already run -- there is no hook that could re-check the +/// sum afterwards. Read-only properties close that gap by construction: the constructor is the only +/// way to obtain an instance, and it validates everything. Build a different weighting with +/// new RankingWeights(...). +/// +/// +/// Weights are the effective weights reported on every retrieved record alongside the +/// component they were applied to, so a host can always see why one record outranked another. +/// +/// +/// Weight of how strongly the record's indexed text matched the task text. +/// Weight of the record's . +/// Weight of how recently the record was last updated. +/// Weight of the record's lifecycle status among the eligible ones. +/// Weight of the record's compatibility with the request's required environment attributes. +public sealed record RankingWeights( + double Relevance, + double Confidence, + double Recency, + double Status, + double EnvironmentCompatibility) +{ + /// + /// How far the weights' sum may sit from 1 and still be accepted, so a set written as ordinary + /// decimal literals is not rejected for binary floating-point rounding alone. + /// + public const double SumTolerance = 1e-6; + + /// + /// The documented default weighting: relevance 0.35, confidence 0.25, recency 0.15, status 0.15, + /// environment compatibility 0.10. + /// + public static RankingWeights Default { get; } = new(0.35, 0.25, 0.15, 0.15, 0.10); + + /// Weight of how strongly the record's indexed text matched the task text. + public double Relevance { get; } = EnsureWeight(Relevance, nameof(Relevance)); + + /// Weight of the record's reuse confidence. + public double Confidence { get; } = EnsureWeight(Confidence, nameof(Confidence)); + + /// Weight of how recently the record was last updated. + public double Recency { get; } = EnsureWeight(Recency, nameof(Recency)); + + /// Weight of the record's lifecycle status among the eligible ones. + public double Status { get; } = EnsureWeight(Status, nameof(Status)); + + /// Weight of the record's compatibility with the request's required environment attributes. + public double EnvironmentCompatibility { get; } = EnsureWeight(EnvironmentCompatibility, nameof(EnvironmentCompatibility)); + + /// + /// The weights' sum. Declared last on purpose: property initializers run in declaration order, so + /// every weight has already been checked to be finite and non-negative by the time the + /// cross-property sum rule is applied, and a negative weight is reported as such rather than as a + /// bad sum. This is also the only place the sum rule can run -- a record has no constructor body + /// to put it in. + /// + public double Sum { get; } = EnsureSum(Relevance, Confidence, Recency, Status, EnvironmentCompatibility); + + private static double EnsureSum(double relevance, double confidence, double recency, double status, double environmentCompatibility) + { + var sum = relevance + confidence + recency + status + environmentCompatibility; + if (Math.Abs(sum - 1d) > SumTolerance) + { + // The out-of-range value is the sum itself, not any one weight, so that is what the + // exception names: no single constructor parameter is at fault. + throw new ArgumentOutOfRangeException( + nameof(Sum), + sum, + string.Format( + CultureInfo.InvariantCulture, + "Ranking weights must sum to 1 within {0}; this set sums to {1}.", + SumTolerance.ToString("R", CultureInfo.InvariantCulture), + sum.ToString("R", CultureInfo.InvariantCulture))); + } + + return sum; + } + + private static double EnsureWeight(double value, string paramName) => + double.IsFinite(value) && value >= 0d + ? value + : throw new ArgumentOutOfRangeException(paramName, value, "Ranking weights must be finite and non-negative."); +} diff --git a/src/AgentExperience.Core/Retrieval/RetrievalPolicy.cs b/src/AgentExperience.Core/Retrieval/RetrievalPolicy.cs new file mode 100644 index 0000000..fd66f6d --- /dev/null +++ b/src/AgentExperience.Core/Retrieval/RetrievalPolicy.cs @@ -0,0 +1,160 @@ +using AgentExperience.Abstractions; + +namespace AgentExperience.Core.Retrieval; + +/// +/// The bounds and thresholds a retrieval call runs under. Every value is validated both at +/// construction and on a with expression (each property's init accessor +/// re-validates via the C# field keyword), exactly as +/// does, because a record's property +/// initializers alone do not re-run when a property is changed with with. An invalid value +/// throws , so a misconfigured policy fails at startup +/// rather than silently widening what may be reused. +/// +/// +/// How long the whole retrieval call may take, measured with the service's injected +/// . Exceeding it is never an exception: the caller gets an empty result +/// carrying a timeout signal. Must be strictly positive and at most . +/// +/// +/// The smallest a record +/// may have and still be a candidate, in [0, 1]. Applied in the database, before ranking. +/// +/// +/// How long since a record's last lifecycle activity it may still be reusable. +/// means records never expire. Must be strictly positive when set. Expiry is +/// decided in Core, not in the database, because it is relative to the clock the service was given. +/// +/// What "age" means here. It is measured from +/// , which every lifecycle commit +/// bumps -- so it is the age of the record's last status change, not of the lesson itself. A +/// years-old lesson reinforced yesterday is treated as one day old and does not expire; a lesson +/// learned yesterday and never touched since ages normally. That is deliberate (recent revalidation is +/// evidence the lesson still holds), but it is not "when this was learned". +/// +/// +/// +/// The last-activity age at which the recency component has decayed to half. Recency is +/// 2^(-age / RecencyHalfLife), so it is always in (0, 1], is 1 for a record just committed, and +/// stays defined whether or not is set. It measures the same +/// as , with +/// the same caveat. Must be strictly positive. +/// +/// +/// The most candidates the search may return for Core to filter and rank. It bounds the work a single +/// retrieval does; the caller's own limit then bounds how many ranked records come back, and may not +/// exceed this. Because the search orders by text relevance before cutting, this is a real +/// recall ceiling: a record with a weaker text match but strong confidence, recency, or status is not +/// ranked at all once this many stronger text matches exist. The service asks the source for one more +/// candidate than this, so it can tell "exactly at the ceiling" from "more existed" and report +/// Truncated; that is why the largest permitted value is one below +/// . +/// +public sealed record RetrievalPolicy( + TimeSpan Timeout, + double MinimumConfidence, + TimeSpan? MaxAge, + TimeSpan RecencyHalfLife, + int CandidateLimit) +{ + /// The documented default timeout: 500 ms. + public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMilliseconds(500); + + /// + /// The largest permitted : one day. Retrieval is a request-scoped bound + /// measured in milliseconds, so a day is already far past any sensible value, and capping it keeps + /// the wait comfortably inside what 's timeout support accepts -- a longer span + /// would throw out of the retrieval call instead of bounding it. + /// + public static readonly TimeSpan MaxTimeout = TimeSpan.FromDays(1); + + /// The documented default eligibility confidence threshold: 0.5. + public const double DefaultMinimumConfidence = 0.5; + + /// The default half-life of the recency component: 30 days. + public static readonly TimeSpan DefaultRecencyHalfLife = TimeSpan.FromDays(30); + + /// The default bound on how many candidates one search may return. + public const int DefaultCandidateLimit = 50; + + /// + /// The largest permitted : one below + /// , because the service asks the source for + /// + 1 candidates to detect truncation. + /// + public const int MaxCandidateLimit = ExperienceCandidateQuery.MaxLimit - 1; + + /// + /// The documented defaults: a 500 ms timeout, a 0.5 confidence threshold, no expiry, a 30-day + /// recency half-life, and at most 50 candidates per search. + /// + public static RetrievalPolicy Default { get; } = new( + DefaultTimeout, + DefaultMinimumConfidence, + MaxAge: null, + DefaultRecencyHalfLife, + DefaultCandidateLimit); + + /// How long the whole retrieval call may take (see the primary constructor's parameter doc). + public TimeSpan Timeout + { + get; + init => field = EnsureTimeout(value); + } = EnsureTimeout(Timeout); + + /// The confidence floor a candidate must clear (see the primary constructor's parameter doc). + public double MinimumConfidence + { + get; + init => field = EnsureUnitInterval(value, nameof(MinimumConfidence)); + } = EnsureUnitInterval(MinimumConfidence, nameof(MinimumConfidence)); + + /// How old a reusable record may be, or for no expiry (see the primary constructor's parameter doc). + public TimeSpan? MaxAge + { + get; + init => field = value is { } age ? EnsurePositive(age, nameof(MaxAge)) : null; + } = MaxAge is { } maxAge ? EnsurePositive(maxAge, nameof(MaxAge)) : null; + + /// The age at which the recency component has decayed to half (see the primary constructor's parameter doc). + public TimeSpan RecencyHalfLife + { + get; + init => field = EnsurePositive(value, nameof(RecencyHalfLife)); + } = EnsurePositive(RecencyHalfLife, nameof(RecencyHalfLife)); + + /// The most candidates one search may return (see the primary constructor's parameter doc). + public int CandidateLimit + { + get; + init => field = EnsureCandidateLimit(value); + } = EnsureCandidateLimit(CandidateLimit); + + private static TimeSpan EnsurePositive(TimeSpan value, string paramName) => + value > TimeSpan.Zero + ? value + // TimeSpan.Zero excludes Timeout.InfiniteTimeSpan (-1 tick) too: an unbounded retrieval is + // exactly what the timeout exists to prevent. + : throw new ArgumentOutOfRangeException(paramName, value, "Retrieval durations must be strictly positive."); + + private static TimeSpan EnsureTimeout(TimeSpan value) + { + EnsurePositive(value, nameof(Timeout)); + return value <= MaxTimeout + ? value + : throw new ArgumentOutOfRangeException(nameof(Timeout), value, $"The retrieval timeout must be at most {MaxTimeout}."); + } + + private static double EnsureUnitInterval(double value, string paramName) => + value >= 0d && value <= 1d + ? value + : throw new ArgumentOutOfRangeException(paramName, value, "The confidence threshold must be between 0 and 1 inclusive."); + + private static int EnsureCandidateLimit(int value) => + value is >= ExperienceCandidateQuery.MinLimit and <= MaxCandidateLimit + ? value + : throw new ArgumentOutOfRangeException( + nameof(CandidateLimit), + value, + $"The candidate limit must be between {ExperienceCandidateQuery.MinLimit} and {MaxCandidateLimit}."); +} diff --git a/src/AgentExperience.Core/Retrieval/RetrievalResults.cs b/src/AgentExperience.Core/Retrieval/RetrievalResults.cs new file mode 100644 index 0000000..fbe019d --- /dev/null +++ b/src/AgentExperience.Core/Retrieval/RetrievalResults.cs @@ -0,0 +1,235 @@ +using AgentExperience.Abstractions; + +namespace AgentExperience.Core.Retrieval; + +/// +/// One request to find experience that applies to a task. +/// +/// What the host has established the caller may do. The request's must lie inside it, or the call is denied before any search is issued. +/// The exact scope to retrieve within. Never treated as authority, and never widened. +/// The task text to match stored experience against. Must be non-blank. +/// +/// Environment attributes a record must carry to be reusable here. Every pair must equal the record's +/// entry exactly (ordinal), and a record missing the key +/// is excluded. or empty means the request names no requirement, and the result +/// is marked . +/// +/// Optional. The host's correlation identifier, echoed on the result -- including on a timeout, so a timed-out retrieval can be tied back to the request that caused it. +/// Optional. The most ranked records to return. Defaults to the policy's candidate limit when ; must be strictly positive and no greater than that limit. +public sealed record RetrieveExperienceRequest( + AuthorizationContext Authorization, + Scope Scope, + string TaskText, + IReadOnlyDictionary? RequiredEnvironmentAttributes = null, + string? CorrelationId = null, + int? Limit = null); + +/// The component axes a retrieved record is scored on. Each is normalized to [0, 1]. +public enum RankingComponentKind +{ + /// How strongly the record's indexed text matched the request's task text. + Relevance, + + /// The record's , which is already in [0, 1]. + Confidence, + + /// + /// How recently the record last saw lifecycle activity, decayed by the policy's recency + /// half-life. Measured from , which every lifecycle commit + /// bumps, so a years-old lesson reinforced yesterday scores as fully recent. It is the age of the + /// last status change, not of the lesson. + /// + Recency, + + /// The record's lifecycle status among the eligible ones. + Status, + + /// How well the record's environment matches the request's required attributes. + EnvironmentCompatibility, +} + +/// +/// One normalized ranking component and the weight actually applied to it, so a host can always see +/// why one record outranked another rather than being handed an opaque score. +/// +/// Which axis this is. +/// The normalized component value, in [0, 1]. +/// The effective weight applied to . +public sealed record RankingComponent(RankingComponentKind Kind, double Value, double Weight) +{ + /// This component's contribution to the record's total score: times . + public double Contribution => Value * Weight; +} + +/// +/// A record that passed every eligibility check, with its score and the components that produced it. +/// +/// The eligible record, exactly as stored. Retrieval never rewrites it. +/// The weighted total of , in [0, 1] whenever the weights sum to 1. +/// Every component, in order, each with the weight applied to it. +public sealed record RankedExperience( + ExperienceRecord Record, + double Score, + IReadOnlyList Components); + +/// Why a candidate the search returned was not ranked. +public enum RetrievalExclusionReason +{ + /// Its status is not one of the eligible ones. Only and are. + IneligibleStatus, + + /// Its is older than the policy's . + Expired, + + /// A required environment attribute differs, or the record does not carry the key at all. + EnvironmentMismatch, +} + +/// +/// One candidate that an eligibility check removed before ranking, named so a host can tell "nothing +/// matched" apart from "something matched but was not reusable here". +/// +/// The excluded record. +/// Which check excluded it. +public sealed record ExcludedExperience(Guid ExperienceId, RetrievalExclusionReason Reason); + +/// What a retrieval call ended as. +public enum RetrievalOutcome +{ + /// The search ran inside the timeout and the result holds every eligible record it found, ranked. + Completed, + + /// The call exceeded the policy's timeout. The result is empty; nothing about it is an error. + TimedOut, + + /// The request scope lies outside the host-established authorization. No search was issued. + Denied, + + /// The search failed (for example the database was unavailable, or a stored record could not be read). The result is empty, never unfiltered. + Failed, +} + +/// +/// Why a retrieval failed. is safe to log or surface: it never carries record +/// content. is not held to that standard -- it is whatever the port +/// threw, and a driver's message can quote SQL text, parameter values, or connection detail. Treat it +/// as local diagnostics only, and do not copy it into a user-visible response or a shared log without +/// deciding that yourself. +/// +/// A human-readable, content-free explanation. +/// The original failure, when one was caught. Diagnostic only; may carry adapter detail. +public sealed record RetrievalFailure(string Reason, Exception? Exception); + +/// +/// Why a retrieval answered from the text channel alone. Every value is an explicit statement that +/// the vector channel did not contribute -- none of them is ever inferred from an empty +/// vector result, because "nothing was semantically similar" and "the vector channel could not be +/// trusted" are different claims and only the second one should make a host look at its wiring. +/// +public enum TextOnlyReason +{ + /// + /// No embedding index or no embedding generator was wired in, so there is no vector channel at + /// all. This is a configuration fact, not a failure: a text-only deployment is a supported one. + /// + NotConfigured, + + /// + /// The embedding provider could not produce a query vector -- it threw, timed out, or returned + /// something unusable -- so no vector comparison was possible. The text channel still answered. + /// + ProviderUnavailable, + + /// + /// The scope's stored embeddings come from a different model than the query vector, so none of + /// them is comparable with it. No vector comparison was attempted. + /// + ModelMismatch, + + /// + /// The scope's stored embeddings are from this model but at a different width, so none of them is + /// comparable with the query vector. No vector comparison was attempted. + /// + DimensionMismatch, + + /// + /// The vector search itself failed, was denied, or was refused as malformed. The text channel + /// still answered, and its candidates are still returned. + /// + VectorSearchFailed, +} + +/// +/// The explicit text-only signal: present exactly when the vector channel contributed nothing to a +/// result, with the reason it did not. It is deliberately not an error -- a retrieval that fell back +/// to text is a complete, usable answer, just a narrower one -- but it is always stated rather than +/// left to be inferred from an empty vector match. +/// +/// Which of the documented fallbacks applied. +/// A human-readable, content-free explanation. Safe to log or surface. +/// +/// The original failure, when one was caught. As with this +/// is not content-free -- a driver or HTTP client message can quote SQL text, parameters, or +/// a request body. Treat it as local diagnostics only. +/// +public sealed record VectorChannelFallback(TextOnlyReason Reason, string Detail, Exception? Exception); + +/// +/// The result of a retrieval call. It is always a complete answer: an empty +/// list with a non- +/// means retrieval declined to answer, never that the caller may proceed with +/// unfiltered experience. +/// +/// What the call ended as. +/// The eligible records in rank order, highest score first, ties broken by ascending. Empty unless is , and bounded by the request's limit and by 's ceiling. +/// +/// Candidates an eligibility check in Core removed before ranking, with the check that +/// removed each. It is not a complete account of everything that was filtered: scope, status, and the +/// reuse-confidence floor are applied in the database, so records they exclude never reach Core and are +/// never itemized here. +/// +/// +/// when the search hit the policy's candidate ceiling and more matching records +/// existed than were considered. Because the search orders by text relevance before cutting, the +/// records beyond the ceiling are simply not ranked -- they are not in either, +/// and one of them may well have outranked what came back. Treat it as "this answer is partial". +/// +/// +/// when the request named no required environment attributes, so every +/// candidate passed that check unconditionally. This is explicit rather than inferred: "no +/// environment requirement" and "every requirement happened to match" are different claims. +/// +/// The request's correlation identifier, echoed back on every outcome including . +/// How long the call took, measured with the service's . +/// Why the call failed, when is ; otherwise . +/// +/// Why the vector channel contributed nothing, when it did not; when both +/// channels ran. Present even on a perfectly good text-only answer, because "this deployment has no +/// vector channel" and "the vector channel could not be trusted this time" are things a host has to +/// be able to tell apart. +/// +public sealed record ExperienceRetrievalResult( + RetrievalOutcome Outcome, + IReadOnlyList Records, + IReadOnlyList Excluded, + bool Truncated, + bool EnvironmentUnrestricted, + string? CorrelationId, + TimeSpan Elapsed, + RetrievalFailure? Failure, + VectorChannelFallback? VectorFallback = null) +{ + /// + /// The timeout signal: exactly when the call ran out of time. A timeout is + /// never an exception and is never reported as a failure, so a host can tell "too slow this time" + /// from "something is broken". + /// + public bool TimedOut => Outcome is RetrievalOutcome.TimedOut; + + /// + /// The text-only signal: exactly when is + /// present, that is, when this answer came from the text channel alone. It is never true merely + /// because the vector channel matched nothing. + /// + public bool TextOnly => VectorFallback is not null; +} diff --git a/src/AgentExperience.Core/Verification/RequiredCheck.cs b/src/AgentExperience.Core/Verification/RequiredCheck.cs new file mode 100644 index 0000000..ab2f6cb --- /dev/null +++ b/src/AgentExperience.Core/Verification/RequiredCheck.cs @@ -0,0 +1,41 @@ +namespace AgentExperience.Core.Verification; + +/// +/// One check a task declares as required for verification, and (optionally) the kind of evaluator +/// that is allowed to satisfy it. +/// +/// +/// +/// Matching a required check by alone lets any producer claim any check: a +/// human approval, for instance, could satisfy a check the task meant to be answered by a test run. +/// closes that gap. When it is non-, +/// counts a piece of +/// for this check only when the evidence's own +/// equals it (ordinal, case-sensitive); +/// evidence of any other kind is ignored entirely, exactly as if it had been recorded for a +/// different . A check whose evidence is all ignored has no evidence at all and +/// therefore resolves to -- a +/// mismatched evaluator can never turn a check into a pass. +/// +/// +/// A accepts evidence of any kind, which is the +/// behaviour this type replaced (checks were previously declared as bare CheckId strings). +/// +/// +/// The task-declared required check ID. Must be non-blank and unique within one aggregation. +/// +/// The that may satisfy this check (e.g. +/// "TestResult", "ToolExitCode", "HumanApproval"), or to +/// accept any kind. When supplied it must be non-blank. +/// +public sealed record RequiredCheck(string CheckId, string? ExpectedKind = null) +{ + /// + /// Whether is allowed to satisfy this check: always + /// when no is named, otherwise an exact + /// ordinal match. + /// + /// The candidate evidence's own Kind. + public bool Accepts(string? evidenceKind) => + ExpectedKind is null || string.Equals(ExpectedKind, evidenceKind, StringComparison.Ordinal); +} diff --git a/src/AgentExperience.Core/Verification/VerificationAggregator.cs b/src/AgentExperience.Core/Verification/VerificationAggregator.cs index ee1d535..ab95502 100644 --- a/src/AgentExperience.Core/Verification/VerificationAggregator.cs +++ b/src/AgentExperience.Core/Verification/VerificationAggregator.cs @@ -25,7 +25,9 @@ namespace AgentExperience.Core.Verification; /// /// /// Per-check resolution (the two AC4 conflict clauses, reconciled -- see this story's Design -/// Notes): for each required CheckId, gather only the selected evidence carrying it. No +/// Notes): for each , gather only the selected evidence carrying its +/// and a its +/// accepts. No /// evidence at all is a missing check (); any /// among it makes the check -- /// dominating even a recorded for the same check in the same @@ -65,26 +67,26 @@ public static class VerificationAggregator public const string RuleVersion = "1.0.0"; /// - /// Aggregates against , reading + /// Aggregates against , reading /// only the evidence in for /// -- see this type's remarks for the full selection, per-check, and overall-verdict rules. /// /// All evidence available to consider, in the order it was produced. Never filtered or reordered by the caller; this call does that filtering itself. A entry is a caller error and throws. - /// The task's declared required check IDs, which must be unique (a duplicate is a caller error and throws, rather than silently skewing the completion score). An empty set always yields . + /// The task's declared required checks, whose s must be unique (a duplicate is a caller error and throws, rather than silently skewing the completion score) and non-blank. A entry is a caller error and throws. An empty set always yields . /// The host-closed verification round and artifact revision to read from, or if the host has not closed a round yet. Never agent-suppliable -- only a host establishes this. /// The artifact's current revision. If it does not match 's own revision, verification is stale. /// When this aggregation is being performed. /// Checked cooperatively; a cancelled call throws rather than returning any . public static VerificationResult Aggregate( IReadOnlyList evidence, - IReadOnlyList requiredCheckIds, + IReadOnlyList requiredChecks, ClosedVerificationRound? closedRound, string currentArtifactRevision, DateTimeOffset evaluatedAt, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(evidence); - ArgumentNullException.ThrowIfNull(requiredCheckIds); + ArgumentNullException.ThrowIfNull(requiredChecks); ArgumentException.ThrowIfNullOrWhiteSpace(currentArtifactRevision); // Invalid input throws -- never silently dropped or tolerated into a fabricated result, per @@ -94,15 +96,30 @@ public static VerificationResult Aggregate( throw new ArgumentException("Evidence must not contain null entries.", nameof(evidence)); } - if (requiredCheckIds.Distinct(StringComparer.Ordinal).Count() != requiredCheckIds.Count) + if (requiredChecks.Any(c => c is null)) { - throw new ArgumentException("Required check IDs must be unique.", nameof(requiredCheckIds)); + throw new ArgumentException("Required checks must not contain null entries.", nameof(requiredChecks)); + } + + if (requiredChecks.Any(c => string.IsNullOrWhiteSpace(c.CheckId))) + { + throw new ArgumentException("Required check IDs must not be null, empty, or whitespace.", nameof(requiredChecks)); + } + + if (requiredChecks.Any(c => c.ExpectedKind is not null && string.IsNullOrWhiteSpace(c.ExpectedKind))) + { + throw new ArgumentException("A required check's ExpectedKind must be null or non-blank.", nameof(requiredChecks)); + } + + if (requiredChecks.Select(c => c.CheckId).Distinct(StringComparer.Ordinal).Count() != requiredChecks.Count) + { + throw new ArgumentException("Required check IDs must be unique.", nameof(requiredChecks)); } cancellationToken.ThrowIfCancellationRequested(); // Stale/unclosed short-circuit -- no round or revision selection from evidence or - // requiredCheckIds themselves is ever consulted here; only the host-supplied closedRound + // requiredChecks themselves is ever consulted here; only the host-supplied closedRound // decides. Nothing is examined further. if (closedRound is null) { @@ -116,7 +133,7 @@ public static VerificationResult Aggregate( evaluatedAt); } - if (requiredCheckIds.Count == 0) + if (requiredChecks.Count == 0) { return UnknownResult("No required checks were declared for this task; an empty required set can never be conclusively verified.", evaluatedAt); } @@ -132,11 +149,17 @@ public static VerificationResult Aggregate( var failedCheckIds = new List(); var unknownCheckIds = new List(); - foreach (var checkId in requiredCheckIds) + foreach (var requiredCheck in requiredChecks) { cancellationToken.ThrowIfCancellationRequested(); - var checkEvidence = selectedEvidence.Where(e => string.Equals(e.CheckId, checkId, StringComparison.Ordinal)).ToList(); + // A named ExpectedKind narrows the evidence for this check: evidence of any other kind is + // ignored outright, so a mismatched evaluator can never satisfy the check (it becomes a + // check with no evidence, i.e. Unknown). + var checkEvidence = selectedEvidence + .Where(e => string.Equals(e.CheckId, requiredCheck.CheckId, StringComparison.Ordinal) && requiredCheck.Accepts(e.Kind)) + .ToList(); + foreach (var e in checkEvidence) { contributingEvidenceIds.Add(e.EvidenceId); @@ -145,11 +168,11 @@ public static VerificationResult Aggregate( switch (ResolveCheck(checkEvidence)) { case CheckResult.Fail: - failedCheckIds.Add(checkId); + failedCheckIds.Add(requiredCheck.CheckId); break; case CheckResult.Unknown: - unknownCheckIds.Add(checkId); + unknownCheckIds.Add(requiredCheck.CheckId); break; case CheckResult.Pass: @@ -158,10 +181,10 @@ public static VerificationResult Aggregate( } } - var completionScore = (double)passingCheckCount / requiredCheckIds.Count; + var completionScore = (double)passingCheckCount / requiredChecks.Count; // The evidence backing the outcome, in the order it was produced (Outcome.Evidence's own - // contract) -- the original evidence list's own order, not the order requiredCheckIds + // contract) -- the original evidence list's own order, not the order requiredChecks // happened to name checks in. Drawn only from the already round/revision-scoped selection. var contributingEvidence = selectedEvidence.Where(e => contributingEvidenceIds.Contains(e.EvidenceId)).ToList(); diff --git a/src/AgentExperience.Core/packages.lock.json b/src/AgentExperience.Core/packages.lock.json index 4030d55..54ebb50 100644 --- a/src/AgentExperience.Core/packages.lock.json +++ b/src/AgentExperience.Core/packages.lock.json @@ -12,6 +12,12 @@ "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.11" } }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Direct", + "requested": "[10.0.11, 10.0.11]", + "resolved": "10.0.11", + "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" + }, "Microsoft.Extensions.Compliance.Abstractions": { "type": "Transitive", "resolved": "10.9.0", @@ -47,11 +53,6 @@ "Microsoft.Extensions.Configuration.Abstractions": "10.0.11" } }, - "Microsoft.Extensions.DependencyInjection.Abstractions": { - "type": "Transitive", - "resolved": "10.0.11", - "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" - }, "Microsoft.Extensions.ObjectPool": { "type": "Transitive", "resolved": "10.0.11", diff --git a/src/AgentExperience.MicrosoftAgentFramework/CaptureScope.cs b/src/AgentExperience.MicrosoftAgentFramework/CaptureScope.cs index 64a4eac..98d26df 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/CaptureScope.cs +++ b/src/AgentExperience.MicrosoftAgentFramework/CaptureScope.cs @@ -4,6 +4,7 @@ using System.Text.Json; using AgentExperience.Abstractions; using AgentExperience.Core.Capture; +using AgentExperience.Core.Finalization; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; @@ -210,7 +211,8 @@ internal void FailToolCall(PendingToolCall pending, Exception exception) => /// /// Finalizes the run exactly once: appends the invocation's single attempt (every buffered tool - /// call, ordered by start) and then completes the run. Bounded by + /// call, ordered by start), completes the run, and -- when the host configured one -- hands the + /// completed run to Core's finalization service to become a durable Experience Record. Bounded by /// with its own token, never the /// caller's. A second call is a no-op. Never throws. /// @@ -311,6 +313,104 @@ private async Task FinalizeCoreAsync(AppendAttemptRequest request, Guid completi if (problems.Count > 0) { ReportFailure(ExperienceCaptureFailureStage.Finalize, string.Join(" ", problems), firstException); + return; + } + + // Only a run whose attempt and completion both landed is worth turning into a durable record: + // finalizing a half-captured run would persist an incomplete history as if it were whole. + if (_options.FinalizationService is { } finalization && _options.ResolveFinalization is { } resolve) + { + await FinalizeExperienceAsync(finalization, resolve, cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Hands the completed run to Core's finalization service, through the host's own request + /// resolver. Like everything else on this type, nothing here throws into MAF or the caller: every + /// problem is reported to and swallowed, + /// and the captured run is left untouched so the host can retry finalization itself. + /// + private async Task FinalizeExperienceAsync( + ExperienceFinalizationService finalization, + Func resolve, + CancellationToken cancellationToken) + { + FinalizeExperienceRequest? request; + try + { + if (!_service.TryGetRun(RunId, out var run)) + { + ReportFailure(ExperienceCaptureFailureStage.Finalization, "The completed run could not be read back for finalization; it is not finalized.", null); + return; + } + + request = resolve(new ExperienceFinalizationContext(run)); + } + catch (Exception ex) + { + ReportFailure(ExperienceCaptureFailureStage.Finalization, $"ResolveFinalization threw {ex.GetType().FullName}; the run is not finalized.", ex); + return; + } + + // A null request is the host declining to finalize this particular run -- not a failure. + if (request is null) + { + return; + } + + // The resolver is host code and could hand back a request for some other captured run, which + // would finalize an unrelated run on this invocation's behalf. + if (request.RunId != RunId) + { + ReportFailure( + ExperienceCaptureFailureStage.Finalization, + "ResolveFinalization returned a request for a different run; the run is not finalized.", + null); + return; + } + + FinalizeExperienceResult result; + try + { + result = await finalization.FinalizeAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + ReportFailure(ExperienceCaptureFailureStage.Finalization, $"Finalizing the run threw {ex.GetType().FullName}.", ex); + return; + } + + if (result is null) + { + ReportFailure(ExperienceCaptureFailureStage.Finalization, "FinalizeAsync returned null.", null); + return; + } + + // Only a genuine defect goes to the failure channel. A host whose policy denies storage, or + // whose authorization refuses a scope, made that decision on purpose and should not get a + // failure callback per invocation -- OnRunFinalized already carries the whole result. + if (result.Outcome is FinalizationOutcome.Failed) + { + ReportFailure( + ExperienceCaptureFailureStage.Finalization, + $"Finalization ended at stage {result.Stage} with outcome {result.Outcome}; no Experience Record is durable for this run.", + result.Failure?.Exception); + } + + // Finalization may have overrun the timeout and already been reported as such; telling the + // host it finished after that would contradict the failure it already saw. + if (cancellationToken.IsCancellationRequested) + { + return; + } + + try + { + _options.OnRunFinalized?.Invoke(result); + } + catch + { + // The host's finalization callback must never affect the agent invocation. } } diff --git a/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureAgentBuilderExtensions.cs b/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureAgentBuilderExtensions.cs index cf67a3b..db5e5b8 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureAgentBuilderExtensions.cs +++ b/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureAgentBuilderExtensions.cs @@ -25,7 +25,11 @@ public static class ExperienceCaptureAgentBuilderExtensions /// executes and completes it exactly once on success, failure, cancellation, or a streaming /// consumer that stops reading early -- and then, when /// is , function - /// middleware that records each tool call into that run. + /// middleware that records each tool call into that run. When + /// and + /// are configured, each successfully + /// captured run is then handed to Core's finalization service to become a durable Experience + /// Record, inside the same timeout-bounded step. /// /// The agent builder. /// The capture service runs are recorded through. @@ -38,6 +42,7 @@ public static class ExperienceCaptureAgentBuilderExtensions /// (a ); throws otherwise. /// /// Any argument, or , , , or , is . + /// Exactly one of and is set. /// is not positive or exceeds the timer maximum. public static AIAgentBuilder UseExperienceCapture( this AIAgentBuilder builder, @@ -56,6 +61,16 @@ public static AIAgentBuilder UseExperienceCapture( throw new ArgumentOutOfRangeException(nameof(options), options.FinalizationTimeout, $"FinalizationTimeout must be positive and at most {uint.MaxValue - 1} milliseconds."); } + // Either half alone could only ever do nothing, silently -- and a resolver without a service is + // the easier mistake to make. Only the host can supply a run's required checks, evidence, + // authorization, and storage decision, so the two are configured together or not at all. + if (options.FinalizationService is null != (options.ResolveFinalization is null)) + { + throw new ArgumentException( + $"{nameof(ExperienceCaptureOptions.FinalizationService)} and {nameof(ExperienceCaptureOptions.ResolveFinalization)} must be set together, or neither set.", + nameof(options)); + } + var middleware = new ExperienceCaptureMiddleware(captureService, options); // The first Use call is the outermost layer: run middleware wraps function middleware. diff --git a/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureOptions.cs b/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureOptions.cs index d285f3e..afccc79 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureOptions.cs +++ b/src/AgentExperience.MicrosoftAgentFramework/ExperienceCaptureOptions.cs @@ -1,5 +1,6 @@ using System.Runtime.InteropServices; using AgentExperience.Abstractions; +using AgentExperience.Core.Finalization; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; @@ -32,6 +33,16 @@ public sealed record ExperienceRunDescriptor( Scope Scope, string? TaskDescription = null); +/// +/// What the host sees when it is asked how a completed, captured run should be finalized into a +/// durable Experience Record. +/// +/// +/// The completed run's sanitized snapshot, read back from the capture service after the invocation's +/// attempt and completion were recorded. Its is set. +/// +public sealed record ExperienceFinalizationContext(ExperienceRun Run); + /// Where in the capture pipeline an happened. public enum ExperienceCaptureFailureStage { @@ -46,6 +57,15 @@ public enum ExperienceCaptureFailureStage /// Capturing a tool call's start, result, or error threw; that tool call is not recorded. ToolCall, + + /// + /// Finalizing the completed run into a durable Experience Record threw, was declined for a + /// foreign run ID, or failed a stage. A host decision (storage denied, or a scope outside the + /// authorization) is not reported here -- it is an expected outcome on + /// . The captured run is unchanged and still + /// available for the host to retry. + /// + Finalization, } /// @@ -89,9 +109,18 @@ public sealed class ExperienceCaptureOptions public bool CaptureToolCalls { get; init; } = true; /// - /// The upper bound on finalization (append attempt, then complete run) per run. Default 5 seconds. - /// Must be positive and at most - 1 milliseconds. Finalization never uses the caller's cancellation token. + /// The upper bound on the whole post-invocation step per run: append the attempt, complete the + /// run, and -- when is configured -- finalize it into a durable + /// Experience Record. Default 5 seconds. Must be positive and at most + /// - 1 milliseconds. It never uses the caller's cancellation token. /// + /// + /// With finalization configured this bound covers database round trips, not just in-memory + /// capture, so 5 seconds may be too tight for a slow or distant database. A timeout is reported + /// through and can leave the Experience Record created but not yet + /// confirmed -- a Candidate, which is never reusable. Finalizing that run again completes + /// the same commit. + /// public TimeSpan FinalizationTimeout { get; init; } = TimeSpan.FromSeconds(5); /// @@ -101,6 +130,40 @@ public sealed class ExperienceCaptureOptions /// public Action? OnCaptureFailure { get; init; } + /// + /// Optional. The Core service that turns each completed, captured run into a durable Experience + /// Record. Leave it to capture only -- the host can still finalize runs + /// itself, whenever it likes, from the capture service's snapshot. + /// + /// + /// Setting this requires too (and vice versa), because only the + /// host knows a run's required checks, its verification evidence, its authorization context, and + /// its storage policy. Finalization runs inside the same once-only, -bounded + /// step as capture finalization, after the run's attempt and completion were recorded, and only + /// when both of those succeeded. It never uses the caller's cancellation token and never changes + /// what the caller of the agent observes. + /// + public ExperienceFinalizationService? FinalizationService { get; init; } + + /// + /// Required when is set (and only valid then): builds the + /// finalize request for one completed run. Returning skips finalizing that + /// run. The request's RunId must be this invocation's run. If it throws, or returns a + /// request for another run, the run is not finalized and the failure is reported through + /// . + /// + public Func? ResolveFinalization { get; init; } + + /// + /// Optional. Receives every finalization result, durable or not -- including an expected + /// or + /// , which are host decisions rather than capture + /// failures and are therefore not reported through . Not called when + /// finalization already overran . Exceptions thrown by the + /// callback are swallowed. + /// + public Action? OnRunFinalized { get; init; } + /// The clock used for run, attempt, and tool-call timestamps, durations, and the finalization timeout. public TimeProvider TimeProvider { get; init; } = TimeProvider.System; diff --git a/src/AgentExperience.MicrosoftAgentFramework/README.md b/src/AgentExperience.MicrosoftAgentFramework/README.md index 56c4cc2..a0cdb57 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/README.md +++ b/src/AgentExperience.MicrosoftAgentFramework/README.md @@ -40,7 +40,8 @@ Call `UseExperienceCapture` first on the builder so capture is the outermost lay its result or exception, into the run for that invocation. Every recorded value goes through the `IExperienceCaptureService` you pass in, so its sanitizer and limits apply -unchanged. This adapter does not evaluate, reflect, or persist runs. +unchanged. This adapter never evaluates, reflects, or persists anything itself; when you configure finalization +(below) it hands the completed run to Core's `ExperienceFinalizationService`, which owns all of that. When the caller passes a session, the run ID is written to it under `ExperienceCaptureAgentBuilderExtensions.RunIdStateKey` (`"AgentExperience.RunId"`) as a `"D"`-formatted GUID @@ -54,11 +55,56 @@ because MAF forwards those to the model provider. | `ResolveRun` | required | Maps messages, session, and agent to a task ID, scope, and task description. If it throws or returns a null descriptor, task ID, or scope, the invocation runs uncaptured and the failure is reported. | | `Environment` | machine name + `RuntimeInformation` | Environment fingerprint recorded on every run. | | `CaptureToolCalls` | `true` | Registers function middleware. Requires a `ChatClientAgent`. | -| `FinalizationTimeout` | 5 s | Upper bound on appending the attempt and completing the run. Must be positive and at most `uint.MaxValue - 1` milliseconds. Finalization uses its own token, never the caller's. | -| `OnCaptureFailure` | none | Called when capture fails, at most once per failure stage (`ResolveRun`, `StartRun`, `Finalize`, `ToolCall`) per run. Exceptions it throws are swallowed. | +| `FinalizationTimeout` | 5 s | Upper bound on the whole post-invocation step: appending the attempt, completing the run, and — when `FinalizationService` is set — finalizing it into a durable Experience Record. With finalization configured this bounds database round trips, not just in-memory capture, so 5 s may be too tight. Must be positive and at most `uint.MaxValue - 1` milliseconds. It uses its own token, never the caller's. | +| `OnCaptureFailure` | none | Called when capture fails, at most once per failure stage per run. The stages are `ResolveRun`, `StartRun`, `Finalize` (recording the attempt and completing the run in memory), `ToolCall`, and `Finalization` (turning the completed run into a durable Experience Record). Exceptions it throws are swallowed. | +| `FinalizationService` | none | Core's `ExperienceFinalizationService`. When set, each successfully captured run is finalized into a durable Experience Record. Requires `ResolveFinalization`. | +| `ResolveFinalization` | none | Builds the `FinalizeExperienceRequest` for one completed run. Return `null` to skip finalizing that run. Required when `FinalizationService` is set. | +| `OnRunFinalized` | none | Receives every `FinalizeExperienceResult`, durable or not — including a host decision such as `StorageDenied`, which is not a capture failure. Not called once finalization has overrun `FinalizationTimeout`. Exceptions it throws are swallowed. | | `TimeProvider` | `TimeProvider.System` | Timestamps, durations, and the finalization timeout. | | `NewId` | `Guid.NewGuid` | Run, attempt, tool-call, and completion-event IDs. Must be thread-safe. | +## Finalizing captured runs + +Capture alone keeps the run in memory. To turn each invocation into a durable Experience Record, give the adapter +Core's finalization service and a resolver that supplies what only the host knows — the required checks, the +verification evidence and the round it was closed in, the authorization context, and the storage decision: + +```csharp +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Verification; + +var options = new ExperienceCaptureOptions +{ + ResolveRun = context => new ExperienceRunDescriptor("triage-ticket", hostScope), + + FinalizationService = finalization, // AgentExperience.Core.Finalization.ExperienceFinalizationService + ResolveFinalization = context => new FinalizeExperienceRequest( + RunId: context.Run.RunId, + Authorization: hostAuthorization, + ClosedRound: new ClosedVerificationRound(roundId, artifactRevision), + RequiredChecks: [new RequiredCheck("unit-tests-pass", ExpectedKind: "TestResult")], + Evidence: evidenceFor(context.Run), + CurrentArtifactRevision: artifactRevision, + StorageDecision: StorageDecision.Permit, + FinalizedAt: DateTimeOffset.UtcNow), + + OnRunFinalized = result => logger.LogInformation( + "Experience {Id} is {Status}", result.ExperienceId, result.Status), +}; +``` + +- **When it runs.** Immediately after the run's attempt and completion were both recorded, inside the same once-only + step and the same `FinalizationTimeout`. A run whose capture reported a problem is never finalized, so a + half-captured run is never persisted as if it were whole. +- **Opting out per run.** `ResolveFinalization` returning `null` skips that run, and is not a failure. +- **Failures.** A throwing resolver, a throwing `FinalizeAsync`, or a non-durable outcome (`StorageDenied`, + `NotAuthorized`, `Failed`, …) is reported through `OnCaptureFailure` with stage `Finalization` and never thrown. + The captured run is left untouched, so the host can retry finalization itself from the capture service. +- **Latency.** Finalization is a database round trip and is awaited inside `FinalizationTimeout`, so it adds + caller-visible latency. Leave `FinalizationService` unset and finalize out of band if that is not acceptable. +- **Setting `FinalizationService` without `ResolveFinalization` throws** at `UseExperienceCapture`, rather than + silently doing nothing. + ## Supported agent types | Agent | Run lifecycle | Tool calls | @@ -89,7 +135,9 @@ because MAF forwards those to the model provider. the agent would produce without capture. Capture does not wrap or re-execute tools, retry, or keep its own session store. - **Capture failures** are reported through `OnCaptureFailure` and never thrown. They include a resolver exception, a - non-success capture outcome, a capture exception, and a finalization timeout. Each failure stage is reported at most + non-success capture outcome, a capture exception, a finalization timeout, and a failed Experience Record + finalization. Note the two similarly named stages: `Finalize` is the in-memory capture step (append the attempt, + complete the run), while `Finalization` is turning that completed run into a durable Experience Record. Each failure stage is reported at most once per run, so an earlier tool-call or session-write failure never hides a later finalization failure. - **Finalization.** Completion is attempted even when appending the attempt fails. If finalization times out, the run may be left without an attempt or completion. diff --git a/src/AgentExperience.MicrosoftAgentFramework/packages.lock.json b/src/AgentExperience.MicrosoftAgentFramework/packages.lock.json index 89f118c..c850164 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/packages.lock.json +++ b/src/AgentExperience.MicrosoftAgentFramework/packages.lock.json @@ -187,7 +187,8 @@ "type": "Project", "dependencies": { "AgentExperience.Abstractions": "[1.0.0, )", - "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )" + "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]" } } } diff --git a/src/AgentExperience.Storage.Postgres.Vectors/AgentExperience.Storage.Postgres.Vectors.csproj b/src/AgentExperience.Storage.Postgres.Vectors/AgentExperience.Storage.Postgres.Vectors.csproj new file mode 100644 index 0000000..22efcda --- /dev/null +++ b/src/AgentExperience.Storage.Postgres.Vectors/AgentExperience.Storage.Postgres.Vectors.csproj @@ -0,0 +1,36 @@ + + + + pgvector adapter for AgentExperience.NET: stores one derived embedding per Experience Record through the IExperienceEmbeddingIndex port, with conditional revision-and-existence-checked writes, scoped idempotent re-indexing, and a scoped nearest-neighbour search that applies the same eligibility filters as the text channel. Split out of AgentExperience.Storage.Postgres because that package's dependency boundary forbids Pgvector and the model-provider abstractions. Pinned to Npgsql 10.0.3, Pgvector 0.3.2, and Microsoft.Extensions.AI.Abstractions 10.9.0. + true + README.md + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/AgentExperience.Storage.Postgres.Vectors/AiExperienceEmbeddingGenerator.cs b/src/AgentExperience.Storage.Postgres.Vectors/AiExperienceEmbeddingGenerator.cs new file mode 100644 index 0000000..ace3afd --- /dev/null +++ b/src/AgentExperience.Storage.Postgres.Vectors/AiExperienceEmbeddingGenerator.cs @@ -0,0 +1,139 @@ +using AgentExperience.Abstractions; +using Microsoft.Extensions.AI; + +namespace AgentExperience.Storage.Postgres.Vectors; + +/// +/// Adapts a Microsoft.Extensions.AI to +/// AgentExperience's own domain-typed . It exists so the +/// model-provider abstraction stops here, at the adapter edge: neither +/// AgentExperience.Abstractions nor AgentExperience.Core ever references +/// Microsoft.Extensions.AI, and a host can swap this for an in-process, deterministic, or +/// bespoke generator without either of them noticing. +/// +/// +/// +/// The model ID and the dimension are fixed at construction. They have to be: the content hash +/// covers the model ID, so "the same text under the same model" must be recognizable before +/// any provider call is made. They are read from the generator's +/// unless the caller states them, and a generator that +/// reports neither is rejected here -- at startup -- rather than producing vectors nobody can decide +/// the comparability of later. +/// +/// +/// It validates what the provider returned. A response with no embedding, or one whose width +/// is not , throws rather than being stored: a descriptor that disagrees with +/// its own vector would make every later comparison against it unsound. The caller treats that throw +/// as a retryable provider failure, exactly like a timeout. +/// +/// +public sealed class AiExperienceEmbeddingGenerator : IExperienceEmbeddingGenerator +{ + private readonly IEmbeddingGenerator> _generator; + private readonly EmbeddingGenerationOptions _options; + + /// + /// Wraps , taking its model ID and dimension from the arguments when + /// given and otherwise from the generator's own metadata. + /// + /// The underlying embedding generator. Its lifetime belongs to the host; this adapter never disposes it. + /// Optional. The model identifier to stamp on every embedding, and to ask the provider for. Defaults to , and may not contradict it when it is reported. + /// Optional. The vector width to require, and to ask the provider for. Defaults to , and may not contradict it when it is reported. + /// is . + /// is blank, contradicts the generator's reported model, or no model ID is available from either source. + /// is outside 1.., contradicts the generator's reported dimension, or no dimension is available from either source. + public AiExperienceEmbeddingGenerator( + IEmbeddingGenerator> generator, + string? modelId = null, + int? dimension = null) + { + ArgumentNullException.ThrowIfNull(generator); + _generator = generator; + + var metadata = generator.GetService(); + + var effectiveModelId = modelId ?? metadata?.DefaultModelId; + if (string.IsNullOrWhiteSpace(effectiveModelId)) + { + throw new ArgumentException( + "The embedding model ID must be supplied, or reported by the generator's EmbeddingGeneratorMetadata: " + + "it is part of every stored embedding's content hash and decides which vectors are comparable.", + nameof(modelId)); + } + + if (effectiveModelId.Length > ExperienceEmbeddingDescriptor.MaxModelIdLength) + { + throw new ArgumentException( + $"The embedding model ID must be at most {ExperienceEmbeddingDescriptor.MaxModelIdLength} characters.", + nameof(modelId)); + } + + var effectiveDimension = dimension ?? metadata?.DefaultModelDimensions; + if (effectiveDimension is not (>= 1 and <= ExperienceEmbeddingDescriptor.MaxDimension)) + { + throw new ArgumentOutOfRangeException( + nameof(dimension), + effectiveDimension, + "The embedding dimension must be supplied, or reported by the generator's EmbeddingGeneratorMetadata, " + + $"and must be between 1 and {ExperienceEmbeddingDescriptor.MaxDimension}."); + } + + // An override that contradicts what the generator actually runs is the one failure the + // descriptor exists to prevent: vectors would be stamped with a model ID the provider never + // used, so two genuinely incomparable sets would look comparable. Rejected at wiring time. + if (metadata?.DefaultModelId is { Length: > 0 } reportedModelId + && !string.Equals(reportedModelId, effectiveModelId, StringComparison.Ordinal)) + { + throw new ArgumentException( + $"The generator reports model '{reportedModelId}', but '{effectiveModelId}' was supplied. Stamping vectors " + + "with a model ID the provider did not produce them under would make incomparable vectors look comparable.", + nameof(modelId)); + } + + if (metadata?.DefaultModelDimensions is { } reportedDimension && reportedDimension != effectiveDimension) + { + throw new ArgumentOutOfRangeException( + nameof(dimension), + effectiveDimension, + $"The generator reports {reportedDimension} dimensions, but {effectiveDimension} was supplied."); + } + + ModelId = effectiveModelId; + Dimension = effectiveDimension.Value; + + // The resolved model and dimension are what the provider is actually asked for, not merely what + // the stored descriptor claims -- otherwise a request would silently run on the provider's own + // default while being stamped with something else. + _options = new EmbeddingGenerationOptions { ModelId = ModelId, Dimensions = Dimension }; + } + + /// + public string ModelId { get; } + + /// + public int Dimension { get; } + + /// + /// is . + /// The generator returned no embedding, or one of the wrong width. + public async Task> GenerateAsync(string text, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(text); + + var generated = await _generator + .GenerateAsync([text], _options, cancellationToken) + .ConfigureAwait(false); + + if (generated is not { Count: > 0 }) + { + throw new InvalidOperationException("The embedding generator returned no embedding for the requested text."); + } + + var vector = generated[0].Vector; + return vector.Length == Dimension + ? vector + : throw new InvalidOperationException( + $"The embedding generator returned a {vector.Length}-component vector where {Dimension} were declared; " + + "a stored descriptor must never disagree with its own vector."); + } +} diff --git a/src/AgentExperience.Storage.Postgres.Vectors/DependencyInjection/AgentExperiencePostgresVectorsServiceCollectionExtensions.cs b/src/AgentExperience.Storage.Postgres.Vectors/DependencyInjection/AgentExperiencePostgresVectorsServiceCollectionExtensions.cs new file mode 100644 index 0000000..879b7c2 --- /dev/null +++ b/src/AgentExperience.Storage.Postgres.Vectors/DependencyInjection/AgentExperiencePostgresVectorsServiceCollectionExtensions.cs @@ -0,0 +1,97 @@ +using AgentExperience.Abstractions; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Npgsql; + +namespace AgentExperience.Storage.Postgres.Vectors.DependencyInjection; + +/// +/// Registers the pgvector embedding index, and optionally an embedding generator over a +/// Microsoft.Extensions.AI one, in a . The adapter owns its +/// own registration, exactly as Core owns AddAgentExperienceCore, so a host wires them +/// together without either package knowing the other's concrete types. +/// +public static class AgentExperiencePostgresVectorsServiceCollectionExtensions +{ + /// + /// Registers as the singleton + /// , over an resolved from + /// the container. + /// + /// + /// The host owns the data source's lifetime and the index never disposes it. The schema is not + /// applied here: the extension and the embedding table live in this package's own + /// 0004_add_experience_embeddings.sql, applied at startup by + /// + /// after the base adapter's ExperienceSchemaMigrator.MigrateAsync. The two are + /// separate calls on purpose: this one needs the privilege to create the vector extension, + /// and a text-only host should never be made to have it. The dimension-specific HNSW index is + /// separate again, and optional; see . + /// + /// The service collection to add to. + /// , for chaining. + /// is . + public static IServiceCollection AddAgentExperiencePostgresEmbeddingIndex(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(provider => + new PostgresExperienceEmbeddingIndex(provider.GetRequiredService())); + + return services; + } + + /// + /// Registers as the singleton + /// over , for a host that + /// keeps its data source outside the container. + /// + /// The service collection to add to. + /// The host-owned data source the index opens connections from. Never disposed by the index. + /// , for chaining. + /// Any argument is . + public static IServiceCollection AddAgentExperiencePostgresEmbeddingIndex( + this IServiceCollection services, + NpgsqlDataSource dataSource) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(dataSource); + + services.TryAddSingleton(new PostgresExperienceEmbeddingIndex(dataSource)); + + return services; + } + + /// + /// Registers as the singleton + /// over an + /// resolved from the container. + /// + /// + /// Registered separately from the index because they are independent decisions: a host may index + /// with one generator and query with another process, or use a deterministic generator in tests + /// while keeping the real index. The model ID and dimension are resolved once, when the singleton + /// is first created, so a generator that reports neither -- or an argument that contradicts what it + /// does report -- fails there rather than mid-query. + /// + /// The service collection to add to. + /// Optional. The model ID to use when the generator reports none. It may not contradict one the generator does report. + /// Optional. The dimension to use when the generator reports none. It may not contradict one the generator does report. + /// , for chaining. + /// is . + public static IServiceCollection AddAgentExperienceEmbeddingGenerator( + this IServiceCollection services, + string? modelId = null, + int? dimension = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(provider => new AiExperienceEmbeddingGenerator( + provider.GetRequiredService>>(), + modelId, + dimension)); + + return services; + } +} diff --git a/src/AgentExperience.Storage.Postgres.Vectors/ExperienceVectorIndexMaintenance.cs b/src/AgentExperience.Storage.Postgres.Vectors/ExperienceVectorIndexMaintenance.cs new file mode 100644 index 0000000..576809e --- /dev/null +++ b/src/AgentExperience.Storage.Postgres.Vectors/ExperienceVectorIndexMaintenance.cs @@ -0,0 +1,127 @@ +using System.Globalization; +using AgentExperience.Abstractions; +using Npgsql; + +namespace AgentExperience.Storage.Postgres.Vectors; + +/// +/// The explicit, out-of-band creation of the approximate-nearest-neighbour index on +/// agent_experience.experience_embeddings. It is a separate, deliberate call rather than part +/// of the schema migration because an HNSW index needs a dimension, and the dimension +/// belongs to whichever embedding model a host configured -- something no shipped migration can know. +/// +/// +/// +/// Why the column is unconstrained and the index is not. 0004 declares +/// embedding vector with no type modifier, so one deployment can hold 384-wide vectors and +/// another 1536-wide ones without a schema fork. pgvector's opclasses, however, refuse a column +/// without a dimension. The index is therefore built over the expression +/// embedding::vector(n) and is partial on dimension = n: the predicate is what +/// makes the cast safe, because the build only ever touches rows that really are that wide. Several +/// dimensions can coexist, each with its own index. +/// +/// +/// It is optional. Every search is correct without it -- pgvector falls back to an exact scan, +/// which is what a small deployment wants anyway. The index changes latency, and it also makes a +/// search approximate: HNSW may miss a true nearest neighbour. Create it when a scope holds +/// enough embeddings for an exact scan to hurt. +/// +/// +/// It matches the search's own expression. The distance +/// computes is cosine +/// (<=>, vector_cosine_ops) over exactly embedding::vector(n), with +/// dimension = n in the predicate. Changing either side alone silently stops the index being +/// used. +/// +/// +/// This runs DDL, so it needs a connection with rights to create an index on the schema. Building an +/// HNSW index over many rows takes minutes and holds a lock that blocks writes to the table for its +/// duration -- run it from a maintenance path, never from request handling. +/// +/// +public static class ExperienceVectorIndexMaintenance +{ + /// The name of the dimension-specific index, so a host can find, monitor, or drop it by name. + /// The vector width the index covers. + /// The index name, unqualified. + /// is not between 1 and . + public static string IndexNameFor(int dimension) => + string.Create(CultureInfo.InvariantCulture, $"ix_experience_embeddings_hnsw_{Ensure(dimension)}"); + + /// + /// Creates the cosine HNSW index for -wide vectors if it does not + /// already exist. Idempotent: calling it again once the index exists does nothing. + /// + /// The host-owned data source. Never disposed here. + /// The vector width to index, which must be the dimension of the model the host embeds with. + /// Cancels the operation. Cancelling does not necessarily stop an index build already running on the server. + /// A task that completes once the index exists. + /// is . + /// is not between 1 and . + /// The index could not be created. + public static async Task EnsureHnswIndexAsync( + NpgsqlDataSource dataSource, + int dimension, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(dataSource); + var width = Ensure(dimension).ToString(CultureInfo.InvariantCulture); + + // The dimension is an int this method has already bounded, so nothing caller-controlled + // reaches the statement text; a pgvector type modifier can never be a parameter anyway. + var sql = + $"CREATE INDEX IF NOT EXISTS {IndexNameFor(dimension)} " + + $"ON {PostgresExperienceEmbeddingIndex.Table} " + + $"USING hnsw ((embedding::vector({width})) vector_cosine_ops) " + + $"WHERE dimension = {width}"; + + try + { + await using var command = dataSource.CreateCommand(sql); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (PostgresExperienceRecordStore.IsInfrastructureFailure(ex, cancellationToken)) + { + throw PostgresExperienceRecordStore.Translate(ex, "embedding index creation", cancellationToken); + } + } + + /// + /// Drops the dimension-specific index if it exists, for a host retiring a model. Dropping it never + /// changes a search's results, only its latency and whether it is approximate. + /// + /// The host-owned data source. Never disposed here. + /// The vector width whose index to drop. + /// Cancels the operation. + /// A task that completes once the index is gone. + /// is . + /// is not between 1 and . + /// The index could not be dropped. + public static async Task DropHnswIndexAsync( + NpgsqlDataSource dataSource, + int dimension, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(dataSource); + var name = IndexNameFor(dimension); + + try + { + await using var command = dataSource.CreateCommand( + $"DROP INDEX IF EXISTS {PostgresExperienceRecordSchema.SchemaName}.{name}"); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (PostgresExperienceRecordStore.IsInfrastructureFailure(ex, cancellationToken)) + { + throw PostgresExperienceRecordStore.Translate(ex, "embedding index removal", cancellationToken); + } + } + + private static int Ensure(int dimension) => + dimension is >= 1 and <= ExperienceEmbeddingDescriptor.MaxDimension + ? dimension + : throw new ArgumentOutOfRangeException( + nameof(dimension), + dimension, + $"The vector dimension must be between 1 and {ExperienceEmbeddingDescriptor.MaxDimension}."); +} diff --git a/src/AgentExperience.Storage.Postgres.Vectors/ExperienceVectorSchema.cs b/src/AgentExperience.Storage.Postgres.Vectors/ExperienceVectorSchema.cs new file mode 100644 index 0000000..08efecb --- /dev/null +++ b/src/AgentExperience.Storage.Postgres.Vectors/ExperienceVectorSchema.cs @@ -0,0 +1,111 @@ +using AgentExperience.Abstractions; +using Npgsql; + +namespace AgentExperience.Storage.Postgres.Vectors; + +/// +/// Access to the schema scripts embedded in this package, and the call that applies them. +/// The embedding schema is owned here rather than by AgentExperience.Storage.Postgres on +/// purpose: it begins with CREATE EXTENSION vector, which is not a trusted extension and so +/// needs a superuser (or an equivalently privileged) role. Putting it in the base adapter's script +/// list would have made that privilege a startup requirement for every host, including text-only ones +/// that never enable the vector channel at all. +/// +public static class ExperienceVectorSchema +{ + /// + /// The script that creates the vector extension and the derived experience_embeddings + /// table the vector retrieval channel reads. Its embedding column is an unconstrained + /// vector: the dimension belongs to whichever model a host configured, so the + /// dimension-specific HNSW index is created out of band by + /// rather than by this script. + /// + /// + /// The number continues the family's sequence past the base adapter's 0001-0003, so a + /// reader can still order the whole schema at a glance, even though the two packages apply their + /// scripts separately. + /// + public const string EmbeddingsScriptName = "0004_add_experience_embeddings.sql"; + + internal const string ResourcePrefix = "AgentExperience.Storage.Postgres.Vectors.Migrations."; + + /// Every embedded script name, in the order they must be applied. + public static IReadOnlyList ScriptNames { get; } = [EmbeddingsScriptName]; + + /// Reads an embedded script's SQL text. + /// One of . + /// The script's SQL. + /// is not an embedded script. + public static string GetScript(string scriptName) + { + ArgumentNullException.ThrowIfNull(scriptName); + if (!ScriptNames.Contains(scriptName, StringComparer.Ordinal)) + { + throw new ArgumentException("Unknown schema script name.", nameof(scriptName)); + } + + using var stream = typeof(ExperienceVectorSchema).Assembly.GetManifestResourceStream(ResourcePrefix + scriptName) + ?? throw new InvalidOperationException($"Embedded schema script '{scriptName}' is missing from the assembly."); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } +} + +/// +/// Applies this package's embedded schema scripts, journaled, exactly the way +/// applies the base adapter's: scripts in name order, one +/// transaction per script, recorded in agent_experience.schema_versions, and serialized across +/// processes by the same PostgreSQL session advisory lock, so the two migrators can run concurrently +/// on one database without racing. +/// +/// +/// +/// The two migrators share a journal table but never share an entry: a journal row records DbUp's +/// script name, which is the full embedded-resource name, and this package's resources live under a +/// different prefix from the base adapter's. +/// +/// +/// Run the base migration first. 0004 declares a foreign key to +/// agent_experience.experience_records, so +/// must have +/// created that table before this call. +/// +/// +/// Privileges. The role running this needs whatever the base migration needs, plus the right +/// to CREATE EXTENSION vector -- pgvector is not a trusted extension, so that is ordinarily a +/// superuser (on a managed service, whichever role that provider designates). A deployment whose +/// operators install the extension out of band can run this as an ordinary role: +/// CREATE EXTENSION IF NOT EXISTS vector is a no-op once it exists. The index itself needs +/// SELECT and INSERT/UPDATE on agent_experience.experience_embeddings and +/// SELECT on agent_experience.experience_records. +/// +/// +public static class ExperienceVectorSchemaMigrator +{ + /// + /// Applies every embedded embedding-schema script that this database has not recorded yet. + /// + /// + /// The host-owned data source for the database to migrate. Never disposed here. It must allow at + /// least two concurrent connections and must not be multiplexing. + /// + /// + /// Cancels opening the lock connection and waiting for the advisory lock. Once scripts start + /// running, cancellation is ignored, so the run finishes and returns normally. + /// + /// The scripts applied by this call. Empty when nothing was pending. + /// is . + /// A script failed, or the database was unreachable. + /// was cancelled. + public static Task MigrateAsync( + NpgsqlDataSource dataSource, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(dataSource); + return ExperienceSchemaMigrator.MigrateAsync( + dataSource, + typeof(ExperienceVectorSchema).Assembly, + ExperienceVectorSchema.ResourcePrefix, + cancellationToken); + } +} diff --git a/src/AgentExperience.Storage.Postgres.Vectors/Migrations/0004_add_experience_embeddings.sql b/src/AgentExperience.Storage.Postgres.Vectors/Migrations/0004_add_experience_embeddings.sql new file mode 100644 index 0000000..6eee58f --- /dev/null +++ b/src/AgentExperience.Storage.Postgres.Vectors/Migrations/0004_add_experience_embeddings.sql @@ -0,0 +1,80 @@ +-- AgentExperience.NET: derived embeddings over Experience Records, for hybrid retrieval (Story 2.6). +-- Applied by ExperienceVectorSchemaMigrator -- this package's own migrator -- and recorded in +-- agent_experience.schema_versions alongside the base adapter's scripts. It is deliberately NOT in the +-- base adapter's script list: CREATE EXTENSION vector needs a superuser, because pgvector is not a +-- trusted extension, and a text-only deployment must never be made to run it for a feature it has not +-- enabled. Run the base migration first: the foreign key below needs experience_records to exist. +-- Every statement is IF NOT EXISTS on purpose, matching 0001-0003, so a database whose schema was +-- applied by hand can still be journaled. Do not edit this script once it has been journaled anywhere; +-- add the next-numbered script instead. +-- +-- This script is append-only with respect to 0001-0003: it creates the pgvector extension and one new +-- table, and alters nothing that already exists. The canonical record table is untouched, so every +-- write path -- the store's INSERT and its lifecycle UPDATE -- is unaffected, and a deployment that +-- never indexes anything simply has an empty table. + +CREATE EXTENSION IF NOT EXISTS vector; + +-- One row per indexed record. The embedding is derived data: the canonical record is created, +-- committed, and text-searchable whether or not a row ever appears here, and nothing in this table +-- takes part in a lifecycle decision. +-- +-- The scope columns are denormalized from the record on purpose, so the scope predicate a search +-- applies can be decided from this table's own index rather than only after the join. They are +-- written from the record row inside the conditional INSERT ... SELECT, never from caller input, so +-- they cannot disagree with the record they copy. +-- +-- model_id, dimension, content_hash, and source_revision are what the embedding *is*, kept separate +-- from lifecycle state (status, revision, reuse confidence all stay on the record): +-- * model_id + dimension decide comparability -- a query vector is only ever compared with vectors +-- from the same model at the same width; +-- * content_hash is SHA-256 over the model ID and the normalized summary that was embedded, so a +-- re-index can skip an unchanged record without calling a provider at all; +-- * source_revision is the record revision the summary was read at, which makes every write +-- conditional: a write only lands while the record is still at exactly that revision. +-- +-- The embedding column is an UNCONSTRAINED `vector`, not `vector(n)`. The dimension is a property of +-- whichever model a host configured, which this schema cannot know, and a typmod would have to be +-- chosen here and then be wrong for everyone else. The dimension is carried in its own column and +-- checked in the search predicate instead; the dimension-specific HNSW index is created out of band +-- (see ExperienceVectorIndexMaintenance) because it, too, needs a dimension this script does not have. +-- +-- ON DELETE CASCADE: an embedding may never outlive the record it describes. Together with the +-- conditional INSERT ... SELECT, which can only insert a row whose record currently exists, this is +-- what makes "a deleted record can never be recreated by an in-flight write" true of the schema and +-- not only of the adapter. +CREATE TABLE IF NOT EXISTS agent_experience.experience_embeddings ( + experience_id uuid PRIMARY KEY + REFERENCES agent_experience.experience_records (experience_id) ON DELETE CASCADE, + tenant_id text NOT NULL, + application_id text NOT NULL, + project_id text NOT NULL, + team_id text NULL, + agent_id text NULL, + user_id text NULL, + model_id text NOT NULL, + dimension integer NOT NULL, + content_hash text NOT NULL, + source_revision bigint NOT NULL, + embedding vector NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT ck_experience_embeddings_dimension CHECK (dimension > 0), + CONSTRAINT ck_experience_embeddings_source_revision CHECK (source_revision >= 0), + -- The stored vector and the dimension column can never disagree. The search filters on `dimension` + -- and then casts `embedding::vector(n)`; without this constraint a row written outside this adapter + -- could claim a width it does not have, and the cast would raise mid-query on a row the predicate + -- had already admitted. It also keeps the partial HNSW index's own expression buildable. + CONSTRAINT ck_experience_embeddings_vector_dims CHECK (vector_dims(embedding) = dimension) +); + +-- The three required scope columns plus the two comparability columns: exactly the predicate a +-- scoped vector search applies before it ever computes a distance, so an incomparable or foreign-scope +-- row is never read. The optional scope columns are deliberately absent for the same reason as in +-- 0003: they are matched with IS NOT DISTINCT FROM, which is not an indexable btree operator. +CREATE INDEX IF NOT EXISTS ix_experience_embeddings_scope_model + ON agent_experience.experience_embeddings + (tenant_id, application_id, project_id, model_id, dimension); + +-- A re-index pass walks a scope's records in a stable order and pages through them; the record table's +-- own scope index serves that walk, so nothing else is needed here. diff --git a/src/AgentExperience.Storage.Postgres.Vectors/PostgresExperienceEmbeddingIndex.cs b/src/AgentExperience.Storage.Postgres.Vectors/PostgresExperienceEmbeddingIndex.cs new file mode 100644 index 0000000..2a69d0f --- /dev/null +++ b/src/AgentExperience.Storage.Postgres.Vectors/PostgresExperienceEmbeddingIndex.cs @@ -0,0 +1,538 @@ +using System.Data.Common; +using System.Globalization; +using AgentExperience.Abstractions; +using Npgsql; +using NpgsqlTypes; +using Pgvector; + +namespace AgentExperience.Storage.Postgres.Vectors; + +/// +/// over PostgreSQL and pgvector. It follows exactly the order +/// uses -- validate the request, check it against the +/// host-established , and only then open a connection and run +/// parameterized SQL whose predicates apply the exact scope -- and translates failures the same way. +/// The schema must already exist: 0004_add_experience_embeddings.sql creates the extension and +/// the experience_embeddings table, and the host applies it by calling +/// +/// after the base adapter's own migration. +/// +/// +/// +/// Writes are conditional, in SQL. A write is an INSERT ... SELECT whose source is the +/// canonical record row itself, matched on the exact scope and on the revision the write +/// names. So a record that has moved on writes nothing () +/// and a record that no longer exists writes nothing and creates no row +/// () -- an in-flight write can never resurrect a deleted +/// record, because there is no row for its SELECT to read. The scope columns stored alongside +/// the vector are copied from that same record row, never from caller input, so they cannot disagree +/// with the record they describe. +/// +/// +/// Never inside the canonical transaction. Every statement here runs on its own connection +/// from the host's pooled data source. This adapter is only ever called after a record's +/// lifecycle commit has landed; it never participates in one. That is a property of the design, not +/// of the driver: embeddings are derived data, and the canonical write must not depend on a provider. +/// +/// +/// Comparability is a predicate, not a check afterwards. A search filters on the query's model +/// ID and on the query vector's width before any distance is computed, so a vector from another model +/// or of another dimension is never compared. When the scope holds embeddings but none of them are +/// comparable, the result says which ( or +/// ) rather than looking like an empty +/// match. +/// +/// +/// Relevance. Distance is pgvector's cosine distance (<=>), which lies in [0, 2]; +/// the reported relevance is 1 - distance / 2, so it is already in [0, 1] with 1 for an exact +/// direction match. Like the text channel's relevance it is a within-search measure. +/// +/// +/// This adapter reads and writes only agent_experience.experience_embeddings. It needs +/// SELECT on agent_experience.experience_records and never writes to it. +/// +/// +public sealed class PostgresExperienceEmbeddingIndex : IExperienceEmbeddingIndex +{ + /// The derived embedding table. Created by 0004_add_experience_embeddings.sql. + internal const string Table = "agent_experience.experience_embeddings"; + + /// + /// qualified with the r alias. + /// Derived from the shared constant rather than retyped, so a column added there cannot silently + /// shift this reader's ordinals: still sees + /// ordinals 0-17 in exactly the documented order, and the distance is appended after them. + /// + private static readonly string RecordColumns = + "r." + PostgresExperienceRecordStore.SelectColumns.Replace(", ", ", r.", StringComparison.Ordinal); + + /// The alias the distance is selected under, read back by name rather than by ordinal. + private const string DistanceColumn = "distance"; + + /// + /// The same exact-scope predicate, qualified with the e alias. It is redundant with the + /// r-aliased one -- the embedding's scope columns are copied from the record row inside the + /// write, so they cannot disagree -- and it is applied anyway, because it is what lets + /// ix_experience_embeddings_scope_model serve the search: a btree on + /// (tenant_id, application_id, project_id, model_id, dimension) cannot be used when the only + /// predicates on the embeddings table are the trailing two columns. + /// + private static readonly string EmbeddingScopePredicate = + PostgresExperienceRecordStore.RecordScopePredicate.Replace("r.", "e.", StringComparison.Ordinal); + + /// + /// The conditional write. The target table is aliased t so the conflict action can name it + /// unambiguously, and the source row is the canonical record itself: nothing is inserted unless + /// that record exists, in exactly this scope, at exactly this revision. + /// + /// The ON CONFLICT guard is a second line of defence for two writes racing each other: an + /// older in-flight write can never overwrite a vector already stored from a newer revision. + /// + /// + private const string WriteSql = + $"INSERT INTO {Table} AS t (experience_id, tenant_id, application_id, project_id, team_id, agent_id, user_id, " + + "model_id, dimension, content_hash, source_revision, embedding, created_at, updated_at) " + + "SELECT r.experience_id, r.tenant_id, r.application_id, r.project_id, r.team_id, r.agent_id, r.user_id, " + + "@model_id, @dimension, @content_hash, @source_revision, CAST(@embedding AS vector), @now, @now " + + $"FROM {PostgresExperienceRecordStore.Table} r " + + $"WHERE r.experience_id = @experience_id AND {PostgresExperienceRecordStore.RecordScopePredicate} " + + "AND r.revision = @source_revision " + + "ON CONFLICT (experience_id) DO UPDATE SET " + + "model_id = EXCLUDED.model_id, dimension = EXCLUDED.dimension, content_hash = EXCLUDED.content_hash, " + + "source_revision = EXCLUDED.source_revision, embedding = EXCLUDED.embedding, updated_at = EXCLUDED.updated_at " + + "WHERE t.source_revision <= EXCLUDED.source_revision"; + + /// + /// Reports a rejected write: the record's current revision, or nothing at all when it is not in + /// this scope. Identical whichever scope actually owns the record, so it reveals nothing. + /// + private const string ProbeRevisionSql = + $"SELECT r.revision FROM {PostgresExperienceRecordStore.Table} r " + + $"WHERE r.experience_id = @experience_id AND {PostgresExperienceRecordStore.RecordScopePredicate}"; + + /// + /// One statement, so a record's revision, the summary read at that revision, and the descriptor of + /// whatever vector is stored for it all come from a single snapshot. The summary is assembled from + /// exactly the three fields 0003 indexes for text, so both channels describe the same claim + /// about the record. A left join keeps a never-indexed record in the list with a null descriptor. + /// + private const string ScanSql = + "SELECT r.experience_id, r.revision, r.task_id, r.payload ->> 'taskSummary', " + + "r.payload -> 'reflection' ->> 'lesson', e.model_id, e.dimension, e.content_hash, e.source_revision " + + $"FROM {PostgresExperienceRecordStore.Table} r " + + $"LEFT JOIN {Table} e ON e.experience_id = r.experience_id " + + $"WHERE {PostgresExperienceRecordStore.RecordScopePredicate} " + + // The search's own predicates, applied here too: a record whose vector could never be returned + // is never embedded, so its summary and lesson never leave the database for a third party. + "AND r.status = ANY(@statuses) AND r.reuse_confidence >= @min_confidence"; + + private const string ScanIdPredicate = " AND r.experience_id = ANY(@experience_ids)"; + + /// The keyset cursor. The sort key is the primary key, so this is a stable, gap-free walk. + private const string ScanCursorPredicate = " AND r.experience_id > @start_after_id"; + + private const string ScanOrderAndLimit = " ORDER BY r.experience_id LIMIT @limit"; + + /// + /// What the scope actually holds, used only when a search matched nothing, to tell "nothing is + /// similar" apart from "nothing here is comparable". Two EXISTS probes in one statement, + /// not an aggregate: each stops at the first matching row, so the healthy "nothing similar" case + /// costs two index probes rather than a scan of every in-scope embedding, and the answer cannot + /// depend on how many distinct groups happened to fit under a limit. + /// + /// The two questions are asked separately and in the right order: is there any comparable + /// population at all, and is there one for this exact model? A scope with 500 models would have + /// made a GROUP BY ... LIMIT report ModelMismatch whenever this model's group fell + /// outside the limit, even though the real cause was the width. + /// + /// + private static readonly string CompatibilityProbeSql = + $"SELECT EXISTS (SELECT 1 FROM {Table} e JOIN {PostgresExperienceRecordStore.Table} r ON r.experience_id = e.experience_id " + + $"WHERE {EmbeddingScopePredicate} AND {PostgresExperienceRecordStore.RecordScopePredicate} " + + "AND r.status = ANY(@statuses) AND r.reuse_confidence >= @min_confidence), " + + $"EXISTS (SELECT 1 FROM {Table} e JOIN {PostgresExperienceRecordStore.Table} r ON r.experience_id = e.experience_id " + + $"WHERE {EmbeddingScopePredicate} AND {PostgresExperienceRecordStore.RecordScopePredicate} " + + "AND r.status = ANY(@statuses) AND r.reuse_confidence >= @min_confidence AND e.model_id = @model_id)"; + + private static readonly IReadOnlyList NoErrors = []; + + private static readonly IReadOnlyList NoCandidates = []; + + private static readonly IReadOnlyList NoTargets = []; + + private readonly NpgsqlDataSource _dataSource; + + /// Creates an embedding index over a host-owned data source. The index never disposes it. + /// The Npgsql data source to open connections from. + /// is . + public PostgresExperienceEmbeddingIndex(NpgsqlDataSource dataSource) + { + ArgumentNullException.ThrowIfNull(dataSource); + _dataSource = dataSource; + } + + /// + public async Task WriteAsync( + AuthorizationContext authorization, + ExperienceIndexWrite write, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(authorization); + ArgumentNullException.ThrowIfNull(write); + + var errors = ExperienceRecordValidator.ValidateIndexWrite(write); + if (errors.Count > 0) + { + return new(ExperienceIndexOutcome.Invalid, 0, errors); + } + + if (!authorization.Permits(write.Scope)) + { + // Fail-closed, and before any connection opens: no statement is issued at all. + return new(ExperienceIndexOutcome.Denied, 0, NoErrors); + } + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + // One connection for the write and, if it wrote nothing, for the probe that explains why. + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + + int written; + await using (var command = new NpgsqlCommand(WriteSql, connection)) + { + var parameters = command.Parameters; + parameters.Add(new NpgsqlParameter("experience_id", write.ExperienceId)); + PostgresExperienceRecordStore.AddScopeParameters(parameters, write.Scope); + parameters.Add(new NpgsqlParameter("model_id", NpgsqlDbType.Text) { TypedValue = write.Descriptor.ModelId }); + parameters.Add(new NpgsqlParameter("dimension", write.Descriptor.Dimension)); + parameters.Add(new NpgsqlParameter("content_hash", NpgsqlDbType.Text) { TypedValue = write.Descriptor.ContentHash }); + parameters.Add(new NpgsqlParameter("source_revision", write.Descriptor.SourceRevision)); + parameters.Add(new NpgsqlParameter("embedding", NpgsqlDbType.Text) { TypedValue = ToVectorLiteral(write.Vector) }); + parameters.Add(new NpgsqlParameter("now", ToStoredTimestamp(DateTimeOffset.UtcNow))); + + written = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + if (written > 0) + { + return new(ExperienceIndexOutcome.Written, 0, NoErrors); + } + + // Nothing was written. Either the record is not in this scope at all, or its revision has + // moved past the one this write was computed from. + var current = await ProbeRevisionAsync(connection, write.Scope, write.ExperienceId, cancellationToken).ConfigureAwait(false); + return current is { } revision + ? new(ExperienceIndexOutcome.Stale, revision, NoErrors) + : new(ExperienceIndexOutcome.Missing, 0, NoErrors); + } + catch (Exception ex) when (PostgresExperienceRecordStore.IsInfrastructureFailure(ex, cancellationToken)) + { + throw PostgresExperienceRecordStore.Translate(ex, "embedding write", cancellationToken); + } + } + + /// + public async Task ScanAsync( + AuthorizationContext authorization, + ExperienceIndexScan scan, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(authorization); + ArgumentNullException.ThrowIfNull(scan); + + var errors = ExperienceRecordValidator.ValidateIndexScan(scan); + if (errors.Count > 0) + { + return new(ExperienceStoreOutcome.Invalid, NoTargets, errors); + } + + if (!authorization.Permits(scan.Scope)) + { + return new(ExperienceStoreOutcome.Denied, NoTargets, NoErrors); + } + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var sql = ScanSql + + (scan.ExperienceIds is null ? string.Empty : ScanIdPredicate) + + (scan.StartAfterId is null ? string.Empty : ScanCursorPredicate) + + ScanOrderAndLimit; + + await using var command = _dataSource.CreateCommand(sql); + var parameters = command.Parameters; + PostgresExperienceRecordStore.AddScopeParameters(parameters, scan.Scope); + parameters.Add(new NpgsqlParameter("statuses", NpgsqlDbType.Array | NpgsqlDbType.Text) + { + TypedValue = [.. scan.EligibleStatuses.Distinct().Select(status => status.ToString())], + }); + parameters.Add(new NpgsqlParameter("min_confidence", scan.MinimumConfidence)); + if (scan.ExperienceIds is not null) + { + parameters.Add(new NpgsqlParameter("experience_ids", NpgsqlDbType.Array | NpgsqlDbType.Uuid) + { + TypedValue = [.. scan.ExperienceIds.Distinct()], + }); + } + + if (scan.StartAfterId is { } startAfterId) + { + parameters.Add(new NpgsqlParameter("start_after_id", startAfterId)); + } + + parameters.Add(new NpgsqlParameter("limit", scan.Limit)); + + var targets = new List(); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + targets.Add(ReadTarget(reader)); + } + + return new( + ExperienceStoreOutcome.Found, + targets, + NoErrors, + targets.Count > 0 ? targets[^1].ExperienceId : null); + } + catch (Exception ex) when (PostgresExperienceRecordStore.IsInfrastructureFailure(ex, cancellationToken)) + { + throw PostgresExperienceRecordStore.Translate(ex, "embedding scan", cancellationToken); + } + } + + /// + public async Task SearchAsync( + AuthorizationContext authorization, + ExperienceVectorQuery query, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(authorization); + ArgumentNullException.ThrowIfNull(query); + + var errors = ExperienceRecordValidator.ValidateVectorQuery(query); + if (errors.Count > 0) + { + return new(ExperienceVectorSearchOutcome.Invalid, NoCandidates, errors); + } + + if (!authorization.Permits(query.Scope)) + { + return new(ExperienceVectorSearchOutcome.Denied, NoCandidates, NoErrors); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var dimension = query.Vector.Length; + var statuses = query.EligibleStatuses.Distinct().Select(status => status.ToString()).ToArray(); + + try + { + await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + + var candidates = new List(); + await using (var command = new NpgsqlCommand(SearchSql(dimension), connection)) + { + var parameters = command.Parameters; + PostgresExperienceRecordStore.AddScopeParameters(parameters, query.Scope); + parameters.Add(new NpgsqlParameter("statuses", NpgsqlDbType.Array | NpgsqlDbType.Text) { TypedValue = statuses }); + parameters.Add(new NpgsqlParameter("min_confidence", query.MinimumConfidence)); + parameters.Add(new NpgsqlParameter("model_id", NpgsqlDbType.Text) { TypedValue = query.ModelId }); + parameters.Add(new NpgsqlParameter("query_vector", NpgsqlDbType.Text) { TypedValue = ToVectorLiteral(query.Vector) }); + parameters.Add(new NpgsqlParameter("limit", query.Limit)); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + candidates.Add(new ExperienceCandidate( + PostgresExperienceRecordStore.ReadRecord(reader), + ReadRelevance(reader))); + } + } + + if (candidates.Count > 0) + { + return new(ExperienceVectorSearchOutcome.Found, candidates, NoErrors); + } + + // Only now -- an empty answer is the one case where "nothing similar" and "nothing + // comparable" look the same from outside, and a host must be able to tell them apart. + var mismatch = await ProbeCompatibilityAsync(connection, query, statuses, cancellationToken).ConfigureAwait(false); + return new(mismatch ?? ExperienceVectorSearchOutcome.Found, NoCandidates, NoErrors); + } + catch (Exception ex) when (PostgresExperienceRecordStore.IsInfrastructureFailure(ex, cancellationToken)) + { + throw PostgresExperienceRecordStore.Translate(ex, "vector search", cancellationToken); + } + } + + /// + /// The nearest-neighbour statement for one dimension. The dimension is written into the SQL rather + /// than parameterized because a pgvector type modifier is part of the type, not a value -- it can + /// never be a parameter. It is an the validator has already bounded, so nothing + /// caller-controlled reaches the statement text. + /// + /// Both the stored vector and the query vector are cast to vector(n) so the expression + /// matches the partial HNSW index creates. The + /// e.dimension = n predicate is what makes the cast safe: a row of another width is filtered + /// out before the distance expression is ever evaluated on it. + /// + /// + /// + /// The exact statement the search issues, for a test to hand to EXPLAIN. Asserting the + /// planner's choice is only meaningful against the real statement: an approximation would prove the + /// index matches something this adapter never runs. + /// + internal static string SearchSqlForTesting(int dimension) => SearchSql(dimension); + + private static string SearchSql(int dimension) + { + var width = dimension.ToString(CultureInfo.InvariantCulture); + return $"SELECT {RecordColumns}, " + + $"(e.embedding::vector({width}) <=> CAST(@query_vector AS vector({width}))) AS {DistanceColumn} " + + $"FROM {Table} e " + + $"JOIN {PostgresExperienceRecordStore.Table} r ON r.experience_id = e.experience_id " + + // Both sides of the join carry the scope. The r-side is the authoritative one; the e-side is + // what makes ix_experience_embeddings_scope_model usable (see EmbeddingScopePredicate). + $"WHERE {EmbeddingScopePredicate} AND {PostgresExperienceRecordStore.RecordScopePredicate} " + + "AND r.status = ANY(@statuses) " + + "AND r.reuse_confidence >= @min_confidence " + + "AND e.model_id = @model_id " + + $"AND e.dimension = {width} " + + // The distance expression is repeated rather than referenced by its alias, and it is the + // only sort key: an index scan can supply this ordering directly, while a tie-break on + // r.experience_id would force the whole join to be sorted and the HNSW index never to be + // used. Exact distance ties are broken arbitrarily here as a result, which costs nothing -- + // Core re-sorts every candidate by score and breaks its own ties on ExperienceId, so the + // order a caller sees is still total and stable. + $"ORDER BY (e.embedding::vector({width}) <=> CAST(@query_vector AS vector({width}))) LIMIT @limit"; + } + + private static async Task ProbeRevisionAsync( + NpgsqlConnection connection, + Scope scope, + Guid experienceId, + CancellationToken cancellationToken) + { + await using var command = new NpgsqlCommand(ProbeRevisionSql, connection); + command.Parameters.Add(new NpgsqlParameter("experience_id", experienceId)); + PostgresExperienceRecordStore.AddScopeParameters(command.Parameters, scope); + + var value = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + return value is long revision ? revision : null; + } + + /// + /// Decides why an empty search was empty. means the scope simply held no + /// comparable-or-otherwise embedding to match, which is an answer rather than a fallback. + /// + private static async Task ProbeCompatibilityAsync( + NpgsqlConnection connection, + ExperienceVectorQuery query, + string[] statuses, + CancellationToken cancellationToken) + { + await using var command = new NpgsqlCommand(CompatibilityProbeSql, connection); + var parameters = command.Parameters; + PostgresExperienceRecordStore.AddScopeParameters(parameters, query.Scope); + parameters.Add(new NpgsqlParameter("statuses", NpgsqlDbType.Array | NpgsqlDbType.Text) { TypedValue = statuses }); + parameters.Add(new NpgsqlParameter("min_confidence", query.MinimumConfidence)); + + parameters.Add(new NpgsqlParameter("model_id", NpgsqlDbType.Text) { TypedValue = query.ModelId }); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return null; + } + + var sawAnything = reader.GetBoolean(0); + var sawThisModel = reader.GetBoolean(1); + + if (!sawAnything) + { + return null; + } + + // The search already filtered on both model and dimension and found nothing, so whichever of + // the two the stored rows disagree on is the one to report. The dimension is named only when + // the model itself matched, so "wrong model" is never reported as "wrong width". + return sawThisModel ? ExperienceVectorSearchOutcome.DimensionMismatch : ExperienceVectorSearchOutcome.ModelMismatch; + } + + private static ExperienceIndexTarget ReadTarget(DbDataReader reader) + { + try + { + var stored = reader.IsDBNull(5) + ? null + : new ExperienceEmbeddingDescriptor( + reader.GetString(5), + reader.GetInt32(6), + reader.GetString(7), + reader.GetInt64(8)); + + return new ExperienceIndexTarget( + reader.GetGuid(0), + reader.GetInt64(1), + ExperienceRetrievalSummary.For( + reader.GetString(2), + reader.IsDBNull(3) ? null : reader.GetString(3), + reader.IsDBNull(4) ? null : reader.GetString(4)), + stored); + } + catch (Exception ex) when (ex is not (ExperienceStoreException or OperationCanceledException or NpgsqlException)) + { + throw new ExperienceStoreException("Stored Experience Record could not be decoded.", ex); + } + } + + /// + /// Turns cosine distance into the normalized [0, 1] relevance every + /// carries: 1 - distance / 2, since <=> + /// lies in [0, 2]. A NaN -- which pgvector returns for a zero-magnitude vector -- scores 0 rather + /// than travelling into Core's ranking arithmetic and poisoning every comparison against it. + /// + private static double ReadRelevance(DbDataReader reader) + { + double distance; + try + { + distance = reader.GetDouble(reader.GetOrdinal(DistanceColumn)); + } + catch (Exception ex) when (ex is not (ExperienceStoreException or OperationCanceledException or NpgsqlException)) + { + throw new ExperienceStoreException("Stored Experience Record could not be decoded.", ex); + } + + if (double.IsNaN(distance)) + { + return 0d; + } + + return Math.Clamp(1d - (distance / 2d), 0d, 1d); + } + + /// + /// Formats a vector as pgvector's own text literal, sent as text and cast in SQL. Going through + /// the literal rather than a mapped CLR type is deliberate: it means this adapter works on any + /// the host built, whether or not UseVector() was called on + /// its builder, so a host cannot misconfigure the two halves of the schema against each other. + /// owns the formatting, which is invariant-culture by construction. + /// + private static string ToVectorLiteral(ReadOnlyMemory vector) => new Vector(vector).ToString(); + + /// + /// Truncates to whole microseconds in UTC, which is the precision PostgreSQL's timestamptz + /// keeps -- matching , so a timestamp read back from + /// either table compares equal to the value that was sent. + /// + private static DateTimeOffset ToStoredTimestamp(DateTimeOffset value) + { + var utcTicks = value.UtcTicks; + return new DateTimeOffset(utcTicks - (utcTicks % 10), TimeSpan.Zero); + } +} diff --git a/src/AgentExperience.Storage.Postgres.Vectors/README.md b/src/AgentExperience.Storage.Postgres.Vectors/README.md new file mode 100644 index 0000000..b959c5c --- /dev/null +++ b/src/AgentExperience.Storage.Postgres.Vectors/README.md @@ -0,0 +1,235 @@ +# AgentExperience.Storage.Postgres.Vectors + +The pgvector half of AgentExperience.NET's PostgreSQL adapter: one derived embedding per Experience Record, a +conditional write that can never overwrite newer state or resurrect a deleted record, an explicit scoped re-index, +and a scoped nearest-neighbour search that applies exactly the same eligibility filters as the text channel. + +It implements `IExperienceEmbeddingIndex` from `AgentExperience.Abstractions` and is consumed by +`AgentExperience.Core`'s `ExperienceIndexingService` and `ExperienceRetrievalService`. + +> **Status: early development.** Nothing is published to NuGet yet, and APIs may change. + +## Why this is a separate package + +`AgentExperience.Storage.Postgres` deliberately takes **no** vector or model-provider dependency — its dependency +boundary test pins its package set exactly and forbids `Pgvector`, `VectorData`, and `Microsoft.Extensions.AI`. A +host that only wants canonical storage and text retrieval should not pull those in. So the vector work lives here, +with its own exact pins, and references the store package for the column list, scope predicate, row decoder, and +failure translation the two channels must share. + +| Package | Version | Why | +| --- | --- | --- | +| `Npgsql` | `[10.0.3]` | Every statement, on the host's own data source | +| `Pgvector` | `[0.3.2]` | The `vector` literal format | +| `Microsoft.Extensions.AI.Abstractions` | `[10.9.0]` | Adapting an `IEmbeddingGenerator` to this library's own port | +| `Microsoft.Extensions.DependencyInjection.Abstractions` | `[10.0.11]` | This package's own `Add…` registrations | + +Every version was verified by Story 1.7's executable PostgreSQL/pgvector compatibility proof before this package +was written. + +## Usage + +```csharp +using AgentExperience.Core.DependencyInjection; +using AgentExperience.Storage.Postgres.DependencyInjection; +using AgentExperience.Storage.Postgres.Vectors.DependencyInjection; + +services.AddSingleton(NpgsqlDataSource.Create(connectionString)); +services.AddAgentExperiencePostgresStore(); // IExperienceRecordStore +services.AddAgentExperiencePostgresCandidateSource(); // IExperienceCandidateSource (text) +services.AddAgentExperiencePostgresEmbeddingIndex(); // IExperienceEmbeddingIndex (vectors) +services.AddAgentExperienceEmbeddingGenerator(); // over a registered IEmbeddingGenerator> +services.AddAgentExperienceCore(sanitizationOptions, captureLimits); +services.AddAgentExperienceIndexing(); // ExperienceIndexingService, and finalization's post-commit hook +services.AddAgentExperienceRetrieval(); // hybrid, because both halves above are registered +``` + +Apply the schema once at startup, in two calls: + +```csharp +await ExperienceSchemaMigrator.MigrateAsync(dataSource, cancellationToken); // 0001-0003, the base schema +await ExperienceVectorSchemaMigrator.MigrateAsync(dataSource, cancellationToken); // 0004, this package's schema +``` + +They are separate on purpose. `0004` begins with `CREATE EXTENSION vector`, and pgvector is **not** a trusted +extension, so that statement ordinarily needs a superuser (on a managed service, whichever role that provider +designates). A text-only deployment never calls the second line and therefore never needs that privilege. If your +operators install the extension out of band, this call runs fine as an ordinary role — `CREATE EXTENSION IF NOT +EXISTS` is a no-op once it exists. Run the base migration first: `0004` has a foreign key to `experience_records`. + +Both migrators share the `agent_experience.schema_versions` journal and the same advisory lock, so they serialize +against each other and against another host, and neither can claim the other's journal entries. + +**No `UseVector()` needed.** This adapter sends vectors as pgvector's own text literal and casts them in SQL, so it +works on whatever `NpgsqlDataSource` the host built. You can still call `UseVector()` for your own queries. + +### Bringing your own generator + +`IExperienceEmbeddingGenerator` is three members — `ModelId`, `Dimension`, and `GenerateAsync(string, …)` — and +speaks domain types only. `AiExperienceEmbeddingGenerator` adapts a `Microsoft.Extensions.AI` +`IEmbeddingGenerator>` to it, taking the model ID and dimension from the generator's +`EmbeddingGeneratorMetadata` unless you pass your own. Both are read **once**, at construction: the content hash +covers the model ID, so "the same text under the same model" has to be recognizable before any provider call. A +generator that reports neither fails at wiring time rather than mid-query. + +Tests, and any deployment that wants reproducibility, can implement the port directly with a deterministic +function — that is exactly what this package's own integration tests do, which is why they need no model +credentials. + +## What is stored, and what is not + +Only the **sanitized retrieval summary** is ever sent to a provider: the task ID, the sanitized task summary, and +the reflection's lesson — the same three fields `0003` indexes for text. Attempts, tool calls, evidence, +provenance, and environment metadata never leave the database. + +The same *fields*, but not necessarily the same *length*: the embedded summary is capped at 8,192 characters +(`ExperienceRetrievalSummary.MaxLength`) while `0003` analyzes the concatenation up to 100,000. A record whose +summary and lesson together run past 8 KB is matched on more of its text by words than by meaning. The cap is +applied before the content hash is taken, so the hashed text and the text the provider sees are always identical, +and it never splits a surrogate pair. + +A record is only embedded at all when a search could return it: the scan applies the same status filter and +confidence floor the search applies, so a `Quarantined`, `Revoked`, `Superseded`, or `Candidate` record's summary +and lesson never reach a provider. + +| Column | What it is | +| --- | --- | +| `experience_id` | Primary key, `REFERENCES experience_records … ON DELETE CASCADE` | +| `tenant_id` … `user_id` | The record's scope, **copied from the record row** inside the write, never from caller input | +| `model_id`, `dimension` | What decides comparability. A query vector is only ever compared with vectors from the same model at the same width | +| `content_hash` | SHA-256 over the model ID and the normalized summary, so an unchanged record can be skipped without calling a provider | +| `source_revision` | The record revision the summary was read at — what makes every write conditional | +| `embedding` | An **unconstrained** `vector`. The dimension belongs to whichever model a host configured, and `CHECK (vector_dims(embedding) = dimension)` keeps the two from ever disagreeing | +| `created_at`, `updated_at` | UTC, truncated to whole microseconds like the rest of the schema | + +None of this takes part in a lifecycle decision. Status, revision, and reuse confidence live on the record and are +never read from or written to this table. + +## Writes are conditional, in SQL + +A write is an `INSERT … SELECT` whose source is the canonical record row itself, matched on the exact scope **and** +on the revision the write names: + +| Situation | Outcome | What happened | +| --- | --- | --- | +| Record present at exactly that revision | `Written` | The vector replaces any previous one for that record | +| Record moved to a newer revision | `Stale` | Nothing written; the result carries the record's current revision | +| Record no longer in this scope | `Missing` | Nothing written, **no row created** — an in-flight write cannot resurrect a deleted record | +| Scope outside the authorization | `Denied` | No statement issued at all | +| Malformed request | `Invalid` | No statement issued; every field path is reported | + +An `ON CONFLICT` guard additionally stops an older in-flight write from overwriting a vector already stored from a +newer revision, and the foreign key's cascade means an embedding can never outlive its record. + +The vector write is **never** inside the canonical transaction. It runs on its own connection, after the record's +lifecycle commit has landed. That is the whole point: embeddings are derived data, and the canonical write must not +depend on a provider being up. + +## Re-indexing + +`ScanAsync` lists, for one scope (optionally narrowed to specific IDs, always bounded), each record's current +revision, its normalized summary, and the descriptor of whatever vector is already stored for it — all from one +snapshot. It applies the **same status filter and confidence floor the search applies**: a record whose vector could +never be returned is never listed, so its task summary and reflection lesson never leave the database for a +third-party provider. + +It is a **keyset walk**, not a repeated first page. Targets come back in ascending `ExperienceId` order and the +result carries `LastExaminedId`; pass that as the next scan's `StartAfterId` (or the next pass's) and a scope larger +than one page is walked to the end. A `null` `LastExaminedId` means the scope is exhausted. + +`ExperienceIndexingService` then decides per record: + +- same model **and** same content hash → **skipped**: no provider call, no write; +- anything else → re-embed and write conditionally. + +So re-running a pass over unchanged records costs one read and nothing else, and running it twice over a changed +record rewrites it exactly once. Re-indexing is always explicit and always scoped; nothing here ever runs on its own. + +## Searching + +```sql +SELECT , (e.embedding::vector(n) <=> CAST(@query_vector AS vector(n))) AS distance +FROM agent_experience.experience_embeddings e +JOIN agent_experience.experience_records r ON r.experience_id = e.experience_id +WHERE AND r.status = ANY(@statuses) AND r.reuse_confidence >= @min_confidence + AND e.model_id = @model_id AND e.dimension = n +ORDER BY distance, r.experience_id LIMIT @limit +``` + +Scope, status, and the confidence floor are the same predicates the text channel applies — both channels filter +identically, in the database, before anything is ranked. Comparability is a predicate too: a vector from another +model or of another width is excluded by the query, so no incompatible comparison is ever attempted. + +The scope predicate is applied to **both** sides of the join. On `r` it is authoritative; on `e` it is redundant +(the embedding's scope columns are copied from the record row inside the write) and exists so +`ix_experience_embeddings_scope_model` can actually serve the query — a btree on +`(tenant_id, application_id, project_id, model_id, dimension)` is useless when the only predicates on the embeddings +table are its trailing two columns. + +The distance expression is the **only** sort key. A tie-break on `experience_id` would force the whole join to be +sorted and the HNSW index never to be used, so exact distance ties are broken arbitrarily here — which costs +nothing, because Core re-sorts every candidate by score and breaks its own ties on `ExperienceId`. + +Relevance is `1 - distance / 2` (cosine distance lies in `[0, 2]`), so it is already in `[0, 1]`, with 1 for an +exact direction match. Like the text channel's relevance it is a within-search measure. + +When a search matches nothing, and only then, one extra statement asks what the scope actually holds, so the caller +can tell "nothing is similar" from "nothing here is comparable". It is two `EXISTS` probes — is there any comparable +population at all, and is there one for this exact model — so the healthy empty case costs two index probes rather +than an aggregate over every in-scope row, and the answer cannot depend on how many distinct models happened to fit +under a limit: + +| Result | Meaning | +| --- | --- | +| `Found` (possibly empty) | The search ran over comparable vectors | +| `ModelMismatch` | The scope holds embeddings, all from other models | +| `DimensionMismatch` | The scope holds this model's embeddings, all at another width | + +Core turns either mismatch into an explicit text-only retrieval result carrying the reason. + +## The HNSW index is created out of band + +`0004` cannot create it: an HNSW index needs a dimension, and a shipped migration does not know which model a host +runs. So it is an explicit call: + +```csharp +await ExperienceVectorIndexMaintenance.EnsureHnswIndexAsync(dataSource, dimension: 1536, cancellationToken); +``` + +which creates, idempotently, + +```sql +CREATE INDEX IF NOT EXISTS ix_experience_embeddings_hnsw_1536 + ON agent_experience.experience_embeddings + USING hnsw ((embedding::vector(1536)) vector_cosine_ops) + WHERE dimension = 1536; +``` + +The index is over an *expression* and is *partial* on the dimension — that predicate is what makes the cast safe on +a table that may hold several widths at once, and it lets several dimensions coexist, each with its own index. The +search's own `ORDER BY` uses exactly the same expression, so changing either side alone silently stops the index +being used. + +It is **optional**: every search is correct without it (pgvector falls back to an exact scan, which is what a small +deployment wants anyway). It changes latency, and it makes search *approximate* — HNSW may miss a true nearest +neighbour. Building it over many rows takes minutes and locks the table against writes, so run it from a +maintenance path, never from request handling. `DropHnswIndexAsync` removes it again. + +## Results and failures + +Expected conditions are typed results, exactly as in `AgentExperience.Storage.Postgres`. Infrastructure failures +throw `ExperienceStoreException` with the driver's exception inside; caller cancellation surfaces as an unwrapped +`OperationCanceledException`. Core catches all of it and reports it as a retryable indexing result or an explicit +text-only retrieval — nothing here can fail a canonical write or a finalization. + +## Testing + +The integration tests run against an ephemeral `pgvector/pgvector:pg16` container through Testcontainers (Docker +required; set `TESTCONTAINERS_RYUK_DISABLED=true` if Ryuk fails under your local Docker setup) and embed through a +deterministic in-test generator, so **no model credentials are ever needed**. One of them runs `EXPLAIN` over the +adapter's real search statement with `enable_seqscan` off and asserts the HNSW index appears in the plan, so "the +index exists" and "the search uses it" stay the same claim. + +## License + +[Apache-2.0](../../LICENSE) diff --git a/src/AgentExperience.Storage.Postgres.Vectors/packages.lock.json b/src/AgentExperience.Storage.Postgres.Vectors/packages.lock.json new file mode 100644 index 0000000..0c4feb9 --- /dev/null +++ b/src/AgentExperience.Storage.Postgres.Vectors/packages.lock.json @@ -0,0 +1,75 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Microsoft.Extensions.AI.Abstractions": { + "type": "Direct", + "requested": "[10.9.0, 10.9.0]", + "resolved": "10.9.0", + "contentHash": "//nASHMCJVxnYfE/WSzfaLOao6/q816kPpgB9rxU0gfSmAny1u3rfQT0D4xAmcIo4yQqJs7rAeBB+M/dIMdZYA==" + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Direct", + "requested": "[10.0.11, 10.0.11]", + "resolved": "10.0.11", + "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" + }, + "Npgsql": { + "type": "Direct", + "requested": "[10.0.3, 10.0.3]", + "resolved": "10.0.3", + "contentHash": "7nb5YzXuvWWJxB0J8DiyL3we+X4FOctZrt0fIBnucOIaIevFEEwGQVZKtiu9olXdlNAK1eNgqSral6r/jlhI4w==", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "10.0.0" + } + }, + "Pgvector": { + "type": "Direct", + "requested": "[0.3.2, 0.3.2]", + "resolved": "0.3.2", + "contentHash": "n7M5LuNejHUmtWky3zCbNO+tP1Gnjiuv9Qtu4LyvB1602dD8RiBxxCQp9jEjM0ZFDxAZF1oOWkNIkXw46KT00Q==", + "dependencies": { + "Npgsql": "8.0.5" + } + }, + "dbup-core": { + "type": "Transitive", + "resolved": "6.1.1", + "contentHash": "kgpuyJVEFJHoIj/slnc994Go88aoeZqNDfGHDBr4sh7CsEWwJhOTCt/FJqO4ziUImL5L0NEY0kxxOiNgPKI2Fw==", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "8.0.0" + } + }, + "dbup-postgresql": { + "type": "Transitive", + "resolved": "7.0.1", + "contentHash": "mRnmENWWPuuMZ538gOd1mZnzucx6FQk0anmw3EABjGfcbp24FDb9QdGepYrDiaM8K9s5/gd49+5cmBOlniH/lg==", + "dependencies": { + "Npgsql": "10.0.1", + "dbup-core": "6.1.1" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "10.0.0", + "contentHash": "FU/IfjDfwaMuKr414SSQNTIti/69bHEMb+QKrskRb26oVqpx3lNFXMjs/RC9ZUuhBhcwDM2BwOgoMw+PZ+beqQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0" + } + }, + "agentexperience.abstractions": { + "type": "Project" + }, + "agentexperience.storage.postgres": { + "type": "Project", + "dependencies": { + "AgentExperience.Abstractions": "[1.0.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]", + "Npgsql": "[10.0.3, 10.0.3]", + "dbup-core": "[6.1.1, 6.1.1]", + "dbup-postgresql": "[7.0.1, 7.0.1]" + } + } + } + } +} \ No newline at end of file diff --git a/src/AgentExperience.Storage.Postgres/AgentExperience.Storage.Postgres.csproj b/src/AgentExperience.Storage.Postgres/AgentExperience.Storage.Postgres.csproj index f024a5d..1bbc771 100644 --- a/src/AgentExperience.Storage.Postgres/AgentExperience.Storage.Postgres.csproj +++ b/src/AgentExperience.Storage.Postgres/AgentExperience.Storage.Postgres.csproj @@ -15,6 +15,9 @@ in lockstep, to the dbup-core version the new dbup-postgresql depends on; DependencyBoundaryTests asserts this exact set of three PackageReferences. --> + + @@ -24,10 +27,16 @@ + + + diff --git a/src/AgentExperience.Storage.Postgres/DependencyInjection/AgentExperiencePostgresServiceCollectionExtensions.cs b/src/AgentExperience.Storage.Postgres/DependencyInjection/AgentExperiencePostgresServiceCollectionExtensions.cs new file mode 100644 index 0000000..4618a8e --- /dev/null +++ b/src/AgentExperience.Storage.Postgres/DependencyInjection/AgentExperiencePostgresServiceCollectionExtensions.cs @@ -0,0 +1,101 @@ +using AgentExperience.Abstractions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Npgsql; + +namespace AgentExperience.Storage.Postgres.DependencyInjection; + +/// +/// Registers the PostgreSQL Experience Record store in a . The +/// adapter owns its own registration, exactly as Core owns AddAgentExperienceCore, so a host +/// wires the two together without either package knowing the other's concrete types. +/// +public static class AgentExperiencePostgresServiceCollectionExtensions +{ + /// + /// Registers as the singleton + /// , over an resolved from the + /// container. + /// + /// + /// The host owns the data source's lifetime and the store never disposes it. The schema is not + /// applied here: call + /// once + /// at startup. + /// + /// The service collection to add to. + /// , for chaining. + /// is . + public static IServiceCollection AddAgentExperiencePostgresStore(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(provider => + new PostgresExperienceRecordStore(provider.GetRequiredService())); + + return services; + } + + /// + /// Registers as the singleton + /// over , for a host that keeps + /// its data source outside the container. + /// + /// The service collection to add to. + /// The host-owned data source the store opens connections from. Never disposed by the store. + /// , for chaining. + /// Any argument is . + public static IServiceCollection AddAgentExperiencePostgresStore(this IServiceCollection services, NpgsqlDataSource dataSource) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(dataSource); + + services.TryAddSingleton(new PostgresExperienceRecordStore(dataSource)); + + return services; + } + + /// + /// Registers as the singleton + /// , over an resolved from + /// the container, so Core's retrieval service has something to search. + /// + /// + /// Registered separately from the store: the two are independent ports, and a host that only + /// writes experience does not need the search index. The schema is not applied here -- the search + /// column and its index live in 0003_add_experience_search.sql, applied by + /// at + /// startup like the rest of the schema. + /// + /// The service collection to add to. + /// , for chaining. + /// is . + public static IServiceCollection AddAgentExperiencePostgresCandidateSource(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(provider => + new PostgresExperienceCandidateSource(provider.GetRequiredService())); + + return services; + } + + /// + /// Registers as the singleton + /// over , for a host that + /// keeps its data source outside the container. + /// + /// The service collection to add to. + /// The host-owned data source the search opens connections from. Never disposed by the source. + /// , for chaining. + /// Any argument is . + public static IServiceCollection AddAgentExperiencePostgresCandidateSource(this IServiceCollection services, NpgsqlDataSource dataSource) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(dataSource); + + services.TryAddSingleton(new PostgresExperienceCandidateSource(dataSource)); + + return services; + } +} diff --git a/src/AgentExperience.Storage.Postgres/ExperienceRecordValidator.cs b/src/AgentExperience.Storage.Postgres/ExperienceRecordValidator.cs index 184ecb7..4f6b5ef 100644 --- a/src/AgentExperience.Storage.Postgres/ExperienceRecordValidator.cs +++ b/src/AgentExperience.Storage.Postgres/ExperienceRecordValidator.cs @@ -150,6 +150,240 @@ public static IReadOnlyList ValidateQuery(ExperienceRecord return errors; } + /// + /// Validates a candidate search: the scope, the task text to match, the caller's eligible status + /// set, the confidence floor, and the bound on how many candidates may come back. An empty status + /// set is rejected rather than widened to "every status", so a caller can never accidentally ask + /// for records it considers ineligible. + /// + public static IReadOnlyList ValidateCandidateQuery(ExperienceCandidateQuery query) + { + var errors = new List(); + ValidateScope(query.Scope, "Scope", errors); + RequireNotBlank(query.TaskText, "TaskText", errors); + + if (query.TaskText is { Length: > ExperienceCandidateQuery.MaxTaskTextLength }) + { + errors.Add(new("TaskText", $"must be at most {ExperienceCandidateQuery.MaxTaskTextLength} characters.")); + } + + if (query.EligibleStatuses is null) + { + errors.Add(new("EligibleStatuses", Required)); + } + else if (query.EligibleStatuses.Count == 0) + { + errors.Add(new("EligibleStatuses", "must contain at least one status.")); + } + else + { + for (var i = 0; i < query.EligibleStatuses.Count; i++) + { + RequireDefined(query.EligibleStatuses[i], $"EligibleStatuses[{i}]", errors); + } + } + + RequireUnitInterval(query.MinimumConfidence, "MinimumConfidence", errors); + + if (query.Limit is < ExperienceCandidateQuery.MinLimit or > ExperienceCandidateQuery.MaxLimit) + { + errors.Add(new("Limit", $"must be between {ExperienceCandidateQuery.MinLimit} and {ExperienceCandidateQuery.MaxLimit}.")); + } + + return errors; + } + + /// + /// Validates a conditional index write: the scope the record must lie in, the record ID, the + /// descriptor that makes the write conditional, and the vector itself. The vector's length is + /// checked against the descriptor's dimension here, so the two can never be stored disagreeing. + /// + public static IReadOnlyList ValidateIndexWrite(ExperienceIndexWrite write) + { + var errors = new List(); + ValidateScope(write.Scope, "Scope", errors); + + if (write.ExperienceId == Guid.Empty) + { + errors.Add(new("ExperienceId", "must not be an empty GUID.")); + } + + if (write.Descriptor is null) + { + errors.Add(new("Descriptor", Required)); + return errors; + } + + ValidateDescriptor(write.Descriptor, "Descriptor", errors); + ValidateVector(write.Vector, "Vector", errors); + + if (write.Descriptor.Dimension > 0 && write.Vector.Length != write.Descriptor.Dimension) + { + errors.Add(new("Vector", "must hold exactly Descriptor.Dimension components.")); + } + + return errors; + } + + /// Validates a scoped index scan: the scope, the model whose descriptors to report, the optional ID filter, and the bound. + public static IReadOnlyList ValidateIndexScan(ExperienceIndexScan scan) + { + var errors = new List(); + ValidateScope(scan.Scope, "Scope", errors); + RequireNotBlank(scan.ModelId, "ModelId", errors); + + if (scan.ModelId is { Length: > ExperienceEmbeddingDescriptor.MaxModelIdLength }) + { + errors.Add(new("ModelId", $"must be at most {ExperienceEmbeddingDescriptor.MaxModelIdLength} characters.")); + } + + if (scan.EligibleStatuses is null) + { + errors.Add(new("EligibleStatuses", Required)); + } + else if (scan.EligibleStatuses.Count == 0) + { + errors.Add(new("EligibleStatuses", "must contain at least one status.")); + } + else + { + for (var i = 0; i < scan.EligibleStatuses.Count; i++) + { + RequireDefined(scan.EligibleStatuses[i], $"EligibleStatuses[{i}]", errors); + } + } + + RequireUnitInterval(scan.MinimumConfidence, "MinimumConfidence", errors); + + if (scan.StartAfterId == Guid.Empty) + { + errors.Add(new("StartAfterId", "must be null or a non-empty GUID.")); + } + + if (scan.ExperienceIds is not null) + { + if (scan.ExperienceIds.Count == 0) + { + errors.Add(new("ExperienceIds", "must contain at least one identifier when supplied.")); + } + else if (scan.ExperienceIds.Count > ExperienceIndexScan.MaxLimit) + { + errors.Add(new("ExperienceIds", $"must hold at most {ExperienceIndexScan.MaxLimit} identifiers.")); + } + else + { + for (var i = 0; i < scan.ExperienceIds.Count; i++) + { + if (scan.ExperienceIds[i] == Guid.Empty) + { + errors.Add(new($"ExperienceIds[{i}]", "must not be an empty GUID.")); + } + } + } + } + + if (scan.Limit is < ExperienceIndexScan.MinLimit or > ExperienceIndexScan.MaxLimit) + { + errors.Add(new("Limit", $"must be between {ExperienceIndexScan.MinLimit} and {ExperienceIndexScan.MaxLimit}.")); + } + + return errors; + } + + /// + /// Validates a scoped vector search. The field paths and the bounds deliberately mirror + /// , so both retrieval channels reject the same requests for + /// the same reasons. + /// + public static IReadOnlyList ValidateVectorQuery(ExperienceVectorQuery query) + { + var errors = new List(); + ValidateScope(query.Scope, "Scope", errors); + RequireNotBlank(query.ModelId, "ModelId", errors); + + if (query.ModelId is { Length: > ExperienceEmbeddingDescriptor.MaxModelIdLength }) + { + errors.Add(new("ModelId", $"must be at most {ExperienceEmbeddingDescriptor.MaxModelIdLength} characters.")); + } + + ValidateVector(query.Vector, "Vector", errors); + + if (query.EligibleStatuses is null) + { + errors.Add(new("EligibleStatuses", Required)); + } + else if (query.EligibleStatuses.Count == 0) + { + errors.Add(new("EligibleStatuses", "must contain at least one status.")); + } + else + { + for (var i = 0; i < query.EligibleStatuses.Count; i++) + { + RequireDefined(query.EligibleStatuses[i], $"EligibleStatuses[{i}]", errors); + } + } + + RequireUnitInterval(query.MinimumConfidence, "MinimumConfidence", errors); + + if (query.Limit is < ExperienceCandidateQuery.MinLimit or > ExperienceCandidateQuery.MaxLimit) + { + errors.Add(new("Limit", $"must be between {ExperienceCandidateQuery.MinLimit} and {ExperienceCandidateQuery.MaxLimit}.")); + } + + return errors; + } + + private static void ValidateDescriptor(ExperienceEmbeddingDescriptor descriptor, string path, List errors) + { + RequireNotBlank(descriptor.ModelId, $"{path}.ModelId", errors); + if (descriptor.ModelId is { Length: > ExperienceEmbeddingDescriptor.MaxModelIdLength }) + { + errors.Add(new($"{path}.ModelId", $"must be at most {ExperienceEmbeddingDescriptor.MaxModelIdLength} characters.")); + } + + if (descriptor.Dimension is < 1 or > ExperienceEmbeddingDescriptor.MaxDimension) + { + errors.Add(new($"{path}.Dimension", $"must be between 1 and {ExperienceEmbeddingDescriptor.MaxDimension}.")); + } + + RequireNotBlank(descriptor.ContentHash, $"{path}.ContentHash", errors); + + if (descriptor.SourceRevision < 0) + { + errors.Add(new($"{path}.SourceRevision", "must not be negative.")); + } + } + + /// + /// A vector must be non-empty, within the dimension ceiling, and entirely finite. A NaN or an + /// infinity would be accepted by pgvector's input parser in some forms and then poison every + /// distance computed against it, so it is rejected before any database access. + /// + private static void ValidateVector(ReadOnlyMemory vector, string path, List errors) + { + if (vector.Length == 0) + { + errors.Add(new(path, "must hold at least one component.")); + return; + } + + if (vector.Length > ExperienceEmbeddingDescriptor.MaxDimension) + { + errors.Add(new(path, $"must hold at most {ExperienceEmbeddingDescriptor.MaxDimension} components.")); + return; + } + + foreach (var component in vector.Span) + { + if (!float.IsFinite(component)) + { + errors.Add(new(path, "must hold only finite components.")); + return; + } + } + } + private static void ValidateScope(Scope? scope, string path, List errors) { if (scope is null) diff --git a/src/AgentExperience.Storage.Postgres/ExperienceSchemaMigrator.cs b/src/AgentExperience.Storage.Postgres/ExperienceSchemaMigrator.cs index 89b2144..ed2080d 100644 --- a/src/AgentExperience.Storage.Postgres/ExperienceSchemaMigrator.cs +++ b/src/AgentExperience.Storage.Postgres/ExperienceSchemaMigrator.cs @@ -27,7 +27,10 @@ namespace AgentExperience.Storage.Postgres; /// CREATE on the database (to create the agent_experience schema) and on that schema (to /// create its tables). The store itself only needs SELECT, INSERT, and UPDATE on /// agent_experience.experience_records and SELECT and INSERT on -/// agent_experience.lifecycle_events. +/// agent_experience.lifecycle_events. No script here needs a superuser, and none creates an +/// extension: this package's schema is text-only, and the derived embedding schema -- which does need +/// CREATE EXTENSION vector -- is applied separately by +/// AgentExperience.Storage.Postgres.Vectors's own migrator, only by hosts that enable it. /// /// /// The wait for the advisory lock is deliberately unbounded and ends only with the caller's token. Each @@ -84,8 +87,12 @@ public static Task MigrateAsync( } /// - /// The script-selection seam: same run, but over an arbitrary assembly and resource prefix. Test - /// only, so a failing script never has to ship in the package. + /// The script-selection seam: same run -- same journal table, same advisory lock -- but over an + /// arbitrary assembly and resource prefix. It is what lets + /// AgentExperience.Storage.Postgres.Vectors apply its own schema without duplicating the + /// journalling and locking, and what lets a test drive a failing script that never ships in the + /// package. Journal entries record DbUp's script name, which is the full resource name, so two + /// prefixes can never claim each other's entries. /// internal static async Task MigrateAsync( NpgsqlDataSource dataSource, diff --git a/src/AgentExperience.Storage.Postgres/Migrations/0003_add_experience_search.sql b/src/AgentExperience.Storage.Postgres/Migrations/0003_add_experience_search.sql new file mode 100644 index 0000000..4a55e60 --- /dev/null +++ b/src/AgentExperience.Storage.Postgres/Migrations/0003_add_experience_search.sql @@ -0,0 +1,64 @@ +-- AgentExperience.NET: full-text search over Experience Records, for text retrieval (Story 2.2). +-- Applied by ExperienceSchemaMigrator and recorded in agent_experience.schema_versions. +-- Every statement is IF NOT EXISTS on purpose, matching 0001 and 0002, so a database whose schema was +-- applied by hand can still be journaled. Do not edit this script once it has been journaled anywhere; +-- add the next-numbered script instead. (This script is still unreleased and has only ever been applied +-- to throwaway test databases, so the length bound below was added in place during review; once this +-- branch ships, the append-only rule applies to it as it does to 0001.) +-- +-- search_vector is a GENERATED ... STORED column, not a trigger and not a column the store writes: it is +-- derived from columns and payload fields that already exist, so it can never disagree with the record it +-- indexes, and no write path has to remember to maintain it. That also means the store's INSERT and its +-- lifecycle UPDATE are unchanged -- PostgreSQL recomputes the vector itself. +-- +-- The indexed text is deliberately narrow: the task identifier, the sanitized task summary, and the +-- reflection's lesson. Those are the fields that say what a record is *about*. Attempts, tool calls, +-- evidence, and environment metadata are not indexed: they are operational detail, they would flood the +-- vector with identifiers and stack-trace-like fragments, and matching on them would make retrieval +-- recall incidental strings rather than applicable experience. +-- +-- to_tsvector's two-argument form is used with a literal configuration ('english'), which is IMMUTABLE and +-- therefore legal in a generated column; the one-argument form depends on default_text_search_config and is +-- only STABLE. Changing the configuration later means a new script that rebuilds the column, because every +-- already-indexed row would otherwise keep the old analysis. +-- +-- The concatenated text is bounded with left(): a tsvector may not exceed 1 MB, and a record with a very +-- long task summary or lesson would otherwise make to_tsvector raise -- which, in a generated column, is +-- not a search failure but a failed INSERT (and a failed migration on a table that already holds such a +-- row). 100k characters is far more than any realistic summary and analyzes to well under the ceiling, so +-- the bound only ever truncates text that would have broken the write. + +ALTER TABLE agent_experience.experience_records + ADD COLUMN IF NOT EXISTS search_vector tsvector + GENERATED ALWAYS AS ( + to_tsvector( + 'english', + left( + coalesce(task_id, '') || ' ' || + coalesce(payload ->> 'taskSummary', '') || ' ' || + coalesce(payload -> 'reflection' ->> 'lesson', ''), + 100000)) + ) STORED; + +-- GIN, not GiST: the vector is read far more often than it is written (a record's text never changes after +-- it is created -- only status, revision, and updated_at do), and GIN answers @@ lookups faster. +CREATE INDEX IF NOT EXISTS ix_experience_records_search + ON agent_experience.experience_records USING GIN (search_vector); + +-- The scope columns are the first predicate every search applies, and a tenant's records are a small +-- fraction of the table, so this composite index keeps the text match from scanning foreign scopes. It +-- carries status and reuse_confidence so the status and confidence filters are decided from the index too. +-- +-- Only the three required scope columns are indexed. team_id, agent_id, and user_id are matched with +-- IS NOT DISTINCT FROM, which is not an indexable btree operator, so adding them would not help: a +-- deployment that scopes records by team, agent, or user still scans every row of its project and filters +-- those three in memory. That is acceptable while a project's record count is modest; a deployment that +-- leans heavily on the optional scope fields should add its own partial or expression index. +-- +-- 0001's index on (tenant_id, application_id, project_id) is now a prefix of this one and therefore +-- redundant. It is deliberately left in place: scripts are append-only, and dropping an index 0001 created +-- would rewrite history for every database that already applied it. The cost is one extra index to +-- maintain on write, which is small next to the rewrite risk. +CREATE INDEX IF NOT EXISTS ix_experience_records_scope_status_confidence + ON agent_experience.experience_records + (tenant_id, application_id, project_id, status, reuse_confidence); diff --git a/src/AgentExperience.Storage.Postgres/PostgresExperienceCandidateSource.cs b/src/AgentExperience.Storage.Postgres/PostgresExperienceCandidateSource.cs new file mode 100644 index 0000000..dafbc9b --- /dev/null +++ b/src/AgentExperience.Storage.Postgres/PostgresExperienceCandidateSource.cs @@ -0,0 +1,154 @@ +using AgentExperience.Abstractions; +using Npgsql; +using NpgsqlTypes; + +namespace AgentExperience.Storage.Postgres; + +/// +/// over PostgreSQL full-text search. It follows exactly the +/// order uses -- validate the request, check it against +/// the host-established , and only then open a connection and run +/// parameterized SQL whose predicates apply the exact scope -- and translates failures the same way. +/// The schema must already exist: 0003_add_experience_search.sql adds the generated +/// search_vector column this searches, and the host applies it by calling +/// . +/// +/// +/// +/// What runs in SQL. The scope predicate, the status filter, the confidence floor, the text +/// match, and the limit. Nothing else: expiry and environment compatibility are Core's decisions, +/// made over what comes back, because they depend on policy and on the request's required +/// attributes rather than on stored state alone. +/// +/// +/// Relevance. websearch_to_tsquery parses the task text (it accepts arbitrary input -- +/// quotes, or, - -- and never raises a syntax error on it), and ts_rank_cd with +/// normalization flag 32 divides the raw rank by itself plus one, so the reported relevance is +/// already in [0, 1). It is a within-search measure: two records' relevances are comparable to each +/// other, not to a relevance from a different query. +/// +/// +/// This source reads and never writes. It needs only SELECT on +/// agent_experience.experience_records. +/// +/// +public sealed class PostgresExperienceCandidateSource : IExperienceCandidateSource +{ + /// + /// The text-search configuration the generated column was built with. It must stay identical to + /// the one in 0003_add_experience_search.sql: querying with a different configuration than + /// the column was analyzed under silently changes which rows match. + /// + internal const string SearchConfiguration = "english"; + + /// + /// The alias the rank is selected under. It is appended after the record columns, so + /// 's ordinals 0-17 are untouched, and it is + /// read back by name rather than by a hard-coded ordinal so that adding a column to + /// cannot silently shift the rank out + /// from under this reader. + /// + private const string RelevanceColumn = "relevance"; + + private const string SearchSql = + $"SELECT {PostgresExperienceRecordStore.SelectColumns}, " + + $"ts_rank_cd(search_vector, websearch_to_tsquery('{SearchConfiguration}', @task_text), 32) AS {RelevanceColumn} " + + $"FROM {PostgresExperienceRecordStore.Table} " + + $"WHERE {PostgresExperienceRecordStore.ScopePredicate} " + + "AND status = ANY(@statuses) " + + "AND reuse_confidence >= @min_confidence " + + $"AND search_vector @@ websearch_to_tsquery('{SearchConfiguration}', @task_text) " + + $"ORDER BY {RelevanceColumn} DESC, experience_id LIMIT @limit"; + + private static readonly IReadOnlyList NoErrors = []; + + private static readonly IReadOnlyList NoCandidates = []; + + private readonly NpgsqlDataSource _dataSource; + + /// Creates a candidate source over a host-owned data source. The source never disposes it. + /// The Npgsql data source to open connections from. + /// is . + public PostgresExperienceCandidateSource(NpgsqlDataSource dataSource) + { + ArgumentNullException.ThrowIfNull(dataSource); + _dataSource = dataSource; + } + + /// + public async Task SearchAsync( + AuthorizationContext authorization, + ExperienceCandidateQuery query, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(authorization); + ArgumentNullException.ThrowIfNull(query); + + var errors = ExperienceRecordValidator.ValidateCandidateQuery(query); + if (errors.Count > 0) + { + return new(ExperienceStoreOutcome.Invalid, NoCandidates, errors); + } + + if (!authorization.Permits(query.Scope)) + { + // Fail-closed, and before any connection opens: no search is issued at all. + return new(ExperienceStoreOutcome.Denied, NoCandidates, NoErrors); + } + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + await using var command = _dataSource.CreateCommand(SearchSql); + var parameters = command.Parameters; + PostgresExperienceRecordStore.AddScopeParameters(parameters, query.Scope); + parameters.Add(new NpgsqlParameter("task_text", NpgsqlDbType.Text) { TypedValue = query.TaskText }); + + var statuses = query.EligibleStatuses.Distinct().Select(status => status.ToString()).ToArray(); + parameters.Add(new NpgsqlParameter("statuses", NpgsqlDbType.Array | NpgsqlDbType.Text) { TypedValue = statuses }); + parameters.Add(new NpgsqlParameter("min_confidence", query.MinimumConfidence)); + parameters.Add(new NpgsqlParameter("limit", query.Limit)); + + var candidates = new List(); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + candidates.Add(new ExperienceCandidate( + PostgresExperienceRecordStore.ReadRecord(reader), + ReadRelevance(reader))); + } + + return new(ExperienceStoreOutcome.Found, candidates, NoErrors); + } + catch (Exception ex) when (PostgresExperienceRecordStore.IsInfrastructureFailure(ex, cancellationToken)) + { + throw PostgresExperienceRecordStore.Translate(ex, "candidate search", cancellationToken); + } + } + + /// + /// Reads the rank and clamps it into [0, 1]. Normalization flag 32 already bounds it, but a rank + /// read back as NaN or out of range would otherwise travel into Core's ranking arithmetic and + /// poison every comparison against it. + /// + private static double ReadRelevance(System.Data.Common.DbDataReader reader) + { + double rank; + try + { + rank = reader.GetDouble(reader.GetOrdinal(RelevanceColumn)); + } + catch (Exception ex) when (ex is not (ExperienceStoreException or OperationCanceledException or NpgsqlException)) + { + throw new ExperienceStoreException("Stored Experience Record could not be decoded.", ex); + } + + if (double.IsNaN(rank)) + { + return 0d; + } + + return Math.Clamp(rank, 0d, 1d); + } +} diff --git a/src/AgentExperience.Storage.Postgres/PostgresExperienceRecordSchema.cs b/src/AgentExperience.Storage.Postgres/PostgresExperienceRecordSchema.cs index 920fe7e..225efec 100644 --- a/src/AgentExperience.Storage.Postgres/PostgresExperienceRecordSchema.cs +++ b/src/AgentExperience.Storage.Postgres/PostgresExperienceRecordSchema.cs @@ -22,10 +22,22 @@ public static class PostgresExperienceRecordSchema /// The script that creates the append-only lifecycle_events table. public const string LifecycleEventsScriptName = "0002_create_lifecycle_events.sql"; + /// + /// The script that adds the generated search_vector column and its GIN index, which + /// matches task text against. + /// + public const string SearchScriptName = "0003_add_experience_search.sql"; + private const string ResourcePrefix = "AgentExperience.Storage.Postgres.Migrations."; - /// Every embedded script name, in the order they must be applied. - public static IReadOnlyList ScriptNames { get; } = [InitialScriptName, LifecycleEventsScriptName]; + /// + /// Every embedded script name, in the order they must be applied. This package's schema is + /// deliberately text-only: the derived embedding schema, which needs the vector extension, + /// is owned and applied by AgentExperience.Storage.Postgres.Vectors instead, so a host that + /// never enables the vector channel never runs a superuser-only CREATE EXTENSION. + /// + public static IReadOnlyList ScriptNames { get; } = + [InitialScriptName, LifecycleEventsScriptName, SearchScriptName]; /// Reads an embedded script's SQL text. /// One of . diff --git a/src/AgentExperience.Storage.Postgres/PostgresExperienceRecordStore.cs b/src/AgentExperience.Storage.Postgres/PostgresExperienceRecordStore.cs index 000ffaa..dc45b3a 100644 --- a/src/AgentExperience.Storage.Postgres/PostgresExperienceRecordStore.cs +++ b/src/AgentExperience.Storage.Postgres/PostgresExperienceRecordStore.cs @@ -32,14 +32,20 @@ namespace AgentExperience.Storage.Postgres; /// public sealed class PostgresExperienceRecordStore : IExperienceRecordStore { - private const string Table = "agent_experience.experience_records"; + /// The canonical record table. Shared with , which reads from it. + internal const string Table = "agent_experience.experience_records"; - private const string SelectColumns = + /// + /// The record columns every read selects, in the order expects (ordinals 0-17). + /// A reader that selects more must append its extra columns after these, never before. + /// + internal const string SelectColumns = "experience_id, source_run_id, tenant_id, application_id, project_id, team_id, agent_id, user_id, task_id, " + "status, reuse_confidence, supporting_validations, contradictions, revision, created_at, updated_at, " + "payload_version, payload"; - private const string ScopePredicate = + /// The exact-scope predicate every statement applies, shared with . + internal const string ScopePredicate = "tenant_id = @tenant_id AND application_id = @application_id AND project_id = @project_id " + "AND team_id IS NOT DISTINCT FROM @team_id AND agent_id IS NOT DISTINCT FROM @agent_id " + "AND user_id IS NOT DISTINCT FROM @user_id"; @@ -96,7 +102,12 @@ public sealed class PostgresExperienceRecordStore : IExperienceRecordStore "e.prior_status, e.current_status, e.reason, e.producer, e.occurred_at, e.recorded_at, e.expected_revision, " + "e.applied_revision"; - private const string RecordScopePredicate = + /// + /// The same exact-scope predicate as , qualified with the r + /// alias for a statement that joins the record table to another one. Shared with the vectors + /// adapter, so both retrieval channels apply a byte-for-byte identical scope match. + /// + internal const string RecordScopePredicate = "r.tenant_id = @tenant_id AND r.application_id = @application_id AND r.project_id = @project_id " + "AND r.team_id IS NOT DISTINCT FROM @team_id AND r.agent_id IS NOT DISTINCT FROM @agent_id " + "AND r.user_id IS NOT DISTINCT FROM @user_id"; @@ -575,7 +586,7 @@ private static void AddEventParameters( parameters.Add(new NpgsqlParameter("applied_revision", appliedRevision)); } - private static void AddScopeParameters(NpgsqlParameterCollection parameters, Scope scope) + internal static void AddScopeParameters(NpgsqlParameterCollection parameters, Scope scope) { parameters.Add(new NpgsqlParameter("tenant_id", NpgsqlDbType.Text) { TypedValue = scope.TenantId }); parameters.Add(new NpgsqlParameter("application_id", NpgsqlDbType.Text) { TypedValue = scope.ApplicationId }); @@ -617,7 +628,7 @@ private static bool ContainsEscapedNul(string json) return false; } - private static ExperienceRecord ReadRecord(DbDataReader reader) + internal static ExperienceRecord ReadRecord(DbDataReader reader) { try { @@ -732,14 +743,14 @@ private static ExperienceRecord DecodeRecord(DbDataReader reader) /// Driver, socket, and timeout failures are translated. An /// caused by the caller's own token is not matched, so it propagates unwrapped with its stack. /// - private static bool IsInfrastructureFailure(Exception ex, CancellationToken cancellationToken) => ex switch + internal static bool IsInfrastructureFailure(Exception ex, CancellationToken cancellationToken) => ex switch { OperationCanceledException => !cancellationToken.IsCancellationRequested, NpgsqlException or SocketException or TimeoutException => true, _ => false, }; - private static Exception Translate(Exception ex, string operation, CancellationToken cancellationToken) + internal static Exception Translate(Exception ex, string operation, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { diff --git a/src/AgentExperience.Storage.Postgres/README.md b/src/AgentExperience.Storage.Postgres/README.md index 2227c5a..f08a4a4 100644 --- a/src/AgentExperience.Storage.Postgres/README.md +++ b/src/AgentExperience.Storage.Postgres/README.md @@ -1,11 +1,13 @@ # AgentExperience.Storage.Postgres -Stores AgentExperience.NET Experience Records in PostgreSQL through the `IExperienceRecordStore` port, using plain -Npgsql. +Stores AgentExperience.NET Experience Records in PostgreSQL through the `IExperienceRecordStore` port and searches +them by task text through the `IExperienceCandidateSource` port, using plain Npgsql. -Pinned to `Npgsql` **10.0.3**, `dbup-postgresql` **7.0.1**, and `dbup-core` **6.1.1** (all exact). Integration tests -run against PostgreSQL 16 (`pgvector/pgvector:pg16`) through `Testcontainers.PostgreSql` 4.15.0. This package does not -use EF Core, Dapper, Pgvector, or the pgvector extension. +Pinned to `Npgsql` **10.0.3**, `dbup-postgresql` **7.0.1**, `dbup-core` **6.1.1**, and +`Microsoft.Extensions.DependencyInjection.Abstractions` **10.0.11** (all exact; the DI package is abstractions only — +no container, no hosting — and exists for this package's own registration extension). Integration tests run against +PostgreSQL 16 (`pgvector/pgvector:pg16`) through `Testcontainers.PostgreSql` 4.15.0. This package does not use EF +Core, Dapper, Pgvector, or the pgvector extension. ## Usage @@ -54,9 +56,48 @@ var commit = await store.CommitLifecycleEventAsync( var history = await store.GetHistoryAsync(authorization, record.Scope, record.ExperienceId, cancellationToken); // history.Events is every transition, oldest first; history.Revision is the record's current revision. + +// Finding records that could apply to a task. A separate, read-only port (see "Text search" below). +IExperienceCandidateSource search = new PostgresExperienceCandidateSource(dataSource); + +var candidates = await search.SearchAsync( + authorization, + new ExperienceCandidateQuery( + Scope: record.Scope, // exact scope, applied in SQL + TaskText: "refund ticket stuck on a lock", // arbitrary text; no escaping needed + EligibleStatuses: [ExperienceStatus.Validated, ExperienceStatus.Reinforced], + MinimumConfidence: 0.5, + Limit: 50), + cancellationToken); +// candidates.Candidates is strongest match first, each with a Relevance in [0, 1]. +``` + +Neither the store nor the search disposes the data source. The host owns it. + +### Registering it + +```csharp +using AgentExperience.Core.DependencyInjection; +using AgentExperience.Storage.Postgres.DependencyInjection; + +services.AddSingleton(NpgsqlDataSource.Create(connectionString)); +services.AddAgentExperiencePostgresStore(); // or AddAgentExperiencePostgresStore(dataSource) +services.AddAgentExperiencePostgresCandidateSource(); // or ...CandidateSource(dataSource) + +// Core's own extensions then supply capture, reflection, lifecycle, finalization, and retrieval over them. +services.AddAgentExperienceCore(sanitizationOptions, captureLimits); +services.AddAgentExperienceRetrieval(); ``` -The store never disposes the data source. The host owns it. +The two ports are registered independently: a host that only writes experience never has to register the search, and +one that only reads never has to register the store. Both registrations are `TryAdd`-based, so a host that has +already registered its own `IExperienceRecordStore` or `IExperienceCandidateSource` keeps it. +It does **not** apply the schema: call `ExperienceSchemaMigrator.MigrateAsync` once at startup (see +[Schema](#schema)). + +Records are normally written by Core's `ExperienceFinalizationService`, which creates the record and commits its +initial lifecycle event; `CreateAsync` and `CommitLifecycleEventAsync` stay available for hosts that orchestrate that +themselves. ## Trusted host boundary @@ -84,6 +125,7 @@ The store never disposes the data source. The host owns it. | Lifecycle event and projection committed together | `Committed` | | Lifecycle `ExpectedRevision` ≠ the record's current `Revision` | `StaleRevision` (nothing written) | | Lifecycle `PriorStatus` ≠ the record's stored `Status` | `StatusMismatch` with the stored status (nothing written) | +| Candidate search ran (no text match is still `Found`) | `Found` with the matching candidates, strongest match first | | Database or driver failure (`NpgsqlException`, `SocketException`, `TimeoutException`) | throws `ExperienceStoreException` with the original as `InnerException` | | Stored row with an unsupported `payload_version` or an unreadable payload | throws `ExperienceStoreException` | | Caller cancellation | throws `OperationCanceledException`, unwrapped | @@ -130,6 +172,44 @@ the transaction opens, exactly as for the store's other operations, and the scop statement, so the revision can never contradict the events even if a commit lands mid-read. Events are append-only: nothing deletes or rewrites them. `GetAsync` and its result are unchanged by this operation. +## Text search + +`PostgresExperienceCandidateSource` answers one question — *which stored records look relevant to this task text?* — +and nothing else. It is a separate port from the store on purpose: it only reads, it needs only `SELECT`, and a host +that never retrieves does not have to register it. + +It runs in the same order as every store operation: validate the query, check the request scope against the +host-established `AuthorizationContext`, and only then open a connection. A scope outside the context is `Denied` +before any connection opens, and the scope predicate is applied in SQL exactly as it is for reads. + +**What runs in the database:** the exact scope predicate, the caller's eligible status set, the reuse-confidence +floor (inclusive), the text match, and the limit (1–200, default 50). Nothing else. Because those filters run in SQL, +records they exclude never reach the caller and are never itemized anywhere — which is the point for scope, and worth +remembering for status and confidence. Expiry and environment +compatibility are Core's decisions, made over the candidates that come back, because they depend on the clock and on +the request rather than on stored state alone. + +**What is indexed:** the task ID, the sanitized task summary, and the reflection's lesson — the fields that say what +a record is *about*. Attempts, tool calls, evidence, and environment metadata are deliberately not indexed: matching +on them would make retrieval recall incidental identifiers and error strings rather than applicable experience. + +**The query text** goes through `websearch_to_tsquery`, which accepts arbitrary user input — quotes, `or`, `-`, +stray punctuation — and never raises a syntax error, so callers do not escape or sanitize around it. Multiple words +are combined with AND, and it is capped at `ExperienceCandidateQuery.MaxTaskTextLength` (4096) characters; longer is +`Invalid` before a connection opens. The text-search configuration is `english`, fixed by the generated column; +changing it means a new migration that rebuilds the column, because already-indexed rows would otherwise keep the +old analysis. + +Because the `english` configuration drops stopwords, **text made only of stopwords matches nothing at all** — `"the +of and"` produces an empty query, and an empty query matches no row by construction. The result is an ordinary +`Found` with no candidates, indistinguishable from "nothing relevant is stored". A caller that wants to tell those +apart has to decide it before calling. + +**Relevance** is `ts_rank_cd` with normalization flag 32 (`rank / (rank + 1)`), so it is already in [0, 1). It is a +within-search measure: two candidates' relevances are comparable to each other, never to a relevance from a different +query. Candidates come back in descending relevance, ties broken by `experience_id`; Core re-sorts with its own +total, ordinal tie-break when it ranks. + ## Schema The schema lives in the embedded scripts under `Migrations/`. @@ -157,6 +237,39 @@ The schema lives in the embedded scripts under `Migrations/`. back by the revision-checked projection update, and a foreign-key violation would report that expected condition as an infrastructure failure instead. +`0003_add_experience_search.sql` makes those records searchable by text: + +- A `search_vector` column, `GENERATED ALWAYS AS ... STORED` over `task_id`, the payload's `taskSummary`, and the + payload's `reflection.lesson`, analyzed with the `english` configuration. Generated, not a trigger and not a column + the store writes: it is derived from state that already exists, so it can never disagree with the record it indexes + and no write path has to maintain it. The store's `INSERT` and its lifecycle `UPDATE` are unchanged. +- A **GIN** index on `search_vector`. The vector is read far more often than written — a record's text never changes + after it is created, only its status, revision, and `updated_at` do — so GIN's faster `@@` lookups are the right + trade. +- A composite index on `(tenant_id, application_id, project_id, status, reuse_confidence)`, so a search decides scope, + status, and the confidence floor from an index rather than scanning foreign scopes. It covers only the three + *required* scope columns: `team_id`, `agent_id`, and `user_id` are matched with `IS NOT DISTINCT FROM`, which is + not an indexable btree operator, so including them would not help. A deployment that scopes records by team, agent, + or user still scans its whole project and filters those three in memory; if that matters at your row counts, add + your own partial or expression index. +- `0001`'s index on `(tenant_id, application_id, project_id)` is now a prefix of that composite and therefore + redundant, but it is deliberately left in place: scripts are append-only, and dropping an index `0001` created + would rewrite history for every database that already applied it. The cost is one extra index maintained on write. +- The concatenated text is bounded with `left(..., 100000)` before it is analyzed. A `tsvector` may not exceed 1 MB, + and in a *generated* column exceeding it is not a search failure but a failed `INSERT` — and a failed migration on + a table that already holds such a row. The bound only ever truncates text that would have broken the write. + +Adding the generated column rewrites the table, so on a large existing deployment apply this script in a maintenance +window like any other rewriting migration. + +**This package's schema stops there, and that is deliberate.** The derived embedding schema — the `vector` +extension and the `experience_embeddings` table — belongs to the companion package +[`AgentExperience.Storage.Postgres.Vectors`](../AgentExperience.Storage.Postgres.Vectors/README.md) and is applied +by *its* migrator, `ExperienceVectorSchemaMigrator.MigrateAsync`. `CREATE EXTENSION vector` needs a superuser, +because pgvector is not a trusted extension; putting it in this script list would make that privilege a startup +requirement for every host, including text-only ones that never enable the vector channel. Nothing here creates an +extension, and nothing here reads or writes the embedding table. + ### Applying it Call `ExperienceSchemaMigrator.MigrateAsync` explicitly at startup, before using the store. The store never migrates @@ -177,8 +290,9 @@ var migration = await ExperienceSchemaMigrator.MigrateAsync(dataSource, cancella - **Serialized across processes.** The whole run holds a PostgreSQL session advisory lock on its own connection, so two hosts starting at once cannot apply the same script twice. The lock is always released. - **Permissions.** The migrating role needs `CREATE` on the database (for the `agent_experience` schema) and on that - schema (for its tables). The store itself only needs `SELECT`, `INSERT`, and `UPDATE` on - `agent_experience.experience_records` and `SELECT` and `INSERT` on `agent_experience.lifecycle_events`. + schema (for its tables). It does **not** need to be a superuser: no script here creates an extension. The store itself only needs `SELECT`, `INSERT`, and `UPDATE` on + `agent_experience.experience_records` and `SELECT` and `INSERT` on `agent_experience.lifecycle_events`; the + candidate source needs only `SELECT` on `agent_experience.experience_records`. - **Connections.** The data source must allow at least two concurrent connections: one for the advisory lock and one for the scripts. A multiplexing data source (`NpgsqlDataSourceBuilder.EnableMultiplexing`) cannot hold a session advisory lock, because its commands do not stay on one physical connection, so it is not supported for migration. @@ -223,5 +337,8 @@ definition, and a rename would reapply it. Change the schema by adding the next- - **Query order** is newest `CreatedAt` first, then `ExperienceId` in PostgreSQL `uuid` byte order, which differs from .NET `Guid` comparison. `Limit` must be from 1 to 500 (default 50). `Statuses` is either null (all statuses) or a non-empty list. +- **Search order** is descending `ts_rank_cd` relevance, then `ExperienceId` in PostgreSQL `uuid` byte order. `Limit` + must be from 1 to 200 (default 50), and `EligibleStatuses` must be non-empty — an empty set is `Invalid` rather + than widened to "every status", so a caller can never accidentally ask for records it considers ineligible. - PostgreSQL cannot store the NUL character (U+0000) in `text` or `jsonb`, so a record or scope containing it is `Invalid` and never reaches the database. diff --git a/src/AgentExperience.Storage.Postgres/packages.lock.json b/src/AgentExperience.Storage.Postgres/packages.lock.json index b0e78c4..f474c1c 100644 --- a/src/AgentExperience.Storage.Postgres/packages.lock.json +++ b/src/AgentExperience.Storage.Postgres/packages.lock.json @@ -21,6 +21,12 @@ "dbup-core": "6.1.1" } }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Direct", + "requested": "[10.0.11, 10.0.11]", + "resolved": "10.0.11", + "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" + }, "Npgsql": { "type": "Direct", "requested": "[10.0.3, 10.0.3]", @@ -30,11 +36,6 @@ "Microsoft.Extensions.Logging.Abstractions": "10.0.0" } }, - "Microsoft.Extensions.DependencyInjection.Abstractions": { - "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "L3AdmZ1WOK4XXT5YFPEwyt0ep6l8lGIPs7F5OOBZc77Zqeo01Of7XXICy47628sdVl0v/owxYJTe86DTgFwKCA==" - }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", "resolved": "10.0.0", diff --git a/tests/AgentExperience.Abstractions.Tests/ContractShapeTests.cs b/tests/AgentExperience.Abstractions.Tests/ContractShapeTests.cs index 97b9c7e..586f709 100644 --- a/tests/AgentExperience.Abstractions.Tests/ContractShapeTests.cs +++ b/tests/AgentExperience.Abstractions.Tests/ContractShapeTests.cs @@ -320,6 +320,55 @@ public void Store_port_operations_take_authorization_and_a_required_cancellation Assert.Equal([typeof(AuthorizationContext), typeof(ExperienceRecordQuery), typeof(CancellationToken)], methods[4].GetParameters().Select(p => p.ParameterType)); } + // Story 2.2: the retrieval candidate-source port. + [Fact] + public void Candidate_source_port_mirrors_the_store_port_and_stays_separate_from_it() + { + var method = Assert.Single(typeof(IExperienceCandidateSource).GetMethods()); + + Assert.Equal("SearchAsync", method.Name); + Assert.Equal(typeof(Task), method.ReturnType); + Assert.Equal( + [typeof(AuthorizationContext), typeof(ExperienceCandidateQuery), typeof(CancellationToken)], + method.GetParameters().Select(p => p.ParameterType)); + Assert.False(method.GetParameters()[^1].HasDefaultValue); + + // A new port, not an extension of the store: retrieval must not change what a writer implements. + Assert.DoesNotContain( + typeof(IExperienceRecordStore).GetMethods(), + m => m.Name.Contains("Search", StringComparison.Ordinal)); + Assert.False(typeof(IExperienceCandidateSource).IsAssignableFrom(typeof(IExperienceRecordStore))); + } + + [Fact] + public void Candidate_query_defaults_to_a_limit_of_50_within_1_to_200_and_a_candidate_carries_a_normalized_relevance() + { + var query = new ExperienceCandidateQuery(new Scope("t", "a", "p"), "refund", [ExperienceStatus.Validated], 0.5); + + Assert.Equal(50, query.Limit); + Assert.Equal(50, ExperienceCandidateQuery.DefaultLimit); + Assert.Equal(1, ExperienceCandidateQuery.MinLimit); + Assert.Equal(200, ExperienceCandidateQuery.MaxLimit); + + var record = new ExperienceRecord( + Guid.NewGuid(), Guid.NewGuid(), query.Scope, "task", null, [], + new Outcome(TaskVerificationStatus.Verified, [], null, Now), 1, null, + new EnvironmentFingerprint("host", "10.0.0", "linux-x64", null, new Dictionary()), + new Provenance("tests", null, Now, null), + ExperienceStatus.Validated, 0.8, 1, 0, 1, Now, Now); + + var found = new ExperienceCandidateSearchResult( + ExperienceStoreOutcome.Found, [new ExperienceCandidate(record, 0.42)], []); + + Assert.Equal(0.42, Assert.Single(found.Candidates).Relevance); + Assert.Same(record, found.Candidates[0].Record); + + // The result reuses the store's outcome enum, which has no timeout member: a retrieval timeout + // is Core's own result type, never a storage outcome. + Assert.DoesNotContain("Timeout", Enum.GetNames()); + Assert.Empty(new ExperienceCandidateSearchResult(ExperienceStoreOutcome.Denied, [], []).Candidates); + } + // Story 2.4: the lifecycle commit and history contracts. [Fact] public void Lifecycle_commit_and_history_results_carry_a_revision_and_ordered_events() diff --git a/tests/AgentExperience.Abstractions.Tests/packages.lock.json b/tests/AgentExperience.Abstractions.Tests/packages.lock.json index 1597822..50c646d 100644 --- a/tests/AgentExperience.Abstractions.Tests/packages.lock.json +++ b/tests/AgentExperience.Abstractions.Tests/packages.lock.json @@ -34,19 +34,6 @@ "resolved": "17.14.1", "contentHash": "pmTrhfFIoplzFVbhVwUquT+77CbGH+h4/3mBpdmIlYtBi9nAB+kKI6dN3A/nV4DFi3wLLx/BlHIPK+MkbQ6Tpg==" }, - "Microsoft.Extensions.DependencyInjection.Abstractions": { - "type": "Transitive", - "resolved": "8.0.0", - "contentHash": "cjWrLkJXK0rs4zofsK4bSdg+jhDLTaxrkXu4gS6Y7MAlCvRyNNgwY/lJi5RDlQOnSZweHqoyvgvbdvQsRIW+hg==" - }, - "Microsoft.Extensions.Logging.Abstractions": { - "type": "Transitive", - "resolved": "8.0.0", - "contentHash": "arDBqTgFCyS0EvRV7O3MZturChstm50OJ0y9bDJvAcmEPJm0FFpFyjU/JLYyStNGGey081DvnQYlncNX5SJJGA==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0" - } - }, "Microsoft.TestPlatform.ObjectModel": { "type": "Transitive", "resolved": "17.14.1", @@ -66,14 +53,6 @@ "resolved": "13.0.3", "contentHash": "HrC5BXdl00IP9zeV+0Z848QWPAoCr9P3bDEZguI+gkLcBKAOxix/tLEAAHC+UvDNPv4a2d18lOReHMOagPa+zQ==" }, - "Npgsql": { - "type": "Transitive", - "resolved": "8.0.3", - "contentHash": "6WEmzsQJCZAlUG1pThKg/RmeF6V+I0DmBBBE/8YzpRtEzhyZzKcK7ulMANDm5CkxrALBEC8H+5plxHWtIL7xnA==", - "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "8.0.0" - } - }, "xunit.abstractions": { "type": "Transitive", "resolved": "2.0.3", @@ -115,10 +94,7 @@ } }, "agentexperience.abstractions": { - "type": "Project", - "dependencies": { - "Npgsql": "[8.0.3, )" - } + "type": "Project" } } } diff --git a/tests/AgentExperience.Core.Tests/AgentExperience.Core.Tests.csproj b/tests/AgentExperience.Core.Tests/AgentExperience.Core.Tests.csproj index ab259fa..e85d756 100644 --- a/tests/AgentExperience.Core.Tests/AgentExperience.Core.Tests.csproj +++ b/tests/AgentExperience.Core.Tests/AgentExperience.Core.Tests.csproj @@ -11,6 +11,9 @@ + + diff --git a/tests/AgentExperience.Core.Tests/CoreServiceRegistrationTests.cs b/tests/AgentExperience.Core.Tests/CoreServiceRegistrationTests.cs new file mode 100644 index 0000000..1725a88 --- /dev/null +++ b/tests/AgentExperience.Core.Tests/CoreServiceRegistrationTests.cs @@ -0,0 +1,309 @@ +using AgentExperience.Core.DependencyInjection; +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Lifecycle; +using AgentExperience.Core.Retrieval; +using Microsoft.Extensions.DependencyInjection; + +namespace AgentExperience.Core.Tests; + +/// +/// Resolves what +/// registers out of a real container, so deleting a registration fails here rather than only at a +/// host's startup. Also pins the two properties a host depends on: the caller's own arguments are the +/// ones that reach the services, and a host implementation registered first still wins. +/// +public class CoreServiceRegistrationTests +{ + private static readonly SanitizationOptions CallerOptions = new(new Dictionary(StringComparer.Ordinal) + { + ["ToolResult"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "value" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 2, + MaxFieldCount: 5, + MaxValueLength: 1_000, + MaxFieldNameLength: 100), + }); + + private static readonly CaptureLimits CallerLimits = new(10, 10, 1_000, 1_000); + + [Fact] + public void AddAgentExperienceCore_registers_every_service_the_finalization_path_needs() + { + var services = new ServiceCollection(); + services.AddSingleton(new StubStore()); // the storage adapter's job + services.AddAgentExperienceCore(CallerOptions, CallerLimits); + + using var provider = services.BuildServiceProvider(); + + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + + // Singletons, so capture's in-memory snapshots survive between resolutions. + Assert.Same(provider.GetRequiredService(), provider.GetRequiredService()); + Assert.Same(provider.GetRequiredService(), provider.GetRequiredService()); + } + + [Fact] + public async Task The_sanitizer_applies_the_policy_the_caller_passed_even_when_the_host_registered_another() + { + // A host-registered SanitizationOptions must not silently replace the policy the caller handed + // to AddAgentExperienceCore: the sanitizer would then enforce a policy nobody passed to it. + var services = new ServiceCollection(); + services.AddSingleton(SanitizationOptions.Empty); // registered first, so TryAdd keeps it + services.AddAgentExperienceCore(CallerOptions, CallerLimits); + + using var provider = services.BuildServiceProvider(); + + var sanitized = await provider.GetRequiredService().SanitizeAsync( + new RawPayload("ToolResult", new Dictionary { ["value"] = "ok" })); + + Assert.Equal(SanitizationDecision.Allowed, sanitized.Decision); + } + + [Fact] + public void A_host_implementation_registered_first_wins() + { + var hostSanitizer = new DefaultSanitizer(CallerOptions); + var hostReflector = new DefaultExperienceReflector(); + + var services = new ServiceCollection(); + services.AddSingleton(new StubStore()); + services.AddSingleton(hostSanitizer); + services.AddSingleton(hostReflector); + services.AddAgentExperienceCore(CallerOptions, CallerLimits); + + using var provider = services.BuildServiceProvider(); + + Assert.Same(hostSanitizer, provider.GetRequiredService()); + Assert.Same(hostReflector, provider.GetRequiredService()); + } + + [Fact] + public void AddAgentExperienceRetrieval_registers_the_retrieval_service_with_the_documented_defaults() + { + var services = new ServiceCollection(); + services.AddSingleton(new StubCandidateSource()); // the storage adapter's job + services.AddAgentExperienceRetrieval(); + + using var provider = services.BuildServiceProvider(); + var retrieval = provider.GetRequiredService(); + + Assert.Same(RetrievalPolicy.Default, provider.GetRequiredService()); + Assert.Same(RankingWeights.Default, provider.GetRequiredService()); + Assert.Same(TimeProvider.System, provider.GetRequiredService()); + Assert.Equal(TimeSpan.FromMilliseconds(500), retrieval.Policy.Timeout); + Assert.Equal(0.35, retrieval.Weights.Relevance); + Assert.Same(retrieval, provider.GetRequiredService()); + } + + [Fact] + public void The_retrieval_service_uses_the_policy_and_weights_the_caller_passed_even_when_the_host_registered_others() + { + var callerPolicy = RetrievalPolicy.Default with { Timeout = TimeSpan.FromSeconds(2) }; + var callerWeights = new RankingWeights(1d, 0d, 0d, 0d, 0d); + + var services = new ServiceCollection(); + services.AddSingleton(new StubCandidateSource()); + services.AddSingleton(RetrievalPolicy.Default); // registered first, so TryAdd keeps it + services.AddAgentExperienceRetrieval(callerPolicy, callerWeights); + + using var provider = services.BuildServiceProvider(); + var retrieval = provider.GetRequiredService(); + + Assert.Same(callerPolicy, retrieval.Policy); + Assert.Same(callerWeights, retrieval.Weights); + Assert.Same(RetrievalPolicy.Default, provider.GetRequiredService()); + } + + [Fact] + public async Task A_host_registered_TimeProvider_is_the_clock_the_retrieval_service_actually_measures_with() + { + // Resolved from the container, not defaulted: a service that quietly used TimeProvider.System + // would judge expiry against the wall clock and return the stale record below. + var now = new DateTimeOffset(2026, 9, 21, 12, 0, 0, TimeSpan.Zero); + var stale = StubCandidateSource.RecordUpdatedAt(now - TimeSpan.FromDays(30)); + + var services = new ServiceCollection(); + services.AddSingleton(new FrozenClock(now)); // registered first, so TryAdd keeps it + services.AddSingleton(new StubCandidateSource(stale)); + services.AddAgentExperienceRetrieval(RetrievalPolicy.Default with { MaxAge = TimeSpan.FromDays(7) }); + + using var provider = services.BuildServiceProvider(); + + var result = await provider.GetRequiredService().RetrieveAsync( + new RetrieveExperienceRequest(StubCandidateSource.Authorization, StubCandidateSource.Scope, "refund")); + + Assert.IsType(provider.GetRequiredService()); + Assert.Empty(result.Records); + Assert.Equal( + [new ExcludedExperience(stale.ExperienceId, RetrievalExclusionReason.Expired)], + result.Excluded); + Assert.Equal(TimeSpan.Zero, result.Elapsed); // measured with the frozen clock too + } + + [Fact] + public void Retrieval_without_a_candidate_source_fails_to_resolve_rather_than_retrieving_nothing() + { + var services = new ServiceCollection(); + services.AddAgentExperienceRetrieval(); + + using var provider = services.BuildServiceProvider(); + + Assert.Throws(() => provider.GetRequiredService()); + } + + [Fact] + public void Null_arguments_throw() + { + Assert.Throws(() => ((IServiceCollection)null!).AddAgentExperienceRetrieval()); + Assert.Throws(() => ((IServiceCollection)null!).AddAgentExperienceCore(CallerOptions, CallerLimits)); + Assert.Throws(() => new ServiceCollection().AddAgentExperienceCore(null!, CallerLimits)); + Assert.Throws(() => new ServiceCollection().AddAgentExperienceCore(CallerOptions, null!)); + } + + /// Stands in for a storage adapter's search registration, answering with whatever it was given. + private sealed class StubCandidateSource(params ExperienceRecord[] records) : IExperienceCandidateSource + { + public static Scope Scope { get; } = new("tenant-1", "app-1", "project-1"); + + public static AuthorizationContext Authorization { get; } = new("tenant-1", "host", [], DateTimeOffset.UnixEpoch); + + public static ExperienceRecord RecordUpdatedAt(DateTimeOffset updatedAt) => new( + Guid.NewGuid(), Guid.NewGuid(), Scope, "refund-ticket", "Resolve a refund", [], + new Outcome(TaskVerificationStatus.Verified, [], null, updatedAt), 1, null, + new EnvironmentFingerprint("host", "10.0.0", "linux-x64", null, new Dictionary()), + new Provenance("tests", null, updatedAt, null), + ExperienceStatus.Validated, 0.9, 1, 0, 1, updatedAt, updatedAt); + + public Task SearchAsync(AuthorizationContext authorization, ExperienceCandidateQuery query, CancellationToken cancellationToken) => + Task.FromResult(new ExperienceCandidateSearchResult( + ExperienceStoreOutcome.Found, + [.. records.Select(record => new ExperienceCandidate(record, 1d))], + [])); + } + + /// A clock frozen at a known instant, standing in for a host's own . + private sealed class FrozenClock(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => now; + + public override long GetTimestamp() => now.UtcTicks; + + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => new NeverFires(); + + private sealed class NeverFires : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) => true; + + public void Dispose() + { + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + } + + // ---------------------------------------------------------------- indexing and hybrid retrieval + + [Fact] + public void AddAgentExperienceIndexing_registers_the_indexing_service_over_an_adapters_index_and_generator() + { + var services = new ServiceCollection(); + services.AddSingleton(new FakeEmbeddingIndex()); // the adapter's job + services.AddSingleton(new FakeEmbeddingGenerator()); + services.AddAgentExperienceIndexing(); + + using var provider = services.BuildServiceProvider(); + + var indexing = provider.GetRequiredService(); + Assert.Equal("fake-embed-v1", indexing.ModelId); + Assert.Equal(4, indexing.Dimension); + Assert.Same(indexing, provider.GetRequiredService()); + } + + [Fact] + public void Finalization_picks_up_an_indexing_hook_registered_in_either_order() + { + foreach (var indexingFirst in new[] { true, false }) + { + var services = new ServiceCollection(); + services.AddSingleton(new StubStore()); + services.AddSingleton(new FakeEmbeddingIndex()); + services.AddSingleton(new FakeEmbeddingGenerator()); + + if (indexingFirst) + { + services.AddAgentExperienceIndexing(); + services.AddAgentExperienceCore(CallerOptions, CallerLimits); + } + else + { + services.AddAgentExperienceCore(CallerOptions, CallerLimits); + services.AddAgentExperienceIndexing(); + } + + using var provider = services.BuildServiceProvider(); + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + } + } + + [Fact] + public void Finalization_resolves_without_an_indexing_hook_at_all() + { + // A text-only deployment registers no embedding index and no generator, and must still work. + var services = new ServiceCollection(); + services.AddSingleton(new StubStore()); + services.AddAgentExperienceCore(CallerOptions, CallerLimits); + + using var provider = services.BuildServiceProvider(); + + Assert.NotNull(provider.GetRequiredService()); + Assert.Null(provider.GetService()); + } + + [Fact] + public void AddAgentExperienceRetrieval_wires_the_vector_channel_only_when_both_halves_are_registered() + { + var textOnly = new ServiceCollection(); + textOnly.AddSingleton(new StubCandidateSource()); + textOnly.AddAgentExperienceRetrieval(); + + var hybrid = new ServiceCollection(); + hybrid.AddSingleton(new StubCandidateSource()); + hybrid.AddSingleton(new FakeEmbeddingIndex()); + hybrid.AddSingleton(new FakeEmbeddingGenerator()); + hybrid.AddAgentExperienceRetrieval(); + + using var textProvider = textOnly.BuildServiceProvider(); + using var hybridProvider = hybrid.BuildServiceProvider(); + + Assert.False(textProvider.GetRequiredService().HybridEnabled); + Assert.True(hybridProvider.GetRequiredService().HybridEnabled); + } + + /// Stands in for a storage adapter's registration; finalization never calls it here. + private sealed class StubStore : IExperienceRecordStore + { + public Task CreateAsync(AuthorizationContext authorization, ExperienceRecord record, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public Task GetAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public Task QueryAsync(AuthorizationContext authorization, ExperienceRecordQuery query, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public Task CommitLifecycleEventAsync(AuthorizationContext authorization, Scope scope, LifecycleEvent lifecycleEvent, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public Task GetHistoryAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) => + throw new NotSupportedException(); + } +} diff --git a/tests/AgentExperience.Core.Tests/DefaultExperienceReflectorTests.cs b/tests/AgentExperience.Core.Tests/DefaultExperienceReflectorTests.cs index 7f10d95..a32c1d6 100644 --- a/tests/AgentExperience.Core.Tests/DefaultExperienceReflectorTests.cs +++ b/tests/AgentExperience.Core.Tests/DefaultExperienceReflectorTests.cs @@ -62,6 +62,9 @@ public static ExperienceRun Run( StartedAt: BaseTime, EndedAt: executionStatus is null ? null : BaseTime.AddMinutes(1)); + /// A required check that accepts evidence of any kind unless a kind is named. + public static RequiredCheck Check(string checkId, string? expectedKind = null) => new(checkId, expectedKind); + public static Evidence Evidence(int index, string checkId, CheckResult result) => new( EvidenceId: Guid.Parse($"aaaaaaaa-0000-0000-0000-{index:D12}"), @@ -77,18 +80,18 @@ public static Evidence Evidence(int index, string checkId, CheckResult result) = public static VerificationResult Verified() => VerificationAggregator.Aggregate( [Evidence(1, "build", CheckResult.Pass), Evidence(2, "tests", CheckResult.Pass)], - ["build", "tests"], Round, Round.ArtifactRevision, BaseTime); + [Check("build"), Check("tests")], Round, Round.ArtifactRevision, BaseTime); public static VerificationResult Failed() => VerificationAggregator.Aggregate( [Evidence(1, "build", CheckResult.Fail), Evidence(2, "tests", CheckResult.Pass)], - ["build", "tests"], Round, Round.ArtifactRevision, BaseTime); + [Check("build"), Check("tests")], Round, Round.ArtifactRevision, BaseTime); /// "build" passes, "tests" has no evidence: Unknown with completion score 0.5. public static VerificationResult Unknown() => VerificationAggregator.Aggregate( [Evidence(1, "build", CheckResult.Pass)], - ["build", "tests"], Round, Round.ArtifactRevision, BaseTime); + [Check("build"), Check("tests")], Round, Round.ArtifactRevision, BaseTime); /// Attempt 0 errors, attempt 1 completes. public static IReadOnlyList RepairAttempts() => @@ -447,7 +450,7 @@ public async Task Tool_calls_are_described_in_sequence_number_order_regardless_o [Fact] public async Task Unknown_with_a_zero_completion_score_has_no_partial_score_warning() { - var evaluation = VerificationAggregator.Aggregate([], ["build"], null, "rev-1", ReflectionFixtures.BaseTime); + var evaluation = VerificationAggregator.Aggregate([], [ReflectionFixtures.Check("build")], null, "rev-1", ReflectionFixtures.BaseTime); Assert.Equal(TaskVerificationStatus.Unknown, evaluation.Outcome.Status); Assert.Equal(0.0, evaluation.CompletionScore); diff --git a/tests/AgentExperience.Core.Tests/DependencyBoundaryTests.cs b/tests/AgentExperience.Core.Tests/DependencyBoundaryTests.cs index acb7904..66bf767 100644 --- a/tests/AgentExperience.Core.Tests/DependencyBoundaryTests.cs +++ b/tests/AgentExperience.Core.Tests/DependencyBoundaryTests.cs @@ -7,8 +7,11 @@ namespace AgentExperience.Core.Tests; /// /// Proves AgentExperience.Core has no dependency on MAF, EF Core, Npgsql, DbUp, OpenTelemetry, /// or a model-provider package (AD-1) -- its only allowed dependencies are -/// AgentExperience.Abstractions and Microsoft.Extensions.Compliance.Redaction (plus -/// that package's own transitive Microsoft.Extensions.* configuration/DI/options graph). +/// AgentExperience.Abstractions, Microsoft.Extensions.Compliance.Redaction, and +/// Microsoft.Extensions.DependencyInjection.Abstractions (plus the redaction package's own +/// transitive Microsoft.Extensions.* configuration/DI/options graph). The DI package is +/// abstractions only -- no container, no hosting -- and exists so Core can ship its own +/// AddAgentExperienceCore registration extension without a host guessing concrete types. /// Mirrors AgentExperience.Abstractions.Tests/DependencyBoundaryTests.cs. This runs in CI on /// every push/PR so the boundary cannot silently regress as later stories/adapters are added to /// the solution. @@ -79,6 +82,28 @@ public void AgentExperience_Core_csproj_declares_no_forbidden_PackageReference() } } + [Fact] + public void AgentExperience_Core_csproj_declares_exactly_the_allowed_PackageReferences() + { + // The forbidden-substring checks above cannot catch a newly added package that is merely + // unwanted rather than forbidden. Pinning the whole declared set makes every future addition + // a deliberate, reviewed change to this list. + var declared = XDocument.Load(GetCoreCsprojPath()) + .Descendants("PackageReference") + .Select(element => $"{element.Attribute("Include")?.Value} {element.Attribute("Version")?.Value}") + .Order(StringComparer.Ordinal) + .ToList(); + + // The DI abstractions pin is exact, matching AgentExperience.Storage.Postgres, so the two + // packages can never resolve different versions of the same dependency. + Assert.Equal( + [ + "Microsoft.Extensions.Compliance.Redaction 10.9.0", + "Microsoft.Extensions.DependencyInjection.Abstractions [10.0.11]", + ], + declared); + } + private static string GetCoreCsprojPath([CallerFilePath] string testSourceFilePath = "") { var testsProjectDirectory = Path.GetDirectoryName(testSourceFilePath)!; diff --git a/tests/AgentExperience.Core.Tests/ExperienceFinalizationServiceTests.cs b/tests/AgentExperience.Core.Tests/ExperienceFinalizationServiceTests.cs new file mode 100644 index 0000000..32d2d44 --- /dev/null +++ b/tests/AgentExperience.Core.Tests/ExperienceFinalizationServiceTests.cs @@ -0,0 +1,993 @@ +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Lifecycle; + +namespace AgentExperience.Core.Tests; + +/// +/// Covers every row of Story 2.5's I/O and edge-case matrix against fakes: the happy path, an +/// unverified run, a throwing reflector, a host storage denial, a run beyond the caller's authority, +/// an unknown run, an unfinished run, a replay, and a store failure in each of the two stages that +/// touch the database. Capture and the lifecycle service are the real implementations; only the +/// record store and (where a failure is being forced) the reflector are doubles. +/// +public class ExperienceFinalizationServiceTests +{ + private const string ArtifactRevision = "rev-1"; + + private static readonly DateTimeOffset Now = new(2026, 9, 18, 10, 0, 0, TimeSpan.Zero); + private static readonly Scope TestScope = new("tenant-1", "app-1", "project-1"); + private static readonly AuthorizationContext Authorization = new("tenant-1", "host-principal", ["experience:write"], Now); + private static readonly ClosedVerificationRound Round = new(Guid.Parse("11111111-1111-1111-1111-111111111111"), ArtifactRevision); + + private static readonly SanitizationOptions PermissiveOptions = new(new Dictionary(StringComparer.Ordinal) + { + ["ToolArguments"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "query" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 3, + MaxFieldCount: 10, + MaxValueLength: 1_000, + MaxFieldNameLength: 100), + ["ToolResult"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "value" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 2, + MaxFieldCount: 5, + MaxValueLength: 1_000, + MaxFieldNameLength: 100), + }); + + private static Evidence PassingEvidence(string checkId = "tests", CheckResult result = CheckResult.Pass) => new( + EvidenceId: Guid.NewGuid(), + VerificationRoundId: Round.RoundId, + ArtifactRevision: ArtifactRevision, + CheckId: checkId, + Kind: "TestResult", + Result: result, + Producer: "ci", + Detail: null, + CapturedAt: Now); + + // --------------------------------------------------------------------------------------------- + // Happy path + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task A_verified_run_a_successful_reflection_and_a_permitting_decision_produce_a_Validated_record() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, result.Stage); + Assert.True(result.IsDurable); + Assert.Null(result.Failure); + + var record = Assert.IsType(result.Record); + Assert.Equal(ExperienceStatus.Validated, record.Status); + Assert.Equal(2d / 3d, record.ReuseConfidence); + Assert.Equal(1, record.SupportingValidations); + Assert.Equal(0, record.Contradictions); + Assert.Equal(TaskVerificationStatus.Verified, record.Outcome.Status); + Assert.NotNull(record.Reflection); + Assert.Equal(harness.RunId, record.SourceRunId); + + // One record, at revision 1, with exactly one lifecycle event. + Assert.Equal(1, result.Revision); + + // The record is *created* as a Candidate; the initial event performs the real transition, so a + // commit that never lands can only ever leave a Candidate behind. + Assert.Equal(ExperienceStatus.Candidate, Assert.Single(harness.Store.Creates).Status); + + var committed = Assert.Single(harness.Store.Commits); + Assert.Equal(ExperienceStatus.Candidate, committed.PriorStatus); + Assert.Equal(ExperienceStatus.Validated, committed.CurrentStatus); + Assert.Equal(0, committed.ExpectedRevision); + Assert.Equal(ExperienceFinalizationService.ProducerIdentity, committed.Producer); + Assert.Equal(1, harness.Store.RevisionOf(record.ExperienceId)); + Assert.Equal(ExperienceStatus.Validated, harness.Store.StatusOf(record.ExperienceId)); + } + + [Fact] + public async Task The_record_copies_the_captured_attempts_unchanged() + { + var harness = await Harness.WithCompletedRunAsync(); + var captured = harness.CapturedRun(); + + var result = await harness.FinalizeAsync(); + + // Finalization never sanitizes: capture already rejected anything unsafe. + Assert.Equal(captured.Attempts, result.Record!.Attempts); + Assert.Equal(captured.TaskId, result.Record.TaskId); + Assert.Equal(captured.TaskDescription, result.Record.TaskSummary); + Assert.Same(captured.Environment, result.Record.Environment); + Assert.Same(captured.Provenance, result.Record.Provenance); + Assert.Same(captured.Scope, result.Record.Scope); + } + + // --------------------------------------------------------------------------------------------- + // Unverified + // --------------------------------------------------------------------------------------------- + + [Theory] + [InlineData(CheckResult.Fail, TaskVerificationStatus.Failed)] + [InlineData(CheckResult.Unknown, TaskVerificationStatus.Unknown)] + public async Task An_unverified_run_is_quarantined_with_failure_metadata_and_no_reflection( + CheckResult evidenceResult, + TaskVerificationStatus expectedStatus) + { + var reflector = new CountingReflector(); + var harness = await Harness.WithCompletedRunAsync(reflector: reflector); + + var result = await harness.FinalizeAsync(evidence: [PassingEvidence(result: evidenceResult)]); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(ExperienceStatus.Quarantined, result.Record!.Status); + Assert.Null(result.Record.Reflection); + Assert.Equal(0d, result.Record.ReuseConfidence); + Assert.Equal(0, result.Record.SupportingValidations); + Assert.Equal(expectedStatus, result.Record.Outcome.Status); + + // Safe failure metadata: which stage decided it, in content-free prose. + var failure = Assert.IsType(result.Failure); + Assert.Equal(FinalizationStage.Evaluate, failure.Stage); + Assert.Contains(expectedStatus.ToString(), failure.Reason, StringComparison.Ordinal); + + // An unverified run is never reflected on, so no unreflected lesson can reach the record. + Assert.Equal(0, reflector.Calls); + Assert.Equal(ExperienceStatus.Quarantined, Assert.Single(harness.Store.Commits).CurrentStatus); + } + + [Fact] + public async Task A_run_with_no_closed_round_is_quarantined_rather_than_validated() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(noRound: true); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Equal(TaskVerificationStatus.Unknown, result.Evaluation!.Outcome.Status); + } + + // --------------------------------------------------------------------------------------------- + // Reflection fails + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task A_throwing_reflector_quarantines_the_record_names_the_stage_and_is_not_rethrown() + { + var harness = await Harness.WithCompletedRunAsync(reflector: new ThrowingReflector()); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Equal(ExperienceStatus.Quarantined, result.Record!.Status); + Assert.Null(result.Record.Reflection); + + var failure = Assert.IsType(result.Failure); + Assert.Equal(FinalizationStage.Reflect, failure.Stage); + Assert.IsType(failure.Exception); + + // The record is still committed, and the verification it was judged against is preserved. + Assert.Equal(TaskVerificationStatus.Verified, result.Record.Outcome.Status); + Assert.Single(harness.Store.Commits); + Assert.Equal(1, result.Revision); + } + + [Fact] + public async Task A_reflector_that_returns_no_reflection_quarantines_the_record_and_names_the_stage() + { + // Returning null is not the same failure mode as throwing, and it is the one a lenient custom + // reflector is most likely to produce. + var harness = await Harness.WithCompletedRunAsync(reflector: new NullReturningReflector()); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Equal(ExperienceStatus.Quarantined, result.Record!.Status); + Assert.Null(result.Record.Reflection); + Assert.Equal(FinalizationStage.Reflect, result.Failure!.Stage); + Assert.Null(result.Failure.Exception); + } + + // --------------------------------------------------------------------------------------------- + // Storage denied + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task A_host_decision_that_denies_storage_writes_nothing_and_returns_a_structured_denial() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(decision: StorageDecision.Deny("retention policy")); + + Assert.Equal(FinalizationOutcome.StorageDenied, result.Outcome); + Assert.Equal(FinalizationStage.Authorize, result.Stage); + Assert.False(result.IsDurable); + Assert.Null(result.Record); + Assert.Null(result.ExperienceId); // no record ID is issued + Assert.Null(result.Event); + Assert.Equal("retention policy", result.Reason); + Assert.Empty(harness.Store.Creates); + Assert.Empty(harness.Store.Commits); + } + + [Fact] + public async Task Storage_is_denied_whatever_the_verification_says() + { + var harness = await Harness.WithCompletedRunAsync(); + + var denied = await harness.FinalizeAsync( + evidence: [PassingEvidence(result: CheckResult.Fail)], + decision: StorageDecision.Deny()); + + Assert.Equal(FinalizationOutcome.StorageDenied, denied.Outcome); + Assert.Empty(harness.Store.Creates); + } + + // --------------------------------------------------------------------------------------------- + // Beyond authority + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task A_run_outside_the_authorization_is_denied_before_any_store_call() + { + var harness = await Harness.WithCompletedRunAsync(); + var otherTenant = new AuthorizationContext("tenant-2", "host-principal", ["experience:write"], Now); + + var result = await harness.FinalizeAsync(authorization: otherTenant); + + Assert.Equal(FinalizationOutcome.NotAuthorized, result.Outcome); + Assert.Equal(FinalizationStage.Authorize, result.Stage); + Assert.Null(result.Record); + Assert.Empty(harness.Store.Creates); + Assert.Empty(harness.Store.Commits); + } + + [Fact] + public async Task A_refused_run_is_never_handed_to_the_reflector() + { + // IExperienceReflector is the documented seam for a model-backed reflector, so both gates run + // before it: a run the host is about to refuse never has its content handed over. + var denied = new CountingReflector(); + var harnessDenied = await Harness.WithCompletedRunAsync(reflector: denied); + await harnessDenied.FinalizeAsync(decision: StorageDecision.Deny("retention policy")); + Assert.Equal(0, denied.Calls); + + var unauthorized = new CountingReflector(); + var harnessUnauthorized = await Harness.WithCompletedRunAsync(reflector: unauthorized); + await harnessUnauthorized.FinalizeAsync( + authorization: new AuthorizationContext("tenant-2", "host-principal", ["experience:write"], Now)); + Assert.Equal(0, unauthorized.Calls); + } + + // --------------------------------------------------------------------------------------------- + // Unknown and unfinished runs + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task An_unknown_run_is_a_structured_not_found_result() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(runId: Guid.NewGuid()); + + Assert.Equal(FinalizationOutcome.RunNotFound, result.Outcome); + Assert.Equal(FinalizationStage.Load, result.Stage); + Assert.Null(result.Evaluation); + Assert.Empty(harness.Store.Creates); + } + + [Fact] + public async Task A_run_with_no_execution_status_is_a_structured_invalid_result_and_writes_nothing() + { + var harness = Harness.Create(); + var runId = harness.StartRun(); + await harness.AppendAttemptAsync(runId); + // Deliberately not completed. + + var result = await harness.FinalizeAsync(runId: runId); + + Assert.Equal(FinalizationOutcome.RunNotFinished, result.Outcome); + Assert.Equal(FinalizationStage.Load, result.Stage); + Assert.Empty(harness.Store.Creates); + Assert.Empty(harness.Store.Commits); + } + + // --------------------------------------------------------------------------------------------- + // Replay + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task Finalizing_the_same_run_twice_creates_one_record_at_revision_one_with_one_event() + { + var harness = await Harness.WithCompletedRunAsync(); + + var first = await harness.FinalizeAsync(); + var second = await harness.FinalizeAsync(finalizedAt: Now.AddMinutes(5)); // a later retry + + Assert.Equal(FinalizationOutcome.Validated, first.Outcome); + Assert.Equal(FinalizationOutcome.AlreadyFinalized, second.Outcome); + + // The second call reports the first call's outcome... + Assert.Equal(first.ExperienceId, second.ExperienceId); + Assert.Equal(ExperienceStatus.Validated, second.Status); + Assert.Equal(1, second.Revision); + Assert.True(second.IsDurable); + + // ...and writes nothing: no second record, no second initial event. + Assert.Single(harness.Store.Creates); + Assert.Single(harness.Store.Commits); + Assert.Equal(1, harness.Store.RevisionOf(first.ExperienceId!.Value)); + } + + [Fact] + public async Task A_quarantined_replay_still_names_the_stage_that_quarantined_it() + { + var harness = await Harness.WithCompletedRunAsync(); + var unverified = new[] { PassingEvidence(result: CheckResult.Fail) }; + + var first = await harness.FinalizeAsync(evidence: unverified); + Assert.Equal(FinalizationOutcome.Quarantined, first.Outcome); + + var second = await harness.FinalizeAsync(evidence: unverified, finalizedAt: Now.AddMinutes(5)); + + Assert.Equal(FinalizationOutcome.AlreadyFinalized, second.Outcome); + Assert.Equal(ExperienceStatus.Quarantined, second.Status); + + // A quarantine always names the stage that decided it, replayed or not. + Assert.Equal(FinalizationStage.Evaluate, second.Failure!.Stage); + } + + [Fact] + public async Task A_resumed_commit_on_a_quarantined_record_names_the_stage_that_quarantined_it() + { + var harness = await Harness.WithCompletedRunAsync(); + var unverified = new[] { PassingEvidence(result: CheckResult.Fail) }; + harness.Store.ThrowOnCommit = () => new ExperienceStoreException("commit unavailable"); + await harness.FinalizeAsync(evidence: unverified); + + harness.Store.ThrowOnCommit = null; + var resumed = await harness.FinalizeAsync(evidence: unverified, finalizedAt: Now.AddMinutes(5)); + + Assert.Equal(FinalizationOutcome.Quarantined, resumed.Outcome); + Assert.Equal(FinalizationStage.Evaluate, resumed.Failure!.Stage); + Assert.Single(harness.Store.Commits); + } + + [Fact] + public async Task The_record_and_initial_event_ids_derive_from_the_run() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), result.ExperienceId); + Assert.Equal(ExperienceFinalizationService.InitialEventIdFor(harness.RunId), result.Event!.EventId); + Assert.Equal(ExperienceFinalizationService.ReflectionIdFor(harness.RunId), result.Record!.Reflection!.ReflectionId); + + // Derivation is stable across calls and distinct per purpose and per run. + Assert.Equal(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), ExperienceFinalizationService.ExperienceIdFor(harness.RunId)); + Assert.NotEqual(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), ExperienceFinalizationService.InitialEventIdFor(harness.RunId)); + Assert.NotEqual(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), ExperienceFinalizationService.ExperienceIdFor(Guid.NewGuid())); + Assert.NotEqual(Guid.Empty, ExperienceFinalizationService.ExperienceIdFor(harness.RunId)); + } + + [Fact] + public async Task A_retry_after_a_failed_initial_commit_finishes_that_commit_rather_than_starting_over() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.ThrowOnCommit = () => new ExperienceStoreException("commit unavailable"); + + var failed = await harness.FinalizeAsync(); + Assert.Equal(FinalizationOutcome.Failed, failed.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, failed.Stage); + + // The record exists but is still a Candidate, so nothing can reuse it in the meantime. + Assert.Equal(ExperienceStatus.Candidate, harness.Store.StatusOf(failed.ExperienceId!.Value)); + + harness.Store.ThrowOnCommit = null; + var retried = await harness.FinalizeAsync(finalizedAt: Now.AddMinutes(5)); + + Assert.Equal(FinalizationOutcome.Validated, retried.Outcome); + Assert.Equal(1, retried.Revision); + Assert.Single(harness.Store.Creates); // still exactly one record + Assert.Single(harness.Store.Commits); // and exactly one initial event + } + + [Fact] + public async Task A_derived_record_id_taken_in_another_scope_is_a_failure_not_a_silent_success() + { + var harness = await Harness.WithCompletedRunAsync(); + var foreignScope = new Scope("tenant-9", "app-1", "project-1"); + harness.Store.Seed(TestRecord(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), foreignScope)); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, result.Stage); + Assert.False(result.IsDurable); + Assert.Empty(harness.Store.Commits); + } + + // --------------------------------------------------------------------------------------------- + // Store failures + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task A_store_failure_while_creating_is_a_failed_stage_and_the_run_stays_retrievable() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.ThrowOnCreate = () => new ExperienceStoreException("database unavailable"); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, result.Stage); + Assert.False(result.IsDurable); + Assert.Null(result.Record); + Assert.IsType(result.Failure!.Exception); + Assert.Empty(harness.Store.Commits); + + // The captured snapshot is still there for the host to retry against. + Assert.True(harness.Capture.TryGetRun(harness.RunId, out _)); + } + + [Fact] + public async Task A_store_failure_while_committing_is_never_reported_as_durable_success() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.ThrowOnCommit = () => new ExperienceStoreException("database unavailable"); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, result.Stage); + Assert.False(result.IsDurable); + Assert.Equal(0, result.Revision); + Assert.IsType(result.Failure!.Exception); + Assert.True(harness.Capture.TryGetRun(harness.RunId, out _)); + + // The record that now exists is reported, so the host can reconcile it rather than guess -- and + // it is still a Candidate, so nothing can reuse it. + Assert.Equal(ExperienceStatus.Candidate, result.Record!.Status); + Assert.Equal(ExperienceStatus.Candidate, harness.Store.StatusOf(result.ExperienceId!.Value)); + } + + [Fact] + public async Task A_store_that_reports_the_record_invalid_is_a_failed_stage_carrying_its_errors() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.CreateResult = new ExperienceRecordCreateResult( + ExperienceStoreOutcome.Invalid, + [new StoreValidationError("TaskId", "must not be empty or whitespace.")]); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, result.Stage); + Assert.Equal("TaskId", Assert.Single(result.Failure!.Errors).Path); + Assert.Empty(harness.Store.Commits); + } + + [Fact] + public async Task A_store_that_denies_the_create_is_reported_as_not_authorized() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.CreateResult = new ExperienceRecordCreateResult(ExperienceStoreOutcome.Denied, []); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.NotAuthorized, result.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, result.Stage); + Assert.Empty(harness.Store.Commits); + } + + [Fact] + public async Task A_lifecycle_commit_that_does_not_commit_is_a_failed_stage() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.CommitResult = new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.StaleRevision, 4, null, []); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, result.Stage); + Assert.Contains("StaleRevision", result.Failure!.Reason, StringComparison.Ordinal); + Assert.Equal(ExperienceStatus.Candidate, result.Record!.Status); + } + + [Fact] + public async Task A_refused_commit_on_a_quarantined_record_still_names_the_stage_that_quarantined_it() + { + var harness = await Harness.WithCompletedRunAsync(); + harness.Store.CommitResult = new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Invalid, 0, null, []); + + var result = await harness.FinalizeAsync(evidence: [PassingEvidence(result: CheckResult.Fail)]); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(ExperienceStatus.Candidate, result.Record!.Status); + + // The reason the record was going to be quarantined is not dropped in favour of the commit's + // own refusal -- the host still learns why no lesson was recorded. + Assert.Equal(FinalizationStage.Evaluate, result.Failure!.Stage); + } + + [Fact] + public async Task A_record_finalized_concurrently_converges_on_AlreadyFinalized_rather_than_failing_forever() + { + var harness = await Harness.WithCompletedRunAsync(); + + // The create lands, then someone else commits the initial event before this call's own commit, + // so the store reports a stale revision against a record that is now durably finalized. + harness.Store.BeforeCommit = store => + { + store.BeforeCommit = null; + store.ForceFinalize(ExperienceFinalizationService.ExperienceIdFor(harness.RunId), ExperienceStatus.Validated); + }; + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.AlreadyFinalized, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(ExperienceStatus.Validated, result.Status); + Assert.Equal(1, result.Revision); + } + + [Fact] + public async Task An_exception_that_is_not_an_ExperienceStoreException_is_still_a_structured_failed_stage() + { + // "Every stage failure comes back as a structured result" is not limited to the store's own + // documented exception type. + var creating = await Harness.WithCompletedRunAsync(); + creating.Store.ThrowOnCreate = () => new ObjectDisposedException("data source"); + var createResult = await creating.FinalizeAsync(); + Assert.Equal(FinalizationOutcome.Failed, createResult.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, createResult.Stage); + Assert.IsType(createResult.Failure!.Exception); + + var committing = await Harness.WithCompletedRunAsync(); + committing.Store.ThrowOnCommit = () => new ObjectDisposedException("data source"); + var commitResult = await committing.FinalizeAsync(); + Assert.Equal(FinalizationOutcome.Failed, commitResult.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, commitResult.Stage); + Assert.IsType(commitResult.Failure!.Exception); + + var loading = Harness.Create(captureService: new ThrowingCaptureService()); + var loadResult = await loading.Service.FinalizeAsync(loading.Request(runId: Guid.NewGuid()), CancellationToken.None); + Assert.Equal(FinalizationOutcome.Failed, loadResult.Outcome); + Assert.Equal(FinalizationStage.Load, loadResult.Stage); + Assert.IsType(loadResult.Failure!.Exception); + } + + // --------------------------------------------------------------------------------------------- + // Evaluation ownership, arguments, cancellation + // --------------------------------------------------------------------------------------------- + + [Fact] + public async Task Malformed_verification_inputs_end_the_evaluate_stage_rather_than_throwing() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(requiredChecks: [new RequiredCheck("tests"), new RequiredCheck("tests")]); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.Evaluate, result.Stage); + Assert.IsType(result.Failure!.Exception); + Assert.Empty(harness.Store.Creates); + } + + [Fact] + public async Task Evidence_of_a_kind_the_required_check_does_not_expect_never_validates_the_record() + { + var harness = await Harness.WithCompletedRunAsync(); + var approval = PassingEvidence() with { Kind = "HumanApproval" }; + + var result = await harness.FinalizeAsync( + requiredChecks: [new RequiredCheck("tests", "TestResult")], + evidence: [approval]); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Equal(TaskVerificationStatus.Unknown, result.Record!.Outcome.Status); + } + + [Fact] + public async Task Evidence_from_another_round_never_finalizes_this_run_as_validated() + { + var harness = await Harness.WithCompletedRunAsync(); + var otherRound = PassingEvidence() with { VerificationRoundId = Guid.NewGuid() }; + + var result = await harness.FinalizeAsync(evidence: [otherRound]); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + } + + [Fact] + public async Task A_round_the_host_closed_for_another_artifact_revision_is_stale_and_never_validates() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(closedRound: new ClosedVerificationRound(Round.RoundId, "rev-2")); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Equal(TaskVerificationStatus.Unknown, result.Evaluation!.Outcome.Status); + } + + [Fact] + public async Task Null_arguments_throw_ArgumentNullException() + { + var harness = await Harness.WithCompletedRunAsync(); + + Assert.Throws(() => new ExperienceFinalizationService(null!, new DefaultExperienceReflector(), harness.Store, new ExperienceLifecycleService(harness.Store))); + Assert.Throws(() => new ExperienceFinalizationService(harness.Capture, null!, harness.Store, new ExperienceLifecycleService(harness.Store))); + Assert.Throws(() => new ExperienceFinalizationService(harness.Capture, new DefaultExperienceReflector(), null!, new ExperienceLifecycleService(harness.Store))); + Assert.Throws(() => new ExperienceFinalizationService(harness.Capture, new DefaultExperienceReflector(), harness.Store, null!)); + + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(null!, CancellationToken.None)); + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { Authorization = null! }, CancellationToken.None)); + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { RequiredChecks = null! }, CancellationToken.None)); + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { Evidence = null! }, CancellationToken.None)); + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { StorageDecision = null! }, CancellationToken.None)); + } + + [Fact] + public async Task A_malformed_request_is_rejected_before_anything_is_stored() + { + var harness = await Harness.WithCompletedRunAsync(); + + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { RunId = Guid.Empty }, CancellationToken.None)); + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { CurrentArtifactRevision = " " }, CancellationToken.None)); + + // An unset FinalizedAt would create a record whose every commit -- including every retry -- the + // store then rejects as Invalid forever, because an unset OccurredAt is not a valid event. + await Assert.ThrowsAsync(() => harness.Service.FinalizeAsync(harness.Request() with { FinalizedAt = default }, CancellationToken.None)); + + Assert.Empty(harness.Store.Creates); + } + + [Fact] + public async Task Cancellation_propagates_rather_than_becoming_a_structured_result() + { + var harness = await Harness.WithCompletedRunAsync(); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => harness.Service.FinalizeAsync(harness.Request(), cts.Token)); + Assert.Empty(harness.Store.Creates); + } + + [Fact] + public async Task A_reflector_that_cancels_for_its_own_reasons_propagates_rather_than_quarantining_silently() + { + // One rule for every stage: an OperationCanceledException always propagates, whoever raised it. + var harness = await Harness.WithCompletedRunAsync(reflector: new CancellingReflector()); + + await Assert.ThrowsAnyAsync(() => harness.Service.FinalizeAsync(harness.Request(), CancellationToken.None)); + Assert.Empty(harness.Store.Creates); + } + + // --------------------------------------------------------------------------------------------- + // Fixtures + // --------------------------------------------------------------------------------------------- + + private static ExperienceRecord TestRecord(Guid experienceId, Scope scope) => new( + ExperienceId: experienceId, + SourceRunId: Guid.NewGuid(), + Scope: scope, + TaskId: "task-1", + TaskSummary: null, + Attempts: [], + Outcome: new Outcome(TaskVerificationStatus.Unknown, [], null, Now), + CompletionScore: 0, + Reflection: null, + Environment: new EnvironmentFingerprint("host", "10.0.0", "linux-x64", null, new Dictionary()), + Provenance: new Provenance("tests", null, Now, null), + Status: ExperienceStatus.Candidate, + ReuseConfidence: 0, + SupportingValidations: 0, + Contradictions: 0, + Revision: 0, + CreatedAt: Now, + UpdatedAt: Now); + + /// Real capture plus a real lifecycle service over an in-memory fake store. + private sealed class Harness + { + public required InMemoryExperienceCaptureService Capture { get; init; } + + public required FakeStore Store { get; init; } + + public required ExperienceFinalizationService Service { get; init; } + + public Guid RunId { get; private set; } + + public static Harness Create(IExperienceReflector? reflector = null, IExperienceCaptureService? captureService = null) + { + var capture = new InMemoryExperienceCaptureService( + new DefaultSanitizer(PermissiveOptions), + new CaptureLimits(50, 50, 10_000, 10_000)); + var store = new FakeStore(); + + return new Harness + { + Capture = capture, + Store = store, + Service = new ExperienceFinalizationService( + captureService ?? capture, + reflector ?? new DefaultExperienceReflector(), + store, + new ExperienceLifecycleService(store)), + }; + } + + public static async Task WithCompletedRunAsync(IExperienceReflector? reflector = null) + { + var harness = Create(reflector); + harness.RunId = harness.StartRun(); + await harness.AppendAttemptAsync(harness.RunId); + var completed = await harness.Capture.CompleteRunAsync(harness.RunId, Guid.NewGuid(), RunExecutionStatus.Completed, Now.AddMinutes(1)); + Assert.Equal(CompleteRunOutcome.Recorded, completed.Outcome); + return harness; + } + + public Guid StartRun() + { + var runId = Guid.NewGuid(); + RunId = runId; + var started = Capture.StartRun( + runId, + taskId: "task-1", + taskDescription: "a test task", + scope: TestScope, + environment: new EnvironmentFingerprint("host-1", "net10.0", "test-os", null, new Dictionary()), + provenance: new Provenance("unit-tests", "1.0.0", Now, null), + startedAt: Now); + Assert.Equal(StartRunOutcome.Started, started.Outcome); + return runId; + } + + public async Task AppendAttemptAsync(Guid runId) + { + var appended = await Capture.AppendAttemptAsync( + runId, + new AppendAttemptRequest(Guid.NewGuid(), Now, TimeSpan.FromSeconds(1), [], "done", null)); + Assert.Equal(AppendAttemptOutcome.Recorded, appended.Outcome); + } + + public ExperienceRun CapturedRun() + { + Assert.True(Capture.TryGetRun(RunId, out var run)); + return run; + } + + public FinalizeExperienceRequest Request( + Guid? runId = null, + AuthorizationContext? authorization = null, + ClosedVerificationRound? closedRound = null, + bool noRound = false, + IReadOnlyList? requiredChecks = null, + IReadOnlyList? evidence = null, + StorageDecision? decision = null, + DateTimeOffset? finalizedAt = null) => new( + RunId: runId ?? RunId, + Authorization: authorization ?? Authorization, + ClosedRound: noRound ? null : closedRound ?? Round, + RequiredChecks: requiredChecks ?? [new RequiredCheck("tests")], + Evidence: evidence ?? [PassingEvidence()], + CurrentArtifactRevision: ArtifactRevision, + StorageDecision: decision ?? StorageDecision.Permit, + FinalizedAt: finalizedAt ?? Now.AddMinutes(2)); + + public Task FinalizeAsync( + Guid? runId = null, + AuthorizationContext? authorization = null, + ClosedVerificationRound? closedRound = null, + bool noRound = false, + IReadOnlyList? requiredChecks = null, + IReadOnlyList? evidence = null, + StorageDecision? decision = null, + DateTimeOffset? finalizedAt = null) => + Service.FinalizeAsync( + Request(runId, authorization, closedRound, noRound, requiredChecks, evidence, decision, finalizedAt), + CancellationToken.None); + } + + private sealed class CountingReflector : IExperienceReflector + { + private readonly DefaultExperienceReflector _inner = new(); + + public int Calls { get; private set; } + + public Task ReflectAsync(ReflectionRequest request, CancellationToken cancellationToken = default) + { + Calls++; + return _inner.ReflectAsync(request, cancellationToken); + } + } + + private sealed class ThrowingReflector : IExperienceReflector + { + public Task ReflectAsync(ReflectionRequest request, CancellationToken cancellationToken = default) => + throw new InvalidOperationException("the reflector template failed"); + } + + /// A lenient custom reflector that declines rather than throwing. + private sealed class NullReturningReflector : IExperienceReflector + { + public Task ReflectAsync(ReflectionRequest request, CancellationToken cancellationToken = default) => + Task.FromResult(null!); + } + + /// A reflector that cancels on a token of its own, not the caller's. + private sealed class CancellingReflector : IExperienceReflector + { + public Task ReflectAsync(ReflectionRequest request, CancellationToken cancellationToken = default) => + throw new OperationCanceledException("the reflector's own budget expired"); + } + + /// A capture service whose snapshot read fails with something other than a store exception. + private sealed class ThrowingCaptureService : IExperienceCaptureService + { + public StartRunResult StartRun(Guid runId, string taskId, string? taskDescription, Scope scope, EnvironmentFingerprint environment, Provenance provenance, DateTimeOffset startedAt) => + throw new InvalidOperationException("Finalization must not start runs."); + + public Task AppendAttemptAsync(Guid runId, AppendAttemptRequest request, CancellationToken cancellationToken = default) => + throw new InvalidOperationException("Finalization must not append attempts."); + + public Task CompleteRunAsync(Guid runId, Guid completionEventId, RunExecutionStatus executionStatus, DateTimeOffset endedAt, CancellationToken cancellationToken = default) => + throw new InvalidOperationException("Finalization must not complete runs."); + + public bool TryGetRun(Guid runId, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out ExperienceRun? run) => + throw new InvalidOperationException("the capture snapshot store is disposed"); + } + + /// + /// A minimal in-memory : create-only inserts, scoped reads, + /// and event-ID-idempotent, revision-checked lifecycle commits -- just enough of the port's real + /// contract that replay and failure behaviour are exercised rather than assumed. Query and history + /// are out of this story's scope and fail loudly if finalization ever calls them. + /// + private sealed class FakeStore : IExperienceRecordStore + { + private readonly Dictionary _records = []; + private readonly Dictionary _events = []; + + public List Creates { get; } = []; + + public List Commits { get; } = []; + + public Func? ThrowOnCreate { get; set; } + + public Func? ThrowOnCommit { get; set; } + + public ExperienceRecordCreateResult? CreateResult { get; set; } + + public ExperienceLifecycleCommitResult? CommitResult { get; set; } + + /// Runs just before a commit is applied, so a test can simulate a concurrent writer. + public Action? BeforeCommit { get; set; } + + public void Seed(ExperienceRecord record) => _records[record.ExperienceId] = record; + + public long RevisionOf(Guid experienceId) => _records[experienceId].Revision; + + public ExperienceStatus StatusOf(Guid experienceId) => _records[experienceId].Status; + + /// Applies someone else's initial commit to a stored record, exactly as a racing caller would. + public void ForceFinalize(Guid experienceId, ExperienceStatus status) + { + var record = _records[experienceId]; + _records[experienceId] = record with { Status = status, Revision = record.Revision + 1 }; + _events[Guid.NewGuid()] = (Event(experienceId, record.Status, status, record.Revision), record.Revision + 1); + } + + private static LifecycleEvent Event(Guid experienceId, ExperienceStatus? prior, ExperienceStatus current, long expectedRevision) => + new(Guid.NewGuid(), experienceId, prior, current, "concurrent finalization", "another host", Now, expectedRevision); + + public Task CreateAsync(AuthorizationContext authorization, ExperienceRecord record, CancellationToken cancellationToken) + { + Assert.NotNull(authorization); + cancellationToken.ThrowIfCancellationRequested(); + + if (ThrowOnCreate is not null) + { + throw ThrowOnCreate(); + } + + if (CreateResult is not null) + { + return Task.FromResult(CreateResult); + } + + if (_records.ContainsKey(record.ExperienceId)) + { + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Conflict, [])); + } + + if (!authorization.Permits(record.Scope)) + { + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Denied, [])); + } + + _records[record.ExperienceId] = record; + Creates.Add(record); + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Created, [])); + } + + public Task GetAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) + { + Assert.NotNull(authorization); + cancellationToken.ThrowIfCancellationRequested(); + + // A record in another scope is indistinguishable from a missing one. + return Task.FromResult(_records.TryGetValue(experienceId, out var record) && record.Scope == scope + ? new ExperienceRecordGetResult(ExperienceStoreOutcome.Found, record, []) + : new ExperienceRecordGetResult(ExperienceStoreOutcome.NotFound, null, [])); + } + + public Task CommitLifecycleEventAsync( + AuthorizationContext authorization, + Scope scope, + LifecycleEvent lifecycleEvent, + CancellationToken cancellationToken) + { + Assert.NotNull(authorization); + cancellationToken.ThrowIfCancellationRequested(); + + if (ThrowOnCommit is not null) + { + throw ThrowOnCommit(); + } + + BeforeCommit?.Invoke(this); + + if (CommitResult is not null) + { + return Task.FromResult(CommitResult); + } + + if (_events.TryGetValue(lifecycleEvent.EventId, out var stored)) + { + // Identical replay reports the original commit and writes nothing; any differing field conflicts. + return Task.FromResult(stored.Event == lifecycleEvent + ? new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Committed, stored.AppliedRevision, null, []) + : new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Conflict, 0, null, [])); + } + + if (!_records.TryGetValue(lifecycleEvent.ExperienceRecordId, out var record) || record.Scope != scope) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.NotFound, 0, null, [])); + } + + if (record.Revision != lifecycleEvent.ExpectedRevision) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.StaleRevision, record.Revision, null, [])); + } + + if (lifecycleEvent.PriorStatus is { } prior && record.Status != prior) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.StatusMismatch, record.Revision, record.Status, [])); + } + + var applied = lifecycleEvent.ExpectedRevision + 1; + _records[record.ExperienceId] = record with + { + Status = lifecycleEvent.CurrentStatus, + Revision = applied, + UpdatedAt = lifecycleEvent.OccurredAt, + }; + _events[lifecycleEvent.EventId] = (lifecycleEvent, applied); + Commits.Add(lifecycleEvent); + + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Committed, applied, null, [])); + } + + public Task QueryAsync(AuthorizationContext authorization, ExperienceRecordQuery query, CancellationToken cancellationToken) => + throw new InvalidOperationException("Finalization must not query records."); + + public Task GetHistoryAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) => + throw new InvalidOperationException("Finalization must not read history."); + } +} diff --git a/tests/AgentExperience.Core.Tests/ExperienceIndexingServiceTests.cs b/tests/AgentExperience.Core.Tests/ExperienceIndexingServiceTests.cs new file mode 100644 index 0000000..181dc40 --- /dev/null +++ b/tests/AgentExperience.Core.Tests/ExperienceIndexingServiceTests.cs @@ -0,0 +1,647 @@ +using AgentExperience.Core.Indexing; +using AgentExperience.Core.Retrieval; + +namespace AgentExperience.Core.Tests; + +/// +/// Covers against a deterministic fake generator and an +/// in-memory index that enforces the real conditional-write contract: one test per row of the story's +/// I/O and edge-case matrix that this service owns -- indexing after a commit, a provider that is +/// down, a stale write, a deleted record, and re-indexing both unchanged and changed records. +/// +public class ExperienceIndexingServiceTests +{ + private static readonly DateTimeOffset Now = new(2026, 9, 21, 12, 0, 0, TimeSpan.Zero); + + private static readonly Scope RequestScope = new("tenant-1", "app-1", "project-1"); + + private static readonly AuthorizationContext Authorization = new("tenant-1", "host-principal", ["experience:write"], Now); + + private const string Summary = "refund-ticket Resolve a refund ticket Retry the refund after releasing the lock"; + + // ---------------------------------------------------------------- matrix: index after commit + + [Fact] + public async Task Indexing_a_committed_record_stores_the_vector_with_its_model_dimension_hash_and_revision() + { + var id = Id(1); + var index = new FakeEmbeddingIndex { Records = { [id] = new(1, Summary) } }; + var generator = new FakeEmbeddingGenerator(); + var service = new ExperienceIndexingService(index, generator); + + var result = await service.IndexAsync(Authorization, RequestScope, id); + + Assert.Equal(ExperienceIndexingOutcome.Indexed, result.Outcome); + Assert.True(result.IsIndexed); + Assert.Null(result.Failure); + + var (descriptor, vector) = index.Stored[id]; + Assert.Equal("fake-embed-v1", descriptor.ModelId); + Assert.Equal(4, descriptor.Dimension); + Assert.Equal(ExperienceEmbeddingDescriptor.ComputeContentHash("fake-embed-v1", Summary), descriptor.ContentHash); + Assert.Equal(1, descriptor.SourceRevision); + Assert.Equal(FakeEmbeddingGenerator.VectorFor(Summary, 4).ToArray(), vector.ToArray()); + Assert.Equal(descriptor, result.Descriptor); + } + + [Fact] + public async Task Only_the_sanitized_retrieval_summary_is_ever_handed_to_the_provider() + { + // The service embeds what the index listed, not a record the caller happened to be holding: + // attempts, tool calls, evidence, and environment never reach a provider because they are + // never in the summary to begin with. + var id = Id(1); + var index = new FakeEmbeddingIndex { Records = { [id] = new(1, Summary) } }; + var generator = new FakeEmbeddingGenerator(); + + await new ExperienceIndexingService(index, generator).IndexAsync(Authorization, RequestScope, id); + + Assert.Equal([Summary], generator.Requests); + } + + // ---------------------------------------------------------------- matrix: provider down + + [Fact] + public async Task A_provider_that_throws_leaves_the_record_untouched_and_is_reported_as_retryable() + { + var id = Id(1); + var index = new FakeEmbeddingIndex { Records = { [id] = new(1, Summary) } }; + var service = new ExperienceIndexingService( + index, + new FakeEmbeddingGenerator { Throws = FakeEmbeddingGenerator.ThrownException }); + + var result = await service.IndexAsync(Authorization, RequestScope, id); + + Assert.Equal(ExperienceIndexingOutcome.ProviderFailed, result.Outcome); + Assert.True(result.IsRetryable); + Assert.False(result.IsIndexed); + Assert.Same(FakeEmbeddingGenerator.ThrownException, result.Failure!.Exception); + Assert.Empty(index.Stored); + Assert.Empty(index.Writes); + } + + [Fact] + public async Task A_provider_that_returns_the_wrong_width_is_a_provider_failure_and_nothing_is_written() + { + // A descriptor that disagreed with its own vector would make every later comparison unsound, + // so it never reaches the index at all. + var id = Id(1); + var index = new FakeEmbeddingIndex { Records = { [id] = new(1, Summary) } }; + var service = new ExperienceIndexingService(index, new FakeEmbeddingGenerator { ReturnDimension = 3 }); + + var result = await service.IndexAsync(Authorization, RequestScope, id); + + Assert.Equal(ExperienceIndexingOutcome.ProviderFailed, result.Outcome); + Assert.Empty(index.Writes); + Assert.Empty(index.Stored); + } + + [Fact] + public async Task An_index_that_throws_is_reported_as_retryable_rather_than_propagating() + { + var id = Id(1); + var index = new FakeEmbeddingIndex + { + Records = { [id] = new(1, Summary) }, + WriteThrows = FakeEmbeddingIndex.ThrownException, + }; + + var result = await new ExperienceIndexingService(index, new FakeEmbeddingGenerator()) + .IndexAsync(Authorization, RequestScope, id); + + Assert.Equal(ExperienceIndexingOutcome.IndexFailed, result.Outcome); + Assert.True(result.IsRetryable); + Assert.Same(FakeEmbeddingIndex.ThrownException, result.Failure!.Exception); + } + + // ---------------------------------------------------------------- matrix: stale write + + [Fact] + public async Task A_record_that_moves_to_a_new_revision_before_the_write_lands_rejects_it_and_keeps_the_stored_vector() + { + var id = Id(1); + FakeEmbeddingIndex? index = null; + index = new FakeEmbeddingIndex + { + Records = { [id] = new(1, Summary) }, + // The record moves to revision 2 while this very write is in flight. + BeforeWrite = _ => index!.Records[id] = new(2, Summary), + }; + + // Seed a vector from revision 1, so the test can prove the in-flight write did not replace it. + var first = new ExperienceEmbeddingDescriptor("fake-embed-v1", 4, "seeded", 1); + index.Stored[id] = (first, FakeEmbeddingGenerator.VectorFor("seeded", 4)); + + var result = await new ExperienceIndexingService(index, new FakeEmbeddingGenerator()) + .IndexAsync(Authorization, RequestScope, id); + + Assert.Equal(ExperienceIndexingOutcome.Stale, result.Outcome); + Assert.True(result.IsRetryable); + Assert.Contains("revision 2", result.Failure!.Reason, StringComparison.Ordinal); + Assert.Equal(first, index.Stored[id].Descriptor); + Assert.Equal(1, index.Stored[id].Descriptor.SourceRevision); + } + + // ---------------------------------------------------------------- matrix: deleted record + + [Fact] + public async Task A_record_deleted_before_the_write_lands_is_never_recreated() + { + var id = Id(1); + FakeEmbeddingIndex? index = null; + index = new FakeEmbeddingIndex + { + Records = { [id] = new(1, Summary) }, + BeforeWrite = _ => index!.Records.Remove(id), + }; + + var result = await new ExperienceIndexingService(index, new FakeEmbeddingGenerator()) + .IndexAsync(Authorization, RequestScope, id); + + Assert.Equal(ExperienceIndexingOutcome.Missing, result.Outcome); + Assert.False(result.IsRetryable); + Assert.Empty(index.Stored); + } + + [Fact] + public async Task A_record_that_never_existed_is_Missing_and_no_provider_is_called() + { + var index = new FakeEmbeddingIndex(); + var generator = new FakeEmbeddingGenerator(); + + var result = await new ExperienceIndexingService(index, generator).IndexAsync(Authorization, RequestScope, Id(9)); + + Assert.Equal(ExperienceIndexingOutcome.Missing, result.Outcome); + Assert.Empty(generator.Requests); + Assert.Empty(index.Writes); + } + + // ---------------------------------------------------------------- matrix: reindex unchanged + + [Fact] + public async Task Reindexing_an_unchanged_record_calls_no_provider_writes_nothing_and_reports_it_skipped() + { + var id = Id(1); + var index = new FakeEmbeddingIndex { Records = { [id] = new(1, Summary) } }; + var generator = new FakeEmbeddingGenerator(); + var service = new ExperienceIndexingService(index, generator); + + var first = await service.ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope)); + Assert.Equal(1, first.Indexed); + + var writesAfterFirst = index.Writes.Count; + var second = await service.ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope)); + + Assert.Equal(ExperienceReindexOutcome.Completed, second.Outcome); + Assert.Equal(1, second.Examined); + Assert.Equal(0, second.Indexed); + Assert.Equal(1, second.Skipped); + Assert.Equal(ExperienceIndexingOutcome.Skipped, Assert.Single(second.Records).Outcome); + + // The whole point: an unchanged content hash under the same model costs exactly one read. + Assert.Single(generator.Requests); + Assert.Equal(writesAfterFirst, index.Writes.Count); + } + + [Fact] + public async Task A_vector_stored_under_a_different_model_is_re_embedded_rather_than_skipped() + { + // The model ID is inside the content hash on purpose: the same text under a different model + // is a different, incomparable vector and must never look "unchanged". + var id = Id(1); + var index = new FakeEmbeddingIndex { Records = { [id] = new(1, Summary) } }; + index.Stored[id] = ( + new ExperienceEmbeddingDescriptor("other-model", 4, ExperienceEmbeddingDescriptor.ComputeContentHash("other-model", Summary), 1), + FakeEmbeddingGenerator.VectorFor(Summary, 4)); + + var generator = new FakeEmbeddingGenerator(); + var result = await new ExperienceIndexingService(index, generator).IndexAsync(Authorization, RequestScope, id); + + Assert.Equal(ExperienceIndexingOutcome.Indexed, result.Outcome); + Assert.Single(generator.Requests); + Assert.Equal("fake-embed-v1", index.Stored[id].Descriptor.ModelId); + } + + // ---------------------------------------------------------------- matrix: reindex changed + + [Fact] + public async Task A_changed_summary_is_re_embedded_and_rewritten_once_and_the_repeat_is_idempotent() + { + var id = Id(1); + var index = new FakeEmbeddingIndex { Records = { [id] = new(1, Summary) } }; + var generator = new FakeEmbeddingGenerator(); + var service = new ExperienceIndexingService(index, generator); + + await service.ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope)); + var firstHash = index.Stored[id].Descriptor.ContentHash; + + const string Changed = Summary + " and confirm the ledger entry"; + index.Records[id] = new(1, Changed); + + var rewritten = await service.ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope)); + Assert.Equal(1, rewritten.Indexed); + Assert.Equal(0, rewritten.Skipped); + Assert.NotEqual(firstHash, index.Stored[id].Descriptor.ContentHash); + Assert.Equal(FakeEmbeddingGenerator.VectorFor(Changed, 4).ToArray(), index.Stored[id].Vector.ToArray()); + Assert.Equal(2, generator.Requests.Count); + + var repeat = await service.ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope)); + Assert.Equal(0, repeat.Indexed); + Assert.Equal(1, repeat.Skipped); + Assert.Equal(2, generator.Requests.Count); + } + + [Fact] + public async Task Whitespace_only_differences_in_the_summary_never_cause_a_re_embedding() + { + // The summary is normalized before it is hashed, so re-indentation is not a content change. + var id = Id(1); + var index = new FakeEmbeddingIndex { Records = { [id] = new(1, "refund ticket lesson") } }; + var generator = new FakeEmbeddingGenerator(); + var service = new ExperienceIndexingService(index, generator); + + await service.ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope)); + + Assert.Equal( + ExperienceEmbeddingDescriptor.ComputeContentHash("fake-embed-v1", ExperienceRetrievalSummary.For("refund", "ticket", "lesson")), + index.Stored[id].Descriptor.ContentHash); + Assert.Equal("refund ticket lesson", ExperienceRetrievalSummary.For(" refund\n\t", " ticket ", "lesson")); + } + + // ---------------------------------------------------------------- scope, authorization, bounds + + [Fact] + public async Task A_scope_outside_the_authorization_is_denied_before_anything_is_read_or_embedded() + { + var id = Id(1); + var index = new FakeEmbeddingIndex { Records = { [id] = new(1, Summary) } }; + var generator = new FakeEmbeddingGenerator(); + var elsewhere = new AuthorizationContext("tenant-1", "p", [], Now, ProjectId: "elsewhere"); + + var result = await new ExperienceIndexingService(index, generator).IndexAsync(elsewhere, RequestScope, id); + + Assert.Equal(ExperienceIndexingOutcome.Denied, result.Outcome); + Assert.False(result.IsRetryable); + Assert.Empty(generator.Requests); + Assert.Empty(index.Writes); + } + + [Fact] + public async Task A_reindex_pass_is_scoped_and_bounded_and_says_so_in_the_scan_it_issues() + { + var index = new FakeEmbeddingIndex(); + for (var n = 1; n <= 5; n++) + { + index.Records[Id(n)] = new(1, $"summary {n}"); + } + + var service = new ExperienceIndexingService(index, new FakeEmbeddingGenerator()); + var result = await service.ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope, Limit: 3)); + + var scan = Assert.Single(index.Scans); + Assert.Equal(RequestScope, scan.Scope); + Assert.Equal("fake-embed-v1", scan.ModelId); + Assert.Equal(3, scan.Limit); + Assert.Null(scan.ExperienceIds); + Assert.Equal(3, result.Examined); + Assert.Equal(3, result.Indexed); + } + + [Fact] + public async Task A_reindex_narrowed_to_specific_records_passes_exactly_those_ids_through() + { + var index = new FakeEmbeddingIndex + { + Records = { [Id(1)] = new(1, "one"), [Id(2)] = new(1, "two"), [Id(3)] = new(1, "three") }, + }; + + var service = new ExperienceIndexingService(index, new FakeEmbeddingGenerator()); + var result = await service.ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope, [Id(1), Id(3)])); + + Assert.Equal([Id(1), Id(3)], Assert.Single(index.Scans).ExperienceIds); + Assert.Equal(2, result.Examined); + Assert.Equal([Id(1), Id(3)], result.Records.Select(record => record.ExperienceId)); + } + + [Fact] + public async Task A_pass_whose_scan_fails_examines_nothing_and_reports_the_failure() + { + var index = new FakeEmbeddingIndex { ScanThrows = FakeEmbeddingIndex.ThrownException }; + + var result = await new ExperienceIndexingService(index, new FakeEmbeddingGenerator()) + .ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope)); + + Assert.Equal(ExperienceReindexOutcome.Failed, result.Outcome); + Assert.Equal(0, result.Examined); + Assert.Empty(result.Records); + Assert.Same(FakeEmbeddingIndex.ThrownException, result.Failure!.Exception); + } + + [Fact] + public async Task A_pass_tallies_every_per_record_outcome_it_saw() + { + var index = new FakeEmbeddingIndex + { + Records = { [Id(1)] = new(1, "one"), [Id(2)] = new(1, "two") }, + }; + + // Id(2) is already indexed under this model with exactly this text, so it is skipped. + index.Stored[Id(2)] = ( + new ExperienceEmbeddingDescriptor("fake-embed-v1", 4, ExperienceEmbeddingDescriptor.ComputeContentHash("fake-embed-v1", "two"), 1), + FakeEmbeddingGenerator.VectorFor("two", 4)); + + var result = await new ExperienceIndexingService(index, new FakeEmbeddingGenerator()) + .ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope)); + + Assert.Equal(ExperienceReindexOutcome.Completed, result.Outcome); + Assert.Equal(2, result.Examined); + Assert.Equal(1, result.Indexed); + Assert.Equal(1, result.Skipped); + Assert.Equal(0, result.Rejected); + Assert.Equal(0, result.Failed); + } + + // ---------------------------------------------------------------- eligibility and paging + + [Fact] + public async Task A_record_a_search_could_never_return_is_never_listed_and_never_embedded() + { + var index = new FakeEmbeddingIndex + { + Records = + { + [Id(1)] = new(1, "eligible"), + [Id(2)] = new(1, "quarantined", ExperienceStatus.Quarantined), + [Id(3)] = new(1, "revoked", ExperienceStatus.Revoked), + [Id(4)] = new(1, "low confidence", ExperienceStatus.Validated, ReuseConfidence: 0.1), + }, + }; + + var generator = new FakeEmbeddingGenerator(); + var pass = await new ExperienceIndexingService(index, generator) + .ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope)); + + Assert.Equal(1, pass.Examined); + Assert.Equal([Id(1)], pass.Records.Select(record => record.ExperienceId)); + Assert.Equal(["eligible"], generator.Requests); + + var scan = Assert.Single(index.Scans); + Assert.Equal([ExperienceStatus.Validated, ExperienceStatus.Reinforced], scan.EligibleStatuses); + Assert.Equal(RetrievalPolicy.DefaultMinimumConfidence, scan.MinimumConfidence); + } + + [Fact] + public void The_indexable_rule_is_exactly_the_one_a_vector_search_applies() + { + var service = new ExperienceIndexingService(new FakeEmbeddingIndex(), new FakeEmbeddingGenerator()); + + Assert.Equal(ExperienceRetrievalService.EligibleStatuses, ExperienceIndexingService.IndexableStatuses); + Assert.Equal(RetrievalPolicy.DefaultMinimumConfidence, service.MinimumConfidence); + Assert.True(service.IsIndexable(ExperienceStatus.Validated, 0.5)); + Assert.True(service.IsIndexable(ExperienceStatus.Reinforced, 1d)); + Assert.False(service.IsIndexable(ExperienceStatus.Validated, 0.49)); + Assert.False(service.IsIndexable(ExperienceStatus.Quarantined, 1d)); + Assert.False(service.IsIndexable(ExperienceStatus.Revoked, 1d)); + + // And it tracks whatever floor retrieval was configured with, not a second copy of the default. + var strict = new ExperienceIndexingService( + new FakeEmbeddingIndex(), + new FakeEmbeddingGenerator(), + RetrievalPolicy.Default with { MinimumConfidence = 0.9 }); + Assert.False(strict.IsIndexable(ExperienceStatus.Validated, 0.8)); + } + + [Fact] + public async Task A_scope_larger_than_one_page_is_walked_to_the_end_by_the_cursor() + { + // Without a cursor every pass re-reads the same first page, so a scope larger than the limit is + // never fully indexed however often the pass runs. + var index = new FakeEmbeddingIndex(); + for (var n = 1; n <= 5; n++) + { + index.Records[Id(n)] = new(1, $"summary {n}"); + } + + var service = new ExperienceIndexingService(index, new FakeEmbeddingGenerator()); + + var first = await service.ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope, Limit: 2)); + Assert.Equal([Id(1), Id(2)], first.Records.Select(record => record.ExperienceId)); + Assert.Equal(Id(2), first.LastExaminedId); + + var second = await service.ReindexAsync( + Authorization, + new ReindexExperienceRequest(RequestScope, Limit: 2, StartAfterId: first.LastExaminedId)); + Assert.Equal([Id(3), Id(4)], second.Records.Select(record => record.ExperienceId)); + + var third = await service.ReindexAsync( + Authorization, + new ReindexExperienceRequest(RequestScope, Limit: 2, StartAfterId: second.LastExaminedId)); + Assert.Equal([Id(5)], third.Records.Select(record => record.ExperienceId)); + + var exhausted = await service.ReindexAsync( + Authorization, + new ReindexExperienceRequest(RequestScope, Limit: 2, StartAfterId: third.LastExaminedId)); + Assert.Equal(0, exhausted.Examined); + Assert.Null(exhausted.LastExaminedId); + + Assert.Equal(5, index.Stored.Count); + } + + // ---------------------------------------------------------------- malformed provider answers + + [Fact] + public async Task A_vector_with_a_non_finite_component_is_a_provider_failure_and_never_reaches_the_index() + { + // Right width, so every later check would pass -- and then the database would reject it, as a + // failure reported retryable, which is a retry loop that never terminates. + var id = Id(1); + var index = new FakeEmbeddingIndex { Records = { [id] = new(1, Summary) } }; + var service = new ExperienceIndexingService(index, new FakeEmbeddingGenerator { ReturnNonFinite = true }); + + var result = await service.IndexAsync(Authorization, RequestScope, id); + + Assert.Equal(ExperienceIndexingOutcome.ProviderFailed, result.Outcome); + Assert.Contains("non-finite", result.Failure!.Reason, StringComparison.Ordinal); + Assert.Empty(index.Writes); + Assert.Empty(index.Stored); + } + + [Fact] + public async Task A_provider_that_cancels_for_its_own_reasons_fails_one_record_not_the_whole_pass() + { + // A client-side request timeout is an OperationCanceledException the caller never asked for. + // Letting it escape would abandon a whole pass with no per-record results at all. + var index = new FakeEmbeddingIndex + { + Records = { [Id(1)] = new(1, "one"), [Id(2)] = new(1, "two") }, + }; + + var pass = await new ExperienceIndexingService( + index, + new FakeEmbeddingGenerator { Throws = new TaskCanceledException("provider request timeout") }) + .ReindexAsync(Authorization, new ReindexExperienceRequest(RequestScope)); + + Assert.Equal(ExperienceReindexOutcome.Completed, pass.Outcome); + Assert.Equal(2, pass.Examined); + Assert.Equal(2, pass.Failed); + Assert.All(pass.Records, record => Assert.Equal(ExperienceIndexingOutcome.ProviderFailed, record.Outcome)); + Assert.All(pass.Records, record => Assert.True(record.IsRetryable)); + } + + // ---------------------------------------------------------------- construction and arguments + + [Fact] + public void A_generator_that_cannot_say_what_it_is_fails_at_construction_rather_than_mid_query() + { + var index = new FakeEmbeddingIndex(); + + Assert.Throws(() => new ExperienceIndexingService(null!, new FakeEmbeddingGenerator())); + Assert.Throws(() => new ExperienceIndexingService(index, null!)); + Assert.Throws(() => new ExperienceIndexingService(index, new FakeEmbeddingGenerator { ModelId = " " })); + Assert.Throws(() => new ExperienceIndexingService(index, new FakeEmbeddingGenerator { Dimension = 0 })); + Assert.Throws(() => new ExperienceIndexingService( + index, + new FakeEmbeddingGenerator { Dimension = ExperienceEmbeddingDescriptor.MaxDimension + 1 })); + } + + [Fact] + public async Task Null_and_empty_arguments_throw() + { + var service = new ExperienceIndexingService(new FakeEmbeddingIndex(), new FakeEmbeddingGenerator()); + + await Assert.ThrowsAsync(() => service.IndexAsync(null!, RequestScope, Id(1))); + await Assert.ThrowsAsync(() => service.IndexAsync(Authorization, null!, Id(1))); + await Assert.ThrowsAsync(() => service.IndexAsync(Authorization, RequestScope, Guid.Empty)); + await Assert.ThrowsAsync(() => service.ReindexAsync(null!, new ReindexExperienceRequest(RequestScope))); + await Assert.ThrowsAsync(() => service.ReindexAsync(Authorization, null!)); + await Assert.ThrowsAsync(() => service.ReindexAsync(Authorization, new ReindexExperienceRequest(null!))); + } + + [Fact] + public async Task Caller_cancellation_propagates_unwrapped_rather_than_becoming_a_reported_failure() + { + var id = Id(1); + var index = new FakeEmbeddingIndex { Records = { [id] = new(1, Summary) } }; + using var cancellation = new CancellationTokenSource(); + var gate = new TaskCompletionSource(); + var service = new ExperienceIndexingService(index, new FakeEmbeddingGenerator { Gate = gate }); + + var indexing = service.IndexAsync(Authorization, RequestScope, id, cancellation.Token); + await cancellation.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => indexing); + Assert.Empty(index.Stored); + } + + // ---------------------------------------------------------------- the content hash itself + + [Fact] + public void The_content_hash_covers_the_model_and_the_text_and_nothing_can_collide_across_the_two() + { + var a = ExperienceEmbeddingDescriptor.ComputeContentHash("model-a", "text"); + var b = ExperienceEmbeddingDescriptor.ComputeContentHash("model-b", "text"); + var c = ExperienceEmbeddingDescriptor.ComputeContentHash("model-a", "other"); + + Assert.Equal(64, a.Length); + Assert.Equal(a, ExperienceEmbeddingDescriptor.ComputeContentHash("model-a", "text")); + Assert.NotEqual(a, b); + Assert.NotEqual(a, c); + + // The separator cannot occur in either part, so "model-a" + "b|text" cannot hash like + // "model-ab" + "text". + Assert.NotEqual( + ExperienceEmbeddingDescriptor.ComputeContentHash("model", "atext"), + ExperienceEmbeddingDescriptor.ComputeContentHash("modela", "text")); + Assert.All(a, character => Assert.Contains(character, "0123456789abcdef")); + } + + [Fact] + public void Truncation_never_splits_a_character_or_leaves_a_dangling_separator() + { + // A raw cut at a UTF-16 index can land inside a surrogate pair, and UTF-8 encoding silently + // replaces a lone surrogate with U+FFFD -- so the text that was hashed and the text the + // provider saw would differ, which is the one thing the content hash exists to rule out. + // One filler short of the ceiling, so the surrogate pair straddles the cut: it is dropped + // whole rather than halved. + var straddling = ExperienceRetrievalSummary.For( + new string('a', ExperienceRetrievalSummary.MaxLength - 1) + string.Concat(Enumerable.Repeat("\U0001F600", 8)), + null, + null); + + Assert.Equal(ExperienceRetrievalSummary.MaxLength - 1, straddling.Length); + Assert.EndsWith("a", straddling, StringComparison.Ordinal); + Assert.DoesNotContain(straddling, char.IsSurrogate); + + // Round-tripping through UTF-8 is exactly what the provider and the hash both do; a lone + // surrogate would come back as U+FFFD and the two would no longer be hashing the same text. + Assert.Equal( + straddling, + System.Text.Encoding.UTF8.GetString(System.Text.Encoding.UTF8.GetBytes(straddling))); + + // An even cut needs no backing off, and a whole pair is kept. + var aligned = ExperienceRetrievalSummary.For( + new string('a', ExperienceRetrievalSummary.MaxLength - 2) + string.Concat(Enumerable.Repeat("\U0001F600", 8)), + null, + null); + Assert.Equal(ExperienceRetrievalSummary.MaxLength, aligned.Length); + Assert.Equal(aligned, System.Text.Encoding.UTF8.GetString(System.Text.Encoding.UTF8.GetBytes(aligned))); + + // And a cut that would have ended on the separator space trims it instead. + var trailing = ExperienceRetrievalSummary.For(new string('a', ExperienceRetrievalSummary.MaxLength - 1), "b", null); + Assert.Equal(ExperienceRetrievalSummary.MaxLength - 1, trailing.Length); + Assert.EndsWith("a", trailing, StringComparison.Ordinal); + } + + [Fact] + public void A_control_character_in_the_summary_can_never_collide_with_the_hash_separator() + { + // The hash joins the model ID and the summary with U+001F. That is only unambiguous while + // neither part can contain it, so the summary strips every control character. + Assert.Equal("model text", ExperienceRetrievalSummary.For("model\u001ftext", null, null)); + Assert.Equal("a b", ExperienceRetrievalSummary.For("a\u0000\u0007\u001fb", null, null)); + + Assert.NotEqual( + ExperienceEmbeddingDescriptor.ComputeContentHash("m", ExperienceRetrievalSummary.For("odel\u001ftext", null, null)), + ExperienceEmbeddingDescriptor.ComputeContentHash("model", ExperienceRetrievalSummary.For("text", null, null))); + } + + [Fact] + public void The_retrieval_summary_is_exactly_the_three_fields_the_text_index_analyzes() + { + var record = TestRecord("refund-ticket", "Resolve a refund ticket", "Release the lock first"); + + Assert.Equal("refund-ticket Resolve a refund ticket Release the lock first", ExperienceRetrievalSummary.For(record)); + + // A record with no reflection contributes no lesson, and a blank field contributes nothing at + // all rather than a stray separator. + Assert.Equal("refund-ticket", ExperienceRetrievalSummary.For(TestRecord("refund-ticket", null, null))); + Assert.Equal("refund-ticket lesson", ExperienceRetrievalSummary.For(TestRecord("refund-ticket", " ", "lesson"))); + Assert.Equal(ExperienceRetrievalSummary.MaxLength, ExperienceRetrievalSummary.For(new string('a', 20_000), null, null).Length); + } + + // ---------------------------------------------------------------- helpers + + private static Guid Id(int n) => Guid.Parse(FormattableString.Invariant($"00000000-0000-0000-0000-{n:000000000000}")); + + private static ExperienceRecord TestRecord(string taskId, string? taskSummary, string? lesson) => new( + ExperienceId: Id(1), + SourceRunId: Guid.NewGuid(), + Scope: RequestScope, + TaskId: taskId, + TaskSummary: taskSummary, + Attempts: [], + Outcome: new Outcome(TaskVerificationStatus.Verified, [], "checks passed", Now), + CompletionScore: 1, + Reflection: lesson is null + ? null + : new Reflection(Guid.NewGuid(), Guid.NewGuid(), lesson, [], [], [], [], null, [], TaskVerificationStatus.Verified, 1, "v1", "tests", Now), + Environment: new EnvironmentFingerprint("worker-01", "10.0.0", "linux-x64", null, new Dictionary()), + Provenance: new Provenance("tests", null, Now, null), + Status: ExperienceStatus.Validated, + ReuseConfidence: 0.5, + SupportingValidations: 1, + Contradictions: 0, + Revision: 1, + CreatedAt: Now, + UpdatedAt: Now); +} diff --git a/tests/AgentExperience.Core.Tests/ExperienceRetrievalServiceTests.cs b/tests/AgentExperience.Core.Tests/ExperienceRetrievalServiceTests.cs new file mode 100644 index 0000000..658189e --- /dev/null +++ b/tests/AgentExperience.Core.Tests/ExperienceRetrievalServiceTests.cs @@ -0,0 +1,808 @@ +using System.Globalization; +using System.Text; +using AgentExperience.Core.Retrieval; + +namespace AgentExperience.Core.Tests; + +/// +/// Covers against a fake : +/// one test per row of the story's I/O and edge-case matrix, a golden fixture pinning the documented +/// default ordering together with every component value and effective weight, and the validation that +/// keeps an impossible weighting from ever reaching a retrieval call. +/// +public class ExperienceRetrievalServiceTests +{ + private static readonly DateTimeOffset Now = new(2026, 9, 21, 12, 0, 0, TimeSpan.Zero); + + private static readonly Scope RequestScope = new("tenant-1", "app-1", "project-1"); + + private static readonly AuthorizationContext Authorization = new("tenant-1", "host-principal", ["experience:read"], Now); + + private const string TaskText = "resolve a refund ticket"; + + // ---------------------------------------------------------------- matrix: relevant match + + [Fact] + public async Task Matching_records_come_back_ranked_with_every_component_and_its_effective_weight() + { + var record = Record(Id(1), confidence: 0.8, updatedAt: Now); + var service = Service(Found(new ExperienceCandidate(record, 0.6))); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.False(result.TimedOut); + Assert.Null(result.Failure); + var ranked = Assert.Single(result.Records); + Assert.Same(record, ranked.Record); + + Assert.Equal( + [ + RankingComponentKind.Relevance, + RankingComponentKind.Confidence, + RankingComponentKind.Recency, + RankingComponentKind.Status, + RankingComponentKind.EnvironmentCompatibility, + ], + ranked.Components.Select(component => component.Kind)); + + var weights = RankingWeights.Default; + Assert.Equal( + [weights.Relevance, weights.Confidence, weights.Recency, weights.Status, weights.EnvironmentCompatibility], + ranked.Components.Select(component => component.Weight)); + + Assert.Equal([0.6, 0.8, 1d, ExperienceRetrievalService.ValidatedStatusScore, 1d], ranked.Components.Select(component => component.Value)); + Assert.Equal(ranked.Components.Sum(component => component.Contribution), ranked.Score, 12); + Assert.All(ranked.Components, component => Assert.InRange(component.Value, 0d, 1d)); + } + + // ---------------------------------------------------------------- matrix: ineligible status + + [Theory] + [InlineData(ExperienceStatus.Candidate)] + [InlineData(ExperienceStatus.Quarantined)] + [InlineData(ExperienceStatus.Contested)] + [InlineData(ExperienceStatus.Stale)] + [InlineData(ExperienceStatus.Superseded)] + [InlineData(ExperienceStatus.Revoked)] + public async Task An_ineligible_status_is_excluded_before_ranking_whatever_its_text_match(ExperienceStatus status) + { + // A perfect text match and full confidence: only the status keeps it out. + var ineligible = Record(Id(1), status: status, confidence: 1d, updatedAt: Now); + var service = Service(Found(new ExperienceCandidate(ineligible, 1d))); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Empty(result.Records); + var excluded = Assert.Single(result.Excluded); + Assert.Equal(new ExcludedExperience(ineligible.ExperienceId, RetrievalExclusionReason.IneligibleStatus), excluded); + } + + [Fact] + public void Only_Validated_and_Reinforced_are_ever_eligible() + { + Assert.Equal([ExperienceStatus.Validated, ExperienceStatus.Reinforced], ExperienceRetrievalService.EligibleStatuses); + } + + // ---------------------------------------------------------------- matrix: low confidence + + [Fact] + public async Task The_confidence_threshold_and_the_eligible_statuses_are_pushed_into_the_search_not_applied_afterwards() + { + // The confidence floor is a database predicate: what never comes back is never scored, and a + // candidate source is never asked to return records Core would only throw away. + var source = new FakeCandidateSource(Found()); + var policy = RetrievalPolicy.Default with { MinimumConfidence = 0.75, CandidateLimit = 7 }; + var service = new ExperienceRetrievalService(source, policy, RankingWeights.Default, new FixedTimeProvider(Now)); + + await service.RetrieveAsync(Request()); + + var query = Assert.Single(source.Queries); + Assert.Equal(0.75, query.MinimumConfidence); + + // One past the ceiling, so the service can tell "exactly 7 matched" from "more than 7 matched". + Assert.Equal(8, query.Limit); + Assert.Equal(RequestScope, query.Scope); + Assert.Equal(TaskText, query.TaskText); + Assert.Equal([ExperienceStatus.Validated, ExperienceStatus.Reinforced], query.EligibleStatuses); + Assert.Equal(0.5, RetrievalPolicy.DefaultMinimumConfidence); + } + + // ---------------------------------------------------------------- matrix: expired + + [Fact] + public async Task A_record_older_than_MaxAge_is_excluded_in_Core() + { + var fresh = Record(Id(1), updatedAt: Now - TimeSpan.FromDays(6)); + var expired = Record(Id(2), updatedAt: Now - TimeSpan.FromDays(8)); + var service = Service( + Found(new ExperienceCandidate(fresh, 1d), new ExperienceCandidate(expired, 1d)), + policy: RetrievalPolicy.Default with { MaxAge = TimeSpan.FromDays(7) }); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal([fresh.ExperienceId], result.Records.Select(ranked => ranked.Record.ExperienceId)); + Assert.Equal( + [new ExcludedExperience(expired.ExperienceId, RetrievalExclusionReason.Expired)], + result.Excluded); + } + + [Fact] + public async Task A_null_MaxAge_means_no_expiry_at_all() + { + var ancient = Record(Id(1), updatedAt: Now - TimeSpan.FromDays(4000)); + var service = Service(Found(new ExperienceCandidate(ancient, 1d))); + + var result = await service.RetrieveAsync(Request()); + + Assert.Null(RetrievalPolicy.Default.MaxAge); + Assert.Equal([ancient.ExperienceId], result.Records.Select(ranked => ranked.Record.ExperienceId)); + Assert.Empty(result.Excluded); + } + + // ---------------------------------------------------------------- matrix: environment mismatch + + [Fact] + public async Task A_differing_or_missing_required_environment_attribute_excludes_the_record_and_the_result_names_the_check() + { + var matching = Record(Id(1), metadata: new Dictionary { ["region"] = "us-east", ["tier"] = "prod" }); + var differing = Record(Id(2), metadata: new Dictionary { ["region"] = "eu-west" }); + var missingKey = Record(Id(3), metadata: new Dictionary { ["tier"] = "prod" }); + var caseDiffering = Record(Id(4), metadata: new Dictionary { ["region"] = "US-EAST" }); + + var service = Service(Found( + new ExperienceCandidate(matching, 1d), + new ExperienceCandidate(differing, 1d), + new ExperienceCandidate(missingKey, 1d), + new ExperienceCandidate(caseDiffering, 1d))); + + var result = await service.RetrieveAsync( + Request(required: new Dictionary { ["region"] = "us-east" })); + + Assert.False(result.EnvironmentUnrestricted); + Assert.Equal([matching.ExperienceId], result.Records.Select(ranked => ranked.Record.ExperienceId)); + Assert.Equal( + [ + new ExcludedExperience(differing.ExperienceId, RetrievalExclusionReason.EnvironmentMismatch), + new ExcludedExperience(missingKey.ExperienceId, RetrievalExclusionReason.EnvironmentMismatch), + new ExcludedExperience(caseDiffering.ExperienceId, RetrievalExclusionReason.EnvironmentMismatch), + ], + result.Excluded); + } + + // ---------------------------------------------------------------- matrix: unrestricted + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task A_request_with_no_required_attributes_passes_every_candidate_and_is_marked_unrestricted(bool emptyRatherThanNull) + { + var record = Record(Id(1), metadata: new Dictionary { ["region"] = "anywhere" }); + var service = Service(Found(new ExperienceCandidate(record, 1d))); + + var result = await service.RetrieveAsync( + Request(required: emptyRatherThanNull ? new Dictionary() : null)); + + Assert.True(result.EnvironmentUnrestricted); + Assert.Single(result.Records); + Assert.Empty(result.Excluded); + } + + // ---------------------------------------------------------------- matrix: foreign scope + + [Fact] + public async Task A_candidate_outside_the_requested_scope_empties_the_whole_result_rather_than_being_dropped_from_it() + { + // Defence in depth: the scope predicate runs in SQL, so this can only happen through a broken + // source -- and a source that answered out of scope once cannot be trusted for the rest either. + var mine = Record(Id(1)); + var foreign = Record(Id(2), scope: new Scope("tenant-2", "app-1", "project-1")); + var service = Service(Found(new ExperienceCandidate(mine, 1d), new ExperienceCandidate(foreign, 1d))); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Failed, result.Outcome); + Assert.Empty(result.Records); + Assert.NotNull(result.Failure); + } + + // ---------------------------------------------------------------- matrix: beyond authority + + [Fact] + public async Task A_scope_outside_the_authorization_is_an_empty_fail_closed_result_and_no_search_is_issued() + { + var source = new FakeCandidateSource(Found(new ExperienceCandidate(Record(Id(1)), 1d))); + var service = new ExperienceRetrievalService(source, RetrievalPolicy.Default, RankingWeights.Default, new FixedTimeProvider(Now)); + + var result = await service.RetrieveAsync(new RetrieveExperienceRequest( + new AuthorizationContext("tenant-1", "host-principal", [], Now, ProjectId: "another-project"), + RequestScope, + TaskText, + CorrelationId: "corr-1")); + + Assert.Equal(RetrievalOutcome.Denied, result.Outcome); + Assert.Empty(result.Records); + Assert.Empty(source.Queries); + Assert.Equal("corr-1", result.CorrelationId); + Assert.Null(result.Failure); + } + + // ---------------------------------------------------------------- matrix: ties + + [Fact] + public async Task Equal_scores_are_ordered_by_ExperienceId_ascending_and_ordinal() + { + // Identical in every scored respect, handed over in reverse order. + var first = Record(Guid.Parse("00000000-0000-0000-0000-0000000000aa"), updatedAt: Now); + var second = Record(Guid.Parse("00000000-0000-0000-0000-0000000000ab"), updatedAt: Now); + var third = Record(Guid.Parse("00000000-0000-0000-0000-0000000000ba"), updatedAt: Now); + var service = Service(Found( + new ExperienceCandidate(third, 0.5), + new ExperienceCandidate(second, 0.5), + new ExperienceCandidate(first, 0.5))); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal( + [first.ExperienceId, second.ExperienceId, third.ExperienceId], + result.Records.Select(ranked => ranked.Record.ExperienceId)); + Assert.Single(result.Records.Select(ranked => ranked.Score).Distinct()); + } + + // ---------------------------------------------------------------- matrix: timeout + + [Fact] + public async Task A_search_that_exceeds_the_timeout_returns_an_empty_result_with_the_timeout_signal_and_the_correlation_id() + { + // Real time here: the point is that the wall clock runs out, not how it is measured. + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var observedCancellation = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var source = new FakeCandidateSource(async (_, token) => + { + entered.TrySetResult(); + using var registration = token.Register(() => observedCancellation.TrySetResult()); + await Task.Delay(Timeout.InfiniteTimeSpan, token); + return Found(); + }); + + var service = new ExperienceRetrievalService( + source, + RetrievalPolicy.Default with { Timeout = TimeSpan.FromMilliseconds(50) }, + RankingWeights.Default, + TimeProvider.System); + + var result = await service.RetrieveAsync(Request(correlationId: "corr-timeout")); + + Assert.Equal(RetrievalOutcome.TimedOut, result.Outcome); + Assert.True(result.TimedOut); + Assert.Empty(result.Records); + Assert.Empty(result.Excluded); + Assert.Equal("corr-timeout", result.CorrelationId); + Assert.Null(result.Failure); // a timeout is not a failure + await entered.Task; + + // The abandoned search is cancelled only after the timeout has been reported. + await observedCancellation.Task.WaitAsync(TimeSpan.FromSeconds(10)); + } + + [Fact] + public async Task The_timeout_is_measured_with_the_injected_TimeProvider() + { + // A 5 ms timeout against a search that takes ten times that in real time. The frozen provider's + // clock never advances, so nothing times out; a service measuring on the wall clock would have. + var clock = new FixedTimeProvider(Now); + var source = new FakeCandidateSource(async (_, _) => + { + await Task.Delay(TimeSpan.FromMilliseconds(50)); + return Found(new ExperienceCandidate(Record(Id(1)), 1d)); + }); + var service = new ExperienceRetrievalService( + source, + RetrievalPolicy.Default with { Timeout = TimeSpan.FromMilliseconds(5) }, + RankingWeights.Default, + clock); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Single(result.Records); + Assert.Equal(TimeSpan.Zero, result.Elapsed); + } + + // ---------------------------------------------------------------- matrix: cancelled + + [Fact] + public async Task Caller_cancellation_propagates_unwrapped_and_is_never_reported_as_a_timeout() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var source = new FakeCandidateSource(async (_, token) => + { + entered.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, token); + return Found(); + }); + + // A timeout long enough that it cannot be what ends the call. + var service = new ExperienceRetrievalService( + source, + RetrievalPolicy.Default with { Timeout = TimeSpan.FromMinutes(5) }, + RankingWeights.Default, + TimeProvider.System); + + using var cancellation = new CancellationTokenSource(); + var retrieval = service.RetrieveAsync(Request(), cancellation.Token); + await entered.Task; + await cancellation.CancelAsync(); + + var exception = await Assert.ThrowsAnyAsync(() => retrieval); + + // Cancellation, not a timeout: the call threw rather than returning a timed-out result. + Assert.IsAssignableFrom(exception); + Assert.True(cancellation.IsCancellationRequested); + } + + [Fact] + public async Task An_already_cancelled_token_throws_before_any_search_is_issued() + { + var source = new FakeCandidateSource(Found()); + var service = new ExperienceRetrievalService(source, RetrievalPolicy.Default, RankingWeights.Default, new FixedTimeProvider(Now)); + using var cancellation = new CancellationTokenSource(); + await cancellation.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => service.RetrieveAsync(Request(), cancellation.Token)); + + Assert.Empty(source.Queries); + } + + // ---------------------------------------------------------------- matrix: store failure + + [Fact] + public async Task A_store_failure_inside_the_timeout_is_an_empty_fail_closed_result_carrying_the_failure() + { + var failure = new ExperienceStoreException("database unavailable", new InvalidOperationException("driver")); + var service = Service(new FakeCandidateSource((_, _) => throw failure)); + + var result = await service.RetrieveAsync(Request(correlationId: "corr-failed")); + + Assert.Equal(RetrievalOutcome.Failed, result.Outcome); + Assert.False(result.TimedOut); + Assert.Empty(result.Records); + Assert.Equal("corr-failed", result.CorrelationId); + Assert.NotNull(result.Failure); + Assert.Same(failure, result.Failure!.Exception); + } + + [Theory] + [InlineData(ExperienceStoreOutcome.Denied)] + [InlineData(ExperienceStoreOutcome.Invalid)] + public async Task A_source_that_refuses_to_answer_empties_the_result_rather_than_reporting_no_matches(ExperienceStoreOutcome outcome) + { + var service = Service(new FakeCandidateSource( + (_, _) => Task.FromResult(new ExperienceCandidateSearchResult(outcome, [], [])))); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Failed, result.Outcome); + Assert.Empty(result.Records); + Assert.NotNull(result.Failure); + Assert.Contains(outcome.ToString(), result.Failure!.Reason, StringComparison.Ordinal); + } + + [Fact] + public async Task An_unreadable_candidate_empties_the_result_rather_than_leaving_it_unfiltered() + { + var readable = Record(Id(1)); + var unreadable = Record(Id(2)) with { Environment = null! }; + var service = Service(Found(new ExperienceCandidate(readable, 1d), new ExperienceCandidate(unreadable, 1d))); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Failed, result.Outcome); + Assert.Empty(result.Records); + Assert.NotNull(result.Failure); + } + + [Fact] + public async Task The_same_record_returned_twice_empties_the_result_rather_than_ranking_it_twice() + { + var record = Record(Id(1), updatedAt: Now); + var service = Service(Found(new ExperienceCandidate(record, 0.9), new ExperienceCandidate(record, 0.2))); + + var result = await service.RetrieveAsync(Request()); + + // Scored twice, it would be ordered arbitrarily against itself and the ranking would stop being + // total -- and which of the two relevances won would be undefined. + Assert.Equal(RetrievalOutcome.Failed, result.Outcome); + Assert.Empty(result.Records); + Assert.NotNull(result.Failure); + } + + [Fact] + public async Task A_source_that_cancels_for_its_own_reasons_is_an_empty_failed_result_not_a_throw() + { + // Neither the caller's token nor the timeout: a port that cancels on its own must not be able to + // make RetrieveAsync throw at a caller who never cancelled anything. + using var unrelated = new CancellationTokenSource(); + await unrelated.CancelAsync(); + var service = Service(new FakeCandidateSource((_, _) => + throw new OperationCanceledException("the source gave up", unrelated.Token))); + + var result = await service.RetrieveAsync(Request(correlationId: "corr-source-cancel")); + + Assert.Equal(RetrievalOutcome.Failed, result.Outcome); + Assert.False(result.TimedOut); + Assert.Empty(result.Records); + Assert.Equal("corr-source-cancel", result.CorrelationId); + Assert.IsType(result.Failure!.Exception); + } + + [Fact] + public async Task A_source_returning_no_result_or_no_candidate_list_is_fail_closed() + { + var noResult = await Service(new FakeCandidateSource( + (_, _) => Task.FromResult(null!))).RetrieveAsync(Request()); + var noList = await Service(new FakeCandidateSource( + (_, _) => Task.FromResult(new ExperienceCandidateSearchResult(ExperienceStoreOutcome.Found, null!, [])))).RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Failed, noResult.Outcome); + Assert.NotNull(noResult.Failure); + Assert.Equal(RetrievalOutcome.Failed, noList.Outcome); + Assert.NotNull(noList.Failure); + } + + [Fact] + public async Task A_source_that_throws_something_other_than_a_store_failure_is_still_fail_closed() + { + var service = Service(new FakeCandidateSource((_, _) => throw new InvalidOperationException("boom"))); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Failed, result.Outcome); + Assert.Empty(result.Records); + Assert.IsType(result.Failure!.Exception); + } + + // ---------------------------------------------------------------- the golden fixture + + /// + /// Pins the documented default ordering, every normalized component, and every effective weight. + /// Each record differs from the baseline (...0004) in exactly one component, so the order + /// below is the statement that relevance outranks confidence outranks recency and status. + /// A change here is a behaviour change for every host that ranks with the defaults. + /// + [Fact] + public async Task Golden_fixture_pins_the_default_ordering_and_every_component_value() + { + const string Expected = """ + 00000000-0000-0000-0000-000000000001 0.800 Relevance=1.000*0.350 Confidence=0.500*0.250 Recency=1.000*0.150 Status=0.500*0.150 EnvironmentCompatibility=1.000*0.100 + 00000000-0000-0000-0000-000000000002 0.750 Relevance=0.500*0.350 Confidence=1.000*0.250 Recency=1.000*0.150 Status=0.500*0.150 EnvironmentCompatibility=1.000*0.100 + 00000000-0000-0000-0000-000000000003 0.700 Relevance=0.500*0.350 Confidence=0.500*0.250 Recency=1.000*0.150 Status=1.000*0.150 EnvironmentCompatibility=1.000*0.100 + 00000000-0000-0000-0000-000000000004 0.625 Relevance=0.500*0.350 Confidence=0.500*0.250 Recency=1.000*0.150 Status=0.500*0.150 EnvironmentCompatibility=1.000*0.100 + 00000000-0000-0000-0000-000000000005 0.550 Relevance=0.500*0.350 Confidence=0.500*0.250 Recency=0.500*0.150 Status=0.500*0.150 EnvironmentCompatibility=1.000*0.100 + """; + + var policy = RetrievalPolicy.Default; + var halfLifeAgo = Now - policy.RecencyHalfLife; + + // Deliberately handed over worst-first, so the assertion is about ranking and not about the + // order the source happened to return. + var service = Service( + Found( + new ExperienceCandidate(Record(Id(5), updatedAt: halfLifeAgo), 0.5), + new ExperienceCandidate(Record(Id(4), updatedAt: Now), 0.5), + new ExperienceCandidate(Record(Id(3), status: ExperienceStatus.Reinforced, updatedAt: Now), 0.5), + new ExperienceCandidate(Record(Id(2), confidence: 1d, updatedAt: Now), 0.5), + new ExperienceCandidate(Record(Id(1), updatedAt: Now), 1d)), + policy: policy); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Equal(Expected, Render(result.Records)); + } + + private static string Render(IReadOnlyList records) + { + var builder = new StringBuilder(); + for (var i = 0; i < records.Count; i++) + { + if (i > 0) + { + builder.Append('\n'); + } + + var ranked = records[i]; + builder.Append(CultureInfo.InvariantCulture, $"{ranked.Record.ExperienceId:D} {ranked.Score:0.000}"); + foreach (var component in ranked.Components) + { + builder.Append(CultureInfo.InvariantCulture, $" {component.Kind}={component.Value:0.000}*{component.Weight:0.000}"); + } + } + + return builder.ToString(); + } + + // ---------------------------------------------------------------- weight and policy validation + + [Theory] + [InlineData(-0.35, 0.25, 0.15, 0.15, 0.10)] // negative + [InlineData(0.35, 0.25, 0.15, 0.15, 0.20)] // sums to 1.1 + [InlineData(0.35, 0.25, 0.15, 0.15, 0.00)] // sums to 0.9 + [InlineData(0.00, 0.00, 0.00, 0.00, 0.00)] // sums to 0 + [InlineData(double.NaN, 0.25, 0.15, 0.15, 0.10)] + [InlineData(double.PositiveInfinity, 0.25, 0.15, 0.15, 0.10)] + public void Negative_non_finite_or_non_unit_sum_weights_throw_at_construction( + double relevance, + double confidence, + double recency, + double status, + double environment) + { + Assert.Throws( + () => new RankingWeights(relevance, confidence, recency, status, environment)); + } + + [Fact] + public void A_zero_weight_is_allowed_as_long_as_the_set_still_sums_to_one() + { + var weights = new RankingWeights(1d, 0d, 0d, 0d, 0d); + + Assert.Equal(0d, weights.Confidence); + Assert.Equal(1d, weights.Sum, 12); + } + + [Fact] + public void The_default_weights_are_the_documented_ones_and_sum_to_one() + { + var weights = RankingWeights.Default; + + Assert.Equal(0.35, weights.Relevance); + Assert.Equal(0.25, weights.Confidence); + Assert.Equal(0.15, weights.Recency); + Assert.Equal(0.15, weights.Status); + Assert.Equal(0.10, weights.EnvironmentCompatibility); + Assert.Equal(1d, weights.Sum, 12); + Assert.True(Math.Abs(weights.Sum - 1d) <= RankingWeights.SumTolerance); + } + + [Fact] + public void Invalid_weights_mean_no_retrieval_service_can_be_built_at_all() + { + // The service takes a constructed RankingWeights, so an invalid set cannot reach a call. + Assert.Throws(() => new ExperienceRetrievalService( + new FakeCandidateSource(Found()), + RetrievalPolicy.Default, + new RankingWeights(0.5, 0.25, 0.15, 0.15, 0.10), + new FixedTimeProvider(Now))); + } + + [Fact] + public void An_invalid_policy_value_throws_at_construction_and_on_a_with_expression() + { + Assert.Throws(() => RetrievalPolicy.Default with { Timeout = TimeSpan.Zero }); + Assert.Throws(() => RetrievalPolicy.Default with { Timeout = TimeSpan.FromMilliseconds(-1) }); + Assert.Throws(() => RetrievalPolicy.Default with { MinimumConfidence = 1.5 }); + Assert.Throws(() => RetrievalPolicy.Default with { MinimumConfidence = -0.1 }); + Assert.Throws(() => RetrievalPolicy.Default with { MaxAge = TimeSpan.Zero }); + Assert.Throws(() => RetrievalPolicy.Default with { RecencyHalfLife = TimeSpan.Zero }); + Assert.Throws(() => RetrievalPolicy.Default with { CandidateLimit = 0 }); + Assert.Throws(() => RetrievalPolicy.Default with { CandidateLimit = RetrievalPolicy.MaxCandidateLimit + 1 }); + + // The ceiling stops one below the port's own maximum, because the service asks for one more. + Assert.Equal(ExperienceCandidateQuery.MaxLimit - 1, RetrievalPolicy.MaxCandidateLimit); + Assert.Equal(RetrievalPolicy.MaxCandidateLimit, (RetrievalPolicy.Default with { CandidateLimit = RetrievalPolicy.MaxCandidateLimit }).CandidateLimit); + + // A timeout past the supported span would throw out of the retrieval call instead of bounding it. + Assert.Throws(() => RetrievalPolicy.Default with { Timeout = RetrievalPolicy.MaxTimeout + TimeSpan.FromSeconds(1) }); + Assert.Throws(() => RetrievalPolicy.Default with { Timeout = TimeSpan.MaxValue }); + Assert.Equal(RetrievalPolicy.MaxTimeout, (RetrievalPolicy.Default with { Timeout = RetrievalPolicy.MaxTimeout }).Timeout); + + Assert.Equal(TimeSpan.FromMilliseconds(500), RetrievalPolicy.Default.Timeout); + Assert.Null((RetrievalPolicy.Default with { MaxAge = TimeSpan.FromDays(1) } with { MaxAge = null }).MaxAge); + } + + // ---------------------------------------------------------------- request validation and limits + + [Fact] + public async Task A_malformed_request_throws_rather_than_quietly_retrieving_nothing() + { + var service = Service(Found()); + + await Assert.ThrowsAsync(() => service.RetrieveAsync(null!)); + await Assert.ThrowsAsync(() => service.RetrieveAsync(new RetrieveExperienceRequest(null!, RequestScope, TaskText))); + await Assert.ThrowsAsync(() => service.RetrieveAsync(new RetrieveExperienceRequest(Authorization, null!, TaskText))); + await Assert.ThrowsAsync(() => service.RetrieveAsync(new RetrieveExperienceRequest(Authorization, RequestScope, " "))); + await Assert.ThrowsAsync(() => service.RetrieveAsync(Request(limit: 0))); + + // Longer than the port will accept: rejected here rather than travelling to the database and + // coming back as an opaque Failed result. + await Assert.ThrowsAsync(() => service.RetrieveAsync( + new RetrieveExperienceRequest(Authorization, RequestScope, new string('a', ExperienceCandidateQuery.MaxTaskTextLength + 1)))); + } + + [Fact] + public async Task A_limit_larger_than_the_candidate_ceiling_is_rejected_rather_than_quietly_capped() + { + var source = new FakeCandidateSource(Found()); + var policy = RetrievalPolicy.Default with { CandidateLimit = 5 }; + var service = new ExperienceRetrievalService(source, policy, RankingWeights.Default, new FixedTimeProvider(Now)); + + // Asking for 6 when the search will only ever consider 5 could never be satisfied; silently + // returning 5 would hide the misconfiguration. + await Assert.ThrowsAsync(() => service.RetrieveAsync(Request(limit: 6))); + Assert.Empty(source.Queries); + + var atTheCeiling = await service.RetrieveAsync(Request(limit: 5)); + Assert.Equal(RetrievalOutcome.Completed, atTheCeiling.Outcome); + } + + // ---------------------------------------------------------------- the candidate ceiling + + [Fact] + public async Task Hitting_the_candidate_ceiling_is_reported_and_the_extra_probe_candidate_is_never_ranked() + { + // The service asks for CandidateLimit + 1; a full extra candidate means more matched than were + // considered. It must not be ranked, and the caller must be told the answer is partial. + var policy = RetrievalPolicy.Default with { CandidateLimit = 3 }; + var candidates = Enumerable.Range(1, 4) + .Select(n => new ExperienceCandidate(Record(Id(n), updatedAt: Now), 1d - (n * 0.1))) + .ToArray(); + var service = Service(Found(candidates), policy); + + var result = await service.RetrieveAsync(Request()); + + Assert.True(result.Truncated); + Assert.Equal([Id(1), Id(2), Id(3)], result.Records.Select(ranked => ranked.Record.ExperienceId)); + Assert.DoesNotContain(Id(4), result.Excluded.Select(excluded => excluded.ExperienceId)); + } + + [Fact] + public async Task A_result_that_did_not_reach_the_ceiling_is_not_truncated() + { + var policy = RetrievalPolicy.Default with { CandidateLimit = 3 }; + var candidates = Enumerable.Range(1, 3) + .Select(n => new ExperienceCandidate(Record(Id(n), updatedAt: Now), 0.5)) + .ToArray(); + + var result = await Service(Found(candidates), policy).RetrieveAsync(Request()); + + // Exactly at the ceiling, with no probe candidate: everything that matched was considered. + Assert.False(result.Truncated); + Assert.Equal(3, result.Records.Count); + } + + [Fact] + public async Task An_empty_result_is_never_marked_truncated() + { + var denied = await Service(Found()).RetrieveAsync(new RetrieveExperienceRequest( + new AuthorizationContext("tenant-1", "p", [], Now, ProjectId: "elsewhere"), RequestScope, TaskText)); + var failed = await Service(new FakeCandidateSource((_, _) => throw new InvalidOperationException("boom"))).RetrieveAsync(Request()); + + Assert.False(denied.Truncated); + Assert.False(failed.Truncated); + } + + [Fact] + public async Task The_requests_limit_bounds_how_many_ranked_records_come_back_after_ranking() + { + var service = Service(Found( + new ExperienceCandidate(Record(Id(1), updatedAt: Now), 0.1), + new ExperienceCandidate(Record(Id(2), updatedAt: Now), 0.9), + new ExperienceCandidate(Record(Id(3), updatedAt: Now), 0.5))); + + var result = await service.RetrieveAsync(Request(limit: 2)); + + // Trimmed after ranking, so the best two survive rather than the first two returned. + Assert.Equal([Id(2), Id(3)], result.Records.Select(ranked => ranked.Record.ExperienceId)); + } + + [Fact] + public void Null_constructor_arguments_throw() + { + var source = new FakeCandidateSource(Found()); + var clock = new FixedTimeProvider(Now); + + Assert.Throws(() => new ExperienceRetrievalService(null!, RetrievalPolicy.Default, RankingWeights.Default, clock)); + Assert.Throws(() => new ExperienceRetrievalService(source, null!, RankingWeights.Default, clock)); + Assert.Throws(() => new ExperienceRetrievalService(source, RetrievalPolicy.Default, null!, clock)); + Assert.Throws(() => new ExperienceRetrievalService(source, RetrievalPolicy.Default, RankingWeights.Default, null!)); + } + + // ---------------------------------------------------------------- helpers + + private static Guid Id(int n) => Guid.Parse(FormattableString.Invariant($"00000000-0000-0000-0000-{n:000000000000}")); + + private static RetrieveExperienceRequest Request( + IReadOnlyDictionary? required = null, + string? correlationId = null, + int? limit = null) => new(Authorization, RequestScope, TaskText, required, correlationId, limit); + + private static ExperienceCandidateSearchResult Found(params ExperienceCandidate[] candidates) => + new(ExperienceStoreOutcome.Found, candidates, []); + + private static ExperienceRetrievalService Service(ExperienceCandidateSearchResult result, RetrievalPolicy? policy = null) => + Service(new FakeCandidateSource(result), policy); + + private static ExperienceRetrievalService Service(FakeCandidateSource source, RetrievalPolicy? policy = null) => + new(source, policy ?? RetrievalPolicy.Default, RankingWeights.Default, new FixedTimeProvider(Now)); + + private static ExperienceRecord Record( + Guid id, + Scope? scope = null, + ExperienceStatus status = ExperienceStatus.Validated, + double confidence = 0.5, + DateTimeOffset? updatedAt = null, + IReadOnlyDictionary? metadata = null) => new( + ExperienceId: id, + SourceRunId: Guid.NewGuid(), + Scope: scope ?? RequestScope, + TaskId: "refund-ticket", + TaskSummary: "Resolve a refund ticket", + Attempts: [], + Outcome: new Outcome(TaskVerificationStatus.Verified, [], "checks passed", Now), + CompletionScore: 1, + Reflection: null, + Environment: new EnvironmentFingerprint("worker-01", "10.0.0", "linux-x64", null, metadata ?? new Dictionary()), + Provenance: new Provenance("tests", null, Now, null), + Status: status, + ReuseConfidence: confidence, + SupportingValidations: 1, + Contradictions: 0, + Revision: 1, + CreatedAt: updatedAt ?? Now, + UpdatedAt: updatedAt ?? Now); + + /// A candidate source that answers with whatever the test scripted, recording what it was asked. + private sealed class FakeCandidateSource( + Func> onSearch) + : IExperienceCandidateSource + { + public FakeCandidateSource(ExperienceCandidateSearchResult result) + : this((_, _) => Task.FromResult(result)) + { + } + + public List Queries { get; } = []; + + public Task SearchAsync( + AuthorizationContext authorization, + ExperienceCandidateQuery query, + CancellationToken cancellationToken) + { + Assert.NotNull(authorization); + lock (Queries) + { + Queries.Add(query); + } + + return onSearch(query, cancellationToken); + } + } + + /// + /// A clock frozen at a known instant. Its timers never fire, so a test that does not mean to + /// exercise the timeout cannot accidentally hit one, and elapsed time is always exactly zero. + /// + private sealed class FixedTimeProvider(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => now; + + public override long GetTimestamp() => now.UtcTicks; + + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => new FrozenTimer(); + + private sealed class FrozenTimer : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) => true; + + public void Dispose() + { + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + } +} diff --git a/tests/AgentExperience.Core.Tests/FinalizationIndexingHookTests.cs b/tests/AgentExperience.Core.Tests/FinalizationIndexingHookTests.cs new file mode 100644 index 0000000..aab0076 --- /dev/null +++ b/tests/AgentExperience.Core.Tests/FinalizationIndexingHookTests.cs @@ -0,0 +1,406 @@ +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Indexing; +using AgentExperience.Core.Lifecycle; + +namespace AgentExperience.Core.Tests; + +/// +/// Covers the post-commit indexing hook on : it indexes a +/// record that was just committed, it never runs on an already-finalized replay, and no failure it +/// can reach -- a provider that throws, an unreachable index, even a cancellation -- is ever allowed +/// to turn a durable finalization into anything else. +/// +public class FinalizationIndexingHookTests +{ + private const string ArtifactRevision = "rev-1"; + + private static readonly DateTimeOffset Now = new(2026, 9, 18, 10, 0, 0, TimeSpan.Zero); + private static readonly Scope TestScope = new("tenant-1", "app-1", "project-1"); + private static readonly AuthorizationContext Authorization = new("tenant-1", "host-principal", ["experience:write"], Now); + private static readonly ClosedVerificationRound Round = new(Guid.Parse("11111111-1111-1111-1111-111111111111"), ArtifactRevision); + + /// The same permissive policy the rest of the finalization tests capture under. + private static readonly SanitizationOptions PermissiveOptions = new(new Dictionary(StringComparer.Ordinal) + { + ["ToolArguments"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "query" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 3, + MaxFieldCount: 10, + MaxValueLength: 1_000, + MaxFieldNameLength: 100), + ["ToolResult"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "value" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 2, + MaxFieldCount: 5, + MaxValueLength: 1_000, + MaxFieldNameLength: 100), + }); + + // ---------------------------------------------------------------- matrix: index after commit + + [Fact] + public async Task A_finalized_run_is_indexed_after_its_initial_event_commits_at_revision_1() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(1, result.Revision); + + var indexing = result.Indexing!; + Assert.Equal(ExperienceIndexingOutcome.Indexed, indexing.Outcome); + Assert.Equal(result.ExperienceId, indexing.ExperienceId); + Assert.Equal(1, indexing.Descriptor!.SourceRevision); + Assert.Equal("fake-embed-v1", indexing.Descriptor.ModelId); + Assert.Equal(4, indexing.Descriptor.Dimension); + Assert.True(harness.Index.Stored.ContainsKey(result.ExperienceId!.Value)); + } + + [Fact] + public async Task Only_the_records_task_id_summary_and_lesson_are_handed_to_the_provider() + { + var harness = await Harness.WithCompletedRunAsync(); + + var result = await harness.FinalizeAsync(); + + var summary = Assert.Single(harness.Generator.Requests); + Assert.Equal(ExperienceRetrievalSummary.For(result.Record!), summary); + Assert.StartsWith("task-1 a test task", summary, StringComparison.Ordinal); + + // Nothing operational travels with it: the run's attempt outcome is not in the summary. + Assert.DoesNotContain("done", summary, StringComparison.Ordinal); + } + + // ---------------------------------------------------------------- matrix: provider down + + [Fact] + public async Task A_provider_that_throws_leaves_the_record_committed_durable_and_text_searchable() + { + var harness = await Harness.WithCompletedRunAsync( + generator: new FakeEmbeddingGenerator { Throws = FakeEmbeddingGenerator.ThrownException }); + + var result = await harness.FinalizeAsync(); + + // Finalization itself is untouched by the outage. + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(ExperienceStatus.Validated, result.Status); + Assert.Equal(1, result.Revision); + Assert.Null(result.Failure); + + // And the failure is reported, and retryable. + Assert.Equal(ExperienceIndexingOutcome.ProviderFailed, result.Indexing!.Outcome); + Assert.True(result.Indexing.IsRetryable); + Assert.Same(FakeEmbeddingGenerator.ThrownException, result.Indexing.Failure!.Exception); + Assert.Empty(harness.Index.Stored); + } + + [Fact] + public async Task An_index_that_throws_is_reported_and_never_fails_finalization() + { + var harness = await Harness.WithCompletedRunAsync( + index: new FakeEmbeddingIndex { ScanThrows = FakeEmbeddingIndex.ThrownException }); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(ExperienceIndexingOutcome.IndexFailed, result.Indexing!.Outcome); + Assert.True(result.Indexing.IsRetryable); + } + + [Fact] + public async Task A_cancellation_inside_the_hook_is_reported_rather_than_denying_a_record_that_is_already_durable() + { + using var cancellation = new CancellationTokenSource(); + var gate = new TaskCompletionSource(); + var harness = await Harness.WithCompletedRunAsync(generator: new FakeEmbeddingGenerator { Gate = gate }); + + var finalizing = harness.Service.FinalizeAsync(harness.Request(), cancellation.Token); + + // Cancel once the provider has actually been entered, so the commit is already done. + while (harness.Generator.Requests.Count == 0) + { + await Task.Yield(); + } + + await cancellation.CancelAsync(); + var result = await finalizing; + + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(ExperienceIndexingOutcome.IndexFailed, result.Indexing!.Outcome); + Assert.Contains("cancelled", result.Indexing.Failure!.Reason, StringComparison.OrdinalIgnoreCase); + gate.TrySetResult(); + } + + [Fact] + public async Task A_hung_provider_cannot_hold_the_call_open_after_the_record_is_durable() + { + // "Derived data never blocks canonical data" includes not blocking the caller's thread once the + // canonical work is done: the hook gets its own budget, not the caller's unbounded token. + var gate = new TaskCompletionSource(); + var harness = await Harness.WithCompletedRunAsync( + generator: new FakeEmbeddingGenerator { Gate = gate }, + indexingTimeout: TimeSpan.FromMilliseconds(50)); + + var finalizing = harness.Service.FinalizeAsync(harness.Request(), CancellationToken.None); + + var result = await finalizing.WaitAsync(TimeSpan.FromSeconds(30)); + + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(ExperienceIndexingOutcome.IndexFailed, result.Indexing!.Outcome); + Assert.True(result.Indexing.IsRetryable); + Assert.Contains("did not finish within", result.Indexing.Failure!.Reason, StringComparison.Ordinal); + gate.TrySetResult(); + } + + // ---------------------------------------------------------------- ineligible records + + [Fact] + public async Task A_quarantined_record_is_never_sent_to_a_provider() + { + // Its vector could never be returned by a search, so embedding it would hand a third party a + // task summary and lesson for nothing. + var harness = await Harness.WithCompletedRunAsync(reflector: new ThrowingReflector()); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(ExperienceIndexingOutcome.Ineligible, result.Indexing!.Outcome); + Assert.False(result.Indexing.IsRetryable); + Assert.Empty(harness.Generator.Requests); + Assert.Empty(harness.Index.Stored); + } + + // ---------------------------------------------------------------- replay + + [Fact] + public async Task An_already_finalized_replay_never_re_embeds_anything() + { + var harness = await Harness.WithCompletedRunAsync(); + + var first = await harness.FinalizeAsync(); + Assert.Equal(FinalizationOutcome.Validated, first.Outcome); + Assert.Single(harness.Generator.Requests); + + var replay = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.AlreadyFinalized, replay.Outcome); + Assert.Null(replay.Indexing); + Assert.Single(harness.Generator.Requests); + } + + [Fact] + public async Task A_finalization_that_never_commits_never_indexes() + { + var harness = await Harness.WithCompletedRunAsync(); + + // The host's storage decision refuses before anything is written, so there is nothing to index. + var result = await harness.FinalizeAsync(decision: StorageDecision.Deny("retention policy")); + + Assert.Equal(FinalizationOutcome.StorageDenied, result.Outcome); + Assert.Null(result.Indexing); + Assert.Empty(harness.Generator.Requests); + } + + [Fact] + public async Task Finalization_without_an_indexing_hook_reports_no_indexing_at_all() + { + var harness = await Harness.WithCompletedRunAsync(withHook: false); + + var result = await harness.FinalizeAsync(); + + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.Null(result.Indexing); + } + + // ---------------------------------------------------------------- harness + + /// Real capture, a real lifecycle service and a real indexing service over in-memory doubles. + private sealed class Harness + { + public required InMemoryExperienceCaptureService Capture { get; init; } + + public required RecordingStore Store { get; init; } + + public required FakeEmbeddingIndex Index { get; init; } + + public required FakeEmbeddingGenerator Generator { get; init; } + + public required ExperienceFinalizationService Service { get; init; } + + public Guid RunId { get; private set; } + + public static async Task WithCompletedRunAsync( + FakeEmbeddingIndex? index = null, + FakeEmbeddingGenerator? generator = null, + bool withHook = true, + IExperienceReflector? reflector = null, + TimeSpan? indexingTimeout = null) + { + var capture = new InMemoryExperienceCaptureService( + new DefaultSanitizer(PermissiveOptions), + new CaptureLimits(50, 50, 10_000, 10_000)); + var embeddingIndex = index ?? new FakeEmbeddingIndex(); + var embeddingGenerator = generator ?? new FakeEmbeddingGenerator(); + var store = new RecordingStore(embeddingIndex); + + var harness = new Harness + { + Capture = capture, + Store = store, + Index = embeddingIndex, + Generator = embeddingGenerator, + Service = new ExperienceFinalizationService( + capture, + reflector ?? new DefaultExperienceReflector(), + store, + new ExperienceLifecycleService(store), + withHook ? new ExperienceIndexingService(embeddingIndex, embeddingGenerator) : null, + indexingTimeout), + }; + + harness.RunId = Guid.NewGuid(); + Assert.Equal( + StartRunOutcome.Started, + capture.StartRun( + harness.RunId, + taskId: "task-1", + taskDescription: "a test task", + scope: TestScope, + environment: new EnvironmentFingerprint("host-1", "net10.0", "test-os", null, new Dictionary()), + provenance: new Provenance("unit-tests", "1.0.0", Now, null), + startedAt: Now).Outcome); + + Assert.Equal( + AppendAttemptOutcome.Recorded, + (await capture.AppendAttemptAsync( + harness.RunId, + new AppendAttemptRequest(Guid.NewGuid(), Now, TimeSpan.FromSeconds(1), [], "done", null))).Outcome); + + Assert.Equal( + CompleteRunOutcome.Recorded, + (await capture.CompleteRunAsync(harness.RunId, Guid.NewGuid(), RunExecutionStatus.Completed, Now.AddMinutes(1))).Outcome); + + return harness; + } + + public FinalizeExperienceRequest Request(StorageDecision? decision = null) => new( + RunId: RunId, + Authorization: Authorization, + ClosedRound: Round, + RequiredChecks: [new RequiredCheck("tests")], + Evidence: + [ + new Evidence( + EvidenceId: Guid.NewGuid(), + VerificationRoundId: Round.RoundId, + ArtifactRevision: ArtifactRevision, + CheckId: "tests", + Kind: "TestResult", + Result: CheckResult.Pass, + Producer: "ci", + Detail: null, + CapturedAt: Now), + ], + CurrentArtifactRevision: ArtifactRevision, + StorageDecision: decision ?? StorageDecision.Permit, + FinalizedAt: Now.AddMinutes(2)); + + public Task FinalizeAsync(StorageDecision? decision = null) => + Service.FinalizeAsync(Request(decision), CancellationToken.None); + } + + /// A reflector that always fails, so the record is committed as Quarantined. + private sealed class ThrowingReflector : IExperienceReflector + { + public Task ReflectAsync(ReflectionRequest request, CancellationToken cancellationToken) => + throw new InvalidOperationException("scripted reflector failure"); + } + + /// + /// A minimal in-memory record store that also keeps the embedding index's view of the canonical + /// world in step, so the conditional-write contract the index enforces is exercised against the + /// same revisions finalization actually committed. + /// + private sealed class RecordingStore(FakeEmbeddingIndex index) : IExperienceRecordStore + { + private readonly Dictionary _records = []; + + public Task CreateAsync( + AuthorizationContext authorization, + ExperienceRecord record, + CancellationToken cancellationToken) + { + if (!_records.TryAdd(record.ExperienceId, record)) + { + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Conflict, [])); + } + + Publish(record); + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Created, [])); + } + + public Task GetAsync( + AuthorizationContext authorization, + Scope scope, + Guid experienceId, + CancellationToken cancellationToken) => + Task.FromResult(_records.TryGetValue(experienceId, out var record) && record.Scope == scope + ? new ExperienceRecordGetResult(ExperienceStoreOutcome.Found, record, []) + : new ExperienceRecordGetResult(ExperienceStoreOutcome.NotFound, null, [])); + + public Task QueryAsync( + AuthorizationContext authorization, + ExperienceRecordQuery query, + CancellationToken cancellationToken) => + Task.FromResult(new ExperienceRecordQueryResult(ExperienceStoreOutcome.Found, [], [])); + + public Task CommitLifecycleEventAsync( + AuthorizationContext authorization, + Scope scope, + LifecycleEvent lifecycleEvent, + CancellationToken cancellationToken) + { + if (!_records.TryGetValue(lifecycleEvent.ExperienceRecordId, out var record) || record.Scope != scope) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.NotFound, 0, null, [])); + } + + if (record.Revision != lifecycleEvent.ExpectedRevision) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.StaleRevision, record.Revision, null, [])); + } + + var applied = lifecycleEvent.ExpectedRevision + 1; + var updated = record with + { + Status = lifecycleEvent.CurrentStatus, + Revision = applied, + UpdatedAt = lifecycleEvent.OccurredAt, + }; + + _records[record.ExperienceId] = updated; + Publish(updated); + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Committed, applied, null, [])); + } + + public Task GetHistoryAsync( + AuthorizationContext authorization, + Scope scope, + Guid experienceId, + CancellationToken cancellationToken) => + Task.FromResult(new ExperienceRecordHistoryResult(ExperienceStoreOutcome.Found, 0, [], [])); + + /// Mirrors a committed record into the index's view of the world, exactly as the real schema's join would see it. + private void Publish(ExperienceRecord record) => + index.Records[record.ExperienceId] = new FakeEmbeddingIndex.Row(record.Revision, ExperienceRetrievalSummary.For(record)); + } +} diff --git a/tests/AgentExperience.Core.Tests/HybridRetrievalTests.cs b/tests/AgentExperience.Core.Tests/HybridRetrievalTests.cs new file mode 100644 index 0000000..1693797 --- /dev/null +++ b/tests/AgentExperience.Core.Tests/HybridRetrievalTests.cs @@ -0,0 +1,738 @@ +using AgentExperience.Core.Retrieval; + +namespace AgentExperience.Core.Tests; + +/// +/// Covers the vector half of : one test per row of the +/// story's I/O and edge-case matrix that retrieval owns -- a hybrid match, a model mismatch, a +/// dimension mismatch, a vector channel that fails, and both channels empty -- plus the rules that +/// keep the merge honest: shared eligibility, dedupe by ID, highest normalized relevance wins, one +/// timeout over both channels, and no sixth ranking weight. +/// +public class HybridRetrievalTests +{ + private static readonly DateTimeOffset Now = new(2026, 9, 21, 12, 0, 0, TimeSpan.Zero); + + private static readonly Scope RequestScope = new("tenant-1", "app-1", "project-1"); + + private static readonly AuthorizationContext Authorization = new("tenant-1", "host-principal", ["experience:read"], Now); + + private const string TaskText = "resolve a refund ticket"; + + // ---------------------------------------------------------------- matrix: hybrid match + + [Fact] + public async Task Both_channels_contribute_and_the_result_is_deduplicated_and_ranked_once() + { + var shared = Record(Id(1)); + var textOnly = Record(Id(2)); + var vectorOnly = Record(Id(3)); + + var service = Service( + text: [Candidate(shared, 0.4), Candidate(textOnly, 0.3)], + vector: [Candidate(shared, 0.9), Candidate(vectorOnly, 0.8)]); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.False(result.TextOnly); + Assert.Null(result.VectorFallback); + Assert.Equal([Id(1), Id(2), Id(3)], result.Records.Select(r => r.Record.ExperienceId).Order()); + + // Deduplicated by ID: the shared record is ranked exactly once, and on its *higher* + // normalized relevance -- the vector channel's 0.9, not the text channel's 0.4. + var ranked = result.Records.Single(r => r.Record.ExperienceId == Id(1)); + Assert.Equal(0.9, ranked.Components.Single(c => c.Kind == RankingComponentKind.Relevance).Value); + } + + [Fact] + public async Task A_record_only_the_vector_channel_found_is_still_ranked_on_all_five_components() + { + // Being found by meaning rather than by words changes nothing about how a record is scored: + // there is no sixth axis and no "found semantically" bonus. + var record = Record(Id(1), confidence: 0.8); + var service = Service(text: [], vector: [Candidate(record, 0.75)]); + + var result = await service.RetrieveAsync(Request()); + + var ranked = Assert.Single(result.Records); + Assert.Equal( + [ + RankingComponentKind.Relevance, + RankingComponentKind.Confidence, + RankingComponentKind.Recency, + RankingComponentKind.Status, + RankingComponentKind.EnvironmentCompatibility, + ], + ranked.Components.Select(component => component.Kind)); + Assert.Equal(5, ranked.Components.Count); + Assert.Equal([0.75, 0.8, 1d, ExperienceRetrievalService.ValidatedStatusScore, 1d], ranked.Components.Select(c => c.Value)); + Assert.Equal(ranked.Components.Sum(c => c.Contribution), ranked.Score, 12); + } + + [Fact] + public async Task The_higher_normalized_relevance_wins_whichever_channel_it_came_from() + { + var record = Record(Id(1)); + var textWins = await Service(text: [Candidate(record, 0.95)], vector: [Candidate(record, 0.2)]).RetrieveAsync(Request()); + var vectorWins = await Service(text: [Candidate(record, 0.2)], vector: [Candidate(record, 0.95)]).RetrieveAsync(Request()); + + Assert.Equal(0.95, Relevance(textWins)); + Assert.Equal(0.95, Relevance(vectorWins)); + } + + // ---------------------------------------------------------------- matrix: shared eligibility + + [Theory] + [InlineData(ExperienceStatus.Candidate)] + [InlineData(ExperienceStatus.Quarantined)] + [InlineData(ExperienceStatus.Revoked)] + public async Task An_ineligible_status_is_excluded_from_the_vector_channel_exactly_as_from_the_text_one(ExperienceStatus status) + { + var ineligible = Record(Id(1), status: status, confidence: 1d); + var result = await Service(text: [], vector: [Candidate(ineligible, 1d)]).RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Empty(result.Records); + Assert.Equal( + new ExcludedExperience(ineligible.ExperienceId, RetrievalExclusionReason.IneligibleStatus), + Assert.Single(result.Excluded)); + } + + [Fact] + public async Task The_vector_channel_is_asked_for_the_same_statuses_confidence_floor_and_ceiling_as_the_text_one() + { + var index = new FakeEmbeddingIndex(); + var policy = RetrievalPolicy.Default with { MinimumConfidence = 0.75, CandidateLimit = 7 }; + var source = new RecordingCandidateSource([]); + var service = new ExperienceRetrievalService( + source, policy, RankingWeights.Default, new FrozenClock(Now), index, new FakeEmbeddingGenerator()); + + await service.RetrieveAsync(Request()); + + var textQuery = Assert.Single(source.Queries); + var vectorQuery = Assert.Single(index.Queries); + + Assert.Equal(textQuery.Scope, vectorQuery.Scope); + Assert.Equal(textQuery.EligibleStatuses, vectorQuery.EligibleStatuses); + Assert.Equal(textQuery.MinimumConfidence, vectorQuery.MinimumConfidence); + Assert.Equal(textQuery.Limit, vectorQuery.Limit); + Assert.Equal(8, vectorQuery.Limit); + Assert.Equal("fake-embed-v1", vectorQuery.ModelId); + Assert.Equal(FakeEmbeddingGenerator.VectorFor(TaskText, 4).ToArray(), vectorQuery.Vector.ToArray()); + } + + [Fact] + public async Task A_record_the_expiry_or_environment_check_removes_is_excluded_whichever_channel_found_it() + { + var expired = Record(Id(1), updatedAt: Now - TimeSpan.FromDays(8)); + var mismatched = Record(Id(2), metadata: new Dictionary { ["region"] = "eu-west" }); + var policy = RetrievalPolicy.Default with { MaxAge = TimeSpan.FromDays(7) }; + + var result = await Service( + text: [], + vector: [Candidate(expired, 1d), Candidate(mismatched, 1d)], + policy: policy) + .RetrieveAsync(Request(required: new Dictionary { ["region"] = "us-east" })); + + Assert.Empty(result.Records); + Assert.Equal( + [ + new ExcludedExperience(Id(1), RetrievalExclusionReason.Expired), + new ExcludedExperience(Id(2), RetrievalExclusionReason.EnvironmentMismatch), + ], + result.Excluded); + } + + // ---------------------------------------------------------------- matrix: model mismatch + + [Fact] + public async Task A_model_mismatch_gives_an_explicit_text_only_result_and_keeps_the_text_candidates() + { + var record = Record(Id(1)); + var service = Service( + text: [Candidate(record, 0.4)], + onSearch: _ => new(ExperienceVectorSearchOutcome.ModelMismatch, [], [])); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.True(result.TextOnly); + Assert.Equal(TextOnlyReason.ModelMismatch, result.VectorFallback!.Reason); + Assert.Null(result.Failure); + Assert.Equal(Id(1), Assert.Single(result.Records).Record.ExperienceId); + } + + // ---------------------------------------------------------------- matrix: dimension mismatch + + [Fact] + public async Task A_dimension_mismatch_gives_an_explicit_text_only_result() + { + var service = Service( + text: [Candidate(Record(Id(1)), 0.4)], + onSearch: _ => new(ExperienceVectorSearchOutcome.DimensionMismatch, [], [])); + + var result = await service.RetrieveAsync(Request()); + + Assert.True(result.TextOnly); + Assert.Equal(TextOnlyReason.DimensionMismatch, result.VectorFallback!.Reason); + Assert.Single(result.Records); + } + + [Fact] + public async Task A_provider_whose_query_vector_is_the_wrong_width_never_reaches_the_index_at_all() + { + // No incompatible comparison is attempted: the mismatch is caught before a query is issued. + var index = new FakeEmbeddingIndex(); + var service = new ExperienceRetrievalService( + new RecordingCandidateSource([Candidate(Record(Id(1)), 0.4)]), + RetrievalPolicy.Default, + RankingWeights.Default, + new FrozenClock(Now), + index, + new FakeEmbeddingGenerator { ReturnDimension = 3 }); + + var result = await service.RetrieveAsync(Request()); + + Assert.True(result.TextOnly); + Assert.Equal(TextOnlyReason.ProviderUnavailable, result.VectorFallback!.Reason); + Assert.Empty(index.Queries); + Assert.Single(result.Records); + } + + // ---------------------------------------------------------------- matrix: vector channel fails + + [Fact] + public async Task A_provider_that_throws_gives_a_text_only_result_flagged_with_the_provider_as_the_reason() + { + var service = Service( + text: [Candidate(Record(Id(1)), 0.4)], + generator: new FakeEmbeddingGenerator { Throws = FakeEmbeddingGenerator.ThrownException }); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.True(result.TextOnly); + Assert.Equal(TextOnlyReason.ProviderUnavailable, result.VectorFallback!.Reason); + Assert.Same(FakeEmbeddingGenerator.ThrownException, result.VectorFallback.Exception); + Assert.Single(result.Records); + } + + [Fact] + public async Task A_vector_search_that_throws_gives_a_text_only_result_and_the_text_candidates_still_come_back() + { + var index = new FakeEmbeddingIndex { SearchThrows = FakeEmbeddingIndex.ThrownException }; + var service = new ExperienceRetrievalService( + new RecordingCandidateSource([Candidate(Record(Id(1)), 0.4), Candidate(Record(Id(2)), 0.3)]), + RetrievalPolicy.Default, + RankingWeights.Default, + new FrozenClock(Now), + index, + new FakeEmbeddingGenerator()); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.True(result.TextOnly); + Assert.Equal(TextOnlyReason.VectorSearchFailed, result.VectorFallback!.Reason); + Assert.Same(FakeEmbeddingIndex.ThrownException, result.VectorFallback.Exception); + Assert.Equal(2, result.Records.Count); + } + + [Theory] + [InlineData(ExperienceVectorSearchOutcome.Denied)] + [InlineData(ExperienceVectorSearchOutcome.Invalid)] + public async Task A_refused_vector_search_is_a_text_only_fallback_not_a_failed_retrieval(ExperienceVectorSearchOutcome outcome) + { + var service = Service( + text: [Candidate(Record(Id(1)), 0.4)], + onSearch: _ => new(outcome, [], [])); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Equal(TextOnlyReason.VectorSearchFailed, result.VectorFallback!.Reason); + Assert.Single(result.Records); + } + + [Fact] + public async Task A_text_channel_failure_still_ends_the_call_even_when_the_vector_channel_answered() + { + // The vector channel cannot stand in for the text one: answering from vectors alone would be + // a result the caller never asked for. + var index = new FakeEmbeddingIndex { OnSearch = _ => new(ExperienceVectorSearchOutcome.Found, [Candidate(Record(Id(1)), 0.9)], []) }; + var service = new ExperienceRetrievalService( + new RecordingCandidateSource((_, _) => throw new InvalidOperationException("boom")), + RetrievalPolicy.Default, + RankingWeights.Default, + new FrozenClock(Now), + index, + new FakeEmbeddingGenerator()); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Failed, result.Outcome); + Assert.Empty(result.Records); + Assert.NotNull(result.Failure); + } + + // ---------------------------------------------------------------- matrix: both channels empty + + [Fact] + public async Task Nothing_matching_either_channel_is_a_completed_empty_result_not_a_failure() + { + var result = await Service(text: [], vector: []).RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Empty(result.Records); + Assert.Empty(result.Excluded); + Assert.Null(result.Failure); + Assert.False(result.TextOnly); + Assert.False(result.TimedOut); + } + + // ---------------------------------------------------------------- fail-closed, both channels + + [Fact] + public async Task A_vector_candidate_outside_the_requested_scope_empties_the_whole_result() + { + var foreign = Record(Id(1), scope: new Scope("tenant-1", "app-1", "other-project")); + var result = await Service(text: [Candidate(Record(Id(2)), 0.4)], vector: [Candidate(foreign, 0.9)]).RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Failed, result.Outcome); + Assert.Empty(result.Records); + Assert.Contains("embedding index", result.Failure!.Reason, StringComparison.Ordinal); + } + + [Fact] + public async Task The_same_record_twice_from_one_channel_is_still_fail_closed() + { + var record = Record(Id(1)); + var result = await Service(text: [], vector: [Candidate(record, 0.9), Candidate(record, 0.8)]).RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Failed, result.Outcome); + Assert.Contains("more than once", result.Failure!.Reason, StringComparison.Ordinal); + } + + [Fact] + public async Task An_unreadable_vector_candidate_empties_the_whole_result() + { + var result = await Service(text: [], vector: [new ExperienceCandidate(null!, 0.9)]).RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Failed, result.Outcome); + Assert.Contains("could not be read", result.Failure!.Reason, StringComparison.Ordinal); + } + + // ---------------------------------------------------------------- bounds shared by both channels + + [Fact] + public async Task Either_channel_reaching_the_ceiling_marks_the_result_truncated() + { + var policy = RetrievalPolicy.Default with { CandidateLimit = 2 }; + var overflowing = Enumerable.Range(1, 3).Select(n => Candidate(Record(Id(n)), 0.5)).ToArray(); + + var fromVector = await Service(text: [], vector: overflowing, policy: policy).RetrieveAsync(Request(limit: 2)); + var fromText = await Service(text: overflowing, vector: [], policy: policy).RetrieveAsync(Request(limit: 2)); + var neither = await Service(text: [overflowing[0]], vector: [overflowing[1]], policy: policy).RetrieveAsync(Request(limit: 2)); + + Assert.True(fromVector.Truncated); + Assert.True(fromText.Truncated); + Assert.False(neither.Truncated); + + // The probe candidate past the ceiling is never ranked, in either channel. + Assert.Equal(2, fromVector.Records.Count); + Assert.DoesNotContain(Id(3), fromVector.Excluded.Select(e => e.ExperienceId)); + } + + [Fact] + public async Task The_whole_hybrid_call_is_bounded_by_the_one_timeout_and_a_hanging_provider_is_not_an_exception() + { + var clock = new ManualClock(Now); + var gate = new TaskCompletionSource(); + var service = new ExperienceRetrievalService( + new RecordingCandidateSource([Candidate(Record(Id(1)), 0.4)]), + RetrievalPolicy.Default, + RankingWeights.Default, + clock, + new FakeEmbeddingIndex(), + new FakeEmbeddingGenerator { Gate = gate }); + + var retrieval = service.RetrieveAsync(Request(correlationId: "trace-1")); + clock.Advance(RetrievalPolicy.DefaultTimeout + TimeSpan.FromMilliseconds(1)); + var result = await retrieval; + + Assert.Equal(RetrievalOutcome.TimedOut, result.Outcome); + Assert.True(result.TimedOut); + Assert.Equal("trace-1", result.CorrelationId); + Assert.Empty(result.Records); + gate.TrySetResult(); + } + + // ---------------------------------------------------------------- no vector channel at all + + [Fact] + public async Task A_deployment_with_no_vector_channel_says_so_explicitly_on_every_result() + { + var service = new ExperienceRetrievalService( + new RecordingCandidateSource([Candidate(Record(Id(1)), 0.4)]), + RetrievalPolicy.Default, + RankingWeights.Default, + new FrozenClock(Now)); + + var result = await service.RetrieveAsync(Request()); + + Assert.False(service.HybridEnabled); + Assert.True(result.TextOnly); + Assert.Equal(TextOnlyReason.NotConfigured, result.VectorFallback!.Reason); + Assert.Single(result.Records); + } + + [Fact] + public async Task Half_a_vector_channel_is_no_vector_channel() + { + // An index with no generator (or the reverse) cannot produce a comparison, so it is the + // not-configured fallback rather than a failure reported on every single call. + var indexOnly = new ExperienceRetrievalService( + new RecordingCandidateSource([]), RetrievalPolicy.Default, RankingWeights.Default, new FrozenClock(Now), + new FakeEmbeddingIndex(), embeddingGenerator: null); + var generatorOnly = new ExperienceRetrievalService( + new RecordingCandidateSource([]), RetrievalPolicy.Default, RankingWeights.Default, new FrozenClock(Now), + embeddingIndex: null, new FakeEmbeddingGenerator()); + + Assert.False(indexOnly.HybridEnabled); + Assert.False(generatorOnly.HybridEnabled); + Assert.Equal(TextOnlyReason.NotConfigured, (await indexOnly.RetrieveAsync(Request())).VectorFallback!.Reason); + Assert.Equal(TextOnlyReason.NotConfigured, (await generatorOnly.RetrieveAsync(Request())).VectorFallback!.Reason); + } + + [Fact] + public async Task A_denied_scope_never_reaches_either_channel() + { + var index = new FakeEmbeddingIndex(); + var source = new RecordingCandidateSource([]); + var generator = new FakeEmbeddingGenerator(); + var service = new ExperienceRetrievalService( + source, RetrievalPolicy.Default, RankingWeights.Default, new FrozenClock(Now), index, generator); + + var result = await service.RetrieveAsync(new RetrieveExperienceRequest( + new AuthorizationContext("tenant-1", "p", [], Now, ProjectId: "elsewhere"), RequestScope, TaskText)); + + Assert.Equal(RetrievalOutcome.Denied, result.Outcome); + Assert.Empty(source.Queries); + Assert.Empty(index.Queries); + Assert.Empty(generator.Requests); + } + + [Fact] + public async Task A_provider_that_cancels_for_its_own_reasons_is_a_fallback_not_a_failed_retrieval() + { + // An HttpClient request timeout surfaces as a TaskCanceledException with the caller's token + // untouched. Letting that escape would turn every retrieval against a merely slow provider into + // Failed with no records -- exactly the answer the text-only fallback exists to prevent. + var service = Service( + text: [Candidate(Record(Id(1)), 0.4)], + generator: new FakeEmbeddingGenerator { Throws = new TaskCanceledException("provider request timeout") }); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Equal(TextOnlyReason.ProviderUnavailable, result.VectorFallback!.Reason); + Assert.Equal(Id(1), Assert.Single(result.Records).Record.ExperienceId); + } + + [Fact] + public async Task A_vector_search_that_cancels_for_its_own_reasons_is_a_fallback_too() + { + var index = new FakeEmbeddingIndex { SearchThrows = new OperationCanceledException("index-side timeout") }; + var service = new ExperienceRetrievalService( + new RecordingCandidateSource([Candidate(Record(Id(1)), 0.4)]), + RetrievalPolicy.Default, + RankingWeights.Default, + new FrozenClock(Now), + index, + new FakeEmbeddingGenerator()); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Equal(TextOnlyReason.VectorSearchFailed, result.VectorFallback!.Reason); + Assert.Single(result.Records); + } + + [Fact] + public async Task The_callers_own_cancellation_still_propagates_from_the_vector_channel() + { + using var cancellation = new CancellationTokenSource(); + var gate = new TaskCompletionSource(); + var service = Service(text: [], generator: new FakeEmbeddingGenerator { Gate = gate }); + + var retrieval = service.RetrieveAsync(Request(), cancellation.Token); + await cancellation.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => retrieval); + gate.TrySetResult(); + } + + [Fact] + public async Task A_non_finite_query_vector_falls_back_without_a_database_round_trip() + { + var index = new FakeEmbeddingIndex(); + var service = new ExperienceRetrievalService( + new RecordingCandidateSource([Candidate(Record(Id(1)), 0.4)]), + RetrievalPolicy.Default, + RankingWeights.Default, + new FrozenClock(Now), + index, + new FakeEmbeddingGenerator { ReturnNonFinite = true }); + + var result = await service.RetrieveAsync(Request()); + + Assert.Equal(TextOnlyReason.ProviderUnavailable, result.VectorFallback!.Reason); + Assert.Empty(index.Queries); + Assert.Single(result.Records); + } + + [Fact] + public async Task The_merge_keeps_the_first_channels_record_snapshot_and_only_raises_the_relevance() + { + // The two channels read the record at different instants. Taking the other snapshot because it + // scored higher would let eligibility be decided on the staler of the two reads. + var textSnapshot = Record(Id(1), status: ExperienceStatus.Validated, confidence: 0.9); + var vectorSnapshot = Record(Id(1), status: ExperienceStatus.Revoked, confidence: 0.1); + + var result = await Service(text: [Candidate(textSnapshot, 0.2)], vector: [Candidate(vectorSnapshot, 0.95)]) + .RetrieveAsync(Request()); + + var ranked = Assert.Single(result.Records); + Assert.Same(textSnapshot, ranked.Record); + Assert.Equal(0.95, ranked.Components.Single(c => c.Kind == RankingComponentKind.Relevance).Value); + Assert.Equal(0.9, ranked.Components.Single(c => c.Kind == RankingComponentKind.Confidence).Value); + } + + // ---------------------------------------------------------------- the flag on every outcome + + [Fact] + public async Task A_denied_or_timed_out_result_still_says_whether_a_vector_channel_exists_at_all() + { + var elsewhere = new AuthorizationContext("tenant-1", "p", [], Now, ProjectId: "elsewhere"); + var deniedRequest = new RetrieveExperienceRequest(elsewhere, RequestScope, TaskText); + + var textOnlyDenied = await TextOnlyService().RetrieveAsync(deniedRequest); + var hybridDenied = await Service(text: [], vector: []).RetrieveAsync(deniedRequest); + + Assert.Equal(RetrievalOutcome.Denied, textOnlyDenied.Outcome); + Assert.True(textOnlyDenied.TextOnly); + Assert.Equal(TextOnlyReason.NotConfigured, textOnlyDenied.VectorFallback!.Reason); + + // A wired-up channel simply did not contribute, and the Denied outcome already says why. + Assert.Equal(RetrievalOutcome.Denied, hybridDenied.Outcome); + Assert.False(hybridDenied.TextOnly); + Assert.Null(hybridDenied.VectorFallback); + + var textOnlyTimedOut = await TimingOutService(hybrid: false); + var hybridTimedOut = await TimingOutService(hybrid: true); + + Assert.Equal(RetrievalOutcome.TimedOut, textOnlyTimedOut.Outcome); + Assert.Equal(TextOnlyReason.NotConfigured, textOnlyTimedOut.VectorFallback!.Reason); + Assert.Equal(RetrievalOutcome.TimedOut, hybridTimedOut.Outcome); + Assert.Null(hybridTimedOut.VectorFallback); + } + + [Fact] + public void The_ranking_weights_still_have_exactly_five_axes() + { + Assert.Equal(5, Enum.GetValues().Length); + var weights = RankingWeights.Default; + Assert.Equal(1d, weights.Relevance + weights.Confidence + weights.Recency + weights.Status + weights.EnvironmentCompatibility, 12); + } + + // ---------------------------------------------------------------- helpers + + private static Guid Id(int n) => Guid.Parse(FormattableString.Invariant($"00000000-0000-0000-0000-{n:000000000000}")); + + private static double Relevance(ExperienceRetrievalResult result) => + Assert.Single(result.Records).Components.Single(c => c.Kind == RankingComponentKind.Relevance).Value; + + private static ExperienceCandidate Candidate(ExperienceRecord record, double relevance) => new(record, relevance); + + private static ExperienceRetrievalService TextOnlyService() => new( + new RecordingCandidateSource([Candidate(Record(Id(1)), 0.4)]), + RetrievalPolicy.Default, + RankingWeights.Default, + new FrozenClock(Now)); + + /// + /// Drives a retrieval to its timeout, hybrid or not. The text channel is what blocks in both + /// cases, so the two differ only in whether a vector channel is wired in at all -- which is exactly + /// what the timed-out result has to keep reporting. + /// + private static async Task TimingOutService(bool hybrid) + { + var clock = new ManualClock(Now); + var blocked = new RecordingCandidateSource(async (_, token) => + { + await Task.Delay(Timeout.Infinite, token); + return new ExperienceCandidateSearchResult(ExperienceStoreOutcome.Found, [], []); + }); + + var service = new ExperienceRetrievalService( + blocked, + RetrievalPolicy.Default, + RankingWeights.Default, + clock, + hybrid ? new FakeEmbeddingIndex() : null, + hybrid ? new FakeEmbeddingGenerator() : null); + + var retrieval = service.RetrieveAsync(Request()); + clock.Advance(RetrievalPolicy.DefaultTimeout + TimeSpan.FromMilliseconds(1)); + return await retrieval; + } + + private static RetrieveExperienceRequest Request( + IReadOnlyDictionary? required = null, + string? correlationId = null, + int? limit = null) => new(Authorization, RequestScope, TaskText, required, correlationId, limit); + + private static ExperienceRetrievalService Service( + IReadOnlyList text, + IReadOnlyList? vector = null, + Func? onSearch = null, + FakeEmbeddingGenerator? generator = null, + RetrievalPolicy? policy = null) => new( + new RecordingCandidateSource(text), + policy ?? RetrievalPolicy.Default, + RankingWeights.Default, + new FrozenClock(Now), + new FakeEmbeddingIndex + { + OnSearch = onSearch ?? (_ => new(ExperienceVectorSearchOutcome.Found, vector ?? [], [])), + }, + generator ?? new FakeEmbeddingGenerator()); + + private static ExperienceRecord Record( + Guid id, + Scope? scope = null, + ExperienceStatus status = ExperienceStatus.Validated, + double confidence = 0.5, + DateTimeOffset? updatedAt = null, + IReadOnlyDictionary? metadata = null) => new( + ExperienceId: id, + SourceRunId: Guid.NewGuid(), + Scope: scope ?? RequestScope, + TaskId: "refund-ticket", + TaskSummary: "Resolve a refund ticket", + Attempts: [], + Outcome: new Outcome(TaskVerificationStatus.Verified, [], "checks passed", Now), + CompletionScore: 1, + Reflection: null, + Environment: new EnvironmentFingerprint("worker-01", "10.0.0", "linux-x64", null, metadata ?? new Dictionary()), + Provenance: new Provenance("tests", null, Now, null), + Status: status, + ReuseConfidence: confidence, + SupportingValidations: 1, + Contradictions: 0, + Revision: 1, + CreatedAt: updatedAt ?? Now, + UpdatedAt: updatedAt ?? Now); + + /// A text candidate source that answers with whatever the test scripted, recording what it was asked. + private sealed class RecordingCandidateSource( + Func> onSearch) + : IExperienceCandidateSource + { + public RecordingCandidateSource(IReadOnlyList candidates) + : this((_, _) => Task.FromResult(new ExperienceCandidateSearchResult(ExperienceStoreOutcome.Found, candidates, []))) + { + } + + public List Queries { get; } = []; + + public Task SearchAsync( + AuthorizationContext authorization, + ExperienceCandidateQuery query, + CancellationToken cancellationToken) + { + Assert.NotNull(authorization); + lock (Queries) + { + Queries.Add(query); + } + + return onSearch(query, cancellationToken); + } + } + + /// A clock whose timers fire only when the test advances it, so a timeout is deterministic. + private sealed class ManualClock(DateTimeOffset start) : TimeProvider + { + private readonly List _timers = []; + private DateTimeOffset _now = start; + + public override DateTimeOffset GetUtcNow() => _now; + + public override long GetTimestamp() => _now.UtcTicks; + + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = new ManualTimer(callback, state, dueTime); + lock (_timers) + { + _timers.Add(timer); + } + + return timer; + } + + public void Advance(TimeSpan by) + { + _now += by; + ManualTimer[] due; + lock (_timers) + { + due = [.. _timers]; + } + + foreach (var timer in due) + { + timer.MaybeFire(by); + } + } + + private sealed class ManualTimer(TimerCallback callback, object? state, TimeSpan dueTime) : ITimer + { + private TimeSpan _remaining = dueTime; + private bool _fired; + + public bool Change(TimeSpan due, TimeSpan period) + { + _remaining = due; + return true; + } + + public void MaybeFire(TimeSpan elapsed) + { + if (_fired || _remaining == Timeout.InfiniteTimeSpan) + { + return; + } + + _remaining -= elapsed; + if (_remaining <= TimeSpan.Zero) + { + _fired = true; + callback(state); + } + } + + public void Dispose() => _fired = true; + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } + } +} diff --git a/tests/AgentExperience.Core.Tests/IndexingTestDoubles.cs b/tests/AgentExperience.Core.Tests/IndexingTestDoubles.cs new file mode 100644 index 0000000..08b3f61 --- /dev/null +++ b/tests/AgentExperience.Core.Tests/IndexingTestDoubles.cs @@ -0,0 +1,266 @@ +using System.Security.Cryptography; +using System.Text; + +namespace AgentExperience.Core.Tests; + +/// +/// A deterministic : the vector is derived from a SHA-256 +/// of the text, so the same text always embeds to the same vector and semantically-unrelated texts +/// embed to unrelated directions -- with no model, no network, and no credentials. Every integration +/// test in this story runs on this rather than on a live provider. +/// +internal sealed class FakeEmbeddingGenerator : IExperienceEmbeddingGenerator +{ + /// The well-known failure a scripted provider outage throws, so a test can assert on identity rather than on a message. + public static readonly InvalidOperationException ThrownException = new("scripted embedding provider failure"); + + public string ModelId { get; init; } = "fake-embed-v1"; + + public int Dimension { get; init; } = 4; + + /// When set, every call throws this instead of embedding. + public Exception? Throws { get; init; } + + /// When set, the returned vector has this many components instead of . + public int? ReturnDimension { get; init; } + + /// When set, the returned vector is the right width but holds a NaN. + public bool ReturnNonFinite { get; init; } + + /// When set, the call blocks on this before returning, so a test can hold the provider open. + public TaskCompletionSource? Gate { get; init; } + + /// Every text this generator was asked to embed, in order. + public List Requests { get; } = []; + + /// The vector this generator produces for , without going through the port. + public static ReadOnlyMemory VectorFor(string text, int dimension) + { + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(text)); + var vector = new float[dimension]; + for (var i = 0; i < dimension; i++) + { + // Two bytes per component, mapped into [-1, 1]. Deterministic and always finite. + var raw = (hash[(i * 2) % hash.Length] << 8) | hash[((i * 2) + 1) % hash.Length]; + vector[i] = ((raw / 65535f) * 2f) - 1f; + } + + return vector; + } + + public async Task> GenerateAsync(string text, CancellationToken cancellationToken) + { + lock (Requests) + { + Requests.Add(text); + } + + if (Gate is not null) + { + await Gate.Task.WaitAsync(cancellationToken); + } + + cancellationToken.ThrowIfCancellationRequested(); + + if (Throws is not null) + { + throw Throws; + } + + var vector = VectorFor(text, ReturnDimension ?? Dimension); + if (ReturnNonFinite) + { + var poisoned = vector.ToArray(); + poisoned[0] = float.NaN; + return poisoned; + } + + return vector; + } +} + +/// +/// An in-memory that reproduces the adapter's contract rather +/// than merely returning canned values: is the canonical world it can see, and a +/// write only lands while the named record is in it at exactly the named revision. Deleting a record +/// from is "the record was deleted"; bumping its revision is "the record moved +/// on". Everything else is scripted through the hooks. +/// +internal sealed class FakeEmbeddingIndex : IExperienceEmbeddingIndex +{ + /// The well-known failure a scripted index outage throws. + public static readonly ExperienceStoreException ThrownException = new("scripted embedding index failure"); + + /// One canonical record as the index can see it. + /// The record's current revision. A write only lands while it still matches. + /// The record's normalized retrieval summary. + /// The record's lifecycle status, which the scan filters on exactly as the search does. + /// The record's reuse confidence, filtered the same way. + public sealed record Row( + long Revision, + string Summary, + ExperienceStatus Status = ExperienceStatus.Validated, + double ReuseConfidence = 0.8); + + /// The records that currently exist, keyed by ID. A missing key is a deleted record. + public Dictionary Records { get; } = []; + + /// The vectors currently stored, keyed by record ID. + public Dictionary Vector)> Stored { get; } = []; + + /// Every write this index was asked to apply, in order -- including the ones it rejected. + public List Writes { get; } = []; + + /// Every scan this index was asked for, in order. + public List Scans { get; } = []; + + /// Every vector search this index was asked for, in order. + public List Queries { get; } = []; + + /// When set, every scan throws this. + public Exception? ScanThrows { get; init; } + + /// When set, every write throws this. + public Exception? WriteThrows { get; init; } + + /// When set, every search throws this. + public Exception? SearchThrows { get; init; } + + /// When set, every scan returns this outcome instead of listing anything. + public ExperienceStoreOutcome? ScanOutcome { get; init; } + + /// When set, runs just before a write is applied -- the seam for "the record moved while the write was in flight". + public Action? BeforeWrite { get; init; } + + /// When set, answers every vector search instead of the default empty result. + public Func? OnSearch { get; init; } + + public Task WriteAsync( + AuthorizationContext authorization, + ExperienceIndexWrite write, + CancellationToken cancellationToken) + { + Assert.NotNull(authorization); + Assert.NotNull(write); + Writes.Add(write); + + if (WriteThrows is not null) + { + throw WriteThrows; + } + + if (!authorization.Permits(write.Scope)) + { + return Task.FromResult(new ExperienceIndexWriteResult(ExperienceIndexOutcome.Denied, 0, [])); + } + + BeforeWrite?.Invoke(write); + + if (!Records.TryGetValue(write.ExperienceId, out var row)) + { + // The conditional INSERT ... SELECT has no source row, so nothing is written and nothing + // is recreated. + return Task.FromResult(new ExperienceIndexWriteResult(ExperienceIndexOutcome.Missing, 0, [])); + } + + if (row.Revision != write.Descriptor.SourceRevision) + { + return Task.FromResult(new ExperienceIndexWriteResult(ExperienceIndexOutcome.Stale, row.Revision, [])); + } + + Stored[write.ExperienceId] = (write.Descriptor, write.Vector); + return Task.FromResult(new ExperienceIndexWriteResult(ExperienceIndexOutcome.Written, 0, [])); + } + + public Task ScanAsync( + AuthorizationContext authorization, + ExperienceIndexScan scan, + CancellationToken cancellationToken) + { + Assert.NotNull(authorization); + Assert.NotNull(scan); + Scans.Add(scan); + + if (ScanThrows is not null) + { + throw ScanThrows; + } + + if (ScanOutcome is { } scripted) + { + return Task.FromResult(new ExperienceIndexScanResult(scripted, [], [])); + } + + if (!authorization.Permits(scan.Scope)) + { + return Task.FromResult(new ExperienceIndexScanResult(ExperienceStoreOutcome.Denied, [], [])); + } + + var ids = (scan.ExperienceIds is null ? Records.Keys : Records.Keys.Intersect(scan.ExperienceIds)) + .Where(id => scan.EligibleStatuses.Contains(Records[id].Status) && Records[id].ReuseConfidence >= scan.MinimumConfidence) + .Where(id => scan.StartAfterId is not { } after || id.CompareTo(after) > 0); + + var targets = ids + .Order() + .Take(scan.Limit) + .Select(id => new ExperienceIndexTarget( + id, + Records[id].Revision, + Records[id].Summary, + Stored.TryGetValue(id, out var stored) ? stored.Descriptor : null)) + .ToArray(); + + return Task.FromResult(new ExperienceIndexScanResult( + ExperienceStoreOutcome.Found, + targets, + [], + targets.Length > 0 ? targets[^1].ExperienceId : null)); + } + + public Task SearchAsync( + AuthorizationContext authorization, + ExperienceVectorQuery query, + CancellationToken cancellationToken) + { + Assert.NotNull(authorization); + Assert.NotNull(query); + lock (Queries) + { + Queries.Add(query); + } + + if (SearchThrows is not null) + { + throw SearchThrows; + } + + return Task.FromResult(OnSearch?.Invoke(query) + ?? new ExperienceVectorSearchResult(ExperienceVectorSearchOutcome.Found, [], [])); + } +} + +/// +/// A clock frozen at a known instant. Its timers never fire, so a test that does not mean to exercise +/// the retrieval timeout cannot accidentally hit one, and elapsed time is always exactly zero. +/// +internal sealed class FrozenClock(DateTimeOffset now) : TimeProvider +{ + public override DateTimeOffset GetUtcNow() => now; + + public override long GetTimestamp() => now.UtcTicks; + + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => new FrozenTimer(); + + private sealed class FrozenTimer : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) => true; + + public void Dispose() + { + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/tests/AgentExperience.Core.Tests/VerificationAggregatorTests.cs b/tests/AgentExperience.Core.Tests/VerificationAggregatorTests.cs index 75c9302..42f41b3 100644 --- a/tests/AgentExperience.Core.Tests/VerificationAggregatorTests.cs +++ b/tests/AgentExperience.Core.Tests/VerificationAggregatorTests.cs @@ -14,6 +14,9 @@ namespace AgentExperience.Core.Tests; /// public class VerificationAggregatorTests { + /// A required check with no expected kind, i.e. one any evidence kind may satisfy. + private static RequiredCheck Check(string checkId, string? expectedKind = null) => new(checkId, expectedKind); + private static Evidence MakeEvidence(Guid roundId, string artifactRevision, string checkId, CheckResult result, string producer = "evaluator") => new( EvidenceId: Guid.NewGuid(), @@ -36,7 +39,7 @@ public void AC2_all_required_checks_passing_in_the_closed_round_yields_Verified( MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass), }; - var result = VerificationAggregator.Aggregate(evidence, ["build", "tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Verified, result.Outcome.Status); Assert.Equal(1.0, result.CompletionScore); @@ -54,7 +57,7 @@ public void AC2_one_required_check_failing_yields_Failed_regardless_of_other_pas MakeEvidence(round.RoundId, round.ArtifactRevision, "lint", CheckResult.Pass), }; - var result = VerificationAggregator.Aggregate(evidence, ["build", "tests", "lint"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests"), Check("lint")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Failed, result.Outcome.Status); Assert.Contains("build", result.Outcome.Reason); @@ -67,7 +70,7 @@ public void AC2_a_required_check_with_no_evidence_at_all_yields_Unknown() var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); var evidence = new[] { MakeEvidence(round.RoundId, round.ArtifactRevision, "build", CheckResult.Pass) }; - var result = VerificationAggregator.Aggregate(evidence, ["build", "tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Equal(0.5, result.CompletionScore); @@ -82,7 +85,7 @@ public void AC1_evidence_for_a_CheckId_outside_the_required_set_never_contribute var required = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass); var unrelated = MakeEvidence(round.RoundId, round.ArtifactRevision, "lint", CheckResult.Fail); - var result = VerificationAggregator.Aggregate([required, unrelated], ["tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([required, unrelated], [Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Verified, result.Outcome.Status); Assert.Same(required, Assert.Single(result.Outcome.Evidence)); @@ -95,7 +98,7 @@ public void Outcome_evidence_keeps_production_order_even_when_required_checks_ar var producedFirst = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass); var producedSecond = MakeEvidence(round.RoundId, round.ArtifactRevision, "build", CheckResult.Pass); - var result = VerificationAggregator.Aggregate([producedFirst, producedSecond], ["build", "tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([producedFirst, producedSecond], [Check("build"), Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal([producedFirst, producedSecond], result.Outcome.Evidence); } @@ -106,7 +109,7 @@ public void An_evidence_Result_outside_the_defined_CheckResult_values_resolves_t var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); var evidence = new[] { MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", (CheckResult)999) }; - var result = VerificationAggregator.Aggregate(evidence, ["tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Equal(0.0, result.CompletionScore); @@ -119,7 +122,7 @@ public void Duplicate_required_check_ids_throw_rather_than_skewing_the_completio var evidence = new[] { MakeEvidence(round.RoundId, round.ArtifactRevision, "build", CheckResult.Pass) }; Assert.Throws(() => - VerificationAggregator.Aggregate(evidence, ["build", "build", "tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + VerificationAggregator.Aggregate(evidence, [Check("build"), Check("build"), Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); } [Fact] @@ -129,7 +132,7 @@ public void A_null_evidence_entry_throws_rather_than_being_silently_dropped() var evidence = new[] { MakeEvidence(round.RoundId, round.ArtifactRevision, "build", CheckResult.Pass), null! }; Assert.Throws(() => - VerificationAggregator.Aggregate(evidence, ["build"], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + VerificationAggregator.Aggregate(evidence, [Check("build")], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); } [Fact] @@ -142,7 +145,7 @@ public void AC2_a_required_check_whose_only_evidence_is_Unknown_yields_overall_U MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Unknown), }; - var result = VerificationAggregator.Aggregate(evidence, ["build", "tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); } @@ -172,7 +175,7 @@ public void AC3_a_later_closed_round_can_verify_success_after_an_earlier_round_f var laterPass = MakeEvidence(laterRoundId, artifactRevision, "tests", CheckResult.Pass); var allEvidence = new[] { earlierFailure, laterPass }; - var result = VerificationAggregator.Aggregate(allEvidence, ["tests"], closedRound, artifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(allEvidence, [Check("tests")], closedRound, artifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Verified, result.Outcome.Status); // The earlier round's failing evidence never contributes to the outcome... @@ -193,7 +196,7 @@ public void AC3_AC4_evidence_from_a_different_artifact_revision_never_contribute // contribute even though VerificationRoundId matches. var staleRevisionEvidence = MakeEvidence(roundId, "rev-1", "tests", CheckResult.Pass); - var result = VerificationAggregator.Aggregate([staleRevisionEvidence], ["tests"], closedRound, "rev-2", DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([staleRevisionEvidence], [Check("tests")], closedRound, "rev-2", DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Empty(result.Outcome.Evidence); @@ -207,7 +210,7 @@ public void AC3_evidence_from_a_non_selected_round_id_never_contributes_even_und var otherRoundEvidence = MakeEvidence(Guid.NewGuid(), artifactRevision, "tests", CheckResult.Pass); - var result = VerificationAggregator.Aggregate([otherRoundEvidence], ["tests"], closedRound, artifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([otherRoundEvidence], [Check("tests")], closedRound, artifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Empty(result.Outcome.Evidence); @@ -216,7 +219,7 @@ public void AC3_evidence_from_a_non_selected_round_id_never_contributes_even_und [Fact] public void AC4_no_closed_round_yields_Unknown() { - var result = VerificationAggregator.Aggregate([], ["tests"], null, "rev-1", DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([], [Check("tests")], null, "rev-1", DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Equal(0.0, result.CompletionScore); @@ -229,7 +232,7 @@ public void AC4_a_closed_round_for_a_different_artifact_revision_than_current_is var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); var evidence = new[] { MakeEvidence(round.RoundId, "rev-2", "tests", CheckResult.Pass) }; - var result = VerificationAggregator.Aggregate(evidence, ["tests"], round, "rev-2", DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("tests")], round, "rev-2", DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); } @@ -244,7 +247,7 @@ public void AC4_mixed_Pass_and_Fail_evidence_for_the_same_required_check_resolve MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Fail), }; - var result = VerificationAggregator.Aggregate(evidence, ["tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Failed, result.Outcome.Status); // Both pieces of conflicting evidence are retained for audit, not discarded. @@ -262,7 +265,7 @@ public void AC5_completion_score_is_the_fraction_of_required_checks_that_conclus // "lint" has no evidence at all -> Unknown, so overall stays Unknown even though 2/3 passed. }; - var result = VerificationAggregator.Aggregate(evidence, ["build", "tests", "lint"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests"), Check("lint")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); // completion score alone never grants verification Assert.Equal(2.0 / 3.0, result.CompletionScore, precision: 10); @@ -297,12 +300,102 @@ public void AC6_an_evaluators_own_internal_failure_is_caught_upstream_and_aggreg Assert.Equal(CheckResult.Unknown, produced.Result); Assert.False(string.IsNullOrWhiteSpace(produced.Detail)); - var result = VerificationAggregator.Aggregate([produced], ["workflow-completes"], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + var result = VerificationAggregator.Aggregate([produced], [Check("workflow-completes")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); Assert.Same(produced, Assert.Single(result.Outcome.Evidence)); } + [Fact] + public void A_required_check_naming_an_ExpectedKind_ignores_evidence_of_any_other_kind() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + + // A human approval claiming the check the task declared must be answered by a test run. Under + // CheckId-only matching this would have verified the task; the expected kind stops it. + var wrongKind = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass) with { Kind = "HumanApproval" }; + + var result = VerificationAggregator.Aggregate( + [wrongKind], [Check("tests", "TestResult")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + + Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); + Assert.Equal(0.0, result.CompletionScore); + Assert.Empty(result.Outcome.Evidence); + } + + [Fact] + public void A_required_check_naming_an_ExpectedKind_is_satisfied_by_matching_evidence() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + var matching = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass); // Kind "TestResult" + + var result = VerificationAggregator.Aggregate( + [matching], [Check("tests", "TestResult")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + + Assert.Equal(TaskVerificationStatus.Verified, result.Outcome.Status); + Assert.Same(matching, Assert.Single(result.Outcome.Evidence)); + } + + [Fact] + public void An_ExpectedKind_match_is_ordinal_and_case_sensitive() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + var wrongCase = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass) with { Kind = "testresult" }; + + var result = VerificationAggregator.Aggregate( + [wrongCase], [Check("tests", "TestResult")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + + Assert.Equal(TaskVerificationStatus.Unknown, result.Outcome.Status); + } + + [Fact] + public void A_null_ExpectedKind_accepts_evidence_of_any_kind() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + var approval = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass) with { Kind = "HumanApproval" }; + + var result = VerificationAggregator.Aggregate( + [approval], [Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + + Assert.Equal(TaskVerificationStatus.Verified, result.Outcome.Status); + } + + [Fact] + public void Evidence_of_the_wrong_kind_cannot_hide_a_Fail_recorded_by_the_expected_kind() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + var failingTest = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Fail); + var approvalPass = MakeEvidence(round.RoundId, round.ArtifactRevision, "tests", CheckResult.Pass) with { Kind = "HumanApproval" }; + + var result = VerificationAggregator.Aggregate( + [failingTest, approvalPass], [Check("tests", "TestResult")], round, round.ArtifactRevision, DateTimeOffset.UtcNow); + + Assert.Equal(TaskVerificationStatus.Failed, result.Outcome.Status); + Assert.Same(failingTest, Assert.Single(result.Outcome.Evidence)); + } + + [Fact] + public void A_malformed_required_check_throws_rather_than_being_tolerated() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + + Assert.Throws(() => + VerificationAggregator.Aggregate([], [null!], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + Assert.Throws(() => + VerificationAggregator.Aggregate([], [Check(" ")], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + Assert.Throws(() => + VerificationAggregator.Aggregate([], [Check("tests", " ")], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + } + + [Fact] + public void Duplicate_check_ids_throw_even_when_their_expected_kinds_differ() + { + var round = new ClosedVerificationRound(Guid.NewGuid(), "rev-1"); + + Assert.Throws(() => VerificationAggregator.Aggregate( + [], [Check("tests", "TestResult"), Check("tests", "HumanApproval")], round, round.ArtifactRevision, DateTimeOffset.UtcNow)); + } + [Fact] public void AC6_cancellation_propagates_as_an_exception_rather_than_returning_any_VerificationResult() { @@ -311,7 +404,7 @@ public void AC6_cancellation_propagates_as_an_exception_rather_than_returning_an cts.Cancel(); Assert.Throws(() => - VerificationAggregator.Aggregate([], ["tests"], round, round.ArtifactRevision, DateTimeOffset.UtcNow, cts.Token)); + VerificationAggregator.Aggregate([], [Check("tests")], round, round.ArtifactRevision, DateTimeOffset.UtcNow, cts.Token)); } [Fact] @@ -323,6 +416,6 @@ public void AC6_cancellation_requested_mid_aggregation_still_propagates_rather_t cts.Cancel(); Assert.Throws(() => - VerificationAggregator.Aggregate(evidence, ["build", "tests", "lint"], round, round.ArtifactRevision, DateTimeOffset.UtcNow, cts.Token)); + VerificationAggregator.Aggregate(evidence, [Check("build"), Check("tests"), Check("lint")], round, round.ArtifactRevision, DateTimeOffset.UtcNow, cts.Token)); } } diff --git a/tests/AgentExperience.Core.Tests/packages.lock.json b/tests/AgentExperience.Core.Tests/packages.lock.json index fb14393..6d950d6 100644 --- a/tests/AgentExperience.Core.Tests/packages.lock.json +++ b/tests/AgentExperience.Core.Tests/packages.lock.json @@ -12,6 +12,15 @@ "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.11" } }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Direct", + "requested": "[10.0.11, )", + "resolved": "10.0.11", + "contentHash": "PSmotV19c7E3lKed++uYo1kSiXFI+uTl37CBSrhq+CfLC3FCHjG7R91+xPnNehQfHS1b0Tzo/CCLPWH3qaEheg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11" + } + }, "Microsoft.NET.Test.Sdk": { "type": "Direct", "requested": "[17.14.1, )", @@ -181,7 +190,8 @@ "type": "Project", "dependencies": { "AgentExperience.Abstractions": "[1.0.0, )", - "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )" + "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]" } } } diff --git a/tests/AgentExperience.MicrosoftAgentFramework.Tests/ExperienceFinalizationWiringTests.cs b/tests/AgentExperience.MicrosoftAgentFramework.Tests/ExperienceFinalizationWiringTests.cs new file mode 100644 index 0000000..b9ddbec --- /dev/null +++ b/tests/AgentExperience.MicrosoftAgentFramework.Tests/ExperienceFinalizationWiringTests.cs @@ -0,0 +1,431 @@ +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Lifecycle; +using AgentExperience.Core.Reflections; +using AgentExperience.Core.Verification; +using AgentExperience.MicrosoftAgentFramework; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace AgentExperience.MicrosoftAgentFramework.Tests; + +/// +/// Story 2.5: the MAF adapter's completed run reaches Core's finalization service. The agent runs +/// for real against the scripted fake model; capture, reflection, and lifecycle are the real +/// implementations, and only the record store is in-memory. Finalization never changes what the +/// caller of the agent observes, and a finalization problem is a reported capture failure, never an +/// exception. +/// +public class ExperienceFinalizationWiringTests +{ + private const string ArtifactRevision = "rev-1"; + + private static readonly Scope TestScope = new("tenant-1", "app-1", "project-1"); + private static readonly AuthorizationContext Authorization = new("tenant-1", "host", ["experience:write"], DateTimeOffset.UnixEpoch); + private static readonly ClosedVerificationRound Round = new(Guid.Parse("22222222-2222-2222-2222-222222222222"), ArtifactRevision); + + private static readonly SanitizationOptions Sanitization = new(new Dictionary(StringComparer.Ordinal) + { + ["ToolResult"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "value" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 2, + MaxFieldCount: 5, + MaxValueLength: 10_000, + MaxFieldNameLength: 100), + }); + + private static Evidence PassingEvidence(CheckResult result = CheckResult.Pass) => new( + EvidenceId: Guid.NewGuid(), + VerificationRoundId: Round.RoundId, + ArtifactRevision: ArtifactRevision, + CheckId: "tests", + Kind: "TestResult", + Result: result, + Producer: "ci", + Detail: null, + CapturedAt: DateTimeOffset.UnixEpoch); + + [Fact] + public async Task A_completed_run_is_finalized_into_a_durable_Validated_record() + { + var harness = new Harness(); + var agent = harness.Capture(new ScriptedChatClient()); + + var response = await agent.RunAsync("task-finalize"); + + Assert.Equal("Hello, world", response.Text); + + var result = Assert.Single(harness.Finalized); + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(1, result.Revision); + Assert.Empty(harness.Failures); + + // The record is the one this invocation's run produced. + var runId = Assert.Single(harness.Service.StartedRunIds); + Assert.Equal(ExperienceFinalizationService.ExperienceIdFor(runId), result.ExperienceId); + Assert.Equal(runId, result.Record!.SourceRunId); + Assert.Equal(ExperienceStatus.Validated, harness.Store.StatusOf(result.ExperienceId!.Value)); + } + + [Fact] + public async Task A_resolver_that_returns_null_skips_finalizing_that_run_without_reporting_a_failure() + { + var harness = new Harness { Resolve = _ => null }; + + await harness.Capture(new ScriptedChatClient()).RunAsync("task-skip"); + + Assert.Empty(harness.Finalized); + Assert.Empty(harness.Failures); + Assert.Empty(harness.Store.Records); + } + + [Fact] + public async Task A_host_storage_denial_is_an_outcome_not_a_capture_failure() + { + var harness = new Harness { Decision = StorageDecision.Deny("host policy") }; + + var response = await harness.Capture(new ScriptedChatClient()).RunAsync("task-denied"); + + // The agent's own result is untouched. + Assert.Equal("Hello, world", response.Text); + + var result = Assert.Single(harness.Finalized); + Assert.Equal(FinalizationOutcome.StorageDenied, result.Outcome); + Assert.Empty(harness.Store.Records); + + // The host decided this on purpose. Reporting it through the capture-failure channel would give + // a host whose policy denies most runs one "failure" per invocation, mixed in with real defects. + Assert.Empty(harness.Failures); + } + + [Fact] + public async Task A_finalization_that_actually_failed_is_reported_as_a_capture_failure_and_never_thrown() + { + var harness = new Harness(); + harness.Store.ThrowOnCreate = true; + + var response = await harness.Capture(new ScriptedChatClient()).RunAsync("task-store-down"); + + Assert.Equal("Hello, world", response.Text); + + var result = Assert.Single(harness.Finalized); + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + + var failure = Assert.Single(harness.Failures); + Assert.Equal(ExperienceCaptureFailureStage.Finalization, failure.Stage); + Assert.Contains("Failed", failure.Reason, StringComparison.Ordinal); + } + + [Fact] + public async Task A_resolver_that_returns_a_request_for_another_run_is_refused() + { + var foreignRunId = Guid.NewGuid(); + var harness = new Harness { Resolve = _ => ForeignRequest(foreignRunId) }; + + await harness.Capture(new ScriptedChatClient()).RunAsync("task-foreign-run"); + + // An unrelated captured run must never be finalized on this invocation's behalf. + Assert.Empty(harness.Finalized); + Assert.Empty(harness.Store.Records); + var failure = Assert.Single(harness.Failures); + Assert.Equal(ExperienceCaptureFailureStage.Finalization, failure.Stage); + Assert.Contains("different run", failure.Reason, StringComparison.Ordinal); + } + + [Fact] + public async Task Exceptions_thrown_by_the_finalized_callback_are_swallowed() + { + var harness = new Harness { ThrowFromOnRunFinalized = true }; + + var response = await harness.Capture(new ScriptedChatClient()).RunAsync("task-callback-throws"); + + Assert.Equal("Hello, world", response.Text); + Assert.Equal(FinalizationOutcome.Validated, Assert.Single(harness.Finalized).Outcome); + Assert.Empty(harness.Failures); + } + + [Fact] + public async Task A_throwing_resolver_reports_a_capture_failure_and_leaves_the_invocation_alone() + { + var harness = new Harness { Resolve = _ => throw new InvalidOperationException("resolver failed") }; + + var response = await harness.Capture(new ScriptedChatClient()).RunAsync("task-resolver-throws"); + + Assert.Equal("Hello, world", response.Text); + Assert.Empty(harness.Finalized); + var failure = Assert.Single(harness.Failures); + Assert.Equal(ExperienceCaptureFailureStage.Finalization, failure.Stage); + Assert.IsType(failure.Exception); + } + + [Fact] + public async Task A_run_whose_capture_failed_is_never_finalized() + { + var harness = new Harness(); + harness.Service.ForcedCompleteOutcome = CompleteRunOutcome.Conflict; + + await harness.Capture(new ScriptedChatClient()).RunAsync("task-capture-failed"); + + // A half-captured run must not be persisted as if it were whole. + Assert.Empty(harness.Finalized); + Assert.Empty(harness.Store.Records); + Assert.Equal(ExperienceCaptureFailureStage.Finalize, Assert.Single(harness.Failures).Stage); + } + + [Fact] + public void Half_configured_finalization_is_rejected_at_wiring_time_in_both_directions() + { + var harness = new Harness(); + + // A service with no resolver can never build a request... + var serviceOnly = new ExperienceCaptureOptions + { + ResolveRun = _ => new ExperienceRunDescriptor("task", TestScope), + FinalizationService = harness.Finalization, + ResolveFinalization = null, + }; + + // ...and a resolver with no service -- the easier mistake -- would silently never finalize. + var resolverOnly = new ExperienceCaptureOptions + { + ResolveRun = _ => new ExperienceRunDescriptor("task", TestScope), + FinalizationService = null, + ResolveFinalization = _ => null, + }; + + foreach (var options in new[] { serviceOnly, resolverOnly }) + { + var exception = Assert.Throws(() => + new ScriptedAgent().AsBuilder().UseExperienceCapture(harness.Service, options).Build()); + + Assert.Contains(nameof(ExperienceCaptureOptions.ResolveFinalization), exception.Message, StringComparison.Ordinal); + Assert.Contains(nameof(ExperienceCaptureOptions.FinalizationService), exception.Message, StringComparison.Ordinal); + } + } + + [Fact] + public async Task A_failed_invocation_is_still_finalized_and_quarantined() + { + var harness = new Harness { Evidence = () => [PassingEvidence(CheckResult.Fail)] }; + var client = new ScriptedChatClient { Throw = true }; + + await Assert.ThrowsAsync(() => harness.Capture(client).RunAsync("task-failed")); + + var result = Assert.Single(harness.Finalized); + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + Assert.Null(result.Record!.Reflection); + Assert.Equal(ExperienceStatus.Quarantined, harness.Store.StatusOf(result.ExperienceId!.Value)); + } + + /// A well-formed request that simply names some other captured run. + private static FinalizeExperienceRequest ForeignRequest(Guid runId) => new( + RunId: runId, + Authorization: Authorization, + ClosedRound: Round, + RequiredChecks: [new RequiredCheck("tests", "TestResult")], + Evidence: [PassingEvidence()], + CurrentArtifactRevision: ArtifactRevision, + StorageDecision: StorageDecision.Permit, + FinalizedAt: DateTimeOffset.UnixEpoch.AddDays(1)); + + private sealed class Harness + { + private readonly List _failures = []; + private readonly List _finalized = []; + + public Harness() + { + Service = new RecordingCaptureService(new InMemoryExperienceCaptureService( + new DefaultSanitizer(Sanitization), + new CaptureLimits(10, 50, 10_000, 10_000))); + Store = new InMemoryRecordStore(); + Finalization = new ExperienceFinalizationService( + Service, + new DefaultExperienceReflector(), + Store, + new ExperienceLifecycleService(Store)); + } + + public RecordingCaptureService Service { get; } + + public InMemoryRecordStore Store { get; } + + public ExperienceFinalizationService Finalization { get; } + + public StorageDecision Decision { get; init; } = StorageDecision.Permit; + + public Func> Evidence { get; init; } = () => [PassingEvidence()]; + + public Func? Resolve { get; init; } + + public bool ThrowFromOnRunFinalized { get; init; } + + public FinalizeExperienceRequest RequestFor(Guid runId) => new( + RunId: runId, + Authorization: Authorization, + ClosedRound: Round, + RequiredChecks: [new RequiredCheck("tests", "TestResult")], + Evidence: Evidence(), + CurrentArtifactRevision: ArtifactRevision, + StorageDecision: Decision, + FinalizedAt: DateTimeOffset.UnixEpoch.AddDays(1)); + + public IReadOnlyList Failures + { + get + { + lock (_failures) + { + return _failures.ToList(); + } + } + } + + public IReadOnlyList Finalized + { + get + { + lock (_finalized) + { + return _finalized.ToList(); + } + } + } + + public AIAgent Capture(ScriptedChatClient client) + { + var inner = new ChatClientAgent(client, new ChatClientAgentOptions()); + return inner.AsBuilder().UseExperienceCapture(Service, Options()).Build(); + } + + private ExperienceCaptureOptions Options() => new() + { + ResolveRun = context => new ExperienceRunDescriptor(context.Messages.Last().Text, TestScope), + CaptureToolCalls = false, + FinalizationService = Finalization, + ResolveFinalization = Resolve ?? (context => RequestFor(context.Run.RunId)), + OnRunFinalized = result => + { + lock (_finalized) + { + _finalized.Add(result); + } + + if (ThrowFromOnRunFinalized) + { + throw new InvalidOperationException("host finalization callback failure"); + } + }, + OnCaptureFailure = failure => + { + lock (_failures) + { + _failures.Add(failure); + } + }, + }; + } + + /// + /// A minimal in-memory : enough of the port's contract for the + /// adapter wiring to be exercised end to end without a database. Query and history are not part of + /// finalization and fail loudly if they are ever called. + /// + private sealed class InMemoryRecordStore : IExperienceRecordStore + { + private readonly Dictionary _records = []; + private readonly HashSet _events = []; + + public IReadOnlyDictionary Records + { + get + { + lock (_records) + { + return _records.ToDictionary(); + } + } + } + + public ExperienceStatus StatusOf(Guid experienceId) + { + lock (_records) + { + return _records[experienceId].Status; + } + } + + public bool ThrowOnCreate { get; set; } + + public Task CreateAsync(AuthorizationContext authorization, ExperienceRecord record, CancellationToken cancellationToken) + { + if (ThrowOnCreate) + { + throw new ExperienceStoreException("database unavailable"); + } + + lock (_records) + { + if (_records.ContainsKey(record.ExperienceId)) + { + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Conflict, [])); + } + + if (!authorization.Permits(record.Scope)) + { + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Denied, [])); + } + + _records[record.ExperienceId] = record; + return Task.FromResult(new ExperienceRecordCreateResult(ExperienceStoreOutcome.Created, [])); + } + } + + public Task GetAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) + { + lock (_records) + { + return Task.FromResult(_records.TryGetValue(experienceId, out var record) && record.Scope == scope + ? new ExperienceRecordGetResult(ExperienceStoreOutcome.Found, record, []) + : new ExperienceRecordGetResult(ExperienceStoreOutcome.NotFound, null, [])); + } + } + + public Task CommitLifecycleEventAsync( + AuthorizationContext authorization, + Scope scope, + LifecycleEvent lifecycleEvent, + CancellationToken cancellationToken) + { + lock (_records) + { + if (!_records.TryGetValue(lifecycleEvent.ExperienceRecordId, out var record) || record.Scope != scope) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.NotFound, 0, null, [])); + } + + if (!_events.Add(lifecycleEvent.EventId)) + { + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Committed, record.Revision, null, [])); + } + + var applied = lifecycleEvent.ExpectedRevision + 1; + _records[record.ExperienceId] = record with + { + Status = lifecycleEvent.CurrentStatus, + Revision = applied, + UpdatedAt = lifecycleEvent.OccurredAt, + }; + + return Task.FromResult(new ExperienceLifecycleCommitResult(ExperienceStoreOutcome.Committed, applied, null, [])); + } + } + + public Task QueryAsync(AuthorizationContext authorization, ExperienceRecordQuery query, CancellationToken cancellationToken) => + throw new InvalidOperationException("Finalization must not query records."); + + public Task GetHistoryAsync(AuthorizationContext authorization, Scope scope, Guid experienceId, CancellationToken cancellationToken) => + throw new InvalidOperationException("Finalization must not read history."); + } +} diff --git a/tests/AgentExperience.MicrosoftAgentFramework.Tests/packages.lock.json b/tests/AgentExperience.MicrosoftAgentFramework.Tests/packages.lock.json index afa1bb7..f68aae2 100644 --- a/tests/AgentExperience.MicrosoftAgentFramework.Tests/packages.lock.json +++ b/tests/AgentExperience.MicrosoftAgentFramework.Tests/packages.lock.json @@ -277,7 +277,8 @@ "type": "Project", "dependencies": { "AgentExperience.Abstractions": "[1.0.0, )", - "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )" + "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]" } }, "agentexperience.microsoftagentframework": { diff --git a/tests/AgentExperience.Storage.Postgres.Tests/AgentExperience.Storage.Postgres.Tests.csproj b/tests/AgentExperience.Storage.Postgres.Tests/AgentExperience.Storage.Postgres.Tests.csproj index 465eb1a..466c280 100644 --- a/tests/AgentExperience.Storage.Postgres.Tests/AgentExperience.Storage.Postgres.Tests.csproj +++ b/tests/AgentExperience.Storage.Postgres.Tests/AgentExperience.Storage.Postgres.Tests.csproj @@ -11,6 +11,9 @@ + + @@ -18,6 +21,12 @@ + + + + + + @@ -32,6 +41,10 @@ + + diff --git a/tests/AgentExperience.Storage.Postgres.Tests/DependencyBoundaryTests.cs b/tests/AgentExperience.Storage.Postgres.Tests/DependencyBoundaryTests.cs index 8623590..28481a6 100644 --- a/tests/AgentExperience.Storage.Postgres.Tests/DependencyBoundaryTests.cs +++ b/tests/AgentExperience.Storage.Postgres.Tests/DependencyBoundaryTests.cs @@ -4,9 +4,11 @@ namespace AgentExperience.Storage.Postgres.Tests; /// -/// Proves AgentExperience.Storage.Postgres uses plain Npgsql plus DbUp for schema migrations and -/// nothing else: no MAF, EF Core, Dapper, Pgvector, or model-provider dependency, in either its compiled -/// references or its csproj. +/// Proves AgentExperience.Storage.Postgres uses plain Npgsql, DbUp for schema migrations, and +/// the dependency-injection abstractions its own AddAgentExperiencePostgresStore +/// extension needs -- and nothing else: no MAF, EF Core, Dapper, Pgvector, or model-provider +/// dependency, in either its compiled references or its csproj. The DI package is abstractions only +/// (no container, no hosting), so the adapter still imposes no composition root on a host. /// public class DependencyBoundaryTests { @@ -42,7 +44,7 @@ public void Storage_Postgres_does_not_reference_a_forbidden_assembly() } [Fact] - public void Storage_Postgres_csproj_declares_only_the_exact_Npgsql_and_DbUp_pins() + public void Storage_Postgres_csproj_declares_only_the_exact_Npgsql_DbUp_and_DI_abstractions_pins() { var csprojPath = GetCsprojPath(); Assert.True(File.Exists(csprojPath), $"Could not locate AgentExperience.Storage.Postgres.csproj at '{csprojPath}'."); @@ -54,7 +56,12 @@ public void Storage_Postgres_csproj_declares_only_the_exact_Npgsql_and_DbUp_pins .ToList(); Assert.Equal( - ["Npgsql [10.0.3]", "dbup-core [6.1.1]", "dbup-postgresql [7.0.1]"], + [ + "Microsoft.Extensions.DependencyInjection.Abstractions [10.0.11]", + "Npgsql [10.0.3]", + "dbup-core [6.1.1]", + "dbup-postgresql [7.0.1]", + ], packages); } diff --git a/tests/AgentExperience.Storage.Postgres.Tests/ExperienceSchemaMigratorTests.cs b/tests/AgentExperience.Storage.Postgres.Tests/ExperienceSchemaMigratorTests.cs index 23527e8..09a090b 100644 --- a/tests/AgentExperience.Storage.Postgres.Tests/ExperienceSchemaMigratorTests.cs +++ b/tests/AgentExperience.Storage.Postgres.Tests/ExperienceSchemaMigratorTests.cs @@ -96,6 +96,75 @@ public async Task Database_whose_initial_script_was_applied_by_hand_is_journaled Assert.Equal(Canonical(record), Canonical(read.Record!)); } + [Fact] + public async Task Search_script_applied_by_hand_first_is_journaled_without_failing_on_the_existing_column() + { + await using var dataSource = await _fixture.CreateDatabaseAsync("search_manual"); + + // The whole shipped schema applied the pre-migrator way, 0003 included, so the generated column + // and both indexes already exist when the runner re-runs the script over them. + foreach (var scriptName in PostgresExperienceRecordSchema.ScriptNames) + { + await using var command = dataSource.CreateCommand(PostgresExperienceRecordSchema.GetScript(scriptName)); + await command.ExecuteNonQueryAsync(); + } + + var store = new PostgresExperienceRecordStore(dataSource); + var tenant = NewTenant(); + var record = Full(Scope(tenant)); + await store.CreateAsync(Authorize(tenant), record, CancellationToken.None); + + // Every statement in 0003 is IF NOT EXISTS, so this is a no-op rather than a duplicate-column error. + var result = await ExperienceSchemaMigrator.MigrateAsync(dataSource, CancellationToken.None); + + Assert.Equal(PostgresExperienceRecordSchema.ScriptNames, result.AppliedScripts); + Assert.Equal(PostgresExperienceRecordSchema.ScriptNames, await JournaledAsync(dataSource, ShippedPrefix)); + + // And the hand-applied column still indexes the row that was written through it. + var search = new PostgresExperienceCandidateSource(dataSource); + var found = await search.SearchAsync( + Authorize(tenant), + new ExperienceCandidateQuery(record.Scope, "refund ticket", [ExperienceStatus.Validated], 0d), + CancellationToken.None); + Assert.Equal(record.ExperienceId, Assert.Single(found.Candidates).Record.ExperienceId); + } + + [Fact] + public async Task A_record_written_before_the_search_script_is_indexed_when_it_is_applied() + { + await using var dataSource = await _fixture.CreateDatabaseAsync("search_backfill"); + + // The state an existing deployment is in: 0001 and 0002 applied, rows written, 0003 not yet run. + foreach (var scriptName in new[] + { + PostgresExperienceRecordSchema.InitialScriptName, + PostgresExperienceRecordSchema.LifecycleEventsScriptName, + }) + { + await using var command = dataSource.CreateCommand(PostgresExperienceRecordSchema.GetScript(scriptName)); + await command.ExecuteNonQueryAsync(); + } + + var store = new PostgresExperienceRecordStore(dataSource); + var tenant = NewTenant(); + var record = Full(Scope(tenant)); + Assert.Equal(ExperienceStoreOutcome.Created, (await store.CreateAsync(Authorize(tenant), record, CancellationToken.None)).Outcome); + + await ExperienceSchemaMigrator.MigrateAsync(dataSource, CancellationToken.None); + + // The generated column is computed for every existing row as the table is rewritten, so records + // that predate the search are searchable without a backfill step of their own. + var search = new PostgresExperienceCandidateSource(dataSource); + var found = await search.SearchAsync( + Authorize(tenant), + new ExperienceCandidateQuery(record.Scope, "refund ticket", [ExperienceStatus.Validated], 0d), + CancellationToken.None); + + var candidate = Assert.Single(found.Candidates); + Assert.Equal(record.ExperienceId, candidate.Record.ExperienceId); + Assert.Equal(Canonical(record), Canonical(candidate.Record)); + } + [Fact] public async Task Concurrent_runs_both_succeed_and_journal_each_script_exactly_once() { diff --git a/tests/AgentExperience.Storage.Postgres.Tests/OfflineStoreTests.cs b/tests/AgentExperience.Storage.Postgres.Tests/OfflineStoreTests.cs index 5d8f2f4..b5f4b92 100644 --- a/tests/AgentExperience.Storage.Postgres.Tests/OfflineStoreTests.cs +++ b/tests/AgentExperience.Storage.Postgres.Tests/OfflineStoreTests.cs @@ -314,7 +314,11 @@ public void Embedded_schema_script_is_available_and_creates_the_versioned_table( var sql = PostgresExperienceRecordSchema.GetScript(PostgresExperienceRecordSchema.InitialScriptName); Assert.Equal( - [PostgresExperienceRecordSchema.InitialScriptName, PostgresExperienceRecordSchema.LifecycleEventsScriptName], + [ + PostgresExperienceRecordSchema.InitialScriptName, + PostgresExperienceRecordSchema.LifecycleEventsScriptName, + PostgresExperienceRecordSchema.SearchScriptName, + ], PostgresExperienceRecordSchema.ScriptNames); Assert.Contains("CREATE SCHEMA IF NOT EXISTS agent_experience", sql, StringComparison.Ordinal); Assert.Contains("payload_version", sql, StringComparison.Ordinal); @@ -342,6 +346,148 @@ public void Lifecycle_script_is_embedded_separately_and_never_edits_the_initial_ PostgresExperienceRecordSchema.ScriptNames); } + [Fact] + public void Search_script_is_embedded_separately_and_only_adds_derived_read_artifacts() + { + var search = PostgresExperienceRecordSchema.GetScript(PostgresExperienceRecordSchema.SearchScriptName); + + // A generated column, so no write path has to maintain it and it can never disagree with the + // record it indexes; the store's INSERT and its lifecycle UPDATE are untouched by this script. + Assert.Contains("ADD COLUMN IF NOT EXISTS search_vector tsvector", search, StringComparison.Ordinal); + Assert.Contains("GENERATED ALWAYS AS", search, StringComparison.Ordinal); + Assert.Contains("STORED", search, StringComparison.Ordinal); + Assert.Contains("CREATE INDEX IF NOT EXISTS ix_experience_records_search", search, StringComparison.Ordinal); + Assert.Contains("USING GIN", search, StringComparison.Ordinal); + + // The query and the generated column must be analyzed with the same configuration: querying with + // a different one silently changes which rows match, so the constant is pinned to the script. + Assert.Contains($"'{PostgresExperienceCandidateSource.SearchConfiguration}'", search, StringComparison.Ordinal); + Assert.Equal("english", PostgresExperienceCandidateSource.SearchConfiguration); + + // A tsvector may not exceed 1 MB, and a generated column that raises fails the INSERT, not the + // search -- so the concatenated text is bounded before it is analyzed. + Assert.Contains("left(", search, StringComparison.Ordinal); + + // The indexed text is exactly the three fields that say what a record is about. + Assert.Contains("task_id", search, StringComparison.Ordinal); + Assert.Contains("payload ->> 'taskSummary'", search, StringComparison.Ordinal); + Assert.Contains("payload -> 'reflection' ->> 'lesson'", search, StringComparison.Ordinal); + + // The "never" assertions below are about what the script *executes*, so the leading comment block + // -- which explains, in prose, why 0001's now-redundant index is not dropped -- is stripped first. + var statements = string.Join( + '\n', + search.Split('\n').Where(line => !line.TrimStart().StartsWith("--", StringComparison.Ordinal))); + + // Append-only: 0003 adds to the table and rewrites nothing 0001 or 0002 created. + Assert.DoesNotContain("DROP", statements, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("ALTER COLUMN", statements, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("lifecycle_events", statements, StringComparison.Ordinal); + + // Story 2.6 owns embeddings; this script must not anticipate them. + Assert.DoesNotContain("embedding", statements, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("CREATE EXTENSION", statements, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("hnsw", statements, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("ivfflat", statements, StringComparison.OrdinalIgnoreCase); + + // 0003 is applied after 0002, which the migrator relies on for ordinal name ordering. + Assert.Equal( + PostgresExperienceRecordSchema.ScriptNames.Order(StringComparer.Ordinal), + PostgresExperienceRecordSchema.ScriptNames); + } + + [Fact] + public async Task Malformed_candidate_search_returns_Invalid_with_every_field_path_and_no_database_call() + { + var tenant = NewTenant(); + var source = new PostgresExperienceCandidateSource(_dataSource); // unreachable: reaching it would hang or throw + + var result = await source.SearchAsync( + Authorize(tenant), + new ExperienceCandidateQuery(new Scope(tenant, "app-1", " "), " ", [], 1.5, 0), + CancellationToken.None); + + Assert.Equal(ExperienceStoreOutcome.Invalid, result.Outcome); + Assert.Empty(result.Candidates); + Assert.Equal( + ["Scope.ProjectId", "TaskText", "EligibleStatuses", "MinimumConfidence", "Limit"], + result.Errors.Select(error => error.Path)); + } + + [Fact] + public async Task Task_text_past_the_maximum_length_is_Invalid_rather_than_reaching_the_parser() + { + var tenant = NewTenant(); + var source = new PostgresExperienceCandidateSource(_dataSource); + + var result = await source.SearchAsync( + Authorize(tenant), + new ExperienceCandidateQuery( + Scope(tenant), + new string('a', ExperienceCandidateQuery.MaxTaskTextLength + 1), + [ExperienceStatus.Validated], + 0.5), + CancellationToken.None); + + // A typed Invalid, decided before any connection opens -- not a multi-megabyte round trip that + // comes back as an opaque infrastructure failure. + Assert.Equal(ExperienceStoreOutcome.Invalid, result.Outcome); + var error = Assert.Single(result.Errors); + Assert.Equal("TaskText", error.Path); + Assert.DoesNotContain("aaaa", error.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task A_candidate_search_outside_the_authorization_is_Denied_before_any_connection_opens() + { + var tenant = NewTenant(); + var source = new PostgresExperienceCandidateSource(_dataSource); + + var result = await source.SearchAsync( + new AuthorizationContext(tenant, "host-principal", [], ColumnTime, ProjectId: "other-project"), + new ExperienceCandidateQuery(Scope(tenant), "refund", [ExperienceStatus.Validated], 0.5), + CancellationToken.None); + + // The data source points at a closed port: any connection attempt would have failed instead. + Assert.Equal(ExperienceStoreOutcome.Denied, result.Outcome); + Assert.Empty(result.Candidates); + Assert.Empty(result.Errors); + } + + [Fact] + public async Task An_unreachable_database_makes_a_candidate_search_throw_ExperienceStoreException() + { + var tenant = NewTenant(); + var source = new PostgresExperienceCandidateSource(_dataSource); + + var ex = await Assert.ThrowsAsync(() => source.SearchAsync( + Authorize(tenant), + new ExperienceCandidateQuery(Scope(tenant), "refund", [ExperienceStatus.Validated], 0.5), + CancellationToken.None)); + + Assert.NotNull(ex.InnerException); + } + + [Fact] + public async Task A_cancelled_candidate_search_surfaces_cancellation_unwrapped() + { + var tenant = NewTenant(); + var source = new PostgresExperienceCandidateSource(_dataSource); + using var cancellation = new CancellationTokenSource(); + await cancellation.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => source.SearchAsync( + Authorize(tenant), + new ExperienceCandidateQuery(Scope(tenant), "refund", [ExperienceStatus.Validated], 0.5), + cancellation.Token)); + } + + [Fact] + public void A_candidate_source_needs_a_data_source() + { + Assert.Throws(() => new PostgresExperienceCandidateSource(null!)); + } + [Fact] public async Task Malformed_lifecycle_commit_returns_Invalid_with_every_field_path_and_no_database_call() { diff --git a/tests/AgentExperience.Storage.Postgres.Tests/PlainPostgresMigrationTests.cs b/tests/AgentExperience.Storage.Postgres.Tests/PlainPostgresMigrationTests.cs new file mode 100644 index 0000000..3a04800 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Tests/PlainPostgresMigrationTests.cs @@ -0,0 +1,97 @@ +using Npgsql; +using Testcontainers.PostgreSql; + +namespace AgentExperience.Storage.Postgres.Tests; + +/// +/// Proves this package's schema needs nothing pgvector provides, against a stock postgres:16 +/// image with no vector extension available at all. +/// +/// +/// This is the regression guard for a real defect: the embedding schema was briefly in this package's +/// script list, which made CREATE EXTENSION vector -- an untrusted extension, so superuser-only +/// -- a startup requirement for every host, including text-only ones that never enable the vector +/// channel. A stock image cannot even satisfy it, so if the embedding script ever comes back here, +/// this test fails rather than a text-only deployment failing at someone's startup. +/// +public sealed class PlainPostgresMigrationTests : IAsyncLifetime +{ + private PostgreSqlContainer? _container; + private NpgsqlDataSource? _dataSource; + + private NpgsqlDataSource DataSource => _dataSource ?? throw new InvalidOperationException("Fixture not initialized."); + + public async Task InitializeAsync() + { + // Stock postgres:16, deliberately not pgvector/pgvector:pg16. + _container = new PostgreSqlBuilder("postgres:16").Build(); + await _container.StartAsync(); + _dataSource = NpgsqlDataSource.Create(_container.GetConnectionString()); + } + + public async Task DisposeAsync() + { + if (_dataSource is not null) + { + await _dataSource.DisposeAsync(); + } + + if (_container is not null) + { + await _container.DisposeAsync(); + } + } + + [Fact] + public async Task The_base_schema_migrates_on_a_PostgreSQL_without_pgvector_available() + { + // Sanity: the extension really is unavailable here, so the assertion below means something. + Assert.Equal( + 0L, + await ScalarAsync("SELECT count(*) FROM pg_available_extensions WHERE name = 'vector'")); + + var applied = await ExperienceSchemaMigrator.MigrateAsync(DataSource, CancellationToken.None); + + Assert.Equal(PostgresExperienceRecordSchema.ScriptNames.Count, applied.AppliedScripts.Count); + Assert.Equal(PostgresExperienceRecordSchema.ScriptNames, applied.AppliedScripts); + + // The store's tables exist, so a text-only deployment is fully usable. + Assert.Equal(1L, await ScalarAsync( + "SELECT count(*) FROM information_schema.tables " + + "WHERE table_schema = 'agent_experience' AND table_name = 'experience_records'")); + Assert.Equal(1L, await ScalarAsync( + "SELECT count(*) FROM information_schema.tables " + + "WHERE table_schema = 'agent_experience' AND table_name = 'lifecycle_events'")); + + // And nothing here created an extension or an embedding table. + Assert.Equal(0L, await ScalarAsync("SELECT count(*) FROM pg_extension WHERE extname = 'vector'")); + Assert.Equal(0L, await ScalarAsync( + "SELECT count(*) FROM information_schema.tables " + + "WHERE table_schema = 'agent_experience' AND table_name = 'experience_embeddings'")); + + // Rerunning is a no-op, as for any other database. + Assert.Empty((await ExperienceSchemaMigrator.MigrateAsync(DataSource, CancellationToken.None)).AppliedScripts); + } + + [Fact] + public void No_script_this_package_ships_creates_an_extension() + { + foreach (var scriptName in PostgresExperienceRecordSchema.ScriptNames) + { + var statements = string.Join( + '\n', + PostgresExperienceRecordSchema.GetScript(scriptName) + .Split('\n') + .Where(line => !line.TrimStart().StartsWith("--", StringComparison.Ordinal))); + + Assert.DoesNotContain("CREATE EXTENSION", statements, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("experience_embeddings", statements, StringComparison.Ordinal); + } + } + + private async Task ScalarAsync(string sql) + { + await using var command = DataSource.CreateCommand(sql); + return (T)(await command.ExecuteScalarAsync())!; + } +} diff --git a/tests/AgentExperience.Storage.Postgres.Tests/PostgresExperienceCandidateSourceTests.cs b/tests/AgentExperience.Storage.Postgres.Tests/PostgresExperienceCandidateSourceTests.cs new file mode 100644 index 0000000..1f2846f --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Tests/PostgresExperienceCandidateSourceTests.cs @@ -0,0 +1,396 @@ +using AgentExperience.Core.Retrieval; +using static AgentExperience.Storage.Postgres.Tests.TestRecords; + +namespace AgentExperience.Storage.Postgres.Tests; + +/// +/// Integration tests for the SQL behind , against a +/// real PostgreSQL 16 container: what the generated search_vector indexes, that scope, status, +/// and confidence are all decided inside the query, and that a matched record still decodes into the +/// full canonical record. Each test uses its own random tenant, so tests sharing the container never +/// see each other's rows. +/// +[Collection(PostgresCollection.Name)] +public sealed class PostgresExperienceCandidateSourceTests +{ + private static readonly DateTimeOffset Now = ColumnTime; + + private readonly PostgresExperienceRecordStore _store; + private readonly PostgresExperienceCandidateSource _source; + + public PostgresExperienceCandidateSourceTests(PostgresFixture fixture) + { + _store = new PostgresExperienceRecordStore(fixture.DataSource); + _source = new PostgresExperienceCandidateSource(fixture.DataSource); + } + + [Fact] + public async Task The_task_id_the_task_summary_and_the_reflection_lesson_are_each_searchable() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var byTaskId = await SeedAsync(scope, taskId: "refund-ticket-triage", summary: "Nothing else to say", lesson: "Nothing else to say"); + var bySummary = await SeedAsync(scope, taskId: "unrelated-a", summary: "Resolve a customer refund", lesson: "Nothing else to say"); + var byLesson = await SeedAsync(scope, taskId: "unrelated-b", summary: "Nothing else to say", lesson: "Issue the refund once the lock clears"); + var unrelated = await SeedAsync(scope, taskId: "deploy-cluster", summary: "Roll out the cluster", lesson: "Drain nodes before rolling"); + + var result = await SearchAsync(tenant, scope, "refund"); + + Assert.Equal(ExperienceStoreOutcome.Found, result.Outcome); + Assert.Empty(result.Errors); + Assert.Equal( + new[] { byTaskId, bySummary, byLesson }.Order(), + result.Candidates.Select(candidate => candidate.Record.ExperienceId).Order()); + Assert.DoesNotContain(unrelated, result.Candidates.Select(candidate => candidate.Record.ExperienceId)); + } + + [Fact] + public async Task Text_that_matches_nothing_is_Found_with_no_candidates() + { + var tenant = NewTenant(); + await SeedAsync(Scope(tenant), taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund"); + + var result = await SearchAsync(tenant, Scope(tenant), "kubernetes autoscaling"); + + Assert.Equal(ExperienceStoreOutcome.Found, result.Outcome); + Assert.Empty(result.Candidates); + } + + [Fact] + public async Task A_record_in_another_tenant_project_or_team_is_never_returned() + { + var tenant = NewTenant(); + var otherTenant = NewTenant(); + var scope = Scope(tenant); + var mine = await SeedAsync(scope, taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund"); + + // Identical text in every neighbouring scope, including one that only differs by an optional field. + await SeedAsync(Scope(otherTenant), taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund"); + await SeedAsync(Scope(tenant, project: "project-2"), taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund"); + await SeedAsync(Scope(tenant, team: "team-1"), taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund"); + + var result = await SearchAsync(tenant, scope, "refund"); + + Assert.Equal([mine], result.Candidates.Select(candidate => candidate.Record.ExperienceId)); + + // And the authorization the host established still bounds the search, whatever scope is asked for. + var denied = await _source.SearchAsync( + Authorize(otherTenant), + new ExperienceCandidateQuery(scope, "refund", [ExperienceStatus.Validated], 0d), + CancellationToken.None); + Assert.Equal(ExperienceStoreOutcome.Denied, denied.Outcome); + Assert.Empty(denied.Candidates); + } + + [Fact] + public async Task Only_the_requested_statuses_come_back() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var validated = await SeedAsync(scope, taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund", status: ExperienceStatus.Validated); + var reinforced = await SeedAsync(scope, taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund", status: ExperienceStatus.Reinforced); + + foreach (var ineligible in new[] + { + ExperienceStatus.Candidate, + ExperienceStatus.Quarantined, + ExperienceStatus.Contested, + ExperienceStatus.Stale, + ExperienceStatus.Superseded, + ExperienceStatus.Revoked, + }) + { + await SeedAsync(scope, taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund", status: ineligible); + } + + var result = await SearchAsync(tenant, scope, "refund"); + + Assert.Equal( + new[] { validated, reinforced }.Order(), + result.Candidates.Select(candidate => candidate.Record.ExperienceId).Order()); + } + + [Fact] + public async Task A_status_change_committed_through_the_store_takes_a_record_out_of_the_eligible_set() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var id = await SeedAsync(scope, taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund", status: ExperienceStatus.Validated); + + Assert.Equal([id], (await SearchAsync(tenant, scope, "refund")).Candidates.Select(candidate => candidate.Record.ExperienceId)); + + var commit = await _store.CommitLifecycleEventAsync( + Authorize(tenant), + scope, + Event(id, ExperienceStatus.Validated, ExperienceStatus.Revoked, expectedRevision: 0, reason: "withdrawn", producer: "tests"), + CancellationToken.None); + Assert.Equal(ExperienceStoreOutcome.Committed, commit.Outcome); + + // The projection update is all it takes: the generated search vector still indexes the same + // text, and the status predicate is what removes the record. + Assert.Empty((await SearchAsync(tenant, scope, "refund")).Candidates); + } + + [Fact] + public async Task A_record_below_the_confidence_floor_is_filtered_in_SQL() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var above = await SeedAsync(scope, taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund", confidence: 0.5); + var below = await SeedAsync(scope, taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund", confidence: 0.49); + + var result = await SearchAsync(tenant, scope, "refund", minimumConfidence: 0.5); + + // The floor is inclusive: a record exactly at the threshold is still a candidate. + Assert.Equal([above], result.Candidates.Select(candidate => candidate.Record.ExperienceId)); + Assert.DoesNotContain(below, result.Candidates.Select(candidate => candidate.Record.ExperienceId)); + } + + [Fact] + public async Task Relevance_is_normalized_positive_for_a_match_and_orders_the_candidates() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var strong = await SeedAsync( + scope, + taskId: "refund-policy-triage", + summary: "Refund policy questions from customers", + lesson: "Apply the refund policy once the ticket lock clears"); + var weak = await SeedAsync( + scope, + taskId: "deployment-review", + summary: "Weekly deployment checklist for the cluster", + lesson: "A refund may be needed when the policy changes after a rollout"); + + var result = await SearchAsync(tenant, scope, "refund policy"); + + Assert.Equal(2, result.Candidates.Count); + Assert.All(result.Candidates, candidate => Assert.InRange(candidate.Relevance, 0d, 1d)); + Assert.All(result.Candidates, candidate => Assert.True(candidate.Relevance > 0d, "a matched record must report a positive relevance.")); + + // Strongest match first, and the reported order is the relevance order. + Assert.Equal([strong, weak], result.Candidates.Select(candidate => candidate.Record.ExperienceId)); + Assert.True(result.Candidates[0].Relevance > result.Candidates[1].Relevance); + } + + [Fact] + public async Task Relevance_stays_strictly_below_one_however_heavily_the_text_repeats_the_query() + { + // ts_rank_cd is unbounded above; only the normalization flag keeps it inside [0, 1). Without it a + // document this saturated ranks far above 1, and the clamp in C# would hide that by reporting + // exactly 1 -- so this asserts a value strictly below 1, which only the flag can produce. + var tenant = NewTenant(); + var scope = Scope(tenant); + var repeated = string.Join(' ', Enumerable.Repeat("refund policy ticket", 200)); + await SeedAsync(scope, taskId: "refund-policy-ticket", summary: repeated, lesson: repeated); + + var result = await SearchAsync(tenant, scope, "refund policy ticket"); + + var candidate = Assert.Single(result.Candidates); + Assert.True(candidate.Relevance > 0d, "a saturated match must report a positive relevance."); + Assert.True( + candidate.Relevance < 1d, + $"relevance must stay strictly below 1; ts_rank_cd's normalization is what bounds it, got {candidate.Relevance}."); + } + + [Fact] + public async Task The_limit_bounds_how_many_candidates_come_back() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + for (var i = 0; i < 5; i++) + { + await SeedAsync(scope, taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund"); + } + + var result = await SearchAsync(tenant, scope, "refund", limit: 2); + + Assert.Equal(2, result.Candidates.Count); + } + + [Fact] + public async Task A_matched_candidate_decodes_into_the_full_canonical_record() + { + var tenant = NewTenant(); + var scope = new Scope(tenant, "app-1", "project-1", "team-1", "agent-1", "user-1"); + var record = Full(scope); // Validated, reuse confidence 2/3, task summary "Resolve refund ticket" + Assert.Equal(ExperienceStoreOutcome.Created, (await _store.CreateAsync(Authorize(tenant), record, CancellationToken.None)).Outcome); + + var result = await SearchAsync(tenant, scope, "refund ticket"); + + var candidate = Assert.Single(result.Candidates); + Assert.Equal(Canonical(record), Canonical(candidate.Record)); + Assert.Equal(record.UpdatedAt, candidate.Record.UpdatedAt); + } + + [Fact] + public async Task Search_text_that_looks_like_query_syntax_is_still_just_text() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var id = await SeedAsync(scope, taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund"); + + // websearch_to_tsquery accepts arbitrary user text: quotes, operators, and punctuation are + // parsed as a search, never as a syntax error the caller has to sanitize around. + foreach (var text in new[] { "\"refund", "refund or", "refund -", "refund & ticket |", "((refund))" }) + { + var result = await SearchAsync(tenant, scope, text); + Assert.Equal(ExperienceStoreOutcome.Found, result.Outcome); + } + + Assert.Equal([id], (await SearchAsync(tenant, scope, "\"refund\"")).Candidates.Select(candidate => candidate.Record.ExperienceId)); + } + + [Fact] + public async Task Text_made_only_of_stopwords_matches_nothing_and_looks_like_nothing_relevant_exists() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + await SeedAsync(scope, taskId: "refund-ticket", summary: "Resolve a refund", lesson: "Retry the refund"); + + // The english configuration drops stopwords, so this parses to an empty query, and an empty + // query matches no row by construction -- reported as an ordinary Found with no candidates, + // indistinguishable from "nothing relevant is stored". + var stopwords = await SearchAsync(tenant, scope, "the of and"); + var nothingRelevant = await SearchAsync(tenant, scope, "kubernetes autoscaling"); + + Assert.Equal(ExperienceStoreOutcome.Found, stopwords.Outcome); + Assert.Empty(stopwords.Candidates); + Assert.Equal(nothingRelevant.Outcome, stopwords.Outcome); + Assert.Equal(nothingRelevant.Candidates.Count, stopwords.Candidates.Count); + } + + [Fact] + public async Task Core_retrieval_over_the_real_search_ranks_only_the_eligible_records() + { + // The full path: Core's service, the real SQL, and a real database. + var tenant = NewTenant(); + var scope = Scope(tenant); + var reinforced = await SeedAsync( + scope, + taskId: "refund-ticket", + summary: "Resolve a refund", + lesson: "Retry the refund", + status: ExperienceStatus.Reinforced, + confidence: 0.9, + metadata: new Dictionary { ["region"] = "us-east" }); + var validated = await SeedAsync( + scope, + taskId: "refund-ticket", + summary: "Resolve a refund", + lesson: "Retry the refund", + status: ExperienceStatus.Validated, + confidence: 0.9, + metadata: new Dictionary { ["region"] = "us-east" }); + var wrongRegion = await SeedAsync( + scope, + taskId: "refund-ticket", + summary: "Resolve a refund", + lesson: "Retry the refund", + status: ExperienceStatus.Validated, + confidence: 0.9, + metadata: new Dictionary { ["region"] = "eu-west" }); + var quarantined = await SeedAsync( + scope, + taskId: "refund-ticket", + summary: "Resolve a refund", + lesson: "Retry the refund", + status: ExperienceStatus.Quarantined, + confidence: 0.9, + metadata: new Dictionary { ["region"] = "us-east" }); + var lowConfidence = await SeedAsync( + scope, + taskId: "refund-ticket", + summary: "Resolve a refund", + lesson: "Retry the refund", + confidence: 0.1, + metadata: new Dictionary { ["region"] = "us-east" }); + + var retrieval = new ExperienceRetrievalService( + _source, + RetrievalPolicy.Default with { Timeout = TimeSpan.FromSeconds(30) }, + RankingWeights.Default, + TimeProvider.System); + + var result = await retrieval.RetrieveAsync( + new RetrieveExperienceRequest( + Authorize(tenant), + scope, + "refund ticket", + new Dictionary { ["region"] = "us-east" }, + CorrelationId: "corr-integration"), + CancellationToken.None); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Equal("corr-integration", result.CorrelationId); + Assert.False(result.EnvironmentUnrestricted); + + // Equal on every other axis, so the reinforced record outranks the validated one. + Assert.Equal([reinforced, validated], result.Records.Select(ranked => ranked.Record.ExperienceId)); + Assert.True(result.Records[0].Score > result.Records[1].Score); + Assert.All(result.Records, ranked => Assert.Equal(5, ranked.Components.Count)); + + // The environment check ran in Core, over what SQL returned; the rest never left the database. + Assert.Equal( + [new ExcludedExperience(wrongRegion, RetrievalExclusionReason.EnvironmentMismatch)], + result.Excluded); + Assert.DoesNotContain(quarantined, result.Records.Select(ranked => ranked.Record.ExperienceId)); + Assert.DoesNotContain(lowConfidence, result.Records.Select(ranked => ranked.Record.ExperienceId)); + } + + private Task SearchAsync( + string tenant, + Scope scope, + string taskText, + double minimumConfidence = 0d, + int limit = ExperienceCandidateQuery.DefaultLimit) => + _source.SearchAsync( + Authorize(tenant), + new ExperienceCandidateQuery( + scope, + taskText, + [ExperienceStatus.Validated, ExperienceStatus.Reinforced], + minimumConfidence, + limit), + CancellationToken.None); + + /// Creates one searchable record and returns its ID. + private async Task SeedAsync( + Scope scope, + string taskId, + string summary, + string lesson, + ExperienceStatus status = ExperienceStatus.Validated, + double confidence = 0.75, + IReadOnlyDictionary? metadata = null) + { + var runId = Guid.NewGuid(); + var record = Minimal(scope, status: status) with + { + SourceRunId = runId, + TaskId = taskId, + TaskSummary = summary, + ReuseConfidence = confidence, + Environment = new EnvironmentFingerprint("worker-01", "10.0.0", "linux-x64", null, metadata ?? new Dictionary()), + Reflection = new Reflection( + Guid.NewGuid(), + runId, + lesson, + [], + [], + [], + [], + null, + [], + TaskVerificationStatus.Verified, + 1, + "v1", + "tests", + Now), + }; + + var created = await _store.CreateAsync(Authorize(scope.TenantId), record, CancellationToken.None); + Assert.Equal(ExperienceStoreOutcome.Created, created.Outcome); + return record.ExperienceId; + } +} diff --git a/tests/AgentExperience.Storage.Postgres.Tests/PostgresFinalizationTests.cs b/tests/AgentExperience.Storage.Postgres.Tests/PostgresFinalizationTests.cs new file mode 100644 index 0000000..99655e6 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Tests/PostgresFinalizationTests.cs @@ -0,0 +1,322 @@ +using static AgentExperience.Storage.Postgres.Tests.TestRecords; + +namespace AgentExperience.Storage.Postgres.Tests; + +/// +/// Story 2.5 end to end against a real PostgreSQL 16 container: capture a run, finalize it through +/// Core's , and read the durable record and its lifecycle +/// history back through the real store. Nothing here is faked below the service under test -- the +/// sanitizer, the capture service, the reflector, the lifecycle service, and the PostgreSQL store are +/// all the shipping implementations. Each test uses its own random tenant, so tests sharing the +/// container never see each other's rows. +/// +[Collection(PostgresCollection.Name)] +public sealed class PostgresFinalizationTests +{ + private const string ArtifactRevision = "rev-1"; + + private static readonly ClosedVerificationRound Round = new(Guid.Parse("33333333-3333-3333-3333-333333333333"), ArtifactRevision); + + private static readonly SanitizationOptions Sanitization = new(new Dictionary(StringComparer.Ordinal) + { + ["ToolArguments"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "query" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 3, + MaxFieldCount: 10, + MaxValueLength: 10_000, + MaxFieldNameLength: 100), + ["ToolResult"] = new SanitizationPolicy( + AllowedFieldNames: new HashSet(StringComparer.Ordinal) { "value" }, + SecretFieldNames: new HashSet(StringComparer.Ordinal), + MaxDepth: 2, + MaxFieldCount: 5, + MaxValueLength: 10_000, + MaxFieldNameLength: 100), + }); + + private readonly PostgresExperienceRecordStore _store; + private readonly InMemoryExperienceCaptureService _capture; + private readonly ExperienceFinalizationService _finalization; + + public PostgresFinalizationTests(PostgresFixture fixture) + { + _store = new PostgresExperienceRecordStore(fixture.DataSource); + _capture = new InMemoryExperienceCaptureService( + new DefaultSanitizer(Sanitization), + new CaptureLimits(MaxAttemptsPerRun: 10, MaxToolCallsPerAttempt: 50, MaxResultLength: 10_000, MaxErrorLength: 10_000)); + _finalization = new ExperienceFinalizationService( + _capture, + new DefaultExperienceReflector(), + _store, + new ExperienceLifecycleService(_store)); + } + + [Fact] + public async Task A_captured_run_finalizes_into_a_durable_record_readable_with_its_history() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + var result = await _finalization.FinalizeAsync(Request(runId, auth), CancellationToken.None); + + Assert.Equal(FinalizationOutcome.Validated, result.Outcome); + Assert.True(result.IsDurable); + Assert.Equal(1, result.Revision); + + // Read the record back through the real store. + var read = await _store.GetAsync(auth, scope, result.ExperienceId!.Value, CancellationToken.None); + Assert.Equal(ExperienceStoreOutcome.Found, read.Outcome); + var stored = read.Record!; + + Assert.Equal(ExperienceStatus.Validated, stored.Status); + Assert.Equal(1, stored.Revision); + Assert.Equal(runId, stored.SourceRunId); + Assert.Equal("task-1", stored.TaskId); + Assert.Equal(2d / 3d, stored.ReuseConfidence, precision: 12); + Assert.Equal(1, stored.SupportingValidations); + Assert.Equal(0, stored.Contradictions); + Assert.Equal(TaskVerificationStatus.Verified, stored.Outcome.Status); + Assert.Equal(1.0, stored.CompletionScore); + + // The reflection round-tripped whole, and is traceable to the evidence it was derived from. + var reflection = Assert.IsType(stored.Reflection); + Assert.Equal(runId, reflection.ExperienceRunId); + Assert.Equal(ExperienceFinalizationService.ReflectionIdFor(runId), reflection.ReflectionId); + Assert.Equal(TaskVerificationStatus.Verified, reflection.VerificationStatus); + Assert.Equal(Assert.Single(stored.Outcome.Evidence).EvidenceId, Assert.Single(reflection.EvidenceIds)); + + // The captured attempt and its tool call came through unchanged. + var attempt = Assert.Single(stored.Attempts); + Assert.Equal("done", attempt.Result); + Assert.Equal("search", Assert.Single(attempt.ToolCalls).ToolName); + + // And so did the lifecycle history: exactly one initial event. + var history = await _store.GetHistoryAsync(auth, scope, stored.ExperienceId, CancellationToken.None); + Assert.Equal(ExperienceStoreOutcome.Found, history.Outcome); + Assert.Equal(1, history.Revision); + var initial = Assert.Single(history.Events); + Assert.Equal(ExperienceFinalizationService.InitialEventIdFor(runId), initial.EventId); + Assert.Equal(ExperienceStatus.Candidate, initial.PriorStatus); // the record was created as a Candidate + Assert.Equal(ExperienceStatus.Validated, initial.CurrentStatus); + Assert.Equal(0, initial.ExpectedRevision); + Assert.Equal(ExperienceFinalizationService.ProducerIdentity, initial.Producer); + } + + [Fact] + public async Task Finalizing_the_same_run_twice_leaves_one_record_at_revision_one_with_one_event() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + var first = await _finalization.FinalizeAsync(Request(runId, auth), CancellationToken.None); + var second = await _finalization.FinalizeAsync( + Request(runId, auth) with { FinalizedAt = DateTimeOffset.UtcNow.AddMinutes(10) }, + CancellationToken.None); + + Assert.Equal(FinalizationOutcome.Validated, first.Outcome); + Assert.Equal(FinalizationOutcome.AlreadyFinalized, second.Outcome); + Assert.Equal(first.ExperienceId, second.ExperienceId); + Assert.Equal(ExperienceStatus.Validated, second.Status); + Assert.Equal(1, second.Revision); + + var records = await _store.QueryAsync(auth, new ExperienceRecordQuery(scope), CancellationToken.None); + Assert.Single(records.Records); + + var history = await _store.GetHistoryAsync(auth, scope, first.ExperienceId!.Value, CancellationToken.None); + Assert.Equal(1, history.Revision); + Assert.Single(history.Events); + } + + [Fact] + public async Task A_denied_storage_decision_leaves_no_record_and_no_event_for_the_run() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + var result = await _finalization.FinalizeAsync( + Request(runId, auth) with { StorageDecision = StorageDecision.Deny("host retention policy") }, + CancellationToken.None); + + Assert.Equal(FinalizationOutcome.StorageDenied, result.Outcome); + Assert.Null(result.ExperienceId); + Assert.Equal("host retention policy", result.Reason); + + var records = await _store.QueryAsync(auth, new ExperienceRecordQuery(scope), CancellationToken.None); + Assert.Empty(records.Records); + + var history = await _store.GetHistoryAsync(auth, scope, ExperienceFinalizationService.ExperienceIdFor(runId), CancellationToken.None); + Assert.Equal(ExperienceStoreOutcome.NotFound, history.Outcome); + } + + [Fact] + public async Task An_unverified_run_finalizes_into_a_quarantined_record_with_no_reflection() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + var result = await _finalization.FinalizeAsync( + Request(runId, auth) with { Evidence = [Evidence(CheckResult.Fail)] }, + CancellationToken.None); + + Assert.Equal(FinalizationOutcome.Quarantined, result.Outcome); + + var stored = (await _store.GetAsync(auth, scope, result.ExperienceId!.Value, CancellationToken.None)).Record!; + Assert.Equal(ExperienceStatus.Quarantined, stored.Status); + Assert.Null(stored.Reflection); + Assert.Equal(0d, stored.ReuseConfidence); + Assert.Equal(TaskVerificationStatus.Failed, stored.Outcome.Status); + Assert.Equal(ExperienceStatus.Quarantined, Assert.Single((await _store.GetHistoryAsync(auth, scope, stored.ExperienceId, CancellationToken.None)).Events).CurrentStatus); + } + + [Fact] + public async Task A_store_failure_during_finalization_is_never_reported_as_durable_success() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + await using var unreachable = Unreachable(); + var offline = new ExperienceFinalizationService( + _capture, + new DefaultExperienceReflector(), + new PostgresExperienceRecordStore(unreachable), + new ExperienceLifecycleService(new PostgresExperienceRecordStore(unreachable))); + + var result = await offline.FinalizeAsync(Request(runId, auth), CancellationToken.None); + + Assert.Equal(FinalizationOutcome.Failed, result.Outcome); + Assert.Equal(FinalizationStage.CreateRecord, result.Stage); + Assert.False(result.IsDurable); + Assert.IsType(result.Failure!.Exception); + + // The captured snapshot is still available for the host to retry with -- and the retry, against + // a reachable database, succeeds. + Assert.True(_capture.TryGetRun(runId, out _)); + var retried = await _finalization.FinalizeAsync(Request(runId, auth), CancellationToken.None); + Assert.Equal(FinalizationOutcome.Validated, retried.Outcome); + } + + [Fact] + public async Task A_record_created_but_never_confirmed_stays_a_Candidate_and_a_retry_completes_its_commit() + { + var tenant = NewTenant(); + var scope = Scope(tenant); + var auth = Authorize(tenant); + var runId = await CaptureRunAsync(scope); + + // The create lands against the real database, but the commit cannot: the lifecycle service is + // pointed at an unreachable one. + await using var unreachable = Unreachable(); + var halfway = new ExperienceFinalizationService( + _capture, + new DefaultExperienceReflector(), + _store, + new ExperienceLifecycleService(new PostgresExperienceRecordStore(unreachable))); + + var interrupted = await halfway.FinalizeAsync(Request(runId, auth), CancellationToken.None); + Assert.Equal(FinalizationOutcome.Failed, interrupted.Outcome); + Assert.Equal(FinalizationStage.CommitInitialEvent, interrupted.Stage); + Assert.False(interrupted.IsDurable); + + // What is stored is a Candidate at revision 0 with no history -- never a reusable record. + var experienceId = ExperienceFinalizationService.ExperienceIdFor(runId); + var unconfirmed = (await _store.GetAsync(auth, scope, experienceId, CancellationToken.None)).Record!; + Assert.Equal(ExperienceStatus.Candidate, unconfirmed.Status); + Assert.Equal(0, unconfirmed.Revision); + Assert.Empty((await _store.GetHistoryAsync(auth, scope, experienceId, CancellationToken.None)).Events); + + // The retry re-derives the very same initial event -- including its OccurredAt, which has been + // through PostgreSQL's microsecond truncation on the way back out -- and finishes that commit. + var retried = await _finalization.FinalizeAsync( + Request(runId, auth) with { FinalizedAt = ColumnTime.AddMinutes(30) }, + CancellationToken.None); + + Assert.Equal(FinalizationOutcome.Validated, retried.Outcome); + Assert.Equal(1, retried.Revision); + + var confirmed = (await _store.GetAsync(auth, scope, experienceId, CancellationToken.None)).Record!; + Assert.Equal(ExperienceStatus.Validated, confirmed.Status); + Assert.Equal(1, confirmed.Revision); + Assert.Equal(unconfirmed.CreatedAt, confirmed.CreatedAt); // the first call's timestamp, not the retry's + + var only = Assert.Single((await _store.GetHistoryAsync(auth, scope, experienceId, CancellationToken.None)).Events); + Assert.Equal(ExperienceFinalizationService.InitialEventIdFor(runId), only.EventId); + Assert.Equal(ExperienceStatus.Candidate, only.PriorStatus); + Assert.Equal(ExperienceStatus.Validated, only.CurrentStatus); + Assert.Equal(unconfirmed.CreatedAt, only.OccurredAt); + + // And finalizing once more is now the plain already-finalized replay. + var again = await _finalization.FinalizeAsync(Request(runId, auth), CancellationToken.None); + Assert.Equal(FinalizationOutcome.AlreadyFinalized, again.Outcome); + Assert.Single((await _store.GetHistoryAsync(auth, scope, experienceId, CancellationToken.None)).Events); + } + + private static Evidence Evidence(CheckResult result) => new( + EvidenceId: Guid.NewGuid(), + VerificationRoundId: Round.RoundId, + ArtifactRevision: ArtifactRevision, + CheckId: "unit-tests-pass", + Kind: "TestResult", + Result: result, + Producer: "ci", + Detail: "42 of 42 passed", + CapturedAt: PayloadTime); + + private static FinalizeExperienceRequest Request(Guid runId, AuthorizationContext auth) => new( + RunId: runId, + Authorization: auth, + ClosedRound: Round, + RequiredChecks: [new RequiredCheck("unit-tests-pass", "TestResult")], + Evidence: [Evidence(CheckResult.Pass)], + CurrentArtifactRevision: ArtifactRevision, + StorageDecision: StorageDecision.Permit, + FinalizedAt: ColumnTime); + + private async Task CaptureRunAsync(Scope scope) + { + var runId = Guid.NewGuid(); + var started = _capture.StartRun( + runId, + taskId: "task-1", + taskDescription: "resolve the ticket", + scope: scope, + environment: new EnvironmentFingerprint("worker-01", "net10.0", "linux-x64", "1.2.3", new Dictionary { ["region"] = "us-east" }), + provenance: new Provenance("integration-tests", "1.0.0", PayloadTime, "trace-1"), + startedAt: PayloadTime); + Assert.Equal(StartRunOutcome.Started, started.Outcome); + + var appended = await _capture.AppendAttemptAsync(runId, new AppendAttemptRequest( + AttemptId: Guid.NewGuid(), + StartedAt: PayloadTime, + Duration: TimeSpan.FromSeconds(2), + ToolCalls: + [ + new RawToolCall( + ToolCallId: Guid.NewGuid(), + ToolName: "search", + Arguments: new Dictionary { ["query"] = "refund policy" }, + StartedAt: PayloadTime, + Duration: TimeSpan.FromMilliseconds(120), + Result: "3 documents", + Error: null), + ], + Result: "done", + Error: null)); + Assert.Equal(AppendAttemptOutcome.Recorded, appended.Outcome); + + var completed = await _capture.CompleteRunAsync(runId, Guid.NewGuid(), RunExecutionStatus.Completed, PayloadTime.AddMinutes(1)); + Assert.Equal(CompleteRunOutcome.Recorded, completed.Outcome); + + return runId; + } +} diff --git a/tests/AgentExperience.Storage.Postgres.Tests/PostgresServiceRegistrationTests.cs b/tests/AgentExperience.Storage.Postgres.Tests/PostgresServiceRegistrationTests.cs new file mode 100644 index 0000000..0aaf735 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Tests/PostgresServiceRegistrationTests.cs @@ -0,0 +1,104 @@ +using AgentExperience.Storage.Postgres.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; +using Npgsql; + +namespace AgentExperience.Storage.Postgres.Tests; + +/// +/// Resolves what +/// +/// registers out of a real container, so deleting the registration fails here rather than only at a +/// host's startup. No database is touched: registering a store does not open a connection. +/// +public class PostgresServiceRegistrationTests +{ + [Fact] + public void The_store_is_resolved_from_a_data_source_in_the_container() + { + using var dataSource = TestRecords.Unreachable(); + + var services = new ServiceCollection(); + services.AddSingleton(dataSource); + services.AddAgentExperiencePostgresStore(); + + using var provider = services.BuildServiceProvider(); + + var store = provider.GetRequiredService(); + Assert.IsType(store); + Assert.Same(store, provider.GetRequiredService()); // singleton + } + + [Fact] + public void The_overload_taking_a_data_source_needs_nothing_else_in_the_container() + { + using var dataSource = TestRecords.Unreachable(); + + var services = new ServiceCollection(); + services.AddAgentExperiencePostgresStore(dataSource); + + using var provider = services.BuildServiceProvider(); + + Assert.IsType(provider.GetRequiredService()); + } + + [Fact] + public void A_host_store_registered_first_wins() + { + using var dataSource = TestRecords.Unreachable(); + var hostStore = new PostgresExperienceRecordStore(dataSource); + + var services = new ServiceCollection(); + services.AddSingleton(hostStore); + services.AddAgentExperiencePostgresStore(dataSource); + + using var provider = services.BuildServiceProvider(); + + Assert.Same(hostStore, provider.GetRequiredService()); + } + + [Fact] + public void The_candidate_source_is_resolved_independently_of_the_store() + { + using var dataSource = TestRecords.Unreachable(); + + var services = new ServiceCollection(); + services.AddSingleton(dataSource); + services.AddAgentExperiencePostgresCandidateSource(); + + using var provider = services.BuildServiceProvider(); + + // Two independent ports: a host that only searches never has to register the writer. + var source = provider.GetRequiredService(); + Assert.IsType(source); + Assert.Same(source, provider.GetRequiredService()); // singleton + Assert.Null(provider.GetService()); + } + + [Fact] + public void The_candidate_source_overload_taking_a_data_source_needs_nothing_else_in_the_container() + { + using var dataSource = TestRecords.Unreachable(); + var hostSource = new PostgresExperienceCandidateSource(dataSource); + + var services = new ServiceCollection(); + services.AddSingleton(hostSource); + services.AddAgentExperiencePostgresCandidateSource(dataSource); + + using var provider = services.BuildServiceProvider(); + + Assert.Same(hostSource, provider.GetRequiredService()); // registered first, so TryAdd keeps it + } + + [Fact] + public void Null_arguments_throw() + { + using var dataSource = TestRecords.Unreachable(); + + Assert.Throws(() => ((IServiceCollection)null!).AddAgentExperiencePostgresStore()); + Assert.Throws(() => ((IServiceCollection)null!).AddAgentExperiencePostgresStore(dataSource)); + Assert.Throws(() => new ServiceCollection().AddAgentExperiencePostgresStore((NpgsqlDataSource)null!)); + Assert.Throws(() => ((IServiceCollection)null!).AddAgentExperiencePostgresCandidateSource()); + Assert.Throws(() => ((IServiceCollection)null!).AddAgentExperiencePostgresCandidateSource(dataSource)); + Assert.Throws(() => new ServiceCollection().AddAgentExperiencePostgresCandidateSource((NpgsqlDataSource)null!)); + } +} diff --git a/tests/AgentExperience.Storage.Postgres.Tests/packages.lock.json b/tests/AgentExperience.Storage.Postgres.Tests/packages.lock.json index 414661c..ea1283f 100644 --- a/tests/AgentExperience.Storage.Postgres.Tests/packages.lock.json +++ b/tests/AgentExperience.Storage.Postgres.Tests/packages.lock.json @@ -2,6 +2,15 @@ "version": 1, "dependencies": { "net10.0": { + "Microsoft.Extensions.DependencyInjection": { + "type": "Direct", + "requested": "[10.0.11, )", + "resolved": "10.0.11", + "contentHash": "PSmotV19c7E3lKed++uYo1kSiXFI+uTl37CBSrhq+CfLC3FCHjG7R91+xPnNehQfHS1b0Tzo/CCLPWH3qaEheg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11" + } + }, "Microsoft.NET.Test.Sdk": { "type": "Direct", "requested": "[17.14.1, )", @@ -126,10 +135,54 @@ "resolved": "17.14.1", "contentHash": "pmTrhfFIoplzFVbhVwUquT+77CbGH+h4/3mBpdmIlYtBi9nAB+kKI6dN3A/nV4DFi3wLLx/BlHIPK+MkbQ6Tpg==" }, + "Microsoft.Extensions.Compliance.Abstractions": { + "type": "Transitive", + "resolved": "10.9.0", + "contentHash": "tuSqNuiJxlln43sZ8c1EDA4WXit1eX4foGadylXso3DnMVc+DtKfaNEwvHuiFXfPsEUZ6Z3GnF0Bfk9vvOsE4Q==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.ObjectPool": "10.0.11" + } + }, + "Microsoft.Extensions.Compliance.Redaction": { + "type": "Transitive", + "resolved": "10.9.0", + "contentHash": "2P0WFFq9WAyhOAZqb0FjTeKW86yL4M2vymSGyuBu5XEBWwDCiEI+BoR4TuAFtyFurDRZs5wG3JtysUy8Svlnmw==", + "dependencies": { + "Microsoft.Extensions.Compliance.Abstractions": "10.9.0", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "wlhRqZW8LcJPa+vk2oLAc/REXDItHtkFQdf/QcXYGZbZOO13izcsKY1pCvuFQYwUiZD+hwSZwsKASjqT+BNaVg==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "fVi053xdpda9Em7vSkmgVxO/PtgC2m78ekReKWsgcyskqY0U82Bz/MONwxpGzI0hElYKJfw+fupqMVeKW3fSaA==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration.Binder": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "rFn8RuszZn3qquPVkDytMUlPc2+rXl9MCoygwc1XmAgC5vg5/oXJ8hkOosOrLoBLsqdTy4lFwP6iQdPS9uSYOA==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11" + } + }, "Microsoft.Extensions.DependencyInjection.Abstractions": { "type": "Transitive", - "resolved": "10.0.0", - "contentHash": "L3AdmZ1WOK4XXT5YFPEwyt0ep6l8lGIPs7F5OOBZc77Zqeo01Of7XXICy47628sdVl0v/owxYJTe86DTgFwKCA==" + "resolved": "10.0.11", + "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", @@ -139,6 +192,37 @@ "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0" } }, + "Microsoft.Extensions.ObjectPool": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "p76ztQFROBOlHgdV1vXfmTjRyu073Av7ZlsiLR93ka6+nzkLCeV2ONXq0DO/BGf71REqGW29Uy/20fzQHAjB7Q==" + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "eY1GAKcTfD2maP27J84X9IovT3yjHJ2dVDzPmDg6/XqYvt3jMzJhtfQCLjG9pVsZGAd+8DQ2QrjaDcs2+VQLGw==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Options.ConfigurationExtensions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "syEhXQ/sEaSBFaqzlp9gDGHX/nk6gkQkh1sIUpBO1mlBj3Phu1rmb4ML1uCiyPW9N6Kxfxv3y5FGObC+bV01Qw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.Configuration.Binder": "10.0.11", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Options": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "SXcz+kF+4Oo9b1+55zntpJFYfwb1jw66ioxptyNOOTDc8g2FHnBFWjZpsWfCvZIhzr0x+4e2trVTs4OKwQfBtw==" + }, "Microsoft.TestPlatform.ObjectModel": { "type": "Transitive", "resolved": "17.14.1", @@ -235,10 +319,19 @@ "agentexperience.abstractions": { "type": "Project" }, + "agentexperience.core": { + "type": "Project", + "dependencies": { + "AgentExperience.Abstractions": "[1.0.0, )", + "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]" + } + }, "agentexperience.storage.postgres": { "type": "Project", "dependencies": { "AgentExperience.Abstractions": "[1.0.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]", "Npgsql": "[10.0.3, 10.0.3]", "dbup-core": "[6.1.1, 6.1.1]", "dbup-postgresql": "[7.0.1, 7.0.1]" diff --git a/tests/AgentExperience.Storage.Postgres.Vectors.Tests/AgentExperience.Storage.Postgres.Vectors.Tests.csproj b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/AgentExperience.Storage.Postgres.Vectors.Tests.csproj new file mode 100644 index 0000000..00f79e8 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/AgentExperience.Storage.Postgres.Vectors.Tests.csproj @@ -0,0 +1,35 @@ + + + + false + true + + $(NoWarn);CS1591 + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/AgentExperience.Storage.Postgres.Vectors.Tests/DependencyBoundaryTests.cs b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/DependencyBoundaryTests.cs new file mode 100644 index 0000000..2eb2e01 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/DependencyBoundaryTests.cs @@ -0,0 +1,113 @@ +using System.Runtime.CompilerServices; +using System.Xml.Linq; + +namespace AgentExperience.Storage.Postgres.Vectors.Tests; + +/// +/// Proves AgentExperience.Storage.Postgres.Vectors takes exactly the pins Story 1.7's proof +/// verified -- plain Npgsql, Pgvector, the model-provider abstractions, and the +/// dependency-injection abstractions its own registration extension needs -- and nothing else: no +/// MAF, EF Core, Dapper, Semantic Kernel, or concrete model-provider dependency. +/// +/// +/// This package exists precisely so those dependencies stay out of +/// AgentExperience.Storage.Postgres, whose own boundary test pins its package set exactly and +/// forbids Pgvector and Microsoft.Extensions.AI. Splitting the two means neither list +/// has to move, and a host that wants only canonical storage never pulls a vector or AI dependency in. +/// +public class DependencyBoundaryTests +{ + private static readonly string[] Forbidden = + [ + "Microsoft.Agents", // Microsoft Agent Framework (MAF) + "Microsoft.EntityFrameworkCore", // EF Core + "Dapper", + "Microsoft.SemanticKernel", // the legacy PgVector connector's home + "OpenAI", + "Azure.AI", + "Anthropic", + ]; + + [Fact] + public void Storage_Postgres_Vectors_does_not_reference_a_forbidden_assembly() + { + var referenced = typeof(PostgresExperienceEmbeddingIndex).Assembly.GetReferencedAssemblies(); + Assert.Contains(referenced, a => a.Name == "Npgsql"); + Assert.Contains(referenced, a => a.Name == "Pgvector"); + + foreach (var name in referenced.Select(a => a.Name ?? string.Empty)) + { + foreach (var forbidden in Forbidden) + { + Assert.False( + name.Contains(forbidden, StringComparison.OrdinalIgnoreCase), + $"AgentExperience.Storage.Postgres.Vectors references '{name}', which matches forbidden dependency '{forbidden}'."); + } + } + } + + [Fact] + public void Storage_Postgres_Vectors_csproj_declares_only_the_exact_verified_pins() + { + var csprojPath = GetCsprojPath(); + Assert.True(File.Exists(csprojPath), $"Could not locate AgentExperience.Storage.Postgres.Vectors.csproj at '{csprojPath}'."); + + var packages = XDocument.Load(csprojPath) + .Descendants("PackageReference") + .Select(e => $"{e.Attribute("Include")?.Value} {e.Attribute("Version")?.Value}") + .Order(StringComparer.Ordinal) + .ToList(); + + // Every version here is an exact pin, and every one was verified by Story 1.7's executable + // Postgres/pgvector proof before this package was written. + Assert.Equal( + [ + "Microsoft.Extensions.AI.Abstractions [10.9.0]", + "Microsoft.Extensions.DependencyInjection.Abstractions [10.0.11]", + "Npgsql [10.0.3]", + "Pgvector [0.3.2]", + ], + packages); + } + + [Fact] + public void The_canonical_store_package_still_takes_no_vector_or_model_provider_dependency() + { + // The whole reason this package is separate. If this ever fails, the split has been undone. + var referenced = typeof(PostgresExperienceRecordStore).Assembly.GetReferencedAssemblies() + .Select(a => a.Name ?? string.Empty) + .ToList(); + + Assert.DoesNotContain(referenced, name => name.Contains("Pgvector", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(referenced, name => name.Contains("VectorData", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(referenced, name => name.Contains("Microsoft.Extensions.AI", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Core_still_takes_no_model_provider_dependency_although_it_now_embeds() + { + // Core embeds through its own domain-typed port; the model-provider abstraction stops at the + // adapter edge, which is what keeps AD-1 true. + var referenced = typeof(Core.Indexing.ExperienceIndexingService).Assembly.GetReferencedAssemblies() + .Select(a => a.Name ?? string.Empty) + .ToList(); + + Assert.DoesNotContain(referenced, name => name.Contains("Microsoft.Extensions.AI", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(referenced, name => name.Contains("Npgsql", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(referenced, name => name.Contains("Pgvector", StringComparison.OrdinalIgnoreCase)); + + // Same for Abstractions, which stays BCL-only even though it now declares the embedding ports. + Assert.All( + typeof(IExperienceEmbeddingIndex).Assembly.GetReferencedAssemblies().Select(a => a.Name ?? string.Empty), + name => Assert.StartsWith("System.", name, StringComparison.Ordinal)); + } + + private static string GetCsprojPath([CallerFilePath] string testSourceFilePath = "") => + Path.GetFullPath(Path.Combine( + Path.GetDirectoryName(testSourceFilePath)!, + "..", + "..", + "src", + "AgentExperience.Storage.Postgres.Vectors", + "AgentExperience.Storage.Postgres.Vectors.csproj")); +} diff --git a/tests/AgentExperience.Storage.Postgres.Vectors.Tests/HybridRetrievalIntegrationTests.cs b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/HybridRetrievalIntegrationTests.cs new file mode 100644 index 0000000..2ae3e31 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/HybridRetrievalIntegrationTests.cs @@ -0,0 +1,230 @@ +using AgentExperience.Core.Retrieval; +using Microsoft.Extensions.DependencyInjection; +using Npgsql; + +namespace AgentExperience.Storage.Postgres.Vectors.Tests; + +/// +/// End-to-end hybrid retrieval over a real PostgreSQL 16 + pgvector: a record found by meaning rather +/// than by words, both channels merging into one ranked answer, and every documented fallback -- a +/// model mismatch, a dimension mismatch, and a provider that is down -- producing an explicit +/// text-only result that still carries the text candidates. All embeddings come from a deterministic +/// in-test generator, so none of this needs model credentials. +/// +[Collection(VectorsCollection.Name)] +public class HybridRetrievalIntegrationTests(VectorsFixture fixture) +{ + /// + /// Task text that shares no word with the indexed record below, so the text channel cannot match + /// it. "Chargeback" and "refund" are the same topic; "contention" and "lock" are the same topic. + /// + private const string SemanticTaskText = "chargeback contention"; + + private NpgsqlDataSource DataSource => fixture.DataSource; + + [Fact] + public async Task A_record_whose_words_do_not_overlap_the_task_text_is_still_found_through_the_vector_channel() + { + var world = await TestWorld.CreateAsync(DataSource); + var id = await world.AddRecordAsync("billing-dispute", "Reimburse a blocked payment", "Release the stuck invoice"); + await world.Indexing.IndexAsync(world.Authorization, world.Scope, id); + + // The text channel on its own finds nothing: not one word is shared. + var textOnly = await world.Retrieval(hybrid: false).RetrieveAsync(Request(world, SemanticTaskText)); + Assert.Equal(RetrievalOutcome.Completed, textOnly.Outcome); + Assert.Empty(textOnly.Records); + Assert.Equal(TextOnlyReason.NotConfigured, textOnly.VectorFallback!.Reason); + + // With the vector channel, the same request finds it. + var hybrid = await world.Retrieval().RetrieveAsync(Request(world, SemanticTaskText)); + + Assert.Equal(RetrievalOutcome.Completed, hybrid.Outcome); + Assert.False(hybrid.TextOnly); + Assert.Equal(id, Assert.Single(hybrid.Records).Record.ExperienceId); + } + + [Fact] + public async Task Both_channels_merge_into_one_ranked_answer_with_each_record_appearing_once() + { + var world = await TestWorld.CreateAsync(DataSource); + var byWords = await world.AddRecordAsync("refund-ticket", "Resolve a chargeback contention case", "Check the ledger"); + var byMeaning = await world.AddRecordAsync("billing-dispute", "Reimburse a blocked payment", "Release the stuck invoice"); + + foreach (var id in new[] { byWords, byMeaning }) + { + await world.Indexing.IndexAsync(world.Authorization, world.Scope, id); + } + + var result = await world.Retrieval().RetrieveAsync(Request(world, SemanticTaskText)); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Equal( + new[] { byWords, byMeaning }.Order(), + result.Records.Select(r => r.Record.ExperienceId).Order()); + Assert.Equal(result.Records.Count, result.Records.Select(r => r.Record.ExperienceId).Distinct().Count()); + + // Still exactly five ranking axes, whichever channel found a record. + Assert.All(result.Records, ranked => Assert.Equal(5, ranked.Components.Count)); + Assert.All(result.Records, ranked => Assert.All(ranked.Components, c => Assert.InRange(c.Value, 0d, 1d))); + } + + [Fact] + public async Task Stored_embeddings_from_another_model_give_a_text_only_result_with_no_comparison_attempted() + { + var world = await TestWorld.CreateAsync(DataSource); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + await world.Indexing.IndexAsync(world.Authorization, world.Scope, id); + + // Everything stored in this scope now claims a model the query will not be produced by. + await world.RestampEmbeddingAsync(id, "some-other-model", 4); + + var result = await world.Retrieval().RetrieveAsync(Request(world, "refund ticket")); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.True(result.TextOnly); + Assert.Equal(TextOnlyReason.ModelMismatch, result.VectorFallback!.Reason); + Assert.Null(result.Failure); + + // The text channel still answered. + Assert.Equal(id, Assert.Single(result.Records).Record.ExperienceId); + } + + [Fact] + public async Task Stored_embeddings_of_another_dimension_give_a_text_only_result() + { + var world = await TestWorld.CreateAsync(DataSource); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + await world.Indexing.IndexAsync(world.Authorization, world.Scope, id); + + // Same model, wrong width. The width predicate keeps the incompatible row out of the distance + // expression entirely, so nothing raises and nothing is compared. + await world.RestampEmbeddingAsync(id, "topic-embed-v1", 9); + + var result = await world.Retrieval().RetrieveAsync(Request(world, "refund ticket")); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.True(result.TextOnly); + Assert.Equal(TextOnlyReason.DimensionMismatch, result.VectorFallback!.Reason); + Assert.Single(result.Records); + } + + [Fact] + public async Task A_query_model_that_nothing_was_indexed_under_is_a_model_mismatch_rather_than_an_empty_match() + { + var world = await TestWorld.CreateAsync(DataSource); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + await world.Indexing.IndexAsync(world.Authorization, world.Scope, id); + + var result = await world + .Retrieval(new FixedEmbeddingGenerator { ModelId = "a-different-model", Dimension = 4 }) + .RetrieveAsync(Request(world, "refund ticket")); + + Assert.True(result.TextOnly); + Assert.Equal(TextOnlyReason.ModelMismatch, result.VectorFallback!.Reason); + Assert.Single(result.Records); + } + + [Fact] + public async Task A_provider_that_is_down_gives_a_text_only_result_and_the_text_candidates_still_come_back() + { + var world = await TestWorld.CreateAsync(DataSource); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + await world.Indexing.IndexAsync(world.Authorization, world.Scope, id); + + var outage = new InvalidOperationException("provider unavailable"); + var result = await world + .Retrieval(new UnavailableGenerator(outage)) + .RetrieveAsync(Request(world, "refund ticket")); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.True(result.TextOnly); + Assert.Equal(TextOnlyReason.ProviderUnavailable, result.VectorFallback!.Reason); + Assert.Same(outage, result.VectorFallback.Exception); + Assert.Equal(id, Assert.Single(result.Records).Record.ExperienceId); + } + + [Fact] + public async Task A_record_whose_indexing_failed_is_still_committed_durable_and_text_searchable() + { + var world = await TestWorld.CreateAsync(DataSource, new TopicEmbeddingGenerator { Throws = new InvalidOperationException("provider down") }); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + + var indexing = await world.Indexing.IndexAsync(world.Authorization, world.Scope, id); + + Assert.Equal(Core.Indexing.ExperienceIndexingOutcome.ProviderFailed, indexing.Outcome); + Assert.True(indexing.IsRetryable); + Assert.Equal(0L, await world.CountEmbeddingsAsync(id)); + + // The canonical record never depended on the provider: it is still stored, still readable, and + // still found by the text channel. + var stored = await world.Store.GetAsync(world.Authorization, world.Scope, id, CancellationToken.None); + Assert.Equal(ExperienceStoreOutcome.Found, stored.Outcome); + + var retrieved = await world.Retrieval().RetrieveAsync(Request(world, "refund ticket")); + Assert.Equal(RetrievalOutcome.Completed, retrieved.Outcome); + Assert.Equal(id, Assert.Single(retrieved.Records).Record.ExperienceId); + Assert.True(retrieved.TextOnly); + Assert.Equal(TextOnlyReason.ProviderUnavailable, retrieved.VectorFallback!.Reason); + } + + [Fact] + public async Task Nothing_matching_either_channel_is_a_completed_empty_result() + { + var world = await TestWorld.CreateAsync(DataSource); + + var result = await world.Retrieval().RetrieveAsync(Request(world, "nothing at all like anything stored")); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Empty(result.Records); + Assert.Null(result.Failure); + Assert.False(result.TextOnly); + } + + [Fact] + public async Task Hybrid_retrieval_never_crosses_a_scope_boundary() + { + var mine = await TestWorld.CreateAsync(DataSource); + var theirs = await TestWorld.CreateAsync(DataSource); + + var foreign = await theirs.AddRecordAsync("billing-dispute", "Reimburse a blocked payment", "Release the stuck invoice"); + await theirs.Indexing.IndexAsync(theirs.Authorization, theirs.Scope, foreign); + + var result = await mine.Retrieval().RetrieveAsync(Request(mine, SemanticTaskText)); + + Assert.Equal(RetrievalOutcome.Completed, result.Outcome); + Assert.Empty(result.Records); + } + + [Fact] + public async Task The_registration_extensions_resolve_a_hybrid_retrieval_service_from_a_real_container() + { + var services = new ServiceCollection(); + services.AddSingleton(DataSource); + services.AddSingleton(new TopicEmbeddingGenerator()); + Vectors.DependencyInjection.AgentExperiencePostgresVectorsServiceCollectionExtensions + .AddAgentExperiencePostgresEmbeddingIndex(services); + Postgres.DependencyInjection.AgentExperiencePostgresServiceCollectionExtensions + .AddAgentExperiencePostgresCandidateSource(services); + Core.DependencyInjection.AgentExperienceCoreServiceCollectionExtensions.AddAgentExperienceRetrieval(services); + Core.DependencyInjection.AgentExperienceCoreServiceCollectionExtensions.AddAgentExperienceIndexing(services); + + await using var provider = services.BuildServiceProvider(); + + Assert.IsType(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + Assert.True(provider.GetRequiredService().HybridEnabled); + } + + private static RetrieveExperienceRequest Request(TestWorld world, string taskText) => + new(world.Authorization, world.Scope, taskText); + + /// A generator that is always down, for the provider-outage fallback. + private sealed class UnavailableGenerator(Exception failure) : IExperienceEmbeddingGenerator + { + public string ModelId => "topic-embed-v1"; + + public int Dimension => 4; + + public Task> GenerateAsync(string text, CancellationToken cancellationToken) => throw failure; + } +} diff --git a/tests/AgentExperience.Storage.Postgres.Vectors.Tests/OfflineVectorsTests.cs b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/OfflineVectorsTests.cs new file mode 100644 index 0000000..ea1f50c --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/OfflineVectorsTests.cs @@ -0,0 +1,235 @@ +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Npgsql; + +namespace AgentExperience.Storage.Postgres.Vectors.Tests; + +/// +/// Everything about this package that needs no database: what its own migration script may and may not +/// contain, and the bridge -- the only in-repo path to a +/// real model provider -- driven over a stub . +/// +public class OfflineVectorsTests +{ + // ---------------------------------------------------------------- the script this package owns + + [Fact] + public void The_embedding_script_is_owned_here_and_not_by_the_base_adapter() + { + // The whole point of the split: a host that never enables the vector channel never runs + // CREATE EXTENSION vector, which needs a superuser. + Assert.Equal([ExperienceVectorSchema.EmbeddingsScriptName], ExperienceVectorSchema.ScriptNames); + Assert.DoesNotContain( + ExperienceVectorSchema.EmbeddingsScriptName, + PostgresExperienceRecordSchema.ScriptNames, + StringComparer.Ordinal); + Assert.Throws(() => ExperienceVectorSchema.GetScript("9999_missing.sql")); + Assert.Throws(() => PostgresExperienceRecordSchema.GetScript(ExperienceVectorSchema.EmbeddingsScriptName)); + } + + [Fact] + public void Embedded_migration_resources_match_the_declared_script_names() + { + var embedded = typeof(ExperienceVectorSchema).Assembly.GetManifestResourceNames() + .Where(name => name.StartsWith(ExperienceVectorSchema.ResourcePrefix, StringComparison.Ordinal) + && name.EndsWith(".sql", StringComparison.Ordinal)) + .Select(name => name[ExperienceVectorSchema.ResourcePrefix.Length..]) + .Order(StringComparer.Ordinal); + + Assert.Equal(ExperienceVectorSchema.ScriptNames.Order(StringComparer.Ordinal), embedded); + } + + [Fact] + public void The_embedding_script_only_adds_derived_write_artifacts() + { + var script = ExperienceVectorSchema.GetScript(ExperienceVectorSchema.EmbeddingsScriptName); + + Assert.Contains("CREATE EXTENSION IF NOT EXISTS vector", script, StringComparison.Ordinal); + Assert.Contains("CREATE TABLE IF NOT EXISTS agent_experience.experience_embeddings", script, StringComparison.Ordinal); + + // What an embedding *is*, stored separately from lifecycle state. + Assert.Contains("model_id text NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("dimension integer NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("content_hash text NOT NULL", script, StringComparison.Ordinal); + Assert.Contains("source_revision bigint NOT NULL", script, StringComparison.Ordinal); + + // Unconstrained, with the dimension carried in its own column and checked against the vector, so + // the search's embedding::vector(n) cast can never meet a row that disagrees. + Assert.Contains("embedding vector NOT NULL", script, StringComparison.Ordinal); + Assert.DoesNotContain("embedding vector(", script, StringComparison.Ordinal); + Assert.Contains("CHECK (vector_dims(embedding) = dimension)", script, StringComparison.Ordinal); + + var statements = string.Join( + '\n', + script.Split('\n').Where(line => !line.TrimStart().StartsWith("--", StringComparison.Ordinal))); + + // An embedding may never outlive the record it describes. + Assert.Contains("ON DELETE CASCADE", statements, StringComparison.Ordinal); + + // Append-only: it adds its own table and rewrites nothing the base adapter created. + Assert.DoesNotContain("DROP", statements, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("ALTER TABLE", statements, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("lifecycle_events", statements, StringComparison.Ordinal); + + // The approximate-nearest-neighbour index needs a dimension this script does not have, so it is + // deliberately absent and created by an explicit adapter call. + Assert.DoesNotContain("hnsw", statements, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("ivfflat", statements, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task The_vectors_migrator_rejects_a_null_data_source() + { + await Assert.ThrowsAsync( + () => ExperienceVectorSchemaMigrator.MigrateAsync(null!, CancellationToken.None)); + } + + // ---------------------------------------------------------------- the model-provider bridge + + [Fact] + public void The_generator_takes_its_model_and_dimension_from_the_providers_metadata() + { + var bridge = new AiExperienceEmbeddingGenerator(new StubEmbeddingGenerator("text-embed-3", 6)); + + Assert.Equal("text-embed-3", bridge.ModelId); + Assert.Equal(6, bridge.Dimension); + } + + [Fact] + public void An_explicit_model_and_dimension_stand_in_for_metadata_the_provider_does_not_report() + { + var bridge = new AiExperienceEmbeddingGenerator( + new StubEmbeddingGenerator(modelId: null, dimensions: null), + modelId: "host-chosen", + dimension: 3); + + Assert.Equal("host-chosen", bridge.ModelId); + Assert.Equal(3, bridge.Dimension); + } + + [Fact] + public void An_argument_that_contradicts_the_provider_is_rejected_at_construction() + { + // Stamping vectors with a model the provider did not produce them under is exactly how two + // incomparable sets come to look comparable -- the one thing the descriptor exists to prevent. + var provider = new StubEmbeddingGenerator("text-embed-3", 6); + + Assert.Throws(() => new AiExperienceEmbeddingGenerator(provider, modelId: "something-else")); + Assert.Throws(() => new AiExperienceEmbeddingGenerator(provider, dimension: 4)); + } + + [Fact] + public void A_provider_that_reports_nothing_and_is_told_nothing_fails_at_construction_not_mid_query() + { + var provider = new StubEmbeddingGenerator(modelId: null, dimensions: null); + + Assert.Throws(() => new AiExperienceEmbeddingGenerator(null!)); + Assert.Throws(() => new AiExperienceEmbeddingGenerator(provider)); + Assert.Throws(() => new AiExperienceEmbeddingGenerator(provider, modelId: " ", dimension: 3)); + Assert.Throws(() => new AiExperienceEmbeddingGenerator(provider, modelId: "m")); + Assert.Throws(() => new AiExperienceEmbeddingGenerator(provider, modelId: "m", dimension: 0)); + } + + [Fact] + public async Task Every_request_tells_the_provider_which_model_and_width_to_answer_with() + { + // Otherwise a host that named the model would get vectors from the provider's own default, + // stamped with the name it asked for. + var provider = new StubEmbeddingGenerator("text-embed-3", 6); + var bridge = new AiExperienceEmbeddingGenerator(provider); + + var vector = await bridge.GenerateAsync("refund ticket", CancellationToken.None); + + Assert.Equal(6, vector.Length); + Assert.Equal("text-embed-3", provider.LastOptions!.ModelId); + Assert.Equal(6, provider.LastOptions.Dimensions); + Assert.Equal(["refund ticket"], provider.Requests); + } + + [Fact] + public async Task A_provider_that_answers_with_the_wrong_width_or_with_nothing_is_rejected() + { + var wrongWidth = new AiExperienceEmbeddingGenerator(new StubEmbeddingGenerator("m", 6) { ReturnDimension = 5 }); + var empty = new AiExperienceEmbeddingGenerator(new StubEmbeddingGenerator("m", 6) { ReturnNothing = true }); + + await Assert.ThrowsAsync(() => wrongWidth.GenerateAsync("text", CancellationToken.None)); + await Assert.ThrowsAsync(() => empty.GenerateAsync("text", CancellationToken.None)); + await Assert.ThrowsAsync( + () => new AiExperienceEmbeddingGenerator(new StubEmbeddingGenerator("m", 6)).GenerateAsync(null!, CancellationToken.None)); + } + + [Fact] + public void AddAgentExperienceEmbeddingGenerator_resolves_the_bridge_over_a_registered_provider() + { + var services = new ServiceCollection(); + services.AddSingleton>>(new StubEmbeddingGenerator("text-embed-3", 6)); + DependencyInjection.AgentExperiencePostgresVectorsServiceCollectionExtensions + .AddAgentExperienceEmbeddingGenerator(services); + + using var provider = services.BuildServiceProvider(); + var resolved = provider.GetRequiredService(); + + Assert.IsType(resolved); + Assert.Equal("text-embed-3", resolved.ModelId); + Assert.Equal(6, resolved.Dimension); + Assert.Same(resolved, provider.GetRequiredService()); + } + + [Fact] + public void AddAgentExperiencePostgresEmbeddingIndex_registers_the_index_over_a_host_owned_data_source() + { + using var dataSource = NpgsqlDataSource.Create("Host=127.0.0.1;Port=1;Username=nobody;Password=nothing;Database=none"); + var services = new ServiceCollection(); + DependencyInjection.AgentExperiencePostgresVectorsServiceCollectionExtensions + .AddAgentExperiencePostgresEmbeddingIndex(services, dataSource); + + using var provider = services.BuildServiceProvider(); + + Assert.IsType(provider.GetRequiredService()); + } + + /// + /// A stub Microsoft.Extensions.AI generator: it reports whatever metadata the test wants, + /// records the options it was called with, and answers deterministically. No provider, no network. + /// + private sealed class StubEmbeddingGenerator(string? modelId, int? dimensions) + : IEmbeddingGenerator> + { + private readonly EmbeddingGeneratorMetadata _metadata = new("stub", providerUri: null, modelId, dimensions); + + public List Requests { get; } = []; + + public EmbeddingGenerationOptions? LastOptions { get; private set; } + + /// When set, the answer has this many components instead of the requested width. + public int? ReturnDimension { get; init; } + + /// When set, the answer holds no embedding at all. + public bool ReturnNothing { get; init; } + + public Task>> GenerateAsync( + IEnumerable values, + EmbeddingGenerationOptions? options = null, + CancellationToken cancellationToken = default) + { + Requests.AddRange(values); + LastOptions = options; + + var generated = new GeneratedEmbeddings>(); + if (!ReturnNothing) + { + var width = ReturnDimension ?? options?.Dimensions ?? dimensions ?? 1; + generated.Add(new Embedding(new float[width])); + } + + return Task.FromResult(generated); + } + + public object? GetService(Type serviceType, object? serviceKey = null) => + serviceKey is null && serviceType == typeof(EmbeddingGeneratorMetadata) ? _metadata : null; + + public void Dispose() + { + } + } +} diff --git a/tests/AgentExperience.Storage.Postgres.Vectors.Tests/PostgresEmbeddingIndexTests.cs b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/PostgresEmbeddingIndexTests.cs new file mode 100644 index 0000000..0f6f947 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/PostgresEmbeddingIndexTests.cs @@ -0,0 +1,503 @@ +using AgentExperience.Core.Indexing; +using Npgsql; + +namespace AgentExperience.Storage.Postgres.Vectors.Tests; + +/// +/// Container-backed coverage of against a real +/// PostgreSQL 16 + pgvector: ingestion after a commit, the conditional write's stale and deleted +/// cases, re-index idempotence, and the scope isolation both retrieval channels have to share. Every +/// embedding is produced by a deterministic in-test generator -- no model, no network, no credentials. +/// +[Collection(VectorsCollection.Name)] +public class PostgresEmbeddingIndexTests(VectorsFixture fixture) +{ + private static readonly DateTimeOffset Now = new DateTimeOffset(2026, 9, 21, 10, 0, 0, TimeSpan.Zero).AddTicks(1_234_560); + + private NpgsqlDataSource DataSource => fixture.DataSource; + + // ---------------------------------------------------------------- the schema itself + + [Fact] + public async Task The_migration_creates_the_vector_extension_and_the_embedding_table() + { + Assert.Equal(1L, await ScalarAsync("SELECT count(*) FROM pg_extension WHERE extname = 'vector'")); + + var columns = await ColumnsAsync("experience_embeddings"); + Assert.Equal( + [ + "agent_id", "application_id", "content_hash", "created_at", "dimension", "embedding", + "experience_id", "model_id", "project_id", "source_revision", "team_id", "tenant_id", + "updated_at", "user_id", + ], + columns.Keys.Order(StringComparer.Ordinal)); + + // Unconstrained on purpose: the dimension belongs to whichever model a host configured. + Assert.Equal("USER-DEFINED", columns["embedding"]); + Assert.Null(await ScalarAsync( + "SELECT atttypmod FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid " + + "JOIN pg_namespace n ON n.oid = c.relnamespace " + + "WHERE n.nspname = 'agent_experience' AND c.relname = 'experience_embeddings' AND a.attname = 'embedding' " + + "AND a.atttypmod <> -1")); + } + + [Fact] + public async Task The_out_of_band_HNSW_index_is_created_explicitly_and_creating_it_twice_is_a_no_op() + { + const int Dimension = 5; + var name = ExperienceVectorIndexMaintenance.IndexNameFor(Dimension); + + try + { + await ExperienceVectorIndexMaintenance.EnsureHnswIndexAsync(DataSource, Dimension); + await ExperienceVectorIndexMaintenance.EnsureHnswIndexAsync(DataSource, Dimension); + + var definition = await ScalarAsync( + $"SELECT indexdef FROM pg_indexes WHERE schemaname = 'agent_experience' AND indexname = '{name}'"); + + Assert.NotNull(definition); + Assert.Contains("hnsw", definition, StringComparison.OrdinalIgnoreCase); + Assert.Contains("vector_cosine_ops", definition, StringComparison.Ordinal); + // Partial on the dimension, which is what makes the cast in the indexed expression safe on + // a table that may hold several widths at once. + Assert.Contains("WHERE (dimension = 5)", definition, StringComparison.Ordinal); + } + finally + { + await ExperienceVectorIndexMaintenance.DropHnswIndexAsync(DataSource, Dimension); + } + + Assert.Equal( + 0L, + await ScalarAsync($"SELECT count(*) FROM pg_indexes WHERE schemaname = 'agent_experience' AND indexname = '{name}'")); + } + + [Fact] + public async Task The_search_actually_uses_the_out_of_band_HNSW_index() + { + // Creating the index is not the claim -- the search *using* it is. The search's ORDER BY and + // the index's expression have to match exactly, and nothing but the planner can confirm that. + var world = await WorldAsync(); + var dimension = world.Generator.Dimension; + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + await world.Indexing.IndexAsync(world.Authorization, world.Scope, id); + + await ExperienceVectorIndexMaintenance.EnsureHnswIndexAsync(DataSource, dimension); + try + { + var plan = await world.ExplainSearchAsync(TopicEmbeddingGenerator.VectorFor("refund stuck on a lock")); + Assert.Contains(ExperienceVectorIndexMaintenance.IndexNameFor(dimension), plan, StringComparison.Ordinal); + // And the scope predicate on the embeddings row is what keeps a foreign scope out of the + // HNSW walk itself rather than only out of the joined result. + Assert.Contains("experience_embeddings e", plan, StringComparison.Ordinal); + } + finally + { + await ExperienceVectorIndexMaintenance.DropHnswIndexAsync(DataSource, dimension); + } + + // Dropped, the same search is still correct -- pgvector simply scans exactly. + var afterDrop = await world.Index.SearchAsync( + world.Authorization, + VectorQuery(world, TopicEmbeddingGenerator.VectorFor("refund stuck on a lock")), + CancellationToken.None); + + Assert.Equal(ExperienceVectorSearchOutcome.Found, afterDrop.Outcome); + Assert.Equal(id, Assert.Single(afterDrop.Candidates).Record.ExperienceId); + } + + // ---------------------------------------------------------------- matrix: index after commit + + [Fact] + public async Task Indexing_a_committed_record_stores_the_vector_with_its_model_dimension_hash_and_revision() + { + var world = await WorldAsync(); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock before retrying"); + + var result = await world.Indexing.IndexAsync(world.Authorization, world.Scope, id); + + Assert.Equal(ExperienceIndexingOutcome.Indexed, result.Outcome); + + var stored = await world.ReadEmbeddingAsync(id); + Assert.Equal("topic-embed-v1", stored.ModelId); + Assert.Equal(4, stored.Dimension); + Assert.Equal(0, stored.SourceRevision); + Assert.Equal( + ExperienceEmbeddingDescriptor.ComputeContentHash( + "topic-embed-v1", + ExperienceRetrievalSummary.For("refund-ticket", "Resolve a refund ticket", "Release the lock before retrying")), + stored.ContentHash); + + // The scope columns are copied from the record row inside the conditional write, never from + // caller input, so they always agree with the record they describe. + Assert.Equal(world.Scope.TenantId, stored.TenantId); + Assert.Equal(world.Scope.ProjectId, stored.ProjectId); + } + + // ---------------------------------------------------------------- matrix: stale write + + [Fact] + public async Task A_write_whose_revision_has_moved_is_rejected_and_the_stored_vector_still_names_the_old_revision() + { + var world = await WorldAsync(); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + await world.BumpRevisionAsync(id, 1); + + Assert.Equal(ExperienceIndexingOutcome.Indexed, (await world.Indexing.IndexAsync(world.Authorization, world.Scope, id)).Outcome); + Assert.Equal(1, (await world.ReadEmbeddingAsync(id)).SourceRevision); + + // The record moves to revision 2, and an in-flight write computed from revision 1 tries to land. + await world.BumpRevisionAsync(id, 2); + + var stale = await world.Index.WriteAsync( + world.Authorization, + new ExperienceIndexWrite( + world.Scope, + id, + new ExperienceEmbeddingDescriptor("topic-embed-v1", 4, "a-different-hash", 1), + TopicEmbeddingGenerator.VectorFor("something else entirely")), + CancellationToken.None); + + Assert.Equal(ExperienceIndexOutcome.Stale, stale.Outcome); + Assert.Equal(2, stale.CurrentRevision); + + var unchanged = await world.ReadEmbeddingAsync(id); + Assert.Equal(1, unchanged.SourceRevision); + Assert.NotEqual("a-different-hash", unchanged.ContentHash); + } + + [Fact] + public async Task A_write_from_an_older_revision_can_never_overwrite_one_already_stored_from_a_newer_one() + { + // The upsert's second line of defence, for two writes racing rather than one arriving late: the + // record is at revision 1, an older in-flight write computed from revision 0 is still valid by + // the INSERT's own predicate only if the record were still at 0 -- so this drives the guard by + // storing from the newer revision first and then replaying the older write at the same revision. + var world = await WorldAsync(); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + + var newer = new ExperienceEmbeddingDescriptor("topic-embed-v1", 4, "newer-hash", 0); + Assert.Equal( + ExperienceIndexOutcome.Written, + (await world.Index.WriteAsync( + world.Authorization, + new ExperienceIndexWrite(world.Scope, id, newer, TopicEmbeddingGenerator.VectorFor("newer")), + CancellationToken.None)).Outcome); + + // Force the stored row to claim a source revision beyond the record's own, which is exactly what + // a write that landed from a newer revision leaves behind for a slower racer to find. + await world.ForceStoredSourceRevisionAsync(id, 5); + + var loser = await world.Index.WriteAsync( + world.Authorization, + new ExperienceIndexWrite( + world.Scope, + id, + new ExperienceEmbeddingDescriptor("topic-embed-v1", 4, "older-hash", 0), + TopicEmbeddingGenerator.VectorFor("older")), + CancellationToken.None); + + Assert.Equal(ExperienceIndexOutcome.Stale, loser.Outcome); + + var stored = await world.ReadEmbeddingAsync(id); + Assert.Equal("newer-hash", stored.ContentHash); + Assert.Equal(5, stored.SourceRevision); + } + + // ---------------------------------------------------------------- matrix: deleted record + + [Fact] + public async Task A_record_deleted_before_the_write_lands_is_never_recreated() + { + var world = await WorldAsync(); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + await world.DeleteRecordAsync(id); + + var missing = await world.Index.WriteAsync( + world.Authorization, + new ExperienceIndexWrite( + world.Scope, + id, + new ExperienceEmbeddingDescriptor("topic-embed-v1", 4, "hash", 0), + TopicEmbeddingGenerator.VectorFor("anything")), + CancellationToken.None); + + Assert.Equal(ExperienceIndexOutcome.Missing, missing.Outcome); + Assert.Equal(0L, await world.CountEmbeddingsAsync(id)); + } + + [Fact] + public async Task Deleting_a_record_takes_its_embedding_with_it() + { + var world = await WorldAsync(); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + await world.Indexing.IndexAsync(world.Authorization, world.Scope, id); + Assert.Equal(1L, await world.CountEmbeddingsAsync(id)); + + await world.DeleteRecordAsync(id); + + // The foreign key cascades, so an embedding can never outlive the record it describes. + Assert.Equal(0L, await world.CountEmbeddingsAsync(id)); + } + + // ---------------------------------------------------------------- matrix: reindex + + [Fact] + public async Task Reindexing_unchanged_records_calls_no_provider_and_rewrites_nothing() + { + var world = await WorldAsync(); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + + var first = await world.Indexing.ReindexAsync(world.Authorization, new ReindexExperienceRequest(world.Scope)); + Assert.Equal(1, first.Indexed); + var writtenAt = (await world.ReadEmbeddingAsync(id)).UpdatedAt; + var callsAfterFirst = world.Generator.Requests.Count; + + var second = await world.Indexing.ReindexAsync(world.Authorization, new ReindexExperienceRequest(world.Scope)); + + Assert.Equal(ExperienceReindexOutcome.Completed, second.Outcome); + Assert.Equal(1, second.Examined); + Assert.Equal(0, second.Indexed); + Assert.Equal(1, second.Skipped); + Assert.Equal(callsAfterFirst, world.Generator.Requests.Count); + Assert.Equal(writtenAt, (await world.ReadEmbeddingAsync(id)).UpdatedAt); + } + + [Fact] + public async Task Reindexing_a_changed_summary_rewrites_it_once_and_the_repeat_is_idempotent() + { + var world = await WorldAsync(); + var id = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + + await world.Indexing.ReindexAsync(world.Authorization, new ReindexExperienceRequest(world.Scope)); + var firstHash = (await world.ReadEmbeddingAsync(id)).ContentHash; + + await world.RewriteLessonAsync(id, "Release the lock and confirm the ledger entry afterwards"); + + var rewritten = await world.Indexing.ReindexAsync(world.Authorization, new ReindexExperienceRequest(world.Scope)); + Assert.Equal(1, rewritten.Indexed); + Assert.NotEqual(firstHash, (await world.ReadEmbeddingAsync(id)).ContentHash); + + var repeat = await world.Indexing.ReindexAsync(world.Authorization, new ReindexExperienceRequest(world.Scope)); + Assert.Equal(0, repeat.Indexed); + Assert.Equal(1, repeat.Skipped); + Assert.Equal(1L, await world.CountEmbeddingsAsync(id)); + } + + [Fact] + public async Task A_scope_larger_than_one_page_is_walked_to_the_end_by_the_cursor() + { + // Without a cursor every pass re-reads the same first page, so a scope larger than the limit is + // never fully indexed however often the pass runs. + var world = await WorldAsync(); + var ids = new List(); + for (var n = 0; n < 5; n++) + { + ids.Add(await world.AddRecordAsync($"task-{n}", $"summary {n}", $"lesson {n}")); + } + + var seen = new List(); + Guid? cursor = null; + for (var page = 0; page < 5; page++) + { + var pass = await world.Indexing.ReindexAsync( + world.Authorization, + new Core.Indexing.ReindexExperienceRequest(world.Scope, Limit: 2, StartAfterId: cursor)); + + Assert.Equal(ExperienceReindexOutcome.Completed, pass.Outcome); + seen.AddRange(pass.Records.Select(record => record.ExperienceId)); + + if (pass.LastExaminedId is null) + { + // The documented way to know the scope is exhausted. + Assert.Equal(0, pass.Examined); + break; + } + + cursor = pass.LastExaminedId; + } + + Assert.Equal(ids.Count, seen.Distinct().Count()); + Assert.Equal(ids.Order(), seen.Order()); + Assert.All(ids, id => Assert.Equal(1L, world.CountEmbeddingsAsync(id).GetAwaiter().GetResult())); + } + + [Fact] + public async Task A_record_a_search_could_never_return_is_never_listed_and_never_embedded() + { + // The scan applies the search's own status and confidence predicates, so a quarantined or + // low-confidence record's task summary and lesson never leave the database for a provider. + var world = await WorldAsync(); + var eligible = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + var quarantined = await world.AddRecordAsync("secret-task", "Quarantined summary", "Quarantined lesson", status: ExperienceStatus.Quarantined); + var lowConfidence = await world.AddRecordAsync("weak-task", "Low confidence summary", "Low confidence lesson", confidence: 0.1); + + var pass = await world.Indexing.ReindexAsync(world.Authorization, new Core.Indexing.ReindexExperienceRequest(world.Scope)); + + Assert.Equal(1, pass.Examined); + Assert.Equal([eligible], pass.Records.Select(record => record.ExperienceId)); + Assert.Equal(0L, await world.CountEmbeddingsAsync(quarantined)); + Assert.Equal(0L, await world.CountEmbeddingsAsync(lowConfidence)); + + // And nothing about them was handed to the provider. + Assert.DoesNotContain(world.Generator.Requests, text => text.Contains("Quarantined", StringComparison.Ordinal)); + Assert.DoesNotContain(world.Generator.Requests, text => text.Contains("Low confidence", StringComparison.Ordinal)); + } + + // ---------------------------------------------------------------- scope and eligibility in SQL + + [Fact] + public async Task A_vector_search_never_returns_a_record_from_another_scope_however_close_its_vector() + { + var mine = await WorldAsync(); + var theirs = await WorldAsync(); + + var ours = await mine.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + var foreign = await theirs.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + await mine.Indexing.IndexAsync(mine.Authorization, mine.Scope, ours); + await theirs.Indexing.IndexAsync(theirs.Authorization, theirs.Scope, foreign); + + var results = await mine.Index.SearchAsync( + mine.Authorization, + new ExperienceVectorQuery( + mine.Scope, + "topic-embed-v1", + TopicEmbeddingGenerator.VectorFor("refund stuck on a lock"), + [ExperienceStatus.Validated, ExperienceStatus.Reinforced], + MinimumConfidence: 0.5, + Limit: 50), + CancellationToken.None); + + Assert.Equal(ExperienceVectorSearchOutcome.Found, results.Outcome); + Assert.Equal([ours], results.Candidates.Select(candidate => candidate.Record.ExperienceId)); + Assert.All(results.Candidates, candidate => Assert.Equal(mine.Scope, candidate.Record.Scope)); + Assert.All(results.Candidates, candidate => Assert.InRange(candidate.Relevance, 0d, 1d)); + } + + [Fact] + public async Task The_status_filter_and_the_confidence_floor_are_applied_in_SQL_on_the_vector_channel_too() + { + var world = await WorldAsync(); + var eligible = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock"); + var quarantined = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock", status: ExperienceStatus.Quarantined); + var lowConfidence = await world.AddRecordAsync("refund-ticket", "Resolve a refund ticket", "Release the lock", confidence: 0.1); + + foreach (var id in new[] { eligible, quarantined, lowConfidence }) + { + await world.Indexing.IndexAsync(world.Authorization, world.Scope, id); + } + + var results = await world.Index.SearchAsync( + world.Authorization, + new ExperienceVectorQuery( + world.Scope, + "topic-embed-v1", + TopicEmbeddingGenerator.VectorFor("refund stuck on a lock"), + [ExperienceStatus.Validated, ExperienceStatus.Reinforced], + MinimumConfidence: 0.5, + Limit: 50), + CancellationToken.None); + + Assert.Equal([eligible], results.Candidates.Select(candidate => candidate.Record.ExperienceId)); + } + + [Fact] + public async Task A_scope_outside_the_authorization_is_denied_before_any_statement_runs() + { + var world = await WorldAsync(); + var elsewhere = new AuthorizationContext(world.Scope.TenantId, "p", [], Now, ProjectId: "elsewhere"); + + var write = await world.Index.WriteAsync( + elsewhere, + new ExperienceIndexWrite(world.Scope, Guid.NewGuid(), new ExperienceEmbeddingDescriptor("m", 4, "h", 0), new float[4]), + CancellationToken.None); + var scan = await world.Index.ScanAsync(elsewhere, new ExperienceIndexScan(world.Scope, "m", [ExperienceStatus.Validated], 0.5), CancellationToken.None); + var search = await world.Index.SearchAsync( + elsewhere, + new ExperienceVectorQuery(world.Scope, "m", new float[4], [ExperienceStatus.Validated], 0.5), + CancellationToken.None); + + Assert.Equal(ExperienceIndexOutcome.Denied, write.Outcome); + Assert.Equal(ExperienceStoreOutcome.Denied, scan.Outcome); + Assert.Equal(ExperienceVectorSearchOutcome.Denied, search.Outcome); + } + + [Fact] + public async Task A_malformed_request_is_Invalid_with_every_field_path_and_no_database_call() + { + var index = new PostgresExperienceEmbeddingIndex(Unreachable()); // reaching it would throw + + var search = await index.SearchAsync( + new AuthorizationContext("t", "p", [], Now), + new ExperienceVectorQuery(new Scope("t", "app-1", " "), " ", ReadOnlyMemory.Empty, [], 1.5, 0), + CancellationToken.None); + + Assert.Equal(ExperienceVectorSearchOutcome.Invalid, search.Outcome); + Assert.Equal( + ["Scope.ProjectId", "ModelId", "Vector", "EligibleStatuses", "MinimumConfidence", "Limit"], + search.Errors.Select(error => error.Path)); + + var write = await index.WriteAsync( + new AuthorizationContext("t", "p", [], Now), + new ExperienceIndexWrite(new Scope("t", "app-1", "project-1"), Guid.Empty, new ExperienceEmbeddingDescriptor(" ", 0, " ", -1), new[] { float.NaN }), + CancellationToken.None); + + Assert.Equal(ExperienceIndexOutcome.Invalid, write.Outcome); + Assert.Equal( + ["ExperienceId", "Descriptor.ModelId", "Descriptor.Dimension", "Descriptor.ContentHash", "Descriptor.SourceRevision", "Vector"], + write.Errors.Select(error => error.Path)); + } + + [Fact] + public async Task Null_arguments_throw() + { + var index = new PostgresExperienceEmbeddingIndex(Unreachable()); + var authorization = new AuthorizationContext("t", "p", [], Now); + + Assert.Throws(() => new PostgresExperienceEmbeddingIndex(null!)); + await Assert.ThrowsAsync(() => index.WriteAsync(null!, new ExperienceIndexWrite(new Scope("t", "a", "p"), Guid.NewGuid(), new ExperienceEmbeddingDescriptor("m", 1, "h", 0), new float[1]), CancellationToken.None)); + await Assert.ThrowsAsync(() => index.WriteAsync(authorization, null!, CancellationToken.None)); + await Assert.ThrowsAsync(() => index.ScanAsync(authorization, null!, CancellationToken.None)); + await Assert.ThrowsAsync(() => index.SearchAsync(authorization, null!, CancellationToken.None)); + await Assert.ThrowsAsync(() => ExperienceVectorIndexMaintenance.EnsureHnswIndexAsync(null!, 4)); + Assert.Throws(() => ExperienceVectorIndexMaintenance.IndexNameFor(0)); + } + + // ---------------------------------------------------------------- helpers + + private static ExperienceVectorQuery VectorQuery(TestWorld world, ReadOnlyMemory vector) => new( + world.Scope, + "topic-embed-v1", + vector, + [ExperienceStatus.Validated, ExperienceStatus.Reinforced], + MinimumConfidence: 0.5, + Limit: 50); + + private static NpgsqlDataSource Unreachable() => + NpgsqlDataSource.Create("Host=127.0.0.1;Port=1;Username=nobody;Password=nothing;Database=none;Timeout=3;Pooling=false"); + + private async Task WorldAsync() => await TestWorld.CreateAsync(DataSource); + + private async Task ScalarAsync(string sql) + { + await using var command = DataSource.CreateCommand(sql); + var value = await command.ExecuteScalarAsync(); + return value is null or DBNull ? default : (T)value; + } + + private async Task> ColumnsAsync(string table) + { + await using var command = DataSource.CreateCommand( + "SELECT column_name, data_type FROM information_schema.columns " + + $"WHERE table_schema = 'agent_experience' AND table_name = '{table}'"); + + var columns = new Dictionary(StringComparer.Ordinal); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + columns[reader.GetString(0)] = reader.GetString(1).ToUpperInvariant(); + } + + return columns; + } +} diff --git a/tests/AgentExperience.Storage.Postgres.Vectors.Tests/TestWorld.cs b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/TestWorld.cs new file mode 100644 index 0000000..7345ab0 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/TestWorld.cs @@ -0,0 +1,237 @@ +using AgentExperience.Core.Indexing; +using AgentExperience.Core.Retrieval; +using Npgsql; +using NpgsqlTypes; + +namespace AgentExperience.Storage.Postgres.Vectors.Tests; + +/// +/// One isolated tenant's worth of the real stack over the shared container: the canonical store, the +/// text candidate source, the pgvector embedding index, and Core's indexing and retrieval services -- +/// all pointed at a scope no other test uses, so tests can run in any order without seeing each +/// other's records. +/// +internal sealed class TestWorld +{ + private static readonly DateTimeOffset Stamp = new DateTimeOffset(2026, 9, 21, 10, 0, 0, TimeSpan.Zero).AddTicks(1_234_560); + + private TestWorld(NpgsqlDataSource dataSource, Scope scope, TopicEmbeddingGenerator generator) + { + DataSource = dataSource; + Scope = scope; + Generator = generator; + Authorization = new AuthorizationContext(scope.TenantId, "host-principal", ["experience:write"], Stamp); + Store = new PostgresExperienceRecordStore(dataSource); + CandidateSource = new PostgresExperienceCandidateSource(dataSource); + Index = new PostgresExperienceEmbeddingIndex(dataSource); + Indexing = new ExperienceIndexingService(Index, generator); + } + + public NpgsqlDataSource DataSource { get; } + + public Scope Scope { get; } + + public AuthorizationContext Authorization { get; } + + public TopicEmbeddingGenerator Generator { get; } + + public PostgresExperienceRecordStore Store { get; } + + public PostgresExperienceCandidateSource CandidateSource { get; } + + public PostgresExperienceEmbeddingIndex Index { get; } + + public ExperienceIndexingService Indexing { get; } + + public static Task CreateAsync(NpgsqlDataSource dataSource, TopicEmbeddingGenerator? generator = null) => + Task.FromResult(new TestWorld( + dataSource, + new Scope("tenant-" + Guid.NewGuid().ToString("N"), "app-1", "project-1"), + generator ?? new TopicEmbeddingGenerator())); + + /// + /// A retrieval service over this world's real text and vector channels. The timeout is generous on + /// purpose: these tests are about what the channels return, not about how fast a container is. + /// + public ExperienceRetrievalService Retrieval(IExperienceEmbeddingGenerator? queryGenerator = null, bool hybrid = true) => new( + CandidateSource, + RetrievalPolicy.Default with { Timeout = TimeSpan.FromSeconds(30) }, + RankingWeights.Default, + TimeProvider.System, + hybrid ? Index : null, + hybrid ? queryGenerator ?? Generator : null); + + /// Creates a record in this world's scope, already eligible for retrieval unless told otherwise. + public async Task AddRecordAsync( + string taskId, + string? taskSummary, + string? lesson, + ExperienceStatus status = ExperienceStatus.Validated, + double confidence = 0.8) + { + var id = Guid.NewGuid(); + var record = new ExperienceRecord( + ExperienceId: id, + SourceRunId: Guid.NewGuid(), + Scope: Scope, + TaskId: taskId, + TaskSummary: taskSummary, + Attempts: [], + Outcome: new Outcome(TaskVerificationStatus.Verified, [], "checks passed", Stamp), + CompletionScore: 1, + Reflection: lesson is null + ? null + : new Reflection( + Guid.NewGuid(), Guid.NewGuid(), lesson, [], [], [], [], null, [], + TaskVerificationStatus.Verified, 1, "v1", "tests", Stamp), + Environment: new EnvironmentFingerprint("worker-01", "10.0.0", "linux-x64", null, new Dictionary()), + Provenance: new Provenance("tests", null, Stamp, null), + Status: status, + ReuseConfidence: confidence, + SupportingValidations: 1, + Contradictions: 0, + Revision: 0, + CreatedAt: Stamp, + UpdatedAt: Stamp); + + var created = await Store.CreateAsync(Authorization, record, CancellationToken.None); + Assert.Equal(ExperienceStoreOutcome.Created, created.Outcome); + return id; + } + + /// The stored embedding row, read straight out of SQL rather than through the port. + public async Task ReadEmbeddingAsync(Guid experienceId) + { + await using var command = DataSource.CreateCommand( + "SELECT model_id, dimension, content_hash, source_revision, tenant_id, project_id, updated_at, " + + "vector_dims(embedding) FROM agent_experience.experience_embeddings WHERE experience_id = @id"); + command.Parameters.Add(new NpgsqlParameter("id", experienceId)); + + await using var reader = await command.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync(), $"No embedding row for {experienceId}."); + return new StoredEmbedding( + reader.GetString(0), + reader.GetInt32(1), + reader.GetString(2), + reader.GetInt64(3), + reader.GetString(4), + reader.GetString(5), + reader.GetFieldValue(6), + reader.GetInt32(7)); + } + + public async Task CountEmbeddingsAsync(Guid experienceId) => + await ScalarAsync("SELECT count(*) FROM agent_experience.experience_embeddings WHERE experience_id = @id", experienceId); + + /// Moves a record to a new revision without going through a lifecycle commit, to stage an in-flight stale write. + public Task BumpRevisionAsync(Guid experienceId, long revision) => + ExecuteAsync("UPDATE agent_experience.experience_records SET revision = @revision WHERE experience_id = @id", experienceId, ("revision", revision)); + + /// Deletes a record, to stage a write that lands after the record is gone. + public Task DeleteRecordAsync(Guid experienceId) => + ExecuteAsync("DELETE FROM agent_experience.experience_records WHERE experience_id = @id", experienceId); + + /// Rewrites the reflection's lesson in place, which is a content change the re-index has to notice. + public Task RewriteLessonAsync(Guid experienceId, string lesson) => + ExecuteAsync( + "UPDATE agent_experience.experience_records " + + "SET payload = jsonb_set(payload, '{reflection,lesson}', to_jsonb(@lesson::text)) WHERE experience_id = @id", + experienceId, + ("lesson", lesson)); + + /// + /// Replaces a stored embedding's descriptor and its vector in place, to stage a model or + /// dimension mismatch. The vector is rewritten to the stated width because the schema's + /// CHECK (vector_dims(embedding) = dimension) will not let the two disagree -- which is the + /// point of that constraint, and why the search's cast can never meet a row it cannot cast. + /// + public Task RestampEmbeddingAsync(Guid experienceId, string modelId, int dimension) => + ExecuteAsync( + "UPDATE agent_experience.experience_embeddings " + + "SET model_id = @model_id, dimension = @dimension, embedding = CAST(@embedding AS vector) WHERE experience_id = @id", + experienceId, + ("model_id", modelId), + ("dimension", dimension), + ("embedding", "[" + string.Join(',', Enumerable.Repeat("0.1", dimension)) + "]")); + + /// + /// Forces the stored row's source_revision past the record's own, which is what a write that + /// landed from a newer revision leaves behind for a slower racer to collide with. + /// + public Task ForceStoredSourceRevisionAsync(Guid experienceId, long sourceRevision) => + ExecuteAsync( + "UPDATE agent_experience.experience_embeddings SET source_revision = @source_revision WHERE experience_id = @id", + experienceId, + ("source_revision", sourceRevision)); + + /// + /// The planner's chosen plan for exactly the statement the adapter issues, with sequential scans + /// disabled for the transaction so "it could have used the index" and "it did" are the same claim. + /// + public async Task ExplainSearchAsync(ReadOnlyMemory queryVector) + { + await using var connection = await DataSource.OpenConnectionAsync(); + await using var transaction = await connection.BeginTransactionAsync(); + + await using (var setting = new NpgsqlCommand("SET LOCAL enable_seqscan = off", connection, transaction)) + { + await setting.ExecuteNonQueryAsync(); + } + + var sql = PostgresExperienceEmbeddingIndex.SearchSqlForTesting(queryVector.Length); + await using var command = new NpgsqlCommand("EXPLAIN " + sql, connection, transaction); + var parameters = command.Parameters; + parameters.Add(new NpgsqlParameter("tenant_id", NpgsqlDbType.Text) { TypedValue = Scope.TenantId }); + parameters.Add(new NpgsqlParameter("application_id", NpgsqlDbType.Text) { TypedValue = Scope.ApplicationId }); + parameters.Add(new NpgsqlParameter("project_id", NpgsqlDbType.Text) { TypedValue = Scope.ProjectId }); + parameters.Add(new NpgsqlParameter("team_id", NpgsqlDbType.Text) { Value = DBNull.Value }); + parameters.Add(new NpgsqlParameter("agent_id", NpgsqlDbType.Text) { Value = DBNull.Value }); + parameters.Add(new NpgsqlParameter("user_id", NpgsqlDbType.Text) { Value = DBNull.Value }); + parameters.Add(new NpgsqlParameter("statuses", NpgsqlDbType.Array | NpgsqlDbType.Text) { TypedValue = ["Validated", "Reinforced"] }); + parameters.Add(new NpgsqlParameter("min_confidence", 0.5)); + parameters.Add(new NpgsqlParameter("model_id", NpgsqlDbType.Text) { TypedValue = Generator.ModelId }); + parameters.Add(new NpgsqlParameter("query_vector", NpgsqlDbType.Text) { TypedValue = "[" + string.Join(',', queryVector.ToArray().Select(v => v.ToString("R", System.Globalization.CultureInfo.InvariantCulture))) + "]" }); + parameters.Add(new NpgsqlParameter("limit", 50)); + + var lines = new List(); + await using (var reader = await command.ExecuteReaderAsync()) + { + while (await reader.ReadAsync()) + { + lines.Add(reader.GetString(0)); + } + } + + await transaction.RollbackAsync(); + return string.Join('\n', lines); + } + + private async Task ExecuteAsync(string sql, Guid experienceId, params (string Name, object Value)[] extra) + { + await using var command = DataSource.CreateCommand(sql); + command.Parameters.Add(new NpgsqlParameter("id", experienceId)); + foreach (var (name, value) in extra) + { + command.Parameters.Add(new NpgsqlParameter { ParameterName = name, Value = value }); + } + + await command.ExecuteNonQueryAsync(); + } + + private async Task ScalarAsync(string sql, Guid experienceId) + { + await using var command = DataSource.CreateCommand(sql); + command.Parameters.Add(new NpgsqlParameter("id", experienceId)); + return (T)(await command.ExecuteScalarAsync())!; + } + + internal sealed record StoredEmbedding( + string ModelId, + int Dimension, + string ContentHash, + long SourceRevision, + string TenantId, + string ProjectId, + DateTimeOffset UpdatedAt, + int VectorDimensions); +} diff --git a/tests/AgentExperience.Storage.Postgres.Vectors.Tests/VectorsFixture.cs b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/VectorsFixture.cs new file mode 100644 index 0000000..8508638 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/VectorsFixture.cs @@ -0,0 +1,154 @@ +using System.Security.Cryptography; +using System.Text; +using Npgsql; +using Testcontainers.PostgreSql; + +namespace AgentExperience.Storage.Postgres.Vectors.Tests; + +/// +/// Starts one ephemeral pgvector/pgvector:pg16 container for the whole collection, migrates its +/// default database with and then with this package's +/// own , and tears the container down afterwards. The two +/// calls are separate exactly as a host's are: the base schema needs no extension privilege, and only +/// this second call creates the vector extension. Set TESTCONTAINERS_RYUK_DISABLED=true +/// if Ryuk fails under a local Docker setup. +/// +public sealed class VectorsFixture : IAsyncLifetime +{ + private PostgreSqlContainer? _container; + private NpgsqlDataSource? _dataSource; + + public NpgsqlDataSource DataSource => _dataSource ?? throw new InvalidOperationException("Fixture not initialized."); + + public async Task InitializeAsync() + { + _container = new PostgreSqlBuilder("pgvector/pgvector:pg16").Build(); + await _container.StartAsync(); + + // Deliberately a plain data source: no UseVector() call. The adapter must work on whatever + // data source the host built, and these tests would not notice if it had silently started + // depending on the Pgvector type mapping being registered. + _dataSource = NpgsqlDataSource.Create(_container.GetConnectionString()); + + await ExperienceSchemaMigrator.MigrateAsync(_dataSource, CancellationToken.None); + await ExperienceVectorSchemaMigrator.MigrateAsync(_dataSource, CancellationToken.None); + } + + public async Task DisposeAsync() + { + if (_dataSource is not null) + { + await _dataSource.DisposeAsync(); + } + + if (_container is not null) + { + await _container.DisposeAsync(); + } + } +} + +[CollectionDefinition(Name)] +public sealed class VectorsCollection : ICollectionFixture +{ + public const string Name = "PostgresVectors"; +} + +/// +/// A deterministic . Every integration test in this project +/// embeds through this rather than a model: there are no credentials, no network, and no variance, so +/// a semantic-retrieval assertion is a statement about the adapter rather than about a provider. +/// +/// +/// The vector is a bag-of-topics projection: each configured topic owns one axis, and a text scores on +/// an axis for every one of that topic's words it contains. Two texts that share no words but +/// belong to the same topic therefore point the same way, which is exactly the property a semantic +/// match has to have for the test to mean anything -- and a SHA-256 of the text is folded into the +/// remaining axis so unrelated texts do not accidentally coincide. +/// +internal sealed class TopicEmbeddingGenerator : IExperienceEmbeddingGenerator +{ + private static readonly string[][] Topics = + [ + ["refund", "chargeback", "reimburse", "money", "payment", "invoice", "billing"], + ["deadlock", "lock", "contention", "blocked", "stuck", "concurrency", "timeout"], + ["deploy", "release", "rollback", "pipeline", "build", "ship"], + ]; + + public string ModelId { get; init; } = "topic-embed-v1"; + + public int Dimension => Topics.Length + 1; + + /// When set, every call throws this instead of embedding. + public Exception? Throws { get; init; } + + /// Every text this generator was asked to embed, in order. + public List Requests { get; } = []; + + public static ReadOnlyMemory VectorFor(string text) + { + var words = text.ToLowerInvariant().Split( + [' ', '\t', '\n', '\r', '.', ',', ';', ':', '!', '?', '-', '(', ')', '/'], + StringSplitOptions.RemoveEmptyEntries); + + var vector = new float[Topics.Length + 1]; + for (var topic = 0; topic < Topics.Length; topic++) + { + foreach (var word in words) + { + if (Topics[topic].Contains(word, StringComparer.Ordinal)) + { + vector[topic] += 1f; + } + } + } + + // A small, deterministic idiosyncrasy per text, so two unrelated texts never come out exactly + // parallel just because neither matched a topic. + vector[^1] = SHA256.HashData(Encoding.UTF8.GetBytes(text))[0] / 255f * 0.25f; + + // Normalized, so cosine distance is a pure direction comparison and the relevance a test reads + // back does not depend on how many words a summary happened to contain. + var magnitude = MathF.Sqrt(vector.Sum(component => component * component)); + if (magnitude > 0f) + { + for (var i = 0; i < vector.Length; i++) + { + vector[i] /= magnitude; + } + } + + return vector; + } + + public Task> GenerateAsync(string text, CancellationToken cancellationToken) + { + lock (Requests) + { + Requests.Add(text); + } + + cancellationToken.ThrowIfCancellationRequested(); + return Throws is not null ? throw Throws : Task.FromResult(VectorFor(text)); + } +} + +/// A generator that answers with a fixed-width vector of a chosen model, for the mismatch tests. +internal sealed class FixedEmbeddingGenerator : IExperienceEmbeddingGenerator +{ + public required string ModelId { get; init; } + + public required int Dimension { get; init; } + + public Task> GenerateAsync(string text, CancellationToken cancellationToken) + { + var vector = new float[Dimension]; + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(text)); + for (var i = 0; i < Dimension; i++) + { + vector[i] = ((hash[i % hash.Length] / 255f) * 2f) - 1f; + } + + return Task.FromResult>(vector); + } +} diff --git a/tests/AgentExperience.Storage.Postgres.Vectors.Tests/packages.lock.json b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/packages.lock.json new file mode 100644 index 0000000..4d98190 --- /dev/null +++ b/tests/AgentExperience.Storage.Postgres.Vectors.Tests/packages.lock.json @@ -0,0 +1,365 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Microsoft.Extensions.DependencyInjection": { + "type": "Direct", + "requested": "[10.0.11, )", + "resolved": "10.0.11", + "contentHash": "PSmotV19c7E3lKed++uYo1kSiXFI+uTl37CBSrhq+CfLC3FCHjG7R91+xPnNehQfHS1b0Tzo/CCLPWH3qaEheg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11" + } + }, + "Microsoft.NET.Test.Sdk": { + "type": "Direct", + "requested": "[17.14.1, )", + "resolved": "17.14.1", + "contentHash": "HJKqKOE+vshXra2aEHpi2TlxYX7Z9VFYkr+E5rwEvHC8eIXiyO+K9kNm8vmNom3e2rA56WqxU+/N9NJlLGXsJQ==", + "dependencies": { + "Microsoft.CodeCoverage": "17.14.1", + "Microsoft.TestPlatform.TestHost": "17.14.1" + } + }, + "Testcontainers.PostgreSql": { + "type": "Direct", + "requested": "[4.15.0, )", + "resolved": "4.15.0", + "contentHash": "45ZvqAzrh9BoI15Z4id7WEfhk1pVKqVeAX76aef8fR5ScglD75fSQtYuXhKFx4kIcgRL5aLKt3pRiqCVIgsL9g==", + "dependencies": { + "Testcontainers": "4.15.0" + } + }, + "xunit": { + "type": "Direct", + "requested": "[2.9.3, )", + "resolved": "2.9.3", + "contentHash": "TlXQBinK35LpOPKHAqbLY4xlEen9TBafjs0V5KnA4wZsoQLQJiirCR4CbIXvOH8NzkW4YeJKP5P/Bnrodm0h9Q==", + "dependencies": { + "xunit.analyzers": "1.18.0", + "xunit.assert": "2.9.3", + "xunit.core": "[2.9.3]" + } + }, + "xunit.runner.visualstudio": { + "type": "Direct", + "requested": "[3.1.4, )", + "resolved": "3.1.4", + "contentHash": "5mj99LvCqrq3CNi06xYdyIAXOEh+5b33F2nErCzI5zWiDdLHXiPXEWFSUAF8zlIv0ZWqjZNCwHTQeAPYbF3pCg==" + }, + "BouncyCastle.Cryptography": { + "type": "Transitive", + "resolved": "2.7.0", + "contentHash": "U+12df8UEWHgBi04YVf/Lgi2dy3SItlIYvHjjEVa/BngCQIzDCDRBk50DDByCfDvSbe5pRNFr3b7UrVK2kMcLw==" + }, + "dbup-core": { + "type": "Transitive", + "resolved": "6.1.1", + "contentHash": "kgpuyJVEFJHoIj/slnc994Go88aoeZqNDfGHDBr4sh7CsEWwJhOTCt/FJqO4ziUImL5L0NEY0kxxOiNgPKI2Fw==", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "8.0.0" + } + }, + "dbup-postgresql": { + "type": "Transitive", + "resolved": "7.0.1", + "contentHash": "mRnmENWWPuuMZ538gOd1mZnzucx6FQk0anmw3EABjGfcbp24FDb9QdGepYrDiaM8K9s5/gd49+5cmBOlniH/lg==", + "dependencies": { + "Npgsql": "10.0.1", + "dbup-core": "6.1.1" + } + }, + "Docker.DotNet.Enhanced": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "nGicLwvd42FhRk+khY5uS6cx49ErNdwYKnYBg0F4m4BDKLp/R77AVmmN9xAiqI3W/wN5ZCHkdUhgxf5ORkZuFQ==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3", + "Docker.DotNet.Enhanced.LegacyHttp": "4.3.3", + "Docker.DotNet.Enhanced.NPipe": "4.3.3", + "Docker.DotNet.Enhanced.NativeHttp": "4.3.3", + "Docker.DotNet.Enhanced.Unix": "4.3.3", + "Microsoft.Extensions.Logging.Abstractions": "8.0.3" + } + }, + "Docker.DotNet.Enhanced.Handler.Abstractions": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "9Cp8hOgtynixcDoAs9lnEaQosluojSYmiW3fsLsLIVfZjlq/fznSIZNUhnmyT4Xo1Iyuok/y49WL/25O47u0Pw==", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "8.0.3" + } + }, + "Docker.DotNet.Enhanced.LegacyHttp": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "7j3M16emv9PAQN7VwFn23xLYNj8GJmwPOcogveHkaWnOCqiC+anRaNKQwqIBNApM1AuwZKivehTKTPmmrjUUnw==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.NativeHttp": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "iNzK+jRFeEMobSA7l/h4ARwCKOOefOWtVN5/RB0ft6/6H6IQXvVUuOgGyZAjYLBT7TsyClRYno2B904f3dtBuQ==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.NPipe": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "ZTLYufuEfY0e6qLOgeH9QgXx2KYuoABRVaY5A8rsggyLgYqbDj9rCRfVAhHPCUv83S7pVxDHy+Tvm/BnxjWVpg==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.Unix": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "ypo8qNbmvHw1t9VfpRTMogCw2vht6VjkXzlGYUUeP2H2bf83USURdla1maW1njn2oq2rfLUFOGMfmt+A37QU2w==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.X509": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "oBDibWezEv4hgj3RIQxI3DVcxkNV1MdrD0d/jhjUu+h3DL+qc0wlkQva15kkwMatXmC/hWp1VP0DMoFXe+BmEw==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Microsoft.CodeCoverage": { + "type": "Transitive", + "resolved": "17.14.1", + "contentHash": "pmTrhfFIoplzFVbhVwUquT+77CbGH+h4/3mBpdmIlYtBi9nAB+kKI6dN3A/nV4DFi3wLLx/BlHIPK+MkbQ6Tpg==" + }, + "Microsoft.Extensions.AI.Abstractions": { + "type": "Transitive", + "resolved": "10.9.0", + "contentHash": "//nASHMCJVxnYfE/WSzfaLOao6/q816kPpgB9rxU0gfSmAny1u3rfQT0D4xAmcIo4yQqJs7rAeBB+M/dIMdZYA==" + }, + "Microsoft.Extensions.Compliance.Abstractions": { + "type": "Transitive", + "resolved": "10.9.0", + "contentHash": "tuSqNuiJxlln43sZ8c1EDA4WXit1eX4foGadylXso3DnMVc+DtKfaNEwvHuiFXfPsEUZ6Z3GnF0Bfk9vvOsE4Q==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.ObjectPool": "10.0.11" + } + }, + "Microsoft.Extensions.Compliance.Redaction": { + "type": "Transitive", + "resolved": "10.9.0", + "contentHash": "2P0WFFq9WAyhOAZqb0FjTeKW86yL4M2vymSGyuBu5XEBWwDCiEI+BoR4TuAFtyFurDRZs5wG3JtysUy8Svlnmw==", + "dependencies": { + "Microsoft.Extensions.Compliance.Abstractions": "10.9.0", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "wlhRqZW8LcJPa+vk2oLAc/REXDItHtkFQdf/QcXYGZbZOO13izcsKY1pCvuFQYwUiZD+hwSZwsKASjqT+BNaVg==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "fVi053xdpda9Em7vSkmgVxO/PtgC2m78ekReKWsgcyskqY0U82Bz/MONwxpGzI0hElYKJfw+fupqMVeKW3fSaA==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration.Binder": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "rFn8RuszZn3qquPVkDytMUlPc2+rXl9MCoygwc1XmAgC5vg5/oXJ8hkOosOrLoBLsqdTy4lFwP6iQdPS9uSYOA==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "10.0.0", + "contentHash": "FU/IfjDfwaMuKr414SSQNTIti/69bHEMb+QKrskRb26oVqpx3lNFXMjs/RC9ZUuhBhcwDM2BwOgoMw+PZ+beqQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0" + } + }, + "Microsoft.Extensions.ObjectPool": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "p76ztQFROBOlHgdV1vXfmTjRyu073Av7ZlsiLR93ka6+nzkLCeV2ONXq0DO/BGf71REqGW29Uy/20fzQHAjB7Q==" + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "eY1GAKcTfD2maP27J84X9IovT3yjHJ2dVDzPmDg6/XqYvt3jMzJhtfQCLjG9pVsZGAd+8DQ2QrjaDcs2+VQLGw==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Options.ConfigurationExtensions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "syEhXQ/sEaSBFaqzlp9gDGHX/nk6gkQkh1sIUpBO1mlBj3Phu1rmb4ML1uCiyPW9N6Kxfxv3y5FGObC+bV01Qw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.Configuration.Binder": "10.0.11", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Options": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "SXcz+kF+4Oo9b1+55zntpJFYfwb1jw66ioxptyNOOTDc8g2FHnBFWjZpsWfCvZIhzr0x+4e2trVTs4OKwQfBtw==" + }, + "Microsoft.TestPlatform.ObjectModel": { + "type": "Transitive", + "resolved": "17.14.1", + "contentHash": "xTP1W6Mi6SWmuxd3a+jj9G9UoC850WGwZUps1Wah9r1ZxgXhdJfj1QqDLJkFjHDCvN42qDL2Ps5KjQYWUU0zcQ==" + }, + "Microsoft.TestPlatform.TestHost": { + "type": "Transitive", + "resolved": "17.14.1", + "contentHash": "d78LPzGKkJwsJXAQwsbJJ7LE7D1wB+rAyhHHAaODF+RDSQ0NgMjDFkSA1Djw18VrxO76GlKAjRUhl+H8NL8Z+Q==", + "dependencies": { + "Microsoft.TestPlatform.ObjectModel": "17.14.1", + "Newtonsoft.Json": "13.0.3" + } + }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.3", + "contentHash": "HrC5BXdl00IP9zeV+0Z848QWPAoCr9P3bDEZguI+gkLcBKAOxix/tLEAAHC+UvDNPv4a2d18lOReHMOagPa+zQ==" + }, + "Npgsql": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "7nb5YzXuvWWJxB0J8DiyL3we+X4FOctZrt0fIBnucOIaIevFEEwGQVZKtiu9olXdlNAK1eNgqSral6r/jlhI4w==", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "10.0.0" + } + }, + "Pgvector": { + "type": "Transitive", + "resolved": "0.3.2", + "contentHash": "n7M5LuNejHUmtWky3zCbNO+tP1Gnjiuv9Qtu4LyvB1602dD8RiBxxCQp9jEjM0ZFDxAZF1oOWkNIkXw46KT00Q==", + "dependencies": { + "Npgsql": "8.0.5" + } + }, + "SharpZipLib": { + "type": "Transitive", + "resolved": "1.4.2", + "contentHash": "yjj+3zgz8zgXpiiC3ZdF/iyTBbz2fFvMxZFEBPUcwZjIvXOf37Ylm+K58hqMfIBt5JgU/Z2uoUS67JmTLe973A==" + }, + "SSH.NET": { + "type": "Transitive", + "resolved": "2026.0.0", + "contentHash": "Yu9dirPq8l3oaat0+OQ7K0nUf5MmYltpia5UGqsApTG4zTPvBC1cxbNnC3NERij26dUST0A3Ef1QdHSn5ArbWQ==", + "dependencies": { + "BouncyCastle.Cryptography": "2.7.0", + "Microsoft.Extensions.Logging.Abstractions": "8.0.3" + } + }, + "Testcontainers": { + "type": "Transitive", + "resolved": "4.15.0", + "contentHash": "8tCZKMm++C/9dHIr8lsE1iDBIBmthbo2XlGFDr4gorT1vgZrJwU36fNkCGuX7h7V39rpQoTK/aG+FhJz3jhY3A==", + "dependencies": { + "Docker.DotNet.Enhanced": "4.3.3", + "Docker.DotNet.Enhanced.X509": "4.3.3", + "Microsoft.Extensions.Logging.Abstractions": "8.0.3", + "SSH.NET": "2026.0.0", + "SharpZipLib": "1.4.2" + } + }, + "xunit.abstractions": { + "type": "Transitive", + "resolved": "2.0.3", + "contentHash": "pot1I4YOxlWjIb5jmwvvQNbTrZ3lJQ+jUGkGjWE3hEFM0l5gOnBWS+H3qsex68s5cO52g+44vpGzhAt+42vwKg==" + }, + "xunit.analyzers": { + "type": "Transitive", + "resolved": "1.18.0", + "contentHash": "OtFMHN8yqIcYP9wcVIgJrq01AfTxijjAqVDy/WeQVSyrDC1RzBWeQPztL49DN2syXRah8TYnfvk035s7L95EZQ==" + }, + "xunit.assert": { + "type": "Transitive", + "resolved": "2.9.3", + "contentHash": "/Kq28fCE7MjOV42YLVRAJzRF0WmEqsmflm0cfpMjGtzQ2lR5mYVj1/i0Y8uDAOLczkL3/jArrwehfMD0YogMAA==" + }, + "xunit.core": { + "type": "Transitive", + "resolved": "2.9.3", + "contentHash": "BiAEvqGvyme19wE0wTKdADH+NloYqikiU0mcnmiNyXaF9HyHmE6sr/3DC5vnBkgsWaE6yPyWszKSPSApWdRVeQ==", + "dependencies": { + "xunit.extensibility.core": "[2.9.3]", + "xunit.extensibility.execution": "[2.9.3]" + } + }, + "xunit.extensibility.core": { + "type": "Transitive", + "resolved": "2.9.3", + "contentHash": "kf3si0YTn2a8J8eZNb+zFpwfoyvIrQ7ivNk5ZYA5yuYk1bEtMe4DxJ2CF/qsRgmEnDr7MnW1mxylBaHTZ4qErA==", + "dependencies": { + "xunit.abstractions": "2.0.3" + } + }, + "xunit.extensibility.execution": { + "type": "Transitive", + "resolved": "2.9.3", + "contentHash": "yMb6vMESlSrE3Wfj7V6cjQ3S4TXdXpRqYeNEI3zsX31uTsGMJjEw6oD5F5u1cHnMptjhEECnmZSsPxB6ChZHDQ==", + "dependencies": { + "xunit.extensibility.core": "[2.9.3]" + } + }, + "agentexperience.abstractions": { + "type": "Project" + }, + "agentexperience.core": { + "type": "Project", + "dependencies": { + "AgentExperience.Abstractions": "[1.0.0, )", + "Microsoft.Extensions.Compliance.Redaction": "[10.9.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]" + } + }, + "agentexperience.storage.postgres": { + "type": "Project", + "dependencies": { + "AgentExperience.Abstractions": "[1.0.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]", + "Npgsql": "[10.0.3, 10.0.3]", + "dbup-core": "[6.1.1, 6.1.1]", + "dbup-postgresql": "[7.0.1, 7.0.1]" + } + }, + "agentexperience.storage.postgres.vectors": { + "type": "Project", + "dependencies": { + "AgentExperience.Storage.Postgres": "[1.0.0, )", + "Microsoft.Extensions.AI.Abstractions": "[10.9.0, 10.9.0]", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.11, 10.0.11]", + "Npgsql": "[10.0.3, 10.0.3]", + "Pgvector": "[0.3.2, 0.3.2]" + } + } + } + } +} \ No newline at end of file