diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/ACTIVE_DOC_INVENTORY.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/ACTIVE_DOC_INVENTORY.md index b354694be..86f8c6bed 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/ACTIVE_DOC_INVENTORY.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/ACTIVE_DOC_INVENTORY.md @@ -27,6 +27,50 @@ the controlled `actor.service.provision` route. Historical review artifacts retain their exact earlier SHAs and observations. No AUTH runtime file is changed by this CON reconciliation. +Before CON-02A review, trusted main advanced through ART PR #141 and AUTH-09C +PR #146 to `0ffdabf`. The live catalogue is now +74-permission/65-action/12-active/53-planned because AUTH-09C activates only +`actor.profile.read` and `actor.identity_link.read`; it adds no CON/outbox +identifier or migration. Historical CON-01 evidence above remains exact. +Trusted main then advanced to `b2b9016` through REV-01 PR #145, which publishes +the canonical review specification without changing the backend migration head +or the CON-02A outbox boundary. +REV-02 PR #147 then advanced trusted main to `f18b620` with planning-only chunk +decomposition and no backend, migration, or 02A boundary change. +AUTH-09D-A PR #148 then advanced trusted main to `99ae4c96`, activated only +three actor-profile lifecycle actions, and added AUTH-owned +`0026_actor_profile_lifecycle`. CON-02A therefore rebases its linear migration +to `0027_shared_transactional_outbox`; the merge adds no CON/outbox action, +permission, evaluator, service identity, or runtime admission. +REV PLAN2 PR #150 then advanced trusted main to `983b9e53` with a +planning/specification-only runtime-readiness refresh. It preserves the +FinalAcceptance-sourced submitter contribution, reviewer contribution on all +three decisions, REV-owned single commit, and shared outbox staging. Its split +future REV child gates are reconciled in CON planning; it changes no backend, +migration, AUTH catalogue, or CON-02A implementation. +ART-02B1 PR #151 then advanced trusted main to `1b5422fc` with the +S3-compatible ArtifactStore adapter, real MinIO integration, inactive AWS +profile support, dependency pins, CI changes, and substantial tests. It adds no +migration or outbox seam and does not change CON-02A behavior, but it requires +fresh repository-wide evidence on the combined tree. +AUTH-09D-B PR #152 then advanced trusted main to `93dd3924`, activating only +identity-link revoke/reactivate and expanding AUTH lifecycle proof. It adds no +migration, CON/task-claim identifier, fixed-service admission, or outbox seam; +at that historical point the contributor foundation and AUTH-09E remained +later gates. +Contributor-foundation PR #153 then advanced trusted main to `8d5eb15b`. It +clean-cuts TaskAssignment and Submission attribution to canonical human +`contributor_id`, adds writer revalidation, and owns +`0027_contributor_foundation`. It adds no CON/outbox identifier, service +admission, dispatcher, review lifecycle, or authority change. CON-02A is now +the linear `0028_shared_transactional_outbox` child; AUTH-09E remains a later +gate. +ART-02C1 PR #154 then advanced trusted main to `44f2467c`. It owns +`0028_artifact_admission` and adds durable artifact-admission and prepared-put +state without changing the generic outbox boundary. CON-02A is therefore the +linear `0029_shared_transactional_outbox` child; ART remains absent from the +outbox append path. + ## Inspected and already aligned The following active documents already describe ContributionPolicy, diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/AUTHORIZATION_HANDOFF.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/AUTHORIZATION_HANDOFF.md index 0e225c9f3..dfd1cd631 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/AUTHORIZATION_HANDOFF.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/AUTHORIZATION_HANDOFF.md @@ -2,11 +2,19 @@ ## Current baseline -Trusted `main` is `053242b` from merged AUTH-09B PR #143, layered on merged REV -PR #128, AUTH-09A/AUTH PR #140, and the earlier WS-XINT PR #139 boundary. The -runtime catalogue contains 74 PermissionIds and 65 ActionIds: ten active and 55 -planned. AUTH-09B activates only `actor.service.provision`; no WS-CON-specific -or task-claim ActionId below is registered. PR #140 still defines the +Trusted `main` is `8d5eb15b` after contributor-foundation PR #153, AUTH-09D-B PR #152, ART-02B1 PR #151, +planning-only REV PLAN2 PR #150, AUTH-09D-A PR #148, REV-02 PR #147 and +REV-01 PR #145, layered on AUTH-09C PR #146, ART PR #141, AUTH-09B PR #143, +merged REV planning PR #128, AUTH-09A/AUTH PR #140, and the earlier WS-XINT PR +#139 boundary. The runtime catalogue contains 74 PermissionIds and 65 ActionIds: +17 active and 48 planned. AUTH-09B activates only `actor.service.provision`; +AUTH-09C activates only `actor.profile.read` and `actor.identity_link.read`; +AUTH-09D-A activates only the three actor-profile lifecycle actions; +AUTH-09D-B activates only identity-link revoke/reactivate. PR #153 changes +TaskAssignment/Submission attribution and canonical-human writer validation +only; it adds no action, permission, grant, evaluator, availability, +fixed-service admission, or review runtime. No +WS-CON-specific or task-claim ActionId below is registered. PR #140 still defines the prepared/custody plan; it does not implement AUTH-PREP, transfer ART/REV custody, register a CON action, or activate a CON feature action. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/CHUNK_MAP.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/CHUNK_MAP.md index db045ef5b..5d5068b6f 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/CHUNK_MAP.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/CHUNK_MAP.md @@ -14,13 +14,13 @@ availability writer. Optional evidence chunks are not part of the core order. | `WS-CON-001-PLAN` | Contribution And Compensation Planning | L0 | None | Complete; unpublished | | `WS-CON-001-PLAN2` | Final Acceptance Reconciliation | L0 | Human FinalAcceptance/no-adjudication direction | Complete; unpublished | | `WS-CON-001-PLAN3` | AUTH/REV Current-Main Reconciliation | L0/L1 | Merged AUTH PR #140 plus AUTH-09A and REV PR #128 at `0302bcf` | Complete; unpublished | -| `WS-CON-001-01` | Canonical Contract Adoption And Architecture Decision | L0/L1 | Reconciled plan and human decisions approved | Complete; awaiting human review | -| `WS-CON-001-02A` | Shared Transactional Outbox Persistence | L1 | 01; event ownership approved | Proposed | +| `WS-CON-001-01` | Canonical Contract Adoption And Architecture Decision | L0/L1 | Reconciled plan and human decisions approved | Complete; merged in PR #144 | +| `WS-CON-001-02A` | Shared Transactional Outbox Persistence | L1 | 01 merged at `e118e33`; trusted head refreshed through contributor-foundation PR #153 and ART-02C1 PR #154 at `44f2467c`; explicitly started by human | Reconciled as `0029` after ART `0028`; bounded proof, exact-SHA review, and GitHub full-suite pending | | `WS-CON-001-02B` | Shared Outbox Dispatcher And Recovery | L1 | 02A; AUTH registers `outbox.dispatch`, approved `workstream.outbox.dispatcher` ServiceIdentity/static row, AUTH-09E admission, prepared protocol; dispatcher remains disabled until AUTH activation | Proposed | | `WS-CON-001-02C` | Shared Lifecycle Audit Participant | L1 | 02B; current AuditEvent contract refreshed | Proposed | | `WS-CON-001-03A` | Project Compensation Adapter-Binding Persistence | L1 | 02C; migration head refreshed | Proposed | | `WS-CON-001-03B` | Contribution Policy Persistence | L1 | 03A; legacy-data rule; must precede REV-03 ReviewLease FK | Proposed | -| `WS-CON-001-03C` | Contribution And Award Persistence | L1 | 03B; merged REV-04 runtime FinalAcceptance/Review/ReviewLease FK targets | Proposed | +| `WS-CON-001-03C` | Contribution And Award Persistence | L1 | 03B; merged REV-04B runtime FinalAcceptance/Review/ReviewLease FK targets | Proposed | | `WS-CON-001-03D` | Delivery, Receipt, And Status Persistence | L1 | 03C; immutable fulfillment root ordinal/generation contract | Proposed | | `WS-CON-001-04A` | Hidden Adapter-Binding Service | L1 | 03A; planned AUTH binding actions/contexts/prepared protocol; callback ServiceIdentity/action/static row approved but inactive | Proposed | | `WS-CON-001-04B` | Hidden Contribution-Policy Service | L1 | 03B, 04A; binding activation merged; planned `contribution.policy.*` actions/contexts/prepared protocol | Proposed | @@ -58,9 +58,12 @@ separate human approval -> refreshed ART/AUTH handoff -> 09A -> 09B AUTH registration -> CON hidden behavior -> AUTH activation -> later consumer/release ``` -- AUTH-09A/09B are merged; AUTH-09B activates only the human administrative - provisioning route and grants no service execution. AUTH-09C through 09E must - still precede protected fixed-service execution. New CON +- AUTH-09A through 09D-B are merged; AUTH-09B activates only the human + administrative provisioning route, AUTH-09C activates only actor/profile + administrative reads, AUTH-09D-A activates only actor-profile lifecycle, and + AUTH-09D-B activates only identity-link revoke/reactivate. None grants + service execution. The contributor foundation is merged; AUTH-09E must still precede protected + fixed-service execution. New CON ServiceIdentity/static-row additions require separate reviewed AUTH contracts before provisioning; no existing ART identity or provisioning result may be reused as CON authority. @@ -87,17 +90,19 @@ AUTH registration -> CON hidden behavior -> AUTH activation -> later consumer/re REV-owned FinalAcceptance. REV stages shared audit/outbox rows, owns the single commit, and supplies stabilized artifact-hash lineage; no ART call is made. -- Merged REV PR #128 is planning authority, not runtime readiness. CON-03B must - precede REV-03; CON-02A/02C precede REV-04; REV-04 precedes CON-03C; CON-06 - precedes REV-06; REV-09B plus CON-03C/07 precede REV-10; and CON-11's exact - obligation hooks/ordinal/drain manifest precedes REV-12A. +- Merged REV PR #128 plus PLAN2 PR #150 are planning authority, not runtime + readiness. CON-03B precedes REV-03A; CON-02A/02C precede REV-04B; REV-04B + precedes CON-03C; CON-06 precedes REV-06A; REV-09B plus CON-03C/07 precede + REV-10; the CON-02B dispatcher/handler registry precedes REV-12P1; CON's + 03D/08A/08B/10B/11 hooks precede REV-12A3; and CON-11 precedes REV-13C. - CON-08A/B and 10C cannot reuse outbox dispatcher authority for delivery, callback, reconciliation, or rebuild execution. - CON-10A owns core PostgreSQL contribution/award reads directly; it does not wait for optional evidence reads. - CON-11 has no ART or evidence-projection prerequisite. It hands mandatory obligation-writer, dispatch, callback, maximum-ordinal, and drain-observation - seams to REV-12A's single shared lifecycle controller and registers no route. + seams to REV-12A1/12A3's single shared lifecycle controller and registers no + route. - AUTH PR #140's complete ART and REV activation-custody transfer contracts are consumed by reference to AUTH/WS-XINT handoffs. The runtime transfers remain upstream gates; WS-CON does not define partial subsets. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/CONFORMANCE_MATRIX.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/CONFORMANCE_MATRIX.md index 6b3b9430b..254659c7c 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/CONFORMANCE_MATRIX.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/CONFORMANCE_MATRIX.md @@ -5,7 +5,7 @@ | Canonical policy model | 01,03B,04B | ContributionPolicy/version/rules/definitions; explicit unpaid; immutable publish; one active policy; NUMERIC(38,18) decimal-string bounds; ISO 4217 money units; project-scoped points units; stable binding references | CON-11 | | Adapter binding | 03A,04A,10B | one active binding per project/instrument; policy definitions and awards reference a binding with matching project/instrument identity while the binding stores no policy/award identifiers; non-secret route; suspend/resume and existing-award callback/replay behavior; retirement refuses active policy, unfinished frozen work, or unfulfilled award dependencies | CON-11 + joint live drill | | Legacy clean cut | 05A,05B | zero semantic consumers before schema removal; deterministic row treatment; no alias/fallback; migration upgrade/downgrade | CON-11 | -| Authorization | AUTH + each feature | current 74-65-10-55 baseline after AUTH-09B; only `actor.service.provision` was newly activated; provisioning grants no service execution or admission; all CON mappings/identities/static rows remain unregistered; full ART/REV custody referenced; one future AUTH custodian; planned denial; exact grant/static row; AUTH-09E; prepared handle bound to session/action/actor-ref/idempotency/request digest with substitution non-consumption; no local role logic | AUTH activation + CON-11 | +| Authorization | AUTH + each feature | current 74 PermissionId / 65 ActionId / 17 active / 48 planned baseline after AUTH-09D-B; contributor-foundation PR #153 changes human attribution and write revalidation only; provisioning grants no service execution or admission; all CON mappings/identities/static rows remain unregistered; full ART/REV custody referenced; one future AUTH custodian; planned denial; exact grant/static row; AUTH-09E; prepared handle bound to session/action/actor-ref/idempotency/request digest with substitution non-consumption; no local role logic | AUTH activation + CON-11 | | Final acceptance | REV + 03C,07 | accept creates one immutable FinalAcceptance per task/Review/Submission; needs_revision/reject create none; no create API/action, reopen, replacement, or adjudication path | joint live drill | | Contribution cardinality | 03C,07 + REV | one completed_review per valid Review with direct Review/lease lineage; one accepted_submission per FinalAcceptance with assignment lineage; mutually exclusive sources; revision Reviews distinct; automated outcomes create none | joint live drill | | Policy freeze | 05A,06 + task/REV | exact submitter/reviewer fields; published version; no drift; publish/suspend races both orders | joint live drill | diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/DECISIONS.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/DECISIONS.md index 6c4fea76a..e256c6124 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/DECISIONS.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/DECISIONS.md @@ -221,14 +221,24 @@ contribution type, branch, action, readiness check, or initiative dependency. ## D16 - AUTH Planning And Provisioning Do Not Activate CON -**Status:** accepted by merged AUTH PR #140 and AUTH-09B PR #143 on 2026-07-17. - -Trusted main `053242b` after AUTH-09B has 74 PermissionIds, 65 ActionIds, ten -active actions, and 55 planned actions, with no registered CON or task-claim +**Status:** accepted by merged AUTH PR #140, AUTH-09B PR #143, AUTH-09C PR +#146, AUTH-09D-A PR #148, AUTH-09D-B PR #152, and contributor-foundation PR +#153 through current main `8d5eb15b`; REV PLAN2 PR #150 and ART-02B1 PR #151 change no AUTH catalogue +fact. + +Trusted main `8d5eb15b` after contributor-foundation PR #153 has 74 +PermissionIds, 65 ActionIds, 17 +active actions, and 48 planned actions, with no registered CON or task-claim ActionId. AUTH-09B activates only `actor.service.provision`; its controlled human-administrator route can create the ActorProfile/ActorIdentityLink for an already-approved closed ServiceIdentity but grants no service execution, -runtime admission, role, grant, or database action assignment. PR #140 supplies +runtime admission, role, grant, or database action assignment. AUTH-09C +activates only administrative `actor.profile.read` and +`actor.identity_link.read`. AUTH-09D-A activates only the three actor-profile +lifecycle actions; AUTH-09D-B activates only identity-link revoke/reactivate. +The contributor foundation is merged and changes attribution/write identity +validation without changing authorization availability. Fixed-service +admission remains planned. PR #140 supplies the exact prepared protocol, complete ART/REV custody maps, and feature-manifest activation rule; those runtime implementations remain upstream work. @@ -266,8 +276,9 @@ the request route or service command commits once. **Status:** accepted from merged REV PR #128 on 2026-07-17. -REV-12A owns the sole PostgreSQL `JointLifecycleReleaseControl` and shared -`JointLifecycleMutationFence`. CON creates no parallel phase/controller. Every +REV-12A1 owns the sole PostgreSQL `JointLifecycleReleaseControl`, and REV-12A3 +composes CON against the shared `JointLifecycleMutationFence`. CON creates no +parallel phase/controller. Every fulfillment-obligation root creation, requeue, successor, and repair writer must acquire that fence before it allocates one immutable, monotonically increasing root ordinal or locks obligation rows. @@ -297,3 +308,19 @@ credentials, URLs, markup, or metadata are never persisted, logged, emitted, exported, or returned. The bounded receipt identifiers are not authentication tokens and may appear only in their canonical receipt/status fields. Unknown provider failures map to the closed generic failure code before persistence. + +## D20 - Repository-Wide Runtime Proof Runs In GitHub CI + +**Status:** accepted by human direction for CON-02A and later runtime chunks. + +Repository-wide tests and repository coverage run in the existing GitHub +Backend full-suite job after a full PR is pushed. Agents do not run the +multi-hour repository suite locally. Local proof remains bounded to the active +chunk's focused real-service tests, subsystem coverage floor, Ruff, migration +head, documentation links, stale-contract scans, and agent-loop gates. + +This changes execution location, not proof strength. GitHub must still run the +unchanged full test selection with isolated PostgreSQL, real MinIO where +required, and the repository 78 percent coverage floor. A focused local pass +cannot waive a missing or failing GitHub result, and the subsystem 90 percent +coverage floor remains required. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/DISCOVERY.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/DISCOVERY.md index a2d2c23f2..04a96f8c8 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/DISCOVERY.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/DISCOVERY.md @@ -2,9 +2,12 @@ ## Baseline inspected -- trusted `origin/main` refreshed to `053242b`, merged AUTH-09B PR #143 and REV - PR #128, including AUTH-09A, AUTH PR #140, and the earlier WS-XINT PR #139 - boundary; +- trusted `origin/main` refreshed to `8d5eb15b`, including contributor-foundation + PR #153, AUTH-09D-B PR #152, + ART-02B1 PR #151, planning-only REV PLAN2 PR #150, AUTH-09D-A PR #148, REV-02 PR #147, + REV-01 PR #145, AUTH-09C PR #146, ART PR #141, CON-01 PR #144, AUTH-09B PR + #143, REV planning PR #128, AUTH-09A, AUTH PR #140, and the earlier WS-XINT + PR #139 boundary; - complete WS-XINT intent, decisions, plan, REV/CON, AUTH/role-service, AUTH/REV, AUTH/ART, and ART/REV handoffs; - current WS-CON initiative package and archival reference inputs; @@ -27,12 +30,19 @@ SubmissionVersion model would duplicate current identity. The handoff field named `submission_version_id` therefore maps to canonical `Submission.id` and is stored as `submission_id`. -- No FinalAcceptance runtime exists yet. Merged REV PR #128 is reviewed planning - authority and defines the exact schema/transaction, but CON-03C still waits - for the REV-04 runtime target. -- The merged AUTH catalogue has 74 PermissionIds and 65 ActionIds. Ten actions - are active and 55 are planned. AUTH-09B activates only - `actor.service.provision`; no WS-CON or task-claim ActionId is registered. +- No FinalAcceptance runtime exists yet. Merged REV PR #128 plus PLAN2 PR #150 + are reviewed planning authority and define the exact schema/transaction, but + CON-03C still waits for the REV-04B runtime target. +- The merged AUTH catalogue has 74 PermissionIds and 65 ActionIds. Seventeen + actions are active and 48 are planned. AUTH-09B activates only + `actor.service.provision`; AUTH-09C activates only `actor.profile.read` and + `actor.identity_link.read`; AUTH-09D-A activates only the three actor-profile + lifecycle actions; AUTH-09D-B activates only `actor.identity_link.revoke` + and `actor.identity_link.reactivate`. No WS-CON or task-claim ActionId is + registered. The contributor-field/canonical-human foundation is merged: + TaskAssignment and Submission now expose `contributor_id`, enforce canonical + human ActorProfile lineage, and revalidate active human writers. AUTH-09E + fixed-service admission remains proposed. - Current AUTH supports actor-self, AdminRoleGrant evaluation, and controlled human-administrator provisioning of an approved fixed service ActorProfile/ActorIdentityLink. Independent ProjectRoleGrant runtime, @@ -47,6 +57,78 @@ - No shared transactional outbox exists with the required generic dispatcher, claim fencing, replay, and typed handler outcome contract. +## CON-02A focused discovery + +- Trusted `main` at `8d5eb15b` ends its migration chain at AUTH-owned + `0027_contributor_foundation`; CON-02A owns linear revision + `0028_shared_transactional_outbox` and must import its model through + `backend/app/db/models.py` so metadata and migration truth agree. +- `app.core.hashing.canonical_json_hash` is the only repository canonical JSON + encoder. It sorts object keys, rejects non-finite numbers, uses compact UTF-8 + JSON, and returns a `sha256:` digest. The outbox must call it directly and + must not introduce another serializer or digest helper. +- `AuthorityIdempotencyRepository` establishes the local concurrency pattern: + insert with PostgreSQL `ON CONFLICT DO NOTHING`, lock the existing namespace + row, compare the canonical digest, and complete through the caller's + `AsyncSession` without committing. Outbox append can reuse this sequence + without importing AUTH or creating a second generic idempotency framework. +- The common event envelope requires stable event ID, type, version, producer, + project, correlation, causation, idempotency key, canonical object payload, + and database-authoritative occurrence time. Delivery attempt/state fields + are operational metadata and must be independently mutable without allowing + immutable envelope drift. +- CON-02B has no migration allowance. CON-02A must therefore land the complete + feature-neutral persistence shape needed later for pending/claimed/retryable/ + acknowledged/dead-letter dispatch, claim generation/lease, attempts, + eligibility, bounded failure evidence, and retention while exposing only + append/replay behavior in this chunk. +- Existing audit and AUTH repositories flush and refresh on the supplied + session but never commit or publish. PostgreSQL triggers are already used for + append-only/immutable custody and guarded downgrade checks; the shared outbox + should follow those repository conventions. +- ART PR #141 adds artifact adapters, startup wiring, and an artifact delivery + executor but no shared outbox module, outbox route, dispatcher registry, + broker publication seam, or outbox permission. CON-02A therefore remains + feature-neutral and authorization-neutral and does not call ART. +- AUTH-09C PR #146 adds serialized administrative actor/profile reads and + activates their already-canonical action/permission pairs. It adds no + migration, CON/outbox identifier, service admission, or dispatcher seam. +- REV-01 PR #145 canonically publishes the Review/revision lifecycle and keeps + REV transaction ownership, ordered CON flush-only participation, + FinalAcceptance source integrity, and shared audit/outbox staging intact. It + changes documentation/gates only and adds no runtime outbox consumer. +- REV-02 PR #147 is planning-only chunk decomposition for future guide, + ReviewPolicy/task, and submission-attribution work. It adds no backend, + migration, test-runner, CON, or outbox behavior. +- REV PLAN2 PR #150 is a planning/specification-only runtime-readiness refresh. + It preserves the two ordered CON operations and REV-owned atomic decision + transaction while splitting future runtime gates: CON-03B precedes REV-03A, + CON-02A/02C precede REV-04B, CON-03C/07 precede REV-10, the shared outbox + dispatcher/handler registry precedes REV-12P1, CON lifecycle hooks precede + REV-12A3, and CON-11 precedes the sole product release in REV-13C. It changes + no 02A runtime or migration. +- ART-02B1 PR #151 adds the S3-compatible adapter, MinIO/AWS configuration, + exact SDK pins, CI MinIO service, and substantial artifact/configuration test + coverage. It adds no migration, outbox import, shared dispatcher seam, or + core CON dependency. It therefore leaves 02A's implementation boundary + unchanged while requiring fresh repository-wide evidence on the larger + dependency and test tree. +- AUTH-09D-B PR #152 activates exactly the two identity-link lifecycle + mutations and expands AUTH routes/tests without adding a migration, CON + action, task-claim action, fixed-service admission, or outbox seam. The + at that point the reviewed contributor foundation followed it; AUTH-09E + remains a later gate. +- Contributor-foundation PR #153 clean-cuts TaskAssignment and Submission + attribution to `contributor_id`, installs database-enforced + canonical-human lineage, and adds transaction-local active-human write + revalidation. It owns migration `0027_contributor_foundation` and therefore + moves CON-02A to linear child `0028_shared_transactional_outbox`. It changes + no ActionId, PermissionId, availability, grant, evaluator, service admission, + review lifecycle, dispatcher seam, or outbox behavior. +- ART-02C1 PR #154 owns `0028_artifact_admission` on trusted main `44f2467c`. + It adds no outbox seam or CON authority, so CON-02A moves unchanged to the + linear child `0029_shared_transactional_outbox`. + ## Canonical merged changes affecting CON 1. `ContributionPolicy`, `ContributionPolicyVersion`, `ContributionRule`, and @@ -88,7 +170,7 @@ participant exposes a reviewer operation before the decision branch and an accept-only submitter operation after FinalAcceptance and accepted task effects. Neither input carries nullable cross-actor source/policy facts. -14. REV-12A requires one shared `JointLifecycleMutationFence`. Every CON +14. REV-12A1/12A3 require one shared `JointLifecycleMutationFence`. Every CON fulfillment-obligation creation/requeue/successor/repair writer must fence before allocating an immutable monotonic root ordinal. CON must expose the current maximum ordinal with drain counts; delivery-draining dispatch and @@ -108,14 +190,14 @@ | `WS-XINT-001/REV_CON_HANDOFF.md` | Exact core participant sequence and optional-evidence boundary | | `WS-REV-001/CON_INTEGRATION_REVIEW.md` | Merged two-operation participant, exact lineage, interleaving, and release-control dependencies | | `WS-REV-001-08/10` chunks | Decision input freeze followed by first hidden canonical Review commit only after exact CON participant merge | -| `WS-REV-001-12A` chunk | Single shared lifecycle fence, obligation-writer ordinal order, cutoff capture, and drain-phase behavior required from CON | +| `WS-REV-001-12A1/12A3` planning children | Single shared lifecycle fence, obligation-writer ordinal order, cutoff capture, and drain-phase behavior required from CON | | `WS-XINT-001/AUTH_ROLE_SERVICE_HANDOFF.md` | Fixed service and project grant contract | | `WS-XINT-001/AUTH_REV_HANDOFF.md` | Full review activation-custody/hidden behavior sequence | | `WS-XINT-001/AUTH_ART_HANDOFF.md` | Full 25-action ART transfer; not a core CON gate | | `backend/app/modules/projects/{models,schemas,repository,service}.py` | Current guide-bound economic fields and consumers to cut over/remove | | `backend/app/modules/tasks/**` | TaskAssignment creation and future submitter policy freeze seam | | `backend/app/modules/tasks/models.py::Submission` | Existing immutable version identity: `id`, integer `version`, and `supersedes_submission_id`; no SubmissionVersion table | -| `backend/app/modules/authorization/{catalogue,policy,kernel,schemas}.py` | Current 74/65/10/55 runtime and stable PermissionIds; only `actor.service.provision` was newly activated; no CON/task-claim ActionId | +| `backend/app/modules/authorization/{catalogue,policy,kernel,schemas}.py` | Current 74/65/17/48 runtime and stable PermissionIds; only AUTH administrative actor/profile/service/link lifecycle actions are active; no CON/task-claim ActionId | | `backend/app/modules/audit/**` | Shared append-only audit extension point | | `backend/app/modules/artifacts/{preparation,sources}.py` | Inactive ART-only preparation; no core CON import | @@ -152,7 +234,7 @@ audit/outbox; single route commit; and no no-op fallback. - REV release control: CON writer/dispatch/callback hooks, immutable root ordinal allocation under the shared fence, and same-session maximum-ordinal/ - drain observation must merge before REV-12A. + drain observation must merge before REV-12A3. - Task/Submission: submitter policy freeze and stable assignment/version lineage. - Shared outbox/audit: caller-transaction append and feature-neutral dispatch. - ADR 0014 adapters: typed capability port, factory, and composition-root diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/INTENT.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/INTENT.md index d3285f5a0..f1a35aa62 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/INTENT.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/INTENT.md @@ -10,8 +10,13 @@ points ledger, or reputation scoring. The supplied WS-CON reference pair is input to reconcile, not authority to accept blindly. The active contract follows trusted repository decisions and -current main `053242b`, including merged AUTH-09B PR #143, REV PR #128, AUTH PR -#140, and the underlying WS-XINT-001 boundary from PR #139. +current main `8d5eb15b`, including the contributor foundation in AUTH PR #153, +AUTH-09D-B PR #152, ART-02B1 PR #151, REV PLAN2 PR #150, AUTH-09D-A PR #148, +REV-02 PR #147, REV-01 PR #145, AUTH-09C PR #146, ART PR #141, AUTH-09B PR +#143, REV planning PR #128, AUTH PR #140, and the underlying WS-XINT-001 +boundary from PR #139. PR #153 establishes canonical human `contributor_id` +lineage for TaskAssignment and Submission; it does not add CON authority, +service admission, review behavior, or outbox execution. ## Success state @@ -33,9 +38,10 @@ current main `053242b`, including merged AUTH-09B PR #143, REV PR #128, AUTH PR - Core contribution creation copies stabilized artifact-hash lineage supplied by REV and has no ART or provider dependency. - Downstream adapters fulfill awards but never determine eligibility. -- Every fulfillment-obligation writer uses REV-12A's one shared lifecycle fence - before monotonic root-ordinal allocation; drain dispatch/callback completes - only same-generation roots at or below the persisted cutoff. +- Every fulfillment-obligation writer uses the one shared lifecycle fence + composed with CON by REV-12A3 before monotonic root-ordinal allocation; drain + dispatch/callback completes only same-generation roots at or below the + persisted cutoff. - Every protected human/service surface uses AUTH's exact grant or ServiceIdentity/static-matrix path, prepared mutation protocol when needed, and AUTH-owned activation. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/JOINT_RELEASE_HANDOFF.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/JOINT_RELEASE_HANDOFF.md index 6e4a9c171..1fcb319e2 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/JOINT_RELEASE_HANDOFF.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/JOINT_RELEASE_HANDOFF.md @@ -9,9 +9,18 @@ CompensationAward, fulfillment behavior, and CON projections. AUTH owns all authorization and activation. The canonical cross-boundary source is merged -`WS-XINT-001/REV_CON_HANDOFF.md`. Merged REV PR #128, originally landed at -`0302bcf` and retained in current main `053242b`, is the reviewed owner contract; -runtime REV behavior remains unimplemented. +`WS-XINT-001/REV_CON_HANDOFF.md`. Merged REV PR #128 originally landed at +`0302bcf`; REV-01 PR #145 canonically published it, REV-02 PR #147 decomposed +the first runtime parent, and planning-only REV PLAN2 PR #150 refreshed the +remaining runtime child gates. They remain the reviewed owner contract in +current main `8d5eb15b`; ART-02B1 PR #151 changes ArtifactStore provider +implementation and proof only, while AUTH-09D-B PR #152 changes administrative +identity-link lifecycle only. Contributor-foundation PR #153 changes canonical +TaskAssignment/Submission attribution to `contributor_id` and enforces human +lineage/write identity but adds no review or contribution transaction behavior; +none enters this transaction; +runtime REV behavior remains +unimplemented. ## Required decision composition @@ -93,9 +102,9 @@ Review.decision. - ContributionPolicy publish/freeze and adapter-binding behavior are merged. - TaskAssignment and ReviewLease carry exact frozen policy-version IDs. -- REV FinalAcceptance persistence and its locked decision-lineage contract are - merged, including exact task/Review/Submission uniqueness and ReviewPolicy - lineage. +- REV-04B FinalAcceptance persistence and its locked decision-lineage contract + are merged, including exact task/Review/Submission uniqueness and + ReviewPolicy lineage. - Shared outbox/audit participants and CON-07 are mandatory and merged. - REV hidden claim/decision composition then consumes CON-06/07 and has no fallback. @@ -107,7 +116,7 @@ Review.decision. - Protected outbox handlers have their own exact service authority; dispatcher authority is not inherited. - Every CON fulfillment-obligation creation, requeue, successor, and repair - writer exposes a mandatory hook that acquires REV-12A's shared + writer exposes a mandatory hook that acquires REV-12A3's shared `JointLifecycleMutationFence` before allocating an immutable monotonically increasing root ordinal or locking obligation rows. - CON dispatch and callback hooks consume that shared fence. In @@ -148,6 +157,6 @@ no adjudication action/state/queue/readiness dependency. ## Ownership and stop -CON-11 publishes the hidden dependency manifest but registers no route. The -reviewed REV release chunk consumes that manifest and owns public activation and -the joint live drill. This handoff starts neither chunk automatically. +CON-11 publishes the hidden dependency manifest but registers no route. +REV-13A consumes it in preflight and REV-13C owns the sole public product +release and final HTTP proof. This handoff starts no chunk automatically. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/PLAN.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/PLAN.md index b2a72139c..744a54e09 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/PLAN.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/PLAN.md @@ -2,9 +2,12 @@ ## Proposed approach -Adopt merged REV PR #128 plus trusted main `053242b`, including AUTH-09A, -AUTH-09B PR #143, AUTH PR #140, and the underlying WS-XINT PR #139 boundary -before runtime work, then deliver WS-CON through hidden, reviewable chunks. The +Adopt contributor-foundation PR #153, AUTH-09D-B PR #152, ART-02B1 PR #151, planning-only REV PLAN2 PR #150, AUTH-09D-A PR #148, +merged REV-02 PR #147, REV-01 PR #145, and the underlying REV planning PR #128 +plus trusted main `8d5eb15b`, including +AUTH-09C PR #146, ART PR #141, AUTH-09A, AUTH-09B PR #143, AUTH PR #140, and +the underlying WS-XINT PR #139 boundary before runtime work, then deliver WS-CON through +hidden, reviewable chunks. The core path is PostgreSQL-local and has no ART dependency: ```text @@ -161,12 +164,22 @@ models, routes, lifecycle decisions, or commits. ## Authorization boundary -Trusted `main` is `053242b`, merging AUTH-09B PR #143 after REV PR #128, -AUTH-09A, AUTH PR #140, and WS-XINT PR #139. Runtime catalogue counts are 74 -PermissionIds, 65 ActionIds, ten active actions, and 55 planned actions. No -WS-CON or task-claim ActionId is registered. AUTH-09B activates only the -controlled human `actor.service.provision` operation and grants no service -execution or runtime admission. PR #140 adds reviewed AUTH +Trusted `main` is `8d5eb15b`, merging contributor-foundation PR #153 after +AUTH-09D-B PR #152 and ART-02B1 PR #151 and after planning-only REV +PLAN2 PR #150, AUTH-09D-A PR #148 and REV-02 PR #147, +REV-01 PR #145, AUTH-09C PR #146, ART PR #141, AUTH-09B PR #143, REV planning +PR #128, AUTH-09A, AUTH PR #140, and WS-XINT PR #139. +Runtime catalogue counts are 74 PermissionIds, 65 ActionIds, 17 active actions, +and 48 planned actions. No WS-CON or task-claim ActionId is registered. AUTH-09B +activates only the controlled human `actor.service.provision` operation; +AUTH-09C activates only administrative `actor.profile.read` and +`actor.identity_link.read`; AUTH-09D-A activates only the three actor-profile +lifecycle actions; AUTH-09D-B activates only identity-link revoke/reactivate. +None grants service execution or runtime admission. The contributor foundation +is merged: TaskAssignment and Submission now use canonical human +`contributor_id`, and contributor writes revalidate an active human profile and +identity link. It changes no ActionId, PermissionId, grant, evaluator, service +admission, or review lifecycle. AUTH-09E remains proposed. PR #140 adds reviewed AUTH custody/PREP/activation contracts only; the custody transfers and prepared protocol remain proposed runtime work. @@ -364,12 +377,17 @@ subsystem owns no contribution, award, adapter, review, or provider semantics. port returning outbox/fulfillment counts and the maximum root ordinal. REV- 12A injects the one shared `JointLifecycleMutationFence`, persists the generation cutoff, and owns release-control state; CON creates no second - controller. REV-13 owns final public release and the joint live drill. + controller. REV-13C owns final public release and the joint live drill. Every chunk refreshes trusted-main SHA, migration custody, exact port/action symbols, and merged dependency evidence. No cross-initiative successor starts automatically. +`REV-12A` and `REV-13` are canonical non-executable parent split records. Their +concrete runtime children control this plan: REV-12A1 persists the sole joint +controller, REV-12A3 composes the CON writer/dispatcher/callback/cutoff/drain +fences, and REV-13C alone releases the public product surface. + ### Merged REV interleaving Merged REV PR #128 fixes cross-initiative gates without starting either @@ -380,22 +398,24 @@ REV-02 immutable Submission/TaskAssignment attribution -> CON-05A/B task freeze and retired-field cutover CON-03B ContributionPolicyVersion persistence - -> REV-03 ReviewLease foreign key + -> REV-03A ReviewLease foreign key CON-02A shared outbox + CON-02C lifecycle audit participant - -> REV-04 Review/FinalAcceptance persistence + -> REV-04B Review/FinalAcceptance persistence -> CON-03C exact contribution source schema CON-06 reviewer policy freeze - -> REV-06 claim composition + -> REV-06A claim composition REV-09B stable lineage + CON-03C schema + CON-07 two-operation participant -> REV-10 first canonical Review-committing transaction -CON-11 writer/dispatch/callback/ordinal/drain manifest + REV-12 observations - -> REV-12A shared lifecycle controller/fence +CON-02B dispatcher/handler registry -> REV-12P1 projection handler + +CON-11 writer/dispatch/callback/ordinal/drain manifest + REV-12P3 observations + -> REV-12A1 controller persistence -> REV-12A3 CON fence composition -> AUTH action-specific activation - -> REV-13 joint release + -> REV-13C joint release ``` The merged REV plan proves ownership and ordering only. Each arrow still waits @@ -405,8 +425,9 @@ for the exact runtime predecessor on then-current trusted main. - Isolated PostgreSQL migration, constraint, rollback, idempotency, and both- order concurrency tests. -- Same-run repository coverage at or above 78 percent and each new/materially - changed subsystem at or above 90 percent. +- Bounded local focused coverage for each new/materially changed subsystem at + or above 90 percent; after PR push, GitHub CI runs the repository-wide suite + and enforces repository coverage at or above 78 percent. - Exact contribution cardinality for all three decisions and repeated/revision Reviews; accept-only FinalAcceptance one-to-one constraints; mutually exclusive reviewer/submitter source shapes; automated checks create none. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/RUNTIME_VERIFICATION.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/RUNTIME_VERIFICATION.md index f017b5c28..6b702195b 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/RUNTIME_VERIFICATION.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/RUNTIME_VERIFICATION.md @@ -1,18 +1,20 @@ # Runtime Verification Contract: WS-CON-001 -Every runtime chunk executes this template exactly after replacing -`` and `` with its row. It then runs one separate focused -report command for every concrete subsystem pattern in that row. The initial -erase, isolated PostgreSQL run, repository threshold, and all focused reports -are one evidence run; a pre-existing `.coverage` file is never accepted. +Repository-wide tests and repository coverage run in the existing GitHub +`Backend full-suite coverage` job after the full PR is pushed. Do not run that +multi-hour suite locally. Local verification is bounded to the chunk's focused +PostgreSQL selectors, focused coverage, Ruff, and repository static gates. + +Every runtime chunk executes the local template below after replacing +``, ``, and `` with its reviewed +contract. The focused test command must start from an erased coverage file, +must use the isolated database runner and real required local services, must +select a non-empty test set, and must generate fresh coverage with +`--cov=app --cov-report=`. A pre-existing `.coverage` file is never accepted. ```bash -ATTEMPT_ID="${ATTEMPT_ID:-$(date -u +%Y%m%dT%H%M%SZ)-$$}" -EVIDENCE_JSON="$(pwd)/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/evidence/-${ATTEMPT_ID}-isolated-tests.json" -mkdir -p "$(dirname "$EVIDENCE_JSON")" -test ! -e "$EVIDENCE_JSON" (cd backend && .venv/bin/python -m coverage erase) -(cd backend && WORKSTREAM_TEST_ADMIN_DATABASE_URL=postgresql+asyncpg://workstream:workstream@localhost:5433/postgres .venv/bin/python scripts/run_isolated_tests.py --metadata-json "$EVIDENCE_JSON" --timeout-seconds 12600 -- .venv/bin/python -m pytest -q --ignore=tests/test_isolated_database_runner.py --cov=app --cov-report=term-missing --cov-fail-under=78) +(cd backend && ) (cd backend && .venv/bin/python -m coverage report --include='' --fail-under=90) (cd backend && .venv/bin/ruff check ) python3 scripts/check_markdown_links.py @@ -22,7 +24,7 @@ git diff --check | Chunk | Separate focused subsystem reports (one `coverage report` per entry) | `` | |---|---|---| -| CON-02A | `app/modules/outbox/*` | `app/modules/outbox app/db/models.py tests/test_outbox.py alembic/versions/.py` | +| CON-02A | `app/modules/outbox/*` | `app/modules/outbox app/db/models.py tests/assertion_helpers.py tests/test_assertion_helpers.py tests/test_outbox.py tests/test_alembic.py alembic/versions/0029_shared_transactional_outbox.py` | | CON-02B | `app/modules/outbox/*`; `app/workers/outbox.py` | `app/modules/outbox app/workers/outbox.py app/workers/celery_app.py app/core/config.py tests/test_outbox.py tests/test_config.py` | | CON-02C | `app/modules/audit/*` | `app/modules/audit tests/test_audit.py` | | CON-03A | `app/modules/compensation/*` | `app/modules/compensation app/db/models.py tests/test_compensation.py alembic/versions/.py` | @@ -47,3 +49,9 @@ git diff --check If a named path differs after prerequisite merges, the chunk stops and its contract is re-reviewed; it may not silently broaden a glob or skip the target. + +The GitHub job remains responsible for the unchanged full test selection, +isolated PostgreSQL database, real MinIO service, and repository 78 percent +coverage floor. Focused local evidence does not replace that job and may not be +used to waive a failing or missing GitHub full-suite result. The subsystem 90 +percent coverage floor remains a local and CI review requirement. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/SOURCE_MANIFEST.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/SOURCE_MANIFEST.md index acbf34c22..709436ff2 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/SOURCE_MANIFEST.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/SOURCE_MANIFEST.md @@ -10,13 +10,28 @@ ## Trusted baseline -- `origin/main` at `053242b90d927ace3fab92eeca72da27a61cecec`, merging - AUTH-09B PR #143 after reviewed REV PR #128, AUTH-09A, AUTH PR #140, and - WS-XINT PR #139. +- `origin/main` at `8d5eb15b384fd75787ce98a099400a1d335d2560`, merging + contributor-foundation PR #153 after AUTH-09D-B PR #152 and ART-02B1 PR #151 + after planning-only REV PLAN2 PR #150, + AUTH-09D-A PR #148, REV-02 + PR #147, REV-01 PR #145, AUTH-09C PR #146, ART PR #141, AUTH-09B PR #143, + reviewed REV planning PR #128, AUTH-09A, AUTH PR #140, and WS-XINT PR #139. - PR #128 remains planning authority, not Review runtime implementation. -- Runtime AUTH is 74 PermissionIds, 65 ActionIds, ten active, 55 planned. - AUTH-09B activates only `actor.service.provision`; no CON or task-claim - ActionId exists, and provisioning grants no service runtime authority. +- REV PLAN2 PR #150 is the current runtime-readiness planning authority. It + splits future REV parent records into executable children but changes no + backend runtime, migration, AUTH catalogue, or 02A outbox contract. +- ART-02B1 PR #151 adds the S3-compatible ArtifactStore adapter and real MinIO + proof plus inactive AWS-profile support. It adds no migration or outbox seam, + and remains outside the core Review-to-CON transaction. +- Runtime AUTH is 74 PermissionIds, 65 ActionIds, 17 active, 48 planned. + AUTH-09B activates only `actor.service.provision`; AUTH-09C activates only + `actor.profile.read` and `actor.identity_link.read`; AUTH-09D-A activates + only the three actor-profile lifecycle actions; AUTH-09D-B activates only + identity-link revoke/reactivate. Contributor-foundation PR #153 clean-cuts + TaskAssignment and Submission attribution to canonical human + `contributor_id` and adds write revalidation without changing catalogue or + availability. Fixed-service admission remains planned. No CON or task-claim ActionId exists, + and these administrative operations grant no service runtime authority. - PR #140 remains the source for AUTH activation-custody, prepared-protocol, revised chunk, operations, and verification contracts. It changes no runtime CON behavior, @@ -30,14 +45,14 @@ - On 2026-07-17 the human fixed the v0.1 shipping path as `Review(accept) -> FinalAcceptance -> accepted_submission` and explicitly excluded adjudication lifecycle/actions/readiness. -- Merged REV PR #128 now plans FinalAcceptance, exact +- Merged REV PR #128 and its PLAN2 PR #150 refresh plan FinalAcceptance, exact `accepted`/`needs_revision`/`rejected` effects, two ordered CON participant operations, and REV-12A lifecycle-control hooks. WS-CON implementation still waits for each exact runtime chunk and consumes no sibling-worktree behavior. - The amendment's `submission_version_id` is normalized to canonical `Submission.id` / `submission_id`; current runtime already stores each immutable version as a Submission row. -- Merged REV-04 retains `policy_context_ref` as the foreign key to exact locked +- Planned REV-04B retains `policy_context_ref` as the foreign key to exact locked `ReviewPolicy.id` and `recorded_by` as the reviewer ActorProfile field; CON consumes those names and REV owns/proves the lineage. - `docs/review_closure.md`, `docs/review_final_adversarial_review.md`, and @@ -75,6 +90,7 @@ - `.agent-loop/initiatives/WS-REV-001-review-revision-lifecycle/chunks/WS-REV-001-08-immutable-decision-kernel.md` - `.agent-loop/initiatives/WS-REV-001-review-revision-lifecycle/chunks/WS-REV-001-10-contribution-integration-hidden-composition.md` - `.agent-loop/initiatives/WS-REV-001-review-revision-lifecycle/chunks/WS-REV-001-12A-joint-lifecycle-release-control.md` +- `.agent-loop/initiatives/WS-REV-001-review-revision-lifecycle/chunks/WS-REV-001-PLAN2-runtime-readiness-plan-refresh.md` - `.agent-loop/policies/*` ## Normative WS-XINT handoffs @@ -95,7 +111,7 @@ and 05B own semantic then physical removal after a human migration decision. - No ContributionPolicy, ContributionRecord, CompensationAward, fulfillment, or WS-CON action runtime exists yet. -- No FinalAcceptance runtime exists yet; merged planning assigns it to REV-04 +- No FinalAcceptance runtime exists yet; merged planning assigns it to REV-04B and makes that exact schema a prerequisite for CON-03C/07. - Existing `Submission` is the versioned identity; no new SubmissionVersion is required. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/STATUS.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/STATUS.md index 6e3cd90bc..1d314329c 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/STATUS.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/STATUS.md @@ -2,6 +2,73 @@ ## Current status +`WS-CON-001-01` merged through PR #144 at trusted-main SHA `e118e33`. The +generated post-merge state on `automation/loop-memory` was signature-verified +against that exact main SHA. The human explicitly started `WS-CON-001-02A` on +2026-07-18. CON-02A is now the only active chunk and is limited to generic +PostgreSQL outbox persistence plus append/replay in a caller-owned transaction. +It introduces no route, dispatcher, delivery executor, Celery registration, +protected handler, feature authority, contribution, compensation, review, or +artifact behavior. Trusted `main` then advanced through ART PR #141 at +`a10d901` and AUTH-09C PR #146 at `0ffdabf`. CON-02A initially followed +ART-owned `0025_artifact_store_v2`; ART's adapter, storage, startup, and +delivery-executor changes do not add an outbox seam or change this boundary. +AUTH-09C activates only the canonical administrative +`actor.profile.read`/`actor.identity_link.read` actions; it adds no CON or +outbox identifier and does not change 02A's authorization-neutral boundary. +Trusted `main` then advanced to `b2b9016` through REV-01 PR #145. Its canonical +review specification preserves the two ordered CON flush-only operations, +accept-only FinalAcceptance source, REV-owned single commit, and same-transaction +shared outbox staging. It adds no backend runtime or migration and therefore +does not change the 02A implementation boundary. +Trusted `main` then advanced to `f18b620` through REV-02 PR #147. That +planning-only merge splits future guide activation, ReviewPolicy/task +lifecycle, and submission attribution work into explicit REV chunks. It adds no +backend runtime, migration, or shared outbox behavior and leaves CON-02A +unchanged. +Trusted `main` then advanced to `99ae4c96` through AUTH-09D-A PR #148. That +merge activates only three actor-profile lifecycle actions and adds AUTH-owned +`0026_actor_profile_lifecycle`; it adds no CON/outbox identifier, evaluator, +service identity, static row, or fixed-service admission. CON-02A is therefore +rebased as linear `0027_shared_transactional_outbox` after AUTH's revision. +Trusted `main` then advanced to `983b9e53` through planning-only REV PLAN2 PR +#150. It splits future REV runtime parents into executable children and updates +their exact CON gates, while preserving the ordered reviewer/submitter +operations, accept-only FinalAcceptance, REV-owned audit/outbox staging, and +single commit. It adds no backend runtime, migration, AUTH catalogue entry, or +02A behavior. CON-02A therefore remains the same `0027` implementation. +Trusted `main` then advanced to `1b5422fc` through ART-02B1 PR #151. That merge +adds the S3-compatible ArtifactStore adapter, MinIO/AWS configuration, exact SDK +pins, CI MinIO service, and substantial backend tests. It adds no migration, +outbox seam, CON identifier, or core transaction dependency, so CON-02A remains +the same `0027` implementation. Because it materially changes dependencies, +CI, and repository tests, the pushed PR must receive fresh GitHub full-suite +evidence. +Trusted `main` then advanced to `93dd3924` through AUTH-09D-B PR #152. It +activates exactly `actor.identity_link.revoke` and +`actor.identity_link.reactivate`, expands AUTH routes/tests, and adds the +reviewed but inactive contributor-foundation contract. It adds no migration, +CON/task-claim action, fixed-service admission, or outbox seam. CON-02A remains +the same `0027` implementation, but repository-wide evidence must rerun in +GitHub CI after the full PR is pushed. +Trusted `main` then advanced to `8d5eb15b` through contributor-foundation PR +#153. That merge clean-cuts TaskAssignment and Submission human attribution to +`contributor_id`, adds canonical-human database lineage and active-human writer +revalidation, and owns `0027_contributor_foundation`. It changes no ActionId, +PermissionId, grant, evaluator, action availability, fixed-service admission, +review lifecycle, dispatcher seam, or outbox behavior. CON-02A therefore moves +to the linear child `0028_shared_transactional_outbox`; its generic, +authorization-neutral behavior is otherwise unchanged. Repository-wide proof +remains GitHub CI-owned. +Trusted `main` then advanced to `44f2467c` through ART-02C1 PR #154. ART owns +`0028_artifact_admission`; it adds no outbox seam or CON authority. CON-02A +therefore moves unchanged to the linear child +`0029_shared_transactional_outbox`. Fresh bounded and exact-SHA review evidence +must bind this reconciliation before PR #155 is republished. +Trusted `main` then advanced to `3b1d6379` through planning-only REV-02A PR +#156. It adds no migration, runtime outbox seam, or CON authority; CON remains +the linear `0029_shared_transactional_outbox` child of ART `0028`. + `WS-CON-001-PLAN3` completed its pre-external-review exact-SHA review at `e968430b0c3b5f1432899c9aa31ef209b774eae0` after current-main reconciliation with merged REV PR #128 at `0302bcf`, which also contains AUTH-09A after AUTH PR @@ -56,9 +123,12 @@ with no findings. Both prior CodeRabbit threads remain resolved and outdated. - CON-09A/09B are deferred optional successors and do not gate the core release. - AUTH PR #140 registers no CON ActionId and activates no feature action. Its exact custody and prepared-protocol contracts remain upstream gates. -- Current main has 74 PermissionIds and 65 ActionIds: ten active and 55 planned. - AUTH-09B activates only `actor.service.provision`; it can provision an - approved fixed identity but grants no runtime admission or feature authority. +- Current main has 74 PermissionIds and 65 ActionIds: 17 active and 48 planned. + AUTH-09B activates only `actor.service.provision`; AUTH-09C activates only + `actor.profile.read` and `actor.identity_link.read`; AUTH-09D-A activates only + the three actor-profile lifecycle actions; AUTH-09D-B activates only + identity-link revoke/reactivate. These administrative capabilities + grant no fixed-service runtime admission or feature authority. No CON or task-claim ActionId exists, and the current fixed identities are ART-only. - `task.claim` activation must follow, not precede, the CON-05A hidden @@ -66,47 +136,66 @@ with no findings. Both prior CodeRabbit threads remain resolved and outdated. - Merged REV planning requires the CON reviewer operation before the decision branch and the accept-only submitter operation afterward; it rejects one nullable omnibus participant input. -- REV-12A requires every CON fulfillment-obligation writer to fence before +- REV-12A3 requires every CON fulfillment-obligation writer to fence before monotonic ordinal allocation and requires same-session maximum-ordinal/drain - observation for the immutable delivery cutoff. + observation for the immutable delivery cutoff. REV-12A is only the canonical + non-executable parent split record; REV-12A1 persists the sole controller. ## Active chunk -`WS-CON-001-01` is complete through internal review after explicit human start. -It publishes the repository-owned active specification and ADR 0016; it changes -no runtime, migration, AUTH/ART/REV-owned contract, or archival reference input. -It awaits the specific PR human checkpoint. CON-02A does not start -automatically. +`WS-CON-001-02A` implementation is reconciled with trusted main `3b1d6379` +after the explicit human start. It adds one linear +`0029_shared_transactional_outbox` migration after ART-owned +`0028_artifact_admission`, the shared outbox model/schema/repository/service, +metadata registration, and PostgreSQL-focused migration/append tests. Fresh +bounded proof passes 73 selected tests with 32 deselected after the Proxy repair and +95.90 percent outbox coverage; the exact AUTH revision-specific lifecycle test +and assertion-helper regression suite also remain recorded independently. A +GitHub full-suite run reached 87.19 percent coverage and 1665 passing tests, +then exposed a mutable-dictionary race in that assertion helper; exact repair +candidate `9be9c88a` snapshots entries, distinguishes real dict storage from +framework Mapping proxies, adds deterministic regression coverage, and passes +all nine internal tracks. GitHub CI must rerun the full repository +suite and 78 percent coverage gate after publication. The first reconciled +full-suite attempt was stopped after two hours solely because PR #150 advanced +trusted main; a second attempt was stopped after 3 hours 7 minutes solely +because ART PR #151 advanced trusted main; a third was stopped after one hour +solely because AUTH PR #152 advanced trusted main. None is counted as evidence. +A fourth current-head local attempt was stopped after approximately 4 hours 15 +minutes by human direction that repository-wide suites run in GitHub CI; its +metadata was removed and it is not evidence. +It stops before dispatcher mechanics and CON-02B. | Chunk | Status | Notes | |---|---|---| | `WS-CON-001-PLAN` | Complete; superseded baseline | Based on PR #139 / `5d353b6`; reviewed content `c4242e0` | | `WS-CON-001-PLAN2` | Complete; unpublished | FinalAcceptance is REV-owned; CON trigger changes only; all required internal tracks pass | | `WS-CON-001-PLAN3` | Complete; externally repaired and internally reviewed | CodeRabbit gates/AUTH scope/09B/trust repairs pass at `a69fad3` | -| `WS-CON-001-01` | Complete; external findings repaired; awaiting human review | Specification and ADR only; stop at the PR checkpoint | -| `WS-CON-001-02A` through `08B`, `10A` through `11` | Proposed | Separate explicit start required after predecessor merge and upstream refresh | +| `WS-CON-001-01` | Complete; merged | PR #144 merged at `e118e33` | +| `WS-CON-001-02A` | PR #155 CI repair ready to publish | Generic persistence/append only; exact repair SHA passes all nine internal tracks; GitHub full-suite and CodeRabbit must rerun; retention remains deferred to 02B | +| `WS-CON-001-02B` through `08B`, `10A` through `11` | Proposed | Separate explicit start required after predecessor merge and upstream refresh | | `WS-CON-001-09A/09B` | Deferred optional | Separate approval and fresh ART/AUTH review required | ## Open gates | Gate | Owner | Required action | |---|---|---| -| FinalAcceptance and decision integration | REV + CON | REV-04 runtime persistence -> CON-03C; REV-09B lineage + CON-07 two-operation participant -> REV-10 hidden single-commit composition -> AUTH activation | -| Active specification/archive handling | Human | Review and approve the specific CON-01 PR; archival inputs remain untouched | +| FinalAcceptance and decision integration | REV + CON | REV-04B runtime persistence -> CON-03C; REV-09B lineage + CON-07 two-operation participant -> REV-10 hidden single-commit composition -> AUTH activation | +| Active specification/archive handling | Complete | CON-01 merged in PR #144; archival inputs remain untouched | | Pre-production legacy rows | Human | Choose deterministic rebuild or explicit classified migration before 05A/05B | | D11 AdminRole candidates | Human + AUTH | Fix award-detail, delivery-recovery, and audit candidates before registration | | Core WS-CON action registration/activation | AUTH | Add reviewed registration and later activation chunks; CON remains hidden | -| Fixed service runtime | AUTH | AUTH-09A/09B are merged; approve/register any new CON identity/static row, then complete AUTH-09C through 09E before protected service calls | +| Fixed service runtime | AUTH | AUTH-09A through 09D-B and the contributor foundation are merged; approve/register any new CON identity/static row, then complete AUTH-09E before protected service calls | | Feature handler authority | Human + AUTH + CON | Approve exact identities/actions/static rows; no dispatcher inheritance | | AUTH prepared protocol | AUTH | Merge AUTH-PREP after AUTH-09E; all CON-sensitive mutations consume its exact opaque handle contract | | task.claim | AUTH + task + CON | Only PermissionId exists; after AUTH-10/PREP and stable task seam, merge CON-05A freeze and task-owned composition; AUTH-13 enumerates/registers/evaluates/activates afterward | | review.claim/review.decision | AUTH + REV + CON | Complete REV custody transfer and AUTH-PREP; merge hidden CON participants and REV composition; AUTH-REV-06/08 activate afterward | | Shared outbox | CON-02A/B | Land generic persistence/dispatcher after approval | | Joint release | REV + CON + AUTH | Consume exact hidden manifest; optional evidence and ART are not prerequisites | -| Fulfillment cutoff/drain | CON + REV-12A | CON-03D ordinal; all writer/dispatch/callback hooks; CON-10B observation; CON-11 manifest -> REV-12A shared fence/controller | +| Fulfillment cutoff/drain | CON + REV-12A1/12A3 | CON-03D ordinal; all writer/dispatch/callback hooks; CON-10B observation; CON-11 manifest -> REV-12A1 controller persistence -> REV-12A3 CON fence composition | ## Stop condition -Do not edit runtime code or begin CON-02A from this chunk. Stop after CON-01 -evidence, internal review, and the human checkpoint; merge still requires -explicit approval for its specific PR. +Implement and review only CON-02A. Stop at its specific PR human checkpoint; +do not begin CON-02B, and do not merge without explicit approval for the +specific CON-02A PR. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/chunks/WS-CON-001-02A-shared-outbox-persistence.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/chunks/WS-CON-001-02A-shared-outbox-persistence.md index a7ed8ce36..623668545 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/chunks/WS-CON-001-02A-shared-outbox-persistence.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/chunks/WS-CON-001-02A-shared-outbox-persistence.md @@ -11,7 +11,7 @@ L1 infrastructure/audit/data risk. backend/app/modules/outbox/{__init__,models,schemas,repository,service}.py backend/app/db/models.py backend/alembic/versions/_shared_transactional_outbox.py -backend/tests/{test_outbox,test_alembic}.py +backend/tests/{test_outbox,test_alembic,assertion_helpers,test_assertion_helpers}.py .agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/** .agent-loop/merge-intents/WS-CON-001-02A.json ``` @@ -26,30 +26,38 @@ new JSON canonicalizer, idempotency framework, dependency or CI weakening ## Acceptance criteria -- [ ] Immutable event identity/type/version/project/correlation/causation, +- [x] Immutable event identity/type/version/project/correlation/causation, canonical payload/digest, idempotency key and occurrence time are separate from mutable delivery state. -- [ ] Reuse `app.core.hashing.canonical_json_hash` and the existing +- [x] Reuse `app.core.hashing.canonical_json_hash` and the existing reserve/lock/complete idempotency shape; no second canonicalizer/framework. -- [ ] Caller AsyncSession append flushes but never commits/publishes; changed +- [x] Caller AsyncSession append flushes but never commits/publishes; changed payload under one identity conflicts; PostgreSQL proves duplicate races. -- [ ] No Celery, handler, broker, review, or compensation behavior is added. +- [x] No Celery, handler, broker, review, or compensation behavior is added. ## Verification and reviewers -Execute the exact clean isolated CON-02A row in `../RUNTIME_VERIFICATION.md`, -replace its migration placeholder with the one new revision, then run: +Required reviewers are senior engineering, QA/test, security/auth, +product/ops, architecture, docs, reuse/dedup, test delta, and CI integrity. +Every track must review the exact implementation SHA and resolve or document +all valid findings before publication. + +Execute the bounded local CON-02A row in `../RUNTIME_VERIFICATION.md`, replace +its migration placeholder with the one new revision, then run: ```bash -(cd backend && .venv/bin/python -m pytest -q tests/test_outbox.py tests/test_alembic.py -k 'outbox and (migration or append or idempotency or duplicate or race or rollback)') +metadata_dir="$(mktemp -d)" +trap 'rm -rf "$metadata_dir"' EXIT +(cd backend && .venv/bin/python -m coverage erase) +(cd backend && WORKSTREAM_TEST_ADMIN_DATABASE_URL=postgresql+asyncpg://workstream:workstream@localhost:5433/postgres .venv/bin/python scripts/run_isolated_tests.py --metadata-json "$metadata_dir/result.json" --timeout-seconds 900 -- .venv/bin/python -m pytest -q tests/test_outbox.py tests/test_alembic.py -k outbox --cov=app --cov-report=) (cd backend && .venv/bin/python -m coverage report --include='app/modules/outbox/*' --fail-under=90) -(cd backend && .venv/bin/ruff check app/modules/outbox app/db/models.py tests/test_outbox.py tests/test_alembic.py) +(cd backend && .venv/bin/ruff check app/modules/outbox app/db/models.py tests/assertion_helpers.py tests/test_assertion_helpers.py tests/test_outbox.py tests/test_alembic.py alembic/versions/0029_shared_transactional_outbox.py) ``` -Pass requires a non-empty selected test set, PostgreSQL upgrade and guarded +Local pass requires a non-empty selected test set, PostgreSQL upgrade and guarded downgrade plus duplicate-race proof, stable exact replay, changed-payload -conflict, caller rollback with no commit/publish, repository coverage at least -78 percent in the same clean run, and focused outbox coverage at least 90 -percent. Baseline plus +conflict, caller rollback with no commit/publish, and focused outbox coverage at +least 90 percent. The pushed full PR must then pass the existing GitHub Backend +full-suite job, including repository coverage at least 78 percent. Baseline plus architecture, security/auth, product/ops, docs, reuse/dedup, test-delta, and CI integrity are required. Stop before dispatcher behavior. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/chunks/WS-CON-001-11-hidden-release-readiness.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/chunks/WS-CON-001-11-hidden-release-readiness.md index 576ac16cf..5896e4481 100644 --- a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/chunks/WS-CON-001-11-hidden-release-readiness.md +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/chunks/WS-CON-001-11-hidden-release-readiness.md @@ -55,7 +55,7 @@ archival input edits - [ ] Manifest names mandatory CON obligation-writer, dispatch, callback, and same-session `FulfillmentLifecycleDrainObservationPort` hooks plus `OutboxClaimValidationPort`, injection seams, phase mappings, denial states, - and fail-closed construction. REV-12A injects the one shared + and fail-closed construction. REV-12A3 injects the one shared `JointLifecycleMutationFence`; CON defines no second controller or optional/ no-op fence. - [ ] The manifest enumerates every fulfillment-obligation root creation, diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-external-review-response.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-external-review-response.md new file mode 100644 index 000000000..ca3353329 --- /dev/null +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-external-review-response.md @@ -0,0 +1,90 @@ +# External Review Response: WS-CON-001-02A + +## Review source + +CodeRabbit review `8be695bd-33ce-4847-8bf1-905a130804ec` on PR #155, +submitted 2026-07-19 against head +`21b3b167b930878abfaabc31f5b556e7a6b9036d`. + +## Comments addressed + +1. Added the nine exact required reviewer tracks to the 02A chunk contract. +2. Reconciled the AUTH runtime baseline to 74 PermissionIds, 65 ActionIds, + 17 active actions, and 48 planned actions. +3. Defined `REV-12A` as a non-executable parent and made REV-12A1 controller + persistence plus REV-12A3 CON fence composition explicit across intent, + plan, and status records. +4. Defined `REV-13` as a non-executable parent and made REV-13C the sole public + product release owner. +5. Replaced the stale FinalAcceptance runtime prerequisite `REV-04` with + `REV-04B` in the source manifest. + +## Comments deferred + +- The suggestion to implement a background physical purge is intentionally + deferred to separately started `WS-CON-001-02B`, whose approved scope owns + retention. CON-02A explicitly forbids dispatcher/retention behavior and its + reviewed custody contract prohibits physical delete/truncate; adding a + trigger-disable purge path here would violate both the chunk boundary and + immutable event-truth decision. CON-02B must reconcile archival-in-place, + sustained-volume operations, and the then-current custody contract before + implementing retention behavior. +- CodeRabbit's 42.50 percent docstring warning is not the repository gate. The + canonical `docstr-coverage --config .docstr.yaml` command passes at 90.4 + percent on this head; no threshold or exclusion is changed. + +## Human decisions needed + +None for these comments. The existing human-only PR merge decision remains. + +## Commands rerun + +- Markdown links: passed for 17 changed Markdown files. +- Stale Workstream, AUTH, ART, and REV contract scans: passed. +- Canonical repository docstring coverage: passed at 90.4 percent. +- Agent-loop gates: 88 passed. +- `git diff --check`: passed. +- Exact repair-SHA internal re-review and evidence rebinding after main + reconciliation: pending. + +After these repairs, contributor-foundation PR #153 advanced trusted main to +`8d5eb15b`. CON is being reconciled onto its `0027_contributor_foundation` +migration as `0028_shared_transactional_outbox`; the merge adds no CON +authorization or outbox behavior. The superseded GitHub Backend run is not +accepted as proof. Fresh bounded local evidence and exact-SHA internal review +must pass before the PR update, then GitHub reruns the full suite. + +Bounded reconciliation proof now passes: 43 selected outbox/migration tests +with 32 deselected, 95.73 percent outbox coverage, eight assertion-helper +tests, the exact AUTH `0026` lifecycle downgrade/reupgrade test, one Alembic +head at `0028_shared_transactional_outbox`, and Ruff. Markdown links, stale +Workstream/AUTH/ART/REV scans, canonical 90.4 percent docstring coverage, 88 +agent-loop gates, and diff hygiene also pass. Exact-SHA internal review remains +before push. + +ART-02C1 PR #154 subsequently merged at trusted main `44f2467c` and owns +`0028_artifact_admission`. CON-02A is now reconciled as +`0029_shared_transactional_outbox`. After REV-02A PR #156 advanced trusted main +to `3b1d6379`, the exact bounded `0029` row passes 73 selected tests with 32 +deselected and 95.90 percent focused outbox coverage; +all nine exact-SHA reviewers were required before publication. + +Exact repair candidate `9be9c88a19c4301f2b7ac606b6782604a48472bb` +passed all nine internal tracks against trusted main `3b1d6379` with no open +findings or reviewer sessions. The prior GitHub Backend run reached 87.19 +percent coverage and 1665 passing tests, but exposed a test-helper dictionary +mutation race in two outbox secrecy tests. The helper now snapshots entries and +a deterministic regression covers the mutation failure. A subsequent run +exposed a Celery Mapping Proxy that spoofs `dict` identity without dict +storage; concrete-type dispatch plus an exact secret-detection regression now +cover that path. GitHub Backend and fresh CodeRabbit state must rerun after +publication. + +## Remaining risks + +- GitHub must complete the full backend suite and repository-wide 78 percent + coverage gate. +- CodeRabbit must review the repair commit and close or supersede the five + actionable threads. +- Physical retention remains explicitly outside 02A and cannot be started as + 02B without separate human authorization and refreshed AUTH prerequisites. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-internal-review-evidence.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-internal-review-evidence.md new file mode 100644 index 000000000..b8d94b93e --- /dev/null +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-internal-review-evidence.md @@ -0,0 +1,273 @@ +# Internal Review Evidence: WS-CON-001-02A + +## Chunk + +`WS-CON-001-02A` - Shared Transactional Outbox Persistence + +Risk: L1 infrastructure, schema, concurrency, audit, and data-integrity risk. + +## Baseline And Scope + +Trusted main SHA: `3b1d63796c086f53fc2b0aeefe096387b82485ec` + +The implementation is limited to one linear PostgreSQL migration after +ART-owned revision `0028_artifact_admission`, the generic outbox persistence/append module, shared +metadata registration, focused tests, +initiative evidence, and exactly one merge intent. It adds no dispatcher, +delivery executor, Celery registration, broker, route, feature handler, AUTH +identifier, contribution, compensation, review, task, project, audit, or ART +behavior. + +The user-owned unstaged deletion of the older contribution reference PDF is +outside the chunk and is excluded from every commit and review. + +REV PLAN2 PR #150 is planning/specification-only. It preserves the exact 02A +runtime and migration while refreshing future CON/REV gate names. The first +post-AUTH full-suite attempt was stopped after two hours when that PR advanced +trusted main; its metadata was removed and it is not evidence. + +ART-02B1 PR #151 changes no 02A code or migration, but it adds locked S3 SDK +dependencies, a real MinIO CI service, and substantial backend tests. A second +full-suite attempt was stopped after 3 hours 7 minutes when that PR advanced +trusted main; its metadata was removed and it is not evidence. + +AUTH-09D-B PR #152 changes no 02A code or migration. It activates exactly two +identity-link lifecycle actions and adds substantial AUTH route/test coverage. +A third full-suite attempt was stopped after one hour when that PR advanced +trusted main; its metadata was removed and it is not evidence. + +A fourth local full-suite attempt on the frozen `93dd3924` baseline was stopped +after approximately 4 hours 15 minutes by human direction to keep +repository-wide suites in GitHub CI. Its metadata was removed and it is not +evidence. The active verification rule now uses bounded focused proof locally +and requires the existing GitHub Backend full-suite job on the pushed PR. + +Contributor-foundation PR #153 advances trusted main to `8d5eb15b`, owns +`0027_contributor_foundation`, clean-cuts TaskAssignment/Submission attribution +to canonical human `contributor_id`, and adds active-human write revalidation. +It changes no CON authority or outbox behavior. CON-02A is therefore reconciled +as linear child `0028_shared_transactional_outbox`; AUTH's revision-specific +`0026` lifecycle tests remain revision-specific, while CON owns separate +outbox-head proof. Fresh bounded verification and exact-SHA reviewer results +below supersede earlier publication evidence. + +ART-02C1 PR #154 advances trusted main to `44f2467c` and owns +`0028_artifact_admission`. It changes no outbox or CON authority behavior. +CON-02A is reconciled as its linear child +`0029_shared_transactional_outbox`; all earlier `0028` outbox results remain +historical until the bounded `0029` row and exact-SHA reviewers pass. + +REV-02A PR #156 then advances trusted main to `3b1d6379` with planning and +contract decomposition only. It adds no migration, runtime model, outbox seam, +or CON authority. The `0029` migration topology and generic outbox behavior +remain unchanged; fresh exact-SHA proof and review bind to this baseline. + +## Current PR #154 Reconciliation Verification Results + +```text +73 passed, 32 deselected (exact bounded isolated outbox/migration row on ART `0028` / CON `0029` after Proxy repair) +outbox coverage: 95.90% (244 statements, 10 missed; required: at least 90%) +10 passed in 0.09s (security-sensitive assertion helper suite after CI Proxy regression repair) +1 passed in 63.77s (AUTH revision-specific 0026 lifecycle downgrade/reupgrade) +Alembic heads: one head, `0029_shared_transactional_outbox` +Ruff on CON-02A, ART `0028`, CON `0029`, and reconciliation tests: passed +repository-wide isolated PostgreSQL plus real-MinIO suite: required in GitHub CI after push +``` + +The first bounded run exposed a test-helper compatibility failure while walking +an opaque Pydantic internal `Mapping`; the outbox exception and formatted +traceback contained no secret. The helper now falls back to recursively +inspecting concrete object state and fails closed when state is unavailable. +Its dedicated regression test proves a raising mapping cannot hide a retained +secret. The repaired exact bounded row passed as recorded above. + +The first GitHub Backend run on PR head `44fc78f3` completed the full suite at +87.19 percent repository coverage but failed two outbox secrecy tests because +recursive helper inspection traversed a live module-globals dictionary while a +nested import mutated it. The assertion now snapshots built-in dictionary +entries before recursion. A deterministic nested-mapping regression proves the +old live iteration failure and the repaired traversal; no product or CI +threshold behavior changed. + +The next GitHub Backend run on PR head `2fba5bab` reached the same 87.19 +percent repository coverage and 1665 passing tests, then exposed a Celery +`Proxy` that reports `isinstance(proxy, dict)` without owning built-in dict +storage. Real dict subclasses are now identified from their concrete type; +framework proxies use the Mapping protocol. The exact regression proves the +Proxy path still detects a retained secret rather than bypassing inspection. + +## Implemented Contract + +- One immutable event envelope contains caller-provided event, aggregate, + project, correlation, causation, idempotency, and payload facts; PostgreSQL + forces producer and occurrence time. +- Operational delivery state is a separate closed shape prepared for a later + migration-free 02B implementation but exposes no transition service here. +- Insert/update/delete/truncate custody prevents forged initial state, immutable + envelope mutation, illegal transitions, evidence regression, physical + deletion, and archival reopening. +- Terminal retention is archival-in-place. A guarded downgrade takes an + `ACCESS EXCLUSIVE` lock and refuses durable rows. +- Payload input is strict, bounded, lower-snake-case JSON with recursive secret + key rejection, ordinary validation mapped to a detached domain error, one + defensive deep snapshot, and stable input/persistence errors with no + payload-bearing cause or context. +- Append reuses `canonical_json_hash`, reserves with PostgreSQL conflict + handling, locks both identities in deterministic order, flushes only the + caller session, and never commits or publishes. +- Exact replay preserves database occurrence time; any immutable drift or split + event/idempotency identity raises `outbox_idempotency_conflict`. + +## Pre-PR #153 Reconciliation Verification Results + +```text +43 passed, 30 deselected in 60.22s (exact bounded isolated outbox row after final review repair) +outbox coverage after final review repair: 95.73% (required: at least 90%) +78 passed in 378.36s on current main's outbox/migration, real-MinIO ART, and AUTH-09D-B suite +pre-repair outbox coverage: 95.43% +8 passed, 56 deselected in 50.92s (exact contract selector) +2 passed in 88.51s (affected AUTH lifecycle downgrade tests) +16 passed in 102.10s (isolated database runner self-tests with admin URL) +real API contract end-to-end on 0027: passed +88 passed (agent-loop gates after AUTH-09D-B) +Ruff: passed +Docstring coverage: passed at 90.4% +Markdown links: passed for 15 changed Markdown files +Workstream/AUTH/ART/REV stale-contract scans: passed after PR #152 reconciliation +merge intent and git diff --check: passed +Alembic heads: one head, 0027_shared_transactional_outbox after PR #152 (historical) +repository-wide isolated PostgreSQL plus real-MinIO suite: required in GitHub CI after push +``` + +## Pre-Reconciliation Verification Results + +These results were produced on the former `f18b620` / outbox-0026 chain. They +prove the implementation before AUTH-09D-A but are not publication evidence for +the reconciled `93dd3924` / outbox-0027 chain. PR #153 supersedes that chain; +fresh bounded `8d5eb15b` / outbox-0028 evidence is recorded separately and +GitHub CI owns exact-head repository-wide proof after the full PR is pushed. + +```text +33 passed in 151.73s on the former ART 0025 -> CON 0026 chain +outbox coverage: 95.43% (required: at least 90%) +8 passed, 51 deselected in 125.42s (exact contract selector) +1 passed, 25 deselected in 96.33s (migration/downgrade guard) +16 passed in 180.79s (isolated database runner self-tests in a quiet window) +real API contract end-to-end: passed +1347 passed in 17741.96s / 4:55:41 (exact isolated PostgreSQL full suite) +repository coverage: 85.35% (required: at least 78%) +isolated evidence database: workstream_test_d513fb2f03b1 +isolated evidence tree: f72bb6e6dc71cb38dcb397b34caf9db6f916db4c +isolated evidence Alembic head: 0026_shared_transactional_outbox (superseded) +87 passed (agent-loop gates) +Ruff: passed +Docstring coverage: passed at 91.5% +Markdown links: passed for 15 changed Markdown files +Workstream stale wording: passed +AUTH stale documentation: passed +ART stale contract: passed at phase artifact_store_cutover +REV stale contract: passed +git diff --check: passed +local roadmap workbook: absent, so the one-sheet export check is not applicable +``` + +The historical full suite used the unchanged test command, isolation rule, +assertions, and coverage thresholds. Its measured 4:55:41 runtime explains why +repository-wide proof is now CI-owned. Exact-SHA reviewer results were +superseded when AUTH-09D-A changed the backend and migration head. + +## Test Delta + +Tests add strict schema/privacy bounds; one core-schema error boundary across +constructor, model methods, and `TypeAdapter` Python/JSON/string modes; valid +mode parity; hostile top-level and nested dict/list/string subclasses; +payload-free outbox traceback locals, including idempotency conflicts; +defensive nested-payload snapshotting across the first await; stable database +error redaction; caller rollback including injected post-reservation failure; +exact replay; immutable drift; split identity; concurrent commit/rollback +races; direct-SQL custody; legal and illegal delivery transitions; terminal +archival; delete/truncate denial; exact migration surface; and concurrent +downgrade writer behavior. Shared secret-retention helper tests prove dict, +list, and string subclasses plus slotted dataclass state cannot bypass deep +inspection through the tested override paths. +Existing assertions, skips, coverage settings, and test commands are unchanged. + +## Current Exact-SHA Internal Review + +Reviewed code SHA: `9be9c88a19c4301f2b7ac606b6782604a48472bb` + +Reviewed against trusted main: +`3b1d63796c086f53fc2b0aeefe096387b82485ec` + +Reviewer runs: `/root/proxy_repair_senior_arch_reuse`, +`/root/proxy_repair_qa_product_docs`, `/root/proxy_repair_security_ci` + +Open sub-agent sessions: none + +Valid findings addressed: yes. The repair loop corrected the ART-parent +downgrade target, closed separated and compact sensitive-key-name bypasses, +preserved benign key-like words, and refreshed the active status baseline and +bounded evidence. The GitHub full-suite repair snapshots mutable framework +state and adds deterministic regression coverage. The final confirmation +passed all nine tracks. + +| Reviewer | Result | Blocking findings | +|---|---|---| +| Senior engineering | PASS | none | +| QA/test | PASS | none | +| Security/auth | PASS | none | +| Product/ops | PASS | none | +| Architecture | PASS | none | +| Docs | PASS | none | +| Reuse/dedup | PASS | none | +| Test delta | PASS | none | +| CI integrity | PASS | none | + +## Superseded Pre-PR #153 Internal Review + +Historical reviewed code SHA: `460573287270965d730c83f5f1e52f3acf1c0671` + +Reviewed at: 2026-07-19T09:15:29Z + +Reviewer run IDs: senior-engineering/architecture/reuse-dedup=/root/con01_arch_senior_reuse; QA-test/product-ops/docs/test-delta=/root/con01_qa_product_docs; security-auth/CI-integrity=/root/con01_security_ci + +Valid findings addressed: yes + +Open sub-agent sessions: none + +This review remains useful implementation history but is not publication +evidence for the `8d5eb15b` / `0028` reconciliation. All nine tracks must bind +to the new exact candidate SHA before push. + +| Reviewer | Result | Blocking findings | Notes | +|---|---|---|---| +| Senior engineering | PASS WITH LOW RISKS | None | Fixed top-level hostile mapping traversal and confirmed valid constructor/model/TypeAdapter modes. Fixed model configuration is intentional; arbitrary runtime string-mode option overrides are not a supported 02A contract. | +| QA/test | PASS | None | Hostile string, slotted state, top-level payload, and conflict-retention regressions close every prior test gap. | +| Security/auth | PASS | None | Payload-bearing validation, persistence, and conflict state is absent from the tested exception and outbox traceback graph; no AUTH surface was added. | +| Product/ops | PASS | None | Generic flush-only persistence adds no product lifecycle, payment, dispatcher, or worker behavior. | +| Architecture | PASS | None | Caller-session ownership and feature-neutral 02A boundary remain intact. | +| Docs | PASS | None | Focused proof, GitHub-only full-suite gate, and complete Ruff scope agree across the contract and trust bundle. | +| Reuse/dedup | PASS | None | Canonical hashing and shared deep-retention assertion are reused; no duplicate framework was introduced. | +| Test delta | PASS | None | All changes are additive; no skip, xfail, threshold, selector, or assertion weakening. | +| CI integrity | PASS | None | Existing GitHub full-suite and 78 percent repository coverage gates remain unchanged and mandatory. | + +The final repair loop resolved every earlier reviewer finding: nested payload +snapshotting, stable payload-free persistence errors, rollback after a +post-reservation failure, detached Pydantic entry points, valid JSON/string +mode parity, `TypeAdapter` coverage, traceback-local scrubbing, hostile nested +and top-level built-in subclasses, shared helper traversal, idempotency- +conflict reservation retention, and complete repeatable Ruff scope. + +## Remaining Human Gates + +- Human approval is required for the specific PR before merge. +- 02B remains a separately started chunk and owns dispatcher/recovery behavior. +- `outbox.dispatch`, its fixed service identity/static row, AUTH-09E admission, + prepared authorization, and activation remain upstream gates for 02B. +- No passing check grants a feature handler authority through dispatcher + authority. + +## Stop Condition + +Stop at the CON-02A full PR human checkpoint. Do not merge without explicit +approval for that PR and do not begin `WS-CON-001-02B` automatically. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-pr-trust-bundle.md new file mode 100644 index 000000000..34d8c7bf6 --- /dev/null +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-pr-trust-bundle.md @@ -0,0 +1,239 @@ +# PR Trust Bundle: WS-CON-001-02A + +## Chunk + +`WS-CON-001-02A` - Shared Transactional Outbox Persistence. + +## Goal + +Land feature-neutral shared PostgreSQL outbox truth and a caller-transaction +append participant, with enough persistence shape for 02B to add dispatcher +mechanics without another migration. + +Risk: L1 infrastructure/data boundary; SLA P1. + +## Human-Approved Intent + +The human explicitly started 02A after CON-01 merged. This chunk must remain +generic and authorization-neutral: AUTH owns every action, permission, +evaluator, service admission, and activation decision, while 02B and later +feature chunks own execution behavior. + +## Why It Changed + +REV-owned lifecycle transactions need one generic durable event participant +that can flush into the caller's transaction without publishing, committing, +or inheriting feature authority. No such shared persistence boundary existed on +trusted main. + +## What Changed + +- Added one linear `0029_shared_transactional_outbox` migration after the + ART-owned `0028_artifact_admission` revision. +- Added the generic outbox model, strict append schemas, reservation repository, + and flush-only service. +- Registered the model in shared SQLAlchemy metadata. +- Added PostgreSQL migration, concurrency, replay, privacy, custody, rollback, + and state-shape tests. +- Updated the WS-CON chunk ledger and added exactly one schema-v2 merge intent + naming 02B with a separate explicit start. + +## Design Chosen + +- Immutable event truth and mutable operational delivery state are separate. +- PostgreSQL, not the caller, owns producer, occurrence time, initial state, + counters, and initial eligibility time. +- Event ID and global idempotency key are independent identities; exact replay + requires every immutable fact to match. +- The repository inserts with conflict suppression, locks matching identities + deterministically, and uses only the supplied `AsyncSession`. +- The service reuses the existing repository-wide `app.core.hashing` canonical + JSON helper, takes one defensive deep payload snapshot before its first await, + and emits only stable input, conflict, or persistence errors with detached + payload-bearing validation/database exception contexts. +- Retention is one-way terminal archival; event truth cannot be deleted or + truncated. +- No route, dispatcher, delivery executor, broker, Celery task, handler, + authorization identifier, or product-domain mutation is present. + +## Scope Control + +The committed diff is limited to the reviewed 02A migration, outbox module, +metadata registration, focused tests, initiative evidence, and one merge +intent. The user-owned local PDF deletion is excluded. No 02B dispatcher work +or protected product behavior is included. + +## Product Behavior + +There is no new public or protected product surface. Existing review, +contribution, compensation, authorization, artifact, task, and project behavior +is unchanged; the new participant is infrastructure for later callers. + +## Alternatives Rejected + +- A new canonicalizer or generic idempotency framework. +- An outbox-owned session, commit, enqueue, publish, or post-commit repair. +- Event identity inferred only from the idempotency key. +- Mutable payload/envelope facts or physical retention deletion. +- A schema too small for 02B that would force a second delivery-state migration. +- Dispatcher authority inherited by protected feature handlers. + +## Acceptance Criteria Proof + +- Current PR #153 reconciliation: 43 passed, 32 deselected in 177.40 seconds + on the exact bounded isolated outbox/migration row; outbox coverage is 95.73 + percent against the 90 percent subsystem floor. +- Current helper regression suite: 8 passed in 0.21 seconds. A Pydantic opaque + mapping compatibility repair preserves fail-closed secret graph inspection. +- Current AUTH revision-specific lifecycle proof: 1 passed in 63.77 seconds, + preserving AUTH's exact `0026` downgrade/reupgrade behavior independently of + repository head. +- Alembic reports exactly one head at `0029_shared_transactional_outbox`, with + parent `0028_artifact_admission`. +- The exact ART `0028` / CON `0029` bounded row passes 73 tests with 32 + deselected after the Proxy repair and 95.90 percent focused outbox coverage. +- REV-02A PR #156 is reconciled at trusted main `3b1d6379`; it changes planning + contracts only and adds no migration, runtime outbox seam, or CON authority. +- The following `0027` rows are retained as historical pre-PR #153 evidence; + they do not replace the current `0028` proof above or GitHub CI. +- Post-review exact bounded row: 43 passed, 30 deselected in 60.22 seconds, + with 95.73% outbox coverage against the 90% subsystem floor. +- Reconciled exact contract selector: 8 passed, 56 deselected. +- Current-main outbox plus migration/lifecycle-guard suite: 36 passed in + 156.17 seconds; the reconciled outbox implementation retains 95.43% focused + coverage. +- After ART-02B1 merged, the combined outbox/migration plus real-MinIO S3 + focused suite passed 76 tests in 243.28 seconds on the current tree. +- After AUTH-09D-B merged, the combined outbox/migration, real-MinIO S3, and + identity-link lifecycle/concurrency suite passed 78 tests in 378.36 seconds. +- Affected AUTH lifecycle downgrade tests: 2 passed, including atomic rollback + to the full `0027` head when AUTH refuses `0026 -> 0025`. +- Alembic historically reported exactly one head: `0027_shared_transactional_outbox`. +- Isolated database runner self-tests: 16 passed in 102.10 seconds with the + required admin URL. +- Real API contract end-to-end on the `0027` chain: passed. +- Pre-reconciliation exact isolated PostgreSQL full suite: 1347 passed in 17741.96 seconds + (4:55:41), with 85.35% repository coverage against the 78% floor. +- The pre-reconciliation isolated evidence records tree `f72bb6e`, database + `workstream_test_d513fb2f03b1`, and the superseded Alembic head + `0026_shared_transactional_outbox`; the superseded `0027` focused proof is + retained above and GitHub CI must supply repository-wide proof after publication. +- Agent-loop gates: 88 passed after AUTH-09D-B. +- Ruff, 90.4% docstring coverage, Markdown links, stale Workstream/AUTH/ART/REV + scans, and diff hygiene pass. +- AUTH-09D-A reconciliation historically moved the migration to `0027`; PR #153 + supersedes that revision with the current `0028` proof above, and GitHub CI + must pass after publication. +- REV PLAN2 PR #150 then advanced trusted main to `983b9e53`. It changes only + planning/specification files, preserves the 02A runtime boundary, and updates + future CON/REV child gates. A two-hour suite on the prior head was stopped, + discarded, and is not counted. +- ART-02B1 PR #151 then advanced trusted main to `1b5422fc`. It adds the real + S3-compatible adapter, MinIO service, SDK pins, CI gates, and a substantial + backend test delta without changing 02A code or migration. The 3-hour + 7-minute run on the prior tree was stopped and discarded. +- AUTH-09D-B PR #152 then advanced trusted main to `93dd3924`. It activates + only identity-link revoke/reactivate and adds AUTH route/test coverage, with + no migration or 02A boundary change. The one-hour run on the prior tree was + stopped and discarded. +- Contributor-foundation PR #153 then advanced trusted main to `8d5eb15b`. It + clean-cuts TaskAssignment/Submission attribution to canonical human + `contributor_id`, adds writer revalidation, and owns `0027`. It changes no + CON authority or outbox behavior; CON's migration is now the linear `0028` + child. Fresh bounded proof and exact-SHA internal review supersede the older + publication evidence, while the full suite remains GitHub CI-owned. +- A fourth local run on the frozen current head was stopped after approximately + 4 hours 15 minutes by human direction that repository-wide suites must run in + GitHub CI. Its metadata was removed and it is not counted. The existing + Backend full-suite job is the required exact-head repository proof. + +## Tests And Checks Run + +The exact isolated PostgreSQL selector covers migration custody, rollback, +replay, collision races, payload privacy, and delivery-state constraints. The +separate helper suite covers deep secret-retention assertions. Ruff, Alembic +head, Markdown links, stale-contract scans, agent-loop tests, docstrings, and +`git diff --check` also pass. + +## Test Delta + +No existing test was deleted, skipped, weakened, or rewritten to accept broken +behavior. + +Internal-review repairs add regression proof for one detached Pydantic +core-schema boundary across model methods and `TypeAdapter`, valid Python/JSON/ +string-mode parity, hostile top-level and nested dict/list/string subclasses, +payload-free outbox traceback locals including idempotency conflicts, nested +payload mutation while reservation is blocked, payload-free database failures, +and caller rollback after an injected post-reservation failure. The shared deep- +retention assertion now inspects dict/list/string subclasses and slotted +dataclass state without invoking the tested hostile overrides. The exact +documented focused command generates fresh coverage before enforcing the +subsystem floor. + +## CI Integrity + +No workflow, dependency, package script, test runner, lint/typecheck command, +coverage threshold, or CI configuration changed. + +Repository-wide tests and the 78 percent repository coverage floor run only in +the existing GitHub Backend full-suite job. Local proof is bounded to focused +real-service tests, the 90 percent outbox coverage floor, Ruff, migrations, and +static gates. This changes execution location only: no test, assertion, +isolation control, or coverage threshold is waived. + +## Reviewer Results + +- Exact repair candidate `9be9c88a19c4301f2b7ac606b6782604a48472bb` + passed senior engineering, QA/test, security/auth, product/ops, + architecture, docs, reuse/dedup, test-delta, and CI-integrity review against + trusted main `3b1d63796c086f53fc2b0aeefe096387b82485ec` with no open findings and no + open reviewer sessions. +- The pre-PR #153 implementation passed all nine tracks at historical code SHA + `460573287270965d730c83f5f1e52f3acf1c0671`. +- That historical review is superseded by the current exact-SHA result above. + +## External Review + +CodeRabbit review `8be695bd-33ce-4847-8bf1-905a130804ec` posted five actionable +documentation-consistency threads. The repair adds explicit reviewer tracks, +aligns AUTH counts, defines REV parent/child aliases, fixes REV-13C release +ownership, and fixes the REV-04B prerequisite. Its physical-purge suggestion is +deferred to separately started 02B because 02A forbids retention behavior and +its custody contract prohibits physical delete/truncate. GitHub Backend remains +in progress; external review does not replace internal review. + +## Remaining Risks + +- GitHub still must prove the full backend suite and repository-wide 78 percent + coverage floor on the published exact head. +- The Pydantic string-mode wrapper supports the model's fixed strict/default + contract; arbitrary per-call runtime option overrides are not a 02A API. +- Dispatcher mechanics, service authority, and recovery remain excluded and + require a separately started 02B chunk. +- Sustained-volume archival/retention behavior remains a reviewed 02B concern; + no trigger-disable purge path is introduced into immutable 02A event truth. + +## Human Review Focus + +1. Is the immutable/operational schema complete for migration-free 02B without + implementing dispatcher behavior early? +2. Do global idempotency and event-ID collisions fail closed under concurrent + commit and rollback orders? +3. Are payload bounds and secret-key rejection sufficient for a generic event + envelope? +4. Do database custody, archival, and guarded downgrade preserve durable event + truth? +5. Does append remain entirely inside the caller-owned transaction with no AUTH + or product-domain boundary expansion? + +## Follow-Up Work + +The same-initiative successor is `WS-CON-001-02B`, Shared Outbox Dispatcher And +Recovery. It requires a separate explicit start after this PR merges and after +its AUTH/service prerequisites refresh from trusted main. + +## Human Merge Ownership + +Only the human owner may approve and merge the specific 02A PR. Passing +reviewers, CI, or CodeRabbit do not authorize merge or the next chunk. diff --git a/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-preimplementation-plan-review.md b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-preimplementation-plan-review.md new file mode 100644 index 000000000..c1d24f665 --- /dev/null +++ b/.agent-loop/initiatives/WS-CON-001-contribution-compensation-boundary/reviews/WS-CON-001-02A-preimplementation-plan-review.md @@ -0,0 +1,362 @@ +# WS-CON-001-02A Preimplementation Plan Review + +## ART-02C1 Current-Main Reconciliation + +ART-02C1 PR #154 merged at trusted main `44f2467c` and owns +`0028_artifact_admission`. The reviewed 02A design and scope remain unchanged; +its migration is now the exact linear child +`0029_shared_transactional_outbox`. Fresh bounded proof and all nine exact-SHA +reviewer tracks are required before PR #155 is republished. + +## Contributor Foundation Current-Main Reconciliation + +Trusted main advanced to `8d5eb15b384fd75787ce98a099400a1d335d2560` +through contributor-foundation PR #153. The merge clean-cuts TaskAssignment and +Submission attribution to canonical human `contributor_id`, adds database +lineage guards and transaction-local active-human writer revalidation, and +owns `0027_contributor_foundation`. It changes no ActionId, PermissionId, +availability, grant, evaluator, fixed-service admission, review lifecycle, +dispatcher seam, or outbox behavior. CON-02A remains authorization-neutral and +moves to the exact linear child `0028_shared_transactional_outbox`. AUTH-09E +remains later. + +The AUTH merge also deliberately makes its actor-lifecycle downgrade tests +revision-specific rather than treating repository `head` as `0026`. CON +preserves that test ownership and adds separate dedicated `0028` migration/head +proof. Repository-wide proof remains GitHub CI-owned; local reconciliation +proof is bounded to the outbox and migration selectors. + +## AUTH-09D-B Historical Reconciliation + +Trusted main advanced to `93dd392484b397cfdfaaa833631dc2c27f591ed7` +through AUTH-09D-B PR #152. The canonical catalogue remains 74 PermissionIds +and 65 ActionIds, now 17 active and 48 planned. The merge activates exactly +`actor.identity_link.revoke` and `actor.identity_link.reactivate`; it adds no +CON/task-claim ActionId, fixed-service admission, migration, or outbox seam. +At that historical point the contributor-field/canonical-human foundation was proposed next, with +AUTH-09E still later. CON-02A therefore remains authorization-neutral at +`0027`. PR #153 supersedes that migration-head statement as described above. + +The exact isolated PostgreSQL plus MinIO suite on the prior `1b5422fc` +baseline was stopped after one hour solely because PR #152 advanced trusted +main. Its metadata was removed and it is not accepted evidence. + +## ART-02B1 Historical Reconciliation + +Trusted main advanced to `1b5422fcaa361152af7c2b1f82a763d99c0e6db5` +through ART-02B1 PR #151. The merge adds the S3-compatible ArtifactStore +adapter, real MinIO integration, inactive AWS-profile support, exact SDK pins, +CI services/coverage gates, and substantial artifact/configuration tests. It +adds no migration, outbox import, dispatcher seam, AUTH/CON identifier, ART +call in the core transaction, or 02A allowed-file change. Alembic therefore +still ends at CON-owned `0027_shared_transactional_outbox`, while the expanded +dependency and test tree requires fresh deterministic evidence. + +The exact isolated suite on the prior `983b9e53` baseline was stopped after 3 +hours 7 minutes solely because PR #151 advanced trusted main. Its metadata was +removed and it is not accepted evidence. Verification must restart on the +frozen `1b5422fc` baseline after installing the newly merged locked runtime +dependencies and starting the real MinIO test provider required by main. + +## REV PLAN2 Historical Reconciliation + +Trusted main advanced to `983b9e534b84f1590fafecc0ce1355cf131257ce` +through planning-only REV PLAN2 PR #150. The merge changes no backend runtime, +migration, test runner, AUTH catalogue, outbox contract, or 02A allowed-file +scope. It preserves the two ordered CON operations, accept-only +FinalAcceptance, REV-owned shared audit/outbox staging, and one caller commit. +It replaces future non-executable REV parent gates with explicit children: +CON-03B precedes REV-03A, CON-02A/02C precede REV-04B, CON-03C/07 precede +REV-10, the shared dispatcher/handler registry precedes REV-12P1, CON fence +hooks precede REV-12A3, and CON-11 precedes REV-13C. No 02A implementation +change is required. + +An exact isolated suite started on the prior `99ae4c96` baseline was stopped +after two hours solely because PR #150 advanced trusted main. It produced no +accepted evidence artifact and is not counted below. Exact verification must +restart on the frozen `983b9e53` baseline. + +## AUTH-09D-A Historical Reconciliation + +Trusted main advanced to `99ae4c963e53f317175dcb308b9e47c93ccf19ed` +through AUTH-09D-A PR #148 before publication. AUTH now owns +`0026_actor_profile_lifecycle`, so CON-02A's reviewed linear migration is +`0027_shared_transactional_outbox` with parent +`0026_actor_profile_lifecycle`. AUTH-09D-A activates only three actor-profile +lifecycle actions; it adds no CON/outbox identifier, evaluator, service +identity, static row, fixed-service admission, or product behavior. +Historical local evidence used a 25,200-second full-suite safety ceiling after +the pre-AUTH-09D-A suite consumed 17,741.96 of 18,000 seconds. Human direction +D20 supersedes that local execution location: repository-wide proof now runs +in GitHub CI, with no selection, assertion, isolation control, or 78/90 +coverage threshold change. + +## Exact baseline and scope + +- Baseline: trusted `origin/main` at `a10d901` after ART PR #141 merged. +- Risk: L1 infrastructure, schema, concurrency, audit, and data-integrity risk. +- Delivery priority: P1 active-sprint prerequisite for REV/CON composition. +- Human checkpoint: required before merge. +- Allowed implementation files and exclusions are exactly those in + `../chunks/WS-CON-001-02A-shared-outbox-persistence.md`. +- The user-owned deleted reference PDF is outside scope and must remain + unstaged and untouched. +- Before final implementation evidence, trusted `main` advanced again to + `0ffdabf` through AUTH-09C PR #146. Its administrative actor/profile reads + activate only existing AUTH-owned identifiers, add no migration or CON + action, and leave this reviewed implementation contract unchanged. +- Trusted `main` later advanced to `b2b9016` through REV-01 PR #145. Its + canonical review specification preserves the exact FinalAcceptance, + two-operation CON participant, shared-outbox, and single-commit boundaries; + it adds no backend runtime or migration and does not alter this plan. +- Trusted `main` later advanced to `f18b620` through planning-only REV-02 PR + #147. Its future REV chunk decomposition adds no runtime, migration, test + runner, CON, or outbox behavior and leaves this plan unchanged. +- Historical local full-suite execution reached 90 percent with no failures + before its process ceiling. D20 now assigns the unchanged complete test set + and 78 percent repository coverage floor to GitHub CI; local proof is bounded + to focused real-service tests and the 90 percent subsystem floor. + +## Proposed implementation + +1. Add one `OutboxEvent` PostgreSQL table and SQLAlchemy model containing the + exact immutable common event envelope and canonical payload digest frozen + below, plus the complete generic operational-delivery shape required by the + later migration-free CON-02B dispatcher chunk. +2. Enforce event/idempotency uniqueness, the exact payload/digest/token bounds + below, database-owned occurrence time, closed delivery-state shapes, + immutable envelope/payload custody, permanent physical delete/truncate + denial, terminal archival-in-place, and a nonempty-table downgrade guard in + linear revision `0028_shared_transactional_outbox` after AUTH-owned + `0027_contributor_foundation`. +3. Add strict typed append input/output schemas. The caller supplies stable + event identity and canonical event facts but not occurrence or delivery + state. The service hashes only the validated payload with + `canonical_json_hash`. +4. Reuse the existing PostgreSQL reserve/lock/replay sequence and typed + conflict/replay semantics, without importing AUTH or creating a generic + idempotency abstraction. Attempt the completed event insert with + `ON CONFLICT DO NOTHING`; insertion itself is the event reservation and + there is no separate pending-to-complete idempotency record. Lock matches in + ascending event-ID order and apply the collision matrix below. Flush/refresh + only; transaction commit remains caller-owned. Never commit, publish, + enqueue, log payloads, or open another session. +5. Register the model in shared metadata and add PostgreSQL tests for linear + migration upgrade/guarded downgrade, schema/trigger invariants, append and + exact replay, the full collision/race matrix, canonical key-order replay, + payload/error privacy bounds, and caller rollback proving no independent + commit or publication. +6. Run the exact bounded isolated CON-02A row, focused coverage at or above 90 + percent, Ruff, stale-wording and link checks, then fan out all required + internal reviewer tracks on one exact commit SHA. After the full PR is + pushed, GitHub CI owns repository-wide tests and coverage at or above 78 + percent. Repair and rerun evidence/review after any implementation change. + +## Explicit non-goals + +- No AUTH catalogue or evaluator changes and no outbox action/permission. +- No dispatcher, Celery task, broker, registry, handler, route, claim action, + retry execution, feature delivery, or external I/O. +- No contribution, compensation, review, task, project, audit, or ART behavior. +- No second JSON canonicalizer, generic idempotency framework, dependency, or + CI/coverage change. +- No CON-02B work. + +## Frozen persistence schema + +### Immutable columns + +| Column | PostgreSQL type | Null/default/bounds | +|---|---|---| +| `event_id` | `UUID` | primary key; caller-supplied | +| `event_type` | `VARCHAR(128)` | non-null; ASCII token `[A-Za-z][A-Za-z0-9._:-]{0,127}` | +| `event_version` | `SMALLINT` | non-null; `1..32767` | +| `producer` | `VARCHAR(32)` | non-null; database-forced `workstream` | +| `aggregate_type` | `VARCHAR(64)` | non-null; lower ASCII token `[a-z][a-z0-9_]{0,63}` | +| `aggregate_id` | `UUID` | non-null; canonical source aggregate identity | +| `project_id` | `VARCHAR(36)` | non-null FK to `projects.id`; canonical lowercase UUID text | +| `correlation_id` | `VARCHAR(200)` | non-null; ASCII token `[A-Za-z0-9._:-]{1,200}` | +| `causation_event_id` | `UUID` | nullable; no FK because the cause may live in another event ledger | +| `idempotency_key` | `VARCHAR(200)` | non-null; globally unique; ASCII token `[A-Za-z0-9._:-]{1,200}` | +| `payload` | `JSONB` | non-null object; generic structural/privacy bounds below | +| `payload_digest` | `VARCHAR(71)` | non-null `sha256:` plus 64 lowercase hex characters | +| `occurred_at` | `TIMESTAMPTZ` | non-null; database-forced `statement_timestamp()` | + +The event ID and globally namespaced idempotency key are independent unique +identities. Every column in this table is immutable except the operational +allowlist below. The database insert trigger overwrites producer, occurrence +time, initial state, counters, eligibility time, and all nullable operational +fields so direct SQL cannot forge an already-claimed or completed event. + +### Mutable operational columns + +| Column | PostgreSQL type | Initial/bounds | +|---|---|---| +| `delivery_state` | `VARCHAR(16)` | `pending`; closed set `pending`, `claimed`, `retryable`, `acknowledged`, `dead_letter`, `cancelled` | +| `attempt_count` | `INTEGER` | `0`; nonnegative and equal to claim generation | +| `next_attempt_at` | `TIMESTAMPTZ` | occurrence time while initially pending; otherwise state-bound | +| `claim_owner` | `VARCHAR(120)` | nullable bounded ASCII token | +| `claim_generation` | `BIGINT` | `0`; nonnegative and incremented with each claim | +| `claimed_at` | `TIMESTAMPTZ` | nullable; equals `last_attempt_at` while claimed | +| `claim_expires_at` | `TIMESTAMPTZ` | nullable; greater than `claimed_at` | +| `last_attempt_at` | `TIMESTAMPTZ` | nullable; database-time attempt evidence | +| `last_error_code` | `VARCHAR(80)` | nullable; `[A-Z][A-Z0-9_]{0,79}` only; never free-form diagnostics | +| `finalized_at` | `TIMESTAMPTZ` | nullable; required for terminal states | +| `archived_at` | `TIMESTAMPTZ` | nullable; allowed only for terminal rows and not before `finalized_at` | + +There is no physical purge seam. Retention in v0.1 means terminal +archival-in-place by setting `archived_at`; immutable event/payload truth is +never deleted or truncated. CON-02B may mutate only this operational allowlist. + +### Closed state shapes + +| State | Required shape | +|---|---| +| `pending` | zero attempts/generation; eligibility present; no claim, attempt, error, final, or archive fields | +| `claimed` | positive equal attempts/generation; owner, claimed/last-attempt, and future lease expiry present; no eligibility/final/archive fields; prior bounded error may remain | +| `retryable` | positive equal attempts/generation; eligibility, last attempt, and bounded error present; no live claim/final/archive fields | +| `acknowledged` | positive equal attempts/generation; last attempt and final time present; no eligibility/live claim; bounded prior error and later archive marker are allowed | +| `dead_letter` | positive equal attempts/generation; last attempt, bounded error, and final time present; no eligibility/live claim; later archive marker allowed | +| `cancelled` | final time present and no eligibility/live claim; either never attempted with zero generation/no attempt evidence or positive equal attempts/generation with last-attempt evidence; later archive marker allowed | + +All operational timestamps are at or after `occurred_at`; claim expiry is +strictly after claim time; finalization is not before the last attempt when one +exists; archival is not before finalization. Expired `claimed` work may recover +to `retryable`, and only an unarchived `dead_letter` terminal may be requeued, +in both cases by clearing live-claim/final fields and satisfying the retryable +shape. `acknowledged` and `cancelled` never reopen. CON-02B owns the transition +service and authorization, not this chunk. + +### Required indexes + +- `(event_type, delivery_state, next_attempt_at, occurred_at, event_id)` for + deterministic eligible-work selection; +- `(aggregate_type, aggregate_id, occurred_at, event_id)` for source-aggregate + reconciliation without inspecting payloads; +- `(claim_expires_at, event_id)` partial on `claimed` for expired-lease recovery; +- `(project_id, delivery_state, occurred_at, event_id)` for same-session drain + counts and project-bounded observation; +- `(finalized_at, event_id)` partial on terminal rows with `archived_at IS NULL` + for retention eligibility; +- the primary key and global unique idempotency-key constraint for replay. + +## Payload and privacy contract + +The generic outbox accepts only a JSON object after producer-owned typed payload +validation. Every object key must use lower ASCII snake case +`[a-z][a-z0-9_]{0,127}`. It independently rejects floats, bytes, non-JSON values, more than +16 container levels, more than 4,096 total nodes, more than 1,024 members in +one object/list, keys over 128 UTF-8 bytes, strings over 16,384 UTF-8 bytes, and +integers outside signed 38-digit magnitude. A conservative traversal budget +must prove the canonical UTF-8 encoding cannot exceed 262,144 bytes without +serializing through a second canonicalizer; PostgreSQL also checks the stored +JSONB text representation is at most 262,144 bytes. + +At every nesting level the generic validator first case-folds keys and converts +hyphens to underscores for denylist comparison, then independently enforces the +lower-snake-case key grammar. It rejects credential-bearing normalized keys: +`authorization`, `cookie`, `credentials`, `password`, `secret`, +`access_token`, `refresh_token`, `id_token`, `bearer_token`, `jwks`, +`signed_url`, `raw_callback`, `raw_provider_response`, `artifact_bytes`, +`request_body`, and `response_body`. Capitalization and hyphen/underscore +variants therefore cannot bypass the privacy check. Producer payload schemas must additionally +exclude bearer tokens, credentials, key material, raw claims, provider URLs or +messages, callback bodies, artifact bytes, and unbounded/free-form sensitive +diagnostics. Bounded non-secret opaque identifiers remain allowed where their +feature specification permits them. + +Validation order is structure/privacy bounds, canonical hash, then database +reservation. Invalid input and idempotency conflict raise typed exceptions +whose messages contain only stable error codes; no exception, log, result, or +failure-code field contains payload values or secrets. The outbox module emits +no logs in this chunk. + +## Replay and collision matrix + +Replay compares event ID, event type, event version, fixed producer, aggregate +type/ID, project, correlation, causation, idempotency key, and canonical payload +digest. It also confirms stored JSONB object equality defensively. It preserves the original +database occurrence time and ignores all operational delivery fields. + +| Locked match | Result | +|---|---| +| no identity exists | insert and return `created` | +| event ID and key resolve to the same row; every immutable fact matches | return that row as `replayed` | +| either identity exists with any immutable fact/digest drift | typed `outbox_idempotency_conflict` | +| event ID and key resolve to two different rows | typed `outbox_idempotency_conflict` | + +Conflict lookup locks every distinct matching row in ascending `event_id` +order. Tests cover same/same, canonical object-key reordering, same ID/different +key, different ID/same key, changed payload, changed aggregate/envelope, +pre-seeded split identities, and independent-session races in both first-reserver +commit and rollback orders. If the first reserver commits, an exact contender +replays and a changed contender conflicts. If the first reserver rolls back, +the contender inserts as `created`; rolled-back truth never produces replay or +conflict. Exactly one row is durable in either order. + +## Operational transition matrix + +Database checks and the custody trigger permit only these future CON-02B +transitions; this chunk exposes no transition method: + +- `pending -> claimed | cancelled`; +- `claimed -> retryable | acknowledged | dead_letter | cancelled`; +- `retryable -> claimed | cancelled`; +- unarchived `dead_letter -> retryable` for later independently authorized + recovery; +- `pending -> pending` or `retryable -> retryable` only to change eligibility + time for later independently authorized recovery/reconciliation; +- terminal same-state update only to set the one-way `archived_at` marker. + +Every claim increments attempt count and claim generation together by exactly +one. Every outcome preserves both values. They never decrease. Requeue clears +finalization and preserves attempt/generation history. `acknowledged` and +`cancelled` never reopen; archived dead letters never reopen. No state may +transition out of archival, and no same-state mutation may change immutable or +unrelated operational facts. + +## Migration and transaction proof details + +- Upgrade from exact revision `0026_actor_profile_lifecycle` and verify + columns, constraints, indexes, triggers, metadata, and head identity. +- Attempt direct SQL mutation of every immutable column and physical delete/ + truncate; each must fail. Exercise every operational column through at least + one complete legal state sequence so the allowlist is not over-constrained, + and directly prove acknowledged/cancelled/archived reopening and illegal + same-state mutation fail. +- A downgrade takes `ACCESS EXCLUSIVE` lock before checking emptiness. Test it + against an uncommitted insert in both commit and rollback orders: committed + truth blocks downgrade; rolled-back truth permits it. +- After the nonempty refusal, remove only test data with triggers explicitly + disabled, prove empty downgrade succeeds, then restore head. +- Inject failure after append insert/flush/refresh and roll back the caller + session; prove zero outbox row, no commit, no broker/publish/enqueue call, and + no other observable side effect. + +## Proof and reviewer routing + +Required tracks: senior engineering, QA/test, security/auth, product/ops, +architecture, docs, reuse/dedup, test-delta, and CI integrity. Review must focus +on caller transaction ownership, PostgreSQL duplicate-race behavior, immutable +versus operational field separation, payload privacy/integrity, forward +compatibility with 02B without implementing it, migration downgrade safety, +and preservation of the exact chunk boundary. + +## Review result + +PASS after repair. Initial review found that the migration-free 02B schema, +retention/delete behavior, replay collision matrix, payload/privacy bounds, +aggregate identity, terminal recovery rules, and rolled-back-reserver outcome +were underspecified. The frozen schema, operational transition matrix, privacy +contract, collision matrix, and proof details above resolve every finding. +Architecture, senior engineering, reuse/dedup, QA/test, product/ops, docs, +security/auth, and CI integrity all returned PASS before implementation began. + +After implementation began, ART PR #141 and AUTH-09D-A advanced trusted `main` +and moved CON through historical revisions 0026 and 0027. Contributor- +foundation PR #153 now owns AUTH revision `0027_contributor_foundation`. +Reconciliation preserves the reviewed schema and behavior while moving only +CON's revision identity and parent to linear +`0028_shared_transactional_outbox`. None of these merges adds a shared outbox +or CON authorization seam, so no implementation boundary or non-goal changes. +Final exact-SHA review must cover this current-main reconciliation. diff --git a/.agent-loop/merge-intents/WS-CON-001-02A.json b/.agent-loop/merge-intents/WS-CON-001-02A.json new file mode 100644 index 000000000..f099bbdd8 --- /dev/null +++ b/.agent-loop/merge-intents/WS-CON-001-02A.json @@ -0,0 +1,9 @@ +{ + "chunk_id": "WS-CON-001-02A", + "chunk_title": "Shared Transactional Outbox Persistence", + "initiative_id": "WS-CON-001", + "next_chunk_id": "WS-CON-001-02B", + "next_chunk_title": "Shared Outbox Dispatcher And Recovery", + "next_requires_explicit_start": true, + "schema_version": 2 +} diff --git a/backend/alembic/versions/0029_shared_transactional_outbox.py b/backend/alembic/versions/0029_shared_transactional_outbox.py new file mode 100644 index 000000000..ea6aba6c8 --- /dev/null +++ b/backend/alembic/versions/0029_shared_transactional_outbox.py @@ -0,0 +1,330 @@ +"""add shared transactional outbox persistence + +Revision ID: 0029_shared_transactional_outbox +Revises: 0028_artifact_admission +Create Date: 2026-07-18 +""" + +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = "0029_shared_transactional_outbox" +down_revision = "0028_artifact_admission" +branch_labels = depends_on = None + + +def upgrade() -> None: + """Create immutable event truth and the closed generic delivery-state shape.""" + op.create_table( + "outbox_events", + sa.Column("event_id", sa.Uuid(), nullable=False), + sa.Column("event_type", sa.String(128), nullable=False), + sa.Column("event_version", sa.SmallInteger(), nullable=False), + sa.Column("producer", sa.String(32), nullable=False, server_default="workstream"), + sa.Column("aggregate_type", sa.String(64), nullable=False), + sa.Column("aggregate_id", sa.Uuid(), nullable=False), + sa.Column("project_id", sa.String(36), nullable=False), + sa.Column("correlation_id", sa.String(200), nullable=False), + sa.Column("causation_event_id", sa.Uuid()), + sa.Column("idempotency_key", sa.String(200), nullable=False), + sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("payload_digest", sa.String(71), nullable=False), + sa.Column( + "occurred_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("statement_timestamp()"), + ), + sa.Column("delivery_state", sa.String(16), nullable=False, server_default="pending"), + sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column( + "next_attempt_at", + sa.DateTime(timezone=True), + nullable=True, + server_default=sa.text("statement_timestamp()"), + ), + sa.Column("claim_owner", sa.String(120)), + sa.Column("claim_generation", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("claimed_at", sa.DateTime(timezone=True)), + sa.Column("claim_expires_at", sa.DateTime(timezone=True)), + sa.Column("last_attempt_at", sa.DateTime(timezone=True)), + sa.Column("last_error_code", sa.String(80)), + sa.Column("finalized_at", sa.DateTime(timezone=True)), + sa.Column("archived_at", sa.DateTime(timezone=True)), + sa.PrimaryKeyConstraint("event_id", name="pk_outbox_events"), + sa.UniqueConstraint("idempotency_key", name="uq_outbox_events_idempotency_key"), + sa.ForeignKeyConstraint( + ["project_id"], ["projects.id"], name="fk_outbox_events_project_id_projects" + ), + sa.CheckConstraint( + "event_type ~ '^[A-Za-z][A-Za-z0-9._:-]{0,127}$'", + name="event_type", + ), + sa.CheckConstraint( + "event_version between 1 and 32767", + name="event_version", + ), + sa.CheckConstraint("producer = 'workstream'", name="producer"), + sa.CheckConstraint( + "aggregate_type ~ '^[a-z][a-z0-9_]{0,63}$'", + name="aggregate_type", + ), + sa.CheckConstraint( + "project_id ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'", + name="project_id", + ), + sa.CheckConstraint( + "correlation_id ~ '^[A-Za-z0-9._:-]{1,200}$'", + name="correlation_id", + ), + sa.CheckConstraint( + "idempotency_key ~ '^[A-Za-z0-9._:-]{1,200}$'", + name="idempotency_key", + ), + sa.CheckConstraint( + "payload_digest ~ '^sha256:[0-9a-f]{64}$'", + name="payload_digest", + ), + sa.CheckConstraint( + "jsonb_typeof(payload) = 'object' and octet_length(payload::text) <= 262144", + name="payload_shape", + ), + sa.CheckConstraint( + "delivery_state in ('pending','claimed','retryable','acknowledged'," + "'dead_letter','cancelled')", + name="delivery_state", + ), + sa.CheckConstraint( + "attempt_count >= 0 and claim_generation >= 0 " + "and attempt_count = claim_generation", + name="delivery_counters", + ), + sa.CheckConstraint( + "claim_owner is null or claim_owner ~ '^[A-Za-z0-9._:-]{1,120}$'", + name="claim_owner", + ), + sa.CheckConstraint( + "last_error_code is null or last_error_code ~ '^[A-Z][A-Z0-9_]{0,79}$'", + name="error_code", + ), + sa.CheckConstraint( + "(next_attempt_at is null or next_attempt_at >= occurred_at) and " + "(claimed_at is null or claimed_at >= occurred_at) and " + "(last_attempt_at is null or last_attempt_at >= occurred_at) and " + "(claim_expires_at is null or claim_expires_at > claimed_at) and " + "(finalized_at is null or finalized_at >= occurred_at) and " + "(finalized_at is null or last_attempt_at is null or finalized_at >= last_attempt_at) and " + "(archived_at is null or archived_at >= finalized_at)", + name="delivery_timestamps", + ), + sa.CheckConstraint(_state_shape(), name="delivery_state_shape"), + ) + _create_indexes() + _create_custody_triggers() + + +def _state_shape() -> str: + """Return the frozen closed-state constraint shared with the ORM model.""" + return ( + "(delivery_state = 'pending' and attempt_count = 0 and next_attempt_at is not null " + "and claim_owner is null and claimed_at is null and claim_expires_at is null " + "and last_attempt_at is null and last_error_code is null and finalized_at is null " + "and archived_at is null) or " + "(delivery_state = 'claimed' and attempt_count > 0 and next_attempt_at is null " + "and claim_owner is not null and claimed_at is not null " + "and claim_expires_at is not null and last_attempt_at = claimed_at " + "and finalized_at is null and archived_at is null) or " + "(delivery_state = 'retryable' and attempt_count > 0 and next_attempt_at is not null " + "and claim_owner is null and claimed_at is null and claim_expires_at is null " + "and last_attempt_at is not null and last_error_code is not null " + "and finalized_at is null and archived_at is null) or " + "(delivery_state = 'acknowledged' and attempt_count > 0 and next_attempt_at is null " + "and claim_owner is null and claimed_at is null and claim_expires_at is null " + "and last_attempt_at is not null and finalized_at is not null) or " + "(delivery_state = 'dead_letter' and attempt_count > 0 and next_attempt_at is null " + "and claim_owner is null and claimed_at is null and claim_expires_at is null " + "and last_attempt_at is not null and last_error_code is not null " + "and finalized_at is not null) or " + "(delivery_state = 'cancelled' and next_attempt_at is null and claim_owner is null " + "and claimed_at is null and claim_expires_at is null and finalized_at is not null " + "and ((attempt_count = 0 and last_attempt_at is null and last_error_code is null) " + "or (attempt_count > 0 and last_attempt_at is not null)))" + ) + + +def _create_indexes() -> None: + op.create_index( + "ix_outbox_events_eligible", + "outbox_events", + ["event_type", "delivery_state", "next_attempt_at", "occurred_at", "event_id"], + postgresql_where=sa.text("delivery_state in ('pending','retryable')"), + ) + op.create_index( + "ix_outbox_events_expired_claims", + "outbox_events", + ["claim_expires_at", "event_id"], + postgresql_where=sa.text("delivery_state = 'claimed'"), + ) + op.create_index( + "ix_outbox_events_project_drain", + "outbox_events", + ["project_id", "delivery_state", "occurred_at", "event_id"], + ) + op.create_index( + "ix_outbox_events_retention", + "outbox_events", + ["finalized_at", "event_id"], + postgresql_where=sa.text( + "delivery_state in ('acknowledged','dead_letter','cancelled') " + "and archived_at is null" + ), + ) + op.create_index( + "ix_outbox_events_aggregate", + "outbox_events", + ["aggregate_type", "aggregate_id", "occurred_at", "event_id"], + ) + + +def _create_custody_triggers() -> None: + op.execute( + """ + create function guard_outbox_event() returns trigger + language plpgsql as $$ + declare event_time timestamptz; + begin + if tg_op = 'TRUNCATE' then + raise exception 'outbox events cannot be truncated' using errcode='55000'; + elsif tg_op = 'DELETE' then + raise exception 'outbox events cannot be deleted' using errcode='55000'; + elsif tg_op = 'INSERT' then + event_time := statement_timestamp(); + new.producer := 'workstream'; + new.occurred_at := event_time; + new.delivery_state := 'pending'; + new.attempt_count := 0; + new.next_attempt_at := event_time; + new.claim_owner := null; + new.claim_generation := 0; + new.claimed_at := null; + new.claim_expires_at := null; + new.last_attempt_at := null; + new.last_error_code := null; + new.finalized_at := null; + new.archived_at := null; + return new; + end if; + + if (new.event_id, new.event_type, new.event_version, new.producer, + new.aggregate_type, new.aggregate_id, new.project_id, + new.correlation_id, new.causation_event_id, new.idempotency_key, + new.payload, new.payload_digest, new.occurred_at) + is distinct from + (old.event_id, old.event_type, old.event_version, old.producer, + old.aggregate_type, old.aggregate_id, old.project_id, + old.correlation_id, old.causation_event_id, old.idempotency_key, + old.payload, old.payload_digest, old.occurred_at) then + raise exception 'outbox event envelope is immutable' using errcode='55000'; + end if; + if new.attempt_count < old.attempt_count + or new.claim_generation < old.claim_generation + or new.attempt_count <> new.claim_generation then + raise exception 'outbox counters cannot regress' using errcode='23514'; + end if; + if old.archived_at is not null and + (new.delivery_state, new.attempt_count, new.next_attempt_at, + new.claim_owner, new.claim_generation, new.claimed_at, + new.claim_expires_at, new.last_attempt_at, new.last_error_code, + new.finalized_at, new.archived_at) + is distinct from + (old.delivery_state, old.attempt_count, old.next_attempt_at, + old.claim_owner, old.claim_generation, old.claimed_at, + old.claim_expires_at, old.last_attempt_at, old.last_error_code, + old.finalized_at, old.archived_at) then + raise exception 'archived outbox event is closed' using errcode='55000'; + end if; + + if old.delivery_state in ('pending', 'retryable') + and new.delivery_state = 'claimed' then + if new.attempt_count <> old.attempt_count + 1 + or new.claim_generation <> old.claim_generation + 1 + or new.last_error_code is distinct from old.last_error_code then + raise exception 'outbox claim generation must increment once' using errcode='23514'; + end if; + elsif old.delivery_state = 'claimed' + and new.delivery_state in ('retryable','acknowledged','dead_letter','cancelled') then + if new.attempt_count <> old.attempt_count + or new.claim_generation <> old.claim_generation + or new.last_attempt_at is distinct from old.last_attempt_at then + raise exception 'outbox outcome cannot change claim generation' using errcode='23514'; + end if; + elsif old.delivery_state = 'dead_letter' + and new.delivery_state = 'retryable' and old.archived_at is null then + if new.attempt_count <> old.attempt_count + or new.claim_generation <> old.claim_generation + or new.last_attempt_at is distinct from old.last_attempt_at + or new.last_error_code is distinct from old.last_error_code then + raise exception 'outbox requeue cannot change claim generation' using errcode='23514'; + end if; + elsif old.delivery_state in ('pending','retryable') + and new.delivery_state = 'cancelled' then + if new.attempt_count <> old.attempt_count + or new.claim_generation <> old.claim_generation + or new.last_attempt_at is distinct from old.last_attempt_at + or new.last_error_code is distinct from old.last_error_code then + raise exception 'outbox cancellation cannot change claim generation' using errcode='23514'; + end if; + elsif old.delivery_state in ('pending','retryable') + and new.delivery_state = old.delivery_state then + if (new.attempt_count, new.claim_owner, new.claim_generation, + new.claimed_at, new.claim_expires_at, new.last_attempt_at, + new.last_error_code, new.finalized_at, new.archived_at) + is distinct from + (old.attempt_count, old.claim_owner, old.claim_generation, + old.claimed_at, old.claim_expires_at, old.last_attempt_at, + old.last_error_code, old.finalized_at, old.archived_at) then + raise exception 'outbox eligibility update changed unrelated state' using errcode='23514'; + end if; + elsif old.delivery_state in ('acknowledged','dead_letter','cancelled') + and new.delivery_state = old.delivery_state then + if (new.attempt_count, new.next_attempt_at, new.claim_owner, + new.claim_generation, new.claimed_at, new.claim_expires_at, + new.last_attempt_at, new.last_error_code, new.finalized_at) + is distinct from + (old.attempt_count, old.next_attempt_at, old.claim_owner, + old.claim_generation, old.claimed_at, old.claim_expires_at, + old.last_attempt_at, old.last_error_code, old.finalized_at) + or (old.archived_at is not null and new.archived_at is distinct from old.archived_at) + or (old.archived_at is null and new.archived_at is null) then + raise exception 'terminal outbox event permits archival only' using errcode='23514'; + end if; + else + raise exception 'illegal outbox delivery transition' using errcode='23514'; + end if; + return new; + end $$ + """ + ) + op.execute( + "create trigger outbox_events_custody before insert or update or delete " + "on outbox_events for each row execute function guard_outbox_event()" + ) + op.execute( + "create trigger outbox_events_reject_truncate before truncate on outbox_events " + "for each statement execute function guard_outbox_event()" + ) + + +def downgrade() -> None: + """Remove the empty outbox only after excluding concurrent append writers.""" + bind = op.get_bind() + bind.execute(sa.text("lock table outbox_events in access exclusive mode")) + if bind.execute(sa.text("select exists(select 1 from outbox_events)")).scalar_one(): + raise RuntimeError("cannot downgrade with shared outbox events") + op.execute("drop trigger outbox_events_reject_truncate on outbox_events") + op.execute("drop trigger outbox_events_custody on outbox_events") + op.execute("drop function guard_outbox_event()") + op.drop_table("outbox_events") diff --git a/backend/app/db/models.py b/backend/app/db/models.py index a8195b918..8fe948bc4 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -21,6 +21,7 @@ AuthorityIdempotencyRecord, ) from app.modules.checkers.models import CheckerResult, CheckerRun # noqa: F401 +from app.modules.outbox.models import OutboxEvent # noqa: F401 from app.modules.projects.models import ( # noqa: F401 EffectiveProjectSubmissionArtifactPolicy, GuideSourceSnapshot, diff --git a/backend/app/modules/outbox/__init__.py b/backend/app/modules/outbox/__init__.py new file mode 100644 index 000000000..49672dc96 --- /dev/null +++ b/backend/app/modules/outbox/__init__.py @@ -0,0 +1,21 @@ +"""Shared transactional outbox persistence and caller-transaction append.""" + +from app.modules.outbox.schemas import ( + OutboxAppendDisposition, + OutboxAppendInput, + OutboxAppendResult, + OutboxIdempotencyConflict, + OutboxInputError, + OutboxPersistenceError, +) +from app.modules.outbox.service import OutboxService + +__all__ = [ + "OutboxAppendDisposition", + "OutboxAppendInput", + "OutboxAppendResult", + "OutboxIdempotencyConflict", + "OutboxInputError", + "OutboxPersistenceError", + "OutboxService", +] diff --git a/backend/app/modules/outbox/models.py b/backend/app/modules/outbox/models.py new file mode 100644 index 000000000..7d10c3385 --- /dev/null +++ b/backend/app/modules/outbox/models.py @@ -0,0 +1,190 @@ +"""SQLAlchemy persistence for the feature-neutral shared outbox.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any +from uuid import UUID + +from sqlalchemy import ( + BigInteger, + CheckConstraint, + DateTime, + ForeignKey, + Index, + Integer, + SmallInteger, + String, + Uuid, + text, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base + + +class OutboxEvent(Base): + """Immutable event envelope plus separately mutable delivery metadata.""" + + __tablename__ = "outbox_events" + __table_args__ = ( + CheckConstraint( + "event_type ~ '^[A-Za-z][A-Za-z0-9._:-]{0,127}$'", + name="event_type", + ), + CheckConstraint("event_version between 1 and 32767", name="event_version"), + CheckConstraint("producer = 'workstream'", name="producer"), + CheckConstraint( + "aggregate_type ~ '^[a-z][a-z0-9_]{0,63}$'", + name="aggregate_type", + ), + CheckConstraint( + "project_id ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'", + name="project_id", + ), + CheckConstraint( + "correlation_id ~ '^[A-Za-z0-9._:-]{1,200}$'", + name="correlation_id", + ), + CheckConstraint( + "idempotency_key ~ '^[A-Za-z0-9._:-]{1,200}$'", + name="idempotency_key", + ), + CheckConstraint( + "payload_digest ~ '^sha256:[0-9a-f]{64}$'", + name="payload_digest", + ), + CheckConstraint( + "jsonb_typeof(payload) = 'object' and octet_length(payload::text) <= 262144", + name="payload_shape", + ), + CheckConstraint( + "delivery_state in ('pending','claimed','retryable','acknowledged'," + "'dead_letter','cancelled')", + name="delivery_state", + ), + CheckConstraint( + "attempt_count >= 0 and claim_generation >= 0 " + "and attempt_count = claim_generation", + name="delivery_counters", + ), + CheckConstraint( + "claim_owner is null or claim_owner ~ '^[A-Za-z0-9._:-]{1,120}$'", + name="claim_owner", + ), + CheckConstraint( + "last_error_code is null or last_error_code ~ '^[A-Z][A-Z0-9_]{0,79}$'", + name="error_code", + ), + CheckConstraint( + "(next_attempt_at is null or next_attempt_at >= occurred_at) and " + "(claimed_at is null or claimed_at >= occurred_at) and " + "(last_attempt_at is null or last_attempt_at >= occurred_at) and " + "(claim_expires_at is null or claim_expires_at > claimed_at) and " + "(finalized_at is null or finalized_at >= occurred_at) and " + "(finalized_at is null or last_attempt_at is null or finalized_at >= last_attempt_at) and " + "(archived_at is null or archived_at >= finalized_at)", + name="delivery_timestamps", + ), + CheckConstraint( + "(delivery_state = 'pending' and attempt_count = 0 and next_attempt_at is not null " + "and claim_owner is null and claimed_at is null and claim_expires_at is null " + "and last_attempt_at is null and last_error_code is null and finalized_at is null " + "and archived_at is null) or " + "(delivery_state = 'claimed' and attempt_count > 0 and next_attempt_at is null " + "and claim_owner is not null and claimed_at is not null " + "and claim_expires_at is not null and last_attempt_at = claimed_at " + "and finalized_at is null and archived_at is null) or " + "(delivery_state = 'retryable' and attempt_count > 0 " + "and next_attempt_at is not null and claim_owner is null and claimed_at is null " + "and claim_expires_at is null and last_attempt_at is not null " + "and last_error_code is not null and finalized_at is null and archived_at is null) or " + "(delivery_state = 'acknowledged' and attempt_count > 0 " + "and next_attempt_at is null and claim_owner is null and claimed_at is null " + "and claim_expires_at is null and last_attempt_at is not null " + "and finalized_at is not null) or " + "(delivery_state = 'dead_letter' and attempt_count > 0 " + "and next_attempt_at is null and claim_owner is null and claimed_at is null " + "and claim_expires_at is null and last_attempt_at is not null " + "and last_error_code is not null and finalized_at is not null) or " + "(delivery_state = 'cancelled' and next_attempt_at is null and claim_owner is null " + "and claimed_at is null and claim_expires_at is null and finalized_at is not null " + "and ((attempt_count = 0 and last_attempt_at is null and last_error_code is null) " + "or (attempt_count > 0 and last_attempt_at is not null)))", + name="delivery_state_shape", + ), + Index( + "ix_outbox_events_eligible", + "event_type", + "delivery_state", + "next_attempt_at", + "occurred_at", + "event_id", + postgresql_where=text("delivery_state in ('pending','retryable')"), + ), + Index( + "ix_outbox_events_expired_claims", + "claim_expires_at", + "event_id", + postgresql_where=text("delivery_state = 'claimed'"), + ), + Index( + "ix_outbox_events_project_drain", + "project_id", + "delivery_state", + "occurred_at", + "event_id", + ), + Index( + "ix_outbox_events_retention", + "finalized_at", + "event_id", + postgresql_where=text( + "delivery_state in ('acknowledged','dead_letter','cancelled') " + "and archived_at is null" + ), + ), + Index( + "ix_outbox_events_aggregate", + "aggregate_type", + "aggregate_id", + "occurred_at", + "event_id", + ), + ) + + event_id: Mapped[UUID] = mapped_column(Uuid(), primary_key=True) + event_type: Mapped[str] = mapped_column(String(128), nullable=False) + event_version: Mapped[int] = mapped_column(SmallInteger, nullable=False) + producer: Mapped[str] = mapped_column( + String(32), nullable=False, server_default=text("'workstream'") + ) + aggregate_type: Mapped[str] = mapped_column(String(64), nullable=False) + aggregate_id: Mapped[UUID] = mapped_column(Uuid(), nullable=False) + project_id: Mapped[str] = mapped_column(ForeignKey("projects.id"), nullable=False) + correlation_id: Mapped[str] = mapped_column(String(200), nullable=False) + causation_event_id: Mapped[UUID | None] = mapped_column(Uuid()) + idempotency_key: Mapped[str] = mapped_column(String(200), nullable=False, unique=True) + payload: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False) + payload_digest: Mapped[str] = mapped_column(String(71), nullable=False) + occurred_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=text("statement_timestamp()") + ) + delivery_state: Mapped[str] = mapped_column( + String(16), nullable=False, server_default=text("'pending'") + ) + attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0")) + next_attempt_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, server_default=text("statement_timestamp()") + ) + claim_owner: Mapped[str | None] = mapped_column(String(120)) + claim_generation: Mapped[int] = mapped_column( + BigInteger, nullable=False, server_default=text("0") + ) + claimed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + claim_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_error_code: Mapped[str | None] = mapped_column(String(80)) + finalized_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + archived_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) diff --git a/backend/app/modules/outbox/repository.py b/backend/app/modules/outbox/repository.py new file mode 100644 index 000000000..5a6245f44 --- /dev/null +++ b/backend/app/modules/outbox/repository.py @@ -0,0 +1,74 @@ +"""Caller-transaction persistence for shared outbox append and replay.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlalchemy import or_, select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from app.modules.outbox.models import OutboxEvent +from app.modules.outbox.schemas import OutboxAppendInput + + +@dataclass(frozen=True, slots=True) +class OutboxReservation: + """Rows locked for one completed insert reservation or replay decision.""" + + created: bool + records: tuple[OutboxEvent, ...] + + +class OutboxRepository: + """Reserve and lock event identities without taking transaction ownership.""" + + def __init__(self, session: AsyncSession) -> None: + """Bind all persistence to the caller's exact session.""" + self._session = session + + async def reserve( + self, + value: OutboxAppendInput, + *, + payload_digest: str, + ) -> OutboxReservation: + """Insert a complete event or lock every conflicting identity in UUID order.""" + created_id = await self._session.scalar( + insert(OutboxEvent) + .values( + event_id=value.event_id, + event_type=value.event_type, + event_version=value.event_version, + aggregate_type=value.aggregate_type, + aggregate_id=value.aggregate_id, + project_id=str(value.project_id), + correlation_id=value.correlation_id, + causation_event_id=value.causation_event_id, + idempotency_key=value.idempotency_key, + payload=value.payload, + payload_digest=payload_digest, + ) + .on_conflict_do_nothing() + .returning(OutboxEvent.event_id) + ) + await self._session.flush() + records = tuple( + ( + await self._session.scalars( + select(OutboxEvent) + .where( + or_( + OutboxEvent.event_id == value.event_id, + OutboxEvent.idempotency_key == value.idempotency_key, + ) + ) + .order_by(OutboxEvent.event_id) + .with_for_update() + .execution_options(populate_existing=True) + ) + ).all() + ) + if not records: + raise RuntimeError("outbox_reservation_not_visible") + return OutboxReservation(created=created_id is not None, records=records) diff --git a/backend/app/modules/outbox/schemas.py b/backend/app/modules/outbox/schemas.py new file mode 100644 index 000000000..aace7f73b --- /dev/null +++ b/backend/app/modules/outbox/schemas.py @@ -0,0 +1,255 @@ +"""Strict feature-neutral inputs and results for shared outbox append.""" + +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum +import re +from typing import Any, NoReturn, Self +from uuid import UUID + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + GetCoreSchemaHandler, + field_validator, + model_validator, +) +from pydantic_core import SchemaValidator, core_schema + +_KEY = re.compile(r"^[a-z][a-z0-9_]{0,127}$") +_SECRET_KEYS = frozenset( + { + "access_token", + "artifact_bytes", + "authorization", + "bearer_token", + "cookie", + "credentials", + "id_token", + "jwks", + "password", + "raw_callback", + "raw_provider_response", + "refresh_token", + "request_body", + "response_body", + "secret", + "signed_url", + } +) +_SECRET_KEY_PARTS = frozenset( + {"credential", "credentials", "password", "secret", "token"} +) +_COMPACT_SECRET_LEXEMES = ( + "credential", + "jwt", + "passphrase", + "password", + "secret", + "token", +) +_COMPACT_KEY_LEXEMES = ( + "accesskey", + "apikey", + "encryptionkey", + "keymaterial", + "privatekey", + "providerkey", + "secretkey", + "signingkey", +) +_KEY_QUALIFIERS = frozenset( + {"access", "api", "encryption", "private", "provider", "secret", "signing"} +) +_KEY_DESCRIPTORS = frozenset({"data", "material", "payload", "value"}) +_MAX_DEPTH = 16 +_MAX_MEMBERS = 1024 +_MAX_NODES = 4096 +_MAX_KEY_BYTES = 128 +_MAX_STRING_BYTES = 16_384 +_MAX_ENCODING_BUDGET = 262_144 +_MAX_INTEGER_MAGNITUDE = 10**38 - 1 + + +class OutboxInputError(TypeError): + """Raised without payload details when append input is invalid.""" + + +class OutboxIdempotencyConflict(RuntimeError): + """Raised without payload details when either event identity drifts.""" + + +class OutboxPersistenceError(RuntimeError): + """Raised without statement parameters when database persistence fails.""" + + +class OutboxAppendDisposition(StrEnum): + """Closed caller-visible outcomes for append or exact replay.""" + + CREATED = "created" + REPLAYED = "replayed" + + +def _raise_input_error() -> NoReturn: + """Raise the public input error from a payload-free frame.""" + raise OutboxInputError("outbox_invalid_input") + + +def _is_sensitive_key(value: str) -> bool: + """Reject explicit and conventionally secret-bearing JSON field names.""" + normalized = value.casefold().replace("-", "_") + ordered_parts = tuple(normalized.split("_")) + parts = frozenset(ordered_parts) + key_compound = any( + part == "key" + and ( + (index > 0 and ordered_parts[index - 1] in _KEY_QUALIFIERS) + or ( + index + 1 < len(ordered_parts) + and ordered_parts[index + 1] in _KEY_DESCRIPTORS + ) + ) + for index, part in enumerate(ordered_parts) + ) + return ( + normalized in _SECRET_KEYS + or bool(parts & _SECRET_KEY_PARTS) + or normalized == "key" + or normalized.endswith("_key") + or key_compound + or any(lexeme in normalized for lexeme in _COMPACT_SECRET_LEXEMES) + or any(lexeme in normalized for lexeme in _COMPACT_KEY_LEXEMES) + ) + + +def _encoding_budget(value: object, *, depth: int, nodes: list[int]) -> int: + """Return a conservative canonical UTF-8 size bound while validating JSON.""" + nodes[0] += 1 + if nodes[0] > _MAX_NODES: + raise ValueError("payload_nodes") + if type(value) is dict: + if depth > _MAX_DEPTH or len(value) > _MAX_MEMBERS: + raise ValueError("payload_container") + budget = 2 + max(0, len(value) - 1) + for key, item in value.items(): + if type(key) is not str: + raise ValueError("payload_key") + if _is_sensitive_key(key): + raise ValueError("payload_sensitive") + key_bytes = key.encode("utf-8") + if len(key_bytes) > _MAX_KEY_BYTES or _KEY.fullmatch(key) is None: + raise ValueError("payload_key") + budget += (6 * len(key_bytes)) + 3 + budget += _encoding_budget(item, depth=depth + 1, nodes=nodes) + return budget + if type(value) is list: + if depth > _MAX_DEPTH or len(value) > _MAX_MEMBERS: + raise ValueError("payload_container") + return 2 + max(0, len(value) - 1) + sum( + _encoding_budget(item, depth=depth + 1, nodes=nodes) for item in value + ) + if type(value) is str: + encoded = value.encode("utf-8") + if len(encoded) > _MAX_STRING_BYTES: + raise ValueError("payload_string") + return (6 * len(encoded)) + 2 + if value is None: + return 4 + if type(value) is bool: + return 5 + if type(value) is int: + if abs(value) > _MAX_INTEGER_MAGNITUDE: + raise ValueError("payload_integer") + return len(str(value)) + raise ValueError("payload_type") + + +def validate_outbox_payload(value: object) -> dict[str, Any]: + """Validate generic structure, privacy, and resource bounds without encoding.""" + if type(value) is not dict: + raise ValueError("payload_object") + if _encoding_budget(value, depth=1, nodes=[0]) > _MAX_ENCODING_BUDGET: + raise ValueError("payload_size") + return value + + +class OutboxAppendInput(BaseModel): + """Immutable logical event facts accepted by the append participant.""" + + model_config = ConfigDict( + extra="forbid", + frozen=True, + hide_input_in_errors=True, + strict=True, + ) + + event_id: UUID + event_type: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9._:-]{0,127}$") + event_version: int = Field(ge=1, le=32767) + aggregate_type: str = Field(pattern=r"^[a-z][a-z0-9_]{0,63}$") + aggregate_id: UUID + project_id: UUID + correlation_id: str = Field(pattern=r"^[A-Za-z0-9._:-]{1,200}$") + causation_event_id: UUID | None = None + idempotency_key: str = Field(pattern=r"^[A-Za-z0-9._:-]{1,200}$") + payload: dict[str, Any] + + @field_validator("payload", mode="before") + @classmethod + def validate_payload_object(cls, value: object) -> object: + """Reject mapping subclasses before Pydantic traverses their overrides.""" + if type(value) is not dict: + raise ValueError("payload_object") + return value + + @classmethod + def __get_pydantic_core_schema__( + cls, + source_type: Any, + handler: GetCoreSchemaHandler, + ) -> core_schema.CoreSchema: + """Apply payload-free failure translation at the Pydantic core boundary.""" + inner_schema = handler(source_type) + string_validator = SchemaValidator(inner_schema) + + def admit( + value: object, + validator: core_schema.ValidatorFunctionWrapHandler, + info: core_schema.ValidationInfo, + ) -> object: + """Preserve validation mode while detaching every rejected input.""" + admitted = None + try: + admitted = ( + string_validator.validate_strings(value) + if info.mode == "string" + else validator(value) + ) + except Exception: # noqa: BLE001 - rejected input must not escape diagnostics + admitted = None + if admitted is None: + value = None + validator = None + _raise_input_error() + return admitted + + return core_schema.with_info_wrap_validator_function(admit, inner_schema) + + @model_validator(mode="after") + def validate_payload(self) -> Self: + """Reject noncanonical, sensitive, or unbounded generic payloads.""" + validate_outbox_payload(self.payload) + return self + + +class OutboxAppendResult(BaseModel): + """Minimal immutable append result; operational state is not exposed.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + event_id: UUID + disposition: OutboxAppendDisposition + payload_digest: str = Field(pattern=r"^sha256:[0-9a-f]{64}$") + occurred_at: datetime diff --git a/backend/app/modules/outbox/service.py b/backend/app/modules/outbox/service.py new file mode 100644 index 000000000..aaed2cae2 --- /dev/null +++ b/backend/app/modules/outbox/service.py @@ -0,0 +1,120 @@ +"""Feature-neutral shared outbox append participant.""" + +from __future__ import annotations + +from typing import NoReturn + +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.hashing import canonical_json_hash +from app.modules.outbox.models import OutboxEvent +from app.modules.outbox.repository import OutboxRepository +from app.modules.outbox.schemas import ( + OutboxAppendDisposition, + OutboxAppendInput, + OutboxAppendResult, + OutboxIdempotencyConflict, + OutboxInputError, + OutboxPersistenceError, +) + + +def _raise_input_error() -> NoReturn: + """Raise one stable input error from a payload-free frame.""" + raise OutboxInputError("outbox_invalid_input") + + +def _raise_persistence_error() -> NoReturn: + """Raise one stable persistence error from a payload-free frame.""" + raise OutboxPersistenceError("outbox_persistence_failed") + + +def _raise_idempotency_conflict() -> NoReturn: + """Raise one stable conflict from a payload-free frame.""" + raise OutboxIdempotencyConflict("outbox_idempotency_conflict") + + +def _validated_input(value: object) -> OutboxAppendInput: + """Revalidate a typed input without reflecting payload details in failures.""" + fields: dict[str, object] | None = None + validated: OutboxAppendInput | None = None + try: + fields = dict(object.__getattribute__(value, "__dict__")) + validated = OutboxAppendInput.model_validate(fields) + except Exception: # noqa: BLE001 - rejected values must not escape diagnostics + pass + if validated is None: + value = None + fields = None + _raise_input_error() + return validated.model_copy(deep=True) + + +def _matches(record: OutboxEvent, value: OutboxAppendInput, digest: str) -> bool: + """Compare every immutable caller fact while ignoring operational metadata.""" + return ( + record.event_id == value.event_id + and record.event_type == value.event_type + and record.event_version == value.event_version + and record.producer == "workstream" + and record.aggregate_type == value.aggregate_type + and record.aggregate_id == value.aggregate_id + and record.project_id == str(value.project_id) + and record.correlation_id == value.correlation_id + and record.causation_event_id == value.causation_event_id + and record.idempotency_key == value.idempotency_key + and record.payload_digest == digest + and record.payload == value.payload + ) + + +class OutboxService: + """Append immutable events by flushing only the caller-owned transaction.""" + + def __init__(self, session: AsyncSession) -> None: + """Bind the participant to one caller session; never commit or publish.""" + self._repository = OutboxRepository(session) + + async def append(self, value: OutboxAppendInput) -> OutboxAppendResult: + """Create one event or return its exact idempotent replay.""" + validated = None + try: + validated = _validated_input(value) + except OutboxInputError: + pass + if validated is None: + del value + _raise_input_error() + digest: str | None = None + try: + digest = canonical_json_hash(validated.payload) + except (TypeError, ValueError): + pass + if digest is None: + del value, validated + _raise_input_error() + reservation = None + try: + reservation = await self._repository.reserve(validated, payload_digest=digest) + except SQLAlchemyError: + pass + if reservation is None: + del value, validated + _raise_persistence_error() + if len(reservation.records) != 1 or not _matches( + reservation.records[0], validated, digest + ): + del value, validated, reservation + _raise_idempotency_conflict() + record = reservation.records[0] + return OutboxAppendResult( + event_id=record.event_id, + disposition=( + OutboxAppendDisposition.CREATED + if reservation.created + else OutboxAppendDisposition.REPLAYED + ), + payload_digest=record.payload_digest, + occurred_at=record.occurred_at, + ) diff --git a/backend/tests/assertion_helpers.py b/backend/tests/assertion_helpers.py index 23cc2fbfc..6ae57f5b4 100644 --- a/backend/tests/assertion_helpers.py +++ b/backend/tests/assertion_helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Mapping +from dataclasses import fields, is_dataclass from pydantic import SecretStr, ValidationError @@ -21,7 +22,7 @@ def assert_secret_not_retained( return seen.add(id(value)) if isinstance(value, str): - assert secret not in value + assert not str.__contains__(value, secret) elif isinstance(value, (bytes, bytearray, memoryview)): assert secret.encode("utf-8") not in bytes(value) elif isinstance(value, SecretStr): @@ -52,8 +53,36 @@ def assert_secret_not_retained( traceback_module_prefixes=traceback_module_prefixes, ) traceback = traceback.tb_next + elif issubclass(type(value), dict): + # Snapshot before recursive inspection: traversing nested framework state + # can lazily import modules and mutate a module-globals dictionary. + for key, item in tuple(dict.items(value)): + assert_secret_not_retained( + key, + secret, + seen, + traceback_module_prefixes=traceback_module_prefixes, + ) + assert_secret_not_retained( + item, + secret, + seen, + traceback_module_prefixes=traceback_module_prefixes, + ) elif isinstance(value, Mapping): - for key, item in value.items(): + try: + items = tuple(value.items()) + except Exception: + state = getattr(value, "__dict__", None) + assert isinstance(state, dict), "mapping state could not be inspected" + assert_secret_not_retained( + state, + secret, + seen, + traceback_module_prefixes=traceback_module_prefixes, + ) + return + for key, item in items: assert_secret_not_retained( key, secret, @@ -66,6 +95,46 @@ def assert_secret_not_retained( seen, traceback_module_prefixes=traceback_module_prefixes, ) + elif isinstance(value, list): + for item in list.__iter__(value): + assert_secret_not_retained( + item, + secret, + seen, + traceback_module_prefixes=traceback_module_prefixes, + ) + elif isinstance(value, tuple): + for item in tuple.__iter__(value): + assert_secret_not_retained( + item, + secret, + seen, + traceback_module_prefixes=traceback_module_prefixes, + ) + elif isinstance(value, set): + for item in set.__iter__(value): + assert_secret_not_retained( + item, + secret, + seen, + traceback_module_prefixes=traceback_module_prefixes, + ) + elif isinstance(value, frozenset): + for item in frozenset.__iter__(value): + assert_secret_not_retained( + item, + secret, + seen, + traceback_module_prefixes=traceback_module_prefixes, + ) + elif is_dataclass(value) and not isinstance(value, type): + for field in fields(value): + assert_secret_not_retained( + object.__getattribute__(value, field.name), + secret, + seen, + traceback_module_prefixes=traceback_module_prefixes, + ) elif isinstance(getattr(value, "__dict__", None), Mapping): assert_secret_not_retained( vars(value), @@ -88,11 +157,3 @@ def assert_secret_not_retained( seen, traceback_module_prefixes=traceback_module_prefixes, ) - elif isinstance(value, (list, tuple, set)): - for item in value: - assert_secret_not_retained( - item, - secret, - seen, - traceback_module_prefixes=traceback_module_prefixes, - ) diff --git a/backend/tests/test_alembic.py b/backend/tests/test_alembic.py index 20547f0ea..c42ae43a6 100644 --- a/backend/tests/test_alembic.py +++ b/backend/tests/test_alembic.py @@ -119,6 +119,106 @@ def test_alembic_upgrade_and_downgrade(isolated_database_env: str, migration_loc command.downgrade(config, "base") +def test_outbox_migration_schema_and_downgrade_writer_guard( + isolated_database_env: str, + migration_lock, +) -> None: + """Prove exact 0029 schema plus ACCESS EXCLUSIVE commit/rollback behavior.""" + config = _alembic_config() + committed_project_id = str(uuid4()) + rolled_back_project_id = str(uuid4()) + with migration_lock(): + try: + command.downgrade(config, "base") + command.upgrade(config, "head") + schema = asyncio.run(_outbox_schema(isolated_database_env)) + assert schema == { + "revision": "0029_shared_transactional_outbox", + "columns": { + "aggregate_id", + "aggregate_type", + "archived_at", + "attempt_count", + "causation_event_id", + "claim_expires_at", + "claim_generation", + "claim_owner", + "claimed_at", + "correlation_id", + "delivery_state", + "event_id", + "event_type", + "event_version", + "finalized_at", + "idempotency_key", + "last_attempt_at", + "last_error_code", + "next_attempt_at", + "occurred_at", + "payload", + "payload_digest", + "producer", + "project_id", + }, + "nullable": { + "next_attempt_at", + "causation_event_id", + "claim_owner", + "claimed_at", + "claim_expires_at", + "last_attempt_at", + "last_error_code", + "finalized_at", + "archived_at", + }, + "indexes": { + "ix_outbox_events_aggregate", + "ix_outbox_events_eligible", + "ix_outbox_events_expired_claims", + "ix_outbox_events_project_drain", + "ix_outbox_events_retention", + "pk_outbox_events", + "uq_outbox_events_idempotency_key", + }, + "triggers": {"outbox_events_custody", "outbox_events_reject_truncate"}, + } + + committed = asyncio.run( + _outbox_downgrade_writer_race( + isolated_database_env, + config, + project_id=committed_project_id, + commit_writer=True, + ) + ) + assert committed == "refused_after_commit" + assert asyncio.run(_current_revision(isolated_database_env)) == ( + "0029_shared_transactional_outbox" + ) + asyncio.run(_remove_outbox_migration_row(isolated_database_env, committed_project_id)) + command.downgrade(config, "0028_artifact_admission") + assert "outbox_events" not in asyncio.run(_fetch_table_names(isolated_database_env)) + + command.upgrade(config, "head") + rolled_back = asyncio.run( + _outbox_downgrade_writer_race( + isolated_database_env, + config, + project_id=rolled_back_project_id, + commit_writer=False, + ) + ) + assert rolled_back == "succeeded_after_rollback" + assert asyncio.run(_current_revision(isolated_database_env)) == ( + "0028_artifact_admission" + ) + finally: + command.upgrade(config, "head") + asyncio.run(_remove_outbox_migration_row(isolated_database_env, committed_project_id)) + asyncio.run(_remove_outbox_migration_row(isolated_database_env, rolled_back_project_id)) + command.downgrade(config, "base") + + def test_current_schema_uses_project_policy_contract( isolated_database_env: str, migration_lock, @@ -163,6 +263,9 @@ def test_current_schema_uses_project_policy_contract( "artifact_replicas.provider_object_ref", "artifact_replicas.storage_namespace_id", "artifact_operation_receipts.request_digest", + "outbox_events.event_id", + "outbox_events.payload_digest", + "outbox_events.delivery_state", }.issubset(columns) discarded_columns = { "projects.base_amount", @@ -1818,7 +1921,7 @@ def test_actor_profile_lifecycle_downgrade_refuses_forward_evidence( isolated_database_env: str, migration_lock, ) -> None: - """Keep 0026 intact for every profile, link, or denial evidence branch.""" + """Keep the full head intact for every lifecycle-evidence branch.""" config = _alembic_config() actor_id = str(uuid4()) @@ -2530,6 +2633,139 @@ def test_authority_idempotency_schema_preserves_audit_and_guards_downgrade( assert preserved == {"revision": "0018_authority_audit_evidence", "records": None, "orphan": 1} +async def _outbox_schema(database_url: str) -> dict[str, object]: + """Return the exact shared-outbox migration surface.""" + engine = create_async_engine(database_url) + try: + async with engine.connect() as connection: + column_rows = ( + await connection.execute( + text( + "select column_name, is_nullable from information_schema.columns " + "where table_schema='public' and table_name='outbox_events'" + ) + ) + ).all() + indexes = set( + ( + await connection.scalars( + text( + "select indexname from pg_indexes " + "where schemaname='public' and tablename='outbox_events'" + ) + ) + ).all() + ) + triggers = set( + ( + await connection.scalars( + text( + "select tgname from pg_trigger " + "where tgrelid='outbox_events'::regclass and not tgisinternal" + ) + ) + ).all() + ) + return { + "revision": str( + await connection.scalar(text("select version_num from alembic_version")) + ), + "columns": {row.column_name for row in column_rows}, + "nullable": { + row.column_name for row in column_rows if row.is_nullable == "YES" + }, + "indexes": indexes, + "triggers": triggers, + } + finally: + await engine.dispose() + + +async def _outbox_downgrade_writer_race( + database_url: str, + config: Config, + *, + project_id: str, + commit_writer: bool, +) -> str: + """Hold one append open while downgrade waits, then commit or roll it back.""" + engine = create_async_engine(database_url) + event_id = str(uuid4()) + try: + async with engine.connect() as connection: + transaction = await connection.begin() + await connection.execute( + text( + "insert into projects(id,name,slug,status) " + "values (:id,'Outbox migration',:slug,'active')" + ), + {"id": project_id, "slug": f"outbox-migration-{project_id}"}, + ) + await connection.execute( + text( + "insert into outbox_events " + "(event_id,event_type,event_version,aggregate_type,aggregate_id,project_id," + "correlation_id,idempotency_key,payload,payload_digest) values " + "(:event_id,'MigrationProbe',1,'migration_probe',:aggregate_id,:project_id," + ":correlation_id,:idempotency_key,'{}'::jsonb,:digest)" + ), + { + "event_id": event_id, + "aggregate_id": str(uuid4()), + "project_id": project_id, + "correlation_id": f"migration:{event_id}", + "idempotency_key": f"migration:{event_id}:v1", + "digest": "sha256:" + ("0" * 64), + }, + ) + downgrade = asyncio.create_task( + asyncio.to_thread( + command.downgrade, + config, + "0028_artifact_admission", + ) + ) + await asyncio.sleep(0.1) + assert not downgrade.done() + if commit_writer: + await transaction.commit() + with pytest.raises(RuntimeError, match="cannot downgrade with shared outbox events"): + await asyncio.wait_for(downgrade, timeout=5) + return "refused_after_commit" + await transaction.rollback() + await asyncio.wait_for(downgrade, timeout=5) + return "succeeded_after_rollback" + finally: + await engine.dispose() + + +async def _remove_outbox_migration_row(database_url: str, project_id: str) -> None: + """Remove only migration-test truth under explicit disabled trigger custody.""" + engine = create_async_engine(database_url) + try: + async with engine.begin() as connection: + table_exists = await connection.scalar( + text("select to_regclass('public.outbox_events') is not null") + ) + if table_exists: + await connection.execute(text("alter table outbox_events disable trigger user")) + await connection.execute( + text("delete from outbox_events where project_id=:project_id"), + {"project_id": project_id}, + ) + await connection.execute(text("alter table outbox_events enable trigger user")) + project_exists = await connection.scalar( + text("select to_regclass('public.projects') is not null") + ) + if project_exists: + await connection.execute( + text("delete from projects where id=:project_id"), + {"project_id": project_id}, + ) + finally: + await engine.dispose() + + async def _fetch_columns(database_url: str) -> set[str]: """Return current public table columns as table.column names.""" engine = create_async_engine(database_url) diff --git a/backend/tests/test_assertion_helpers.py b/backend/tests/test_assertion_helpers.py index 54f8fe255..fd6128393 100644 --- a/backend/tests/test_assertion_helpers.py +++ b/backend/tests/test_assertion_helpers.py @@ -1,5 +1,7 @@ """Tests for shared security-sensitive assertion helpers.""" +from collections.abc import Iterator, Mapping +from dataclasses import dataclass from pydantic import SecretStr import pytest from types import SimpleNamespace @@ -29,3 +31,107 @@ def test_public_object_state_rejects_nested_forbidden_value() -> None: with pytest.raises(AssertionError): assert_secret_not_retained(value, "forbidden") + + +def test_builtin_container_subclasses_cannot_hide_forbidden_values() -> None: + """Inspect builtin storage without invoking hostile iteration overrides.""" + + class HostileDict(dict[str, str]): + def items(self): + raise RuntimeError("hostile items") + + class HostileList(list[str]): + def __iter__(self): + raise RuntimeError("hostile iterator") + + class HostileString(str): + def __contains__(self, item: object) -> bool: + raise RuntimeError("hostile membership") + + for value in ( + HostileDict(value="forbidden"), + HostileList(["forbidden"]), + HostileString("forbidden"), + ): + with pytest.raises(AssertionError): + assert_secret_not_retained(value, "forbidden") + + +def test_slotted_dataclass_state_rejects_forbidden_value() -> None: + """Inspect slotted reservation-style records without requiring __dict__.""" + + @dataclass(frozen=True, slots=True) + class SlottedRecord: + payload: str + + with pytest.raises(AssertionError): + assert_secret_not_retained(SlottedRecord(payload="forbidden"), "forbidden") + + +def test_raising_mapping_falls_back_to_inspecting_object_state() -> None: + """Fail closed on retained secrets when a framework mapping cannot iterate.""" + + class RaisingMapping(Mapping[str, str]): + def __init__(self, payload: str) -> None: + self.payload = payload + + def __getitem__(self, key: str) -> str: + raise RuntimeError(key) + + def __iter__(self) -> Iterator[str]: + raise RuntimeError("opaque framework mapping") + + def __len__(self) -> int: + return 1 + + with pytest.raises(AssertionError): + assert_secret_not_retained(RaisingMapping("forbidden"), "forbidden") + + +def test_nested_mapping_can_mutate_parent_dict_during_inspection() -> None: + """Snapshot parent entries before nested framework state mutates them.""" + parent: dict[str, object] = {} + + class ParentMutatingMapping(Mapping[str, str]): + def __getitem__(self, key: str) -> str: + raise KeyError(key) + + def __iter__(self) -> Iterator[str]: + return iter(()) + + def __len__(self) -> int: + return 0 + + def items(self): + parent["lazily_imported"] = "safe" + return ().__iter__() + + parent["framework_state"] = ParentMutatingMapping() + + assert_secret_not_retained(parent, "forbidden") + + +def test_mapping_proxy_that_spoofs_dict_uses_mapping_protocol() -> None: + """Do not apply built-in dict descriptors to framework proxy objects.""" + + class DictSpoofingProxy(Mapping[str, str]): + @property + def __class__(self): + return dict + + def __getitem__(self, key: str) -> str: + if key != "payload": + raise KeyError(key) + return "forbidden" + + def __iter__(self) -> Iterator[str]: + return iter(("payload",)) + + def __len__(self) -> int: + return 1 + + proxy = DictSpoofingProxy() + assert isinstance(proxy, dict) + + with pytest.raises(AssertionError): + assert_secret_not_retained(proxy, "forbidden") diff --git a/backend/tests/test_outbox.py b/backend/tests/test_outbox.py new file mode 100644 index 000000000..933fdc8c6 --- /dev/null +++ b/backend/tests/test_outbox.py @@ -0,0 +1,981 @@ +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +import json +from pathlib import Path +import traceback +from typing import Any, cast +from uuid import UUID, uuid4 + +import pytest +from alembic import command +from alembic.config import Config +from pydantic import TypeAdapter +from sqlalchemy import text +from sqlalchemy.exc import DBAPIError +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.modules.outbox.schemas import ( + OutboxAppendDisposition, + OutboxAppendInput, + OutboxIdempotencyConflict, + OutboxInputError, + OutboxPersistenceError, +) +from app.modules.outbox.repository import OutboxRepository +from app.modules.outbox.service import OutboxService +from tests.assertion_helpers import assert_secret_not_retained + + +def _alembic_config() -> Config: + backend_root = Path(__file__).resolve().parents[1] + config = Config(str(backend_root / "alembic.ini")) + config.set_main_option("script_location", str(backend_root / "alembic")) + return config + + +@pytest.fixture +def outbox_database_env( + isolated_database_env: str, + migration_lock, +) -> str: + """Upgrade the isolated database to the exact shared-outbox head.""" + with migration_lock(): + command.upgrade(_alembic_config(), "head") + return isolated_database_env + + +@pytest.fixture +async def outbox_factory( + outbox_database_env: str, +) -> AsyncIterator[tuple[async_sessionmaker[AsyncSession], UUID]]: + """Provide one project-scoped session factory and privileged local cleanup.""" + engine = create_async_engine(outbox_database_env) + factory = async_sessionmaker(engine, expire_on_commit=False) + project_id = uuid4() + async with engine.begin() as connection: + await connection.execute( + text( + "insert into projects(id, name, slug, status) " + "values (:id, 'Outbox test', :slug, 'active')" + ), + {"id": str(project_id), "slug": f"outbox-{project_id}"}, + ) + try: + yield factory, project_id + finally: + async with engine.begin() as connection: + await connection.execute(text("alter table outbox_events disable trigger user")) + await connection.execute( + text("delete from outbox_events where project_id=:project_id"), + {"project_id": str(project_id)}, + ) + await connection.execute(text("alter table outbox_events enable trigger user")) + await connection.execute( + text("delete from projects where id=:project_id"), + {"project_id": str(project_id)}, + ) + await engine.dispose() + + +def _event(project_id: UUID, **changes: Any) -> OutboxAppendInput: + values: dict[str, Any] = { + "event_id": uuid4(), + "event_type": "ContributionRecorded", + "event_version": 1, + "aggregate_type": "contribution_record", + "aggregate_id": uuid4(), + "project_id": project_id, + "correlation_id": f"request:{uuid4()}", + "causation_event_id": uuid4(), + "idempotency_key": f"contribution:{uuid4()}:recorded:v1", + "payload": {"contribution_record_id": str(uuid4()), "award_ids": []}, + } + values.update(changes) + return OutboxAppendInput(**values) + + +def _unsafe_event(project_id: UUID, payload: object) -> OutboxAppendInput: + valid = _event(project_id) + values = valid.model_dump() + values["payload"] = payload + return OutboxAppendInput.model_construct(**values) + + +def test_outbox_input_requires_closed_tokens_and_object_payload() -> None: + project_id = uuid4() + with pytest.raises(OutboxInputError, match="^outbox_invalid_input$"): + _event(project_id, event_type="bad event") + with pytest.raises(OutboxInputError, match="^outbox_invalid_input$"): + _event(project_id, aggregate_type="BadAggregate") + with pytest.raises(OutboxInputError, match="^outbox_invalid_input$"): + _event(project_id, payload=[]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "payload", + [ + {"Authorization": "Bearer secret"}, + {"access-token": "secret"}, + {"nested": {"refresh_token": "secret"}}, + {"api_key": "secret"}, + {"private_key": "secret"}, + {"client_secret": "secret"}, + {"provider_token": "secret"}, + {"session_token": "secret"}, + {"provider_credentials": "secret"}, + {"apikey": "secret"}, + {"privatekey": "secret"}, + {"clientsecret": "secret"}, + {"providertoken": "secret"}, + {"sessiontoken": "secret"}, + {"providercredential": "secret"}, + {"jwt": "secret"}, + {"passphrase": "secret"}, + {"providertokenvalue": "secret"}, + {"sessiontokendata": "secret"}, + {"clientsecretvalue": "secret"}, + {"providercredentialvalue": "secret"}, + {"passwordvalue": "secret"}, + {"jwtvalue": "secret"}, + {"keymaterial": "secret"}, + {"api_key_material": "secret"}, + {"private_key_material": "secret"}, + {"provider_key_value": "secret"}, + {"access_key_data": "secret"}, + {"encryption_key_payload": "secret"}, + {"signing_key_material": "secret"}, + {"ratio": 1.5}, + {"blob": b"secret"}, + {"huge_integer": 10**38}, + {"oversized": "x" * 16_385}, + {"BadKey": "value"}, + ], +) +async def test_outbox_invalid_payload_errors_never_echo_values( + payload: object, +) -> None: + service = OutboxService(cast(AsyncSession, None)) + with pytest.raises(OutboxInputError) as raised: + await service.append(_unsafe_event(uuid4(), payload)) + assert str(raised.value) == "outbox_invalid_input" + assert "secret" not in str(raised.value) + + +@pytest.mark.parametrize("key", ["hockey", "monkey", "turnkey"]) +def test_outbox_allows_benign_words_ending_in_key_letters(key: str) -> None: + """Keep ordinary generic payload names outside the credential-key policy.""" + value = _event(uuid4(), payload={key: "safe"}) + assert value.payload == {key: "safe"} + + +def test_outbox_normal_validation_detaches_rejected_secret_input() -> None: + values = _event(uuid4()).model_dump() + marker = f"private-marker-{uuid4()}" + values["payload"] = {"authorization": marker} + with pytest.raises(OutboxInputError, match="^outbox_invalid_input$") as raised: + OutboxAppendInput(**values) + rendered = "".join(traceback.format_exception(raised.value)) + assert marker not in rendered + assert_secret_not_retained( + raised.value, + marker, + traceback_module_prefixes=("app.modules.outbox",), + ) + assert raised.value.__context__ is None + assert raised.value.__cause__ is None + + +def test_outbox_all_validation_entry_points_detach_rejected_input() -> None: + marker = f"private-marker-{uuid4()}" + + class ExplodingDict(dict[str, object]): + def items(self): + raise RuntimeError(marker) + + class ExplodingList(list[object]): + def __iter__(self): + raise RuntimeError(marker) + + class ExplodingString(str): + def encode(self, *args: object, **kwargs: object): + raise RuntimeError(marker) + + adapter = TypeAdapter(OutboxAppendInput) + hostile_payloads = ( + {"nested": ExplodingDict({"value": "safe"})}, + {"nested": ExplodingList(["safe"])}, + {"nested": ExplodingString("safe")}, + ) + for payload in hostile_payloads: + values = _event(uuid4()).model_dump() + values["payload"] = payload + calls = ( + lambda: OutboxAppendInput(**values), + lambda: OutboxAppendInput.model_validate(values), + lambda: OutboxAppendInput.model_validate_strings(values), + lambda: adapter.validate_python(values), + ) + for call in calls: + with pytest.raises( + OutboxInputError, + match="^outbox_invalid_input$", + ) as raised: + call() + assert_secret_not_retained( + raised.value, + marker, + traceback_module_prefixes=("app.modules.outbox",), + ) + assert raised.value.__context__ is None + assert raised.value.__cause__ is None + + json_values = _event(uuid4()).model_dump(mode="json") + json_values["payload"] = {"authorization": marker} + document = json.dumps(json_values) + for call in ( + lambda: OutboxAppendInput.model_validate_json(document), + lambda: adapter.validate_json(document), + ): + with pytest.raises(OutboxInputError, match="^outbox_invalid_input$") as raised: + call() + assert_secret_not_retained( + raised.value, + marker, + traceback_module_prefixes=("app.modules.outbox",), + ) + + +def test_outbox_rejects_hostile_top_level_payload_before_traversal() -> None: + marker = "top-level-payload-override-must-not-run" + calls: list[str] = [] + + class ExplodingDict(dict[str, object]): + def items(self): + calls.append(marker) + raise RuntimeError(marker) + + adapter = TypeAdapter(OutboxAppendInput) + values = _event(uuid4()).model_dump() + values["payload"] = ExplodingDict(value="safe") + string_values = _event(uuid4()).model_dump(mode="json") + string_values["event_version"] = str(string_values["event_version"]) + string_values["payload"] = ExplodingDict(value="safe") + for call in ( + lambda: OutboxAppendInput(**values), + lambda: OutboxAppendInput.model_validate(values), + lambda: OutboxAppendInput.model_validate_strings(string_values), + lambda: adapter.validate_python(values), + lambda: adapter.validate_strings(string_values), + ): + with pytest.raises(OutboxInputError, match="^outbox_invalid_input$"): + call() + assert calls == [] + + +@pytest.mark.asyncio +async def test_outbox_service_detaches_hostile_nested_container_failure() -> None: + marker = f"private-marker-{uuid4()}" + + class ExplodingDict(dict[str, object]): + def items(self): + raise RuntimeError(marker) + + class ExplodingList(list[object]): + def __iter__(self): + raise RuntimeError(marker) + + class ExplodingString(str): + def encode(self, *args: object, **kwargs: object): + raise RuntimeError(marker) + + for payload in ( + {"nested": ExplodingDict({"value": "safe"})}, + {"nested": ExplodingList(["safe"])}, + {"nested": ExplodingString("safe")}, + ): + value = _unsafe_event(uuid4(), payload) + with pytest.raises(OutboxInputError, match="^outbox_invalid_input$") as raised: + await OutboxService(cast(AsyncSession, None)).append(value) + assert_secret_not_retained( + raised.value, + marker, + traceback_module_prefixes=("app.modules.outbox",), + ) + assert raised.value.__context__ is None + assert raised.value.__cause__ is None + + +def test_outbox_validation_entry_points_preserve_valid_modes() -> None: + expected = _event(uuid4(), payload={"marker": "safe"}) + adapter = TypeAdapter(OutboxAppendInput) + python_value = OutboxAppendInput.model_validate(expected.model_dump()) + json_value = OutboxAppendInput.model_validate_json( + json.dumps(expected.model_dump(mode="json")) + ) + strings = expected.model_dump(mode="json") + strings["event_version"] = str(strings["event_version"]) + string_value = OutboxAppendInput.model_validate_strings(strings) + adapter_python = adapter.validate_python(expected.model_dump()) + adapter_json = adapter.validate_json(json.dumps(expected.model_dump(mode="json"))) + adapter_strings = adapter.validate_strings(strings) + assert all( + value == expected + for value in ( + python_value, + json_value, + string_value, + adapter_python, + adapter_json, + adapter_strings, + ) + ) + + +@pytest.mark.asyncio +async def test_outbox_payload_depth_nodes_members_and_budget_are_bounded() -> None: + project_id = uuid4() + nested: dict[str, Any] = {} + cursor = nested + for _ in range(17): + cursor["nested"] = {} + cursor = cursor["nested"] + cases = ( + nested, + {"items": list(range(1025))}, + {f"key_{index}": index for index in range(1025)}, + {"items": [[index] for index in range(4096)]}, + {"one": "x" * 16_000, "two": "y" * 16_000, "three": "z" * 16_000}, + ) + for payload in cases: + with pytest.raises(OutboxInputError, match="^outbox_invalid_input$"): + await OutboxService(cast(AsyncSession, None)).append( + _unsafe_event(project_id, payload) + ) + + +@pytest.mark.asyncio +async def test_outbox_append_flushes_pending_event_without_committing( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + value = _event(project_id) + async with factory() as session: + async with session.begin(): + result = await OutboxService(session).append(value) + row = ( + await session.execute( + text( + "select producer, aggregate_type, aggregate_id, payload, " + "payload_digest, delivery_state, attempt_count, claim_generation, " + "occurred_at, next_attempt_at from outbox_events where event_id=:id" + ), + {"id": value.event_id}, + ) + ).one() + assert result.disposition is OutboxAppendDisposition.CREATED + assert row.producer == "workstream" + assert row.aggregate_type == value.aggregate_type + assert row.aggregate_id == value.aggregate_id + assert row.payload == value.payload + assert row.payload_digest == result.payload_digest + assert row.delivery_state == "pending" + assert row.attempt_count == row.claim_generation == 0 + assert row.occurred_at == row.next_attempt_at == result.occurred_at + + +@pytest.mark.asyncio +async def test_outbox_insert_trigger_rejects_preforged_operational_state( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + event_id = uuid4() + async with factory() as session: + async with session.begin(): + await session.execute( + text( + "insert into outbox_events " + "(event_id,event_type,event_version,producer,aggregate_type,aggregate_id," + "project_id,correlation_id,idempotency_key,payload,payload_digest,occurred_at," + "delivery_state,attempt_count,next_attempt_at,claim_owner,claim_generation," + "claimed_at,claim_expires_at,last_attempt_at,last_error_code,finalized_at," + "archived_at) values " + "(:event_id,'ForgedProbe',1,'forged','forged_probe',:aggregate_id,:project_id," + ":correlation_id,:idempotency_key,'{}'::jsonb,:digest,'2000-01-01Z'," + "'acknowledged',99,null,'forged:worker',99,'2000-01-01Z','2099-01-01Z'," + "'2000-01-01Z','FORGED','2000-01-01Z','2000-01-01Z')" + ), + { + "event_id": event_id, + "aggregate_id": uuid4(), + "project_id": str(project_id), + "correlation_id": f"forged:{event_id}", + "idempotency_key": f"forged:{event_id}:v1", + "digest": "sha256:" + ("0" * 64), + }, + ) + row = ( + await session.execute( + text( + "select producer,occurred_at,delivery_state,attempt_count," + "next_attempt_at,claim_owner,claim_generation,claimed_at," + "claim_expires_at,last_attempt_at,last_error_code,finalized_at,archived_at " + "from outbox_events where event_id=:event_id" + ), + {"event_id": event_id}, + ) + ).one() + assert row.producer == "workstream" + assert row.occurred_at == row.next_attempt_at + assert row.occurred_at.year >= 2026 + assert row.delivery_state == "pending" + assert row.attempt_count == row.claim_generation == 0 + assert all( + value is None + for value in ( + row.claim_owner, + row.claimed_at, + row.claim_expires_at, + row.last_attempt_at, + row.last_error_code, + row.finalized_at, + row.archived_at, + ) + ) + + +@pytest.mark.asyncio +async def test_outbox_caller_rollback_removes_flushed_event( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + value = _event(project_id) + async with factory() as session: + transaction = await session.begin() + await OutboxService(session).append(value) + assert await session.scalar( + text("select count(*) from outbox_events where event_id=:id"), + {"id": value.event_id}, + ) == 1 + await transaction.rollback() + async with factory() as observer: + assert await observer.scalar( + text("select count(*) from outbox_events where event_id=:id"), + {"id": value.event_id}, + ) == 0 + + +@pytest.mark.asyncio +async def test_outbox_post_reservation_failure_rolls_back_caller_transaction( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], + monkeypatch: pytest.MonkeyPatch, +) -> None: + factory, project_id = outbox_factory + value = _event(project_id) + original_reserve = OutboxRepository.reserve + + async def fail_after_reservation( + repository: OutboxRepository, + event: OutboxAppendInput, + *, + payload_digest: str, + ) -> None: + await original_reserve(repository, event, payload_digest=payload_digest) + raise RuntimeError("injected_outbox_failure") + + monkeypatch.setattr(OutboxRepository, "reserve", fail_after_reservation) + async with factory() as session: + transaction = await session.begin() + with pytest.raises(RuntimeError, match="^injected_outbox_failure$"): + await OutboxService(session).append(value) + await transaction.rollback() + async with factory() as observer: + assert await observer.scalar( + text("select count(*) from outbox_events where event_id=:id"), + {"id": value.event_id}, + ) == 0 + + +@pytest.mark.asyncio +async def test_outbox_database_error_never_reflects_payload( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, _ = outbox_factory + marker = f"private-marker-{uuid4()}" + value = _event(uuid4(), payload={"private_marker": marker}) + async with factory() as session: + transaction = await session.begin() + with pytest.raises( + OutboxPersistenceError, + match="^outbox_persistence_failed$", + ) as raised: + await OutboxService(session).append(value) + rendered = "".join(traceback.format_exception(raised.value)) + assert marker not in rendered + assert_secret_not_retained( + raised.value, + marker, + traceback_module_prefixes=("app.modules.outbox",), + ) + assert raised.value.__context__ is None + assert raised.value.__cause__ is None + await transaction.rollback() + + +@pytest.mark.asyncio +async def test_outbox_snapshots_nested_payload_before_first_await( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], + monkeypatch: pytest.MonkeyPatch, +) -> None: + factory, project_id = outbox_factory + value = _event(project_id, payload={"nested": {"state": "before"}}) + reserve_started = asyncio.Event() + release_reserve = asyncio.Event() + original_reserve = OutboxRepository.reserve + + async def blocked_reserve( + repository: OutboxRepository, + event: OutboxAppendInput, + *, + payload_digest: str, + ): + reserve_started.set() + await release_reserve.wait() + return await original_reserve(repository, event, payload_digest=payload_digest) + + monkeypatch.setattr(OutboxRepository, "reserve", blocked_reserve) + async with factory() as session: + async with session.begin(): + append_task = asyncio.create_task(OutboxService(session).append(value)) + await reserve_started.wait() + value.payload["nested"]["state"] = "after" + release_reserve.set() + result = await append_task + row = ( + await session.execute( + text( + "select payload, payload_digest from outbox_events " + "where event_id=:event_id" + ), + {"event_id": value.event_id}, + ) + ).one() + assert result.disposition is OutboxAppendDisposition.CREATED + assert row.payload == {"nested": {"state": "before"}} + assert row.payload_digest == result.payload_digest + + +@pytest.mark.asyncio +async def test_outbox_exact_replay_uses_canonical_payload_and_original_time( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + value = _event( + project_id, + payload={"nested": {"second": 2, "first": 1}, "items": ["a", "b"]}, + ) + async with factory() as session: + async with session.begin(): + created = await OutboxService(session).append(value) + replay_payload = {"items": ["a", "b"], "nested": {"first": 1, "second": 2}} + replay = OutboxAppendInput(**{**value.model_dump(), "payload": replay_payload}) + async with factory() as session: + async with session.begin(): + result = await OutboxService(session).append(replay) + assert result.disposition is OutboxAppendDisposition.REPLAYED + assert result.event_id == created.event_id + assert result.payload_digest == created.payload_digest + assert result.occurred_at == created.occurred_at + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "field", + [ + "event_id", + "event_type", + "event_version", + "aggregate_type", + "aggregate_id", + "project_id", + "correlation_id", + "causation_event_id", + "idempotency_key", + "payload", + ], +) +async def test_outbox_reused_identity_with_immutable_drift_conflicts( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], + field: str, +) -> None: + factory, project_id = outbox_factory + value = _event(project_id) + async with factory() as session: + async with session.begin(): + await OutboxService(session).append(value) + changes: dict[str, Any] = { + "event_id": uuid4(), + "event_type": "CompensationAwardCreated", + "event_version": 2, + "aggregate_type": "compensation_award", + "aggregate_id": uuid4(), + "project_id": uuid4(), + "correlation_id": f"request:{uuid4()}", + "causation_event_id": uuid4(), + "idempotency_key": f"changed:{uuid4()}", + "payload": {"contribution_record_id": str(uuid4()), "award_ids": []}, + } + drift = OutboxAppendInput(**{**value.model_dump(), field: changes[field]}) + async with factory() as session: + async with session.begin(): + with pytest.raises( + OutboxIdempotencyConflict, + match="^outbox_idempotency_conflict$", + ): + await OutboxService(session).append(drift) + + +@pytest.mark.asyncio +async def test_outbox_conflict_does_not_retain_stored_payload( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + marker = f"stored-conflict-{uuid4()}" + value = _event(project_id, payload={"detail": marker}) + async with factory() as session: + async with session.begin(): + await OutboxService(session).append(value) + drift = OutboxAppendInput( + **{**value.model_dump(), "payload": {"detail": "changed"}} + ) + async with factory() as session: + async with session.begin(): + with pytest.raises( + OutboxIdempotencyConflict, + match="^outbox_idempotency_conflict$", + ) as raised: + await OutboxService(session).append(drift) + assert_secret_not_retained( + raised.value, + marker, + traceback_module_prefixes=("app.modules.outbox",), + ) + assert raised.value.__context__ is None + assert raised.value.__cause__ is None + + +@pytest.mark.asyncio +async def test_outbox_split_event_and_idempotency_identities_conflict( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + first = _event(project_id) + second = _event(project_id) + async with factory() as session: + async with session.begin(): + await OutboxService(session).append(first) + await OutboxService(session).append(second) + crossed = OutboxAppendInput( + **{ + **first.model_dump(), + "idempotency_key": second.idempotency_key, + } + ) + async with factory() as session: + async with session.begin(): + with pytest.raises(OutboxIdempotencyConflict): + await OutboxService(session).append(crossed) + + +async def _blocked_append( + session: AsyncSession, + value: OutboxAppendInput, +) -> tuple[OutboxAppendDisposition | None, Exception | None]: + try: + result = await OutboxService(session).append(value) + return result.disposition, None + except Exception as error: # noqa: BLE001 - test returns exact typed race outcome + return None, error + + +@pytest.mark.asyncio +async def test_outbox_duplicate_race_replays_after_first_reserver_commits( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + value = _event(project_id) + async with factory() as winner, factory() as contender: + await winner.begin() + await contender.begin() + first = await OutboxService(winner).append(value) + blocked = asyncio.create_task(_blocked_append(contender, value)) + await asyncio.sleep(0.05) + assert not blocked.done() + await winner.commit() + disposition, error = await asyncio.wait_for(blocked, timeout=3) + await contender.commit() + assert first.disposition is OutboxAppendDisposition.CREATED + assert disposition is OutboxAppendDisposition.REPLAYED + assert error is None + + +@pytest.mark.asyncio +async def test_outbox_duplicate_race_creates_after_first_reserver_rolls_back( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + value = _event(project_id) + async with factory() as reserver, factory() as contender: + await reserver.begin() + await contender.begin() + await OutboxService(reserver).append(value) + blocked = asyncio.create_task(_blocked_append(contender, value)) + await asyncio.sleep(0.05) + assert not blocked.done() + await reserver.rollback() + disposition, error = await asyncio.wait_for(blocked, timeout=3) + await contender.commit() + assert disposition is OutboxAppendDisposition.CREATED + assert error is None + + +@pytest.mark.asyncio +async def test_outbox_changed_payload_race_conflicts_after_first_reserver_commits( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + value = _event(project_id) + changed = OutboxAppendInput( + **{**value.model_dump(), "payload": {"contribution_record_id": str(uuid4())}} + ) + async with factory() as winner, factory() as contender: + await winner.begin() + await contender.begin() + await OutboxService(winner).append(value) + blocked = asyncio.create_task(_blocked_append(contender, changed)) + await asyncio.sleep(0.05) + assert not blocked.done() + await winner.commit() + disposition, error = await asyncio.wait_for(blocked, timeout=3) + await contender.rollback() + assert disposition is None + assert isinstance(error, OutboxIdempotencyConflict) + + +@pytest.mark.asyncio +async def test_outbox_custody_allows_closed_sequence_and_denies_terminal_reopen( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + value = _event(project_id) + async with factory() as session: + async with session.begin(): + await OutboxService(session).append(value) + await session.execute( + text( + "update outbox_events set delivery_state='claimed', attempt_count=1, " + "claim_generation=1, next_attempt_at=null, claim_owner='worker:1', " + "claimed_at=statement_timestamp(), last_attempt_at=statement_timestamp(), " + "claim_expires_at=statement_timestamp()+interval '30 seconds' " + "where event_id=:id" + ), + {"id": value.event_id}, + ) + await session.execute( + text( + "update outbox_events set delivery_state='retryable', " + "next_attempt_at=clock_timestamp()+interval '1 second', " + "claim_owner=null, claimed_at=null, claim_expires_at=null, " + "last_error_code='PROVIDER_UNAVAILABLE' where event_id=:id" + ), + {"id": value.event_id}, + ) + await session.execute( + text( + "update outbox_events set delivery_state='claimed', attempt_count=2, " + "claim_generation=2, next_attempt_at=null, claim_owner='worker:2', " + "claimed_at=statement_timestamp(), last_attempt_at=statement_timestamp(), " + "claim_expires_at=statement_timestamp()+interval '30 seconds' " + "where event_id=:id" + ), + {"id": value.event_id}, + ) + await session.execute( + text( + "update outbox_events set delivery_state='acknowledged', " + "claim_owner=null, claimed_at=null, claim_expires_at=null, " + "finalized_at=clock_timestamp() where event_id=:id" + ), + {"id": value.event_id}, + ) + with pytest.raises(DBAPIError, match="illegal outbox delivery transition"): + async with session.begin(): + await session.execute( + text( + "update outbox_events set delivery_state='retryable', " + "next_attempt_at=clock_timestamp(), finalized_at=null, " + "last_error_code='RETRY_REQUESTED' where event_id=:id" + ), + {"id": value.event_id}, + ) + async with session.begin(): + await session.execute( + text("update outbox_events set archived_at=clock_timestamp() where event_id=:id"), + {"id": value.event_id}, + ) + with pytest.raises(DBAPIError, match="archived outbox event is closed"): + async with session.begin(): + await session.execute( + text("update outbox_events set archived_at=null where event_id=:id"), + {"id": value.event_id}, + ) + + +@pytest.mark.asyncio +async def test_outbox_immutable_columns_delete_and_truncate_are_guarded( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + value = _event(project_id) + async with factory() as session: + async with session.begin(): + await OutboxService(session).append(value) + immutable_updates = ( + f"event_id='{uuid4()}'", + "event_type='ChangedEvent'", + "event_version=2", + "producer='other'", + "aggregate_type='other'", + f"aggregate_id='{uuid4()}'", + f"project_id='{uuid4()}'", + "correlation_id='changed'", + f"causation_event_id='{uuid4()}'", + "idempotency_key='changed:key'", + "payload='{}'::jsonb", + "payload_digest='sha256:" + ("0" * 64) + "'", + "occurred_at=clock_timestamp()", + ) + for assignment in immutable_updates: + with pytest.raises(DBAPIError, match="outbox event envelope is immutable"): + async with session.begin(): + await session.execute( + text(f"update outbox_events set {assignment} where event_id=:id"), + {"id": value.event_id}, + ) + with pytest.raises(DBAPIError, match="outbox events cannot be deleted"): + async with session.begin(): + await session.execute( + text("delete from outbox_events where event_id=:id"), + {"id": value.event_id}, + ) + with pytest.raises(DBAPIError, match="outbox events cannot be truncated"): + async with session.begin(): + await session.execute(text("truncate table outbox_events")) + + +@pytest.mark.asyncio +async def test_outbox_pending_cancellation_is_terminal_and_archivable( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + value = _event(project_id) + async with factory() as session: + async with session.begin(): + await OutboxService(session).append(value) + await session.execute( + text( + "update outbox_events set delivery_state='cancelled', " + "next_attempt_at=null, finalized_at=clock_timestamp() where event_id=:id" + ), + {"id": value.event_id}, + ) + with pytest.raises(DBAPIError, match="illegal outbox delivery transition"): + async with session.begin(): + await session.execute( + text( + "update outbox_events set delivery_state='claimed', attempt_count=1, " + "claim_generation=1, claim_owner='worker:1', " + "claimed_at=statement_timestamp(), last_attempt_at=statement_timestamp(), " + "claim_expires_at=statement_timestamp()+interval '30 seconds', " + "finalized_at=null where event_id=:id" + ), + {"id": value.event_id}, + ) + async with session.begin(): + await session.execute( + text("update outbox_events set archived_at=clock_timestamp() where event_id=:id"), + {"id": value.event_id}, + ) + + +@pytest.mark.asyncio +async def test_outbox_unarchived_dead_letter_can_requeue_with_identity_preserved( + outbox_factory: tuple[async_sessionmaker[AsyncSession], UUID], +) -> None: + factory, project_id = outbox_factory + value = _event(project_id) + async with factory() as session: + async with session.begin(): + await OutboxService(session).append(value) + await session.execute( + text( + "update outbox_events set delivery_state='claimed', attempt_count=1, " + "claim_generation=1, next_attempt_at=null, claim_owner='worker:1', " + "claimed_at=statement_timestamp(), last_attempt_at=statement_timestamp(), " + "claim_expires_at=statement_timestamp()+interval '30 seconds' " + "where event_id=:id" + ), + {"id": value.event_id}, + ) + await session.execute( + text( + "update outbox_events set delivery_state='dead_letter', claim_owner=null, " + "claimed_at=null, claim_expires_at=null, last_error_code='ATTEMPTS_EXHAUSTED', " + "finalized_at=clock_timestamp() where event_id=:id" + ), + {"id": value.event_id}, + ) + await session.execute( + text( + "update outbox_events set delivery_state='retryable', " + "next_attempt_at=clock_timestamp(), finalized_at=null where event_id=:id" + ), + {"id": value.event_id}, + ) + row = ( + await session.execute( + text( + "select event_id,idempotency_key,attempt_count,claim_generation," + "delivery_state,last_error_code from outbox_events where event_id=:id" + ), + {"id": value.event_id}, + ) + ).one() + assert row.event_id == value.event_id + assert row.idempotency_key == value.idempotency_key + assert row.attempt_count == row.claim_generation == 1 + assert row.delivery_state == "retryable" + assert row.last_error_code == "ATTEMPTS_EXHAUSTED" + with pytest.raises(DBAPIError, match="outbox claim generation must increment once"): + async with session.begin(): + await session.execute( + text( + "update outbox_events set delivery_state='claimed', attempt_count=2, " + "claim_generation=2, next_attempt_at=null, claim_owner='worker:2', " + "claimed_at=statement_timestamp(), last_attempt_at=statement_timestamp(), " + "claim_expires_at=statement_timestamp()+interval '30 seconds', " + "last_error_code='CHANGED_DURING_CLAIM' where event_id=:id" + ), + {"id": value.event_id}, + ) + + +def test_outbox_module_contains_no_dispatch_publish_or_commit_surface() -> None: + root = Path(__file__).resolve().parents[1] / "app/modules/outbox" + source = "\n".join(path.read_text(encoding="utf-8") for path in root.glob("*.py")) + assert "celery" not in source.lower() + assert "broker" not in source.lower() + assert ".commit(" not in source + assert "def publish" not in source