From 498e41cef0bbb62eec93029cf1eff6d743071211 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 18 Aug 2026 12:40:49 -0400 Subject: [PATCH] fix: require passing claim evidence --- .github/workflows/ci.yml | 19 ++ .github/workflows/validate-claims.yml | 14 +- claims.yaml | 5 +- docs/VERIFICATION.md | 34 ++-- docs/verification.json | 75 ++++++++ public-artifacts.json | 8 +- scripts/validate_claims.py | 261 ++++++++++++++++++++------ tests/test_ci_workflow_contract.py | 20 ++ tests/test_validate_claims.py | 76 +++++++- 9 files changed, 419 insertions(+), 93 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8211f83..15481107 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -358,8 +358,17 @@ jobs: run: | mkdir -p runs pytest -q --ignore=tests/e2e --basetemp=runs/ci \ + --junitxml=runs/unit-claims-junit.xml \ --cov=openadapt_flow --cov-report= + # A supported claim is not a function of file existence. Bind every + # cited unit-test file to this job's real Pytest result. Missing and + # all-skipped files fail closed even when another test passed. + - name: Validate passing unit claim evidence + run: | + python scripts/validate_claims.py --check \ + --ci-job test --junit runs/unit-claims-junit.xml + # Enforced RATCHET floor on the safety-critical path (branch-inclusive). # Scoped via --include to exactly the safety modules (compiler, identity, # effects, policy, replayer, resolver). 85% sits just under the fast @@ -384,6 +393,7 @@ jobs: runs/**/BENCH.md runs/**/report.json runs/**/*.png + runs/unit-claims-junit.xml if-no-files-found: warn # --- Browser record -> compile -> replay E2E (REQUIRED on PRs) ----------- @@ -447,8 +457,16 @@ jobs: mkdir -p runs pytest -q tests/e2e \ --ignore=tests/e2e/test_free_path_e2e.py \ + --junitxml=runs/e2e-claims-junit.xml \ --basetemp=runs/ci + # The browser maturity claims cite the record/compile/replay E2E file. + # Require that file to appear and pass in this required job's own JUnit. + - name: Validate passing browser claim evidence + run: | + python scripts/validate_claims.py --check \ + --ci-job e2e-browser --junit runs/e2e-claims-junit.xml + - name: Upload run artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -459,6 +477,7 @@ jobs: runs/**/BENCH.md runs/**/report.json runs/**/*.png + runs/e2e-claims-junit.xml if-no-files-found: warn # --- Native Linux: real GTK3 + AT-SPI under isolated X11 (REQUIRED) ------ diff --git a/.github/workflows/validate-claims.yml b/.github/workflows/validate-claims.yml index aaf3d480..6130f043 100644 --- a/.github/workflows/validate-claims.yml +++ b/.github/workflows/validate-claims.yml @@ -2,13 +2,15 @@ name: Validate claims # Make every public maturity claim a FUNCTION of automated evidence. # -# - pull_request: run the fast claim->evidence GATE -# (scripts/validate_claims.py --check) on every PR. This is a +# - pull_request: run the fast claim->evidence STRUCTURE GATE +# (scripts/validate_claims.py --check --structure-only) on every PR. This is a # required-check CANDIDATE: wire it into branch protection as the context # "gate" (the actual CheckRun job name from the "Validate claims" workflow) # to block a PR that overclaims (a `supported` claim whose only backing is # an opt-in/infra-gated test, a `field` result labeled `supported`, or a -# missing evidence path). It NEVER touches infra. +# missing evidence path). The required CI `test` and `e2e-browser` jobs +# separately bind every supported test file to their real JUnit results. +# It NEVER touches infra. # - schedule + workflow_dispatch: the infra-gated leg that provisions a real # Windows VM (via oa-vm) and runs the OPT-IN desktop + Citrix e2e that back # the `validating` tiers, then regenerates the evidence report. Where the @@ -55,8 +57,8 @@ jobs: - name: Install PyYAML run: pip install "pyyaml>=6" - - name: Claim -> evidence gate - run: python scripts/validate_claims.py --check + - name: Claim -> evidence structure gate + run: python scripts/validate_claims.py --check --structure-only # Prove the doc is regenerable and in sync with the registry: regenerate # docs/VERIFICATION.md + docs/verification.json into a temp dir would be @@ -154,7 +156,7 @@ jobs: if: steps.infra.outputs.available == 'true' run: | python scripts/validate_claims.py --report \ - --junit runs/validating-junit.xml + --ci-job validating --junit runs/validating-junit.xml - name: Upload refreshed evidence if: steps.infra.outputs.available == 'true' diff --git a/claims.yaml b/claims.yaml index 653992d5..349b8f58 100644 --- a/claims.yaml +++ b/claims.yaml @@ -13,7 +13,8 @@ # ----------------------------------------------------- # Evidence strength is derived from the repo, not asserted here: # * a test file with NO module-level env skipif, that exists -> "supported" -# (it actually runs, and can be green, in CI) +# candidate evidence; its required `test` or `e2e-browser` job must bind the +# file to at least one passing case in that job's real JUnit result # * a test file gated by a module-level `pytestmark` env skipif -> "validating" # (opt-in / infra-gated: grounded, but never runs on default CI) # * a doc / benchmark artifact (.md or a benchmark dir) -> "roadmap" @@ -24,7 +25,7 @@ # result that is not CI-reproducible is never presented as "supported"). # # TIERS -# supported — CI-proven today on the default (non-opt-in) suite. +# supported — bound to a real passing result in required default CI. # validating — grounded in a REAL opt-in/infra-gated proof or a field test; # being validated (incl. with design partners). NOT "supported". # roadmap — designed/specified, not yet proven by a running test. diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index a34f294e..28aaee27 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -3,20 +3,21 @@ > GENERATED by `scripts/validate_claims.py --report` from `claims.yaml`. Do not edit by hand — edit the registry and regenerate. - Generated at: **committed registry state (regenerate: scripts/validate_claims.py --report)** -- Green-check against a junit artifact: **not run** (no `--junit` artifact supplied) -- Gate: `python scripts/validate_claims.py --check` (a claim whose tier outranks its strongest backing evidence fails CI). +- Green-check against a junit artifact: **not embedded in this generated registry view** (required CI jobs enforce pass evidence) +- Structure gate: `python scripts/validate_claims.py --check --structure-only` (a claim whose tier outranks its strongest backing evidence fails CI). +- Pass gates: required `test` and `e2e-browser` jobs supply their own JUnit files; an absent, all-skipped, or failed supported evidence file fails that required job. -**What this harness does and does not do.** It makes each public maturity claim a *function* of automated evidence: a `supported` claim must be backed by a test that actually runs on the default (non-opt-in) CI suite; a `validating` claim must be grounded in a REAL opt-in / infra-gated proof or a field test, and is never presented as supported. It does not replace workflow- and deployment-specific acceptance: application controls, identity rules, effect oracles, and live transport conditions remain bound to their counted evidence. +**What this harness does and does not do.** It makes each public maturity claim a *function* of automated evidence: a `supported` claim must be backed by a test file that has a real passing case in its required default CI job; a `validating` claim must be grounded in a REAL opt-in / infra-gated proof or a field test, and is never presented as supported. It does not replace workflow- and deployment-specific acceptance: application controls, identity rules, effect oracles, and live transport conditions remain bound to their counted evidence. -## What is CI-proven today vs. being validated +## What is bound to required CI vs. being validated -- **CI-proven today (11):** `web-supported`, `deterministic-zero-model-replay`, `effect-verification-silent-writes`, `identity-gate-halt-armed`, `identifier-crop-compile-emission`, `halt-teach-promote`, `typed-business-decisions-supported`, `reviewed-judgment-cases-supported`, `qualified-remote-decision-v2-supported`, `mockmed-benchmark-ci-reproducible`, `substrate-runtime-validation-supported` +- **Bound to required CI pass evidence (11):** `web-supported`, `deterministic-zero-model-replay`, `effect-verification-silent-writes`, `identity-gate-halt-armed`, `identifier-crop-compile-emission`, `halt-teach-promote`, `typed-business-decisions-supported`, `reviewed-judgment-cases-supported`, `qualified-remote-decision-v2-supported`, `mockmed-benchmark-ci-reproducible`, `substrate-runtime-validation-supported` - **Being validated — opt-in / infra-gated or field (8):** `effect-verifier-kit`, `windows-desktop-validating`, `macos-native-validating`, `linux-native-validating`, `rdp-validating`, `desktop-recording-validating`, `citrix-pixel-validating`, `openemr-field-benchmark` - **Roadmap / research (1):** `win32-window-replay-roadmap` ## Claims -### `web-supported` — supported — CI-proven today +### `web-supported` — supported — bound to required CI pass evidence > Web (browser) workflows are supported today: record a GUI workflow once, then replay it deterministically and locally. @@ -34,7 +35,7 @@ - "Supported" is scoped to the reference headless-browser backend in this registry. Desktop and remote-display workflows use the separately scoped acceptance and code-qualified claims below. - The full record->compile->replay browser suite runs in the required e2e-browser PR gate and repeats in the weekly compatibility matrix. -### `deterministic-zero-model-replay` — supported — CI-proven today +### `deterministic-zero-model-replay` — supported — bound to required CI pass evidence > Replay is deterministic, local, and $0 — zero model calls on either the record or the replay side. @@ -51,7 +52,7 @@ - A VLM rung exists as the lowest fallback of the ladder; it is not invoked on a clean deterministic replay, and never on the record side. -### `effect-verification-silent-writes` — supported — CI-proven today +### `effect-verification-silent-writes` — supported — bound to required CI pass evidence > Effect verification against the system of record catches silent wrong writes the screen-only oracle misses (5 of 7 transactional fault classes). @@ -90,7 +91,7 @@ - CONTRACT-PROVEN, not live-proven: the SQL verifier is exercised against sqlite fixtures (no production MariaDB/Postgres), the SFTP arrival path against a fake transport (no real SFTP server), and the worked-example configs are templates — only the FHIR substrate has an additional opt-in live-OpenEMR test. Per-verifier claims are exactly as strong as these tests. -### `identity-gate-halt-armed` — supported — CI-proven today +### `identity-gate-halt-armed` — supported — bound to required CI pass evidence > On ambiguity the runtime halts instead of guessing, and an identity gate refuses a wrong-entity (wrong-patient) click — on ARMED steps only. @@ -108,7 +109,7 @@ - Identity verification covers ONLY armed steps. Real bundles arm a MINORITY of clicks (4 of 12 on a recent live OpenEMR bundle); an UNARMED click proceeds with no identity check whatsoever. Coverage is an auditable per-step metric, but disclosure does not close the gap. -### `identifier-crop-compile-emission` — supported — CI-proven today +### `identifier-crop-compile-emission` — supported — bound to required CI pass evidence > The compiler emits a pixel identifier crop (anchor.identifier_crop) for identity-armed steps without structured identity — and for any step whose identifying region is marked at record time (--identifier) — so the pixel-compare identity tier arms on remote-display/pixel replays; every crop-less identity-applicable step records an explicit degrade reason, and lint surfaces per-bundle pixel-identity coverage. @@ -125,7 +126,7 @@ - Mechanism proven on synthetic fixtures and the bundled demo app only — this does NOT re-qualify any desktop/remote-display substrate, and the pixel tier remains MISMATCH-or-ABSTAIN (it can add a safe halt on a wrong identifier, never authorize a match; PIXEL_VERIFY_ENABLED stays off pending a jitter-robust distance). - Automatic emission requires an OCR-readable identity band at compile time; icon-only or unreadable rows still compile crop-less (with the recorded reason) and fall back to the OCR band tier at replay. -### `halt-teach-promote` — supported — CI-proven today +### `halt-teach-promote` — supported — bound to required CI pass evidence > A halt is learnable: halt -> teach a correction -> promote a guarded branch, refusing underdetermined fixes. @@ -141,7 +142,7 @@ - Promotion is governed by a regression gate that blocks identity- weakening corrections; the loop refuses to guess when the fix is underdetermined. -### `typed-business-decisions-supported` — supported — CI-proven today +### `typed-business-decisions-supported` — supported — bound to required CI pass evidence > A qualification client can add or update a typed finite business-policy decision without editing an internal manifest. The compiled workflow can then pause at that decision, accept one authorized and attributed choice, retain a signed durable receipt, revalidate the live application, and continue only through the certified successor branch. @@ -158,7 +159,7 @@ - The engine validates a principal and its roles; it does not authenticate a self-asserted user. Desktop, Cloud, or a customer-local identity route must supply the authenticated principal. - A human choice is control authority only. It cannot satisfy entity identity, a postcondition, or effect verification, and it cannot turn a screen statement or a human assertion into VERIFIED. -### `reviewed-judgment-cases-supported` — supported — CI-proven today +### `reviewed-judgment-cases-supported` — supported — bound to required CI pass evidence > A qualification project can bind typed local facts, evidence hashes, reviewer provenance, and the exact decision contract to reviewed examples and counterfactuals. It can retain permanent human authority, record a reviewed automatic-rule candidate, or refuse certification until more evidence exists, without converting one historical choice into policy. @@ -176,7 +177,7 @@ - An automatic-rule case names only a reviewed rule identifier and finite option. Flow does not infer or generate executable policy from the case; the rule must be authored and qualified through the normal program path. - A retained human-node case preserves runtime human authority. A more-evidence-required case refuses certification instead of guessing. -### `qualified-remote-decision-v2-supported` — supported — CI-proven today +### `qualified-remote-decision-v2-supported` — supported — bound to required CI pass evidence > A negotiated V2 attended task binds optional reviewed entity wording to the exact qualification, bundle, step, policy, and pause authority. An unqualified or unrecognized entity class stays local and the remote task uses the signed neutral record or item fallback. @@ -194,7 +195,7 @@ - V2 requires openadapt-types 0.10.x and explicit peer negotiation. The dependency does not upgrade a V1 consumer, and the byte-compatible V1 path remains available. - The entity class is presentation metadata only. Actual record identity remains inside the customer-controlled runner and is revalidated before any resumed action. -### `mockmed-benchmark-ci-reproducible` — supported — CI-proven today +### `mockmed-benchmark-ci-reproducible` — supported — bound to required CI pass evidence > The MockMed benchmark is CI-reproducible: compiled replay finishes ~4.9s p50 with zero model calls versus ~37.5s p50 (~$0.27/run) for the agent. Measured 2026-07-08 on Flow 0.1.0, a pre-v0.2.0 source build; not re-measured on a later release. @@ -289,7 +290,7 @@ - Acceptance covers the named 1280x800 Windows Run/file task and exact VM environment, not arbitrary applications, identity policies, or Citrix. - The multi-window campaign uses deterministic synthetic applications. It exercises the real FreeRDP pixel and input path but does not qualify a named customer application or environment. -### `substrate-runtime-validation-supported` — supported — CI-proven today +### `substrate-runtime-validation-supported` — supported — bound to required CI pass evidence > Runtime-validation v3 binds Standard and Regulated hosted activation to the exact governed authorization template reproduced from the sealed qualification. It also binds the successful run's resolved target kind (`web|windows|macos|linux|rdp|citrix`), artifact, compiler, parameter schema, strict lint, policy, risk, report, and opaque environment. Web retains its exact HTTPS/host boundary. Native and remote execution emits no app, window, host, readiness, or backend-hint values to Cloud. Local verification retains signed v1 and v2 compatibility. @@ -385,4 +386,3 @@ - The historical result uses OCR evidence from a visible saved message row. It does not use an out-of-band OpenEMR system-of-record read. - FIELD TEST, NOT CI-reproducible: the head-to-head ran against a SHARED public demo that other users mutate and that resets daily; the sample is small (10 agent runs). Only the verifier + task-prompt units run in CI. `reproducibility: field` forbids ever labeling this `supported`. - diff --git a/docs/verification.json b/docs/verification.json index f47e56f5..f01c9349 100644 --- a/docs/verification.json +++ b/docs/verification.json @@ -1,6 +1,7 @@ { "generated_at": "committed registry state (regenerate: scripts/validate_claims.py --report)", "green_check_run": false, + "green_check_job": null, "ok": true, "claims": [ { @@ -27,6 +28,7 @@ "gating": "ci (required PR gate (e2e-browser))", "node": null, "node_found": null, + "ci_job": "e2e-browser", "junit_status": null, "proves": "Records the MockMed browser demo once, compiles it, and replays it under baseline + theme/move/rename drift and parameter substitution through the headless-browser Backend." }, @@ -38,6 +40,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The reference browser demo app and its drift screens render deterministically (no CSS transitions), so replay is repeatable." }, @@ -49,6 +52,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The deterministic replayer resolves steps, substitutes parameters, enforces postconditions and the risk gate \u2014 no model in the loop." } @@ -78,6 +82,7 @@ "gating": "ci (required PR gate (e2e-browser))", "node": null, "node_found": null, + "ci_job": "e2e-browser", "junit_status": null, "proves": "Baseline replay x3 resolves every step on the `template` rung with 0 heals and 0 model calls; healed bundles replay clean afterward." }, @@ -89,6 +94,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The replayer is a deterministic resolution ladder; the VLM rung is a fallback, not on the default path \u2014 the core runtime is model-free." }, @@ -100,6 +106,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "CI-reproducible MockMed run artifact: 100/100 compiled replays at 4.9s p50 with zero model calls, measured 2026-07-08 on Flow 0.1.0 (pre-v0.2.0 source build); field/artifact reference." } @@ -128,6 +135,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The REST/FHIR/document-hash verifiers confirm exactly-one write and refute duplicate / missing / partial / collateral-loss writes." }, @@ -139,6 +147,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Each silent-wrong-action class is silent under the screen-only oracle and is CAUGHT once effects are declared and verified." }, @@ -150,6 +159,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The FHIR effect verifier's read/confirm/refute/indeterminate contract (the live-OpenEMR smoke is a separate opt-in function in this file)." }, @@ -161,6 +171,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The 7-class transactional fault-model study: 5/7 classes silently mishandled by screen-only, all 5 halt through the real replayer." } @@ -188,6 +199,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The read-only SQL verifier's whitelist (mutation/stacking/comment smuggling refused at construction) and full verdict contract, plus the exact table-delta audit promoted from the Frappe Lending reference matrix \u2014 against LOCAL sqlite fixtures only." }, @@ -199,6 +211,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "File-arrival verdicts (pattern / size>0 / mtime freshness / content probe / duplicate export) against real temp directories, and the SFTP path against an in-memory fake paramiko-shaped transport." }, @@ -210,6 +223,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The declarative construction path: env-var auth references fail loud when unset, run-parameter references resolve at build (and refuse to construct unresolved), and pre-kit configs build byte-identically." }, @@ -221,6 +235,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI constructs and preflights all reviewed candidates before input, pins the strongest verifier and its pre-state, preserves the binding through durable resume, and refuses an unavailable selected verifier rather than falling back after an action." }, @@ -232,6 +247,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "lint warns per consequential step lacking an effect contract and reports coverage %; certify fails the same gap only when the policy sets require_effects_for_irreversible (warn-vs-fail is policy-configurable)." }, @@ -243,6 +259,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Every escalated verification failure emits a typed ReconciliationTask carrying the one-way contract hash and verdict evidence \u2014 never the resolved selector values." }, @@ -254,6 +271,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The operator-facing kit contract: config reference plus Frappe (REST/SQL) and OpenEMR (FHIR/SQL) worked examples targeting the reference fixtures." } @@ -282,6 +300,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Wrong-entity / near-name sibling / generic-band / param-in-band probes: the identity check refuses to verify a mismatched target and never arms on non-discriminative bands." }, @@ -293,6 +312,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Underdetermined targets surface as explicit disambiguation rather than a guessed click." }, @@ -304,6 +324,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Public property-based checks preserve the never-false-accept invariant for collapsible identifiers while requiring clean identifiers to remain verifiable." }, @@ -315,6 +336,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The identity resolution ladder (structural/template/OCR) escalation." } @@ -343,6 +365,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Synthetic-fixture proof of the emission mechanism: a pixel-only recording compiles WITH a crop under templates/identifiers/ (sealed + manifest-hashed like every image crop); a structured recording writes NO identity pixels and records why; explicit --identifier markings (event rect / meta region) win; encrypted save seals the crop and the sealed crop still reaches the tier; the compiled crop drives a wrong-MRN MISMATCH through the real replayer ladder while pixel VERIFY stays hard-gated off (zero-false-accept preserved)." }, @@ -354,6 +377,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Lint coverage surfacing: identity-armed steps with vs without crops (missing_identifier_crop findings carry the compiler's degrade reason; warn on band-only identity, info under structured identity)." } @@ -382,6 +406,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "A surprise modal halts and emits a learnable trace; teaching promotes a guarded dismiss branch that replays without regression; an underdetermined correction is refused." }, @@ -393,6 +418,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The `teach` CLI resolves a halt via a correction spec or a recording, refuses an underdetermined fix, and errors on a non-halted run." } @@ -421,6 +447,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI covers typed qualification authoring and replacement, qualification-revision and certification invalidation, ambiguous insertion refusal, the closed runtime decision contract, role and evidence checks, expiry, renewal, idempotency, crash recovery, signed branch authority, scoped outputs, fresh live-state revalidation, normal downstream action gates, and repair-bypass refusal." }, @@ -432,6 +459,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The public engine contract separates a declared business choice from an operational halt and defines the Desktop, Cloud, and customer-local integration boundary." } @@ -460,6 +488,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI covers closed fact schemas, exact workflow and decision bindings, local-only evidence references, conflict refusal, reciprocal contrast coverage for automatic-rule candidates, retained human authority, unresolved-evidence refusal, and the invariant that case evaluation never edits or synthesizes the executable rule." }, @@ -471,6 +500,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI binds the reviewed local evidence bytes and review note into certification, reproduces that exact contract when a qualified workflow is saved, and refuses changed case or evidence digests." }, @@ -482,6 +512,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The public contract separates qualification-time judgment capture from one live runtime answer and describes the scriptable authoring and judgment-case interfaces." } @@ -510,6 +541,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI covers the V1-compatible and V2 signed envelopes, provenance binding, remote-safe entity vocabulary, idempotent decisions, and fresh revalidation before resumed actuation." }, @@ -521,6 +553,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI requires the current exact qualification contract before the V2 producer can emit its reviewed presentation metadata." }, @@ -532,6 +565,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The delivery contract defines explicit V2 negotiation, neutral V1 fallback, and the local boundary for actual entity identifiers." } @@ -560,6 +594,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The MockMed app + drift screens are deterministic and reachable, the substrate the benchmark replays against." }, @@ -571,6 +606,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The benchmark harness / report accounting that produces the numbers." }, @@ -582,6 +618,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "Full numbers, methodology, and caveats for the CI-reproducible MockMed comparison (dated run artifact)." } @@ -612,6 +649,7 @@ "gating": "opt-in (OAFLOW_PARALLELS_E2E)", "node": null, "node_found": null, + "ci_job": "validating", "junit_status": null, "proves": "OPT-IN live proof on a real Win11-ARM VM: record->compile->replay through WindowsBackend with the structural rung resolving every click by AutomationId (armed_coverage == 1.0)." }, @@ -623,6 +661,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The 4-method Backend protocol on Windows against a MOCKED WAA server (runs in CI post-merge; proves the wire, not a live desktop)." }, @@ -634,6 +673,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "Desktop drift benchmark: the structural rung resolved 21/21 targets where visual replay alone managed 6/21." }, @@ -645,6 +685,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "Accepted candidate 20260717-candidate-56759c8-v2: 3/3 exact task and SQLite effects, 12 UIA receipts, and 3/3 stale plus 3/3 ambiguity refusals with zero silent incorrect success, over-halt, or model calls." } @@ -675,6 +716,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "CI covers unique-window binding, foreground/focus proof, exact-element text delivery, and refusal before physical input when proof fails." }, @@ -686,6 +728,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "CI validates the qualification oracle, failure taxonomy, evidence hashes, cleanup adjudication, and ambiguity-refusal accounting." }, @@ -697,6 +740,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "Candidate b1b61a5 completed 3/3 exact-byte TextEdit effects and refused two ambiguous windows without modifying either file." } @@ -726,6 +770,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Default CI covers exact app/window binding, bounded candidate enumeration, ambiguity and stale-target refusal, native invoke/focus and editable-text delivery, window-scoped capture, physical-input opt-in, and Wayland portal refusal through an injected Linux client." }, @@ -737,6 +782,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Default CI enforces the live qualification's fixed three-trial matrix, independent exact-file and absence oracles, and separate silent-incorrect-success, over-halt, and refusal-failure accounting." }, @@ -748,6 +794,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The required linux-atspi-x11 job drives the real GTK3 fixture through AT-SPI on an isolated Xvfb display and session D-Bus, emitting a per-trial JSON evidence artifact." }, @@ -759,6 +806,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The operator contract, X11 and Wayland boundaries, configuration, delivery-only receipt semantics, and required real GTK/AT-SPI CI qualification contract." } @@ -789,6 +837,7 @@ "gating": "opt-in (OAFLOW_PARALLELS_RDP_E2E)", "node": null, "node_found": null, + "ci_job": "validating", "junit_status": null, "proves": "Opt-in snapshot-safe real RDP qualification harness with exactly three trials, independent guest-tools oracle, failure taxonomy, and cleanup." }, @@ -800,6 +849,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "CI covers the backend/transport contract, framebuffer conversion, pointer and keyboard delivery, and record-compile-replay conformance." }, @@ -811,6 +861,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI covers the bounded 27-trial FreeRDP campaign contract, independent SQLite, CSV, and Maildir oracles, fail-closed visual action preflight, uncertain-delivery handling, and result accounting." }, @@ -822,6 +873,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The public synthetic multi-window campaign defines the workflow, faults, independent persisted surfaces, and acceptance denominator." }, @@ -833,6 +885,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "Candidate 82a658a passed 3/3 at 51.845, 10.467, and 7.477 seconds, with zero failures, silent incorrect successes, over-halts, or model calls." } @@ -862,6 +915,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI exercises all six target kinds, validates v3 against its closed JSON Schema, refuses a Standard or Regulated activation without an exact template, binds the template SHA to the HMAC, preserves web URL/host checks, keeps native/remote execution empty, verifies the canonical HMAC vector, and retains signed v1/v2 compatibility." }, @@ -873,6 +927,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI carries the resolved web, Windows, and Citrix backend token through durable resume into the new run report rather than trusting an upload-time relabel." }, @@ -884,6 +939,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The public closed envelope requires target_kind and the governed authorization template SHA, and conditionally permits browser execution fields only for web." } @@ -914,6 +970,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The live-capture orchestration + CLI wiring (record --backend windows|macos|linux|rdp|citrix -> openadapt-capture -> convert_capture), and that a desktop-shaped recording COMPILES into a bundle and REPLAYS to completion through the desktop backend path, resolving each click to its recorded target (runs on default CI)." }, @@ -925,6 +982,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The capture->recording bridge over a REAL openadapt-capture session (its own event-processing pipeline + frame extraction) feeding the UNMODIFIED compiler (runs on default CI: the `test` job installs the `capture` extra; openadapt-capture >=0.5.4 imports clean headless)." }, @@ -936,6 +994,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The capture-assisted and live-observer paths, fail-closed secret handling, RDP coordinate binding, and reuse of openadapt-capture + the capture adapter." } @@ -967,6 +1026,7 @@ "gating": "opt-in (OAFLOW_CITRIX_PIXEL_E2E)", "node": null, "node_found": null, + "ci_job": "validating", "junit_status": null, "proves": "OPT-IN pixel-only proof: structural_armed_coverage == 0, replay resolves on template/ocr/geometry only, on-screen OCR read-back verifies the write, the identity gate HALTs on a look-alike patient, and render drift triggers halt-on-ambiguity." }, @@ -978,6 +1038,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The pixel-substrate identity probe behind the on-pixels identity gate." }, @@ -989,6 +1050,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI covers the dedicated backend preset, exact owner/title overrides, readiness probe construction, pixel-only capability boundary, backend factory, and product CLI selectors." }, @@ -1000,6 +1062,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI refuses governed Citrix execution before action when its readiness binding is absent or blank, admits a sealed bundle carrying a recorded readiness binding, and reports only the resolved backend token rather than sensitive target strings." }, @@ -1011,6 +1074,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI restores the recorded Citrix owner, exact title, and readiness binding through durable approve/resume, and refuses before backend construction when resumed configuration is incomplete." }, @@ -1022,6 +1086,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI binds a completed local Citrix report to the closed `citrix` execution token without copying target owner, title, or readiness values into the hosted summary." }, @@ -1033,6 +1098,7 @@ "gating": "opt-in (OAFLOW_CITRIX_STANDIN_E2E)", "node": null, "node_found": null, + "ci_job": "validating", "junit_status": null, "proves": "The dedicated Citrix backend passes three healthy effect-confirmed record->compile->replay trials and three severe-drift safe-halts over a no-DOM surface, with zero model calls, false completion, or silent incorrect success." }, @@ -1044,6 +1110,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI validates the public real-ICA campaign preflight, distinct authority keys, executable and oracle attestations, one-use nonce journal, crash recovery, uncertain dispatch, and fail-closed report contract without provisioning infrastructure." }, @@ -1055,6 +1122,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The retained six-trial record reports code_readiness_accepted=true and ica_hdx_accepted=false, keeping driver readiness distinct from a counted live ICA/HDX qualification." }, @@ -1066,6 +1134,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The Citrix driver model, reusable evidence, exact-deployment acceptance contract, independent-effect boundary, and customer-controlled posture." } @@ -1093,6 +1162,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "MOCK-ONLY conformance: the win32 WindowClient implements the remote-display WindowClient seam (exact process/title/class selection with ambiguity halt, client-area coordinate mapping under per-monitor DPI awareness, PrintWindow-then-BitBlt capture fallback, focus-verification refusal, and the UIPI elevation guard) against a scripted Win32 API \u2014 never against a real Windows host." }, @@ -1104,6 +1174,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The remote-display contract the client implements and the evidence ladder for qualifying it (Parallels/RDP client-window analog first, then a counted live Citrix host run)." } @@ -1134,6 +1205,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The CI-reproducible half: the saved-message-row verifier, its unsaved-entry-form rejection, and the intent-not-coordinates task prompt that both benchmark arms use." }, @@ -1145,6 +1217,7 @@ "gating": "opt-in (OPENADAPT_OPENEMR_FINALS_DIR)", "node": null, "node_found": null, + "ci_job": "validating", "junit_status": null, "proves": "When the local-only retained final frames are mounted, the guard replays all 30 with the current verifier and requires 19/20 compiled, 10/10 agent, and exactly one corrected legacy false success." }, @@ -1156,6 +1229,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The corrected field-test numbers, saved-row screen-oracle contract, methodology, and cost caps for the 20-vs-10 head-to-head run." }, @@ -1167,6 +1241,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The correctness-only field findings (fresh browsers, zero model calls, closed-loop scrolling)." } diff --git a/public-artifacts.json b/public-artifacts.json index 383afae7..e74259c4 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -89,7 +89,7 @@ }, { "path": ".github/workflows/ci.yml", - "sha256": "6ababed7e1d90cbc63b681011712ddf5c335429044508836bdd47a113d7331f8" + "sha256": "9f2e22636273b7fc387ecea41db6cb64c5697da2e08dee0b2883cac1d549dce3" }, { "path": ".github/workflows/citrix-workspace-standin.yml", @@ -121,7 +121,7 @@ }, { "path": ".github/workflows/validate-claims.yml", - "sha256": "085faf897b2c9f31afe4b1297454606bec8af3ac0165616a6351f06e9ce1fe6a" + "sha256": "7030f250fe66f36e71f3b44d52713e7f550a10e04c10f37ebdc745fa9a97c69a" }, { "path": ".pre-commit-config.yaml", @@ -601,7 +601,7 @@ }, { "path": "claims.yaml", - "sha256": "bf33b2ef84654c8563937828e38e389e065efcea5ec2654d4888423c1ede0142" + "sha256": "5423f4f10ef289dc3e232630c8558f12a76b3f11ebbb497feb923a928dc230b7" }, { "path": "deploy/on-prem/docker-compose.yml", @@ -1809,7 +1809,7 @@ }, { "path": "docs/verification.json", - "sha256": "b27d6f6a848d1dd75149155bc5f33dc4b797634def65da5ffe4f62fe0053f44e" + "sha256": "c8275336c9581532a3841a57913228b7584ab1487cdcc9c7b7ab884326769842" }, { "path": "openadapt_flow/console/static/console.css", diff --git a/scripts/validate_claims.py b/scripts/validate_claims.py index 97a69f4b..224946f8 100644 --- a/scripts/validate_claims.py +++ b/scripts/validate_claims.py @@ -4,30 +4,34 @@ `scripts/check_consistency.py` stops the README from carrying stale *strings*. This script stops it from carrying stale *maturity claims*. It reads the machine-readable registry `claims.yaml` (each claim -> a `tier` -> the backing -test(s)/benchmark(s)) and enforces a tier<->evidence contract, so a "supported" -claim whose proof is only an opt-in/infra-gated test — or is missing entirely — -is a hard CI failure instead of a thing a design partner discovers. +test(s)/benchmark(s)) and enforces a tier<->evidence contract. A "supported" +claim whose proof is only an opt-in/infra-gated test, is missing, is absent from +the required job's JUnit result, is skipped, or fails is a hard CI failure. The evidence STRENGTH of each artifact is derived from the repo, never asserted by the registry (which therefore cannot lie about it): * a test file with NO module-level env skipif, that exists -> ``supported`` - (it actually runs, and can be green, on the default CI suite) + candidate evidence (the required ``test`` or ``e2e-browser`` job must also + supply a JUnit result that proves the cited file actually passed) * a test file gated by a module-level ``pytestmark`` env skipif -> ``validating`` (opt-in / infra-gated: grounded in a real proof, but never on default CI) * a doc / benchmark artifact (``.md`` or a benchmark dir) -> ``roadmap`` (design / field evidence; cannot by itself prove a running capability) -A claim FAILS when its ``tier`` OUTRANKS its strongest evidence, or when a -claim marked ``reproducibility: field`` is labeled ``supported`` (a result that -is not CI-reproducible is never presented as "supported"), or when any evidence -path is missing (registry rot). +A claim FAILS when its ``tier`` OUTRANKS its strongest evidence, when a claim +marked ``reproducibility: field`` is labeled ``supported`` (a result that is not +CI-reproducible is never presented as "supported"), or when any evidence path +is missing (registry rot). In a required test job, it also fails unless every +supported test assigned to that job has at least one passing case, no failing +case, and a real entry in that job's JUnit result. Usage:: - python scripts/validate_claims.py --check # gate (exit 1 on violation) + python scripts/validate_claims.py --check --structure-only + python scripts/validate_claims.py --check --ci-job test \ + --junit runs/unit-claims-junit.xml python scripts/validate_claims.py --report # (re)write docs/VERIFICATION.md + .json - python scripts/validate_claims.py --check --junit runs/ci/junit.xml # + green-check The public functions are importable so ``tests/test_validate_claims.py`` can drive them with controlled registries (catching registry rot before CI does). @@ -58,10 +62,11 @@ VALID_TIERS = set(TIER_RANK) # Evidence strength labels reuse the tier vocabulary (same rank scale). -STRENGTH_CI = "supported" # non-opt-in test that exists -> runs on default CI +STRENGTH_CI = "supported" # eligible test; required-job JUnit must prove its pass STRENGTH_OPTIN = "validating" # opt-in / infra-gated test -> grounded, not on CI STRENGTH_DOC = "roadmap" # doc/benchmark artifact -> design/field evidence only STRENGTH_MISSING = "research" # nothing backing it +CI_JOBS = {"test", "e2e-browser", "validating"} # --------------------------------------------------------------------------- # @@ -200,7 +205,8 @@ class EvidenceResult: gating: str # human-readable: "ci (required PR gate)", "opt-in (ENV)", ... node: Optional[str] = None node_found: Optional[bool] = None - junit_status: Optional[str] = None # "passed" | "failed" | None (unknown) + ci_job: Optional[str] = None + junit_status: Optional[str] = None # "passed" | "failed" | "skipped" | "unknown" @dataclass @@ -240,8 +246,19 @@ def load_registry(path: Path = REGISTRY) -> dict[str, Any]: return data +def _required_ci_job(path: str, strength: str) -> Optional[str]: + if strength == STRENGTH_OPTIN: + return "validating" + if strength != STRENGTH_CI: + return None + return "e2e-browser" if path.startswith("tests/e2e/") else "test" + + def _classify_evidence( - ev: dict[str, Any], repo_root: Path, junit: Optional[dict[str, str]] + ev: dict[str, Any], + repo_root: Path, + junit: Optional[dict[str, str]], + ci_job: Optional[str], ) -> EvidenceResult: path = str(ev["path"]) kind = ev.get("kind") or _infer_kind(path) @@ -253,6 +270,7 @@ def _classify_evidence( strength = STRENGTH_MISSING gating = "missing" node_found: Optional[bool] = None + required_ci_job: Optional[str] = None junit_status: Optional[str] = None if not exists: @@ -275,8 +293,9 @@ def _classify_evidence( else "required PR gate (test)" ) gating = f"ci ({stage})" - if junit is not None: - junit_status = junit.get(Path(path).name, "unknown") + required_ci_job = _required_ci_job(path, strength) + if junit is not None and required_ci_job == ci_job: + junit_status = junit.get(path, "unknown") else: # doc / benchmark artifact: design or field evidence, never a run proof. strength = STRENGTH_DOC @@ -291,6 +310,7 @@ def _classify_evidence( gating=gating, node=str(node) if node else None, node_found=node_found, + ci_job=required_ci_job, junit_status=junit_status, ) @@ -299,6 +319,7 @@ def validate_claim( raw: dict[str, Any], repo_root: Path = REPO_ROOT, junit: Optional[dict[str, str]] = None, + ci_job: Optional[str] = None, ) -> ClaimResult: """Validate a single registry entry, returning a ClaimResult with errors.""" cid = str(raw.get("id", "")) @@ -320,7 +341,7 @@ def validate_claim( return result for ev in raw.get("evidence", []) or []: - result.evidence.append(_classify_evidence(ev, repo_root, junit)) + result.evidence.append(_classify_evidence(ev, repo_root, junit, ci_job)) # 1) registry rot: every evidence path must exist. for e in result.evidence: @@ -346,13 +367,27 @@ def validate_claim( f"'supported' (result is not CI-reproducible)" ) - # 4) green-check (only when a junit artifact is supplied): a supported - # claim's CI tests must not be red. - if junit is not None and tier == "supported": + # 4) Required-job proof. File existence makes a test eligible to back a + # supported claim. It does not prove that the test ran. The job-scoped + # JUnit result must contain a real passing case from every cited file. + if junit is not None and ci_job is not None and tier == "supported": for e in result.evidence: - if e.strength == STRENGTH_CI and e.junit_status == "failed": + if e.strength != STRENGTH_CI or e.ci_job != ci_job: + continue + if e.junit_status == "failed": + result.errors.append( + f"[{cid}] supported claim's backing test is RED in " + f"{ci_job} JUnit: {e.path}" + ) + elif e.junit_status == "skipped": + result.errors.append( + f"[{cid}] supported claim's backing test was SKIPPED in " + f"{ci_job} JUnit: {e.path}" + ) + elif e.junit_status != "passed": result.errors.append( - f"[{cid}] supported claim's backing test is RED in junit: {e.path}" + f"[{cid}] supported claim's backing test is ABSENT from " + f"{ci_job} JUnit: {e.path}" ) return result @@ -378,37 +413,89 @@ def validate_all( registry: dict[str, Any], repo_root: Path = REPO_ROOT, junit: Optional[dict[str, str]] = None, + ci_job: Optional[str] = None, ) -> list[ClaimResult]: return [ - validate_claim(raw, repo_root=repo_root, junit=junit) + validate_claim(raw, repo_root=repo_root, junit=junit, ci_job=ci_job) for raw in registry.get("claims", []) ] # --------------------------------------------------------------------------- # -# optional junit parse (confirm supported claims are green) +# JUnit parse (prove supported claims passed in their required CI job) # --------------------------------------------------------------------------- # -def parse_junit(path: Path) -> dict[str, str]: - """Map test-file basename -> "passed"|"failed" from a junit XML artifact. +class JunitEvidenceError(ValueError): + """A required CI result is absent, malformed, or carries no test cases.""" + + +def _junit_case_path(case: Any, repo_root: Path) -> Optional[str]: + """Return a repo-relative Python test path from one JUnit testcase.""" + + file_attr = str(case.get("file") or "").replace("\\", "/").lstrip("./") + if file_attr: + if "/tests/" in file_attr: + file_attr = "tests/" + file_attr.split("/tests/", 1)[1] + candidate = Path(file_attr) + if candidate.is_absolute(): + try: + candidate = candidate.relative_to(repo_root) + except ValueError: + return None + normalized = candidate.as_posix() + if normalized.startswith("tests/") and normalized.endswith(".py"): + return normalized + + classname = str(case.get("classname") or "") + parts = classname.split(".") if classname else [] + while len(parts) >= 2: + candidate = "/".join(parts) + ".py" + if candidate.startswith("tests/") and (repo_root / candidate).is_file(): + return candidate + parts.pop() + return None + + +def parse_junit(path: Path, repo_root: Path = REPO_ROOT) -> dict[str, str]: + """Map repo-relative test file -> passed, failed, or skipped. - Best-effort and coarse (file granularity): if ANY case in a file failed or - errored, the file is "failed". Used only to red-flag a `supported` claim. + Pytest's default xUnit2 output omits the ``file`` attribute. Resolve its + dotted ``classname`` against the repository instead of silently returning + an empty map. A file is failed when any case failed or errored. It is passed + when at least one case passed and no case failed. It is skipped only when + every mapped case was skipped. """ import xml.etree.ElementTree as ET + if not path.is_file(): + raise JunitEvidenceError(f"required JUnit artifact is missing: {path}") + try: + root = ET.parse(path).getroot() + except (OSError, ET.ParseError) as exc: + raise JunitEvidenceError( + f"required JUnit artifact cannot be read: {path}: {exc}" + ) from exc + status: dict[str, str] = {} - root = ET.parse(path).getroot() + rank = {"skipped": 1, "passed": 2, "failed": 3} for case in root.iter("testcase"): - file_attr = case.get("file") or case.get("classname", "") - name = Path(file_attr).name if file_attr else "" - if not name.endswith(".py"): + test_path = _junit_case_path(case, repo_root) + if test_path is None: continue - failed = any(child.tag in ("failure", "error") for child in case) - prev = status.get(name) - if failed: - status[name] = "failed" - elif prev != "failed": - status[name] = "passed" + child_tags = {child.tag.rsplit("}", 1)[-1] for child in case} + case_status = ( + "failed" + if child_tags & {"failure", "error"} + else "skipped" + if "skipped" in child_tags + else "passed" + ) + previous = status.get(test_path) + if previous is None or rank[case_status] > rank[previous]: + status[test_path] = case_status + if not status: + raise JunitEvidenceError( + f"required JUnit artifact contains no repository test cases: {path}" + ) return status @@ -440,14 +527,19 @@ def resolve_now(explicit: Optional[str]) -> str: # report generation # --------------------------------------------------------------------------- # _TIER_BADGE = { - "supported": "supported — CI-proven today", + "supported": "supported — bound to required CI pass evidence", "validating": "validating — opt-in / infra-gated or field test", "roadmap": "roadmap — designed, not yet proven", "research": "research — open question", } -def render_markdown(results: list[ClaimResult], now: str, junit_used: bool) -> str: +def render_markdown( + results: list[ClaimResult], + now: str, + junit_used: bool, + junit_job: Optional[str] = None, +) -> str: lines: list[str] = [] lines.append("# VERIFICATION — maturity claims backed by tests") lines.append("") @@ -459,18 +551,29 @@ def render_markdown(results: list[ClaimResult], now: str, junit_used: bool) -> s lines.append(f"- Generated at: **{now}**") lines.append( "- Green-check against a junit artifact: " - + ("**run**" if junit_used else "**not run** (no `--junit` artifact supplied)") + + ( + f"**run for `{junit_job}`**" + if junit_used and junit_job + else "**not embedded in this generated registry view** " + "(required CI jobs enforce pass evidence)" + ) + ) + lines.append( + "- Structure gate: `python scripts/validate_claims.py --check " + "--structure-only` (a claim whose tier outranks its strongest backing " + "evidence fails CI)." ) lines.append( - "- Gate: `python scripts/validate_claims.py --check` " - "(a claim whose tier outranks its strongest backing evidence fails CI)." + "- Pass gates: required `test` and `e2e-browser` jobs supply their own " + "JUnit files; an absent, all-skipped, or failed supported evidence file " + "fails that required job." ) lines.append("") lines.append( "**What this harness does and does not do.** It makes each public " "maturity claim a *function* of automated evidence: a `supported` claim " - "must be backed by a test that actually runs on the default (non-opt-in) " - "CI suite; a `validating` claim must be grounded in a REAL opt-in / " + "must be backed by a test file that has a real passing case in its " + "required default CI job; a `validating` claim must be grounded in a REAL opt-in / " "infra-gated proof or a field test, and is never presented as " "supported. It does not replace workflow- and deployment-specific " "acceptance: application controls, identity rules, effect oracles, and " @@ -482,10 +585,11 @@ def render_markdown(results: list[ClaimResult], now: str, junit_used: bool) -> s ci = [r for r in results if r.tier == "supported"] val = [r for r in results if r.tier == "validating"] other = [r for r in results if r.tier in ("roadmap", "research")] - lines.append("## What is CI-proven today vs. being validated") + lines.append("## What is bound to required CI vs. being validated") lines.append("") lines.append( - f"- **CI-proven today ({len(ci)}):** " + ", ".join(f"`{r.id}`" for r in ci) + f"- **Bound to required CI pass evidence ({len(ci)}):** " + + ", ".join(f"`{r.id}`" for r in ci) ) lines.append( f"- **Being validated — opt-in / infra-gated or field ({len(val)}):** " @@ -537,15 +641,19 @@ def render_markdown(results: list[ClaimResult], now: str, junit_used: bool) -> s lines.append(f"- ❌ {err}") lines.append("") - return "\n".join(lines) + "\n" + return "\n".join(lines).rstrip() + "\n" def render_json( - results: list[ClaimResult], now: str, junit_used: bool + results: list[ClaimResult], + now: str, + junit_used: bool, + junit_job: Optional[str] = None, ) -> dict[str, Any]: return { "generated_at": now, "green_check_run": junit_used, + "green_check_job": junit_job, "ok": all(r.ok for r in results), "claims": [ { @@ -565,6 +673,7 @@ def render_json( "gating": e.gating, "node": e.node, "node_found": e.node_found, + "ci_job": e.ci_job, "junit_status": e.junit_status, "proves": e.proves, } @@ -583,11 +692,7 @@ def render_json( def _collect_junit(junit_path: Optional[str]) -> Optional[dict[str, str]]: if not junit_path: return None - p = Path(junit_path) - if not p.exists(): - print(f"warning: --junit artifact not found, skipping green-check: {p}") - return None - return parse_junit(p) + return parse_junit(Path(junit_path)) def main(argv: Optional[list[str]] = None) -> int: @@ -602,7 +707,18 @@ def main(argv: Optional[list[str]] = None) -> int: parser.add_argument( "--junit", default=None, - help="optional junit XML to confirm supported claims are green", + help="JUnit XML from the required job named by --ci-job", + ) + parser.add_argument( + "--ci-job", + choices=sorted(CI_JOBS), + default=None, + help="required CI job that produced --junit", + ) + parser.add_argument( + "--structure-only", + action="store_true", + help="check registry structure without claiming that supported tests passed", ) parser.add_argument( "--now", @@ -614,18 +730,39 @@ def main(argv: Optional[list[str]] = None) -> int: if not (args.check or args.report): args.check = True # default action is the gate + if args.structure_only and (args.junit or args.ci_job): + print("Claims gate FAILED: --structure-only cannot consume a JUnit result") + return 1 + if bool(args.junit) != bool(args.ci_job): + print("Claims gate FAILED: --junit and --ci-job must be supplied together") + return 1 + if args.check and not args.structure_only and not args.junit: + print( + "Claims gate FAILED: a supported-tier check requires --junit and " + "--ci-job; use --structure-only only for the separate registry-shape gate" + ) + return 1 + registry = load_registry(Path(args.registry)) - junit = _collect_junit(args.junit) - results = validate_all(registry, junit=junit) + try: + junit = _collect_junit(args.junit) + except JunitEvidenceError as exc: + print(f"Claims gate FAILED: {exc}") + return 1 + results = validate_all(registry, junit=junit, ci_job=args.ci_job) now = resolve_now(args.now) if args.report: DOC_OUT.parent.mkdir(parents=True, exist_ok=True) DOC_OUT.write_text( - render_markdown(results, now, junit is not None), encoding="utf-8" + render_markdown(results, now, junit is not None, args.ci_job), + encoding="utf-8", ) JSON_OUT.write_text( - json.dumps(render_json(results, now, junit is not None), indent=2) + "\n", + json.dumps( + render_json(results, now, junit is not None, args.ci_job), indent=2 + ) + + "\n", encoding="utf-8", ) print( @@ -642,8 +779,12 @@ def main(argv: Optional[list[str]] = None) -> int: n = len(results) proven = sum(1 for r in results if r.tier == "supported") print( - f"Claims gate passed: {n} claims, {proven} supported (CI-proven), " - "each tier backed by evidence of at least equal strength." + f"Claims gate passed: {n} claims, {proven} marked supported; " + + ( + f"all {args.ci_job} claim evidence passed." + if args.ci_job + else "registry structure is consistent; no live pass was claimed." + ) ) return 1 if errors else 0 diff --git a/tests/test_ci_workflow_contract.py b/tests/test_ci_workflow_contract.py index 9cee24af..1de98246 100644 --- a/tests/test_ci_workflow_contract.py +++ b/tests/test_ci_workflow_contract.py @@ -124,6 +124,26 @@ def test_exhaustive_identity_ladder_corpus_runs_in_the_slow_lane_only() -> None: assert workflow.count(f'{flag}: "1"') == 1 +def test_supported_claims_consume_their_required_jobs_real_junit() -> None: + """The two required test jobs must fail when cited evidence did not run.""" + + workflow = CI.read_text(encoding="utf-8") + unit_start = workflow.index("- name: Test (fast unit suite)") + unit_end = workflow.index("- name: Coverage (whole-package visibility)") + unit = workflow[unit_start:unit_end] + assert "--junitxml=runs/unit-claims-junit.xml" in unit + assert "--ci-job test --junit runs/unit-claims-junit.xml" in unit + + browser_start = workflow.index("- name: E2E (browser record -> compile -> replay)") + browser_end = workflow.index("- name: Upload run artifacts", browser_start) + browser = workflow[browser_start:browser_end] + assert "--junitxml=runs/e2e-claims-junit.xml" in browser + assert "--ci-job e2e-browser --junit runs/e2e-claims-junit.xml" in browser + + claims = VALIDATE_CLAIMS.read_text(encoding="utf-8") + assert "validate_claims.py --check --structure-only" in claims + + def test_clean_machine_lifecycle_declares_utf8_on_every_os() -> None: workflow = QUICKSTART.read_text(encoding="utf-8") lifecycle_start = workflow.index(" lifecycle:") diff --git a/tests/test_validate_claims.py b/tests/test_validate_claims.py index 94817aa0..9d42da93 100644 --- a/tests/test_validate_claims.py +++ b/tests/test_validate_claims.py @@ -196,7 +196,7 @@ def test_unknown_tier_fails() -> None: # --------------------------------------------------------------------------- # -# green-check via a junit artifact (optional path) +# required-job proof via a JUnit artifact # --------------------------------------------------------------------------- # def test_junit_green_check_flags_red_supported_test(tmp_path: Path) -> None: junit = tmp_path / "junit.xml" @@ -210,10 +210,71 @@ def test_junit_green_check_flags_red_supported_test(tmp_path: Path) -> None: encoding="utf-8", ) parsed = vc.parse_junit(junit) - assert parsed.get("test_replayer.py") == "failed" - result = vc.validate_claim(_claim(), junit=parsed) + assert parsed.get("tests/test_replayer.py") == "failed" + result = vc.validate_claim(_claim(), junit=parsed, ci_job="test") assert not result.ok - assert any("RED in junit" in e for e in result.errors) + assert any("RED in test JUnit" in e for e in result.errors) + + +def test_pytest_xunit2_classname_is_mapped_without_file_attribute( + tmp_path: Path, +) -> None: + junit = tmp_path / "junit.xml" + junit.write_text( + """ + + + """, + encoding="utf-8", + ) + + assert vc.parse_junit(junit) == {"tests/test_replayer.py": "passed"} + + +def test_supported_evidence_must_be_present_and_not_all_skipped() -> None: + missing = vc.validate_claim(_claim(), junit={}, ci_job="test") + skipped = vc.validate_claim(_claim(), junit={CI_TEST: "skipped"}, ci_job="test") + passed = vc.validate_claim(_claim(), junit={CI_TEST: "passed"}, ci_job="test") + + assert any("ABSENT" in error for error in missing.errors) + assert any("SKIPPED" in error for error in skipped.errors) + assert passed.ok, passed.errors + + +def test_junit_file_is_passed_when_one_case_passes_and_an_optional_case_skips( + tmp_path: Path, +) -> None: + junit = tmp_path / "junit.xml" + junit.write_text( + """ + + + + """, + encoding="utf-8", + ) + + assert vc.parse_junit(junit)[CI_TEST] == "passed" + + +def test_missing_or_empty_junit_fails_closed(tmp_path: Path) -> None: + missing = tmp_path / "missing.xml" + empty = tmp_path / "empty.xml" + empty.write_text("", encoding="utf-8") + + for path in (missing, empty): + try: + vc.parse_junit(path) + except vc.JunitEvidenceError: + pass + else: + raise AssertionError(f"{path} did not fail closed") + + +def test_cli_requires_junit_for_non_structural_supported_check() -> None: + assert vc.main(["--check"]) == 1 + assert vc.main(["--check", "--structure-only"]) == 0 + assert vc.main(["--check", "--junit", "missing.xml"]) == 1 def test_report_renders_without_crashing() -> None: @@ -221,6 +282,13 @@ def test_report_renders_without_crashing() -> None: md = vc.render_markdown(results, now="2026-07-14T00:00:00Z", junit_used=False) assert "VERIFICATION" in md assert "web-supported" in md + assert md.endswith("\n") and not md.endswith("\n\n") blob = vc.render_json(results, now="2026-07-14T00:00:00Z", junit_used=False) assert blob["ok"] is True + assert blob["green_check_job"] is None + assert all( + "ci_job" in evidence + for claim in blob["claims"] + for evidence in claim["evidence"] + ) assert {c["id"] for c in blob["claims"]} # non-empty