From 463f5e611bce957a5e333b6e278285db767463bd Mon Sep 17 00:00:00 2001 From: saagpatel Date: Sat, 18 Jul 2026 00:10:31 -0700 Subject: [PATCH 1/3] ci(research): enforce integrity gates --- .github/workflows/research-integrity.yml | 71 ++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/research-integrity.yml diff --git a/.github/workflows/research-integrity.yml b/.github/workflows/research-integrity.yml new file mode 100644 index 0000000..7c7b2ae --- /dev/null +++ b/.github/workflows/research-integrity.yml @@ -0,0 +1,71 @@ +name: Engraph research integrity + +on: + pull_request: + branches: + - feat/engraph-p3-perf + paths: + - models/** + - research-evidence/** + - scripts/research_receipt.py + - scripts/test_research_receipt.py + - scripts/test_verify_*.py + - scripts/verify_*.py + - examples/p3_actor_parity.rs + - src/llm.rs + - src/serve.rs + - Cargo.lock + - target/release/examples/p3_actor_parity + - .github/workflows/research-integrity.yml + push: + branches: + - feat/engraph-p3-perf + paths: + - models/** + - research-evidence/** + - scripts/research_receipt.py + - scripts/test_research_receipt.py + - scripts/test_verify_*.py + - scripts/verify_*.py + - examples/p3_actor_parity.rs + - src/llm.rs + - src/serve.rs + - Cargo.lock + - target/release/examples/p3_actor_parity + - .github/workflows/research-integrity.yml + +permissions: + contents: read + +concurrency: + group: engraph-research-integrity-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + PYTHONDONTWRITEBYTECODE: "1" + PYTHONPATH: scripts + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Verify shared manifest contract + run: python3 research-evidence/contracts/v2/verify_contract.py + - name: Verify model provenance + run: python3 scripts/verify_model_provenance.py + - name: Verify claim durability + run: python3 scripts/verify_claim_durability.py + - name: Verify authentic P3 correctness receipt + run: python3 scripts/research_receipt.py verify research-evidence/runs/p3-actor-parity-v2-20260717-a2.receipt.json + - name: Audit the authentic P3 attempt + run: python3 scripts/research_receipt.py audit-plans research-evidence/runs --attempt-id 076b69d2-3a1f-40d6-9555-9c7cc6c69302 + - name: Run integrity tests + run: python3 -m unittest discover -s scripts -p "test_*.py" -q + - name: Run integrity tests without assertions + run: python3 -O -m unittest discover -s scripts -p "test_*.py" -q From b8cb198298fc7f26052055b3eb5e983a0c4cde46 Mon Sep 17 00:00:00 2001 From: saagpatel Date: Sat, 18 Jul 2026 00:18:26 -0700 Subject: [PATCH 2/3] fix(ci): separate receipt record from custody --- .github/workflows/research-integrity.yml | 8 +++---- research-evidence/README.md | 16 +++++++++++++ scripts/research_receipt.py | 30 +++++++++++++++++++++++- scripts/test_research_receipt.py | 16 +++++++++++++ 4 files changed, 65 insertions(+), 5 deletions(-) diff --git a/.github/workflows/research-integrity.yml b/.github/workflows/research-integrity.yml index 7c7b2ae..d65263f 100644 --- a/.github/workflows/research-integrity.yml +++ b/.github/workflows/research-integrity.yml @@ -49,10 +49,10 @@ jobs: PYTHONDONTWRITEBYTECODE: "1" PYTHONPATH: scripts steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" - name: Verify shared manifest contract @@ -62,9 +62,9 @@ jobs: - name: Verify claim durability run: python3 scripts/verify_claim_durability.py - name: Verify authentic P3 correctness receipt - run: python3 scripts/research_receipt.py verify research-evidence/runs/p3-actor-parity-v2-20260717-a2.receipt.json + run: python3 scripts/research_receipt.py verify-record research-evidence/runs/p3-actor-parity-v2-20260717-a2.receipt.json - name: Audit the authentic P3 attempt - run: python3 scripts/research_receipt.py audit-plans research-evidence/runs --attempt-id 076b69d2-3a1f-40d6-9555-9c7cc6c69302 + run: python3 scripts/research_receipt.py audit-plans research-evidence/runs --attempt-id 076b69d2-3a1f-40d6-9555-9c7cc6c69302 --record-only - name: Run integrity tests run: python3 -m unittest discover -s scripts -p "test_*.py" -q - name: Run integrity tests without assertions diff --git a/research-evidence/README.md b/research-evidence/README.md index cdbc7a7..78aebd4 100644 --- a/research-evidence/README.md +++ b/research-evidence/README.md @@ -86,6 +86,22 @@ complete fail-closed envelope and globally unique event identity. Use `--attempt-id ` to evaluate one new attempt without allowing an unrelated historical failure to become either its pass or its failure. +Clean clones and CI can validate the committed receipt envelope and repository +artifacts without claiming custody of unavailable model or tokenizer bytes: + +```sh +python3 scripts/research_receipt.py verify-record \ + research-evidence/runs/p3-actor-parity-v2-20260717-a2.receipt.json +python3 scripts/research_receipt.py audit-plans research-evidence/runs \ + --attempt-id 076b69d2-3a1f-40d6-9555-9c7cc6c69302 \ + --record-only +``` + +Record-only verification filters only explicitly classified unavailable +external artifacts. Malformed, changed, unsafe, or hash-mismatched evidence +still fails. The strict `verify` and default `audit-plans` commands remain the +custody checks and must not be replaced by record-only results. + Receipts contain a canonical self-digest for accidental corruption detection. The digest is stored beside editable metadata and is not authentication. They are not signed, externally timestamped, or stored on immutable media, so diff --git a/scripts/research_receipt.py b/scripts/research_receipt.py index 28ef081..9f30f5b 100644 --- a/scripts/research_receipt.py +++ b/scripts/research_receipt.py @@ -1422,6 +1422,15 @@ def verify_receipt(path: Path) -> list[str]: return errors +def verify_receipt_record(path: Path) -> list[str]: + """Verify a committed receipt without claiming external-byte custody.""" + return [ + error + for error in verify_receipt(path) + if error not in TERMINAL_UNAVAILABLE_ERRORS + ] + + def _write_exclusive_json(path: Path, value: dict[str, Any]) -> None: path = path.resolve() path.parent.mkdir(parents=True, exist_ok=True) @@ -1586,6 +1595,7 @@ def incomplete_attempts( directory: Path, *, attempt_id_filter: str | None = None, + record_only: bool = False, ) -> list[dict[str, Any]]: resolved = directory.resolve() try: @@ -1619,7 +1629,12 @@ def incomplete_attempts( errors.append("planned event unexpectedly has a parent") planned.setdefault(attempt_id, []).append((path, errors)) elif value.get("record_type") == "attempt_result": - terminal.setdefault(attempt_id, []).append((path, verify_receipt(path))) + receipt_errors = ( + verify_receipt_record(path) + if record_only + else verify_receipt(path) + ) + terminal.setdefault(attempt_id, []).append((path, receipt_errors)) terminal_plan_required[attempt_id] = ( terminal_plan_required.get(attempt_id, False) or value.get("execution_provenance") == "tool_executed" @@ -1801,9 +1816,16 @@ def common(run_parser: argparse.ArgumentParser) -> None: run.add_argument("measured_command", nargs=argparse.REMAINDER) verify = sub.add_parser("verify") verify.add_argument("receipt", type=Path) + verify_record = sub.add_parser("verify-record") + verify_record.add_argument("receipt", type=Path) audit = sub.add_parser("audit-plans") audit.add_argument("directory", type=Path) audit.add_argument("--attempt-id") + audit.add_argument( + "--record-only", + action="store_true", + help="do not claim custody of unavailable external artifacts", + ) args = parser.parse_args() if args.action == "create": print(json.dumps(create_receipt(args), sort_keys=True)) @@ -1814,10 +1836,16 @@ def common(run_parser: argparse.ArgumentParser) -> None: if errors: raise SystemExit("\n".join(errors)) print("OK") + elif args.action == "verify-record": + errors = verify_receipt_record(args.receipt) + if errors: + raise SystemExit("\n".join(errors)) + print("OK: receipt record verified; external byte custody not claimed") else: issues = incomplete_attempts( args.directory, attempt_id_filter=args.attempt_id, + record_only=args.record_only, ) print(json.dumps(issues, indent=2, sort_keys=True)) if issues: diff --git a/scripts/test_research_receipt.py b/scripts/test_research_receipt.py index 008bd3d..70f132e 100644 --- a/scripts/test_research_receipt.py +++ b/scripts/test_research_receipt.py @@ -14,6 +14,22 @@ class ReceiptTests(unittest.TestCase): + def test_committed_a2_record_is_clone_verifiable(self) -> None: + receipt = ( + rr.ROOT + / "research-evidence/runs/" + "p3-actor-parity-v2-20260717-a2.receipt.json" + ) + self.assertEqual(rr.verify_receipt_record(receipt), []) + self.assertEqual( + rr.incomplete_attempts( + rr.ROOT / "research-evidence/runs", + attempt_id_filter="076b69d2-3a1f-40d6-9555-9c7cc6c69302", + record_only=True, + ), + [], + ) + def test_v2_verifier_rejects_rights_and_artifact_overclaims(self) -> None: representatives = json.loads( ( From 63e18cf4c9e2f4ee1d577f935803a49503b77c6a Mon Sep 17 00:00:00 2001 From: saagpatel Date: Sat, 18 Jul 2026 00:20:19 -0700 Subject: [PATCH 3/3] fix(ci): derive home path in receipt test --- scripts/test_research_receipt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/test_research_receipt.py b/scripts/test_research_receipt.py index 70f132e..0e31138 100644 --- a/scripts/test_research_receipt.py +++ b/scripts/test_research_receipt.py @@ -117,8 +117,9 @@ def test_command_evidence_rejects_secrets_without_false_tokenizer_match( rr._safe_command_argv(["tool", "--tokenizer", "embedded"]), ["tool", "--tokenizer", "embedded"], ) + home_input = Path.home() / "private" / "file" self.assertEqual( - rr._safe_command_argv(["tool", "--input=/Users/d/private/file"])[1], + rr._safe_command_argv(["tool", f"--input={home_input}"])[1], "--input=$HOME/private/file", ) with self.assertRaisesRegex(ValueError, "secret-bearing"):