From 5bedae43dd3282f843804aed4f107ca58e2e4a27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 22:28:52 +0900 Subject: [PATCH 1/4] test(opencode): separate semantic and merge decisions --- .../opencode-review-decision-quality-ci.yml | 82 ++++++++++ .../opencode_review_decision_test_support.py | 95 ++++++++++++ .../test_opencode_review_decision_channels.py | 143 ++++++++++++++++++ tests/test_opencode_review_decision_cli.py | 119 +++++++++++++++ ...est_opencode_review_decision_validation.py | 118 +++++++++++++++ 5 files changed, 557 insertions(+) create mode 100644 .github/workflows/opencode-review-decision-quality-ci.yml create mode 100644 tests/opencode_review_decision_test_support.py create mode 100644 tests/test_opencode_review_decision_channels.py create mode 100644 tests/test_opencode_review_decision_cli.py create mode 100644 tests/test_opencode_review_decision_validation.py diff --git a/.github/workflows/opencode-review-decision-quality-ci.yml b/.github/workflows/opencode-review-decision-quality-ci.yml new file mode 100644 index 000000000..aa6739f54 --- /dev/null +++ b/.github/workflows/opencode-review-decision-quality-ci.yml @@ -0,0 +1,82 @@ +name: OpenCode Review Decision Quality CI + +on: + pull_request: + branches: + - main + - feat/opencode-review-gold-corpus + paths: + - ".github/workflows/opencode-review-decision-quality-ci.yml" + - "scripts/ci/opencode_review_decision.py" + - "tests/opencode_review_decision_test_support.py" + - "tests/test_opencode_review_decision_*.py" + - "docs/doctoring/opencode-review-decision-envelope.md" + - "CHANGELOG.md" + +permissions: + contents: read + +concurrency: + group: opencode-review-decision-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + decision-envelope-quality: + name: decision-envelope-quality + if: github.event_name != 'pull_request' || github.event.action != 'closed' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout exact source revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install exact hash-verified test runner dependencies + env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PIP_NO_INPUT: "1" + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + cat >"${RUNNER_TEMP}/opencode-review-decision-requirements.txt" <<'REQEOF' + coverage==7.15.2 --hash=sha256:b9a6367e4aff723e8ee8190836836124284e8fcd4265e307c844010cfa074f3f + iniconfig==2.1.0 --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 + packaging==26.2 --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e + pluggy==1.6.0 --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + pygments==2.20.0 --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + pytest==9.1.1 --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c + REQEOF + python -m pip install \ + --only-binary=:all: \ + --require-hashes \ + -r "${RUNNER_TEMP}/opencode-review-decision-requirements.txt" + + - name: Verify independent decision channels + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + test "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha || github.sha }}" + python -m coverage run \ + --branch \ + --source=scripts/ci \ + -m pytest \ + tests/test_opencode_review_decision_channels.py \ + tests/test_opencode_review_decision_validation.py \ + tests/test_opencode_review_decision_cli.py \ + -q + python -m coverage report \ + --include='scripts/ci/opencode_review_decision.py' \ + --fail-under=100 \ + --show-missing + python -m compileall -q \ + scripts/ci/opencode_review_decision.py \ + tests/opencode_review_decision_test_support.py \ + tests/test_opencode_review_decision_channels.py \ + tests/test_opencode_review_decision_validation.py \ + tests/test_opencode_review_decision_cli.py + git diff --exit-code diff --git a/tests/opencode_review_decision_test_support.py b/tests/opencode_review_decision_test_support.py new file mode 100644 index 000000000..e805c253c --- /dev/null +++ b/tests/opencode_review_decision_test_support.py @@ -0,0 +1,95 @@ +"""Shared fixtures for OpenCode decision-envelope tests.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from types import ModuleType +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "scripts/ci/opencode_review_decision.py" + + +def load_module() -> ModuleType: + """Load the exact decision module without package import side effects.""" + spec = importlib.util.spec_from_file_location("opencode_review_decision", MODULE_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +decision = load_module() + + +def finding( + identifier: str = "finding_001", + *, + severity: str = "high", + blocking: bool = True, +) -> dict[str, Any]: + """Build one complete semantic source finding.""" + return { + "finding_id": identifier, + "defect_class": "correctness", + "severity": severity, + "blocking": blocking, + "path": "scripts/ci/example.py", + "line": 12, + "trigger": "The input contains a duplicate exact-head identity.", + "impact": "The benchmark counts one pull request twice.", + "root_cause": "The identity set is not checked before aggregation.", + "fix_direction": "Reject duplicate repository, PR, and head tuples.", + "regression_target": "Add a duplicate exact-head fixture.", + } + + +def check( + name: str = "CI", + *, + state: str = "success", + required: bool = True, + head_sha: str | None = None, +) -> dict[str, Any]: + """Build one exact-head check evidence record.""" + return { + "name": name, + "state": state, + "required": required, + "head_sha": head_sha or "b" * 40, + } + + +def envelope( + *, + semantic_status: str = "complete", + findings: list[dict[str, Any]] | None = None, + coverage_state: str = "success", + approval_state: str = "success", + protection_state: str = "success", + checks: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Build one decision input with all evidence bound to one immutable head.""" + complete = semantic_status == "complete" + return { + "schema_version": "1.0", + "decision_id": "decision_001", + "quality_policy_version": "opencode-review-quality-v1", + "repository": "ContextualWisdomLab/example", + "pull_request_number": 42, + "base_sha": "a" * 40, + "head_sha": "b" * 40, + "semantic_review": { + "status": semantic_status, + "reviewed_head_sha": "b" * 40 if complete else None, + "findings": findings if findings is not None else [], + }, + "merge_evidence": { + "evidence_head_sha": "b" * 40, + "coverage_state": coverage_state, + "independent_approval_state": approval_state, + "branch_protection_state": protection_state, + "required_checks": checks if checks is not None else [check()], + }, + } diff --git a/tests/test_opencode_review_decision_channels.py b/tests/test_opencode_review_decision_channels.py new file mode 100644 index 000000000..32ada2c1c --- /dev/null +++ b/tests/test_opencode_review_decision_channels.py @@ -0,0 +1,143 @@ +"""Behavior tests for independent semantic and merge-readiness channels.""" + +from __future__ import annotations + +import copy + +from opencode_review_decision_test_support import check, decision, envelope, finding + + +def test_coverage_failure_cannot_create_a_source_finding() -> None: + """Coverage failure must block readiness without becoming a line-level defect.""" + report = decision.build_decision(envelope(coverage_state="failure")) + assert report["review_verdict"] == "APPROVE" + assert report["merge_readiness"] == "BLOCKED" + assert report["findings"] == [] + assert report["semantic_status"] == "complete" + assert report["infrastructure_blockers"] == [ + { + "blocker_code": "coverage_not_successful", + "evidence_name": "coverage", + "state": "failure", + "check_name": None, + } + ] + assert all( + "path" not in blocker and "line" not in blocker + for blocker in report["infrastructure_blockers"] + ) + + +def test_semantic_finding_survives_independent_coverage_failure() -> None: + """A real source defect and infrastructure blocker remain separate channels.""" + report = decision.build_decision( + envelope(findings=[finding()], coverage_state="failure") + ) + assert report["review_verdict"] == "REQUEST_CHANGES" + assert report["merge_readiness"] == "BLOCKED" + assert [item["finding_id"] for item in report["findings"]] == ["finding_001"] + assert report["infrastructure_blockers"][0]["evidence_name"] == "coverage" + + +def test_semantic_verdict_matrix_is_independent_of_merge_evidence() -> None: + """Complete semantic review alone chooses approve, comment, or request changes.""" + assert decision.build_decision(envelope())["review_verdict"] == "APPROVE" + assert ( + decision.build_decision(envelope(findings=[finding(blocking=False)]))[ + "review_verdict" + ] + == "COMMENT" + ) + assert ( + decision.build_decision(envelope(findings=[finding(blocking=True)]))[ + "review_verdict" + ] + == "REQUEST_CHANGES" + ) + for status in ("unavailable", "failed"): + report = decision.build_decision(envelope(semantic_status=status)) + assert report["review_verdict"] == "ABSTAIN" + assert report["findings"] == [] + + +def test_merge_readiness_ready_blocked_and_unknown_states() -> None: + """Readiness distinguishes hard failure from latency or absent evidence.""" + assert decision.build_decision(envelope())["merge_readiness"] == "READY" + assert ( + decision.build_decision(envelope(findings=[finding()]))["merge_readiness"] + == "BLOCKED" + ) + assert ( + decision.build_decision(envelope(checks=[check(state="cancelled")]))[ + "merge_readiness" + ] + == "BLOCKED" + ) + for state in ("pending", "queued", "absent"): + assert ( + decision.build_decision(envelope(checks=[check(state=state)]))[ + "merge_readiness" + ] + == "UNKNOWN" + ) + assert ( + decision.build_decision(envelope(semantic_status="unavailable"))[ + "merge_readiness" + ] + == "UNKNOWN" + ) + + +def test_required_and_advisory_checks_are_classified_separately() -> None: + """Only required checks block readiness, while advisory evidence is recorded.""" + report = decision.build_decision( + envelope( + checks=[ + check("required-ci", state="success", required=True), + check("advisory-lint", state="failure", required=False), + ] + ) + ) + assert report["merge_readiness"] == "READY" + assert report["infrastructure_blockers"] == [] + manifest = report["evidence_manifest"] + assert manifest["required_check_count"] == 1 + assert manifest["successful_required_check_count"] == 1 + assert manifest["advisory_check_count"] == 1 + + +def test_every_non_successful_policy_surface_produces_non_source_blockers() -> None: + """Coverage, approval, protection, and required checks report stable blockers.""" + report = decision.build_decision( + envelope( + coverage_state="neutral", + approval_state="absent", + protection_state="pending", + checks=[check("unit", state="failure"), check("security", state="skipped")], + ) + ) + assert report["merge_readiness"] == "BLOCKED" + assert { + (item["evidence_name"], item["state"], item["check_name"]) + for item in report["infrastructure_blockers"] + } == { + ("coverage", "neutral", None), + ("independent_approval", "absent", None), + ("branch_protection", "pending", None), + ("required_check", "failure", "unit"), + ("required_check", "skipped", "security"), + } + assert all( + "path" not in item and "line" not in item + for item in report["infrastructure_blockers"] + ) + + +def test_output_is_deterministic_and_receipt_bound() -> None: + """Equivalent exact-head input produces one stable content-addressed decision.""" + value = envelope(findings=[finding(blocking=False)]) + first = decision.build_decision(copy.deepcopy(value)) + second = decision.build_decision(copy.deepcopy(value)) + assert first == second + assert first["decision_sha256"].startswith("sha256:") + assert first["evidence_manifest"]["input_sha256"].startswith("sha256:") diff --git a/tests/test_opencode_review_decision_cli.py b/tests/test_opencode_review_decision_cli.py new file mode 100644 index 000000000..e89b3722e --- /dev/null +++ b/tests/test_opencode_review_decision_cli.py @@ -0,0 +1,119 @@ +"""Serialization and CLI tests for OpenCode decision envelopes.""" + +from __future__ import annotations + +import json +import runpy +from pathlib import Path + +import pytest + +from opencode_review_decision_test_support import MODULE_PATH, decision, envelope, finding + + +def test_markdown_keeps_findings_and_infrastructure_blockers_separate() -> None: + """Human summaries never render infrastructure failure as a source line.""" + report = decision.build_decision( + envelope(findings=[finding()], coverage_state="failure") + ) + markdown = decision.render_markdown(report) + assert "## Semantic findings" in markdown + assert "scripts/ci/example.py:12" in markdown + assert "## Infrastructure and policy blockers" in markdown + blocker_section = markdown.split("## Infrastructure and policy blockers", 1)[1] + assert "coverage" in blocker_section + assert ".github/workflows/opencode-review.yml:1" not in blocker_section + + +def test_strict_json_rejects_duplicate_keys_and_nonfinite_numbers(tmp_path: Path) -> None: + """Decision evidence rejects ambiguous JSON and numeric extensions.""" + duplicate = tmp_path / "duplicate.json" + duplicate.write_text('{"schema_version":"1.0","schema_version":"1.0"}') + with pytest.raises(decision.DecisionValidationError, match="duplicate JSON key"): + decision.load_json(duplicate) + + nonfinite = tmp_path / "nonfinite.json" + nonfinite.write_text('{"line": NaN}') + with pytest.raises(decision.DecisionValidationError, match="non-finite JSON number"): + decision.load_json(nonfinite) + + +def test_cli_writes_atomic_json_and_markdown_with_stable_errors( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """The CLI publishes both decision views atomically or rejects the input.""" + source = tmp_path / "input.json" + json_output = tmp_path / "nested" / "decision.json" + markdown_output = tmp_path / "nested" / "decision.md" + source.write_text(json.dumps(envelope()), encoding="utf-8") + assert ( + decision.main( + [ + "--input", + str(source), + "--json-output", + str(json_output), + "--markdown-output", + str(markdown_output), + ] + ) + == 0 + ) + assert json.loads(json_output.read_text(encoding="utf-8"))["merge_readiness"] == "READY" + assert "Review verdict: **APPROVE**" in markdown_output.read_text(encoding="utf-8") + assert not json_output.with_name(f".{json_output.name}.tmp").exists() + assert not markdown_output.with_name(f".{markdown_output.name}.tmp").exists() + + source.write_text("[]", encoding="utf-8") + assert ( + decision.main( + [ + "--input", + str(source), + "--json-output", + str(json_output), + "--markdown-output", + str(markdown_output), + ] + ) + == 2 + ) + assert "decision evidence rejected" in capsys.readouterr().err + + +def test_module_entrypoint_routes_through_main( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Direct script execution uses the same tested CLI boundary.""" + source = tmp_path / "input.json" + json_output = tmp_path / "decision.json" + markdown_output = tmp_path / "decision.md" + source.write_text(json.dumps(envelope()), encoding="utf-8") + monkeypatch.setattr( + "sys.argv", + [ + str(MODULE_PATH), + "--input", + str(source), + "--json-output", + str(json_output), + "--markdown-output", + str(markdown_output), + ], + ) + with pytest.raises(SystemExit, match="0"): + runpy.run_path(str(MODULE_PATH), run_name="__main__") + assert json_output.exists() and markdown_output.exists() + + +def test_public_production_callables_have_docstrings() -> None: + """Every production class and function remains beginner-readable.""" + missing = [ + name + for name, value in vars(decision).items() + if not name.startswith("_") + and (isinstance(value, type) or callable(value)) + and getattr(value, "__module__", None) == decision.__name__ + and not getattr(value, "__doc__", None) + ] + assert missing == [] diff --git a/tests/test_opencode_review_decision_validation.py b/tests/test_opencode_review_decision_validation.py new file mode 100644 index 000000000..b30d76af3 --- /dev/null +++ b/tests/test_opencode_review_decision_validation.py @@ -0,0 +1,118 @@ +"""Validation tests for exact-head OpenCode decision evidence.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from opencode_review_decision_test_support import check, decision, envelope, finding + + +def test_exact_head_binding_rejects_stale_semantic_and_merge_evidence() -> None: + """No semantic, check, or policy evidence may transfer from another head.""" + stale_semantic = envelope() + stale_semantic["semantic_review"]["reviewed_head_sha"] = "c" * 40 + with pytest.raises(decision.DecisionValidationError, match="reviewed_head_sha"): + decision.build_decision(stale_semantic) + + stale_merge = envelope() + stale_merge["merge_evidence"]["evidence_head_sha"] = "c" * 40 + with pytest.raises(decision.DecisionValidationError, match="evidence_head_sha"): + decision.build_decision(stale_merge) + + stale_check = envelope() + stale_check["merge_evidence"]["required_checks"][0]["head_sha"] = "c" * 40 + with pytest.raises( + decision.DecisionValidationError, match=r"required_checks\[0\].head_sha" + ): + decision.build_decision(stale_check) + + +def test_incomplete_semantic_review_cannot_carry_findings_or_head_claim() -> None: + """Unavailable or failed reviews abstain without synthetic source evidence.""" + for status in ("unavailable", "failed"): + with_findings = envelope(semantic_status=status, findings=[finding()]) + with pytest.raises( + decision.DecisionValidationError, match="must not contain findings" + ): + decision.build_decision(with_findings) + + with_head = envelope(semantic_status=status) + with_head["semantic_review"]["reviewed_head_sha"] = "b" * 40 + with pytest.raises(decision.DecisionValidationError, match="must be null"): + decision.build_decision(with_head) + + +def test_complete_semantic_review_requires_exact_head() -> None: + """A completed semantic verdict without an exact reviewed head is invalid.""" + value = envelope() + value["semantic_review"]["reviewed_head_sha"] = None + with pytest.raises(decision.DecisionValidationError, match="reviewed_head_sha"): + decision.build_decision(value) + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda value: value.update({"unexpected": True}), "unknown fields"), + ( + lambda value: value["semantic_review"].update({"unexpected": True}), + "unknown fields", + ), + ( + lambda value: value["merge_evidence"].update({"unexpected": True}), + "unknown fields", + ), + ( + lambda value: value["semantic_review"]["findings"][0].update( + {"unexpected": True} + ), + "unknown fields", + ), + ( + lambda value: value["merge_evidence"]["required_checks"][0].update( + {"unexpected": True} + ), + "unknown fields", + ), + (lambda value: value.update({"pull_request_number": True}), "integer"), + ( + lambda value: value["semantic_review"].update({"status": "running"}), + "semantic_review.status", + ), + ( + lambda value: value["merge_evidence"].update( + {"coverage_state": "green"} + ), + "coverage_state", + ), + ( + lambda value: value["semantic_review"]["findings"][0].update( + {"path": "../secret"} + ), + "relative source path", + ), + ( + lambda value: value["semantic_review"]["findings"][0].update({"line": 0}), + "positive integer", + ), + ], +) +def test_strict_schema_rejects_unknown_fields_and_scalar_confusion( + mutate: Any, message: str +) -> None: + """Every evidence layer fails closed on malformed control data.""" + value = envelope(findings=[finding()]) + mutate(value) + with pytest.raises(decision.DecisionValidationError, match=message): + decision.build_decision(value) + + +def test_duplicate_finding_and_check_names_are_rejected() -> None: + """Duplicate semantic or check identities cannot inflate evidence counts.""" + with pytest.raises(decision.DecisionValidationError, match="finding_id"): + decision.build_decision(envelope(findings=[finding(), finding()])) + + with pytest.raises(decision.DecisionValidationError, match="check name"): + decision.build_decision(envelope(checks=[check("CI"), check("ci")])) From 48a609d177b2fcd5d8a261570bc845ad5abb1af3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 22:32:30 +0900 Subject: [PATCH 2/4] test(opencode): load decision fixtures deterministically --- tests/test_opencode_review_decision_channels.py | 6 ++++++ tests/test_opencode_review_decision_cli.py | 5 +++++ tests/test_opencode_review_decision_validation.py | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/tests/test_opencode_review_decision_channels.py b/tests/test_opencode_review_decision_channels.py index 32ada2c1c..acf1632dd 100644 --- a/tests/test_opencode_review_decision_channels.py +++ b/tests/test_opencode_review_decision_channels.py @@ -3,6 +3,12 @@ from __future__ import annotations import copy +import sys +from pathlib import Path + +TEST_DIR = Path(__file__).resolve().parent +if str(TEST_DIR) not in sys.path: + sys.path.insert(0, str(TEST_DIR)) from opencode_review_decision_test_support import check, decision, envelope, finding diff --git a/tests/test_opencode_review_decision_cli.py b/tests/test_opencode_review_decision_cli.py index e89b3722e..56592d137 100644 --- a/tests/test_opencode_review_decision_cli.py +++ b/tests/test_opencode_review_decision_cli.py @@ -4,10 +4,15 @@ import json import runpy +import sys from pathlib import Path import pytest +TEST_DIR = Path(__file__).resolve().parent +if str(TEST_DIR) not in sys.path: + sys.path.insert(0, str(TEST_DIR)) + from opencode_review_decision_test_support import MODULE_PATH, decision, envelope, finding diff --git a/tests/test_opencode_review_decision_validation.py b/tests/test_opencode_review_decision_validation.py index b30d76af3..d86421cfc 100644 --- a/tests/test_opencode_review_decision_validation.py +++ b/tests/test_opencode_review_decision_validation.py @@ -2,10 +2,16 @@ from __future__ import annotations +import sys +from pathlib import Path from typing import Any import pytest +TEST_DIR = Path(__file__).resolve().parent +if str(TEST_DIR) not in sys.path: + sys.path.insert(0, str(TEST_DIR)) + from opencode_review_decision_test_support import check, decision, envelope, finding From 1b48704a09afa017aceb337264ca6fb826bbf706 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 22:43:36 +0900 Subject: [PATCH 3/4] feat(opencode): build independent decision envelope --- .../opencode-review-decision-quality-ci.yml | 6 +- scripts/ci/opencode_review_decision.py | 228 +++++++++++++++++ .../ci/opencode_review_decision_primitives.py | 167 ++++++++++++ .../ci/opencode_review_decision_validation.py | 239 ++++++++++++++++++ tests/test_opencode_review_decision_cli.py | 10 + ...est_opencode_review_decision_validation.py | 25 ++ 6 files changed, 674 insertions(+), 1 deletion(-) create mode 100644 scripts/ci/opencode_review_decision.py create mode 100644 scripts/ci/opencode_review_decision_primitives.py create mode 100644 scripts/ci/opencode_review_decision_validation.py diff --git a/.github/workflows/opencode-review-decision-quality-ci.yml b/.github/workflows/opencode-review-decision-quality-ci.yml index aa6739f54..1e6d2ccc3 100644 --- a/.github/workflows/opencode-review-decision-quality-ci.yml +++ b/.github/workflows/opencode-review-decision-quality-ci.yml @@ -8,6 +8,8 @@ on: paths: - ".github/workflows/opencode-review-decision-quality-ci.yml" - "scripts/ci/opencode_review_decision.py" + - "scripts/ci/opencode_review_decision_primitives.py" + - "scripts/ci/opencode_review_decision_validation.py" - "tests/opencode_review_decision_test_support.py" - "tests/test_opencode_review_decision_*.py" - "docs/doctoring/opencode-review-decision-envelope.md" @@ -70,11 +72,13 @@ jobs: tests/test_opencode_review_decision_cli.py \ -q python -m coverage report \ - --include='scripts/ci/opencode_review_decision.py' \ + --include='scripts/ci/opencode_review_decision.py,scripts/ci/opencode_review_decision_primitives.py,scripts/ci/opencode_review_decision_validation.py' \ --fail-under=100 \ --show-missing python -m compileall -q \ scripts/ci/opencode_review_decision.py \ + scripts/ci/opencode_review_decision_primitives.py \ + scripts/ci/opencode_review_decision_validation.py \ tests/opencode_review_decision_test_support.py \ tests/test_opencode_review_decision_channels.py \ tests/test_opencode_review_decision_validation.py \ diff --git a/scripts/ci/opencode_review_decision.py b/scripts/ci/opencode_review_decision.py new file mode 100644 index 000000000..2d0763b9d --- /dev/null +++ b/scripts/ci/opencode_review_decision.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Build independent semantic-review and merge-readiness decisions.""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +MODULE_DIR = Path(__file__).resolve().parent +if str(MODULE_DIR) not in sys.path: + sys.path.insert(0, str(MODULE_DIR)) + +from opencode_review_decision_primitives import ( # noqa: E402 + HARD_BLOCKING_STATES, + UNKNOWN_STATES, + DecisionValidationError, + array_value, + bool_value, + commit_sha_value, + content_digest, + load_json, + reject_constant, + strict_pairs, + text_value, + write_text, +) +from opencode_review_decision_validation import validate_decision_input # noqa: E402 + + +def blocker( + blocker_code: str, evidence_name: str, state: str, check_name: str | None = None +) -> dict[str, Any]: + """Build one path-free infrastructure or policy blocker.""" + return { + "blocker_code": blocker_code, + "evidence_name": evidence_name, + "state": state, + "check_name": check_name, + } + + +def classify_review_verdict(semantic_review: Mapping[str, Any]) -> str: + """Choose a semantic verdict without consulting merge-readiness evidence.""" + if semantic_review["status"] != "complete": + return "ABSTAIN" + findings = semantic_review["findings"] + if any(item["blocking"] for item in findings): + return "REQUEST_CHANGES" + return "COMMENT" if findings else "APPROVE" + + +def collect_blockers(merge_evidence: Mapping[str, Any]) -> list[dict[str, Any]]: + """Collect non-successful policy evidence without source-location authority.""" + blockers: list[dict[str, Any]] = [] + policy_surfaces = ( + ( + "coverage_state", + "coverage_not_successful", + "coverage", + ), + ( + "independent_approval_state", + "independent_approval_not_successful", + "independent_approval", + ), + ( + "branch_protection_state", + "branch_protection_not_successful", + "branch_protection", + ), + ) + for field, code, evidence_name in policy_surfaces: + state = merge_evidence[field] + if state != "success": + blockers.append(blocker(code, evidence_name, state)) + for check in merge_evidence["required_checks"]: + if check["required"] and check["state"] != "success": + blockers.append( + blocker( + "required_check_not_successful", + "required_check", + check["state"], + check["name"], + ) + ) + return blockers + + +def classify_merge_readiness( + review_verdict: str, blockers: Sequence[Mapping[str, Any]] +) -> str: + """Classify merge readiness using fail-closed policy evidence and latency states.""" + if review_verdict == "REQUEST_CHANGES": + return "BLOCKED" + blocker_states = {item["state"] for item in blockers} + if blocker_states & HARD_BLOCKING_STATES: + return "BLOCKED" + if review_verdict == "ABSTAIN" or blocker_states & UNKNOWN_STATES: + return "UNKNOWN" + return "READY" + + +def build_decision(raw_value: Any) -> dict[str, Any]: + """Build one deterministic exact-head decision with independent channels.""" + value = validate_decision_input(raw_value) + semantic_review = value["semantic_review"] + merge_evidence = value["merge_evidence"] + review_verdict = classify_review_verdict(semantic_review) + blockers = collect_blockers(merge_evidence) + required_checks = [item for item in merge_evidence["required_checks"] if item["required"]] + advisory_checks = [item for item in merge_evidence["required_checks"] if not item["required"]] + report_without_digest = { + "schema_version": "1.0", + "decision_id": value["decision_id"], + "quality_policy_version": value["quality_policy_version"], + "repository": value["repository"], + "pull_request_number": value["pull_request_number"], + "base_sha": value["base_sha"], + "head_sha": value["head_sha"], + "semantic_status": semantic_review["status"], + "review_verdict": review_verdict, + "merge_readiness": classify_merge_readiness(review_verdict, blockers), + "findings": semantic_review["findings"], + "infrastructure_blockers": blockers, + "evidence_manifest": { + "input_sha256": content_digest(value), + "semantic_reviewed_head_sha": semantic_review["reviewed_head_sha"], + "merge_evidence_head_sha": merge_evidence["evidence_head_sha"], + "coverage_state": merge_evidence["coverage_state"], + "independent_approval_state": merge_evidence[ + "independent_approval_state" + ], + "branch_protection_state": merge_evidence["branch_protection_state"], + "required_check_count": len(required_checks), + "successful_required_check_count": sum( + item["state"] == "success" for item in required_checks + ), + "advisory_check_count": len(advisory_checks), + "checks": merge_evidence["required_checks"], + }, + } + return { + **report_without_digest, + "decision_sha256": content_digest(report_without_digest), + } + + +def render_markdown(report: Mapping[str, Any]) -> str: + """Render a human-readable decision without turning blockers into source defects.""" + lines = [ + "# OpenCode review decision", + "", + f"Review verdict: **{report['review_verdict']}** ", + f"Merge readiness: **{report['merge_readiness']}** ", + f"Semantic status: **{report['semantic_status']}** ", + f"Exact head: `{report['head_sha']}`", + "", + "## Semantic findings", + "", + ] + if report["findings"]: + for finding in report["findings"]: + lines.extend( + [ + f"- **{finding['severity'].upper()}** `{finding['path']}:{finding['line']}` — {finding['trigger']}", + f" - Impact: {finding['impact']}", + f" - Root cause: {finding['root_cause']}", + f" - Fix direction: {finding['fix_direction']}", + f" - Regression target: {finding['regression_target']}", + ] + ) + else: + lines.append("- None.") + lines.extend(["", "## Infrastructure and policy blockers", ""]) + if report["infrastructure_blockers"]: + for item in report["infrastructure_blockers"]: + suffix = f" / check `{item['check_name']}`" if item["check_name"] else "" + lines.append( + f"- `{item['evidence_name']}` — `{item['state']}` ({item['blocker_code']}){suffix}" + ) + else: + lines.append("- None.") + lines.extend( + [ + "", + "## Evidence receipt", + "", + f"- Input: `{report['evidence_manifest']['input_sha256']}`", + f"- Decision: `{report['decision_sha256']}`", + "", + ] + ) + return "\n".join(lines) + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the decision CLI and return a stable validation status.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--json-output", type=Path, required=True) + parser.add_argument("--markdown-output", type=Path, required=True) + arguments = parser.parse_args(argv) + try: + report = build_decision(load_json(arguments.input)) + except DecisionValidationError as error: + print(f"decision evidence rejected: {error}", file=sys.stderr) + return 2 + write_text( + arguments.json_output, + json.dumps( + report, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + + "\n", + ) + write_text(arguments.markdown_output, render_markdown(report)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/opencode_review_decision_primitives.py b/scripts/ci/opencode_review_decision_primitives.py new file mode 100644 index 000000000..170d82f00 --- /dev/null +++ b/scripts/ci/opencode_review_decision_primitives.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Strict primitives for independent OpenCode review decisions.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from pathlib import Path, PurePosixPath +from typing import Any + +REPOSITORY_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +COMMIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +VALID_SEMANTIC_STATUSES = {"complete", "unavailable", "failed"} +VALID_SEVERITIES = {"critical", "high", "medium", "low"} +VALID_EVIDENCE_STATES = { + "success", + "failure", + "pending", + "queued", + "absent", + "cancelled", + "skipped", + "neutral", +} +HARD_BLOCKING_STATES = {"failure", "cancelled", "skipped", "neutral"} +UNKNOWN_STATES = {"pending", "queued", "absent"} + + +class DecisionValidationError(ValueError): + """Signal malformed or internally inconsistent decision evidence.""" + + +def reject(message: str) -> None: + """Raise one stable decision validation error.""" + raise DecisionValidationError(message) + + +def object_value(value: Any, path: str) -> Mapping[str, Any]: + """Return a JSON object or reject its shape.""" + if not isinstance(value, Mapping): + reject(f"{path} must be an object") + return value + + +def array_value(value: Any, path: str) -> list[Any]: + """Return a JSON array or reject its shape.""" + if not isinstance(value, list): + reject(f"{path} must be an array") + return value + + +def require_exact_fields( + value: Mapping[str, Any], path: str, allowed_fields: set[str] +) -> None: + """Reject unreviewed extension fields at one governed schema layer.""" + unknown = sorted(set(value) - allowed_fields) + if unknown: + reject(f"{path} has unknown fields: {', '.join(unknown)}") + + +def text_value(value: Any, path: str) -> str: + """Return stripped non-empty text or reject it.""" + if not isinstance(value, str) or not value.strip(): + reject(f"{path} must be non-empty text") + return value.strip() + + +def bool_value(value: Any, path: str) -> bool: + """Return an actual Boolean rather than an integer lookalike.""" + if not isinstance(value, bool): + reject(f"{path} must be boolean") + return value + + +def positive_int_value(value: Any, path: str) -> int: + """Return a strictly positive integer without Boolean coercion.""" + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + reject(f"{path} must be a positive integer") + return value + + +def commit_sha_value(value: Any, path: str) -> str: + """Return one full lowercase commit SHA.""" + result = text_value(value, path) + if not COMMIT_SHA_RE.fullmatch(result): + reject(f"{path} must be a 40-character lowercase commit SHA") + return result + + +def optional_commit_sha_value(value: Any, path: str) -> str | None: + """Return ``None`` or one full lowercase commit SHA.""" + return None if value is None else commit_sha_value(value, path) + + +def enum_value(value: Any, path: str, allowed: set[str]) -> str: + """Return a normalized enumerated value or reject it.""" + result = text_value(value, path).casefold() + if result not in allowed: + reject(f"{path} is invalid: {result!r}") + return result + + +def source_path_value(value: Any, path: str) -> str: + """Return a safe repository-relative POSIX source path.""" + result = text_value(value, path) + pure = PurePosixPath(result) + if ( + pure.is_absolute() + or "\\" in result + or any(part in {"", ".", ".."} for part in pure.parts) + ): + reject(f"{path} must be a safe relative source path") + return pure.as_posix() + + +def canonical_json(value: Any) -> str: + """Serialize JSON deterministically for content-addressed receipts.""" + return json.dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + allow_nan=False, + ) + + +def content_digest(value: Any) -> str: + """Return the canonical SHA-256 digest for a JSON-compatible value.""" + encoded = canonical_json(value).encode("utf-8") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def strict_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Build one JSON object while rejecting duplicate member names.""" + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + reject(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def reject_constant(value: str) -> None: + """Reject non-finite constants accepted by Python's permissive JSON parser.""" + reject(f"non-finite JSON number: {value}") + + +def load_json(path: Path) -> Any: + """Load strict UTF-8 JSON with bounded stable validation errors.""" + try: + return json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=strict_pairs, + parse_constant=reject_constant, + ) + except (OSError, json.JSONDecodeError) as error: + reject(f"cannot load decision evidence: {error}") + + +def write_text(path: Path, content: str) -> None: + """Atomically replace one UTF-8 output after creating its parent directory.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text(content, encoding="utf-8") + temporary.replace(path) diff --git a/scripts/ci/opencode_review_decision_validation.py b/scripts/ci/opencode_review_decision_validation.py new file mode 100644 index 000000000..30c8c44ed --- /dev/null +++ b/scripts/ci/opencode_review_decision_validation.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Validate exact-head semantic and merge-policy evidence.""" + +from __future__ import annotations + +from typing import Any + +from opencode_review_decision_primitives import ( + REPOSITORY_RE, + VALID_EVIDENCE_STATES, + VALID_SEMANTIC_STATUSES, + VALID_SEVERITIES, + array_value, + bool_value, + commit_sha_value, + enum_value, + object_value, + optional_commit_sha_value, + positive_int_value, + reject, + require_exact_fields, + source_path_value, + text_value, +) + + +def validate_finding(raw_value: Any, path: str) -> dict[str, Any]: + """Validate one complete semantic source finding.""" + value = object_value(raw_value, path) + require_exact_fields( + value, + path, + { + "finding_id", + "defect_class", + "severity", + "blocking", + "path", + "line", + "trigger", + "impact", + "root_cause", + "fix_direction", + "regression_target", + }, + ) + return { + "finding_id": text_value(value.get("finding_id"), f"{path}.finding_id"), + "defect_class": text_value( + value.get("defect_class"), f"{path}.defect_class" + ).casefold(), + "severity": enum_value( + value.get("severity"), f"{path}.severity", VALID_SEVERITIES + ), + "blocking": bool_value(value.get("blocking"), f"{path}.blocking"), + "path": source_path_value(value.get("path"), f"{path}.path"), + "line": positive_int_value(value.get("line"), f"{path}.line"), + "trigger": text_value(value.get("trigger"), f"{path}.trigger"), + "impact": text_value(value.get("impact"), f"{path}.impact"), + "root_cause": text_value(value.get("root_cause"), f"{path}.root_cause"), + "fix_direction": text_value( + value.get("fix_direction"), f"{path}.fix_direction" + ), + "regression_target": text_value( + value.get("regression_target"), f"{path}.regression_target" + ), + } + + +def validate_semantic_review( + raw_value: Any, expected_head_sha: str +) -> dict[str, Any]: + """Validate semantic review evidence independently from merge policy evidence.""" + value = object_value(raw_value, "semantic_review") + require_exact_fields( + value, + "semantic_review", + {"status", "reviewed_head_sha", "findings"}, + ) + status = enum_value( + value.get("status"), "semantic_review.status", VALID_SEMANTIC_STATUSES + ) + reviewed_head_sha = optional_commit_sha_value( + value.get("reviewed_head_sha"), "semantic_review.reviewed_head_sha" + ) + raw_findings = array_value(value.get("findings"), "semantic_review.findings") + if status != "complete": + if reviewed_head_sha is not None: + reject( + "semantic_review.reviewed_head_sha must be null when semantic review is incomplete" + ) + if raw_findings: + reject("incomplete semantic review must not contain findings") + return { + "status": status, + "reviewed_head_sha": None, + "findings": [], + } + if reviewed_head_sha != expected_head_sha: + reject( + "semantic_review.reviewed_head_sha must equal the exact decision head_sha" + ) + findings: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + for index, raw_finding in enumerate(raw_findings): + finding = validate_finding( + raw_finding, f"semantic_review.findings[{index}]" + ) + identity = finding["finding_id"].casefold() + if identity in seen_ids: + reject( + f"semantic_review.findings[{index}].finding_id duplicates {finding['finding_id']!r}" + ) + seen_ids.add(identity) + findings.append(finding) + findings.sort(key=lambda item: item["finding_id"].casefold()) + return { + "status": status, + "reviewed_head_sha": reviewed_head_sha, + "findings": findings, + } + + +def validate_check(raw_value: Any, path: str, expected_head_sha: str) -> dict[str, Any]: + """Validate one exact-head required or advisory check record.""" + value = object_value(raw_value, path) + require_exact_fields(value, path, {"name", "state", "required", "head_sha"}) + head_sha = commit_sha_value(value.get("head_sha"), f"{path}.head_sha") + if head_sha != expected_head_sha: + reject(f"{path}.head_sha must equal the exact decision head_sha") + return { + "name": text_value(value.get("name"), f"{path}.name"), + "state": enum_value( + value.get("state"), f"{path}.state", VALID_EVIDENCE_STATES + ), + "required": bool_value(value.get("required"), f"{path}.required"), + "head_sha": head_sha, + } + + +def validate_merge_evidence( + raw_value: Any, expected_head_sha: str +) -> dict[str, Any]: + """Validate exact-head coverage, approval, protection, and check evidence.""" + value = object_value(raw_value, "merge_evidence") + require_exact_fields( + value, + "merge_evidence", + { + "evidence_head_sha", + "coverage_state", + "independent_approval_state", + "branch_protection_state", + "required_checks", + }, + ) + evidence_head_sha = commit_sha_value( + value.get("evidence_head_sha"), "merge_evidence.evidence_head_sha" + ) + if evidence_head_sha != expected_head_sha: + reject("merge_evidence.evidence_head_sha must equal the exact decision head_sha") + checks: list[dict[str, Any]] = [] + seen_names: set[str] = set() + for index, raw_check in enumerate( + array_value(value.get("required_checks"), "merge_evidence.required_checks") + ): + path = f"merge_evidence.required_checks[{index}]" + check = validate_check(raw_check, path, expected_head_sha) + normalized_name = check["name"].casefold() + if normalized_name in seen_names: + reject(f"{path}.name duplicates check name {check['name']!r}") + seen_names.add(normalized_name) + checks.append(check) + checks.sort(key=lambda item: item["name"].casefold()) + return { + "evidence_head_sha": evidence_head_sha, + "coverage_state": enum_value( + value.get("coverage_state"), + "merge_evidence.coverage_state", + VALID_EVIDENCE_STATES, + ), + "independent_approval_state": enum_value( + value.get("independent_approval_state"), + "merge_evidence.independent_approval_state", + VALID_EVIDENCE_STATES, + ), + "branch_protection_state": enum_value( + value.get("branch_protection_state"), + "merge_evidence.branch_protection_state", + VALID_EVIDENCE_STATES, + ), + "required_checks": checks, + } + + +def validate_decision_input(raw_value: Any) -> dict[str, Any]: + """Validate and normalize one complete exact-head decision input.""" + value = object_value(raw_value, "decision") + require_exact_fields( + value, + "decision", + { + "schema_version", + "decision_id", + "quality_policy_version", + "repository", + "pull_request_number", + "base_sha", + "head_sha", + "semantic_review", + "merge_evidence", + }, + ) + if value.get("schema_version") != "1.0": + reject("decision.schema_version must equal '1.0'") + repository = text_value(value.get("repository"), "decision.repository") + if not REPOSITORY_RE.fullmatch(repository): + reject("decision.repository must use owner/name") + head_sha = commit_sha_value(value.get("head_sha"), "decision.head_sha") + normalized = { + "schema_version": "1.0", + "decision_id": text_value(value.get("decision_id"), "decision.decision_id"), + "quality_policy_version": text_value( + value.get("quality_policy_version"), "decision.quality_policy_version" + ), + "repository": repository, + "pull_request_number": positive_int_value( + value.get("pull_request_number"), "decision.pull_request_number" + ), + "base_sha": commit_sha_value(value.get("base_sha"), "decision.base_sha"), + "head_sha": head_sha, + "semantic_review": validate_semantic_review( + value.get("semantic_review"), head_sha + ), + "merge_evidence": validate_merge_evidence( + value.get("merge_evidence"), head_sha + ), + } + return normalized diff --git a/tests/test_opencode_review_decision_cli.py b/tests/test_opencode_review_decision_cli.py index 56592d137..352870c3f 100644 --- a/tests/test_opencode_review_decision_cli.py +++ b/tests/test_opencode_review_decision_cli.py @@ -122,3 +122,13 @@ def test_public_production_callables_have_docstrings() -> None: and not getattr(value, "__doc__", None) ] assert missing == [] + + +def test_load_json_wraps_syntax_and_filesystem_errors(tmp_path: Path) -> None: + """Malformed or unavailable evidence files must produce bounded stable errors.""" + malformed = tmp_path / "malformed.json" + malformed.write_text("{", encoding="utf-8") + with pytest.raises(decision.DecisionValidationError, match="cannot load"): + decision.load_json(malformed) + with pytest.raises(decision.DecisionValidationError, match="cannot load"): + decision.load_json(tmp_path / "absent.json") diff --git a/tests/test_opencode_review_decision_validation.py b/tests/test_opencode_review_decision_validation.py index d86421cfc..7efae66f2 100644 --- a/tests/test_opencode_review_decision_validation.py +++ b/tests/test_opencode_review_decision_validation.py @@ -122,3 +122,28 @@ def test_duplicate_finding_and_check_names_are_rejected() -> None: with pytest.raises(decision.DecisionValidationError, match="check name"): decision.build_decision(envelope(checks=[check("CI"), check("ci")])) + + +def test_validation_helpers_reject_remaining_invalid_shapes_and_scalars() -> None: + """Primitive schema helpers must reject unsupported JSON shapes and scalar values.""" + with pytest.raises(decision.DecisionValidationError, match="must be an array"): + decision.array_value({}, "array") + with pytest.raises(decision.DecisionValidationError, match="non-empty text"): + decision.text_value(" ", "text") + with pytest.raises(decision.DecisionValidationError, match="must be boolean"): + decision.bool_value(1, "flag") + with pytest.raises(decision.DecisionValidationError, match="commit SHA"): + decision.commit_sha_value("main", "head") + + +def test_top_level_schema_and_repository_coordinates_are_strict() -> None: + """Decision identity must use the exact schema version and owner/name repository form.""" + wrong_version = envelope() + wrong_version["schema_version"] = "2.0" + with pytest.raises(decision.DecisionValidationError, match="schema_version"): + decision.build_decision(wrong_version) + + invalid_repository = envelope() + invalid_repository["repository"] = "missing-slash" + with pytest.raises(decision.DecisionValidationError, match="owner/name"): + decision.build_decision(invalid_repository) From c3133887df775605bfca2aaaa8b94094aec609fc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 22:56:36 +0900 Subject: [PATCH 4/4] docs(opencode): record independent decision boundary --- CHANGELOG.md | 1 + .../opencode-review-decision-envelope.md | 291 ++++++++++++++++++ 2 files changed, 292 insertions(+) create mode 100644 docs/doctoring/opencode-review-decision-envelope.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 675ecb8d7..3ab3862b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Semantic Versioning where the repository publishes a release. ### Added +- Added an exact-head OpenCode decision envelope that keeps semantic source verdicts independent from coverage, checks, approval, and branch-protection merge readiness; emits path-free infrastructure blockers; fails closed on stale or malformed evidence; and preserves 100% production statement, branch, and public-docstring evidence. - Added deterministic exact-head corpus sampling and blinded two-expert-plus-adjudicator gold-freeze tooling, with strict JSON, immutable evidence receipts, hard language/size/risk/defect coverage, atomic outputs, stable failure classes, and permanent 100% production statement/branch/docstring evidence. - Added an empirical OpenCode review-quality benchmark, fail-closed scorer, exact-head quality workflow, and APA 7th doctoring that keep lifecycle-yield evidence separate from head-matched expert-gold precision and recall, require Wilson-bound non-inferiority before any CodeRabbit-parity claim, and preserve 100% production statement/branch/docstring evidence. - Added a trusted pull-request comment router for `@cwl-noema-review` and review-only `@opencode-agent` dispatches, with an organization sweep, exact-head receipts, repository allowlisting, fixed runners, immutable checkout pins, and a permanent 100% statement/branch/docstring quality gate. diff --git a/docs/doctoring/opencode-review-decision-envelope.md b/docs/doctoring/opencode-review-decision-envelope.md new file mode 100644 index 000000000..9aae39b60 --- /dev/null +++ b/docs/doctoring/opencode-review-decision-envelope.md @@ -0,0 +1,291 @@ +# OpenCode review semantic and merge-readiness decision envelope + +Status: Proposed operational architecture +Date: 2026-08-08 +Owner: ContextualWisdomLab central review infrastructure + +## Decision summary + +OpenCode Review must represent two independent decisions: + +1. **Semantic review verdict** — whether exact-head source and connected context contain a substantiated defect. +2. **Merge readiness** — whether exact-head checks, coverage, independent approval, branch protection, and repository policy permit integration. + +Infrastructure or policy failure may block integration. It must never be converted into a source finding, severity, path, or line number. A failed coverage collector is evidence that coverage has not been proven, not evidence that the pull-request implementation contains a high-severity defect. + +This change introduces an offline, deterministic decision-composition module. It does not yet modify the production OpenCode dispatch because other active branches own that large workflow. Production integration requires a later test-first slice after the writer lease clears. + +## Problem + +The current central review workflow can construct a synthetic `REQUEST_CHANGES` finding when coverage evidence acquisition fails. That behavior combines two different questions: + +```text +Does the changed source contain a defect? + ≠ +May this exact head merge under repository policy? +``` + +The conflation creates several operational failure modes: + +- repeated infrastructure-only reviews that provide no semantic source value; +- fabricated source anchors such as a workflow line that did not cause the product defect; +- inability to distinguish reviewer abstention from a negative code judgment; +- false defect counts in quality evaluation; +- developer confusion about whether to repair code or review infrastructure; and +- stale-head evidence accidentally appearing authoritative after a new commit. + +The empirical lifecycle pilot in `benchmarks/opencode_review/pilot_baseline_v1.json` directly observed eight completed OpenCode attempts that produced only infrastructure review output and no source findings. That pilot is not a head-matched precision or recall study, but it establishes that the decision-channel conflation is operationally material. + +## Versioned input contract + +The decision module accepts one strict JSON object: + +```json +{ + "schema_version": "1.0", + "decision_id": "decision_001", + "quality_policy_version": "opencode-review-quality-v1", + "repository": "ContextualWisdomLab/example", + "pull_request_number": 42, + "base_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "head_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "semantic_review": { + "status": "complete", + "reviewed_head_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "findings": [] + }, + "merge_evidence": { + "evidence_head_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "coverage_state": "success", + "independent_approval_state": "success", + "branch_protection_state": "success", + "required_checks": [] + } +} +``` + +### Semantic review input + +`semantic_review.status` is one of: + +- `complete` — semantic review finished on the exact current head; +- `unavailable` — no trusted semantic review result was available; or +- `failed` — the semantic review process failed before producing a valid result. + +A complete review must bind `reviewed_head_sha` to the decision's exact head. An unavailable or failed review must use a null reviewed head and must contain no findings. This prevents a partial or failed process from publishing synthetic source claims. + +Each semantic finding must contain: + +- stable finding identifier; +- defect class and calibrated severity; +- explicit blocking classification; +- repository-relative changed or connected source path; +- positive source line; +- trigger condition; +- observable impact; +- source-backed root cause; +- minimal fix direction; and +- exact regression target. + +### Merge-evidence input + +The merge channel carries only policy evidence: + +- coverage state; +- independent-approval state; +- branch-protection state; and +- named required or advisory checks. + +Every record must bind to the same exact head. A trusted composition root determines which checks are required according to repository policy; the pure decision module does not infer branch protection or fetch GitHub state itself. + +The accepted evidence states are: + +```text +success +failure +pending +queued +absent +cancelled +skipped +neutral +``` + +This vocabulary deliberately distinguishes hard negative evidence, latency, and absence. It does not reinterpret GitHub conclusions as product defects. + +## Versioned output contract + +The output preserves both decisions and both evidence classes: + +```json +{ + "schema_version": "1.0", + "review_verdict": "APPROVE", + "merge_readiness": "BLOCKED", + "semantic_status": "complete", + "findings": [], + "infrastructure_blockers": [ + { + "blocker_code": "coverage_not_successful", + "evidence_name": "coverage", + "state": "failure", + "check_name": null + } + ], + "evidence_manifest": {}, + "decision_sha256": "sha256:..." +} +``` + +Infrastructure blockers have no `path`, `line`, `severity`, `trigger`, `root_cause`, or fix authority. They identify the evidence surface and state only. + +## Semantic verdict rules + +| Semantic status and findings | `review_verdict` | +|---|---| +| Complete, no finding | `APPROVE` | +| Complete, only non-blocking findings | `COMMENT` | +| Complete, one or more blocking findings | `REQUEST_CHANGES` | +| Unavailable or failed | `ABSTAIN` | + +The semantic verdict never reads coverage, checks, approval, or branch-protection state. + +`APPROVE` in this envelope means that the semantic channel found no blocking source defect. It is not a formal GitHub approval and must not be submitted as a review by the change author or any non-independent identity. + +## Merge-readiness rules + +| Evidence | `merge_readiness` | +|---|---| +| Semantic review complete without blocking findings; all required policy evidence successful | `READY` | +| Blocking semantic finding | `BLOCKED` | +| Required evidence `failure`, `cancelled`, `skipped`, or `neutral` | `BLOCKED` | +| Required evidence `pending`, `queued`, or `absent` | `UNKNOWN` | +| Semantic review unavailable or failed, with no hard policy failure | `UNKNOWN` | + +Advisory-check failure is recorded in the evidence manifest but does not block unless repository policy marks that check required. + +A hard policy failure takes precedence over latency. For example, one failed required check and one pending check produce `BLOCKED`, not `UNKNOWN`. + +## Exact-head and evidence-integrity rules + +The module rejects: + +- a semantic review for another head; +- coverage, approval, protection, or check evidence for another head; +- duplicate case-insensitive finding identifiers; +- duplicate case-insensitive check names; +- unsafe or parent-traversing source paths; +- zero or negative line numbers; +- Boolean values passed as integers; +- unknown fields at every governed schema layer; +- duplicate JSON member names; +- Python's non-standard `NaN`, `Infinity`, and `-Infinity` JSON extensions; and +- incomplete semantic reviews that claim findings or an exact reviewed head. + +Canonical strict JSON produces an input SHA-256 receipt. The normalized decision, excluding its own digest, produces a separate decision SHA-256 receipt. JSON and Markdown outputs use temporary sibling files followed by atomic replacement. + +## Markdown presentation + +The human-readable output has physically separate sections: + +```text +Semantic findings + +Infrastructure and policy blockers +``` + +Only semantic findings may render `path:line`. Infrastructure blockers display the evidence name, state, stable blocker code, and optional check name. This presentation rule prevents a later renderer from reintroducing a synthetic source defect even when the underlying JSON remains separated. + +## Security and privacy boundary + +The decision module: + +- runs offline; +- does not execute repository code, model output, commands, or patches; +- does not call GitHub or another network service; +- does not read a secret, cookie, token, environment credential, or model credential; +- accepts only policy-normalized evidence from a trusted caller; +- records no source body or personal data beyond bounded finding text and repository identity; and +- introduces no `COPILOT_GITHUB_TOKEN` use. + +Scheduled OpenCode model execution, when used elsewhere, continues to use the existing `NVIDIA_NIM_API_KEY` credential boundary. This pure module neither selects nor invokes a model. + +## Production integration boundary + +This pull request does not edit `.github/workflows/opencode-review-dispatch.yml`. At the time of implementation, active central branches `#789`, `#812`, `#816`, and `#827` modify that workflow. A competing edit would violate the one-writer lease and could discard exact-head coverage and toolchain repairs. + +After those branches integrate or relinquish the file, a separate implementation must begin with a failing production contract that requires the dispatch to: + +1. continue bounded semantic review whenever safe exact-head source evidence exists; +2. emit `ABSTAIN` rather than a source finding when semantic review cannot complete; +3. place coverage and check failures only in merge-readiness evidence; +4. pass the versioned decision envelope to publication; +5. publish source comments only from validated semantic findings; +6. expose infrastructure blockers through check summary or a non-source status surface; +7. preserve reviewer identity and credential chains; and +8. keep merge, auto-merge, branch update, and release authority separately controlled. + +No predecessor-head result from this pure module authorizes that later production integration. + +## Verification + +The exact-head quality workflow runs on Python 3.14 with immutable action pins, read-only repository permission, persisted checkout credentials disabled, and hash-verified test dependencies. It requires: + +- behavior and adversarial schema tests; +- exact-head checkout verification; +- production statement coverage 100%; +- production branch coverage 100%; +- public production callable docstrings 100%; +- `compileall`; and +- a clean Git worktree. + +Regression cases cover the exact operational failure: coverage failure with a complete defect-free semantic review yields `APPROVE` plus `BLOCKED`, with no source finding. + +## Monitoring after integration + +When the envelope reaches the production dispatch, monitor at least: + +- semantic completion, failure, and abstention rates; +- infrastructure-only blocker rate; +- source findings per completed semantic review; +- current-head duplicate publication rate; +- stale-head evidence rejection count; +- blocked versus unknown readiness rates; +- time to first useful semantic comment; +- developer dismissal and resolution rates; and +- any occurrence of a path or line in an infrastructure blocker. + +The last metric must remain zero. + +## Rollback + +The pure decision module can be removed from a caller without changing reviewer credentials, model selection, or GitHub branch protection. Rollback must not restore the old synthetic source-finding path. If the envelope cannot be consumed safely, the caller should fail closed with: + +- semantic verdict `ABSTAIN`; and +- merge readiness `UNKNOWN` or `BLOCKED` according to independently available policy evidence. + +## Limitations + +- The module does not discover which GitHub checks are required; a trusted policy collector must supply that classification. +- It does not prove that a formal independent approval is valid; it validates only normalized approval state and exact-head identity supplied by a trusted caller. +- It does not itself collect coverage or branch-protection evidence. +- It does not calibrate semantic severity or verify model findings; detector-verifier orchestration and expert-gold evaluation remain separate work. +- A `READY` output is a deterministic policy composition result, not authority to bypass GitHub rulesets or merge administratively. + +## References + +Booth, H., Souppaya, M., Vassilev, A., Ogata, M., Stanley, M., Scarfone, K., & Dodson, D. (2024). *Secure software development practices for generative AI and dual-use foundation models: An SSDF community profile* (NIST Special Publication 800-218A). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-218A + +GitHub. (n.d.). *About protected branches*. GitHub Docs. Retrieved August 8, 2026, from https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/about-protected-branches + +GitHub. (n.d.). *Approving a pull request with required reviews*. GitHub Docs. Retrieved August 8, 2026, from https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/approving-a-pull-request-with-required-reviews + +GitHub. (n.d.). *Status checks*. GitHub Docs. Retrieved August 8, 2026, from https://docs.github.com/en/pull-requests/committing-changes-to-your-project/troubleshooting-commits/status-checks + +GitHub. (n.d.). *Troubleshooting required status checks*. GitHub Docs. Retrieved August 8, 2026, from https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/troubleshooting-required-status-checks + +SLSA Community. (2025). *SLSA specification, version 1.2*. https://slsa.dev/spec/v1.2/ + +Souppaya, M., Scarfone, K., & Dodson, D. (2022). *Secure Software Development Framework (SSDF) version 1.1: Recommendations for mitigating the risk of software vulnerabilities* (NIST Special Publication 800-218). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-218 + +Sun, T., Xu, J., Li, Y., Yan, Z., Zhang, G., Xie, L., Geng, L., Wang, Z., Chen, Y., Lin, Q., Duan, W., & Sui, K. (2025). BitsAI-CR: Automated code review via LLM in practice. In *Proceedings of the 33rd ACM International Conference on the Foundations of Software Engineering*. https://doi.org/10.1145/3696630.3728552