diff --git a/.github/workflows/opencode-review-corpus-quality-ci.yml b/.github/workflows/opencode-review-corpus-quality-ci.yml new file mode 100644 index 000000000..45ba8eb09 --- /dev/null +++ b/.github/workflows/opencode-review-corpus-quality-ci.yml @@ -0,0 +1,94 @@ +name: OpenCode Review Corpus Quality CI + +on: + pull_request: + branches: + - main + - feat/opencode-review-quality-gate + paths: + - ".github/workflows/opencode-review-corpus-quality-ci.yml" + - "scripts/ci/opencode_review_sample.py" + - "scripts/ci/opencode_review_adjudicate.py" + - "scripts/ci/opencode_review_adjudication_primitives.py" + - "scripts/ci/opencode_review_adjudication_annotations.py" + - "scripts/ci/opencode_review_adjudication_decisions.py" + - "tests/test_opencode_review_sample.py" + - "tests/test_opencode_review_sample_coverage.py" + - "tests/test_opencode_review_adjudicate.py" + - "tests/test_opencode_review_adjudicate_coverage.py" + - "docs/doctoring/opencode-review-annotation-guide.md" + - "benchmarks/opencode_review/head_matched_v1/**" + - "CHANGELOG.md" + +permissions: + contents: read + +concurrency: + group: opencode-review-corpus-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + gold-corpus-quality: + name: gold-corpus-quality + if: github.event_name != 'pull_request' || github.event.action != 'closed' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout exact source revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install exact hash-verified test runner dependencies + env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PIP_NO_INPUT: "1" + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + cat >"${RUNNER_TEMP}/opencode-review-corpus-requirements.txt" <<'REQEOF' + coverage==7.15.2 --hash=sha256:b9a6367e4aff723e8ee8190836836124284e8fcd4265e307c844010cfa074f3f + iniconfig==2.1.0 --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 + packaging==26.2 --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e + pluggy==1.6.0 --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + pygments==2.20.0 --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + pytest==9.1.1 --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c + REQEOF + python -m pip install \ + --only-binary=:all: \ + --require-hashes \ + -r "${RUNNER_TEMP}/opencode-review-corpus-requirements.txt" + + - name: Verify head-matched corpus contracts + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + test "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha || github.sha }}" + python -m coverage run \ + --branch \ + --source=scripts/ci \ + -m pytest \ + tests/test_opencode_review_sample.py \ + tests/test_opencode_review_sample_coverage.py \ + tests/test_opencode_review_adjudicate.py \ + tests/test_opencode_review_adjudicate_coverage.py \ + -q + python -m coverage report \ + --include='scripts/ci/opencode_review_sample.py,scripts/ci/opencode_review_adjudicate.py,scripts/ci/opencode_review_adjudication_primitives.py,scripts/ci/opencode_review_adjudication_annotations.py,scripts/ci/opencode_review_adjudication_decisions.py' \ + --fail-under=100 \ + --show-missing + python -m compileall -q \ + scripts/ci/opencode_review_sample.py \ + scripts/ci/opencode_review_adjudicate.py \ + scripts/ci/opencode_review_adjudication_primitives.py \ + scripts/ci/opencode_review_adjudication_annotations.py \ + scripts/ci/opencode_review_adjudication_decisions.py \ + tests/test_opencode_review_sample.py \ + tests/test_opencode_review_sample_coverage.py \ + tests/test_opencode_review_adjudicate.py \ + tests/test_opencode_review_adjudicate_coverage.py + git diff --exit-code diff --git a/CHANGELOG.md b/CHANGELOG.md index fbfdc13c6..675ecb8d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Semantic Versioning where the repository publishes a release. ### Added +- Added deterministic exact-head corpus sampling and blinded two-expert-plus-adjudicator gold-freeze tooling, with strict JSON, immutable evidence receipts, hard language/size/risk/defect coverage, atomic outputs, stable failure classes, and permanent 100% production statement/branch/docstring evidence. - Added an empirical OpenCode review-quality benchmark, fail-closed scorer, exact-head quality workflow, and APA 7th doctoring that keep lifecycle-yield evidence separate from head-matched expert-gold precision and recall, require Wilson-bound non-inferiority before any CodeRabbit-parity claim, and preserve 100% production statement/branch/docstring evidence. - Added a trusted pull-request comment router for `@cwl-noema-review` and review-only `@opencode-agent` dispatches, with an organization sweep, exact-head receipts, repository allowlisting, fixed runners, immutable checkout pins, and a permanent 100% statement/branch/docstring quality gate. - Added exact-base `uv.lock` materialization that reconstructs standalone nested projects with a checksum-pinned official `uv` exporter, isolated frozen/offline execution, strict exact-pin and SHA-256 output validation, and complete Python 3.10/3.14 quality evidence. diff --git a/benchmarks/opencode_review/head_matched_v1/README.md b/benchmarks/opencode_review/head_matched_v1/README.md new file mode 100644 index 000000000..c27a5480e --- /dev/null +++ b/benchmarks/opencode_review/head_matched_v1/README.md @@ -0,0 +1,95 @@ +# Head-matched OpenCode review corpus v1 + +This directory is the versioned landing area for real, exact-head code-review +quality evidence. It intentionally contains no fabricated expert annotations or +synthetic parity result. + +## Required lifecycle + +```text +trusted exact-head inventory +→ deterministic policy-complete sample +→ two independent blinded expert annotations +→ third-party blinded adjudication +→ immutable gold freeze +→ same-head OpenCode and CodeRabbit capture +→ scorer input assembly +→ frozen benchmark report +``` + +Gold must be frozen before either automated reviewer output is disclosed to the +experts or adjudicator. Lifecycle pilot data under `../pilot_baseline_v1.json` +remains operational telemetry and must not be copied into this expert-gold +answer key. + +## Expected layout + +```text +head_matched_v1/ +├── README.md +├── inventory.json +├── sample.json +├── partition_manifest.json +└── cases/ + └── case_001/ + ├── context_manifest.json + ├── expert_a.json + ├── expert_b.json + ├── adjudication.json + ├── gold.json + ├── opencode_review.json + └── coderabbit_review.json +``` + +The repository may store large or access-restricted context bundles outside Git +when required. `context_manifest.json` must then retain immutable object +identities, content digests, access classification, and reproduction +instructions without embedding credentials. + +## Tooling + +Select an eligible sample: + +```bash +python scripts/ci/opencode_review_sample.py \ + --input benchmarks/opencode_review/head_matched_v1/inventory.json \ + --output benchmarks/opencode_review/head_matched_v1/sample.json \ + --seed opencode-review-head-matched-v1 +``` + +Freeze one adjudicated case: + +```bash +python scripts/ci/opencode_review_adjudicate.py \ + --expert-a benchmarks/opencode_review/head_matched_v1/cases/case_001/expert_a.json \ + --expert-b benchmarks/opencode_review/head_matched_v1/cases/case_001/expert_b.json \ + --adjudication benchmarks/opencode_review/head_matched_v1/cases/case_001/adjudication.json \ + --output benchmarks/opencode_review/head_matched_v1/cases/case_001/gold.json +``` + +Both tools are offline, strict-JSON, deterministic utilities. They do not call +GitHub, execute repository code, inspect model credentials, or create human +judgments. + +## Admission rules + +- Same repository, pull request, base SHA, head SHA, diff digest, and context + digest across every record. +- Complete repository context available to both experts. +- Same-head OpenCode and CodeRabbit review possible. +- Two distinct independent expert annotators and a distinct adjudicator. +- Required language, diff-size, risk, and defect-class strata represented. +- Every expert finding adjudicated exactly once. +- Every accepted gold finding carries path, positive line, class, severity, + trigger, impact, root cause, fix direction, and regression target. +- No secrets, cookies, tokens, or unrelated personal data. +- No related-change family leakage across development, calibration, and held-out + partitions. + +## Current status + +`EMPTY_PENDING_REAL_COLLECTION` + +The absence of cases is not a passing parity result. OpenCode Review remains +`INSUFFICIENT_EVIDENCE` until a frozen corpus meets the configured minimum case +and gold-finding floors and the separately versioned statistical gate passes. diff --git a/docs/doctoring/opencode-review-annotation-guide.md b/docs/doctoring/opencode-review-annotation-guide.md new file mode 100644 index 000000000..849435d18 --- /dev/null +++ b/docs/doctoring/opencode-review-annotation-guide.md @@ -0,0 +1,238 @@ +# OpenCode review head-matched corpus annotation guide + +Status: Proposed operational protocol +Date: 2026-08-08 +Owner: ContextualWisdomLab central review infrastructure + +## Purpose + +This protocol defines how ContextualWisdomLab may create an expert-gold corpus +for comparing OpenCode Review with a commercial reference reviewer on the same +immutable pull-request head. It implements the collection and expert-gold stages +of the commercial parity program in +[`opencode-review-quality-evaluation.md`](opencode-review-quality-evaluation.md). + +The protocol does **not** declare parity, generate synthetic human judgments, or +allow lifecycle telemetry to masquerade as precision and recall. A frozen case +is admissible only after two independent experts have reviewed complete +repository context without access to either automated reviewer output and a +third independent adjudicator has resolved every reported candidate finding. + +## Authority separation + +| Role | Authority | Prohibited authority | +|---|---|---| +| Trusted collector | Capture immutable repository, pull request, base, head, diff, and context receipts | Select findings, adjudicate defects, expose secrets to annotators | +| Offline sampler | Deterministically choose policy-complete eligible cases | Fetch GitHub, inspect credentials, alter source evidence | +| Expert A and Expert B | Independently discover defects from the complete exact-head context | View OpenCode or CodeRabbit output, coordinate findings, change the branch | +| Blinded adjudicator | Accept or reject every expert discovery and define canonical gold evidence | View automated-review identity or output, create unreported defects without a new expert round | +| Benchmark operator | Add frozen reviewer outputs and run the scorer | Change frozen gold after observing test-set results | + +The collector, experts, adjudicator, and benchmark operator may be the same +organization, but the two experts and adjudicator must be three distinct +pseudonymous actors for each case. Repository authors, last pushers, automated +review bots, and model-generated annotations do not satisfy independent expert +roles. + +## Trusted collection boundary + +Live GitHub acquisition is outside the sampler and adjudicator processes. The +trusted collector must create a strict UTF-8 JSON inventory whose candidate +records include: + +- repository in `owner/name` form; +- pull request number; +- exact base and contributor-head commit SHAs; +- canonical SHA-256 digest of the reviewed diff artifact; +- canonical SHA-256 digest of the complete repository-context bundle; +- primary implementation language; +- small, medium, or large diff-size bucket; +- primary risk class and target defect classes; +- changed-file, addition, and deletion counts; +- explicit evidence that complete context is available; +- explicit evidence that both reviewers can inspect the same head; and +- explicit confirmation that two independent experts and an adjudicator are + available. + +The context bundle should contain the changed source, connected callers and +callees, tests, schemas, migrations, relevant issue or requirement text, +repository guidance, and architecture or security documentation needed to judge +the change. The bundle must not contain tokens, cookies, passwords, raw secrets, +or unrelated personal data. + +An inventory is ineligible rather than partially trusted when any of the three +capacity flags is false. The offline sampler never upgrades an ineligible case. + +## Deterministic sampling + +Run the sampler with a frozen, versioned seed: + +```bash +python scripts/ci/opencode_review_sample.py \ + --input benchmarks/opencode_review/head_matched_v1/inventory.json \ + --output benchmarks/opencode_review/head_matched_v1/sample.json \ + --seed opencode-review-head-matched-v1 +``` + +The sampler: + +1. rejects duplicate JSON members, non-finite JSON numbers, unknown fields, + malformed commit or digest identities, and duplicate exact-head cases; +2. excludes cases without full context, same-head comparability, or confirmed + independent expert capacity; +3. requires the configured minimum number of primary languages; +4. requires every configured diff-size, risk, and defect-class stratum; +5. uses a seed-bound deterministic priority for ties; +6. rotates across language, size, and risk strata while filling remaining + capacity; and +7. emits source-inventory and selected-sample SHA-256 receipts. + +A valid but underpowered inventory returns an insufficient-corpus status rather +than silently relaxing quotas. The initial commercial parity floor remains at +least 50 head-matched pull requests and 50 accepted expert-gold findings; the +floor is not a substitute for later power and sensitivity analysis. + +## Independent annotation procedure + +Each expert receives the same exact-head context bundle under pseudonymous case +and expert identifiers. Automated reviewer output, reviewer identity, benchmark +score, prior expert annotations, and adjudication decisions remain hidden. + +Each expert must: + +1. inspect the complete context bundle rather than only the changed lines; +2. inspect executable and non-executable claims, including tests, workflows, + documentation, migrations, and data contracts; +3. record every defect they can substantiate; +4. record a positive source line and repository-relative path; +5. state a concrete trigger, observable impact, source-backed root cause, + minimal fix direction, and exact regression target; and +6. affirm `no_additional_findings: true` only after completing the review. + +A finding is not accepted merely because it is plausible or because a tool made +the same claim. Unsupported style preferences, desired enhancements, missing CI +results, rate limits, and infrastructure availability are not source defects. + +## Blinded adjudication procedure + +The adjudicator receives the exact case evidence and both pseudonymous expert +annotations, but not OpenCode or CodeRabbit identity or output. Every expert +finding must appear in exactly one adjudication decision. + +For each decision, the adjudicator must: + +- map zero or more Expert A findings and zero or more Expert B findings into one + reviewed candidate, with at least one source finding present; +- accept only when the defect is supported by the exact source and context; +- assign one unique canonical gold-finding identifier to an accepted defect; +- provide the canonical path, positive line, defect class, severity, trigger, + impact, root cause, fix direction, and regression target; +- reject non-defects with an explicit reason and without creating a gold ID; and +- merge duplicate expert reports into one accepted gold finding rather than + counting agreement as two defects. + +An unresolved disagreement is not guessed into the gold set. The case remains +unfrozen until a new evidence or expert round resolves it. The adjudicator may +not invent an unseen defect; it must return the case to independent annotation +when a new candidate is discovered during adjudication. + +Freeze the case with: + +```bash +python scripts/ci/opencode_review_adjudicate.py \ + --expert-a cases/case_001/expert_a.json \ + --expert-b cases/case_001/expert_b.json \ + --adjudication cases/case_001/adjudication.json \ + --output cases/case_001/gold.json +``` + +The output contains deterministic annotation receipts, an adjudication receipt, +agreement counts, canonical accepted gold findings, and a freeze SHA-256. The +source annotation and adjudication records remain immutable evidence; corrected +records create a new corpus version and freeze rather than overwriting published +test evidence. + +## Reviewer capture after gold freeze + +Only after the case freeze is immutable may the benchmark operator capture +OpenCode and CodeRabbit outputs. Both reviewers must receive the same: + +- repository and pull request; +- exact base and head SHAs; +- reviewed diff and context bundle; +- allowed tools and repository instructions; and +- time-bounded opportunity to complete. + +Record each reviewer's configuration, profile, model or provider identifier when +observable, timestamp, exact reviewed head, status, source comments, and rate +limit or infrastructure evidence. A status-only signal, skipped draft review, +predecessor-head review, synthetic merge result, or incomplete attempt is not a +completed same-head semantic review. + +Reviewer findings are mapped to the already frozen gold set without modifying +the gold. Unmatched actionable findings are false-positive candidates and must +be independently assessed in a later corpus version rather than retroactively +added to the held-out answer key. + +## Dataset partition and calibration discipline + +Before tuning prompts, routing, thresholds, deduplication, or severity mapping, +freeze case assignments into development, calibration, and held-out test sets at +the repository or related-change-family level. Revisions, stacked branches, +backports, copied fixes, and materially equivalent incidents must not cross +partitions. + +The held-out test set is evaluated only after the review system and parity policy +version are frozen. A failed held-out evaluation produces a new development +cycle and a new future held-out version; it does not permit tuning against the +same disclosed answer key. + +## Privacy, security, and retention + +- Treat pull-request text, source, comments, and downloaded documents as + untrusted input. +- Do not execute commands or patches proposed in annotation records. +- Keep secrets and raw credentials outside context and corpus artifacts. +- Use pseudonymous expert identifiers in exportable artifacts while retaining a + separately controlled independence audit. +- Minimize personal data and retain only evidence needed to reproduce the + quality decision. +- Sign or content-address corpus releases and record repository policy, tool + configuration, dependency lock, and Git commit identities. +- `NVIDIA_NIM_API_KEY` remains the scheduled model credential; this protocol + introduces no `COPILOT_GITHUB_TOKEN` use. + +## Quality-control checklist + +A case may enter a frozen benchmark version only when all answers are yes: + +- Are the repository, pull request, base, head, diff, and context identities + immutable and mutually consistent? +- Did both experts review the same complete context without reviewer outputs? +- Are the experts and adjudicator distinct? +- Did each expert certify completion and enumerate all substantiated findings? +- Was every source finding adjudicated exactly once? +- Does every accepted gold defect include the complete source/fix/regression + contract? +- Are rejected reports prevented from creating gold? +- Are all receipts reproducible from canonical strict JSON? +- Is the case partition free of related-change leakage? +- Were automated reviewer outputs captured only after gold freeze? + +## Limitations + +Expert gold is still an estimate. Reviewers can miss defects, disagree on +severity, or lack specialized domain knowledge. The protocol reduces circular +ground truth and stale-head bias; it does not make review quality a context-free +or permanent property. Commercial reference products and OpenCode configurations +change, so every benchmark release must preserve configuration and time evidence. + +## References + +Hu, R., Wang, X., Wen, X.-C., Zhang, Z., Jiang, B., Gao, P., Peng, C., & Gao, C. (2025). *Benchmarking LLMs for fine-grained code review with enriched context in practice* (arXiv:2511.07017). arXiv. https://doi.org/10.48550/arXiv.2511.07017 + +Kumar, S. P., Bararia, S., & Raj, K. (2026). *Bigger isn't always better: A comparative evaluation of LLMs for automated code review* (arXiv:2606.15689). arXiv. https://arxiv.org/abs/2606.15689 + +Sun, T., Xu, J., Li, Y., Yan, Z., Zhang, G., Xie, L., Geng, L., Wang, Z., Chen, Y., Lin, Q., Duan, W., & Sui, K. (2025). BitsAI-CR: Automated code review via LLM in practice. In *Proceedings of the 33rd ACM International Conference on the Foundations of Software Engineering*. https://doi.org/10.1145/3696630.3728552 + +Zhang, L., Yu, Y., Yu, M., Guo, X., Zhuang, Z., Rong, G., Shao, D., Shen, H., Kuang, H., Li, Z., Wang, B., Zhang, G., Xiang, B., & Xu, X. (2026). *AACR-Bench: Evaluating automatic code review with holistic repository-level context* (arXiv:2601.19494). arXiv. https://arxiv.org/abs/2601.19494 diff --git a/scripts/ci/opencode_review_adjudicate.py b/scripts/ci/opencode_review_adjudicate.py new file mode 100644 index 000000000..6fc09feb8 --- /dev/null +++ b/scripts/ci/opencode_review_adjudicate.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Freeze blinded independent expert annotations into exact-head gold evidence.""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +MODULE_DIR = Path(__file__).resolve().parent +if str(MODULE_DIR) not in sys.path: + sys.path.insert(0, str(MODULE_DIR)) + +from opencode_review_adjudication_primitives import ( # noqa: E402 + AdjudicationError, + array_value, + bool_value, + commit_sha_value, + content_digest, + digest_value, + object_value, + reject, + reject_constant, + strict_pairs, + text_value, + unique_text_values, +) +from opencode_review_adjudication_annotations import ( # noqa: E402 + validate_case, + validate_expert, +) +from opencode_review_adjudication_decisions import ( # noqa: E402 + validate_adjudication, +) + +def reference_once( + identifier: str, + known: set[str], + used: set[str], + path: str, + expert_label: str, +) -> None: + """Require one known expert finding and consume it exactly once.""" + if identifier not in known: + reject(f"{path} references unknown expert {expert_label} finding {identifier!r}") + if identifier in used: + reject(f"{path} references expert {expert_label} finding {identifier!r} more than once") + used.add(identifier) + + +def adjudicate_case( + raw_expert_a: Any, raw_expert_b: Any, raw_adjudication: Any +) -> dict[str, Any]: + """Validate, reconcile, and freeze one complete exact-head gold case.""" + expert_a = validate_expert(raw_expert_a, "expert_a") + expert_b = validate_expert(raw_expert_b, "expert_b") + decision_record = validate_adjudication(raw_adjudication) + if not (expert_a["case"] == expert_b["case"] == decision_record["case"]): + reject("expert and adjudicator case identity must match exactly") + actors = { + expert_a["expert_id"], + expert_b["expert_id"], + decision_record["adjudicator_id"], + } + if len(actors) != 3: + reject("two experts and adjudicator must be distinct pseudonymous roles") + + a_known = {item["expert_finding_id"] for item in expert_a["findings"]} + b_known = {item["expert_finding_id"] for item in expert_b["findings"]} + a_used: set[str] = set() + b_used: set[str] = set() + gold: list[dict[str, Any]] = [] + accepted_both = 0 + accepted_a_only = 0 + accepted_b_only = 0 + rejected_sources = 0 + for index, item in enumerate(decision_record["decisions"]): + path = f"adjudication.decisions[{index}]" + for identifier in item["expert_a_finding_ids"]: + reference_once(identifier, a_known, a_used, path, "A") + for identifier in item["expert_b_finding_ids"]: + reference_once(identifier, b_known, b_used, path, "B") + if item["outcome"] == "accept": + has_a = bool(item["expert_a_finding_ids"]) + has_b = bool(item["expert_b_finding_ids"]) + accepted_both += int(has_a and has_b) + accepted_a_only += int(has_a and not has_b) + accepted_b_only += int(has_b and not has_a) + gold.append( + { + "finding_id": item["gold_finding_id"], + "path": item["path"], + "line": item["line"], + "defect_class": item["defect_class"], + "severity": item["severity"], + "trigger": item["trigger"], + "impact": item["impact"], + "root_cause": item["root_cause"], + "fix_direction": item["fix_direction"], + "regression_target": item["regression_target"], + "source_decision_id": item["decision_id"], + } + ) + else: + rejected_sources += len(item["expert_a_finding_ids"]) + len( + item["expert_b_finding_ids"] + ) + uncovered_a = sorted(a_known - a_used) + uncovered_b = sorted(b_known - b_used) + if uncovered_a or uncovered_b: + details = [] + if uncovered_a: + details.append(f"expert A: {', '.join(uncovered_a)}") + if uncovered_b: + details.append(f"expert B: {', '.join(uncovered_b)}") + reject(f"expert findings are uncovered by adjudication: {'; '.join(details)}") + + gold.sort(key=lambda item: item["finding_id"]) + annotation_receipts = sorted( + [ + { + "annotation_id": expert_a["annotation_id"], + "expert_id": expert_a["expert_id"], + "annotation_sha256": content_digest(expert_a), + }, + { + "annotation_id": expert_b["annotation_id"], + "expert_id": expert_b["expert_id"], + "annotation_sha256": content_digest(expert_b), + }, + ], + key=lambda item: item["expert_id"], + ) + adjudication_receipt = { + "adjudication_id": decision_record["adjudication_id"], + "adjudicator_id": decision_record["adjudicator_id"], + "adjudication_sha256": content_digest(decision_record), + } + report_without_digest = { + "schema_version": "1.0", + "case": expert_a["case"], + "gold_findings": gold, + "agreement_metrics": { + "expert_a_findings": len(a_known), + "expert_b_findings": len(b_known), + "accepted_gold_findings": len(gold), + "accepted_by_both_experts": accepted_both, + "accepted_from_expert_a_only": accepted_a_only, + "accepted_from_expert_b_only": accepted_b_only, + "rejected_source_findings": rejected_sources, + }, + "annotation_receipts": annotation_receipts, + "adjudication_receipt": adjudication_receipt, + } + return { + **report_without_digest, + "freeze_sha256": content_digest(report_without_digest), + } + + + +def load_json(path: Path) -> Any: + """Load strict UTF-8 JSON with bounded stable validation errors.""" + try: + return json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=strict_pairs, + parse_constant=reject_constant, + ) + except (OSError, json.JSONDecodeError) as error: + reject(f"cannot load adjudication evidence: {error}") + + +def write_text(path: Path, content: str) -> None: + """Atomically replace one UTF-8 output after creating its parent directory.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text(content, encoding="utf-8") + temporary.replace(path) + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the adjudicator CLI and return a stable validation status.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--expert-a", type=Path, required=True) + parser.add_argument("--expert-b", type=Path, required=True) + parser.add_argument("--adjudication", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args(argv) + try: + report = adjudicate_case( + load_json(arguments.expert_a), + load_json(arguments.expert_b), + load_json(arguments.adjudication), + ) + except AdjudicationError as error: + print(f"adjudication evidence rejected: {error}", file=sys.stderr) + return 2 + write_text( + arguments.output, + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False) + + "\n", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/opencode_review_adjudication_annotations.py b/scripts/ci/opencode_review_adjudication_annotations.py new file mode 100644 index 000000000..b0fc84bb3 --- /dev/null +++ b/scripts/ci/opencode_review_adjudication_annotations.py @@ -0,0 +1,160 @@ +"""Validate blinded expert annotations for exact-head code-review gold evidence.""" + +from __future__ import annotations + +from typing import Any + +from opencode_review_adjudication_primitives import ( + VALID_SEVERITIES, + REPOSITORY_RE, + array_value, + bool_value, + commit_sha_value, + count_value, + digest_value, + object_value, + reject, + require_exact_fields, + source_path_value, + text_value, +) + +def validate_case(raw_value: Any, path: str) -> dict[str, Any]: + """Validate one immutable repository, pull request, and evidence identity.""" + value = object_value(raw_value, path) + require_exact_fields( + value, + path, + { + "case_id", + "repository", + "pull_request_number", + "base_sha", + "head_sha", + "diff_sha256", + "context_sha256", + }, + ) + repository = text_value(value.get("repository"), f"{path}.repository") + if not REPOSITORY_RE.fullmatch(repository): + reject(f"{path}.repository must use owner/name") + return { + "case_id": text_value(value.get("case_id"), f"{path}.case_id"), + "repository": repository, + "pull_request_number": count_value( + value.get("pull_request_number"), + f"{path}.pull_request_number", + positive=True, + ), + "base_sha": commit_sha_value(value.get("base_sha"), f"{path}.base_sha"), + "head_sha": commit_sha_value(value.get("head_sha"), f"{path}.head_sha"), + "diff_sha256": digest_value( + value.get("diff_sha256"), f"{path}.diff_sha256" + ), + "context_sha256": digest_value( + value.get("context_sha256"), f"{path}.context_sha256" + ), + } + + +def finding_fields( + value: Mapping[str, Any], path: str, *, identifier_field: str +) -> dict[str, Any]: + """Validate the complete source-defect contract shared by experts and gold.""" + severity = text_value(value.get("severity"), f"{path}.severity").casefold() + if severity not in VALID_SEVERITIES: + reject(f"{path}.severity is invalid") + return { + identifier_field: text_value( + value.get(identifier_field), f"{path}.{identifier_field}" + ), + "path": source_path_value(value.get("path"), f"{path}.path"), + "line": count_value(value.get("line"), f"{path}.line", positive=True), + "defect_class": text_value( + value.get("defect_class"), f"{path}.defect_class" + ).casefold(), + "severity": severity, + "trigger": text_value(value.get("trigger"), f"{path}.trigger"), + "impact": text_value(value.get("impact"), f"{path}.impact"), + "root_cause": text_value(value.get("root_cause"), f"{path}.root_cause"), + "fix_direction": text_value( + value.get("fix_direction"), f"{path}.fix_direction" + ), + "regression_target": text_value( + value.get("regression_target"), f"{path}.regression_target" + ), + } + + +def validate_expert(raw_value: Any, path: str) -> dict[str, Any]: + """Validate one blinded, exhaustive, full-context independent annotation.""" + value = object_value(raw_value, path) + require_exact_fields( + value, + path, + { + "schema_version", + "annotation_id", + "expert_id", + "case", + "reviewer_outputs_hidden", + "full_repository_context_reviewed", + "no_additional_findings", + "findings", + }, + ) + if value.get("schema_version") != "1.0": + reject(f"{path}.schema_version must equal '1.0'") + if not bool_value( + value.get("reviewer_outputs_hidden"), f"{path}.reviewer_outputs_hidden" + ): + reject(f"{path}.reviewer_outputs_hidden must be true") + if not bool_value( + value.get("full_repository_context_reviewed"), + f"{path}.full_repository_context_reviewed", + ): + reject(f"{path}.full_repository_context_reviewed must be true") + if not bool_value( + value.get("no_additional_findings"), f"{path}.no_additional_findings" + ): + reject(f"{path}.no_additional_findings must be true") + seen: set[str] = set() + findings: list[dict[str, Any]] = [] + finding_allowed = { + "expert_finding_id", + "path", + "line", + "defect_class", + "severity", + "trigger", + "impact", + "root_cause", + "fix_direction", + "regression_target", + } + for index, raw_finding in enumerate(array_value(value.get("findings"), f"{path}.findings")): + finding_path = f"{path}.findings[{index}]" + finding_value = object_value(raw_finding, finding_path) + require_exact_fields(finding_value, finding_path, finding_allowed) + finding = finding_fields( + finding_value, finding_path, identifier_field="expert_finding_id" + ) + identifier = finding["expert_finding_id"] + if identifier in seen: + reject(f"{finding_path}.expert_finding_id duplicates {identifier!r}") + seen.add(identifier) + findings.append(finding) + return { + "schema_version": "1.0", + "annotation_id": text_value( + value.get("annotation_id"), f"{path}.annotation_id" + ), + "expert_id": text_value(value.get("expert_id"), f"{path}.expert_id"), + "case": validate_case(value.get("case"), f"{path}.case"), + "reviewer_outputs_hidden": True, + "full_repository_context_reviewed": True, + "no_additional_findings": True, + "findings": findings, + } + + diff --git a/scripts/ci/opencode_review_adjudication_decisions.py b/scripts/ci/opencode_review_adjudication_decisions.py new file mode 100644 index 000000000..26aca313a --- /dev/null +++ b/scripts/ci/opencode_review_adjudication_decisions.py @@ -0,0 +1,166 @@ +"""Validate blinded adjudication decisions for exact-head gold evidence.""" + +from __future__ import annotations + +from typing import Any + +from opencode_review_adjudication_primitives import ( + VALID_SEVERITIES, + array_value, + bool_value, + object_value, + optional_positive_count, + optional_text, + reject, + require_exact_fields, + source_path_value, + text_value, + unique_text_values, +) +from opencode_review_adjudication_annotations import validate_case + +def validate_decision(raw_value: Any, path: str) -> dict[str, Any]: + """Validate one source-linked accepted-gold or rejected-report decision.""" + value = object_value(raw_value, path) + fields = { + "decision_id", + "expert_a_finding_ids", + "expert_b_finding_ids", + "outcome", + "gold_finding_id", + "path", + "line", + "defect_class", + "severity", + "trigger", + "impact", + "root_cause", + "fix_direction", + "regression_target", + "reason", + } + require_exact_fields(value, path, fields) + a_ids = unique_text_values( + value.get("expert_a_finding_ids"), f"{path}.expert_a_finding_ids" + ) + b_ids = unique_text_values( + value.get("expert_b_finding_ids"), f"{path}.expert_b_finding_ids" + ) + if not a_ids and not b_ids: + reject(f"{path} must reference at least one expert finding") + outcome = text_value(value.get("outcome"), f"{path}.outcome").casefold() + if outcome not in {"accept", "reject"}: + reject(f"{path}.outcome must equal accept or reject") + decision = { + "decision_id": text_value(value.get("decision_id"), f"{path}.decision_id"), + "expert_a_finding_ids": a_ids, + "expert_b_finding_ids": b_ids, + "outcome": outcome, + "gold_finding_id": optional_text( + value.get("gold_finding_id"), f"{path}.gold_finding_id" + ), + "path": optional_text(value.get("path"), f"{path}.path"), + "line": optional_positive_count(value.get("line"), f"{path}.line"), + "defect_class": optional_text( + value.get("defect_class"), f"{path}.defect_class" + ), + "severity": optional_text(value.get("severity"), f"{path}.severity"), + "trigger": optional_text(value.get("trigger"), f"{path}.trigger"), + "impact": optional_text(value.get("impact"), f"{path}.impact"), + "root_cause": optional_text( + value.get("root_cause"), f"{path}.root_cause" + ), + "fix_direction": optional_text( + value.get("fix_direction"), f"{path}.fix_direction" + ), + "regression_target": optional_text( + value.get("regression_target"), f"{path}.regression_target" + ), + "reason": text_value(value.get("reason"), f"{path}.reason"), + } + evidence_fields = ( + "path", + "line", + "defect_class", + "severity", + "trigger", + "impact", + "root_cause", + "fix_direction", + "regression_target", + ) + if outcome == "accept": + if decision["gold_finding_id"] is None: + reject(f"{path}.gold_finding_id is required for accepted gold") + for name in evidence_fields: + if decision[name] is None: + reject(f"accepted decision {path} requires {name}") + decision["path"] = source_path_value(decision["path"], f"{path}.path") + decision["defect_class"] = decision["defect_class"].casefold() + decision["severity"] = decision["severity"].casefold() + if decision["severity"] not in VALID_SEVERITIES: + reject(f"{path}.severity is invalid") + else: + if decision["gold_finding_id"] is not None: + reject(f"rejected decision {path} must not mint gold") + if any(decision[name] is not None for name in evidence_fields): + reject(f"rejected decision {path} must not carry accepted gold evidence") + return decision + + +def validate_adjudication(raw_value: Any, path: str = "adjudication") -> dict[str, Any]: + """Validate one identity-blinded decision record and its unique decisions.""" + value = object_value(raw_value, path) + require_exact_fields( + value, + path, + { + "schema_version", + "adjudication_id", + "adjudicator_id", + "case", + "reviewer_identities_hidden", + "decisions", + }, + ) + if value.get("schema_version") != "1.0": + reject(f"{path}.schema_version must equal '1.0'") + if not bool_value( + value.get("reviewer_identities_hidden"), + f"{path}.reviewer_identities_hidden", + ): + reject(f"{path}.reviewer_identities_hidden must be true") + decisions: list[dict[str, Any]] = [] + decision_ids: set[str] = set() + gold_ids: set[str] = set() + for index, raw_decision in enumerate( + array_value(value.get("decisions"), f"{path}.decisions") + ): + decision_path = f"{path}.decisions[{index}]" + decision = validate_decision(raw_decision, decision_path) + decision_id = decision["decision_id"] + if decision_id in decision_ids: + reject(f"{decision_path}.decision_id duplicates {decision_id!r}") + decision_ids.add(decision_id) + gold_id = decision["gold_finding_id"] + if gold_id is not None: + if gold_id in gold_ids: + reject(f"{decision_path}.gold_finding_id duplicates {gold_id!r}") + gold_ids.add(gold_id) + decisions.append(decision) + if not decisions: + reject(f"{path}.decisions must not be empty") + return { + "schema_version": "1.0", + "adjudication_id": text_value( + value.get("adjudication_id"), f"{path}.adjudication_id" + ), + "adjudicator_id": text_value( + value.get("adjudicator_id"), f"{path}.adjudicator_id" + ), + "case": validate_case(value.get("case"), f"{path}.case"), + "reviewer_identities_hidden": True, + "decisions": decisions, + } + + diff --git a/scripts/ci/opencode_review_adjudication_primitives.py b/scripts/ci/opencode_review_adjudication_primitives.py new file mode 100644 index 000000000..a7673d011 --- /dev/null +++ b/scripts/ci/opencode_review_adjudication_primitives.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Strict primitive validators for blinded code-review gold evidence.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path, PurePosixPath +from typing import Any + +REPOSITORY_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +COMMIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +VALID_SEVERITIES = {"critical", "high", "medium", "low"} + + +class AdjudicationError(ValueError): + """Signal malformed, incomplete, or internally inconsistent gold evidence.""" + + +def reject(message: str) -> None: + """Raise one stable adjudication validation error.""" + raise AdjudicationError(message) + + +def object_value(value: Any, path: str) -> Mapping[str, Any]: + """Return a JSON object or reject one schema-shape mismatch.""" + if not isinstance(value, Mapping): + reject(f"{path} must be an object") + return value + + +def array_value(value: Any, path: str) -> list[Any]: + """Return a JSON array or reject one schema-shape mismatch.""" + if not isinstance(value, list): + reject(f"{path} must be an array") + return value + + +def require_exact_fields( + value: Mapping[str, Any], path: str, allowed_fields: set[str] +) -> None: + """Reject unreviewed extension fields at one governed schema layer.""" + unknown = sorted(set(value) - allowed_fields) + if unknown: + reject(f"{path} has unknown fields: {', '.join(unknown)}") + + +def text_value(value: Any, path: str) -> str: + """Return stripped non-empty text or reject it.""" + if not isinstance(value, str) or not value.strip(): + reject(f"{path} must be non-empty text") + return value.strip() + + +def optional_text(value: Any, path: str) -> str | None: + """Return ``None`` or stripped non-empty text without scalar coercion.""" + return None if value is None else text_value(value, path) + + +def bool_value(value: Any, path: str) -> bool: + """Return an actual Boolean rather than an integer lookalike.""" + if not isinstance(value, bool): + reject(f"{path} must be boolean") + return value + + +def count_value(value: Any, path: str, *, positive: bool = False) -> int: + """Return a non-negative or positive integer without Boolean coercion.""" + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + reject(f"{path} must be a non-negative integer") + if positive and value == 0: + reject(f"{path} must be a positive integer") + return value + + +def optional_positive_count(value: Any, path: str) -> int | None: + """Return ``None`` or one strictly positive integer.""" + return None if value is None else count_value(value, path, positive=True) + + +def commit_sha_value(value: Any, path: str) -> str: + """Return one lowercase full commit SHA.""" + result = text_value(value, path) + if not COMMIT_SHA_RE.fullmatch(result): + reject(f"{path} must be a 40-character lowercase commit SHA") + return result + + +def digest_value(value: Any, path: str) -> str: + """Return one canonical SHA-256 evidence digest.""" + result = text_value(value, path) + if not DIGEST_RE.fullmatch(result): + reject(f"{path} must use sha256:<64 lowercase hex characters>") + return result + + +def source_path_value(value: Any, path: str) -> str: + """Return a safe repository-relative POSIX source path.""" + result = text_value(value, path) + pure = PurePosixPath(result) + if ( + pure.is_absolute() + or "\\" in result + or any(part in {"", ".", ".."} for part in pure.parts) + ): + reject(f"{path} must be a safe relative source path") + return pure.as_posix() + + +def unique_text_values(value: Any, path: str) -> list[str]: + """Return unique non-empty identifiers while preserving declaration order.""" + output: list[str] = [] + seen: set[str] = set() + for index, item in enumerate(array_value(value, path)): + identifier = text_value(item, f"{path}[{index}]") + if identifier in seen: + reject(f"{path} duplicates {identifier!r}") + seen.add(identifier) + output.append(identifier) + return output + + +def canonical_json(value: Any) -> str: + """Serialize JSON deterministically for content-addressed evidence receipts.""" + return json.dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + allow_nan=False, + ) + + +def content_digest(value: Any) -> str: + """Return a canonical SHA-256 digest for a JSON-compatible value.""" + return f"sha256:{hashlib.sha256(canonical_json(value).encode('utf-8')).hexdigest()}" + + + +def strict_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Build one JSON object while rejecting duplicate member names.""" + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + reject(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def reject_constant(value: str) -> None: + """Reject non-finite constants accepted by Python's permissive JSON parser.""" + reject(f"non-finite JSON number: {value}") + + diff --git a/scripts/ci/opencode_review_sample.py b/scripts/ci/opencode_review_sample.py new file mode 100644 index 000000000..5fda5860e --- /dev/null +++ b/scripts/ci/opencode_review_sample.py @@ -0,0 +1,503 @@ +#!/usr/bin/env python3 +"""Deterministically select exact-head cases for review-quality annotation.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from collections import defaultdict +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +VALID_BUCKETS = {"small", "medium", "large"} +REPOSITORY_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +COMMIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") + + +class CorpusSamplingError(ValueError): + """Signal malformed or internally inconsistent corpus inventory evidence.""" + + +class InsufficientCorpusError(CorpusSamplingError): + """Signal a valid inventory that cannot satisfy its hard sampling policy.""" + + +def reject(message: str) -> None: + """Raise one stable corpus validation error.""" + raise CorpusSamplingError(message) + + +def require_exact_fields( + value: Mapping[str, Any], path: str, allowed_fields: set[str] +) -> None: + """Reject unreviewed extension fields at one governed schema layer.""" + unknown = sorted(set(value) - allowed_fields) + if unknown: + reject(f"{path} has unknown fields: {', '.join(unknown)}") + + +def object_value(value: Any, path: str) -> Mapping[str, Any]: + """Return a JSON object or reject a shape mismatch.""" + if not isinstance(value, Mapping): + reject(f"{path} must be an object") + return value + + +def array_value(value: Any, path: str) -> list[Any]: + """Return a JSON array or reject a shape mismatch.""" + if not isinstance(value, list): + reject(f"{path} must be an array") + return value + + +def text_value(value: Any, path: str) -> str: + """Return stripped non-empty text or reject it.""" + if not isinstance(value, str) or not value.strip(): + reject(f"{path} must be non-empty text") + return value.strip() + + +def bool_value(value: Any, path: str) -> bool: + """Return an actual Boolean rather than an integer lookalike.""" + if not isinstance(value, bool): + reject(f"{path} must be boolean") + return value + + +def count_value(value: Any, path: str, *, positive: bool = False) -> int: + """Return a non-negative or positive integer count.""" + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + reject(f"{path} must be a non-negative integer") + if positive and value == 0: + reject(f"{path} must be a positive integer") + return value + + +def normalized_unique_texts(value: Any, path: str) -> list[str]: + """Return case-folded unique text values while preserving declaration order.""" + output: list[str] = [] + seen: set[str] = set() + for index, item in enumerate(array_value(value, path)): + normalized = text_value(item, f"{path}[{index}]").casefold() + if normalized in seen: + reject(f"{path} duplicates {normalized!r}") + seen.add(normalized) + output.append(normalized) + if not output: + reject(f"{path} must not be empty") + return output + + +def commit_sha_value(value: Any, path: str) -> str: + """Return one lowercase full commit SHA.""" + result = text_value(value, path) + if not COMMIT_SHA_RE.fullmatch(result): + reject(f"{path} must be a 40-character lowercase commit SHA") + return result + + +def digest_value(value: Any, path: str) -> str: + """Return one canonical SHA-256 evidence digest.""" + result = text_value(value, path) + if not DIGEST_RE.fullmatch(result): + reject(f"{path} must use sha256:<64 lowercase hex characters>") + return result + + +def canonical_json(value: Any) -> str: + """Serialize JSON deterministically for content-addressed evidence receipts.""" + return json.dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + allow_nan=False, + ) + + +def content_digest(value: Any) -> str: + """Return a canonical SHA-256 digest for a JSON-compatible value.""" + encoded = canonical_json(value).encode("utf-8") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def validate_inventory(raw_value: Any) -> dict[str, Any]: + """Validate and normalize one offline exact-head candidate inventory.""" + value = object_value(raw_value, "inventory") + require_exact_fields( + value, + "inventory", + {"schema_version", "inventory_id", "generated_at", "sampling_policy", "candidates"}, + ) + if value.get("schema_version") != "1.0": + reject("inventory.schema_version must equal '1.0'") + inventory_id = text_value(value.get("inventory_id"), "inventory.inventory_id") + generated_at = text_value(value.get("generated_at"), "inventory.generated_at") + if not TIMESTAMP_RE.fullmatch(generated_at): + reject("inventory.generated_at must use UTC YYYY-MM-DDTHH:MM:SSZ") + + raw_policy = object_value(value.get("sampling_policy"), "sampling_policy") + require_exact_fields( + raw_policy, + "sampling_policy", + { + "sample_size", + "minimum_primary_languages", + "required_diff_size_buckets", + "required_risk_classes", + "required_defect_classes", + }, + ) + policy = { + "sample_size": count_value( + raw_policy.get("sample_size"), "sampling_policy.sample_size", positive=True + ), + "minimum_primary_languages": count_value( + raw_policy.get("minimum_primary_languages"), + "sampling_policy.minimum_primary_languages", + positive=True, + ), + "required_diff_size_buckets": normalized_unique_texts( + raw_policy.get("required_diff_size_buckets"), + "sampling_policy.required_diff_size_buckets", + ), + "required_risk_classes": normalized_unique_texts( + raw_policy.get("required_risk_classes"), + "sampling_policy.required_risk_classes", + ), + "required_defect_classes": normalized_unique_texts( + raw_policy.get("required_defect_classes"), + "sampling_policy.required_defect_classes", + ), + } + invalid_buckets = set(policy["required_diff_size_buckets"]) - VALID_BUCKETS + if invalid_buckets: + reject("sampling_policy.required_diff_size_buckets contains invalid values") + if policy["minimum_primary_languages"] > policy["sample_size"]: + reject("sampling_policy.minimum_primary_languages exceeds sample_size") + + candidate_fields = { + "case_id", + "repository", + "pull_request_number", + "base_sha", + "head_sha", + "diff_sha256", + "context_sha256", + "primary_language", + "diff_size_bucket", + "risk_class", + "defect_class_targets", + "changed_files", + "additions", + "deletions", + "full_repository_context_available", + "same_head_review_possible", + "independent_expert_capacity_confirmed", + } + candidates: list[dict[str, Any]] = [] + seen_cases: set[str] = set() + seen_heads: set[tuple[str, int, str]] = set() + for index, raw_candidate in enumerate(array_value(value.get("candidates"), "candidates")): + path = f"candidates[{index}]" + candidate_value = object_value(raw_candidate, path) + require_exact_fields(candidate_value, path, candidate_fields) + case_id = text_value(candidate_value.get("case_id"), f"{path}.case_id") + if case_id in seen_cases: + reject(f"{path}.case_id duplicates {case_id!r}") + seen_cases.add(case_id) + repository = text_value(candidate_value.get("repository"), f"{path}.repository") + if not REPOSITORY_RE.fullmatch(repository): + reject(f"{path}.repository must use owner/name") + pull_request_number = count_value( + candidate_value.get("pull_request_number"), + f"{path}.pull_request_number", + positive=True, + ) + head_sha = commit_sha_value(candidate_value.get("head_sha"), f"{path}.head_sha") + exact_identity = (repository.casefold(), pull_request_number, head_sha) + if exact_identity in seen_heads: + reject(f"{path} duplicates an exact-head identity") + seen_heads.add(exact_identity) + bucket = text_value( + candidate_value.get("diff_size_bucket"), f"{path}.diff_size_bucket" + ).casefold() + if bucket not in VALID_BUCKETS: + reject(f"{path}.diff_size_bucket is invalid") + candidate = { + "case_id": case_id, + "repository": repository, + "pull_request_number": pull_request_number, + "base_sha": commit_sha_value( + candidate_value.get("base_sha"), f"{path}.base_sha" + ), + "head_sha": head_sha, + "diff_sha256": digest_value( + candidate_value.get("diff_sha256"), f"{path}.diff_sha256" + ), + "context_sha256": digest_value( + candidate_value.get("context_sha256"), f"{path}.context_sha256" + ), + "primary_language": text_value( + candidate_value.get("primary_language"), f"{path}.primary_language" + ).casefold(), + "diff_size_bucket": bucket, + "risk_class": text_value( + candidate_value.get("risk_class"), f"{path}.risk_class" + ).casefold(), + "defect_class_targets": normalized_unique_texts( + candidate_value.get("defect_class_targets"), + f"{path}.defect_class_targets", + ), + "changed_files": count_value( + candidate_value.get("changed_files"), f"{path}.changed_files", positive=True + ), + "additions": count_value( + candidate_value.get("additions"), f"{path}.additions" + ), + "deletions": count_value( + candidate_value.get("deletions"), f"{path}.deletions" + ), + "full_repository_context_available": bool_value( + candidate_value.get("full_repository_context_available"), + f"{path}.full_repository_context_available", + ), + "same_head_review_possible": bool_value( + candidate_value.get("same_head_review_possible"), + f"{path}.same_head_review_possible", + ), + "independent_expert_capacity_confirmed": bool_value( + candidate_value.get("independent_expert_capacity_confirmed"), + f"{path}.independent_expert_capacity_confirmed", + ), + } + candidates.append(candidate) + if not candidates: + reject("candidates must not be empty") + return { + "schema_version": "1.0", + "inventory_id": inventory_id, + "generated_at": generated_at, + "sampling_policy": policy, + "candidates": candidates, + } + + +def candidate_priority(candidate: Mapping[str, Any], seed: str) -> str: + """Return a deterministic seed-bound ordering key for one candidate.""" + payload = "\0".join((seed, candidate["case_id"], candidate["head_sha"])) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def coverage_tokens(candidate: Mapping[str, Any]) -> set[str]: + """Return hard policy dimensions satisfied by one candidate.""" + tokens = { + f"bucket:{candidate['diff_size_bucket']}", + f"risk:{candidate['risk_class']}", + } + tokens.update(f"defect:{item}" for item in candidate["defect_class_targets"]) + return tokens + + +def sample_inventory(raw_value: Any, *, seed: str) -> dict[str, Any]: + """Select a deterministic policy-complete sample from one valid inventory.""" + normalized_seed = text_value(seed, "seed") + inventory = validate_inventory(raw_value) + policy = inventory["sampling_policy"] + eligible = [ + item + for item in inventory["candidates"] + if item["full_repository_context_available"] + and item["same_head_review_possible"] + and item["independent_expert_capacity_confirmed"] + ] + if len(eligible) < policy["sample_size"]: + raise InsufficientCorpusError( + "eligible candidate count is below sampling_policy.sample_size" + ) + + available_languages = {item["primary_language"] for item in eligible} + if len(available_languages) < policy["minimum_primary_languages"]: + raise InsufficientCorpusError( + "eligible inventory cannot satisfy minimum_primary_languages" + ) + + required = { + *(f"bucket:{item}" for item in policy["required_diff_size_buckets"]), + *(f"risk:{item}" for item in policy["required_risk_classes"]), + *(f"defect:{item}" for item in policy["required_defect_classes"]), + } + available = set().union(*(coverage_tokens(item) for item in eligible)) + missing = sorted(required - available) + if missing: + raise InsufficientCorpusError( + f"eligible inventory lacks required coverage: {', '.join(missing)}" + ) + + priorities = {item["case_id"]: candidate_priority(item, normalized_seed) for item in eligible} + selected: list[dict[str, Any]] = [] + selected_ids: set[str] = set() + uncovered = set(required) + languages: set[str] = set() + minimum_languages = policy["minimum_primary_languages"] + while uncovered or len(languages) < minimum_languages: + ranked = sorted( + (item for item in eligible if item["case_id"] not in selected_ids), + key=lambda item: ( + -( + len(coverage_tokens(item) & uncovered) + + int( + len(languages) < minimum_languages + and item["primary_language"] not in languages + ) + ), + -int( + len(languages) < minimum_languages + and item["primary_language"] not in languages + ), + -len(coverage_tokens(item) & uncovered), + priorities[item["case_id"]], + item["case_id"], + ), + ) + choice = ranked[0] + hard_gain = coverage_tokens(choice) & uncovered + language_gain = ( + len(languages) < minimum_languages + and choice["primary_language"] not in languages + ) + selected.append(choice) + selected_ids.add(choice["case_id"]) + uncovered -= hard_gain + languages.add(choice["primary_language"]) + if len(selected) > policy["sample_size"]: + raise InsufficientCorpusError( + "sampling_policy.sample_size is too small for required coverage and language policy" + ) + + groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list) + for item in eligible: + if item["case_id"] not in selected_ids: + key = ( + item["primary_language"], + item["diff_size_bucket"], + item["risk_class"], + ) + groups[key].append(item) + for values in groups.values(): + values.sort(key=lambda item: (priorities[item["case_id"]], item["case_id"])) + group_order = sorted( + groups, + key=lambda key: hashlib.sha256( + f"{normalized_seed}\0{'|'.join(key)}".encode("utf-8") + ).hexdigest(), + ) + fill_candidates = sorted( + ( + (item_position, group_position, item) + for group_position, key in enumerate(group_order) + for item_position, item in enumerate(groups[key]) + ), + key=lambda entry: (entry[0], entry[1]), + ) + needed = policy["sample_size"] - len(selected) + selected.extend(item for _, _, item in fill_candidates[:needed]) + + selected.sort(key=lambda item: item["case_id"]) + counts: dict[tuple[str, str, str], int] = defaultdict(int) + for item in selected: + counts[(item["primary_language"], item["diff_size_bucket"], item["risk_class"])] += 1 + stratum_counts = [ + { + "primary_language": key[0], + "diff_size_bucket": key[1], + "risk_class": key[2], + "case_count": count, + } + for key, count in sorted(counts.items()) + ] + report_without_digest = { + "schema_version": "1.0", + "sample_id": f"{inventory['inventory_id']}__{content_digest(normalized_seed)[7:19]}", + "source_inventory_id": inventory["inventory_id"], + "source_inventory_sha256": content_digest(inventory), + "selection_seed": normalized_seed, + "sample_size": len(selected), + "eligible_candidate_count": len(eligible), + "excluded_candidate_count": len(inventory["candidates"]) - len(eligible), + "sampling_policy": policy, + "stratum_counts": stratum_counts, + "selected_cases": selected, + } + return { + **report_without_digest, + "selection_sha256": content_digest(report_without_digest), + } + + +def strict_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Build one JSON object while rejecting duplicate member names.""" + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + reject(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def reject_constant(value: str) -> None: + """Reject non-finite constants accepted by Python's permissive JSON parser.""" + reject(f"non-finite JSON number: {value}") + + +def load_json(path: Path) -> Any: + """Load strict UTF-8 JSON with bounded stable validation errors.""" + try: + return json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=strict_pairs, + parse_constant=reject_constant, + ) + except (OSError, json.JSONDecodeError) as error: + reject(f"cannot load corpus inventory: {error}") + + +def write_text(path: Path, content: str) -> None: + """Atomically replace one UTF-8 output after creating its parent directory.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text(content, encoding="utf-8") + temporary.replace(path) + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the sampler CLI and return stable malformed/insufficient statuses.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--seed", default="opencode-review-corpus-v1") + arguments = parser.parse_args(argv) + try: + report = sample_inventory(load_json(arguments.input), seed=arguments.seed) + except InsufficientCorpusError as error: + print(f"corpus inventory insufficient: {error}", file=sys.stderr) + return 3 + except CorpusSamplingError as error: + print(f"corpus inventory rejected: {error}", file=sys.stderr) + return 2 + write_text( + arguments.output, + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False) + + "\n", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_opencode_review_adjudicate.py b/tests/test_opencode_review_adjudicate.py new file mode 100644 index 000000000..bb2a5213f --- /dev/null +++ b/tests/test_opencode_review_adjudicate.py @@ -0,0 +1,414 @@ +"""Tests for blinded expert-gold code-review adjudication.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "scripts/ci/opencode_review_adjudicate.py" + + +def load_module() -> ModuleType: + """Load the exact adjudicator module without package import side effects.""" + spec = importlib.util.spec_from_file_location( + "opencode_review_adjudicate", MODULE_PATH + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +adjudicate = load_module() + + +def case_identity() -> dict[str, Any]: + """Build one immutable review case identity shared by every evidence record.""" + return { + "case_id": "case_001", + "repository": "ContextualWisdomLab/example", + "pull_request_number": 42, + "base_sha": "a" * 40, + "head_sha": "b" * 40, + "diff_sha256": f"sha256:{'c' * 64}", + "context_sha256": f"sha256:{'d' * 64}", + } + + +def source_finding(identifier: str, *, line: int = 12) -> dict[str, Any]: + """Build one complete expert-discovered source defect.""" + return { + "expert_finding_id": identifier, + "path": "scripts/ci/example.py", + "line": line, + "defect_class": "correctness", + "severity": "high", + "trigger": "The input contains a duplicate exact-head identity.", + "impact": "The benchmark counts one pull request twice.", + "root_cause": "The identity set is not checked before aggregation.", + "fix_direction": "Reject duplicate repository, PR, and head tuples.", + "regression_target": "Add a duplicate exact-head fixture to the sampler tests.", + } + + +def expert( + annotation_id: str, expert_id: str, findings: list[dict[str, Any]] +) -> dict[str, Any]: + """Build one blinded independent expert annotation.""" + return { + "schema_version": "1.0", + "annotation_id": annotation_id, + "expert_id": expert_id, + "case": case_identity(), + "reviewer_outputs_hidden": True, + "full_repository_context_reviewed": True, + "no_additional_findings": True, + "findings": findings, + } + + +def decision( + identifier: str, + *, + expert_a_ids: list[str], + expert_b_ids: list[str], + outcome: str, + gold_id: str | None, + line: int | None = 12, +) -> dict[str, Any]: + """Build one accepted or rejected blinded adjudication decision.""" + accepted = outcome == "accept" + return { + "decision_id": identifier, + "expert_a_finding_ids": expert_a_ids, + "expert_b_finding_ids": expert_b_ids, + "outcome": outcome, + "gold_finding_id": gold_id, + "path": "scripts/ci/example.py" if accepted else None, + "line": line if accepted else None, + "defect_class": "correctness" if accepted else None, + "severity": "high" if accepted else None, + "trigger": "The input contains a duplicate exact-head identity." + if accepted + else None, + "impact": "The benchmark counts one pull request twice." if accepted else None, + "root_cause": "The identity set is not checked before aggregation." + if accepted + else None, + "fix_direction": "Reject duplicate repository, PR, and head tuples." + if accepted + else None, + "regression_target": "Add a duplicate exact-head fixture to the sampler tests." + if accepted + else None, + "reason": "Independent experts and source evidence support this defect." + if accepted + else "The report describes intended behavior rather than a defect.", + } + + +def adjudication(decisions: list[dict[str, Any]]) -> dict[str, Any]: + """Build one identity-blinded adjudication record.""" + return { + "schema_version": "1.0", + "adjudication_id": "adjudication_001", + "adjudicator_id": "adjudicator_gamma", + "case": case_identity(), + "reviewer_identities_hidden": True, + "decisions": decisions, + } + + +def valid_inputs() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + """Build two experts and complete decisions covering every source finding once.""" + expert_a = expert( + "annotation_a", + "expert_alpha", + [source_finding("a_shared"), source_finding("a_only", line=30)], + ) + expert_b = expert( + "annotation_b", + "expert_beta", + [source_finding("b_shared"), source_finding("b_rejected", line=44)], + ) + decisions = [ + decision( + "decision_shared", + expert_a_ids=["a_shared"], + expert_b_ids=["b_shared"], + outcome="accept", + gold_id="gold_shared", + ), + decision( + "decision_a_only", + expert_a_ids=["a_only"], + expert_b_ids=[], + outcome="accept", + gold_id="gold_a_only", + line=30, + ), + decision( + "decision_b_rejected", + expert_a_ids=[], + expert_b_ids=["b_rejected"], + outcome="reject", + gold_id=None, + line=None, + ), + ] + return expert_a, expert_b, adjudication(decisions) + + +def test_adjudication_freezes_complete_gold_with_receipts_and_agreement() -> None: + """Accepted findings and all source receipts must form one deterministic freeze.""" + expert_a, expert_b, decision_record = valid_inputs() + first = adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + second = adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + assert first == second + assert [item["finding_id"] for item in first["gold_findings"]] == [ + "gold_a_only", + "gold_shared", + ] + assert first["agreement_metrics"] == { + "expert_a_findings": 2, + "expert_b_findings": 2, + "accepted_gold_findings": 2, + "accepted_by_both_experts": 1, + "accepted_from_expert_a_only": 1, + "accepted_from_expert_b_only": 0, + "rejected_source_findings": 1, + } + assert first["freeze_sha256"].startswith("sha256:") + assert {item["expert_id"] for item in first["annotation_receipts"]} == { + "expert_alpha", + "expert_beta", + } + assert all( + item["annotation_sha256"].startswith("sha256:") + for item in first["annotation_receipts"] + ) + assert first["adjudication_receipt"]["adjudication_sha256"].startswith( + "sha256:" + ) + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + ( + lambda a, b, d: a.update({"unexpected": True}), + "unknown fields", + ), + ( + lambda a, b, d: a["case"].update({"unexpected": True}), + "unknown fields", + ), + ( + lambda a, b, d: a["findings"][0].update({"unexpected": True}), + "unknown fields", + ), + ( + lambda a, b, d: d["decisions"][0].update({"unexpected": True}), + "unknown fields", + ), + ( + lambda a, b, d: a.update({"reviewer_outputs_hidden": False}), + "reviewer_outputs_hidden", + ), + ( + lambda a, b, d: b.update({"full_repository_context_reviewed": False}), + "full_repository_context_reviewed", + ), + ( + lambda a, b, d: a.update({"no_additional_findings": False}), + "no_additional_findings", + ), + ( + lambda a, b, d: d.update({"reviewer_identities_hidden": False}), + "reviewer_identities_hidden", + ), + ], +) +def test_adjudicator_rejects_extensible_or_unblinded_evidence( + mutate: Any, message: str +) -> None: + """Every annotation and decision layer must fail closed and preserve blinding.""" + expert_a, expert_b, decision_record = valid_inputs() + mutate(expert_a, expert_b, decision_record) + with pytest.raises(adjudicate.AdjudicationError, match=message): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + +def test_adjudicator_requires_exact_same_case_identity() -> None: + """Expert and adjudicator evidence must bind to one immutable base and head.""" + expert_a, expert_b, decision_record = valid_inputs() + expert_b["case"]["head_sha"] = "e" * 40 + with pytest.raises(adjudicate.AdjudicationError, match="case identity"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + +def test_adjudicator_requires_independent_pseudonymous_roles() -> None: + """The two experts and adjudicator must be three distinct pseudonymous actors.""" + expert_a, expert_b, decision_record = valid_inputs() + expert_b["expert_id"] = expert_a["expert_id"] + with pytest.raises(adjudicate.AdjudicationError, match="distinct"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + expert_a, expert_b, decision_record = valid_inputs() + decision_record["adjudicator_id"] = expert_a["expert_id"] + with pytest.raises(adjudicate.AdjudicationError, match="distinct"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + +def test_adjudicator_requires_every_source_finding_exactly_once() -> None: + """No expert discovery may be dropped or reused across decisions.""" + expert_a, expert_b, decision_record = valid_inputs() + decision_record["decisions"].pop() + with pytest.raises(adjudicate.AdjudicationError, match="uncovered"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + expert_a, expert_b, decision_record = valid_inputs() + decision_record["decisions"][1]["expert_a_finding_ids"] = ["a_shared"] + with pytest.raises(adjudicate.AdjudicationError, match="more than once"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + expert_a, expert_b, decision_record = valid_inputs() + decision_record["decisions"][0]["expert_a_finding_ids"] = ["unknown"] + with pytest.raises(adjudicate.AdjudicationError, match="unknown expert"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + +def test_adjudicator_rejects_empty_decisions_and_duplicate_ids() -> None: + """Every decision needs source evidence and stable unique identifiers.""" + expert_a, expert_b, decision_record = valid_inputs() + decision_record["decisions"][0]["expert_a_finding_ids"] = [] + decision_record["decisions"][0]["expert_b_finding_ids"] = [] + with pytest.raises(adjudicate.AdjudicationError, match="at least one"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + expert_a, expert_b, decision_record = valid_inputs() + decision_record["decisions"][1]["decision_id"] = "decision_shared" + with pytest.raises(adjudicate.AdjudicationError, match="duplicates"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + expert_a, expert_b, decision_record = valid_inputs() + decision_record["decisions"][1]["gold_finding_id"] = "gold_shared" + with pytest.raises(adjudicate.AdjudicationError, match="gold_finding_id"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + +def test_accept_and_reject_contracts_are_mutually_exclusive() -> None: + """Accepted gold needs complete source evidence; rejected reports must not mint gold.""" + expert_a, expert_b, decision_record = valid_inputs() + decision_record["decisions"][0]["path"] = None + with pytest.raises(adjudicate.AdjudicationError, match="accepted.*path"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + expert_a, expert_b, decision_record = valid_inputs() + rejected = decision_record["decisions"][2] + rejected["gold_finding_id"] = "gold_invalid" + with pytest.raises(adjudicate.AdjudicationError, match="rejected.*gold"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + +def test_adjudicator_rejects_unsafe_path_line_and_scalar_types() -> None: + """Gold evidence must remain source-relative, positively anchored, and type strict.""" + expert_a, expert_b, decision_record = valid_inputs() + decision_record["decisions"][0]["path"] = "../secret" + with pytest.raises(adjudicate.AdjudicationError, match="relative source path"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + expert_a, expert_b, decision_record = valid_inputs() + decision_record["decisions"][0]["line"] = 0 + with pytest.raises(adjudicate.AdjudicationError, match="positive integer"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + expert_a, expert_b, decision_record = valid_inputs() + expert_a["case"]["pull_request_number"] = True + with pytest.raises(adjudicate.AdjudicationError, match="integer"): + adjudicate.adjudicate_case(expert_a, expert_b, decision_record) + + +def test_load_json_rejects_duplicate_keys_and_nonfinite_numbers(tmp_path: Path) -> None: + """Annotation files must reject duplicate keys and non-standard JSON constants.""" + duplicate = tmp_path / "duplicate.json" + duplicate.write_text('{"schema_version":"1.0","schema_version":"1.0"}') + with pytest.raises(adjudicate.AdjudicationError, match="duplicate JSON key"): + adjudicate.load_json(duplicate) + + nonfinite = tmp_path / "nonfinite.json" + nonfinite.write_text('{"line": Infinity}') + with pytest.raises(adjudicate.AdjudicationError, match="non-finite JSON number"): + adjudicate.load_json(nonfinite) + + +def test_cli_writes_atomic_frozen_case_and_rejects_invalid_input( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """The CLI must publish one atomic freeze or return a stable validation status.""" + expert_a, expert_b, decision_record = valid_inputs() + a_path = tmp_path / "a.json" + b_path = tmp_path / "b.json" + d_path = tmp_path / "d.json" + output = tmp_path / "nested" / "gold.json" + for path, value in ( + (a_path, expert_a), + (b_path, expert_b), + (d_path, decision_record), + ): + path.write_text(json.dumps(value), encoding="utf-8") + assert ( + adjudicate.main( + [ + "--expert-a", + str(a_path), + "--expert-b", + str(b_path), + "--adjudication", + str(d_path), + "--output", + str(output), + ] + ) + == 0 + ) + assert json.loads(output.read_text(encoding="utf-8"))["gold_findings"] + assert not output.with_name(f".{output.name}.tmp").exists() + + decision_record["case"]["head_sha"] = "e" * 40 + d_path.write_text(json.dumps(decision_record), encoding="utf-8") + assert ( + adjudicate.main( + [ + "--expert-a", + str(a_path), + "--expert-b", + str(b_path), + "--adjudication", + str(d_path), + "--output", + str(output), + ] + ) + == 2 + ) + assert "adjudication evidence rejected" in capsys.readouterr().err + + +def test_public_adjudicator_callables_have_docstrings() -> None: + """Every production class and function must remain beginner-readable.""" + missing = [ + name + for name, value in vars(adjudicate).items() + if not name.startswith("_") + and (isinstance(value, type) or callable(value)) + and getattr(value, "__module__", None) == adjudicate.__name__ + and not getattr(value, "__doc__", None) + ] + assert missing == [] diff --git a/tests/test_opencode_review_adjudicate_coverage.py b/tests/test_opencode_review_adjudicate_coverage.py new file mode 100644 index 000000000..59dc83647 --- /dev/null +++ b/tests/test_opencode_review_adjudicate_coverage.py @@ -0,0 +1,177 @@ +"""Close branch coverage for blinded review-gold adjudication.""" + +from __future__ import annotations + +import importlib.util +import json +import runpy +from pathlib import Path +from types import ModuleType + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SUPPORT_PATH = ROOT / "tests/test_opencode_review_adjudicate.py" +MODULE_PATH = ROOT / "scripts/ci/opencode_review_adjudicate.py" + + +def load_support() -> ModuleType: + """Load the primary adjudicator test support without requiring a package.""" + spec = importlib.util.spec_from_file_location( + "opencode_review_adjudicate_support", SUPPORT_PATH + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +support = load_support() +adjudicate = support.adjudicate +case_identity = support.case_identity +valid_inputs = support.valid_inputs + + +def test_adjudicator_helper_boundaries_reject_invalid_shapes_and_scalars() -> None: + """Low-level adjudication helpers must reject unsupported JSON values.""" + with pytest.raises(adjudicate.AdjudicationError, match="must be an object"): + adjudicate.object_value([], "object") + with pytest.raises(adjudicate.AdjudicationError, match="must be an array"): + adjudicate.array_value({}, "array") + with pytest.raises(adjudicate.AdjudicationError, match="non-empty text"): + adjudicate.text_value(" ", "text") + with pytest.raises(adjudicate.AdjudicationError, match="must be boolean"): + adjudicate.bool_value(1, "flag") + with pytest.raises(adjudicate.AdjudicationError, match="commit SHA"): + adjudicate.commit_sha_value("main", "head") + with pytest.raises(adjudicate.AdjudicationError, match="sha256"): + adjudicate.digest_value("sha256:invalid", "digest") + with pytest.raises(adjudicate.AdjudicationError, match="duplicates"): + adjudicate.unique_text_values(["same", "same"], "identifiers") + + +def test_case_and_finding_metadata_reject_invalid_repository_and_severity() -> None: + """Source evidence must keep a valid repository coordinate and severity scale.""" + value = case_identity() + value["repository"] = "invalid" + with pytest.raises(adjudicate.AdjudicationError, match="owner/name"): + adjudicate.validate_case(value, "case") + + expert_a, _, _ = valid_inputs() + expert_a["findings"][0]["severity"] = "urgent" + with pytest.raises(adjudicate.AdjudicationError, match="severity"): + adjudicate.validate_expert(expert_a, "expert") + + +def test_expert_annotation_rejects_version_and_duplicate_finding_identity() -> None: + """Expert records need the exact schema and unique local finding identifiers.""" + expert_a, _, _ = valid_inputs() + expert_a["schema_version"] = "2.0" + with pytest.raises(adjudicate.AdjudicationError, match="schema_version"): + adjudicate.validate_expert(expert_a, "expert") + + expert_a, _, _ = valid_inputs() + expert_a["findings"][1]["expert_finding_id"] = "a_shared" + with pytest.raises(adjudicate.AdjudicationError, match="duplicates"): + adjudicate.validate_expert(expert_a, "expert") + + +def test_decision_contract_rejects_invalid_outcome_and_incomplete_acceptance() -> None: + """Accepted and rejected decisions must satisfy mutually exclusive evidence forms.""" + _, _, record = valid_inputs() + record["decisions"][0]["outcome"] = "maybe" + with pytest.raises(adjudicate.AdjudicationError, match="accept or reject"): + adjudicate.validate_adjudication(record) + + _, _, record = valid_inputs() + record["decisions"][0]["gold_finding_id"] = None + with pytest.raises(adjudicate.AdjudicationError, match="gold_finding_id"): + adjudicate.validate_adjudication(record) + + _, _, record = valid_inputs() + record["decisions"][0]["severity"] = "urgent" + with pytest.raises(adjudicate.AdjudicationError, match="severity"): + adjudicate.validate_adjudication(record) + + _, _, record = valid_inputs() + record["decisions"][2]["path"] = "scripts/ci/example.py" + with pytest.raises(adjudicate.AdjudicationError, match="rejected.*evidence"): + adjudicate.validate_adjudication(record) + + +def test_adjudication_record_rejects_version_and_empty_decisions() -> None: + """The decision record must use the exact schema and contain reviewed outcomes.""" + _, _, record = valid_inputs() + record["schema_version"] = "2.0" + with pytest.raises(adjudicate.AdjudicationError, match="schema_version"): + adjudicate.validate_adjudication(record) + + _, _, record = valid_inputs() + record["decisions"] = [] + with pytest.raises(adjudicate.AdjudicationError, match="must not be empty"): + adjudicate.validate_adjudication(record) + + +def test_uncovered_diagnostics_cover_each_expert_side() -> None: + """Incomplete adjudication diagnostics must identify A-only and B-only omissions.""" + expert_a, expert_b, record = valid_inputs() + record["decisions"] = [ + item + for item in record["decisions"] + if item["decision_id"] != "decision_a_only" + ] + with pytest.raises(adjudicate.AdjudicationError, match="expert A"): + adjudicate.adjudicate_case(expert_a, expert_b, record) + + expert_a, expert_b, record = valid_inputs() + record["decisions"] = [ + item + for item in record["decisions"] + if item["decision_id"] != "decision_b_rejected" + ] + with pytest.raises(adjudicate.AdjudicationError, match="expert B"): + adjudicate.adjudicate_case(expert_a, expert_b, record) + + +def test_load_json_wraps_syntax_and_filesystem_errors(tmp_path: Path) -> None: + """Malformed or unavailable evidence files must return bounded stable errors.""" + malformed = tmp_path / "malformed.json" + malformed.write_text("{") + with pytest.raises(adjudicate.AdjudicationError, match="cannot load"): + adjudicate.load_json(malformed) + with pytest.raises(adjudicate.AdjudicationError, match="cannot load"): + adjudicate.load_json(tmp_path / "absent.json") + + +def test_adjudicator_module_entrypoint_uses_main( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Direct script execution must route through the tested adjudicator CLI.""" + expert_a, expert_b, decision_record = valid_inputs() + a_path = tmp_path / "a.json" + b_path = tmp_path / "b.json" + d_path = tmp_path / "d.json" + output = tmp_path / "gold.json" + for path, value in ( + (a_path, expert_a), + (b_path, expert_b), + (d_path, decision_record), + ): + path.write_text(json.dumps(value), encoding="utf-8") + monkeypatch.setattr( + "sys.argv", + [ + str(MODULE_PATH), + "--expert-a", + str(a_path), + "--expert-b", + str(b_path), + "--adjudication", + str(d_path), + "--output", + str(output), + ], + ) + with pytest.raises(SystemExit, match="0"): + runpy.run_path(str(MODULE_PATH), run_name="__main__") + assert output.exists() diff --git a/tests/test_opencode_review_sample.py b/tests/test_opencode_review_sample.py new file mode 100644 index 000000000..f0507b30b --- /dev/null +++ b/tests/test_opencode_review_sample.py @@ -0,0 +1,323 @@ +"""Tests for deterministic head-matched review corpus sampling.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "scripts/ci/opencode_review_sample.py" + + +def load_module() -> ModuleType: + """Load the exact sampler module without package import side effects.""" + spec = importlib.util.spec_from_file_location("opencode_review_sample", MODULE_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +sample = load_module() + + +def candidate( + index: int, + *, + language: str, + bucket: str, + risk: str, + defects: list[str] | None = None, + eligible: bool = True, +) -> dict[str, Any]: + """Build one exact-head candidate with deterministic unique identities.""" + hex_index = f"{index:064x}" + return { + "case_id": f"case_{index:03d}", + "repository": "ContextualWisdomLab/example", + "pull_request_number": index + 1, + "base_sha": f"{index + 1:040x}", + "head_sha": f"{index + 2:040x}", + "diff_sha256": f"sha256:{hex_index}", + "context_sha256": f"sha256:{(index + 1000):064x}", + "primary_language": language, + "diff_size_bucket": bucket, + "risk_class": risk, + "defect_class_targets": defects or [risk], + "changed_files": index + 1, + "additions": 10 + index, + "deletions": index, + "full_repository_context_available": eligible, + "same_head_review_possible": eligible, + "independent_expert_capacity_confirmed": eligible, + } + + +def inventory(*, sample_size: int = 6) -> dict[str, Any]: + """Build a policy-complete inventory spanning required strata.""" + values = [ + candidate(0, language="python", bucket="small", risk="security"), + candidate(1, language="rust", bucket="medium", risk="correctness"), + candidate( + 2, + language="typescript", + bucket="large", + risk="performance", + defects=["performance", "workflow"], + ), + candidate( + 3, + language="python", + bucket="medium", + risk="workflow", + defects=["workflow", "documentation"], + ), + candidate( + 4, + language="rust", + bucket="large", + risk="data_model", + defects=["data_model"], + ), + candidate( + 5, + language="go", + bucket="small", + risk="documentation", + defects=["documentation"], + ), + candidate(6, language="go", bucket="medium", risk="security"), + candidate(7, language="typescript", bucket="small", risk="correctness"), + ] + return { + "schema_version": "1.0", + "inventory_id": "inventory_2026_08_08", + "generated_at": "2026-08-08T11:30:00Z", + "sampling_policy": { + "sample_size": sample_size, + "minimum_primary_languages": 4, + "required_diff_size_buckets": ["small", "medium", "large"], + "required_risk_classes": [ + "security", + "correctness", + "performance", + "workflow", + "documentation", + "data_model", + ], + "required_defect_classes": [ + "security", + "correctness", + "performance", + "workflow", + "documentation", + "data_model", + ], + }, + "candidates": values, + } + + +def test_sampler_is_deterministic_and_covers_required_policy() -> None: + """A frozen seed must reproduce one policy-complete exact-head sample.""" + first = sample.sample_inventory(inventory(), seed="frozen-seed-v1") + second = sample.sample_inventory(inventory(), seed="frozen-seed-v1") + assert first == second + assert first["sample_size"] == 6 + assert len(first["selected_cases"]) == 6 + assert first["selection_sha256"].startswith("sha256:") + assert first["source_inventory_sha256"].startswith("sha256:") + selected = first["selected_cases"] + assert {item["diff_size_bucket"] for item in selected} == { + "small", + "medium", + "large", + } + assert len({item["primary_language"] for item in selected}) >= 4 + assert {item["risk_class"] for item in selected} == { + "security", + "correctness", + "performance", + "workflow", + "documentation", + "data_model", + } + covered_defects = { + defect for item in selected for defect in item["defect_class_targets"] + } + assert covered_defects == { + "security", + "correctness", + "performance", + "workflow", + "documentation", + "data_model", + } + + +def test_sampler_excludes_ineligible_candidates_and_reports_counts() -> None: + """Candidates without all three evidence capacities must not enter the sample.""" + value = inventory(sample_size=6) + value["candidates"].append( + candidate( + 99, + language="kotlin", + bucket="small", + risk="security", + eligible=False, + ) + ) + report = sample.sample_inventory(value, seed="seed") + assert report["eligible_candidate_count"] == 8 + assert report["excluded_candidate_count"] == 1 + assert "case_099" not in {item["case_id"] for item in report["selected_cases"]} + + +def test_different_seeds_change_tie_breaks_without_breaking_policy() -> None: + """Seeded tie-breaking may vary membership while every hard quota remains true.""" + value = inventory(sample_size=6) + for index in range(8, 20): + value["candidates"].append( + candidate( + index, + language=("python", "rust", "typescript", "go")[index % 4], + bucket=("small", "medium", "large")[index % 3], + risk=( + "security", + "correctness", + "performance", + "workflow", + "documentation", + "data_model", + )[index % 6], + ) + ) + first = sample.sample_inventory(value, seed="seed-a") + second = sample.sample_inventory(value, seed="seed-b") + assert first["selection_sha256"] != second["selection_sha256"] + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda value: value.update({"unexpected": True}), "unknown fields"), + ( + lambda value: value["sampling_policy"].update({"unexpected": True}), + "unknown fields", + ), + ( + lambda value: value["candidates"][0].update({"unexpected": True}), + "unknown fields", + ), + ( + lambda value: value["candidates"][0].update({"head_sha": "main"}), + "head_sha", + ), + ( + lambda value: value["candidates"][0].update( + {"full_repository_context_available": 1} + ), + "boolean", + ), + ( + lambda value: value["sampling_policy"].update({"sample_size": True}), + "integer", + ), + ( + lambda value: value["sampling_policy"].update( + {"required_risk_classes": ["security", "Security"]} + ), + "duplicates", + ), + ], +) +def test_sampler_rejects_malformed_or_extensible_evidence( + mutate: Any, message: str +) -> None: + """Sampling evidence must fail closed at every governed schema layer.""" + value = inventory() + mutate(value) + with pytest.raises(sample.CorpusSamplingError, match=message): + sample.validate_inventory(value) + + +def test_sampler_rejects_duplicate_exact_head_identity() -> None: + """The same repository, PR, and immutable head must not appear twice.""" + value = inventory() + duplicate = dict(value["candidates"][0]) + duplicate["case_id"] = "different_case_id" + value["candidates"].append(duplicate) + with pytest.raises(sample.CorpusSamplingError, match="exact-head identity"): + sample.validate_inventory(value) + + +def test_sampler_rejects_insufficient_eligible_policy_coverage() -> None: + """An underpowered inventory must be distinguished from malformed evidence.""" + value = inventory() + value["candidates"] = value["candidates"][:3] + with pytest.raises(sample.InsufficientCorpusError, match="sample_size"): + sample.sample_inventory(value, seed="seed") + + +def test_sampler_rejects_missing_required_stratum() -> None: + """Selection must fail rather than silently omit a required defect stratum.""" + value = inventory() + value["candidates"] = [ + item for item in value["candidates"] if item["risk_class"] != "data_model" + ] + with pytest.raises(sample.InsufficientCorpusError, match="required coverage"): + sample.sample_inventory(value, seed="seed") + + +def test_load_json_rejects_duplicate_keys_and_nonfinite_numbers(tmp_path: Path) -> None: + """JSON evidence must reject ambiguous keys and Python non-finite extensions.""" + duplicate = tmp_path / "duplicate.json" + duplicate.write_text('{"schema_version":"1.0","schema_version":"1.0"}') + with pytest.raises(sample.CorpusSamplingError, match="duplicate JSON key"): + sample.load_json(duplicate) + + nonfinite = tmp_path / "nonfinite.json" + nonfinite.write_text('{"value": NaN}') + with pytest.raises(sample.CorpusSamplingError, match="non-finite JSON number"): + sample.load_json(nonfinite) + + +def test_cli_writes_atomic_report_and_uses_stable_exit_codes( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """The CLI must distinguish success, malformed input, and insufficient inventory.""" + source = tmp_path / "inventory.json" + output = tmp_path / "nested" / "sample.json" + source.write_text(json.dumps(inventory()), encoding="utf-8") + assert sample.main(["--input", str(source), "--output", str(output), "--seed", "s"]) == 0 + assert json.loads(output.read_text(encoding="utf-8"))["sample_size"] == 6 + assert not output.with_name(f".{output.name}.tmp").exists() + + malformed = tmp_path / "malformed.json" + malformed.write_text("[]", encoding="utf-8") + assert sample.main(["--input", str(malformed), "--output", str(output)]) == 2 + assert "corpus inventory rejected" in capsys.readouterr().err + + insufficient = inventory() + insufficient["candidates"] = insufficient["candidates"][:2] + source.write_text(json.dumps(insufficient), encoding="utf-8") + assert sample.main(["--input", str(source), "--output", str(output)]) == 3 + assert "corpus inventory insufficient" in capsys.readouterr().err + + +def test_public_sampler_callables_have_docstrings() -> None: + """Every production class and function must remain beginner-readable.""" + missing = [ + name + for name, value in vars(sample).items() + if not name.startswith("_") + and (isinstance(value, type) or callable(value)) + and getattr(value, "__module__", None) == sample.__name__ + and not getattr(value, "__doc__", None) + ] + assert missing == [] diff --git a/tests/test_opencode_review_sample_coverage.py b/tests/test_opencode_review_sample_coverage.py new file mode 100644 index 000000000..e3dadd4e9 --- /dev/null +++ b/tests/test_opencode_review_sample_coverage.py @@ -0,0 +1,145 @@ +"""Close branch coverage for deterministic review-corpus sampling.""" + +from __future__ import annotations + +import importlib.util +import json +import runpy +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SUPPORT_PATH = ROOT / "tests/test_opencode_review_sample.py" +MODULE_PATH = ROOT / "scripts/ci/opencode_review_sample.py" + + +def load_support() -> ModuleType: + """Load the primary sampler test support without requiring a package.""" + spec = importlib.util.spec_from_file_location("opencode_review_sample_support", SUPPORT_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +support = load_support() +sample = support.sample +inventory = support.inventory + + +def test_sampler_helper_boundaries_reject_invalid_shapes_and_scalars() -> None: + """Low-level schema helpers must reject every unsupported JSON shape.""" + with pytest.raises(sample.CorpusSamplingError, match="must be an array"): + sample.array_value({}, "array") + with pytest.raises(sample.CorpusSamplingError, match="non-empty text"): + sample.text_value(" ", "text") + with pytest.raises(sample.CorpusSamplingError, match="positive integer"): + sample.count_value(0, "count", positive=True) + with pytest.raises(sample.CorpusSamplingError, match="must not be empty"): + sample.normalized_unique_texts([], "values") + with pytest.raises(sample.CorpusSamplingError, match="sha256"): + sample.digest_value("sha256:nope", "digest") + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda value: value.update({"schema_version": "2.0"}), "schema_version"), + (lambda value: value.update({"generated_at": "yesterday"}), "generated_at"), + ( + lambda value: value["sampling_policy"].update( + {"required_diff_size_buckets": ["tiny"]} + ), + "invalid values", + ), + ( + lambda value: value["sampling_policy"].update( + {"minimum_primary_languages": 7} + ), + "exceeds sample_size", + ), + ( + lambda value: value["candidates"].append( + {**value["candidates"][0], "head_sha": "f" * 40} + ), + "case_id duplicates", + ), + ( + lambda value: value["candidates"][0].update({"repository": "missing-slash"}), + "owner/name", + ), + ( + lambda value: value["candidates"][0].update( + {"diff_size_bucket": "tiny"} + ), + "diff_size_bucket", + ), + (lambda value: value.update({"candidates": []}), "must not be empty"), + ], +) +def test_inventory_rejects_remaining_contract_violations( + mutate: Any, message: str +) -> None: + """Inventory metadata, policy, and candidate identities must remain strict.""" + value = inventory() + mutate(value) + with pytest.raises(sample.CorpusSamplingError, match=message): + sample.validate_inventory(value) + + +def test_sampler_detects_unavailable_languages_and_too_small_policy() -> None: + """Valid inventories that cannot fit hard quotas must fail as insufficient.""" + unavailable = inventory() + for item in unavailable["candidates"]: + item["primary_language"] = "python" + with pytest.raises(sample.InsufficientCorpusError, match="cannot satisfy"): + sample.sample_inventory(unavailable, seed="seed") + + too_small = inventory(sample_size=5) + too_small["sampling_policy"]["minimum_primary_languages"] = 4 + with pytest.raises(sample.InsufficientCorpusError, match="too small"): + sample.sample_inventory(too_small, seed="seed") + + +def test_sampler_round_robin_fills_remaining_capacity() -> None: + """After hard quotas, deterministic strata rotation must fill the requested size.""" + report = sample.sample_inventory(inventory(sample_size=8), seed="fill-seed") + assert report["sample_size"] == 8 + assert sum(item["case_count"] for item in report["stratum_counts"]) == 8 + + +def test_load_json_wraps_syntax_and_filesystem_errors(tmp_path: Path) -> None: + """Malformed or unavailable inventory files must produce stable bounded errors.""" + malformed = tmp_path / "malformed.json" + malformed.write_text("{") + with pytest.raises(sample.CorpusSamplingError, match="cannot load corpus inventory"): + sample.load_json(malformed) + with pytest.raises(sample.CorpusSamplingError, match="cannot load corpus inventory"): + sample.load_json(tmp_path / "absent.json") + + +def test_sampler_module_entrypoint_uses_main( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Direct script execution must route through the tested CLI main function.""" + source = tmp_path / "inventory.json" + output = tmp_path / "sample.json" + source.write_text(json.dumps(inventory()), encoding="utf-8") + monkeypatch.setattr( + "sys.argv", + [ + str(MODULE_PATH), + "--input", + str(source), + "--output", + str(output), + "--seed", + "entrypoint", + ], + ) + with pytest.raises(SystemExit, match="0"): + runpy.run_path(str(MODULE_PATH), run_name="__main__") + assert output.exists()