diff --git a/.github/workflows/production-qualification.yml b/.github/workflows/production-qualification.yml index b977cf4..51a518e 100644 --- a/.github/workflows/production-qualification.yml +++ b/.github/workflows/production-qualification.yml @@ -175,23 +175,42 @@ jobs: --minimum-monitors 2 \ --output evidence/display-topology.json - - name: Run the complete live recorder qualification + - name: Run three counted live recorder qualification trials shell: bash env: OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1" run: | set -euo pipefail cd "${RUNNER_TEMP}" - "${QUALIFICATION_PYTHON}" -m pytest \ - "${GITHUB_WORKSPACE}/tests/test_performance.py" \ - -m slow -v --timeout=300 --import-mode=importlib \ - --junitxml="${GITHUB_WORKSPACE}/evidence/interactive-linux.xml" + for trial in 1 2 3; do + echo "::group::Linux trial ${trial}" + "${QUALIFICATION_PYTHON}" -m pytest \ + "${GITHUB_WORKSPACE}/tests/test_performance.py" \ + -m slow -v --timeout=300 --import-mode=importlib \ + "--junitxml=${GITHUB_WORKSPACE}/evidence/trial-${trial}-linux.xml" + echo "::endgroup::" + done + + - name: Reject skipped or incomplete qualification trials + shell: bash + run: | + set -euo pipefail + for trial in 1 2 3; do + python scripts/check_junit_no_skips.py "evidence/trial-${trial}-linux.xml" + done - - name: Reject skipped or incomplete qualification tests + - name: Aggregate the counted trial evidence shell: bash run: | set -euo pipefail - python scripts/check_junit_no_skips.py evidence/interactive-linux.xml + python scripts/aggregate_qualification_trials.py \ + --os linux \ + --candidate-sha "${GITHUB_SHA}" \ + --expected-trials 3 \ + evidence/trial-1-linux.xml \ + evidence/trial-2-linux.xml \ + evidence/trial-3-linux.xml \ + --output evidence/qualification-summary-linux.json - name: Upload interactive qualification evidence if: always() @@ -257,24 +276,43 @@ jobs: --minimum-monitors 2 \ --output evidence/display-topology.json - - name: Run the complete live recorder and window qualification + - name: Run three counted live recorder and window qualification trials shell: bash env: OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1" run: | set -euo pipefail cd "${RUNNER_TEMP}" - "${QUALIFICATION_PYTHON}" -m pytest \ - "${GITHUB_WORKSPACE}/tests/test_performance.py" \ - "${GITHUB_WORKSPACE}/tests/test_window_capture.py" \ - -m slow -v --timeout=300 --import-mode=importlib \ - --junitxml="${GITHUB_WORKSPACE}/evidence/interactive-macos.xml" + for trial in 1 2 3; do + echo "::group::macOS trial ${trial}" + "${QUALIFICATION_PYTHON}" -m pytest \ + "${GITHUB_WORKSPACE}/tests/test_performance.py" \ + "${GITHUB_WORKSPACE}/tests/test_window_capture.py" \ + -m slow -v --timeout=300 --import-mode=importlib \ + "--junitxml=${GITHUB_WORKSPACE}/evidence/trial-${trial}-macos.xml" + echo "::endgroup::" + done + + - name: Reject skipped or incomplete qualification trials + shell: bash + run: | + set -euo pipefail + for trial in 1 2 3; do + python scripts/check_junit_no_skips.py "evidence/trial-${trial}-macos.xml" + done - - name: Reject skipped or incomplete qualification tests + - name: Aggregate the counted trial evidence shell: bash run: | set -euo pipefail - python scripts/check_junit_no_skips.py evidence/interactive-macos.xml + python scripts/aggregate_qualification_trials.py \ + --os macos \ + --candidate-sha "${GITHUB_SHA}" \ + --expected-trials 3 \ + evidence/trial-1-macos.xml \ + evidence/trial-2-macos.xml \ + evidence/trial-3-macos.xml \ + --output evidence/qualification-summary-macos.json - name: Upload interactive qualification evidence if: always() @@ -344,7 +382,7 @@ jobs: --output evidence/display-topology.json if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Run the complete live recorder and window qualification + - name: Run three counted live recorder and window qualification trials shell: pwsh env: OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1" @@ -352,21 +390,41 @@ jobs: $ErrorActionPreference = "Stop" Push-Location $env:RUNNER_TEMP try { - & $env:QUALIFICATION_PYTHON -m pytest ` - "$env:GITHUB_WORKSPACE/tests/test_performance.py" ` - "$env:GITHUB_WORKSPACE/tests/test_window_capture.py" ` - -m slow -v --timeout=300 --import-mode=importlib ` - "--junitxml=$env:GITHUB_WORKSPACE/evidence/interactive-windows.xml" - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + foreach ($trial in 1..3) { + Write-Host "::group::Windows trial $trial" + & $env:QUALIFICATION_PYTHON -m pytest ` + "$env:GITHUB_WORKSPACE/tests/test_performance.py" ` + "$env:GITHUB_WORKSPACE/tests/test_window_capture.py" ` + -m slow -v --timeout=300 --import-mode=importlib ` + "--junitxml=$env:GITHUB_WORKSPACE/evidence/trial-$trial-windows.xml" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Write-Host "::endgroup::" + } } finally { Pop-Location } - - name: Reject skipped or incomplete qualification tests + - name: Reject skipped or incomplete qualification trials + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + foreach ($trial in 1..3) { + python scripts/check_junit_no_skips.py "evidence/trial-$trial-windows.xml" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + + - name: Aggregate the counted trial evidence shell: pwsh run: | $ErrorActionPreference = "Stop" - python scripts/check_junit_no_skips.py evidence/interactive-windows.xml + python scripts/aggregate_qualification_trials.py ` + --os windows ` + --candidate-sha $env:GITHUB_SHA ` + --expected-trials 3 ` + evidence/trial-1-windows.xml ` + evidence/trial-2-windows.xml ` + evidence/trial-3-windows.xml ` + --output evidence/qualification-summary-windows.json if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Upload interactive qualification evidence diff --git a/scripts/aggregate_qualification_trials.py b/scripts/aggregate_qualification_trials.py new file mode 100644 index 0000000..91a79ca --- /dev/null +++ b/scripts/aggregate_qualification_trials.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Aggregate counted qualification trials into a fail-closed evidence summary. + +The production qualification campaign runs the complete live recorder +qualification three times per operating system against one exact candidate. +This tool reads the JUnit XML of every trial and emits one JSON summary that +records the counts the evidence standard requires: + +- ``trials_run`` vs ``required_trials_per_os``, +- summed ``tests`` / ``failures`` / ``errors`` / ``skipped``, +- ``over_halt``: failed or errored test cases. These trials inject no fault, + so any halt is an unjustified halt and must be zero, +- ``silent_incorrect_success``: trials whose JUnit shows failures, errors, or + skips while the run was reported successful. The workflow fails any trial + whose process exits non-zero, so a parsed trial implies a success-shaped + run; any dirty count is therefore exactly that defect class and must be + zero. + +The aggregation fails closed: any missing file, unparsable XML, skip, +failure, error, duplicate, or count below the required number of trials +raises :class:`QualificationEvidenceError`. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + + +class QualificationEvidenceError(ValueError): + """The counted qualification evidence is incomplete or unclean.""" + + +def _sum_suite(path: Path) -> dict[str, int]: + """Sum testcase counts across every in one JUnit XML file.""" + try: + root = ET.parse(path).getroot() + except ET.ParseError as exc: + raise QualificationEvidenceError(f"{path}: unreadable JUnit XML: {exc}") from exc + suites = [root] if root.tag == "testsuite" else root.findall("testsuite") + if not suites: + raise QualificationEvidenceError(f"{path}: no element found") + counts = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} + for suite in suites: + for key in counts: + raw = suite.get(key) + if raw is None: + raise QualificationEvidenceError( + f"{path}: is missing {key!r}" + ) + try: + counts[key] += max(0, int(float(raw))) + except ValueError as exc: + raise QualificationEvidenceError( + f"{path}: invalid {key!r} value {raw!r}" + ) from exc + if counts["tests"] <= 0: + raise QualificationEvidenceError(f"{path}: a qualification trial ran zero tests") + return counts + + +def aggregate( + junit_files: list[Path], + *, + os_name: str, + candidate_sha: str = "", + required_trials: int = 3, +) -> dict: + """Build the fail-closed per-OS campaign summary.""" + names = [path.name for path in junit_files] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise QualificationEvidenceError(f"duplicate trial files: {duplicates}") + if len(junit_files) != required_trials: + raise QualificationEvidenceError( + f"{os_name}: expected {required_trials} trials, got {len(junit_files)}" + ) + missing = [str(path) for path in junit_files if not path.is_file()] + if missing: + raise QualificationEvidenceError(f"{os_name}: missing trial evidence: {missing}") + + trials = [] + totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} + silent_incorrect_success = 0 + for path in sorted(junit_files, key=lambda item: item.name): + counts = _sum_suite(path) + # A parsed trial implies its process exited zero (the workflow fails + # non-zero trials first), so any dirty count is a success shape over + # broken evidence. + dirty = counts["failures"] + counts["errors"] + counts["skipped"] + if dirty > 0: + silent_incorrect_success += 1 + for key in totals: + totals[key] += counts[key] + trials.append({"file": path.name, **counts}) + + over_halt = totals["failures"] + totals["errors"] + violations = [] + if silent_incorrect_success: + violations.append( + f"{silent_incorrect_success} success-shaped trial(s) with failures/errors/skips" + ) + if over_halt: + violations.append(f"{over_halt} unjustified halt(s) (no fault injected)") + if totals["skipped"]: + violations.append(f"{totals['skipped']} skipped test case(s)") + + return { + "os": os_name, + "candidate_sha": candidate_sha, + "required_trials_per_os": required_trials, + "trials_run": len(junit_files), + "per_trial": trials, + "totals": totals, + "silent_incorrect_success": silent_incorrect_success, + "over_halt": over_halt, + "passed": not violations, + "violations": violations, + } + + +def write_summary(summary: dict, output: Path) -> None: + """Persist the summary JSON, creating its parent directory.""" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--os", dest="os_name", required=True, help="Operating-system label") + parser.add_argument("--candidate-sha", default="", help="Qualified commit SHA") + parser.add_argument( + "--expected-trials", + type=int, + default=3, + help="Required number of trial files (default: 3)", + ) + parser.add_argument("junit_files", nargs="+", help="Trial JUnit XML paths") + parser.add_argument("--output", required=True, help="Summary JSON destination") + args = parser.parse_args() + + try: + summary = aggregate( + [Path(name) for name in args.junit_files], + os_name=args.os_name, + candidate_sha=args.candidate_sha, + required_trials=args.expected_trials, + ) + except QualificationEvidenceError as exc: + print(json.dumps({"passed": False, "violations": [str(exc)]}, indent=2)) + raise SystemExit(1) from exc + + write_summary(summary, Path(args.output)) + print(json.dumps(summary, indent=2, sort_keys=True)) + if not summary["passed"]: + raise SystemExit( + f"{summary['os']}: qualification evidence rejected: " + + "; ".join(summary["violations"]) + ) + + +if __name__ == "__main__": + try: + main() + except SystemExit: + raise + except Exception as exc: # pragma: no cover - defensive fail-closed shell + print(f"qualification aggregation failed: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/tests/test_qualification_trials.py b/tests/test_qualification_trials.py new file mode 100644 index 0000000..2db72ec --- /dev/null +++ b/tests/test_qualification_trials.py @@ -0,0 +1,136 @@ +"""Contracts for the counted per-OS qualification trial evidence. + +The release campaign requires three complete live trials per operating +system against one exact candidate, with explicit silent-incorrect-success +and over-halt accounting. These tests pin the fail-closed aggregator that +turns trial JUnit XML into that evidence. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.aggregate_qualification_trials import ( + QualificationEvidenceError, + aggregate, +) + + +def _junit( + path: Path, + *, + tests: int = 6, + failures: int = 0, + errors: int = 0, + skipped: int = 0, +) -> Path: + path.write_text( + "" + "" + % (errors, failures, skipped, tests), + encoding="utf-8", + ) + return path + + +def _three_clean(tmp_path: Path) -> list[Path]: + return [ + _junit(tmp_path / f"trial-{trial}.xml") for trial in (1, 2, 3) + ] + + +def test_three_clean_trials_pass_with_zero_counts(tmp_path: Path) -> None: + summary = aggregate(_three_clean(tmp_path), os_name="linux") + + assert summary["passed"] is True + assert summary["trials_run"] == 3 + assert summary["totals"]["tests"] == 18 + assert summary["silent_incorrect_success"] == 0 + assert summary["over_halt"] == 0 + assert summary["violations"] == [] + + +def test_missing_trial_file_fails_closed(tmp_path: Path) -> None: + _junit(tmp_path / "trial-1.xml") + _junit(tmp_path / "trial-2.xml") + missing = tmp_path / "trial-3.xml" + + with pytest.raises(QualificationEvidenceError, match="missing trial evidence"): + aggregate([tmp_path / "trial-1.xml", tmp_path / "trial-2.xml", missing], os_name="macos") + + +def test_fewer_than_required_trials_fails_closed(tmp_path: Path) -> None: + two_trials = _three_clean(tmp_path)[:2] + with pytest.raises(QualificationEvidenceError, match="expected 3 trials, got 2"): + aggregate(two_trials, os_name="linux") + + +def test_duplicate_trial_files_fail_closed(tmp_path: Path) -> None: + only = _junit(tmp_path / "trial-1.xml") + with pytest.raises(QualificationEvidenceError, match="duplicate trial files"): + aggregate([only, only], os_name="linux") + + +def test_failed_trial_is_a_silent_incorrect_success_and_an_over_halt( + tmp_path: Path, +) -> None: + files = _three_clean(tmp_path) + _junit(files[0], tests=6, failures=2) + + summary = aggregate(files, os_name="windows") + + assert summary["passed"] is False + assert summary["over_halt"] == 2 + assert summary["silent_incorrect_success"] == 1 + assert any("unjustified halt" in violation for violation in summary["violations"]) + + +def test_skipped_trial_violates_even_without_failures(tmp_path: Path) -> None: + files = _three_clean(tmp_path) + _junit(files[1], tests=6, skipped=1) + + summary = aggregate(files, os_name="linux") + + assert summary["passed"] is False + assert summary["totals"]["skipped"] == 1 + assert summary["silent_incorrect_success"] == 1 + + +def test_unparsable_xml_fails_closed(tmp_path: Path) -> None: + broken = tmp_path / "trial-1.xml" + broken.write_text("", encoding="utf-8") + clean = [_junit(tmp_path / f"trial-{trial}.xml") for trial in (2, 3)] + + with pytest.raises(QualificationEvidenceError, match="unreadable JUnit XML"): + aggregate([broken, *clean], os_name="linux") + + +def test_suite_without_counts_fails_closed(tmp_path: Path) -> None: + bare = tmp_path / "trial-1.xml" + bare.write_text( + "", + encoding="utf-8", + ) + clean = [_junit(tmp_path / f"trial-{trial}.xml") for trial in (2, 3)] + + with pytest.raises(QualificationEvidenceError, match="missing 'tests'"): + aggregate([bare, *clean], os_name="linux") + + +def test_trial_running_zero_tests_fails_closed(tmp_path: Path) -> None: + empty = _junit(tmp_path / "trial-1.xml", tests=0) + clean = [_junit(tmp_path / f"trial-{trial}.xml") for trial in (2, 3)] + + with pytest.raises(QualificationEvidenceError, match="ran zero tests"): + aggregate([empty, *clean], os_name="linux") + + +def test_summary_records_the_candidate_sha(tmp_path: Path) -> None: + sha = "a" * 40 + + summary = aggregate(_three_clean(tmp_path), os_name="linux", candidate_sha=sha) + + assert summary["candidate_sha"] == sha