From 6cc65db5c63bec6aad01f9f397f8af083eff1f96 Mon Sep 17 00:00:00 2001 From: abrichr Date: Fri, 21 Aug 2026 16:03:47 -0400 Subject: [PATCH] ci: require three counted live qualification trials per OS Each interactive qualification job now runs the complete live suite three times against the exact candidate, rejects skipped trials, and aggregates the JUnit evidence into a fail-closed per-OS summary that records trials_run, totals, silent_incorrect_success, and over_halt. These trials inject no fault, so any halt is unjustified and must be zero; a success-shaped report over failures, errors, or skips counts as a silent incorrect success and also blocks. The seven-job release evidence contract is unchanged: the trial loop lives inside each existing interactive job, so check_release_ci.py keeps validating the same job set. --- .../workflows/production-qualification.yml | 106 ++++++++--- scripts/aggregate_qualification_trials.py | 172 ++++++++++++++++++ tests/test_qualification_trials.py | 136 ++++++++++++++ 3 files changed, 390 insertions(+), 24 deletions(-) create mode 100644 scripts/aggregate_qualification_trials.py create mode 100644 tests/test_qualification_trials.py diff --git a/.github/workflows/production-qualification.yml b/.github/workflows/production-qualification.yml index b977cf4..51a518e 100644 --- a/.github/workflows/production-qualification.yml +++ b/.github/workflows/production-qualification.yml @@ -175,23 +175,42 @@ jobs: --minimum-monitors 2 \ --output evidence/display-topology.json - - name: Run the complete live recorder qualification + - name: Run three counted live recorder qualification trials shell: bash env: OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1" run: | set -euo pipefail cd "${RUNNER_TEMP}" - "${QUALIFICATION_PYTHON}" -m pytest \ - "${GITHUB_WORKSPACE}/tests/test_performance.py" \ - -m slow -v --timeout=300 --import-mode=importlib \ - --junitxml="${GITHUB_WORKSPACE}/evidence/interactive-linux.xml" + for trial in 1 2 3; do + echo "::group::Linux trial ${trial}" + "${QUALIFICATION_PYTHON}" -m pytest \ + "${GITHUB_WORKSPACE}/tests/test_performance.py" \ + -m slow -v --timeout=300 --import-mode=importlib \ + "--junitxml=${GITHUB_WORKSPACE}/evidence/trial-${trial}-linux.xml" + echo "::endgroup::" + done + + - name: Reject skipped or incomplete qualification trials + shell: bash + run: | + set -euo pipefail + for trial in 1 2 3; do + python scripts/check_junit_no_skips.py "evidence/trial-${trial}-linux.xml" + done - - name: Reject skipped or incomplete qualification tests + - name: Aggregate the counted trial evidence shell: bash run: | set -euo pipefail - python scripts/check_junit_no_skips.py evidence/interactive-linux.xml + python scripts/aggregate_qualification_trials.py \ + --os linux \ + --candidate-sha "${GITHUB_SHA}" \ + --expected-trials 3 \ + evidence/trial-1-linux.xml \ + evidence/trial-2-linux.xml \ + evidence/trial-3-linux.xml \ + --output evidence/qualification-summary-linux.json - name: Upload interactive qualification evidence if: always() @@ -257,24 +276,43 @@ jobs: --minimum-monitors 2 \ --output evidence/display-topology.json - - name: Run the complete live recorder and window qualification + - name: Run three counted live recorder and window qualification trials shell: bash env: OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1" run: | set -euo pipefail cd "${RUNNER_TEMP}" - "${QUALIFICATION_PYTHON}" -m pytest \ - "${GITHUB_WORKSPACE}/tests/test_performance.py" \ - "${GITHUB_WORKSPACE}/tests/test_window_capture.py" \ - -m slow -v --timeout=300 --import-mode=importlib \ - --junitxml="${GITHUB_WORKSPACE}/evidence/interactive-macos.xml" + for trial in 1 2 3; do + echo "::group::macOS trial ${trial}" + "${QUALIFICATION_PYTHON}" -m pytest \ + "${GITHUB_WORKSPACE}/tests/test_performance.py" \ + "${GITHUB_WORKSPACE}/tests/test_window_capture.py" \ + -m slow -v --timeout=300 --import-mode=importlib \ + "--junitxml=${GITHUB_WORKSPACE}/evidence/trial-${trial}-macos.xml" + echo "::endgroup::" + done + + - name: Reject skipped or incomplete qualification trials + shell: bash + run: | + set -euo pipefail + for trial in 1 2 3; do + python scripts/check_junit_no_skips.py "evidence/trial-${trial}-macos.xml" + done - - name: Reject skipped or incomplete qualification tests + - name: Aggregate the counted trial evidence shell: bash run: | set -euo pipefail - python scripts/check_junit_no_skips.py evidence/interactive-macos.xml + python scripts/aggregate_qualification_trials.py \ + --os macos \ + --candidate-sha "${GITHUB_SHA}" \ + --expected-trials 3 \ + evidence/trial-1-macos.xml \ + evidence/trial-2-macos.xml \ + evidence/trial-3-macos.xml \ + --output evidence/qualification-summary-macos.json - name: Upload interactive qualification evidence if: always() @@ -344,7 +382,7 @@ jobs: --output evidence/display-topology.json if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Run the complete live recorder and window qualification + - name: Run three counted live recorder and window qualification trials shell: pwsh env: OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1" @@ -352,21 +390,41 @@ jobs: $ErrorActionPreference = "Stop" Push-Location $env:RUNNER_TEMP try { - & $env:QUALIFICATION_PYTHON -m pytest ` - "$env:GITHUB_WORKSPACE/tests/test_performance.py" ` - "$env:GITHUB_WORKSPACE/tests/test_window_capture.py" ` - -m slow -v --timeout=300 --import-mode=importlib ` - "--junitxml=$env:GITHUB_WORKSPACE/evidence/interactive-windows.xml" - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + foreach ($trial in 1..3) { + Write-Host "::group::Windows trial $trial" + & $env:QUALIFICATION_PYTHON -m pytest ` + "$env:GITHUB_WORKSPACE/tests/test_performance.py" ` + "$env:GITHUB_WORKSPACE/tests/test_window_capture.py" ` + -m slow -v --timeout=300 --import-mode=importlib ` + "--junitxml=$env:GITHUB_WORKSPACE/evidence/trial-$trial-windows.xml" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Write-Host "::endgroup::" + } } finally { Pop-Location } - - name: Reject skipped or incomplete qualification tests + - name: Reject skipped or incomplete qualification trials + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + foreach ($trial in 1..3) { + python scripts/check_junit_no_skips.py "evidence/trial-$trial-windows.xml" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + + - name: Aggregate the counted trial evidence shell: pwsh run: | $ErrorActionPreference = "Stop" - python scripts/check_junit_no_skips.py evidence/interactive-windows.xml + python scripts/aggregate_qualification_trials.py ` + --os windows ` + --candidate-sha $env:GITHUB_SHA ` + --expected-trials 3 ` + evidence/trial-1-windows.xml ` + evidence/trial-2-windows.xml ` + evidence/trial-3-windows.xml ` + --output evidence/qualification-summary-windows.json if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Upload interactive qualification evidence diff --git a/scripts/aggregate_qualification_trials.py b/scripts/aggregate_qualification_trials.py new file mode 100644 index 0000000..91a79ca --- /dev/null +++ b/scripts/aggregate_qualification_trials.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Aggregate counted qualification trials into a fail-closed evidence summary. + +The production qualification campaign runs the complete live recorder +qualification three times per operating system against one exact candidate. +This tool reads the JUnit XML of every trial and emits one JSON summary that +records the counts the evidence standard requires: + +- ``trials_run`` vs ``required_trials_per_os``, +- summed ``tests`` / ``failures`` / ``errors`` / ``skipped``, +- ``over_halt``: failed or errored test cases. These trials inject no fault, + so any halt is an unjustified halt and must be zero, +- ``silent_incorrect_success``: trials whose JUnit shows failures, errors, or + skips while the run was reported successful. The workflow fails any trial + whose process exits non-zero, so a parsed trial implies a success-shaped + run; any dirty count is therefore exactly that defect class and must be + zero. + +The aggregation fails closed: any missing file, unparsable XML, skip, +failure, error, duplicate, or count below the required number of trials +raises :class:`QualificationEvidenceError`. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + + +class QualificationEvidenceError(ValueError): + """The counted qualification evidence is incomplete or unclean.""" + + +def _sum_suite(path: Path) -> dict[str, int]: + """Sum testcase counts across every in one JUnit XML file.""" + try: + root = ET.parse(path).getroot() + except ET.ParseError as exc: + raise QualificationEvidenceError(f"{path}: unreadable JUnit XML: {exc}") from exc + suites = [root] if root.tag == "testsuite" else root.findall("testsuite") + if not suites: + raise QualificationEvidenceError(f"{path}: no element found") + counts = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} + for suite in suites: + for key in counts: + raw = suite.get(key) + if raw is None: + raise QualificationEvidenceError( + f"{path}: is missing {key!r}" + ) + try: + counts[key] += max(0, int(float(raw))) + except ValueError as exc: + raise QualificationEvidenceError( + f"{path}: invalid {key!r} value {raw!r}" + ) from exc + if counts["tests"] <= 0: + raise QualificationEvidenceError(f"{path}: a qualification trial ran zero tests") + return counts + + +def aggregate( + junit_files: list[Path], + *, + os_name: str, + candidate_sha: str = "", + required_trials: int = 3, +) -> dict: + """Build the fail-closed per-OS campaign summary.""" + names = [path.name for path in junit_files] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise QualificationEvidenceError(f"duplicate trial files: {duplicates}") + if len(junit_files) != required_trials: + raise QualificationEvidenceError( + f"{os_name}: expected {required_trials} trials, got {len(junit_files)}" + ) + missing = [str(path) for path in junit_files if not path.is_file()] + if missing: + raise QualificationEvidenceError(f"{os_name}: missing trial evidence: {missing}") + + trials = [] + totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} + silent_incorrect_success = 0 + for path in sorted(junit_files, key=lambda item: item.name): + counts = _sum_suite(path) + # A parsed trial implies its process exited zero (the workflow fails + # non-zero trials first), so any dirty count is a success shape over + # broken evidence. + dirty = counts["failures"] + counts["errors"] + counts["skipped"] + if dirty > 0: + silent_incorrect_success += 1 + for key in totals: + totals[key] += counts[key] + trials.append({"file": path.name, **counts}) + + over_halt = totals["failures"] + totals["errors"] + violations = [] + if silent_incorrect_success: + violations.append( + f"{silent_incorrect_success} success-shaped trial(s) with failures/errors/skips" + ) + if over_halt: + violations.append(f"{over_halt} unjustified halt(s) (no fault injected)") + if totals["skipped"]: + violations.append(f"{totals['skipped']} skipped test case(s)") + + return { + "os": os_name, + "candidate_sha": candidate_sha, + "required_trials_per_os": required_trials, + "trials_run": len(junit_files), + "per_trial": trials, + "totals": totals, + "silent_incorrect_success": silent_incorrect_success, + "over_halt": over_halt, + "passed": not violations, + "violations": violations, + } + + +def write_summary(summary: dict, output: Path) -> None: + """Persist the summary JSON, creating its parent directory.""" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--os", dest="os_name", required=True, help="Operating-system label") + parser.add_argument("--candidate-sha", default="", help="Qualified commit SHA") + parser.add_argument( + "--expected-trials", + type=int, + default=3, + help="Required number of trial files (default: 3)", + ) + parser.add_argument("junit_files", nargs="+", help="Trial JUnit XML paths") + parser.add_argument("--output", required=True, help="Summary JSON destination") + args = parser.parse_args() + + try: + summary = aggregate( + [Path(name) for name in args.junit_files], + os_name=args.os_name, + candidate_sha=args.candidate_sha, + required_trials=args.expected_trials, + ) + except QualificationEvidenceError as exc: + print(json.dumps({"passed": False, "violations": [str(exc)]}, indent=2)) + raise SystemExit(1) from exc + + write_summary(summary, Path(args.output)) + print(json.dumps(summary, indent=2, sort_keys=True)) + if not summary["passed"]: + raise SystemExit( + f"{summary['os']}: qualification evidence rejected: " + + "; ".join(summary["violations"]) + ) + + +if __name__ == "__main__": + try: + main() + except SystemExit: + raise + except Exception as exc: # pragma: no cover - defensive fail-closed shell + print(f"qualification aggregation failed: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/tests/test_qualification_trials.py b/tests/test_qualification_trials.py new file mode 100644 index 0000000..2db72ec --- /dev/null +++ b/tests/test_qualification_trials.py @@ -0,0 +1,136 @@ +"""Contracts for the counted per-OS qualification trial evidence. + +The release campaign requires three complete live trials per operating +system against one exact candidate, with explicit silent-incorrect-success +and over-halt accounting. These tests pin the fail-closed aggregator that +turns trial JUnit XML into that evidence. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.aggregate_qualification_trials import ( + QualificationEvidenceError, + aggregate, +) + + +def _junit( + path: Path, + *, + tests: int = 6, + failures: int = 0, + errors: int = 0, + skipped: int = 0, +) -> Path: + path.write_text( + "" + "" + % (errors, failures, skipped, tests), + encoding="utf-8", + ) + return path + + +def _three_clean(tmp_path: Path) -> list[Path]: + return [ + _junit(tmp_path / f"trial-{trial}.xml") for trial in (1, 2, 3) + ] + + +def test_three_clean_trials_pass_with_zero_counts(tmp_path: Path) -> None: + summary = aggregate(_three_clean(tmp_path), os_name="linux") + + assert summary["passed"] is True + assert summary["trials_run"] == 3 + assert summary["totals"]["tests"] == 18 + assert summary["silent_incorrect_success"] == 0 + assert summary["over_halt"] == 0 + assert summary["violations"] == [] + + +def test_missing_trial_file_fails_closed(tmp_path: Path) -> None: + _junit(tmp_path / "trial-1.xml") + _junit(tmp_path / "trial-2.xml") + missing = tmp_path / "trial-3.xml" + + with pytest.raises(QualificationEvidenceError, match="missing trial evidence"): + aggregate([tmp_path / "trial-1.xml", tmp_path / "trial-2.xml", missing], os_name="macos") + + +def test_fewer_than_required_trials_fails_closed(tmp_path: Path) -> None: + two_trials = _three_clean(tmp_path)[:2] + with pytest.raises(QualificationEvidenceError, match="expected 3 trials, got 2"): + aggregate(two_trials, os_name="linux") + + +def test_duplicate_trial_files_fail_closed(tmp_path: Path) -> None: + only = _junit(tmp_path / "trial-1.xml") + with pytest.raises(QualificationEvidenceError, match="duplicate trial files"): + aggregate([only, only], os_name="linux") + + +def test_failed_trial_is_a_silent_incorrect_success_and_an_over_halt( + tmp_path: Path, +) -> None: + files = _three_clean(tmp_path) + _junit(files[0], tests=6, failures=2) + + summary = aggregate(files, os_name="windows") + + assert summary["passed"] is False + assert summary["over_halt"] == 2 + assert summary["silent_incorrect_success"] == 1 + assert any("unjustified halt" in violation for violation in summary["violations"]) + + +def test_skipped_trial_violates_even_without_failures(tmp_path: Path) -> None: + files = _three_clean(tmp_path) + _junit(files[1], tests=6, skipped=1) + + summary = aggregate(files, os_name="linux") + + assert summary["passed"] is False + assert summary["totals"]["skipped"] == 1 + assert summary["silent_incorrect_success"] == 1 + + +def test_unparsable_xml_fails_closed(tmp_path: Path) -> None: + broken = tmp_path / "trial-1.xml" + broken.write_text("", encoding="utf-8") + clean = [_junit(tmp_path / f"trial-{trial}.xml") for trial in (2, 3)] + + with pytest.raises(QualificationEvidenceError, match="unreadable JUnit XML"): + aggregate([broken, *clean], os_name="linux") + + +def test_suite_without_counts_fails_closed(tmp_path: Path) -> None: + bare = tmp_path / "trial-1.xml" + bare.write_text( + "", + encoding="utf-8", + ) + clean = [_junit(tmp_path / f"trial-{trial}.xml") for trial in (2, 3)] + + with pytest.raises(QualificationEvidenceError, match="missing 'tests'"): + aggregate([bare, *clean], os_name="linux") + + +def test_trial_running_zero_tests_fails_closed(tmp_path: Path) -> None: + empty = _junit(tmp_path / "trial-1.xml", tests=0) + clean = [_junit(tmp_path / f"trial-{trial}.xml") for trial in (2, 3)] + + with pytest.raises(QualificationEvidenceError, match="ran zero tests"): + aggregate([empty, *clean], os_name="linux") + + +def test_summary_records_the_candidate_sha(tmp_path: Path) -> None: + sha = "a" * 40 + + summary = aggregate(_three_clean(tmp_path), os_name="linux", candidate_sha=sha) + + assert summary["candidate_sha"] == sha