refactor(Scripts): extract Xcode report policy

AGENTS.md

@@ -54,7 +54,8 @@ easy to corrupt by hand. `./simulator` owns a per-checkout device (see the
 [`running-tests`](.agents/skills/running-tests/SKILL.md) skill).
 
 Retained Python and Ruby implementations are importable and directly tested
-under `Tools/Tests`; shell around them is bootstrap only. In particular,
+under `Tools/Tests`; shell around them is limited to public argument handling,
+bootstrap, and process orchestration. In particular,
 `tla-check` owns discovery and the pinned TLC download while
 `Tools/tla_check.py` owns manifest validation, TLC argv, and result policy.
 

Tools/ADVERSARIAL_TEST_PLAN.md

@@ -240,3 +240,15 @@ The stack is ready only when:
 - Apple Bash 3.2, system Python 3.9, and pinned Ruby all pass.
 - Commands work from local and CI-style environments.
 - Every rebased PR has fresh required checks.
+
+## Mutation evidence
+
+These mutations were applied locally, the named test failed, and the mutation
+was removed before commit:
+
+| Removed or weakened behavior | Test that killed the mutation |
+|---|---|
+| Treat a successful Xcode run with zero tests as success | `test_test_rejects_a_successful_xcode_run_that_matched_zero_tests` |
+| Replace the `xcodebuild` pipeline status with zero | `test_test_preserves_xcode_failure_through_the_progress_pipeline` |
+| Ignore a failed `tee` or progress stage after successful Xcode | `test_test_surfaces_a_progress_process_failure_after_xcode_succeeds` |
+| Trust a schema-shifted xcresult root instead of validating it | `test_tolerates_unknown_nodes_parameterized_names_and_missing_fields` |

Tools/README.md

@@ -2,15 +2,19 @@
 
 This directory contains importable implementations and direct tests for the
 repository's retained Python and Ruby developer tooling. Public commands stay
-at their established paths in the repository root; shell launchers own process
-orchestration and bootstrap, while structured parsing and policy live here.
+at their established paths in the repository root; shell launchers own public
+argument handling, process orchestration, and bootstrap, while structured
+parsing and reporting policy live here.
 
 The existing CircleCI artifact, JUnit, and snapshot-shard helpers remain Python
 because they are already integrated and directly tested. `tla_check.py`
 similarly owns TLA+ manifest validation, isolated translation, TLC argv, and
-result reporting without requiring Java in its tests. Filesystem-heavy Ruby
-generators are require-safe so their behavior can be exercised against
-temporary repositories.
+result reporting without requiring Java in its tests. The Xcode-facing root
+commands keep process and simulator orchestration in shell; `xcode_results.py`
+shares xcresult traversal, `snapshot_reports.py` shares capture reports, and
+the command-specific Python modules retain each command's distinct policy.
+Filesystem-heavy Ruby generators are require-safe so their behavior can be
+exercised against temporary repositories.
 
 ## Testing
 

Tools/Tests/test_flaky_results.py

@@ -0,0 +1,157 @@
+from datetime import datetime, timezone
+import io
+import json
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from flaky_results import (
+    analyze_suite_documents,
+    console_report,
+    flaky_rows,
+    main,
+    markdown_report,
+    tight_counts,
+)
+
+
+class FlakyResultsTests(unittest.TestCase):
+    def test_analyzes_suite_runs_with_stable_only_testing_identifiers(self):
+        documents = [
+            self.document("Passed"),
+            self.document("Failed"),
+        ]
+
+        stats = analyze_suite_documents(documents)
+
+        self.assertEqual(
+            stats["CoreTests/ValueTests/works()"],
+            {
+                "bundle": "CoreTests",
+                "name": "works()",
+                "fails": 1,
+                "seen": 2,
+            },
+        )
+
+    def test_tight_counts_prefers_iterations_then_falls_back_to_summary(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "tight_1.tests.json").write_text(
+                json.dumps(
+                    {
+                        "testNodes": [
+                            self.case("Passed"),
+                            self.case("Failed"),
+                            self.case("Passed"),
+                        ]
+                    }
+                )
+            )
+            (root / "tight_1.summary.json").write_text(
+                json.dumps({"failedTests": 9, "passedTests": 1})
+            )
+            (root / "tight_2.tests.json").write_text(
+                json.dumps({"testNodes": [self.case("Failed")]})
+            )
+            (root / "tight_2.summary.json").write_text(
+                json.dumps({"failedTests": 2, "passedTests": 3})
+            )
+
+            self.assertEqual(tight_counts(root, "1"), (1, 3))
+            self.assertEqual(tight_counts(root, "2"), (2, 5))
+
+    def test_reports_only_tests_with_both_passes_and_failures(self):
+        suite = {
+            "flaky": {"bundle": "CoreTests", "name": "flaky()", "fails": 1, "seen": 2},
+            "broken": {"bundle": "CoreTests", "name": "broken()", "fails": 2, "seen": 2},
+            "passing": {"bundle": "CoreTests", "name": "passing()", "fails": 0, "seen": 2},
+        }
+
+        rows = flaky_rows(
+            suite,
+            {"flaky": (1, 4), "broken": (4, 4), "passing": (0, 4)},
+            suite_runs=2,
+        )
+
+        self.assertEqual([row["id"] for row in rows], ["flaky"])
+        self.assertEqual(rows[0]["flake_rate"], 0.375)
+        self.assertIn("37.5%", console_report(rows, top=1))
+
+        markdown = markdown_report(
+            rows,
+            top=1,
+            suite_runs=2,
+            iterations=4,
+            relaunch="YES",
+            device="iPhone 17",
+            os_version="27.0",
+            generated_at=datetime(2026, 8, 17, 12, 0, tzinfo=timezone.utc),
+        )
+        self.assertIn("2026-08-17T12:00:00Z", markdown)
+        self.assertIn("| `flaky` | CoreTests | 1/2 | 1/4 | 38% |", markdown)
+
+    def test_analyze_suite_keeps_valid_partial_results_and_warns_about_malformed_files(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            suite = root / "suite"
+            suite.mkdir()
+            (suite / "run_1.json").write_text(json.dumps(self.document("Failed")))
+            (suite / "run_2.json").write_text("{")
+            suspects = root / "suspects.txt"
+            counts = root / "counts.json"
+            stderr = io.StringIO()
+
+            with patch("sys.stderr", stderr):
+                status = main(
+                    [
+                        "analyze-suite",
+                        "--suite-dir",
+                        str(suite),
+                        "--suspects",
+                        str(suspects),
+                        "--counts",
+                        str(counts),
+                    ]
+                )
+
+            self.assertEqual(status, 0)
+            self.assertIn("couldn't read", stderr.getvalue())
+            self.assertEqual(suspects.read_text(), "CoreTests/ValueTests/works()\n")
+            self.assertEqual(
+                json.loads(counts.read_text())["CoreTests/ValueTests/works()"]["seen"],
+                1,
+            )
+
+    def document(self, result):
+        return {
+            "testNodes": [
+                {
+                    "nodeType": "Unit test bundle",
+                    "name": "CoreTests",
+                    "children": [
+                        {
+                            "nodeType": "Test Suite",
+                            "name": "ValueTests",
+                            "children": [self.case(result)],
+                        }
+                    ],
+                }
+            ]
+        }
+
+    def case(self, result):
+        return {
+            "nodeType": "Test Case",
+            "name": "works()",
+            "nodeIdentifier": "ValueTests/works()",
+            "result": result,
+        }
+
+
+if __name__ == "__main__":
+    unittest.main()

Tools/Tests/test_profile_results.py

@@ -0,0 +1,60 @@
+from pathlib import Path
+import sys
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from profile_results import build_report, test_report
+
+
+class ProfileResultsTests(unittest.TestCase):
+    def test_build_report_orders_phases_and_typecheck_sites(self):
+        log = """SwiftCompile (2 tasks) | 4.0 seconds
+Link (1 task) | 1.0 seconds
+/tmp/repo/Where/Feature/File.swift:2:3: warning: expression took 250ms to type-check
+/tmp/repo/Shared/Core/File.swift:4:5: warning: function took 125ms to type-check
+"""
+
+        report = build_report(log, 100)
+
+        self.assertLess(report.index("SwiftCompile"), report.index("Link"))
+        self.assertIn("80%  SwiftCompile", report)
+        self.assertLess(report.index("250ms"), report.index("125ms"))
+        self.assertIn("Where/Feature/File.swift:2:3", report)
+
+    def test_test_report_merges_documents_and_excludes_skipped_cases(self):
+        documents = [
+            self.document("CoreTests", "fast()", "Passed", 0.05),
+            self.document("UITests", "slow()", "Failed", 0.5),
+            self.document("UITests", "skipped()", "Skipped", 10),
+        ]
+
+        report = test_report(documents, top=2, threshold=0.1)
+
+        self.assertIn("2 tests, summed self-time 0.55s", report)
+        self.assertLess(report.index("slow()"), report.index("fast()"))
+        self.assertNotIn("skipped()", report)
+        self.assertIn("UITests (1 tests)", report)
+        self.assertIn("1 test(s) at/over the 0.1s threshold", report)
+
+    def document(self, bundle, name, result, duration):
+        return {
+            "testNodes": [
+                {
+                    "nodeType": "Unit test bundle",
+                    "name": bundle,
+                    "children": [
+                        {
+                            "nodeType": "Test Case",
+                            "name": name,
+                            "result": result,
+                            "durationInSeconds": duration,
+                        }
+                    ],
+                }
+            ]
+        }
+
+
+if __name__ == "__main__":
+    unittest.main()

Tools/Tests/test_snapshot_reports.py

@@ -0,0 +1,79 @@
+import json
+from pathlib import Path
+import sys
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from snapshot_reports import difference_report, timing_report
+
+
+class SnapshotReportsTests(unittest.TestCase):
+    def test_timing_report_shares_common_summary_and_optional_detail(self):
+        rows = [
+            {
+                "id": "Suite/first",
+                "total": 2.0,
+                "phases": {"render": 1.5, "intrinsicMeasure": 0.5},
+                "settlePasses": 2,
+                "sizing": "fullContent",
+                "measurementReadiness": "ready",
+                "captureSettle": "stable",
+            },
+            {
+                "id": "Suite/second",
+                "total": 1.0,
+                "phases": {"render": 1.0},
+                "settlePasses": 1,
+                "sizing": "fixed",
+                "measurementReadiness": "ready",
+                "captureSettle": "stable",
+            },
+        ]
+
+        concise = timing_report(rows, detailed=False, empty_message="none")
+        detailed = timing_report(rows, detailed=True, empty_message="none")
+
+        self.assertIn("2 captures, 3.0s total, 1.500s per image", concise)
+        self.assertIn("render", concise)
+        self.assertIn("settle passes: min 1, max 2, mean 1.5", concise)
+        self.assertNotIn("measurement readiness", concise)
+        self.assertIn("measurement readiness", detailed)
+        self.assertIn("0.50s  ready (2 captures)", detailed)
+
+    def test_difference_report_orders_real_differences_and_hides_recording_misses(self):
+        rows = [
+            {
+                "outcome": "referenceMissing",
+                "reference": "missing.png",
+            },
+            {
+                "outcome": "differs",
+                "maxChannelDelta": 40,
+                "differingPixels": 2,
+                "differingFraction": 0.25,
+                "region": [1, 2, 3, 4],
+                "reference": "/tmp/__Snapshots__/Suite/image.png",
+            },
+        ]
+
+        report = difference_report(rows, is_recording=True)
+
+        self.assertIn("1 capture(s)", report)
+        self.assertIn("40", report)
+        self.assertIn("Suite/image.png", report)
+        self.assertNotIn("referenceMissing", report)
+
+    def test_empty_reports_preserve_public_messages(self):
+        self.assertEqual(
+            timing_report([], detailed=False, empty_message="no timing lines found"),
+            "  (no timing lines found)",
+        )
+        self.assertEqual(
+            difference_report([], is_recording=False),
+            "  Every capture matched its reference byte for byte.",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()