diff --git a/docs/QUALITY_SNAPSHOTS.md b/docs/QUALITY_SNAPSHOTS.md index 79461ffa..0d20456e 100644 --- a/docs/QUALITY_SNAPSHOTS.md +++ b/docs/QUALITY_SNAPSHOTS.md @@ -15,13 +15,15 @@ A metric blocks when it exceeds both its absolute and relative budget: | Metric | Linux | Windows | macOS | | --- | ---: | ---: | ---: | | Server startup P50 | 5 ms / 100% | 10 ms / 50% | 100 ms / 50% | -| Server startup P95 | 50 ms / 200% | 50 ms / 100% | 10,000 ms / 100% | +| Server startup P95 | 50 ms / 200% | 50 ms / 100% | 750 ms / 100% | | Full refresh P50 | 25 ms / 30% | 50 ms / 30% | 100 ms / 50% | -| Full refresh P95 | 1,000 ms / 100% | 5,000 ms / 100% | 5,000 ms / 25% | +| Full refresh P95 | 1,000 ms / 100% | 5,000 ms / 100% | 1,000 ms / 50% | | Time to first environment P50 | 20 ms / 100% | 25 ms / 50% | 150 ms / 50% | -| Time to first environment P95 | 250 ms / 100% | 500 ms / 100% | 10,000 ms / 100% | +| Time to first environment P95 | 250 ms / 100% | 500 ms / 100% | 750 ms / 100% | -Each cell is `absolute / relative`. The budgets reflect observed GitHub-hosted runner variance from 11 consecutive main-branch baselines. Tighten them when a noisy path is fixed rather than normalizing a known regression into the baseline. +Each cell is `absolute / relative`. The Linux and Windows budgets plus the macOS P50 budgets reflect observed GitHub-hosted runner variance from 11 consecutive main-branch baselines. Tighten them when a noisy path is fixed rather than normalizing a known regression into the baseline. + +The macOS P95 budget recalibration is tracked by issue #507 and follows PR #506's fix for issue #504. It uses three unchanged-content pull-request runs and the exact merged baseline at `f0c62d9`; the resulting absolute headroom is four to six times the observed post-fix run-to-run range. The dual budget avoids failing on tiny percentage changes while still blocking material latency regressions. Tail metrics remain mandatory; a healthy median does not excuse a degraded P95. @@ -50,4 +52,4 @@ Phase and locator telemetry is collected in separate, untimed refreshes so diagn ## Known investigations -The macOS cold-refresh tail is tracked by issue #504. Phase and locator distributions plus privacy-safe interpreter timeout counts verify that the tail does not recur. +The macOS cold-refresh tail fixed by PR #506 (tracked in issue #504) remains guarded by phase and locator distributions plus privacy-safe interpreter timeout counts. diff --git a/scripts/quality_snapshot.py b/scripts/quality_snapshot.py index 250eaf49..6e77a5c3 100644 --- a/scripts/quality_snapshot.py +++ b/scripts/quality_snapshot.py @@ -80,11 +80,11 @@ def regressed(self) -> bool: ), 'macos': ( RegressionBudget(100, 50), - RegressionBudget(10_000, 100), + RegressionBudget(750, 100), RegressionBudget(100, 50), - RegressionBudget(5_000, 25), + RegressionBudget(1_000, 50), RegressionBudget(150, 50), - RegressionBudget(10_000, 100), + RegressionBudget(750, 100), ), } COVERAGE_BUDGET_PERCENTAGE_POINTS = 0.01 diff --git a/scripts/tests/test_quality_snapshot.py b/scripts/tests/test_quality_snapshot.py index 6a5cb6a5..97ad8562 100644 --- a/scripts/tests/test_quality_snapshot.py +++ b/scripts/tests/test_quality_snapshot.py @@ -64,6 +64,27 @@ def test_p95_regression_fails_even_when_p50_is_unchanged(self): _, failures = compare_performance(current, performance_snapshot(refresh_p95=500), 'Windows') self.assertTrue(any('Full refresh P95' in failure for failure in failures)) + def test_post_fix_macos_tail_variance_passes(self): + baseline = performance_snapshot(startup_p95=621, refresh_p95=1_343, first_p95=649) + current = performance_snapshot(startup_p95=691, refresh_p95=1_435, first_p95=745) + + _, failures = compare_performance(current, baseline, 'macOS') + + self.assertEqual(failures, []) + + def test_tightened_macos_tail_budgets_reject_multi_second_regressions(self): + baseline = performance_snapshot(startup_p95=621, refresh_p95=1_343, first_p95=649) + current = performance_snapshot(startup_p95=1_500, refresh_p95=2_500, first_p95=1_500) + + _, failures = compare_performance(current, baseline, 'macOS') + + for label in ( + 'Server startup P95', + 'Full refresh P95', + 'Time to first environment P95', + ): + self.assertTrue(any(label in failure for failure in failures)) + def test_noise_inside_absolute_budget_passes(self): current = performance_snapshot(refresh_p50=140) _, failures = compare_performance(current, performance_snapshot(refresh_p50=100), 'Windows')