From 9f438e6cf62d925a09de0e3436f098f035579337 Mon Sep 17 00:00:00 2001 From: Karthik Nadig Date: Tue, 11 Aug 2026 07:53:35 -0700 Subject: [PATCH 1/4] test: tighten post-fix macOS performance budgets (Fixes #507) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QUALITY_SNAPSHOTS.md | 10 ++++++---- scripts/quality_snapshot.py | 6 +++--- scripts/tests/test_quality_snapshot.py | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/docs/QUALITY_SNAPSHOTS.md b/docs/QUALITY_SNAPSHOTS.md index 79461ffa..dac9c76d 100644 --- a/docs/QUALITY_SNAPSHOTS.md +++ b/docs/QUALITY_SNAPSHOTS.md @@ -15,14 +15,16 @@ A metric blocks when it exceeds both its absolute and relative budget: | Metric | Linux | Windows | macOS | | --- | ---: | ---: | ---: | | Server startup P50 | 5 ms / 100% | 10 ms / 50% | 100 ms / 50% | -| Server startup P95 | 50 ms / 200% | 50 ms / 100% | 10,000 ms / 100% | +| Server startup P95 | 50 ms / 200% | 50 ms / 100% | 750 ms / 100% | | Full refresh P50 | 25 ms / 30% | 50 ms / 30% | 100 ms / 50% | -| Full refresh P95 | 1,000 ms / 100% | 5,000 ms / 100% | 5,000 ms / 25% | +| Full refresh P95 | 1,000 ms / 100% | 5,000 ms / 100% | 1,000 ms / 50% | | Time to first environment P50 | 20 ms / 100% | 25 ms / 50% | 150 ms / 50% | -| Time to first environment P95 | 250 ms / 100% | 500 ms / 100% | 10,000 ms / 100% | +| Time to first environment P95 | 250 ms / 100% | 500 ms / 100% | 750 ms / 100% | Each cell is `absolute / relative`. The budgets reflect observed GitHub-hosted runner variance from 11 consecutive main-branch baselines. Tighten them when a noisy path is fixed rather than normalizing a known regression into the baseline. +The macOS P95 budgets were recalibrated after #504 using three unchanged-content pull-request runs and the exact merged baseline. Their absolute headroom is four to six times the observed post-fix run-to-run range. + The dual budget avoids failing on tiny percentage changes while still blocking material latency regressions. Tail metrics remain mandatory; a healthy median does not excuse a degraded P95. ## Coverage gate @@ -50,4 +52,4 @@ Phase and locator telemetry is collected in separate, untimed refreshes so diagn ## Known investigations -The macOS cold-refresh tail is tracked by issue #504. Phase and locator distributions plus privacy-safe interpreter timeout counts verify that the tail does not recur. +The macOS cold-refresh tail fixed by issue #504 remains guarded by phase and locator distributions plus privacy-safe interpreter timeout counts. diff --git a/scripts/quality_snapshot.py b/scripts/quality_snapshot.py index 250eaf49..6e77a5c3 100644 --- a/scripts/quality_snapshot.py +++ b/scripts/quality_snapshot.py @@ -80,11 +80,11 @@ def regressed(self) -> bool: ), 'macos': ( RegressionBudget(100, 50), - RegressionBudget(10_000, 100), + RegressionBudget(750, 100), RegressionBudget(100, 50), - RegressionBudget(5_000, 25), + RegressionBudget(1_000, 50), RegressionBudget(150, 50), - RegressionBudget(10_000, 100), + RegressionBudget(750, 100), ), } COVERAGE_BUDGET_PERCENTAGE_POINTS = 0.01 diff --git a/scripts/tests/test_quality_snapshot.py b/scripts/tests/test_quality_snapshot.py index 6a5cb6a5..97ad8562 100644 --- a/scripts/tests/test_quality_snapshot.py +++ b/scripts/tests/test_quality_snapshot.py @@ -64,6 +64,27 @@ def test_p95_regression_fails_even_when_p50_is_unchanged(self): _, failures = compare_performance(current, performance_snapshot(refresh_p95=500), 'Windows') self.assertTrue(any('Full refresh P95' in failure for failure in failures)) + def test_post_fix_macos_tail_variance_passes(self): + baseline = performance_snapshot(startup_p95=621, refresh_p95=1_343, first_p95=649) + current = performance_snapshot(startup_p95=691, refresh_p95=1_435, first_p95=745) + + _, failures = compare_performance(current, baseline, 'macOS') + + self.assertEqual(failures, []) + + def test_tightened_macos_tail_budgets_reject_multi_second_regressions(self): + baseline = performance_snapshot(startup_p95=621, refresh_p95=1_343, first_p95=649) + current = performance_snapshot(startup_p95=1_500, refresh_p95=2_500, first_p95=1_500) + + _, failures = compare_performance(current, baseline, 'macOS') + + for label in ( + 'Server startup P95', + 'Full refresh P95', + 'Time to first environment P95', + ): + self.assertTrue(any(label in failure for failure in failures)) + def test_noise_inside_absolute_budget_passes(self): current = performance_snapshot(refresh_p50=140) _, failures = compare_performance(current, performance_snapshot(refresh_p50=100), 'Windows') From 0ef48b5c10b915c1ea9304896bc1113acdceef76 Mon Sep 17 00:00:00 2001 From: Karthik Nadig Date: Tue, 11 Aug 2026 07:59:00 -0700 Subject: [PATCH 2/4] docs: clarify macOS budget calibration source (PR #508) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QUALITY_SNAPSHOTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/QUALITY_SNAPSHOTS.md b/docs/QUALITY_SNAPSHOTS.md index dac9c76d..7dea7109 100644 --- a/docs/QUALITY_SNAPSHOTS.md +++ b/docs/QUALITY_SNAPSHOTS.md @@ -23,7 +23,7 @@ A metric blocks when it exceeds both its absolute and relative budget: Each cell is `absolute / relative`. The budgets reflect observed GitHub-hosted runner variance from 11 consecutive main-branch baselines. Tighten them when a noisy path is fixed rather than normalizing a known regression into the baseline. -The macOS P95 budgets were recalibrated after #504 using three unchanged-content pull-request runs and the exact merged baseline. Their absolute headroom is four to six times the observed post-fix run-to-run range. +The macOS P95 budgets were recalibrated after PR #506 (tracking issue #504) using three unchanged-content pull-request runs and the exact merged baseline at `f0c62d9`. Their absolute headroom is four to six times the observed post-fix run-to-run range. The dual budget avoids failing on tiny percentage changes while still blocking material latency regressions. Tail metrics remain mandatory; a healthy median does not excuse a degraded P95. @@ -52,4 +52,4 @@ Phase and locator telemetry is collected in separate, untimed refreshes so diagn ## Known investigations -The macOS cold-refresh tail fixed by issue #504 remains guarded by phase and locator distributions plus privacy-safe interpreter timeout counts. +The macOS cold-refresh tail fixed by PR #506 (tracked in issue #504) remains guarded by phase and locator distributions plus privacy-safe interpreter timeout counts. From e49ed7673de52b62f8a529bb52c576143bcb5c1a Mon Sep 17 00:00:00 2001 From: Karthik Nadig Date: Tue, 11 Aug 2026 08:05:32 -0700 Subject: [PATCH 3/4] docs: distinguish macOS P95 calibration data (PR #508) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QUALITY_SNAPSHOTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/QUALITY_SNAPSHOTS.md b/docs/QUALITY_SNAPSHOTS.md index 7dea7109..2b1c0e5e 100644 --- a/docs/QUALITY_SNAPSHOTS.md +++ b/docs/QUALITY_SNAPSHOTS.md @@ -21,7 +21,7 @@ A metric blocks when it exceeds both its absolute and relative budget: | Time to first environment P50 | 20 ms / 100% | 25 ms / 50% | 150 ms / 50% | | Time to first environment P95 | 250 ms / 100% | 500 ms / 100% | 750 ms / 100% | -Each cell is `absolute / relative`. The budgets reflect observed GitHub-hosted runner variance from 11 consecutive main-branch baselines. Tighten them when a noisy path is fixed rather than normalizing a known regression into the baseline. +Each cell is `absolute / relative`. The Linux and Windows budgets plus the macOS P50 budgets reflect observed GitHub-hosted runner variance from 11 consecutive main-branch baselines. Tighten them when a noisy path is fixed rather than normalizing a known regression into the baseline. The macOS P95 budgets were recalibrated after PR #506 (tracking issue #504) using three unchanged-content pull-request runs and the exact merged baseline at `f0c62d9`. Their absolute headroom is four to six times the observed post-fix run-to-run range. From 412b102c99f3ec9ea590086d1c811a9da5d833a4 Mon Sep 17 00:00:00 2001 From: Karthik Nadig Date: Tue, 11 Aug 2026 08:09:17 -0700 Subject: [PATCH 4/4] docs: identify macOS budget tracking issue (PR #508) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/QUALITY_SNAPSHOTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/QUALITY_SNAPSHOTS.md b/docs/QUALITY_SNAPSHOTS.md index 2b1c0e5e..0d20456e 100644 --- a/docs/QUALITY_SNAPSHOTS.md +++ b/docs/QUALITY_SNAPSHOTS.md @@ -23,7 +23,7 @@ A metric blocks when it exceeds both its absolute and relative budget: Each cell is `absolute / relative`. The Linux and Windows budgets plus the macOS P50 budgets reflect observed GitHub-hosted runner variance from 11 consecutive main-branch baselines. Tighten them when a noisy path is fixed rather than normalizing a known regression into the baseline. -The macOS P95 budgets were recalibrated after PR #506 (tracking issue #504) using three unchanged-content pull-request runs and the exact merged baseline at `f0c62d9`. Their absolute headroom is four to six times the observed post-fix run-to-run range. +The macOS P95 budget recalibration is tracked by issue #507 and follows PR #506's fix for issue #504. It uses three unchanged-content pull-request runs and the exact merged baseline at `f0c62d9`; the resulting absolute headroom is four to six times the observed post-fix run-to-run range. The dual budget avoids failing on tiny percentage changes while still blocking material latency regressions. Tail metrics remain mandatory; a healthy median does not excuse a degraded P95.