diff --git a/docs/adr/0011-enforce-performance-budgets-with-playwright.md b/docs/adr/0011-enforce-performance-budgets-with-playwright.md index 528391f..eaae679 100644 --- a/docs/adr/0011-enforce-performance-budgets-with-playwright.md +++ b/docs/adr/0011-enforce-performance-budgets-with-playwright.md @@ -1,6 +1,6 @@ # ADR-0011: Enforce performance budgets with Playwright -- Status: Accepted +- Status: Superseded by ADR-0012 - Date: 2026-08-14 In the context of guarding the static Astro site's performance as published pages increase, diff --git a/docs/adr/0012-stabilize-performance-measurements.md b/docs/adr/0012-stabilize-performance-measurements.md new file mode 100644 index 0000000..61c13d7 --- /dev/null +++ b/docs/adr/0012-stabilize-performance-measurements.md @@ -0,0 +1,15 @@ +# ADR-0012: Stabilize performance measurements + +- Status: Accepted +- Date: 2026-08-15 +- Supersedes: ADR-0011 + +In the context of enforcing browser performance budgets on variable GitHub-hosted runners, +facing a paint-time outlier consistent with runtime network variability while resource and +execution metrics remained stable, we decided for self-hosted rendering assets and adaptive +Playwright sampling against fixed three-run sampling, generic test retries, relaxed budgets, and +Lighthouse CI: measure three fresh browser contexts, add two confirmation measurements only when +the initial median exceeds a timing or layout budget, and evaluate the median of all samples. We +retain every sample, mark recovered results as unstable, and report ranges and sample counts, +accepting two additional measurements for suspicious pages and larger report artifacts in exchange +for fewer false positives without hiding the original failure evidence. diff --git a/package.json b/package.json index 074baae..ae0fb26 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "@astrojs/rss": "^4.0.19", "@astrojs/sitemap": "^3.7.2", "@fontsource-variable/inter": "^5.2.8", + "@fontsource-variable/noto-sans-jp": "^5.2.8", "astro": "^7.1.0", "mermaid": "^11.16.1", "sharp": "^0.35.3", diff --git a/playwright_performance.config.ts b/playwright_performance.config.ts index cc70e12..23ef23a 100644 --- a/playwright_performance.config.ts +++ b/playwright_performance.config.ts @@ -6,7 +6,7 @@ export default defineConfig({ fullyParallel: false, workers: 1, retries: 0, - timeout: 120_000, + timeout: 180_000, outputDir: "test-results/performance", reporter: [["list"], ["./tests/performance/performance_reporter.ts"]], webServer: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5345d93..5e219b0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -35,6 +35,9 @@ importers: '@fontsource-variable/inter': specifier: ^5.2.8 version: 5.2.8 + '@fontsource-variable/noto-sans-jp': + specifier: ^5.2.8 + version: 5.3.0 astro: specifier: ^7.1.0 version: 7.1.6(@astrojs/markdown-remark@7.2.2)(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.2)(@types/node@25.5.2)(jiti@2.6.1)(rollup@4.60.1)(yaml@2.8.3) @@ -466,6 +469,9 @@ packages: '@fontsource-variable/inter@5.2.8': resolution: {integrity: sha512-kOfP2D+ykbcX/P3IFnokOhVRNoTozo5/JxhAIVYLpea/UBmCQ/YWPBfWIDuBImXX/15KH+eKh4xpEUyS2sQQGQ==} + '@fontsource-variable/noto-sans-jp@5.3.0': + resolution: {integrity: sha512-XVL9GfGBRjj523v9w3Bctu4mNq7E5rUa+AMZm8DK1EM3LAGlKC3DZMvuqkLn4DB4TwwoAjcsJUCiRpUTDzzOkw==} + '@iconify/types@2.0.0': resolution: {integrity: sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==} @@ -3681,6 +3687,8 @@ snapshots: '@fontsource-variable/inter@5.2.8': {} + '@fontsource-variable/noto-sans-jp@5.3.0': {} + '@iconify/types@2.0.0': {} '@iconify/utils@3.1.4': diff --git a/public/_headers b/public/_headers index 7500084..4c33e40 100644 --- a/public/_headers +++ b/public/_headers @@ -1,5 +1,5 @@ /* - Content-Security-Policy: default-src 'self'; base-uri 'self'; connect-src 'self'; font-src 'self' https://fonts.gstatic.com; form-action 'self'; frame-ancestors 'none'; img-src 'self' data:; object-src 'none'; script-src 'self' 'unsafe-inline' https://static.cloudflareinsights.com; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; upgrade-insecure-requests + Content-Security-Policy: default-src 'self'; base-uri 'self'; connect-src 'self'; font-src 'self'; form-action 'self'; frame-ancestors 'none'; img-src 'self' data:; object-src 'none'; script-src 'self' 'unsafe-inline' https://static.cloudflareinsights.com; style-src 'self' 'unsafe-inline'; upgrade-insecure-requests Permissions-Policy: camera=(), geolocation=(), microphone=(), payment=(), usb=() Referrer-Policy: strict-origin-when-cross-origin Strict-Transport-Security: max-age=31536000 diff --git a/src/components/seo_head.astro b/src/components/seo_head.astro index 1ac6c6e..c7420fc 100644 --- a/src/components/seo_head.astro +++ b/src/components/seo_head.astro @@ -65,13 +65,6 @@ const socialPreviewUrl = `${origin}${socialPreview.path}`; href={`${BASE_PATH}/rss.xml`} /> - - - - diff --git a/src/layouts/base_layout.astro b/src/layouts/base_layout.astro index 3fdea25..f0cf026 100644 --- a/src/layouts/base_layout.astro +++ b/src/layouts/base_layout.astro @@ -1,5 +1,6 @@ --- import "@fontsource-variable/inter/wght.css"; +import "@fontsource-variable/noto-sans-jp/wght.css"; import "@styles/global.css"; import type { Lang } from "@i18n/ui"; import { useTranslations } from "@i18n/utils"; diff --git a/src/styles/global.css b/src/styles/global.css index 9c32942..f594402 100644 --- a/src/styles/global.css +++ b/src/styles/global.css @@ -6,7 +6,7 @@ --color-text-secondary: #a1a1aa; --color-accent: #4af2c8; - --font-sans: "Inter Variable", "Noto Sans JP", system-ui, sans-serif; + --font-sans: "Inter Variable", "Noto Sans JP Variable", system-ui, sans-serif; --font-mono: ui-monospace, "SFMono-Regular", "Menlo", monospace; --width-content: 56rem; diff --git a/tests/e2e/site.spec.ts b/tests/e2e/site.spec.ts index c6f0798..35d127c 100644 --- a/tests/e2e/site.spec.ts +++ b/tests/e2e/site.spec.ts @@ -113,6 +113,25 @@ test("publishes generated social preview metadata on an article page", async ({ ); }); +test("loads critical article resources only from the site origin", async ({ page }) => { + const externalOrigins = new Set(); + page.on("request", (request) => { + const url = new URL(request.url()); + const isCriticalResource = ["font", "script", "stylesheet"].includes(request.resourceType()); + if ( + isCriticalResource && + url.protocol.startsWith("http") && + url.origin !== "http://127.0.0.1:4321" + ) { + externalOrigins.add(url.origin); + } + }); + + await page.goto("/en/blog/modern-terminal-environment/", { waitUntil: "networkidle" }); + + expect([...externalOrigins]).toEqual([]); +}); + test("unknown paths return the custom not-found page", async ({ page }) => { const response = await page.goto("/missing-page/"); diff --git a/tests/performance/performance.spec.ts b/tests/performance/performance.spec.ts index e97bbb3..653786c 100644 --- a/tests/performance/performance.spec.ts +++ b/tests/performance/performance.spec.ts @@ -1,7 +1,7 @@ import { readdirSync } from "node:fs"; import { join, relative, sep } from "node:path"; -import { chromium, expect, test, type CDPSession, type Page } from "@playwright/test"; +import { expect, test, type Browser, type CDPSession, type Page } from "@playwright/test"; import { type PerformanceBudget as Budget, @@ -10,10 +10,10 @@ import { type ResourceKind, } from "./performance_report"; import { PERFORMANCE_RESULT_ATTACHMENT } from "./performance_reporter"; +import { collectAdaptiveMetrics } from "./performance_sampler"; const DIST_DIRECTORY = join(import.meta.dirname, "..", "..", "dist"); const BASE_URL = "http://127.0.0.1:4321"; -const RUNS_PER_PAGE = 3; const NETWORK_LATENCY_MS = 150; const NETWORK_THROUGHPUT_BYTES_PER_SECOND = (1_600 * 1_024) / 8; const CPU_SLOWDOWN_RATE = 4; @@ -58,39 +58,36 @@ const ARTICLE_BUDGET: Budget = { const routes = discoverRoutes(DIST_DIRECTORY); for (const route of routes) { - test(`${route} remains within its performance budget`, async () => { - const samples = await collectSamples(route); - const metrics = medianMetrics(samples); + test(`${route} remains within its performance budget`, async ({ browser }) => { const budget = budgetFor(route); + const { metrics, samples, wasExtended } = await collectAdaptiveMetrics( + () => measurePage(browser, route), + budget, + ); - const measurement = { budget, metrics, route } satisfies PerformanceMeasurement; + const measurement = { + budget, + metrics, + route, + samples, + wasExtended, + } satisfies PerformanceMeasurement; await test.info().attach(PERFORMANCE_RESULT_ATTACHMENT, { body: Buffer.from(JSON.stringify(measurement)), contentType: "application/json", }); - printMetrics(route, metrics); + printMetrics(route, metrics, samples.length); expectMetricsWithinBudget(metrics, budget); }); } -async function collectSamples(route: string): Promise { - const samples: Metrics[] = []; - - for (let run = 0; run < RUNS_PER_PAGE; run += 1) { - samples.push(await measurePage(route)); - } - - return samples; -} - -async function measurePage(route: string): Promise { - const browser = await chromium.launch(); +async function measurePage(browser: Browser, route: string): Promise { + const context = await browser.newContext({ + serviceWorkers: "block", + viewport: { width: 1_280, height: 720 }, + }); try { - const context = await browser.newContext({ - serviceWorkers: "block", - viewport: { width: 1_280, height: 720 }, - }); const page = await context.newPage(); const session = await context.newCDPSession(page); const resources = observeNetworkResources(session); @@ -109,7 +106,7 @@ async function measurePage(route: string): Promise { return await readMetrics(page, resources); } finally { - await browser.close(); + await context.close(); } } @@ -214,29 +211,6 @@ function resourceKind(resourceType: string | undefined): Exclude sample.firstContentfulPaint)), - largestContentfulPaint: median(samples.map((sample) => sample.largestContentfulPaint)), - cumulativeLayoutShift: median(samples.map((sample) => sample.cumulativeLayoutShift)), - blockingTime: median(samples.map((sample) => sample.blockingTime)), - resources: Object.fromEntries( - (["total", "script", "image", "font"] satisfies ResourceKind[]).map((kind) => [ - kind, - { - size: median(samples.map((sample) => sample.resources[kind].size)), - count: median(samples.map((sample) => sample.resources[kind].count)), - }, - ]), - ) as Metrics["resources"], - }; -} - -function median(values: number[]): number { - const sorted = values.toSorted((left, right) => left - right); - return sorted[Math.floor(sorted.length / 2)]; -} - function expectMetricsWithinBudget(metrics: Metrics, budget: Budget): void { expect(metrics.firstContentfulPaint, "first contentful paint (ms)").toBeGreaterThan(0); expect(metrics.firstContentfulPaint, "first contentful paint (ms)").toBeLessThanOrEqual( @@ -281,9 +255,10 @@ function listHtmlFiles(directory: string): string[] { }); } -function printMetrics(route: string, metrics: Metrics): void { +function printMetrics(route: string, metrics: Metrics, sampleCount: number): void { console.table({ route, + samples: sampleCount, fcpMs: Math.round(metrics.firstContentfulPaint), lcpMs: Math.round(metrics.largestContentfulPaint), cls: metrics.cumulativeLayoutShift.toFixed(3), diff --git a/tests/performance/performance_report.test.ts b/tests/performance/performance_report.test.ts index a3088f3..0dc8498 100644 --- a/tests/performance/performance_report.test.ts +++ b/tests/performance/performance_report.test.ts @@ -16,9 +16,9 @@ describe("performance report", () => { expect(summary).toBe(`## Performance budgets -| Status | Page | FCP | LCP | CLS | Blocking | Transfer | Requests | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| ✅ | \`/ja/\` | 1.23 s | 1.45 s | 0.012 | 88 ms | 512 KB | 20 | +| Status | Page | FCP | LCP | CLS | Blocking | Transfer | Requests | Samples | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| ✅ | \`/ja/\` | 1.23 s | 1.45 s | 0.012 | 88 ms | 512 KB | 20 | 1 | `); }); @@ -55,10 +55,34 @@ describe("performance report", () => { rmSync(directory, { recursive: true }); } }); + + test("marks a recovered measurement as unstable", () => { + const entry = { + ...reportEntry(), + samples: Array.from({ length: 5 }, () => reportEntry().metrics), + wasExtended: true, + }; + + expect(renderPerformanceSummary([entry])).toContain("| ⚠️ unstable | `/ja/` |"); + }); + + test("shows the sample count and timing range", () => { + const entry = reportEntry(); + entry.samples = [ + { ...entry.metrics, firstContentfulPaint: 1_000 }, + entry.metrics, + { ...entry.metrics, firstContentfulPaint: 1_500 }, + ]; + + const summary = renderPerformanceSummary([entry]); + + expect(summary).toContain("1.23 s (1.00–1.50)"); + expect(summary).toContain("| 3 |"); + }); }); function reportEntry(): PerformanceReportEntry { - return { + const entry = { route: "/ja/", status: "passed", metrics: { @@ -85,5 +109,7 @@ function reportEntry(): PerformanceReportEntry { font: { size: 700_000, count: 35 }, }, }, - }; + } satisfies Omit; + + return { ...entry, samples: [entry.metrics], wasExtended: false }; } diff --git a/tests/performance/performance_report.ts b/tests/performance/performance_report.ts index 1db08e1..0bec966 100644 --- a/tests/performance/performance_report.ts +++ b/tests/performance/performance_report.ts @@ -17,7 +17,9 @@ export type PerformanceReportEntry = { budget: PerformanceBudget; metrics: PerformanceMetrics; route: string; + samples: PerformanceMetrics[]; status: "failed" | "interrupted" | "passed" | "skipped" | "timedOut"; + wasExtended: boolean; }; export type PerformanceMeasurement = Omit; @@ -27,8 +29,8 @@ export function renderPerformanceSummary(entries: PerformanceReportEntry[]): str return [ "## Performance budgets", "", - "| Status | Page | FCP | LCP | CLS | Blocking | Transfer | Requests |", - "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |", + "| Status | Page | FCP | LCP | CLS | Blocking | Transfer | Requests | Samples |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", ...rows, "", ].join("\n"); @@ -52,21 +54,47 @@ export function writePerformanceReports( } function renderRow(entry: PerformanceReportEntry): string { - const { metrics } = entry; + const { metrics, samples } = entry; return [ - `| ${statusSymbol(entry.status)} | \`${entry.route}\``, - `${seconds(metrics.firstContentfulPaint)} s`, - `${seconds(metrics.largestContentfulPaint)} s`, - metrics.cumulativeLayoutShift.toFixed(3), - `${Math.round(metrics.blockingTime)} ms`, - `${Math.round(metrics.resources.total.size / 1_024)} KB`, - `${metrics.resources.total.count} |`, + `| ${statusSymbol(entry)} | \`${entry.route}\``, + secondsWithRange( + metrics.firstContentfulPaint, + samples.map(({ firstContentfulPaint }) => firstContentfulPaint), + ), + secondsWithRange( + metrics.largestContentfulPaint, + samples.map(({ largestContentfulPaint }) => largestContentfulPaint), + ), + valueWithRange( + metrics.cumulativeLayoutShift, + samples.map(({ cumulativeLayoutShift }) => cumulativeLayoutShift), + (value) => value.toFixed(3), + ), + unitWithRange( + metrics.blockingTime, + samples.map(({ blockingTime }) => blockingTime), + "ms", + Math.round, + ), + unitWithRange( + metrics.resources.total.size, + samples.map(({ resources }) => resources.total.size), + "KB", + (value) => Math.round(value / 1_024), + ), + valueWithRange( + metrics.resources.total.count, + samples.map(({ resources }) => resources.total.count), + String, + ), + `${samples.length} |`, ].join(" | "); } -function statusSymbol(status: PerformanceReportEntry["status"]): string { - if (status === "passed") return "✅"; - if (status === "skipped") return "➖"; +function statusSymbol(entry: PerformanceReportEntry): string { + if (entry.status === "passed" && entry.wasExtended) return "⚠️ unstable"; + if (entry.status === "passed") return "✅"; + if (entry.status === "skipped") return "➖"; return "❌"; } @@ -74,6 +102,35 @@ function seconds(milliseconds: number): string { return (milliseconds / 1_000).toFixed(2); } +function secondsWithRange(median: number, samples: number[]): string { + return unitWithRange(median, samples, "s", seconds); +} + +function unitWithRange( + median: number, + samples: number[], + unit: string, + format: (value: number) => string | number, +): string { + const minimum = Math.min(...samples); + const maximum = Math.max(...samples); + const formattedMedian = format(median); + if (minimum === maximum) return `${formattedMedian} ${unit}`; + return `${formattedMedian} ${unit} (${format(minimum)}–${format(maximum)})`; +} + +function valueWithRange( + median: number, + samples: number[], + format: (value: number) => string | number, +): string { + const minimum = Math.min(...samples); + const maximum = Math.max(...samples); + const formattedMedian = format(median); + if (minimum === maximum) return String(formattedMedian); + return `${formattedMedian} (${format(minimum)}–${format(maximum)})`; +} + function byRoute(left: PerformanceReportEntry, right: PerformanceReportEntry): number { return left.route.localeCompare(right.route); } diff --git a/tests/performance/performance_reporter.test.ts b/tests/performance/performance_reporter.test.ts index 125aad7..4d9303d 100644 --- a/tests/performance/performance_reporter.test.ts +++ b/tests/performance/performance_reporter.test.ts @@ -7,10 +7,13 @@ import { parsePerformanceResult } from "./performance_reporter"; describe("performance reporter", () => { test("combines an attached measurement with the final Playwright status", () => { + const measuredMetrics = metrics(1_200); const measurement: Omit = { budget: metrics(2_800), - metrics: metrics(1_200), + metrics: measuredMetrics, route: "/ja/", + samples: [measuredMetrics], + wasExtended: false, }; expect(parsePerformanceResult(Buffer.from(JSON.stringify(measurement)), "failed")).toEqual({ diff --git a/tests/performance/performance_sampler.test.ts b/tests/performance/performance_sampler.test.ts new file mode 100644 index 0000000..2aa8df1 --- /dev/null +++ b/tests/performance/performance_sampler.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, test } from "vitest"; + +import type { PerformanceBudget, PerformanceMetrics } from "./performance_report"; +import { collectAdaptiveMetrics } from "./performance_sampler"; + +describe("adaptive performance sampling", () => { + test("uses three samples when the initial median is within the variable budgets", async () => { + const samples = [metrics(1_000), metrics(1_200), metrics(1_100)]; + + const measurement = await collectAdaptiveMetrics(sequence(samples), budget()); + + expect(measurement).toMatchObject({ + metrics: { firstContentfulPaint: 1_100 }, + samples, + wasExtended: false, + }); + }); + + test("adds two samples when the initial median exceeds a variable budget", async () => { + const samples = [ + metrics(3_000), + metrics(6_000), + metrics(6_100), + metrics(1_000), + metrics(1_200), + ]; + + const measurement = await collectAdaptiveMetrics(sequence(samples), budget()); + + expect(measurement).toMatchObject({ + metrics: { firstContentfulPaint: 3_000 }, + samples, + wasExtended: true, + }); + }); + + test("does not add samples for a resource-only budget failure", async () => { + const samples = [metrics(1_000), metrics(1_100), metrics(1_200)]; + const resourceBudget = budget(); + resourceBudget.resources.total.size = 1; + + const measurement = await collectAdaptiveMetrics(sequence(samples), resourceBudget); + + expect(measurement).toMatchObject({ samples, wasExtended: false }); + }); +}); + +function sequence(samples: PerformanceMetrics[]): () => Promise { + let index = 0; + return async () => samples[index++]; +} + +function metrics(duration: number): PerformanceMetrics { + return { + blockingTime: duration / 10, + cumulativeLayoutShift: 0.01, + firstContentfulPaint: duration, + largestContentfulPaint: duration, + resources: { + font: { count: 1, size: 1_024 }, + image: { count: 1, size: 2_048 }, + script: { count: 1, size: 3_072 }, + total: { count: 3, size: 6_144 }, + }, + }; +} + +function budget(): PerformanceBudget { + return metrics(3_500); +} diff --git a/tests/performance/performance_sampler.ts b/tests/performance/performance_sampler.ts new file mode 100644 index 0000000..7b102d6 --- /dev/null +++ b/tests/performance/performance_sampler.ts @@ -0,0 +1,70 @@ +import type { PerformanceBudget, PerformanceMetrics, ResourceKind } from "./performance_report"; + +const INITIAL_SAMPLE_COUNT = 3; +const CONFIRMATION_SAMPLE_COUNT = 2; + +type AdaptiveMeasurement = { + metrics: PerformanceMetrics; + samples: PerformanceMetrics[]; + wasExtended: boolean; +}; + +export async function collectAdaptiveMetrics( + measure: () => Promise, + budget: PerformanceBudget, +): Promise { + const samples = await collectSamples(measure, INITIAL_SAMPLE_COUNT); + const initialMetrics = medianMetrics(samples); + if (variableMetricsWithinBudget(initialMetrics, budget)) { + return { metrics: initialMetrics, samples, wasExtended: false }; + } + + samples.push(...(await collectSamples(measure, CONFIRMATION_SAMPLE_COUNT))); + return { metrics: medianMetrics(samples), samples, wasExtended: true }; +} + +async function collectSamples( + measure: () => Promise, + count: number, +): Promise { + const samples: PerformanceMetrics[] = []; + for (let index = 0; index < count; index += 1) { + samples.push(await measure()); + } + return samples; +} + +function medianMetrics(samples: PerformanceMetrics[]): PerformanceMetrics { + return { + firstContentfulPaint: median(samples.map((sample) => sample.firstContentfulPaint)), + largestContentfulPaint: median(samples.map((sample) => sample.largestContentfulPaint)), + cumulativeLayoutShift: median(samples.map((sample) => sample.cumulativeLayoutShift)), + blockingTime: median(samples.map((sample) => sample.blockingTime)), + resources: Object.fromEntries( + (["total", "script", "image", "font"] satisfies ResourceKind[]).map((kind) => [ + kind, + { + size: median(samples.map((sample) => sample.resources[kind].size)), + count: median(samples.map((sample) => sample.resources[kind].count)), + }, + ]), + ) as PerformanceMetrics["resources"], + }; +} + +function median(values: number[]): number { + const sorted = values.toSorted((left, right) => left - right); + return sorted[Math.floor(sorted.length / 2)]; +} + +function variableMetricsWithinBudget( + metrics: PerformanceMetrics, + budget: PerformanceBudget, +): boolean { + return ( + metrics.firstContentfulPaint <= budget.firstContentfulPaint && + metrics.largestContentfulPaint <= budget.largestContentfulPaint && + metrics.cumulativeLayoutShift <= budget.cumulativeLayoutShift && + metrics.blockingTime <= budget.blockingTime + ); +}