From 2102efe99b359bc529b979a5a9200d50cdf7d2ee Mon Sep 17 00:00:00 2001 From: Sourabh Choraria Date: Sat, 1 Aug 2026 00:30:22 +0100 Subject: [PATCH] test(web-e2e): say WHICH subsystem failed when the deep route never serves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CI failure on 2026-08-01 reported `last probe: 404 (no Location)` for the full 180-second budget — the catch-all answered every poll, so the deepest route never served. That message then blamed the scan, and it cannot know that. "Never served" has two unrelated causes: · the SCAN never reached the route — next dev is still walking src/app, a filesystem-walk problem; or · the route WAS registered and never compiled or served — a different subsystem entirely, with the scan exonerated. The message asserted the first. Reading it sends you through src/app discovery even when next dev's own route table already listed the route, which is exactly the wrong-subsystem trip this file elsewhere goes to some length to prevent. `awaitRouteScan` now returns which of three things it saw and the failure quotes it: `listed` (registered, still 404 — not the scan), `missing-route` (the table was being written and this route never appeared — the scan really did not reach it), or `absent` (no readable types file, so the gate was INERT and proved nothing either way — stated explicitly, because "the scan was fine" is the one conclusion an inert gate cannot support). Read/sleep are injected so all three outcomes are tested without a dev server or a real clock, and the early-return case is pinned so the gate cannot quietly start burning its whole budget on success. ⚠️ This does not fix the flake and is not meant to. It converts one ambiguous timeout into a message that names a subsystem, so the NEXT occurrence is evidence instead of another guess. Verified by forcing the timeout: the harness printed `route scan: listed — … so this is not the scan`, which is correct locally, where the scan completes in well under a second. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BRmGUnxeYsQoG9c8BCZcae --- apps/web/playwright/deepest-route.test.ts | 48 +++++++++++++++++++++++ apps/web/playwright/deepest-route.ts | 43 ++++++++++++++++++++ apps/web/playwright/global-setup.ts | 39 ++++++++++-------- 3 files changed, 114 insertions(+), 16 deletions(-) diff --git a/apps/web/playwright/deepest-route.test.ts b/apps/web/playwright/deepest-route.test.ts index f823ec1f..245e06ea 100644 --- a/apps/web/playwright/deepest-route.test.ts +++ b/apps/web/playwright/deepest-route.test.ts @@ -7,6 +7,7 @@ import { ROUTE_TYPES_FILE, appRoutes, authOriginMismatch, + awaitRouteScan, byDepthDesc, deepestAppRoute, effectiveAuthOrigin, @@ -347,3 +348,50 @@ describe("origin comparison is exact, not prefix-based", () => { expect(isRouteTableReady(307, "/login", "http://localhost:3001")).toBe(false); }); }); + +// ── Which of the two scan failures happened ──────────────────────────────────────────────────────── +// A CI failure on 2026-08-01 reported `last probe: 404 (no Location)` for the full 180s: the catch-all +// answered throughout, so the deep route never served. But "never served" has two very different causes +// and the message cannot tell them apart — +// +// · the SCAN never reached the route (next dev is still walking src/app), or +// · the scan finished and the route registered, but it never compiled/served. +// +// One is a slow filesystem walk, the other is a compile problem. They point at different subsystems, and +// guessing between them is how a flake stays open. The gate now reports which it saw. +describe("awaitRouteScan reports WHY it gave up", () => { + const route = ["org", "[orgId]"]; + const listed = 'type Routes = "/org/[orgId]" | "/other"'; + + it("says `listed` when the route reaches the types file", async () => { + expect(await awaitRouteScan(async () => listed, route, { budgetMs: 50, sleepMs: 1 })).toBe( + "listed", + ); + }); + + it("says `absent` when the types file never appears — the gate was INERT", async () => { + // Not a failure: the file is undocumented and version-dependent, so this degrades to "probe anyway". + // But it means the gate contributed nothing, which is worth knowing before blaming the scan. + const missing = async () => { + throw new Error("ENOENT"); + }; + expect(await awaitRouteScan(missing, route, { budgetMs: 30, sleepMs: 1 })).toBe("absent"); + }); + + it("says `missing-route` when the file is readable but never lists the route", async () => { + // The genuinely diagnostic case: next dev IS writing its route table and this route is not in it, so + // the scan really did not reach it within the budget. + const other = async () => 'type Routes = "/other"'; + expect(await awaitRouteScan(other, route, { budgetMs: 30, sleepMs: 1 })).toBe("missing-route"); + }); + + it("returns as soon as the route appears, rather than burning the budget", async () => { + let calls = 0; + const eventually = async () => { + calls += 1; + return calls < 3 ? 'type Routes = "/other"' : listed; + }; + expect(await awaitRouteScan(eventually, route, { budgetMs: 5_000, sleepMs: 1 })).toBe("listed"); + expect(calls).toBe(3); + }); +}); diff --git a/apps/web/playwright/deepest-route.ts b/apps/web/playwright/deepest-route.ts index 4fb5aece..0bf05d3d 100644 --- a/apps/web/playwright/deepest-route.ts +++ b/apps/web/playwright/deepest-route.ts @@ -232,3 +232,46 @@ export function authOriginMismatch( ): boolean { return status === 307 && locationHasOrigin(location, authOrigin) === false; } + +/** What the route-scan gate saw before it stopped waiting. */ +export type RouteScanOutcome = "listed" | "absent" | "missing-route"; + +/** + * Wait until `next dev` has scanned `src/app` far enough to know about `route` — and REPORT WHICH of the + * three things happened, because the probe's failure message cannot distinguish them on its own. + * + * A CI failure reported `last probe: 404` for the full 180-second budget: the catch-all answered every + * poll, so the deep route never served. That single sentence covers two unrelated causes — + * + * · `missing-route` — next dev is writing its route table and this route is not in it yet, so the SCAN + * genuinely did not reach it. A filesystem-walk problem. + * · `listed` — the route WAS in the table and still 404s, so it registered and never compiled or served. + * A completely different subsystem, and the scan is exonerated. + * + * plus `absent`, which is not a failure but is worth saying out loud: the types file is undocumented and + * version-dependent, so when it never appears this gate contributes nothing and we fall through to the + * probe exactly as before. Reporting it stops someone concluding "the scan was fine" from a gate that was + * never able to say so. + * + * Injectable read/sleep so all three outcomes are testable without a running dev server or a real clock. + */ +export async function awaitRouteScan( + readRouteTypes: () => Promise, + route: readonly string[], + { budgetMs, sleepMs = 250 }: { budgetMs: number; sleepMs?: number }, +): Promise { + const deadline = Date.now() + budgetMs; + let everReadable = false; + do { + try { + const text = await readRouteTypes(); + everReadable = true; + if (routeTypesInclude(text, route)) return "listed"; + } catch { + /* not written yet — keep waiting, and remember we never saw it */ + } + if (Date.now() >= deadline) break; + await new Promise((r) => setTimeout(r, sleepMs)); + } while (Date.now() < deadline); + return everReadable ? "missing-route" : "absent"; +} diff --git a/apps/web/playwright/global-setup.ts b/apps/web/playwright/global-setup.ts index 869937c9..5bd91d4b 100644 --- a/apps/web/playwright/global-setup.ts +++ b/apps/web/playwright/global-setup.ts @@ -12,12 +12,13 @@ import { setupSchema } from "../../../packages/db/test/migrate"; import { startEphemeralPostgres, type EphemeralPostgres } from "../../../packages/db/test/pg"; import { ROUTE_TYPES_FILE, + type RouteScanOutcome, authOriginMismatch, + awaitRouteScan, deepestAppRoute, effectiveAuthOrigin, isRouteTableReady, probeUrlFor, - routeTypesInclude, } from "./deepest-route"; import { BASE_URL, PORT, writeFixture, type Fixture } from "./fixture"; @@ -203,24 +204,16 @@ async function routeTableIsComplete(probe: string): Promise { const SCAN_BUDGET_MS = 60_000; /** - * Resolve once `next dev` has scanned `src/app` far enough to know about `route` — or once the budget - * lapses, whichever comes first. + * Wait for the route scan, and REMEMBER what it saw — see `awaitRouteScan`. The outcome is what turns one + * ambiguous timeout into a message that names a subsystem. * * Never throws. A missing or unreadable file is not a failure: the types file is undocumented and * version-dependent (`typedRoutes` defaults to false, yet Next 16.2 emits it in dev anyway), so this must * degrade to "probe anyway" rather than become a new way for the suite to fail. */ -async function waitForRouteScan(route: readonly string[]): Promise { +async function waitForRouteScan(route: readonly string[]): Promise { const file = resolve(process.cwd(), ROUTE_TYPES_FILE); - const deadline = Date.now() + SCAN_BUDGET_MS; - while (Date.now() < deadline) { - try { - if (routeTypesInclude(await readFile(file, "utf8"), route)) return; - } catch { - /* not written yet — keep waiting */ - } - await sleep(250); - } + return await awaitRouteScan(() => readFile(file, "utf8"), route, { budgetMs: SCAN_BUDGET_MS }); } /** Boot `next dev` against the seeded database and resolve once it answers. */ @@ -275,7 +268,7 @@ async function startDevServer(appConnectionString: string): Promise = { + listed: + "the route WAS in next dev's route table (its types file listed it) and STILL answered 404 — so " + + "this is not the scan. Look at compilation/serving of that route, not at src/app discovery.", + "missing-route": + "next dev was writing its route table and this route never appeared in it within " + + `${SCAN_BUDGET_MS}ms — the scan of src/app genuinely did not reach the deepest route.`, + absent: + "next dev never wrote a readable route-types file, so the scan gate was INERT here and proved " + + "nothing either way. Do not read this as 'the scan was fine'.", + }; throw new Error( `e2e: next dev answered on ${BASE_URL} but ${probeUrlFor(deepestRoute)} never entered its route ` + `table within ${BOOT_TIMEOUT_MS}ms — last probe: ${lastProbe.status} ` + - `${lastProbe.location ?? "(no Location)"}, expected 307 to ${AUTH_ORIGIN}. The catch-all answered ` + - `instead, so next dev began serving before its scan of src/app reached the deepest route.`, + `${lastProbe.location ?? "(no Location)"}, expected 307 to ${AUTH_ORIGIN}.\n` + + ` route scan: ${scan} — ${diagnosis[scan]}`, ); }