From b8bd78798c8ad75fcd5ad8fb4efa70f2b034b6b4 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Mon, 10 Aug 2026 19:37:29 +0530 Subject: [PATCH] fix: stop CSP allow-lists triggering challenge detection (#264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isChallengeResponse matched challenge markers against every response header, so any site whose CSP names cdnjs.cloudflare.com or google.com/recaptcha (HN, among many) was classified as a challenge: two wasted impit fetches and a misleading FETCH_BLOCKED for a page that was never blocked. Header evidence is now limited to headers that describe this response (server, cf-mitigated, cf-chl-*, x-datadome*, set-cookie), and a 200 needs body evidence — headers alone never prove a challenge on a served page. Co-Authored-By: Claude Opus 5 --- src/fetch/classify.test.ts | 10 ++++++++++ src/fetch/classify.ts | 10 ++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/fetch/classify.test.ts b/src/fetch/classify.test.ts index 7d4f42f7..d118ea2f 100644 --- a/src/fetch/classify.test.ts +++ b/src/fetch/classify.test.ts @@ -5,6 +5,16 @@ describe('fetch classification', () => { it('recognizes explicit challenges but not bare forbidden responses', () => { expect(isChallengeResponse(403, { server: 'cloudflare' }, 'Just a moment...')).toBe(true); expect(isChallengeResponse(403, {}, 'forbidden')).toBe(false); + expect(isChallengeResponse(403, { server: 'cloudflare' }, 'forbidden')).toBe(true); + }); + it('ignores third-party allow-lists in CSP and friends', () => { + const csp = "default-src 'self'; script-src https://www.google.com/recaptcha/ https://cdnjs.cloudflare.com/"; + expect(isChallengeResponse(200, { 'content-security-policy': csp }, 'Hacker News')).toBe(false); + expect(isChallengeResponse(403, { 'content-security-policy': csp }, 'forbidden')).toBe(false); + }); + it('does not treat a served 200 as a challenge on headers alone', () => { + expect(isChallengeResponse(200, { server: 'cloudflare' }, 'real page')).toBe(false); + expect(isChallengeResponse(200, { server: 'cloudflare' }, 'Just a moment...')).toBe(true); }); it('recognizes script-heavy app shells', () => expect(isJavaScriptShell('
')).toBe(true)); }); diff --git a/src/fetch/classify.ts b/src/fetch/classify.ts index 4051c13c..c8474fdb 100644 --- a/src/fetch/classify.ts +++ b/src/fetch/classify.ts @@ -1,8 +1,14 @@ const challengeMarkers = /cloudflare|cf-chl|datadome|perimeterx|px-captcha|akamai|captcha|just a moment|verify you are human/i; +// Headers that say something about *this* response. CSP/report-to/link are allow-lists of third +// parties (cdnjs.cloudflare.com, google.com/recaptcha) and are evidence of nothing. +const signalHeaders = /^(?:server|cf-mitigated|cf-chl-[\w-]+|x-datadome[\w-]*|set-cookie)$/i; export function isChallengeResponse(status: number, headers: Record, body: string): boolean { - const evidence = `${Object.entries(headers).map(([key, value]) => `${key}:${value}`).join('\n')}\n${body.slice(0, 20_000)}`; - return challengeMarkers.test(evidence) && (status === 403 || status === 429 || status === 503 || status === 200); + if (status !== 403 && status !== 429 && status !== 503 && status !== 200) return false; + if (challengeMarkers.test(body.slice(0, 20_000))) return true; + // A 200 with a real body is a served page; headers alone (server: cloudflare) never prove otherwise. + if (status === 200) return false; + return Object.entries(headers).some(([key, value]) => signalHeaders.test(key) && challengeMarkers.test(value)); } export function isJavaScriptShell(body: string): boolean {