Resolve the gutendex 403: WAF-aware live-smoke, baseUrl for self-hosting, docs

.changeset/live-smoke-waf-403.md

@@ -0,0 +1,17 @@
+---
+"@refkit/provider-testkit": patch
+"@refkit/provider-gutendex": patch
+---
+
+Stop the weekly live-smoke from crying wolf on WAF blocks: `liveSmoke` gains an
+opt-in `tolerateUpstreamBlock` that skips (with a warning) when the source's WAF
+returns HTTP 403 from the runner's datacenter IP — 404s, 5xx, schema changes,
+and empty results still fail. Applied to gutendex only, whose Cloudflare front
+blocks GitHub Actions IPs regardless of User-Agent (verified with both a
+descriptive bot UA and a browser UA). gutendex requests also send an explicit
+`Accept: application/json` now.
+
+gutendex additionally gains a `baseUrl` config: the upstream docs frame
+gutendex.com as a test instance ("You should run your own server, but you can
+test queries at gutendex.com"), so production/datacenter consumers can now
+point the provider at a self-hosted Gutendex.

README.md

@@ -199,7 +199,7 @@ createRefkit({ providers, cache: myKvCache, cacheTtlMs: 60_000, cacheRaw: false
 | `@refkit/provider-unsplash` | Unsplash | image | API key | Unsplash |
 | `@refkit/provider-pexels` | Pexels | image · video | API key | Pexels |
 | `@refkit/provider-pixabay` | Pixabay | image · video | API key | Pixabay |
-| `@refkit/provider-gutendex` | Project Gutenberg | text | keyless | per-item PD |
+| `@refkit/provider-gutendex` | Project Gutenberg | text | keyless¹ | per-item PD |
 | `@refkit/provider-poetrydb` | PoetryDB | text | keyless | PD |
 | `@refkit/provider-brave` | Brave web search (discovery) | image (web) | API key | unknown → needs-review |
 | `@refkit/provider-rijksmuseum` | Rijksmuseum | image | keyless | CC0 / PD |
@@ -209,6 +209,8 @@ createRefkit({ providers, cache: myKvCache, cacheTtlMs: 60_000, cacheRaw: false
 | `@refkit/provider-europeana` | Europeana | image | API key | per-item CC / PD / rights-statement |
 | `@refkit/provider-internet-archive` | Internet Archive | video · text | keyless | per-item CC (dirty) → unknown |
 
+¹ gutendex's default host (`gutendex.com`) is the upstream maintainer's **test instance** — its docs say "You should run your own server", and its Cloudflare front blocks datacenter IPs. Desktop/local use works out of the box; for production or server-side traffic, [self-host Gutendex](https://github.com/garethbjohnson/gutendex) and pass `gutendex({ baseUrl: 'https://your-instance' })`. When blocked, the source degrades gracefully (a `failed` entry in `meta.providers`; other sources still return).
+
 Audio/video are extra factories on existing packages: `openverseAudio()`, `pexelsVideo()`, `pixabayVideo()`. Modality routing is automatic — an `['audio']` search only hits audio-capable providers.
 
 ## Architecture

packages/provider-gutendex/README.md

@@ -19,3 +19,13 @@ const refs = await refkit.search({ query: 'whale', modalities: ['text'] })
 ```
 
 Gate by intended use with `refkit.evaluateUse(ref, 'commercial-product')`. See [`@refkit/core`](https://www.npmjs.com/package/@refkit/core) for the full API.
+
+## Hosting note (production / server-side use)
+
+The default host, `gutendex.com`, is the upstream maintainer's **test instance** — the [Gutendex docs](https://github.com/garethbjohnson/gutendex) say "You should run your own server, but you can test queries at gutendex.com", and its Cloudflare front blocks datacenter IPs (e.g. CI runners, cloud servers) regardless of headers. Desktop/local use works out of the box. For production or server-side traffic, [self-host Gutendex](https://github.com/garethbjohnson/gutendex/wiki/Installation-Guide) and point the provider at your instance:
+
+```ts
+gutendex({ baseUrl: 'https://gutendex.your-domain.example' })
+```
+
+When the public instance blocks a request, the search degrades gracefully: this source is reported as `failed` in `meta.providers` and results from other providers still return.

packages/provider-gutendex/src/__tests__/gutendex.test.ts

@@ -65,6 +65,24 @@ describe('gutendex provider', () => {
     expect(url.searchParams.get('page')).toBe('2')
   })
 
+  it('baseUrl points the provider at a self-hosted instance (docs: gutendex.com is test-only)', async () => {
+    let calledUrl = ''
+    const ctx: ProviderContext = {
+      fetch: (async (input: Parameters<typeof fetch>[0]) => {
+        calledUrl = String(input)
+        return new Response(JSON.stringify({ results: [] }), { status: 200 })
+      }) as typeof fetch,
+    }
+    await gutendex({ baseUrl: 'https://gutendex.internal.example' }).search({ text: 'x', modalities: ['text'] }, ctx)
+    expect(calledUrl.startsWith('https://gutendex.internal.example/books/?')).toBe(true)
+    // trailing-slash base joins identically
+    await gutendex({ baseUrl: 'https://gutendex.internal.example/' }).search({ text: 'x', modalities: ['text'] }, ctx)
+    expect(calledUrl.startsWith('https://gutendex.internal.example/books/?')).toBe(true)
+    // default stays the public test instance
+    await gutendex().search({ text: 'x', modalities: ['text'] }, ctx)
+    expect(calledUrl.startsWith('https://gutendex.com/books/?')).toBe(true)
+  })
+
   it('forwards documented Gutendex search options', async () => {
     let calledUrl = ''
     const ctx: ProviderContext = {

packages/provider-gutendex/src/__tests__/live.test.ts

@@ -1,4 +1,7 @@
 import { liveSmoke } from '@refkit/provider-testkit/live'
 import { gutendex } from '../index'
 
-liveSmoke('gutendex', () => gutendex(), { query: 'love' })
+// tolerateUpstreamBlock: gutendex.com's Cloudflare 403s datacenter IPs
+// regardless of UA (verified across live-smoke runs 1-3) — treat as
+// inconclusive, not as drift.
+liveSmoke('gutendex', () => gutendex(), { query: 'love', tolerateUpstreamBlock: true })

packages/provider-gutendex/src/index.ts

@@ -6,6 +6,13 @@ import {
 } from '@refkit/core'
 
 export interface GutendexConfig {
+  /** Root of the Gutendex instance to query. Defaults to the maintainer's public
+   *  instance (https://gutendex.com), which its own docs frame as for TESTING —
+   *  "You should run your own server, but you can test queries at gutendex.com" —
+   *  and whose Cloudflare front blocks datacenter IPs regardless of headers.
+   *  For production or CI traffic, self-host Gutendex
+   *  (https://github.com/garethbjohnson/gutendex) and point this at it. */
+  baseUrl?: string
   /** Gutendex/Cloudflare 403s without a real User-Agent; override if you want your own. */
   userAgent?: string
 }
@@ -75,7 +82,8 @@ export function gutendex(config: GutendexConfig = {}) {
     modalities: ['text'],
     capabilities: { controls: ['language', 'text.copyright', 'page'] },
     async search(q: NormalizedQuery, ctx: ProviderContext): Promise<Reference[]> {
-      const url = new URL('https://gutendex.com/books/')
+      const base = config.baseUrl ?? 'https://gutendex.com'
+      const url = new URL('books/', base.endsWith('/') ? base : `${base}/`)
       url.searchParams.set('search', q.text)
       if (q.controls?.language) url.searchParams.set('languages', q.controls.language)
       if (q.controls?.text?.copyright === 'public-domain') url.searchParams.set('copyright', 'false')
@@ -92,7 +100,15 @@ export function gutendex(config: GutendexConfig = {}) {
       setIfString(url, 'topic', opts?.topic)
       setIfPositiveInt(url, 'page', opts?.page)
       const res = await ctx.fetch(url.toString(), {
-        headers: { 'User-Agent': config.userAgent ?? 'refkit (+https://github.com/refkitjs/refkit)' },
+        // Cloudflare in front of gutendex.com intermittently 403s datacenter IPs
+        // (e.g. GitHub Actions) at the fingerprint level — verified that neither a
+        // descriptive bot UA nor a browser UA gets through (live-smoke runs 1-3),
+        // so we keep the honest UA. Residential/user traffic is unaffected; the
+        // live-smoke suite treats these 403s as inconclusive rather than as drift.
+        headers: {
+          'User-Agent': config.userAgent ?? 'refkit (+https://github.com/refkitjs/refkit)',
+          Accept: 'application/json',
+        },
         signal: ctx.signal,
       })
       if (!res.ok) throw new Error(`gutendex search failed: ${res.status}`)

packages/provider-testkit/src/live.ts

@@ -4,16 +4,34 @@ import { searchConformant, type ConformanceOptions } from './index'
 
 /** Register an env-gated live smoke suite for a provider. Runs only with
  *  REFKIT_LIVE=1 (and, if keyEnv given, that env var set). One real query,
- *  full conformance assertions, 30s timeout. */
+ *  full conformance assertions, 30s timeout.
+ *
+ *  `tolerateUpstreamBlock`: for sources behind a WAF that 403s datacenter IPs
+ *  (gutendex/Cloudflare — verified unfixable client-side: descriptive and
+ *  browser UAs are both blocked). A WAF 403 says nothing about API drift, which
+ *  is what this suite exists to detect, so it SKIPS with a warning instead of
+ *  failing the weekly run. Strictly scoped: only HTTP 403 is tolerated — 404s,
+ *  5xx, schema changes, and empty results still fail. */
 export function liveSmoke(
   name: string,
   make: () => ReferenceProvider,
-  opts: ConformanceOptions & { keyEnv?: string } = {},
+  opts: ConformanceOptions & { keyEnv?: string; tolerateUpstreamBlock?: boolean } = {},
 ): void {
   const enabled = process.env.REFKIT_LIVE === '1' && (!opts.keyEnv || !!process.env[opts.keyEnv])
   describe.skipIf(!enabled)(`live smoke: ${name}`, () => {
-    it('returns conformant references from the real API', { timeout: 30_000 }, async () => {
-      const refs = await searchConformant(make(), globalThis.fetch, opts)
+    it('returns conformant references from the real API', { timeout: 30_000 }, async (t) => {
+      let refs
+      try {
+        refs = await searchConformant(make(), globalThis.fetch, opts)
+      } catch (e) {
+        const message = e instanceof Error ? e.message : String(e)
+        if (opts.tolerateUpstreamBlock && /\b403\b/.test(message)) {
+          console.warn(`[live-smoke] ${name}: upstream WAF returned 403 from this runner — inconclusive for drift, skipping. (${message})`)
+          t.skip()
+          return
+        }
+        throw e
+      }
       expect(refs.length).toBeGreaterThan(0)
     })
   })