From de9b567a6eb6386fb8c2b49b0502556972cf7e42 Mon Sep 17 00:00:00 2001 From: Matthew Podwysocki Date: Mon, 3 Aug 2026 12:54:30 -0400 Subject: [PATCH 1/3] chore: add script to report broken llms.txt links from docs.mapbox.com Discovers every llms.txt exposed via the root index, checks each for 2xx, and flags when docsSearchIndex.ts's curated list has drifted from the live root index (e.g. studio-manual/llms.txt vs console-tools/llms.txt). Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + package.json | 1 + scripts/check-llms-links.cjs | 127 +++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+) create mode 100644 scripts/check-llms-links.cjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 99d9396..1cde11a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Unreleased +- chore: add `scripts/check-llms-links.cjs` (`npm run check-llms-links`) to report broken links across all `llms.txt` files exposed from docs.mapbox.com, and flag drift between the curated list in `docsSearchIndex.ts` and the live root index (#TBD) - docs: note in CONTRIBUTING.md that unsolicited third-party directory/discovery listing PRs are out of scope and will be closed without review ## 0.3.1 - 2026-06-11 diff --git a/package.json b/package.json index 84f6212..ae3521e 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "scripts": { "build": "npm run prepare && tshy && npm run generate-version && node scripts/add-shebang.cjs", "changelog:prepare-release": "node scripts/prepare-changelog-release.cjs", + "check-llms-links": "node scripts/check-llms-links.cjs", "format": "prettier --check \"./src/**/*.{ts,tsx,js,json,md}\"", "format:fix": "prettier --write \"./src/**/*.{ts,tsx,js,json,md}\"", "generate-version": "node scripts/build-helpers.cjs generate-version", diff --git a/scripts/check-llms-links.cjs b/scripts/check-llms-links.cjs new file mode 100644 index 0000000..2a9a0e1 --- /dev/null +++ b/scripts/check-llms-links.cjs @@ -0,0 +1,127 @@ +#!/usr/bin/env node +// Copyright (c) Mapbox, Inc. +// Licensed under the MIT License. + +/** + * Checks every llms.txt file exposed by docs.mapbox.com for broken links. + * + * Discovers the full set of product llms.txt URLs by: + * 1. Fetching the root https://docs.mapbox.com/llms.txt index + * 2. Extracting every linked *.../llms.txt URL from it + * 3. Cross-checking against the curated list in src/utils/docsSearchIndex.ts, + * flagging any curated URL that the live root index no longer links to + * (a sign docsSearchIndex.ts has drifted from docs.mapbox.com) + * + * Every discovered URL (root + linked + curated) is then requested and its + * HTTP status recorded. Exits non-zero if any URL does not return 2xx. + * + * Usage: + * node scripts/check-llms-links.cjs + * npm run check-llms-links + */ + +const fs = require('node:fs'); +const path = require('node:path'); +const process = require('node:process'); + +const ROOT_URL = 'https://docs.mapbox.com/llms.txt'; +const LLMS_TXT_LINK_RE = /https:\/\/docs\.mapbox\.com\/[^\s)]*llms\.txt/g; +const REQUEST_TIMEOUT_MS = 15000; + +function extractLlmsTxtUrls(text) { + const matches = text.match(LLMS_TXT_LINK_RE) || []; + return [...new Set(matches)].sort(); +} + +function readCuratedUrls() { + const indexPath = path.join(process.cwd(), 'src/utils/docsSearchIndex.ts'); + if (!fs.existsSync(indexPath)) { + return []; + } + const content = fs.readFileSync(indexPath, 'utf8'); + return extractLlmsTxtUrls(content); +} + +async function fetchWithTimeout(url) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); + try { + const response = await fetch(url, { signal: controller.signal }); + const text = await response.text(); + return { status: response.status, ok: response.ok, text }; + } catch (error) { + return { status: null, ok: false, error: error.message }; + } finally { + clearTimeout(timer); + } +} + +async function main() { + console.log(`Fetching root index: ${ROOT_URL}`); + const rootResult = await fetchWithTimeout(ROOT_URL); + if (!rootResult.ok) { + console.error( + `Error: could not fetch root llms.txt (status: ${rootResult.status ?? 'n/a'}${ + rootResult.error ? `, ${rootResult.error}` : '' + })` + ); + process.exit(1); + } + + const linkedUrls = extractLlmsTxtUrls(rootResult.text); + const curatedUrls = readCuratedUrls(); + + const allUrls = [ + ...new Set([ROOT_URL, ...linkedUrls, ...curatedUrls]) + ].sort(); + + const staleCurated = curatedUrls.filter( + (url) => !linkedUrls.includes(url) && url !== ROOT_URL + ); + + console.log( + `Discovered ${allUrls.length} llms.txt URL(s) (${linkedUrls.length} linked from root, ${curatedUrls.length} curated in docsSearchIndex.ts)\n` + ); + + const results = await Promise.all( + allUrls.map(async (url) => ({ url, ...(await fetchWithTimeout(url)) })) + ); + + const failures = results.filter((r) => !r.ok); + + const statusLabel = (r) => (r.status !== null ? String(r.status) : `ERR`); + const urlColumnWidth = Math.max(...allUrls.map((u) => u.length)); + for (const r of results) { + const marker = r.ok ? ' ' : '✗'; + console.log( + `${marker} ${statusLabel(r).padEnd(4)} ${r.url.padEnd(urlColumnWidth)}${ + r.error ? ` (${r.error})` : '' + }` + ); + } + + console.log(''); + if (staleCurated.length > 0) { + console.log( + `Note: docsSearchIndex.ts references ${staleCurated.length} llms.txt URL(s) no longer linked from the root index (may have moved/renamed):` + ); + for (const url of staleCurated) { + console.log(` - ${url}`); + } + console.log(''); + } + + if (failures.length > 0) { + console.error( + `✗ ${failures.length} of ${allUrls.length} llms.txt URL(s) failed:` + ); + for (const f of failures) { + console.error(` - ${statusLabel(f)} ${f.url}`); + } + process.exit(1); + } + + console.log(`✓ All ${allUrls.length} llms.txt URL(s) returned 2xx`); +} + +main(); From e895d15a0e37a1ba9c730a04ef664ff2338264ef Mon Sep 17 00:00:00 2001 From: Matthew Podwysocki Date: Mon, 3 Aug 2026 13:00:38 -0400 Subject: [PATCH 2/3] feat: add --deep mode to crawl docs.mapbox.com sub-links Weekly 404 reports only surface broken sub-links customers already hit through the MCP server, not ones nobody's found yet. --deep extracts every docs.mapbox.com link referenced inside each llms.txt (~1,400 URLs today) and checks it with bounded concurrency, so we can catch a broken doc page before a customer does. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 2 +- package.json | 1 + scripts/check-llms-links.cjs | 142 +++++++++++++++++++++++++++++++---- 3 files changed, 130 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cde11a..2ac6d9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Unreleased -- chore: add `scripts/check-llms-links.cjs` (`npm run check-llms-links`) to report broken links across all `llms.txt` files exposed from docs.mapbox.com, and flag drift between the curated list in `docsSearchIndex.ts` and the live root index (#TBD) +- chore: add `scripts/check-llms-links.cjs` (`npm run check-llms-links`) to report broken links across all `llms.txt` files exposed from docs.mapbox.com, and flag drift between the curated list in `docsSearchIndex.ts` and the live root index; `npm run check-llms-links:deep` (`--deep`) additionally crawls every docs.mapbox.com sub-link referenced in those files (currently ~1,400 URLs) to proactively catch broken doc pages before customers hit them (#TBD) - docs: note in CONTRIBUTING.md that unsolicited third-party directory/discovery listing PRs are out of scope and will be closed without review ## 0.3.1 - 2026-06-11 diff --git a/package.json b/package.json index ae3521e..e38bf3d 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "build": "npm run prepare && tshy && npm run generate-version && node scripts/add-shebang.cjs", "changelog:prepare-release": "node scripts/prepare-changelog-release.cjs", "check-llms-links": "node scripts/check-llms-links.cjs", + "check-llms-links:deep": "node scripts/check-llms-links.cjs --deep", "format": "prettier --check \"./src/**/*.{ts,tsx,js,json,md}\"", "format:fix": "prettier --write \"./src/**/*.{ts,tsx,js,json,md}\"", "generate-version": "node scripts/build-helpers.cjs generate-version", diff --git a/scripts/check-llms-links.cjs b/scripts/check-llms-links.cjs index 2a9a0e1..890fda5 100644 --- a/scripts/check-llms-links.cjs +++ b/scripts/check-llms-links.cjs @@ -15,9 +15,16 @@ * Every discovered URL (root + linked + curated) is then requested and its * HTTP status recorded. Exits non-zero if any URL does not return 2xx. * + * Pass --deep to additionally crawl every docs.mapbox.com page linked from + * those llms.txt files (hundreds to low-thousands of URLs) — this catches + * broken sub-links proactively, rather than waiting for a customer to hit + * one through the MCP server. + * * Usage: * node scripts/check-llms-links.cjs + * node scripts/check-llms-links.cjs --deep * npm run check-llms-links + * npm run check-llms-links:deep */ const fs = require('node:fs'); @@ -26,13 +33,27 @@ const process = require('node:process'); const ROOT_URL = 'https://docs.mapbox.com/llms.txt'; const LLMS_TXT_LINK_RE = /https:\/\/docs\.mapbox\.com\/[^\s)]*llms\.txt/g; +const MARKDOWN_LINK_RE = /\[[^\]]*\]\((https?:\/\/[^)\s]+)\)/g; const REQUEST_TIMEOUT_MS = 15000; +const SUBLINK_TIMEOUT_MS = 10000; +const SUBLINK_CONCURRENCY = 10; +const SUBLINK_HOSTNAME = 'docs.mapbox.com'; + +const deep = process.argv.includes('--deep'); function extractLlmsTxtUrls(text) { const matches = text.match(LLMS_TXT_LINK_RE) || []; return [...new Set(matches)].sort(); } +function extractMarkdownLinks(text) { + const urls = []; + for (const match of text.matchAll(MARKDOWN_LINK_RE)) { + urls.push(match[1]); + } + return urls; +} + function readCuratedUrls() { const indexPath = path.join(process.cwd(), 'src/utils/docsSearchIndex.ts'); if (!fs.existsSync(indexPath)) { @@ -42,20 +63,54 @@ function readCuratedUrls() { return extractLlmsTxtUrls(content); } -async function fetchWithTimeout(url) { +async function fetchWithTimeout( + url, + { method = 'GET', timeoutMs = REQUEST_TIMEOUT_MS } = {} +) { const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); + const timer = setTimeout(() => controller.abort(), timeoutMs); try { - const response = await fetch(url, { signal: controller.signal }); - const text = await response.text(); + const response = await fetch(url, { method, signal: controller.signal }); + const text = method === 'GET' ? await response.text() : ''; return { status: response.status, ok: response.ok, text }; } catch (error) { - return { status: null, ok: false, error: error.message }; + return { status: null, ok: false, error: error.message, text: '' }; } finally { clearTimeout(timer); } } +// Prefers a cheap HEAD request; falls back to GET when a server doesn't +// support HEAD (some static hosts return 405/501, or drop the connection). +async function checkLink(url) { + const head = await fetchWithTimeout(url, { + method: 'HEAD', + timeoutMs: SUBLINK_TIMEOUT_MS + }); + if (head.status === 405 || head.status === 501 || head.status === null) { + return fetchWithTimeout(url, { + method: 'GET', + timeoutMs: SUBLINK_TIMEOUT_MS + }); + } + return head; +} + +async function runWithConcurrency(items, limit, worker) { + const results = new Array(items.length); + let next = 0; + async function runNext() { + while (next < items.length) { + const i = next++; + results[i] = await worker(items[i]); + } + } + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, runNext) + ); + return results; +} + async function main() { console.log(`Fetching root index: ${ROOT_URL}`); const rootResult = await fetchWithTimeout(ROOT_URL); @@ -83,15 +138,19 @@ async function main() { `Discovered ${allUrls.length} llms.txt URL(s) (${linkedUrls.length} linked from root, ${curatedUrls.length} curated in docsSearchIndex.ts)\n` ); - const results = await Promise.all( - allUrls.map(async (url) => ({ url, ...(await fetchWithTimeout(url)) })) + const indexResults = await Promise.all( + allUrls.map(async (url) => + url === ROOT_URL + ? { url, ...rootResult } + : { url, ...(await fetchWithTimeout(url)) } + ) ); - const failures = results.filter((r) => !r.ok); + const indexFailures = indexResults.filter((r) => !r.ok); const statusLabel = (r) => (r.status !== null ? String(r.status) : `ERR`); const urlColumnWidth = Math.max(...allUrls.map((u) => u.length)); - for (const r of results) { + for (const r of indexResults) { const marker = r.ok ? ' ' : '✗'; console.log( `${marker} ${statusLabel(r).padEnd(4)} ${r.url.padEnd(urlColumnWidth)}${ @@ -111,17 +170,72 @@ async function main() { console.log(''); } - if (failures.length > 0) { + if (indexFailures.length > 0) { console.error( - `✗ ${failures.length} of ${allUrls.length} llms.txt URL(s) failed:` + `✗ ${indexFailures.length} of ${allUrls.length} llms.txt URL(s) failed:` ); - for (const f of failures) { + for (const f of indexFailures) { console.error(` - ${statusLabel(f)} ${f.url}`); } - process.exit(1); + } else { + console.log(`✓ All ${allUrls.length} llms.txt URL(s) returned 2xx`); + } + + if (!deep) { + if (indexFailures.length > 0) process.exit(1); + return; + } + + console.log( + `\nDeep mode: crawling ${SUBLINK_HOSTNAME} sub-links referenced in each llms.txt file...\n` + ); + + const sourcesByUrl = new Map(); + for (const result of indexResults) { + if (!result.ok || !result.text) continue; + for (const link of extractMarkdownLinks(result.text)) { + let hostname; + try { + hostname = new URL(link).hostname; + } catch { + continue; + } + if (hostname !== SUBLINK_HOSTNAME) continue; + if (allUrls.includes(link)) continue; // already checked above as an index file + if (!sourcesByUrl.has(link)) sourcesByUrl.set(link, new Set()); + sourcesByUrl.get(link).add(result.url); + } + } + + const subLinks = [...sourcesByUrl.keys()].sort(); + console.log( + `Discovered ${subLinks.length} unique ${SUBLINK_HOSTNAME} sub-link(s)\n` + ); + + const subResults = await runWithConcurrency( + subLinks, + SUBLINK_CONCURRENCY, + async (url) => ({ url, ...(await checkLink(url)) }) + ); + + const subFailures = subResults.filter((r) => !r.ok); + + if (subFailures.length > 0) { + console.error( + `✗ ${subFailures.length} of ${subLinks.length} sub-link(s) failed:` + ); + for (const f of subFailures) { + const sources = [...sourcesByUrl.get(f.url)].join(', '); + console.error(` - ${statusLabel(f)} ${f.url}`); + console.error(` linked from: ${sources}`); + } + } else { + console.log(`✓ All ${subLinks.length} sub-link(s) returned 2xx`); } - console.log(`✓ All ${allUrls.length} llms.txt URL(s) returned 2xx`); + if (indexFailures.length > 0 || subFailures.length > 0) { + process.exit(1); + } } main(); From 2dde98a821ead3c73f53e3a305fb7f0c637fe338 Mon Sep 17 00:00:00 2001 From: Matthew Podwysocki Date: Mon, 3 Aug 2026 13:01:42 -0400 Subject: [PATCH 3/3] docs: fill in PR number in CHANGELOG entry Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ac6d9e..06f4ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Unreleased -- chore: add `scripts/check-llms-links.cjs` (`npm run check-llms-links`) to report broken links across all `llms.txt` files exposed from docs.mapbox.com, and flag drift between the curated list in `docsSearchIndex.ts` and the live root index; `npm run check-llms-links:deep` (`--deep`) additionally crawls every docs.mapbox.com sub-link referenced in those files (currently ~1,400 URLs) to proactively catch broken doc pages before customers hit them (#TBD) +- chore: add `scripts/check-llms-links.cjs` (`npm run check-llms-links`) to report broken links across all `llms.txt` files exposed from docs.mapbox.com, and flag drift between the curated list in `docsSearchIndex.ts` and the live root index; `npm run check-llms-links:deep` (`--deep`) additionally crawls every docs.mapbox.com sub-link referenced in those files (currently ~1,400 URLs) to proactively catch broken doc pages before customers hit them (#39) - docs: note in CONTRIBUTING.md that unsolicited third-party directory/discovery listing PRs are out of scope and will be closed without review ## 0.3.1 - 2026-06-11