diff --git a/CHANGELOG.md b/CHANGELOG.md index 99d9396..06f4ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Unreleased +- chore: add `scripts/check-llms-links.cjs` (`npm run check-llms-links`) to report broken links across all `llms.txt` files exposed from docs.mapbox.com, and flag drift between the curated list in `docsSearchIndex.ts` and the live root index; `npm run check-llms-links:deep` (`--deep`) additionally crawls every docs.mapbox.com sub-link referenced in those files (currently ~1,400 URLs) to proactively catch broken doc pages before customers hit them (#39) - docs: note in CONTRIBUTING.md that unsolicited third-party directory/discovery listing PRs are out of scope and will be closed without review ## 0.3.1 - 2026-06-11 diff --git a/package.json b/package.json index 84f6212..e38bf3d 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,8 @@ "scripts": { "build": "npm run prepare && tshy && npm run generate-version && node scripts/add-shebang.cjs", "changelog:prepare-release": "node scripts/prepare-changelog-release.cjs", + "check-llms-links": "node scripts/check-llms-links.cjs", + "check-llms-links:deep": "node scripts/check-llms-links.cjs --deep", "format": "prettier --check \"./src/**/*.{ts,tsx,js,json,md}\"", "format:fix": "prettier --write \"./src/**/*.{ts,tsx,js,json,md}\"", "generate-version": "node scripts/build-helpers.cjs generate-version", diff --git a/scripts/check-llms-links.cjs b/scripts/check-llms-links.cjs new file mode 100644 index 0000000..890fda5 --- /dev/null +++ b/scripts/check-llms-links.cjs @@ -0,0 +1,241 @@ +#!/usr/bin/env node +// Copyright (c) Mapbox, Inc. +// Licensed under the MIT License. + +/** + * Checks every llms.txt file exposed by docs.mapbox.com for broken links. + * + * Discovers the full set of product llms.txt URLs by: + * 1. Fetching the root https://docs.mapbox.com/llms.txt index + * 2. Extracting every linked *.../llms.txt URL from it + * 3. Cross-checking against the curated list in src/utils/docsSearchIndex.ts, + * flagging any curated URL that the live root index no longer links to + * (a sign docsSearchIndex.ts has drifted from docs.mapbox.com) + * + * Every discovered URL (root + linked + curated) is then requested and its + * HTTP status recorded. Exits non-zero if any URL does not return 2xx. + * + * Pass --deep to additionally crawl every docs.mapbox.com page linked from + * those llms.txt files (hundreds to low-thousands of URLs) — this catches + * broken sub-links proactively, rather than waiting for a customer to hit + * one through the MCP server. + * + * Usage: + * node scripts/check-llms-links.cjs + * node scripts/check-llms-links.cjs --deep + * npm run check-llms-links + * npm run check-llms-links:deep + */ + +const fs = require('node:fs'); +const path = require('node:path'); +const process = require('node:process'); + +const ROOT_URL = 'https://docs.mapbox.com/llms.txt'; +const LLMS_TXT_LINK_RE = /https:\/\/docs\.mapbox\.com\/[^\s)]*llms\.txt/g; +const MARKDOWN_LINK_RE = /\[[^\]]*\]\((https?:\/\/[^)\s]+)\)/g; +const REQUEST_TIMEOUT_MS = 15000; +const SUBLINK_TIMEOUT_MS = 10000; +const SUBLINK_CONCURRENCY = 10; +const SUBLINK_HOSTNAME = 'docs.mapbox.com'; + +const deep = process.argv.includes('--deep'); + +function extractLlmsTxtUrls(text) { + const matches = text.match(LLMS_TXT_LINK_RE) || []; + return [...new Set(matches)].sort(); +} + +function extractMarkdownLinks(text) { + const urls = []; + for (const match of text.matchAll(MARKDOWN_LINK_RE)) { + urls.push(match[1]); + } + return urls; +} + +function readCuratedUrls() { + const indexPath = path.join(process.cwd(), 'src/utils/docsSearchIndex.ts'); + if (!fs.existsSync(indexPath)) { + return []; + } + const content = fs.readFileSync(indexPath, 'utf8'); + return extractLlmsTxtUrls(content); +} + +async function fetchWithTimeout( + url, + { method = 'GET', timeoutMs = REQUEST_TIMEOUT_MS } = {} +) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { method, signal: controller.signal }); + const text = method === 'GET' ? await response.text() : ''; + return { status: response.status, ok: response.ok, text }; + } catch (error) { + return { status: null, ok: false, error: error.message, text: '' }; + } finally { + clearTimeout(timer); + } +} + +// Prefers a cheap HEAD request; falls back to GET when a server doesn't +// support HEAD (some static hosts return 405/501, or drop the connection). +async function checkLink(url) { + const head = await fetchWithTimeout(url, { + method: 'HEAD', + timeoutMs: SUBLINK_TIMEOUT_MS + }); + if (head.status === 405 || head.status === 501 || head.status === null) { + return fetchWithTimeout(url, { + method: 'GET', + timeoutMs: SUBLINK_TIMEOUT_MS + }); + } + return head; +} + +async function runWithConcurrency(items, limit, worker) { + const results = new Array(items.length); + let next = 0; + async function runNext() { + while (next < items.length) { + const i = next++; + results[i] = await worker(items[i]); + } + } + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, runNext) + ); + return results; +} + +async function main() { + console.log(`Fetching root index: ${ROOT_URL}`); + const rootResult = await fetchWithTimeout(ROOT_URL); + if (!rootResult.ok) { + console.error( + `Error: could not fetch root llms.txt (status: ${rootResult.status ?? 'n/a'}${ + rootResult.error ? `, ${rootResult.error}` : '' + })` + ); + process.exit(1); + } + + const linkedUrls = extractLlmsTxtUrls(rootResult.text); + const curatedUrls = readCuratedUrls(); + + const allUrls = [ + ...new Set([ROOT_URL, ...linkedUrls, ...curatedUrls]) + ].sort(); + + const staleCurated = curatedUrls.filter( + (url) => !linkedUrls.includes(url) && url !== ROOT_URL + ); + + console.log( + `Discovered ${allUrls.length} llms.txt URL(s) (${linkedUrls.length} linked from root, ${curatedUrls.length} curated in docsSearchIndex.ts)\n` + ); + + const indexResults = await Promise.all( + allUrls.map(async (url) => + url === ROOT_URL + ? { url, ...rootResult } + : { url, ...(await fetchWithTimeout(url)) } + ) + ); + + const indexFailures = indexResults.filter((r) => !r.ok); + + const statusLabel = (r) => (r.status !== null ? String(r.status) : `ERR`); + const urlColumnWidth = Math.max(...allUrls.map((u) => u.length)); + for (const r of indexResults) { + const marker = r.ok ? ' ' : '✗'; + console.log( + `${marker} ${statusLabel(r).padEnd(4)} ${r.url.padEnd(urlColumnWidth)}${ + r.error ? ` (${r.error})` : '' + }` + ); + } + + console.log(''); + if (staleCurated.length > 0) { + console.log( + `Note: docsSearchIndex.ts references ${staleCurated.length} llms.txt URL(s) no longer linked from the root index (may have moved/renamed):` + ); + for (const url of staleCurated) { + console.log(` - ${url}`); + } + console.log(''); + } + + if (indexFailures.length > 0) { + console.error( + `✗ ${indexFailures.length} of ${allUrls.length} llms.txt URL(s) failed:` + ); + for (const f of indexFailures) { + console.error(` - ${statusLabel(f)} ${f.url}`); + } + } else { + console.log(`✓ All ${allUrls.length} llms.txt URL(s) returned 2xx`); + } + + if (!deep) { + if (indexFailures.length > 0) process.exit(1); + return; + } + + console.log( + `\nDeep mode: crawling ${SUBLINK_HOSTNAME} sub-links referenced in each llms.txt file...\n` + ); + + const sourcesByUrl = new Map(); + for (const result of indexResults) { + if (!result.ok || !result.text) continue; + for (const link of extractMarkdownLinks(result.text)) { + let hostname; + try { + hostname = new URL(link).hostname; + } catch { + continue; + } + if (hostname !== SUBLINK_HOSTNAME) continue; + if (allUrls.includes(link)) continue; // already checked above as an index file + if (!sourcesByUrl.has(link)) sourcesByUrl.set(link, new Set()); + sourcesByUrl.get(link).add(result.url); + } + } + + const subLinks = [...sourcesByUrl.keys()].sort(); + console.log( + `Discovered ${subLinks.length} unique ${SUBLINK_HOSTNAME} sub-link(s)\n` + ); + + const subResults = await runWithConcurrency( + subLinks, + SUBLINK_CONCURRENCY, + async (url) => ({ url, ...(await checkLink(url)) }) + ); + + const subFailures = subResults.filter((r) => !r.ok); + + if (subFailures.length > 0) { + console.error( + `✗ ${subFailures.length} of ${subLinks.length} sub-link(s) failed:` + ); + for (const f of subFailures) { + const sources = [...sourcesByUrl.get(f.url)].join(', '); + console.error(` - ${statusLabel(f)} ${f.url}`); + console.error(` linked from: ${sources}`); + } + } else { + console.log(`✓ All ${subLinks.length} sub-link(s) returned 2xx`); + } + + if (indexFailures.length > 0 || subFailures.length > 0) { + process.exit(1); + } +} + +main();