diff --git a/.gitignore b/.gitignore index 0bda0de5..5ae92795 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,6 @@ test *.zip *.csv link-reports/ +CLAUDE.md diff --git a/docusaurus.config.mjs b/docusaurus.config.mjs index 82c4eeb1..9035e19e 100644 --- a/docusaurus.config.mjs +++ b/docusaurus.config.mjs @@ -65,6 +65,59 @@ export default { 'docs/desktop/**', 'docs/references/**', ], + customLLMFiles: [ + { + filename: 'llms-api.txt', + includePatterns: [ + 'docs/develop/upload-and-download.md', + 'docs/develop/files.md', + 'docs/develop/tools-and-features/buy-a-stamp-batch.md', + 'docs/develop/tools-and-features/feeds.md', + 'docs/develop/tools-and-features/pss.md', + 'docs/develop/tools-and-features/gsoc.md', + 'docs/develop/tools-and-features/chunk-types.md', + 'docs/develop/tools-and-features/manifests.md', + 'docs/develop/tools-and-features/pinning.md', + 'docs/develop/tools-and-features/store-with-encryption.md', + 'docs/develop/tools-and-features/erasure-coding.md', + 'docs/develop/tools-and-features/bee-js.md', + 'docs/develop/tools-and-features/gateway-proxy.md', + 'docs/develop/access-control.md', + 'docs/develop/dynamic-content.md', + 'docs/develop/ultra-light-nodes.md', + 'docs/develop/host-your-website.md', + 'docs/develop/routing.md', + 'docs/develop/multi-author-blog.md', + 'docs/develop/resources-md.md', + 'docs/bee/working-with-bee/bee-api.md', + ], + fullContent: true, + title: 'Swarm API & Development', + description: 'API usage, uploads, downloads, stamps, feeds, chunks, and integration patterns — for agents building on Swarm.', + rootContent: 'Focused documentation slice for integration-focused agents. Covers the Bee HTTP API, data operations, stamps, feeds, encryption, and developer tooling. For the full documentation, see llms-full.txt.', + }, + { + filename: 'llms-node-ops.txt', + includePatterns: [ + 'docs/bee/installation/**', + 'docs/bee/working-with-bee/configuration.md', + 'docs/bee/working-with-bee/node-types.md', + 'docs/bee/working-with-bee/monitoring.md', + 'docs/bee/working-with-bee/staking.md', + 'docs/bee/working-with-bee/cashing-out.md', + 'docs/bee/working-with-bee/backups.md', + 'docs/bee/working-with-bee/upgrade.md', + 'docs/bee/working-with-bee/logs-and-files.md', + 'docs/bee/working-with-bee/uninstalling-bee.md', + 'docs/bee/working-with-bee/bcrypt.md', + 'docs/bee/faq.md', + ], + fullContent: true, + title: 'Swarm Node Operations', + description: 'Installation, configuration, monitoring, staking, backups, and troubleshooting — for DevOps and sysadmin agents.', + rootContent: 'Focused documentation slice for node operators. Covers installation methods, configuration, monitoring, staking, backups, upgrades, and FAQ. For the full documentation, see llms-full.txt.', + }, + ], } ], [ diff --git a/scripts/validate-llms-txt.mjs b/scripts/validate-llms-txt.mjs index 87e76b56..af5f6437 100644 --- a/scripts/validate-llms-txt.mjs +++ b/scripts/validate-llms-txt.mjs @@ -103,3 +103,49 @@ if (warnings > 0) { } else { console.log('llms.txt validation: all links verified, full coverage ✓'); } + +// 3. Verify slice file references in llms.txt point to files that will be generated +const sliceRe = /https:\/\/docs\.ethswarm\.org\/(llms-[a-z-]+\.txt)/g; +const llmsTxtContent = readFileSync(LLMS_TXT, 'utf-8'); +let sliceMatch; +while ((sliceMatch = sliceRe.exec(llmsTxtContent)) !== null) { + console.log(` ℹ Slice file referenced: ${sliceMatch[1]} (generated by docusaurus-plugin-llms at build time)`); +} + +// 4. Verify customLLMFiles includePatterns resolve to real files. +// Parses docusaurus.config.mjs as text to extract patterns, then checks +// each against the filesystem. Catches typos like "act.md" when the real +// file is "access-control.md". +import { globSync } from 'node:fs'; + +const configText = readFileSync(join(ROOT, 'docusaurus.config.mjs'), 'utf-8'); +const patternRe = /includePatterns:\s*\[([\s\S]*?)\]/g; +let patternBlock; +let patternWarnings = 0; + +while ((patternBlock = patternRe.exec(configText)) !== null) { + const patterns = patternBlock[1].match(/'([^']+)'/g)?.map(s => s.replace(/'/g, '')) ?? []; + for (const pattern of patterns) { + // For glob patterns (contain *), use globSync; for exact paths, check existence + if (pattern.includes('*')) { + const matches = globSync(pattern, { cwd: ROOT }); + if (matches.length === 0) { + console.warn(` ⚠ includePattern matches no files: ${pattern}`); + patternWarnings++; + } + } else { + try { + readFileSync(join(ROOT, pattern)); + } catch { + console.warn(` ⚠ includePattern file not found: ${pattern}`); + patternWarnings++; + } + } + } +} + +if (patternWarnings > 0) { + console.log(`includePatterns validation: ${patternWarnings} pattern(s) match no files`); +} else { + console.log('includePatterns validation: all patterns resolve ✓'); +} diff --git a/static/llms.txt b/static/llms.txt index 5f2b397b..cb6b6320 100644 --- a/static/llms.txt +++ b/static/llms.txt @@ -10,6 +10,11 @@ This is the documentation for [Swarm](https://www.ethswarm.org/) and its referen - [The Book of Swarm](https://www.ethswarm.org/The-Book-of-Swarm.pdf): Comprehensive technical reference - [Swarm Whitepaper](https://www.ethswarm.org/swarm-whitepaper.pdf): Original design paper +## Focused Documentation + +- [API & Development](https://docs.ethswarm.org/llms-api.txt): API usage, uploads, stamps, feeds, chunks, and integration patterns — for agents building on Swarm +- [Node Operations](https://docs.ethswarm.org/llms-node-ops.txt): Installation, configuration, monitoring, staking, backups, and troubleshooting — for DevOps agents + ## Install and run a Bee node - [Getting Started](https://docs.ethswarm.org/docs/bee/installation/getting-started): Overview of node requirements and installation options