From c08ba960846f449808727a6023ff86f677e367f2 Mon Sep 17 00:00:00 2001 From: developerjamiu Date: Wed, 12 Aug 2026 16:26:19 +0100 Subject: [PATCH 1/2] feat: Publish a markdown export of every docs page with llms.txt --- .github/workflows/deploy.yml | 12 +- .github/workflows/test-build.yml | 4 +- README.md | 11 + docusaurus.config.js | 91 +------ package-lock.json | 29 ++- package.json | 7 +- plugins/markdown-export/flatten.js | 397 +++++++++++++++++++++++++++++ plugins/markdown-export/index.js | 317 +++++++++++++++++++++++ plugins/markdown-export/llms.js | 63 +++++ plugins/markdown-export/urls.js | 51 ++++ redirects.js | 89 +++++++ static/robots.txt | 11 + util/verify_markdown_export.js | 286 +++++++++++++++++++++ 13 files changed, 1268 insertions(+), 100 deletions(-) create mode 100644 plugins/markdown-export/flatten.js create mode 100644 plugins/markdown-export/index.js create mode 100644 plugins/markdown-export/llms.js create mode 100644 plugins/markdown-export/urls.js create mode 100644 redirects.js create mode 100644 static/robots.txt create mode 100644 util/verify_markdown_export.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index afb18f43..87bd7c0e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -26,6 +26,10 @@ jobs: cd src npm ci npm run build + - name: Verify markdown export + run: | + cd src + node util/verify_markdown_export.js - uses: actions/checkout@v4 with: path: target @@ -36,9 +40,13 @@ jobs: run: | git config --global user.email "serverpod_docs@serverpod.dev" git config --global user.name "serverpod_docs" - rm -rf target/docs/* + # Dotfiles must deploy too (.nojekyll keeps GitHub Pages from + # running Jekyll over the generated .md files), so copy the + # directory contents rather than a dotfile-skipping glob. + rm -rf target/docs + mkdir -p target/docs cp target/CNAME target/docs - cp -r src/build/* target/docs + cp -a src/build/. target/docs cd target git add . if git diff --cached --quiet; then diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 690bfef2..5caa36ac 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -32,4 +32,6 @@ jobs: - name: Install dependencies run: npm ci - name: Build - run: npm run build \ No newline at end of file + run: npm run build + - name: Verify markdown export + run: node util/verify_markdown_export.js \ No newline at end of file diff --git a/README.md b/README.md index 7484bb70..d44bbfee 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,17 @@ To maintain link integrity when relocating or renaming documentation pages, it's Once a PR is merged into the `main` branch of this repository, a GitHub action is triggered that builds the documentation and pushes the build to the `docs` directory within the `serverpod.github.io` repository. The built documentation is committed as a new commit to the `main` branch and is then deployed to Github pages by the `serverpod.github.io` repository. +### Markdown export (llms.txt and per-page .md) + +The local plugin in `plugins/markdown-export` publishes a clean markdown version of every doc page at its page URL with `.md` appended, plus `llms.txt`, `llms-full.txt`, and `cloud/llms-full.txt` at the site root. The "Copy as Markdown" button on every page fetches these files. Things to know: + +- The llms files always describe the current stable version, derived from `versions.json`; nothing needs updating when a new version is cut. The plugin logs the export size and duration on every build. +- Links between doc pages inside the exports point at the target's `.md` version, so an agent reading one page can follow links to more markdown. +- Renamed pages keep their HTML redirect only. The old `.md` URL becomes a one-line "moved to" stub, generated from `redirects.js`. +- `static/robots.txt` disallows crawling of `*.md` to keep the duplicates out of search engines; `llms.txt` stays crawlable on purpose. +- `node util/verify_markdown_export.js` checks the export after a build. CI runs it on every PR. +- GitHub Pages only serves the `.md` files raw because Jekyll is disabled via `.nojekyll`; the deploy workflow copies that file explicitly. + ### Formatting To ensure consistent formatting, we use markdownlint [(VS Code Extension)](https://marketplace.visualstudio.com/items?itemName=DavidAnson.vscode-markdownlint) diff --git a/docusaurus.config.js b/docusaurus.config.js index 70112c00..3e16f281 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -4,6 +4,7 @@ import { themes } from 'prism-react-renderer'; +import {SITE_URL, BASE_URL} from './plugins/markdown-export/urls.js'; const lightCodeTheme = themes.github; const darkCodeTheme = themes.dracula; @@ -11,8 +12,8 @@ const darkCodeTheme = themes.dracula; const config = { title: 'Serverpod', tagline: 'The missing server for Flutter', - url: 'https://docs.serverpod.dev', - baseUrl: '/', + url: SITE_URL, + baseUrl: BASE_URL, onBrokenLinks: 'throw', onBrokenAnchors: 'throw', favicon: 'img/favicon.png', @@ -147,6 +148,7 @@ const config = { breadcrumbs: false, }, ], + './plugins/markdown-export', [ 'docusaurus-plugin-snipsync', { @@ -182,90 +184,7 @@ const config = { [ '@docusaurus/plugin-client-redirects', { - redirects: [{ - // Moved in version 1.1.1 - from: ['/concepts/authentication'], - to: '/concepts/authentication/setup', - }, - { - // Moved in version 1.1.1, 2.1.0 and 2.9.0 - from: ['/tutorials', '/tutorials/videos', '/tutorials/first-app'], - to: '/tutorials/tutorials/fundamentals', - }, - { - // Moved in version 1.2.0 - from: ['/concepts/database-communication'], - to: '/concepts/database/connection', - }, - { - // Moved in version 2.1.0 - from: ['/insights'], - to: '/tools/insights', - }, - { - // Moved in version 2.1.0 - from: ['/roadmap'], - to: '/contribute', - }, - { - // Moved in version 2.7.0 - from: ['/get-started'], - to: '/get-started/creating-endpoints', - }, - { - // Moved when scheduling was reorganized from a single page to a directory - from: ['/concepts/scheduling'], - to: '/concepts/scheduling/setup', - }, - { - from: ['/cloud/reference/deployment/deploying-your-application'], - to: '/cloud/concepts/deployments', - }, - { - from: ['/cloud/guides/logs', '/cloud/reference/logging'], - to: '/cloud/concepts/logs', - }, - { - from: ['/cloud/guides/passwords'], - to: '/cloud/concepts/passwords-secrets-env-vars', - }, - { - from: ['/cloud/guides/custom-domains'], - to: '/cloud/concepts/custom-domains', - }, - { - from: ['/cloud/guides/database'], - to: '/cloud/concepts/database', - }, - { - from: ['/cloud/reference/personal-access-tokens'], - to: '/cloud/concepts/personal-access-tokens', - }, - { - from: ['/cloud/reference/deployment/assets'], - to: '/cloud/guides/ship-non-dart-files', - }, - { - from: ['/cloud/reference/deployment/deployment-hooks'], - to: '/cloud/concepts/deployment-hooks', - }, - { - from: ['/cloud/reference/deployment/github-automation'], - to: '/cloud/guides/deploy-from-ci-with-github-actions', - }, - { - from: ['/cloud/reference/deployment/handling-private-dependencies'], - to: '/cloud/reference/private-dependencies', - }, - { - from: ['/cloud/reference/deployment/dart-sdk-versions'], - to: '/cloud/reference/dart-sdk-versions', - }, - { - from: ['/cloud/reference/project-id'], - to: '/cloud/reference/project-id-rules', - }, - ], + redirects: require('./redirects'), }, ], ], diff --git a/package-lock.json b/package-lock.json index 0fd9fa23..59f7ac2f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,12 @@ "prism-react-renderer": "^2.4.1", "react": "^19.2.8", "react-dom": "^19.2.8", - "react-player": "^3.4.0" + "react-player": "^3.4.0", + "remark-gfm": "^4.0.1", + "remark-mdx": "^3.1.1", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.5" }, "devDependencies": { "docusaurus-plugin-snipsync": "^1.0.0", @@ -16910,9 +16915,10 @@ } }, "node_modules/remark-mdx": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/remark-mdx/-/remark-mdx-3.0.0.tgz", - "integrity": "sha512-O7yfjuC6ra3NHPbRVxfflafAj3LTwx3b73aBvkEFU5z4PsD6FD4vrqJAkE5iNGLz71GdjXfgRqm3SQ0h0VuE7g==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/remark-mdx/-/remark-mdx-3.1.1.tgz", + "integrity": "sha512-Pjj2IYlUY3+D8x00UJsIOg5BEvfMyeI+2uLPn9VO9Wg4MEtN/VTIq2NEJQfde9PnX15KgtHyl9S0BcTnWrIuWg==", + "license": "MIT", "dependencies": { "mdast-util-mdx": "^3.0.0", "micromark-extension-mdxjs": "^3.0.0" @@ -16926,6 +16932,7 @@ "version": "11.0.0", "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "license": "MIT", "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-from-markdown": "^2.0.0", @@ -18561,9 +18568,10 @@ } }, "node_modules/unified": { - "version": "11.0.4", - "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.4.tgz", - "integrity": "sha512-apMPnyLjAX+ty4OrNap7yumyVAMlKx5IWU2wlzzUdYJO9A8f1p9m/gywF/GM2ZDFcjQPrx59Mc90KwmxsoklxQ==", + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "license": "MIT", "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", @@ -18655,9 +18663,10 @@ } }, "node_modules/unist-util-visit": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.0.0.tgz", - "integrity": "sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", + "integrity": "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==", + "license": "MIT", "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0", diff --git a/package.json b/package.json index ebbc72f7..899abb6c 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,12 @@ "prism-react-renderer": "^2.4.1", "react": "^19.2.8", "react-dom": "^19.2.8", - "react-player": "^3.4.0" + "react-player": "^3.4.0", + "remark-gfm": "^4.0.1", + "remark-mdx": "^3.1.1", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.5" }, "engines": { "node": ">=18.0" diff --git a/plugins/markdown-export/flatten.js b/plugins/markdown-export/flatten.js new file mode 100644 index 00000000..b4fdcc4b --- /dev/null +++ b/plugins/markdown-export/flatten.js @@ -0,0 +1,397 @@ +// Flattens a doc's MDX/markdown source into clean, self-contained markdown: +// partial imports inlined, Tabs unwrapped, video embeds turned into links, +// snipsync markers and HTML comments stripped, and links rewritten to +// absolute URLs. +// +// Front matter and titles go through @docusaurus/utils so this pipeline +// parses sources exactly the way the site build does. The remark toolchain +// is ESM-only, so it is loaded through dynamic import from this CommonJS +// module. + +const fs = require('fs'); +const path = require('path'); +const { + DEFAULT_PARSE_FRONT_MATTER, + escapeMarkdownHeadingIds, + parseMarkdownContentTitle, +} = require('@docusaurus/utils'); + +const urls = require('./urls'); + +let toolchainPromise; +function toolchain() { + if (!toolchainPromise) { + toolchainPromise = (async () => { + const [ + {unified}, + {default: remarkParse}, + {default: remarkStringify}, + {default: remarkMdx}, + {default: remarkGfm}, + ] = await Promise.all([ + import('unified'), + import('remark-parse'), + import('remark-stringify'), + import('remark-mdx'), + import('remark-gfm'), + ]); + return { + mdxParser: unified().use(remarkParse).use(remarkMdx).use(remarkGfm), + mdParser: unified().use(remarkParse).use(remarkGfm), + stringifier: unified().use(remarkGfm).use(remarkStringify, { + bullet: '-', + fences: true, + resourceLink: true, + }), + }; + })(); + } + return toolchainPromise; +} + +const IMPORT_RE = /import\s+(\w+)\s+from\s+['"]([^'"]+)['"]/g; + +/** + * Trim whitespace at the edges of a node list, so unwrapped JSX like + * ` Click…` does not serialize an escaped leading space. + */ +function trimTextEdges(nodes) { + const first = nodes[0]; + if (first && first.type === 'text') { + first.value = first.value.replace(/^\s+/, ''); + } + const last = nodes[nodes.length - 1]; + if (last && last.type === 'text') { + last.value = last.value.replace(/\s+$/, ''); + } +} + +function jsxAttr(node, name) { + for (const attr of node.attributes || []) { + if (attr.type === 'mdxJsxAttribute' && attr.name === name && typeof attr.value === 'string') { + return attr.value; + } + } + return undefined; +} + +function textOf(node) { + if (node.type === 'text' || node.type === 'inlineCode') { + return node.value; + } + return (node.children || []).map(textOf).join(''); +} + +function splitAnchor(url) { + const hash = url.indexOf('#'); + if (hash === -1) { + return { target: url, anchor: '' }; + } + return { target: url.slice(0, hash), anchor: url.slice(hash) }; +} + +function isExternal(url) { + return /^(?:[a-z][a-z0-9+.-]*:|\/\/)/i.test(url); +} + +/** Resolve an import/link source spec to an absolute file path. */ +function resolveSourcePath(spec, fileDir, siteDir) { + if (spec.startsWith('@site/')) { + return path.join(siteDir, spec.slice('@site/'.length)); + } + return path.resolve(fileDir, spec); +} + +/** + * Rewrite a file-style link (ending .md/.mdx) found in `filePath` to an + * absolute .md URL, using the source-to-permalink map. Unknown targets pass + * through and surface later as broken absolute links in the CI link check. + */ +function rewriteFileLink(url, filePath, shared) { + const { target, anchor } = splitAnchor(url); + if (!/\.(md|mdx)$/.test(target) || isExternal(target)) { + return url; + } + if (target.startsWith('/') || (target.startsWith('@') && !target.startsWith('@site/'))) { + return url; + } + const resolved = resolveSourcePath(target, path.dirname(filePath), shared.siteDir); + const permalink = shared.sourceToPermalink.get(resolved); + if (!permalink) { + return url; + } + return `${urls.mdUrl(permalink)}${anchor}`; +} + +/** + * Remove HTML comments outside fenced code blocks (the MDX loader does the + * same before compiling; this also strips the snipsync markers). Comments + * inside code fences are sample code and stay untouched. + */ +function stripHtmlComments(content) { + const out = []; + let inFence = false; + let inComment = false; + for (let line of content.split('\n')) { + if (inFence) { + out.push(line); + if (/^\s*(```|~~~)/.test(line)) { + inFence = false; + } + continue; + } + if (inComment) { + const close = line.indexOf('-->'); + if (close === -1) { + continue; + } + line = line.slice(close + 3); + inComment = false; + } + line = line.replace(//g, ''); + const open = line.indexOf('