From 73aa1376ce30692457926bcda41aaa8c0175fd1b Mon Sep 17 00:00:00 2001 From: kzoeps Date: Thu, 11 Jun 2026 15:08:00 +0600 Subject: [PATCH 01/16] docs: add remote markdown and mermaid rendering --- components/CopyRawButton.js | 10 +- components/Layout.js | 2 +- components/MermaidDiagram.js | 116 +++ components/RemoteMarkdown.js | 274 +++++++ components/TableOfContents.js | 55 +- docs/remote-markdown.md | 33 + lib/generate-raw-pages.js | 45 +- lib/generate-search-index.js | 89 ++- markdoc/nodes/fence.markdoc.js | 20 + markdoc/tags/index.js | 2 + markdoc/tags/remote-doc.markdoc.js | 12 + package-lock.json | 1094 ++++++++++++++++++++++++++++ package.json | 1 + pages/_app.js | 5 + pages/architecture/epds.md | 332 +-------- styles/globals.css | 56 ++ 16 files changed, 1755 insertions(+), 391 deletions(-) create mode 100644 components/MermaidDiagram.js create mode 100644 components/RemoteMarkdown.js create mode 100644 docs/remote-markdown.md create mode 100644 markdoc/tags/remote-doc.markdoc.js diff --git a/components/CopyRawButton.js b/components/CopyRawButton.js index b8ec7b3..76f3b13 100644 --- a/components/CopyRawButton.js +++ b/components/CopyRawButton.js @@ -1,18 +1,22 @@ import { useState } from 'react'; import { useRouter } from 'next/router'; -function getRawUrl(currentPath) { +function getGeneratedRawUrl(currentPath) { if (currentPath === '/') return '/raw/index.md'; return `/raw${currentPath}.md`; } -export function CopyRawButton() { +/** + * Render page-level actions for copying or viewing the Markdown source for the current docs page. + * Pages that render canonical Markdown from another repository can set `rawUrl` in frontmatter so these actions use that source instead of the generated local fallback. + */ +export function CopyRawButton({ frontmatter }) { const [copied, setCopied] = useState(false); const [copyError, setCopyError] = useState(false); const [isCopying, setIsCopying] = useState(false); const router = useRouter(); const currentPath = router.asPath.split('#')[0].split('?')[0] || '/'; - const rawUrl = getRawUrl(currentPath); + const rawUrl = frontmatter?.rawUrl || getGeneratedRawUrl(currentPath); const handleCopy = async () => { setIsCopying(true); diff --git a/components/Layout.js b/components/Layout.js index a5e0dc4..39601ec 100644 --- a/components/Layout.js +++ b/components/Layout.js @@ -201,7 +201,7 @@ export default function Layout({ children, frontmatter }) {
- {frontmatter && } + {frontmatter && }
{children}
diff --git a/components/MermaidDiagram.js b/components/MermaidDiagram.js new file mode 100644 index 0000000..0809191 --- /dev/null +++ b/components/MermaidDiagram.js @@ -0,0 +1,116 @@ +import { useEffect, useMemo, useState } from 'react'; +import { CodeBlock } from './CodeBlock'; + +let mermaidModulePromise; +let diagramIdCounter = 0; + +function getMermaid() { + if (!mermaidModulePromise) { + mermaidModulePromise = import('mermaid').then((module) => { + const mermaid = module.default || module; + return mermaid; + }); + } + + return mermaidModulePromise; +} + +function getDiagramId() { + diagramIdCounter += 1; + return `mermaid-diagram-${diagramIdCounter}`; +} + +function getPreferredMermaidTheme() { + if (typeof document !== 'undefined' && document.documentElement.classList.contains('dark')) { + return 'dark'; + } + + return 'neutral'; +} + +/** + * Render a Mermaid fenced code block as an SVG diagram in the browser. + * Use this for Markdown fences with `mermaid` as the language; invalid diagrams fall back to copyable code with an actionable syntax error. + */ +export function MermaidDiagram({ chart, children }) { + const source = (chart || children || '').trim(); + const diagramId = useMemo(getDiagramId, []); + const [svg, setSvg] = useState(''); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(Boolean(source)); + const [theme, setTheme] = useState('neutral'); + + useEffect(() => { + const updateTheme = () => setTheme(getPreferredMermaidTheme()); + updateTheme(); + + const observer = new MutationObserver(updateTheme); + observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class'] }); + + return () => observer.disconnect(); + }, []); + + useEffect(() => { + if (!source) { + setLoading(false); + setError(new Error('No Mermaid source was provided. Add diagram text inside the mermaid code fence.')); + return undefined; + } + + let cancelled = false; + setLoading(true); + setError(null); + setSvg(''); + + async function renderDiagram() { + try { + const mermaid = await getMermaid(); + mermaid.initialize({ + startOnLoad: false, + securityLevel: 'strict', + theme, + }); + const result = await mermaid.render(diagramId, source); + if (cancelled) return; + setSvg(result.svg); + } catch (err) { + if (cancelled) return; + setError(err); + } finally { + if (!cancelled) setLoading(false); + } + } + + renderDiagram(); + + return () => { + cancelled = true; + }; + }, [diagramId, source, theme]); + + if (error) { + return ( +
+

+ Could not render Mermaid diagram. Check the diagram syntax in the source Markdown and reload the page. Details: {error.message} +

+ +
+ ); + } + + if (loading) { + return ( +
+ Rendering Mermaid diagram… +
+ ); + } + + return ( +
+ ); +} diff --git a/components/RemoteMarkdown.js b/components/RemoteMarkdown.js new file mode 100644 index 0000000..94b6592 --- /dev/null +++ b/components/RemoteMarkdown.js @@ -0,0 +1,274 @@ +import React, { useEffect, useMemo, useState, useContext } from 'react'; +import { useRouter } from 'next/router'; +import Markdoc from '@markdoc/markdoc'; +import tags from '../markdoc/tags'; +import { fence, heading, link } from '../markdoc/nodes'; +import { Callout } from './Callout'; +import { Columns } from './Columns'; +import { Column } from './Column'; +import { Figure } from './Figure'; +import { Heading } from './Heading'; +import { CardLink } from './CardLink'; +import { CodeBlock } from './CodeBlock'; +import { Link } from './Link'; +import { DotPattern } from './DotPattern'; +import { HeroBanner } from './HeroBanner'; +import { CardGrid } from './CardGrid'; +import { MermaidDiagram } from './MermaidDiagram'; + +const SOURCE_ORG = 'hypercerts-org'; +const RemoteMarkdownContext = React.createContext(null); + +const markdocConfig = { + tags, + nodes: { fence, heading, link }, +}; + +/** + * Convert an allowed GitHub Markdown URL into the raw URL used by the browser fetch. + * Only hypercerts-org GitHub sources are allowed so docs pages cannot become an open proxy. + */ +function resolveGitHubMarkdownSource(source) { + const url = new URL(source); + + if (url.hostname === 'github.com') { + const [, owner, repo, marker, ref, ...fileParts] = url.pathname.split('/'); + if (owner?.toLowerCase() !== SOURCE_ORG || marker !== 'blob' || !repo || !ref || fileParts.length === 0) { + throw new Error('Remote docs must use a hypercerts-org GitHub blob URL, for example https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md.'); + } + + const filePath = fileParts.join('/'); + return { + sourceUrl: url.toString(), + rawUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${filePath}`, + owner, + repo, + ref, + filePath, + }; + } + + if (url.hostname === 'raw.githubusercontent.com') { + const [, owner, repo, ref, ...fileParts] = url.pathname.split('/'); + if (owner?.toLowerCase() !== SOURCE_ORG || !repo || !ref || fileParts.length === 0) { + throw new Error('Remote docs must use a raw.githubusercontent.com URL under hypercerts-org.'); + } + + const filePath = fileParts.join('/'); + return { + sourceUrl: `https://github.com/${owner}/${repo}/blob/${ref}/${filePath}`, + rawUrl: url.toString(), + owner, + repo, + ref, + filePath, + }; + } + + throw new Error('Remote docs can only be loaded from github.com or raw.githubusercontent.com.'); +} + +/** + * Remove optional YAML frontmatter before parsing remote Markdown with Markdoc. + */ +function stripFrontmatter(markdown) { + if (!markdown.startsWith('---\n')) return markdown; + + const end = markdown.indexOf('\n---\n', 4); + if (end === -1) return markdown; + + return markdown.slice(end + 5).trimStart(); +} + +/** + * Resolve relative links in remotely-rendered docs back to the source GitHub repo. + */ +function resolveRemoteHref(href, source) { + if (!href || href.startsWith('#') || href.startsWith('/') || /^[a-z][a-z0-9+.-]*:/i.test(href)) { + return href; + } + + const sourceDir = source.filePath.split('/').slice(0, -1).join('/'); + const basePath = `/${source.owner}/${source.repo}/blob/${source.ref}/${sourceDir ? `${sourceDir}/` : ''}`; + const resolved = new URL(href, `https://github.com${basePath}`); + const [, owner, repo, , ref, ...repoPathParts] = resolved.pathname.split('/'); + const repoPath = repoPathParts.join('/'); + const lastSegment = repoPathParts[repoPathParts.length - 1] || ''; + const mode = /\.[a-z0-9]+$/i.test(lastSegment) ? 'blob' : 'tree'; + + return `https://github.com/${owner}/${repo}/${mode}/${ref}/${repoPath}${resolved.hash}`; +} + +function RemoteLink({ href, children, ...props }) { + const source = useContext(RemoteMarkdownContext); + const resolvedHref = source ? resolveRemoteHref(href, source) : href; + + return ( + + {children} + + ); +} + +const remoteMarkdocComponents = { + Callout, + Columns, + Column, + Figure, + Heading, + CardLink, + CodeBlock, + Fence: CodeBlock, + Link: RemoteLink, + DotPattern, + HeroBanner, + CardGrid, + MermaidDiagram, +}; + +function getRawCacheUrl(currentPath) { + if (currentPath === '/') return '/raw/index.md'; + return `/raw${currentPath.replace(/\.html$/, '')}.md`; +} + +async function fetchMarkdown(url, signal, errorPrefix) { + const response = await fetch(url, { + cache: 'no-store', + signal, + }); + + if (!response.ok) { + throw new Error(`${errorPrefix} returned ${response.status} ${response.statusText || 'without the Markdown file'}.`); + } + + return response.text(); +} + +function transformMarkdown(markdown) { + const ast = Markdoc.parse(stripFrontmatter(markdown)); + return Markdoc.transform(ast, markdocConfig); +} + +/** + * Runtime Markdoc renderer for Markdown files that live in Hypercerts service repos. + * It tries the live GitHub raw source first, then the build-time `/raw` cache, then the wrapped local fallback. + */ +export function RemoteMarkdown({ source, children }) { + const router = useRouter(); + const currentPath = router.asPath.split('#')[0].split('?')[0] || '/'; + const rawCacheUrl = getRawCacheUrl(currentPath); + const [markdown, setMarkdown] = useState(null); + const [error, setError] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const sourceState = useMemo(() => { + try { + return { sourceInfo: resolveGitHubMarkdownSource(source), sourceError: null }; + } catch (err) { + return { sourceInfo: null, sourceError: err }; + } + }, [source]); + const { sourceInfo, sourceError } = sourceState; + + useEffect(() => { + if (!sourceInfo) { + setIsLoading(false); + return undefined; + } + + const controller = new AbortController(); + setMarkdown(null); + setError(null); + setIsLoading(true); + + async function loadRemoteMarkdown() { + let githubError = null; + + try { + const nextMarkdown = await fetchMarkdown(sourceInfo.rawUrl, controller.signal, 'GitHub'); + transformMarkdown(nextMarkdown); + setMarkdown(nextMarkdown); + return; + } catch (err) { + if (err.name === 'AbortError') return; + githubError = err; + } + + try { + const cachedMarkdown = await fetchMarkdown(rawCacheUrl, controller.signal, 'Build-time raw cache'); + transformMarkdown(cachedMarkdown); + setMarkdown(cachedMarkdown); + } catch (err) { + if (err.name === 'AbortError') return; + setError(new Error(`${githubError.message} The build-time raw cache also failed: ${err.message}`)); + } finally { + setIsLoading(false); + } + } + + loadRemoteMarkdown().finally(() => { + if (!controller.signal.aborted) setIsLoading(false); + }); + + return () => controller.abort(); + }, [rawCacheUrl, sourceInfo]); + + const renderedState = useMemo(() => { + if (!markdown || !sourceInfo) return { renderedContent: null, renderError: null }; + + try { + const content = transformMarkdown(markdown); + return { + renderedContent: Markdoc.renderers.react(content, React, { + components: remoteMarkdocComponents, + }), + renderError: null, + }; + } catch (err) { + return { renderedContent: null, renderError: err }; + } + }, [markdown, sourceInfo]); + const { renderedContent, renderError } = renderedState; + const displayError = error || sourceError || renderError; + + useEffect(() => { + if (!renderedContent) return undefined; + + const frame = window.requestAnimationFrame(() => { + window.dispatchEvent(new CustomEvent('remote-docs:loaded')); + }); + + return () => window.cancelAnimationFrame(frame); + }, [renderedContent]); + + if (renderedContent) { + return ( + + {renderedContent} + + ); + } + + if (isLoading && !displayError) { + return ( +
+ Loading the canonical docs from GitHub… +
+ ); + } + + return ( + <> +
+ Could not load the canonical docs.{' '} + The browser could not load the live GitHub raw file or the build-time raw cache. Showing the local fallback below. Try refreshing, or edit{' '} + {sourceInfo ? ( + the source file + ) : ( + 'the configured source URL' + )}{' '} + if the URL is wrong. Details: {displayError?.message || 'No renderable Markdown source was available.'} +
+ {children} + + ); +} diff --git a/components/TableOfContents.js b/components/TableOfContents.js index 21f78ba..71ca2ac 100644 --- a/components/TableOfContents.js +++ b/components/TableOfContents.js @@ -8,31 +8,44 @@ export function TableOfContents() { const router = useRouter(); const currentPath = router.asPath.split("#")[0].split("?")[0]; - // Extract H2 and H3 headings from the DOM after render + // Extract H2 and H3 headings from the DOM after render and after runtime docs load. useEffect(() => { - if (typeof window === "undefined") return; + if (typeof window === "undefined") return undefined; const article = document.querySelector(".layout-content article"); if (!article) { setHeadings([]); - return; + return undefined; } - const elements = article.querySelectorAll("h2, h3"); - const items = Array.from(elements).map((el) => { - if (!el.id) { - el.id = el.textContent - .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") - .replace(/(^-|-$)/g, ""); - } - return { - id: el.id, - text: el.textContent, - level: el.tagName === "H3" ? 3 : 2, - }; - }); - setHeadings(items); - setActiveId(""); + const collectHeadings = () => { + const elements = article.querySelectorAll("h2, h3, h4"); + const items = Array.from(elements).map((el) => { + if (!el.id) { + el.id = el.textContent + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/(^-|-$)/g, ""); + } + return { + id: el.id, + text: el.textContent, + level: Number(el.tagName.slice(1)), + }; + }); + setHeadings(items); + setActiveId(""); + }; + + collectHeadings(); + + const observer = new MutationObserver(collectHeadings); + observer.observe(article, { childList: true, subtree: true }); + window.addEventListener("remote-docs:loaded", collectHeadings); + + return () => { + observer.disconnect(); + window.removeEventListener("remote-docs:loaded", collectHeadings); + }; }, [currentPath]); // Scroll spy using scroll position @@ -94,8 +107,8 @@ export function TableOfContents() { { e.preventDefault(); const target = document.getElementById(id); diff --git a/docs/remote-markdown.md b/docs/remote-markdown.md new file mode 100644 index 0000000..1139217 --- /dev/null +++ b/docs/remote-markdown.md @@ -0,0 +1,33 @@ +# Runtime remote Markdown + +Use the `{% remote-doc %}` Markdoc tag when a page should render canonical Markdown from a Hypercerts service repository without copying that Markdown into this documentation repo. + +```md +--- +rawUrl: "https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/tutorial.md" +--- + +{% remote-doc source="https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md" %} + +A short unavailable-state fallback goes here. Keep it brief so the documentation repo does not become a second source of truth. + +{% /remote-doc %} +``` + +## How it works + +- The site remains a static Next.js export. +- The browser fetches the source file from `raw.githubusercontent.com` at runtime first. +- If the live GitHub fetch fails, the browser falls back to the build-time copy in `/raw`. +- The fetched Markdown is parsed with the same Markdoc tags and nodes used by local pages. +- Fenced `mermaid` diagrams render as SVGs in the browser through the Mermaid npm package. +- Relative links in the remote Markdown point back to the source GitHub repository. +- `Copy raw` and `View raw` use the page's `rawUrl` frontmatter when it is set. +- The wrapped local Markdown is the last-resort fallback content. Keep it to a short unavailable-state message, not a copy of the canonical docs. + +## Constraints + +- Sources must be in `hypercerts-org` GitHub repositories. +- The current implementation is browser-runtime fetching, not server-side rendering. +- Build-time search and raw-page generation fetch `rawUrl`, so search and `/raw` use the canonical Markdown instead of the local fallback. +- If runtime docs become permanent for many pages, add build-time indexing for the remote sources too. diff --git a/lib/generate-raw-pages.js b/lib/generate-raw-pages.js index bda1cbc..a3ed888 100644 --- a/lib/generate-raw-pages.js +++ b/lib/generate-raw-pages.js @@ -33,15 +33,44 @@ function getRawOutputPath(filePath) { return join(OUTPUT_DIR, outputRel); } -rmSync(OUTPUT_DIR, { recursive: true, force: true }); -mkdirSync(OUTPUT_DIR, { recursive: true }); +function getFrontmatterRawUrl(markdown) { + const match = markdown.match(/^---\n([\s\S]*?)\n---\n/); + if (!match) return null; -const files = walkDir(PAGES_DIR); + const rawUrlMatch = match[1].match(/^rawUrl:\s*["']?([^"'\n]+)["']?\s*$/m); + return rawUrlMatch?.[1] || null; +} + +async function getRawMarkdown(file) { + const localMarkdown = readFileSync(file, 'utf-8'); + const rawUrl = getFrontmatterRawUrl(localMarkdown); + + if (!rawUrl) return localMarkdown; + + const response = await fetch(rawUrl, { cache: 'no-store' }); + if (!response.ok) { + throw new Error(`Failed to fetch rawUrl for ${relative(PAGES_DIR, file)}: ${rawUrl} returned ${response.status} ${response.statusText || ''}`.trim()); + } + + return response.text(); +} + +async function main() { + rmSync(OUTPUT_DIR, { recursive: true, force: true }); + mkdirSync(OUTPUT_DIR, { recursive: true }); + + const files = walkDir(PAGES_DIR); + + for (const file of files) { + const outputPath = getRawOutputPath(file); + mkdirSync(dirname(outputPath), { recursive: true }); + writeFileSync(outputPath, await getRawMarkdown(file)); + } -for (const file of files) { - const outputPath = getRawOutputPath(file); - mkdirSync(dirname(outputPath), { recursive: true }); - writeFileSync(outputPath, readFileSync(file, 'utf-8')); + console.log(`Generated raw markdown files for ${files.length} pages`); } -console.log(`Generated raw markdown files for ${files.length} pages`); +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/lib/generate-search-index.js b/lib/generate-search-index.js index e6afca6..45377b2 100644 --- a/lib/generate-search-index.js +++ b/lib/generate-search-index.js @@ -18,17 +18,23 @@ function walkDir(dir) { return results; } +function stripYamlValue(value) { + return value.trim().replace(/^['"]|['"]$/g, ""); +} + function extractFrontmatter(content) { const fmMatch = content.match(/^---\n([\s\S]*?)\n---/); - if (!fmMatch) return { title: "", description: "" }; + if (!fmMatch) return { title: "", description: "", rawUrl: "" }; const frontmatter = fmMatch[1]; const titleMatch = frontmatter.match(/^title:\s*(.+)$/m); const descMatch = frontmatter.match(/^description:\s*(.+)$/m); + const rawUrlMatch = frontmatter.match(/^rawUrl:\s*(.+)$/m); return { - title: titleMatch ? titleMatch[1].trim() : "", - description: descMatch ? descMatch[1].trim() : "", + title: titleMatch ? stripYamlValue(titleMatch[1]) : "", + description: descMatch ? stripYamlValue(descMatch[1]) : "", + rawUrl: rawUrlMatch ? stripYamlValue(rawUrlMatch[1]) : "", }; } @@ -100,40 +106,59 @@ function getSection(path) { return "Other"; } -const files = walkDir(PAGES_DIR); -const index = []; +async function getIndexContent(file, localContent, rawUrl) { + if (!rawUrl) return localContent; -for (const file of files) { - const content = readFileSync(file, "utf-8"); - const rel = "/" + relative(PAGES_DIR, file).replace(/\.md$/, ""); - const path = rel === "/index" ? "/" : rel; + const response = await fetch(rawUrl, { cache: "no-store" }); + if (!response.ok) { + throw new Error(`Failed to fetch rawUrl for search index ${relative(PAGES_DIR, file)}: ${rawUrl} returned ${response.status} ${response.statusText || ""}`.trim()); + } - const { title, description } = extractFrontmatter(content); - const headings = extractHeadings(content); - const section = getSection(path); + return response.text(); +} - // For the home page, only include title (body is mostly card markup) - let body = ""; - if (path !== "/") { - body = stripMarkdown(content); - if (body.length > MAX_BODY_LENGTH) { - body = body.substring(0, MAX_BODY_LENGTH); +async function main() { + const files = walkDir(PAGES_DIR); + const index = []; + + for (const file of files) { + const localContent = readFileSync(file, "utf-8"); + const rel = "/" + relative(PAGES_DIR, file).replace(/\.md$/, ""); + const path = rel === "/index" ? "/" : rel; + + const { title, description, rawUrl } = extractFrontmatter(localContent); + const content = await getIndexContent(file, localContent, rawUrl); + const headings = extractHeadings(content); + const section = getSection(path); + + // For the home page, only include title (body is mostly card markup) + let body = ""; + if (path !== "/") { + body = stripMarkdown(content); + if (body.length > MAX_BODY_LENGTH) { + body = body.substring(0, MAX_BODY_LENGTH); + } } + + index.push({ + path, + title, + description: description || "", + section, + headings, + body, + }); } - index.push({ - path, - title, - description: description || "", - section, - headings, - body, - }); + writeFileSync(OUTPUT, JSON.stringify(index, null, 2) + "\n"); + console.log( + `Generated search index for ${index.length} pages (${ + Buffer.byteLength(JSON.stringify(index)) / 1024 + } KB)` + ); } -writeFileSync(OUTPUT, JSON.stringify(index, null, 2) + "\n"); -console.log( - `Generated search index for ${index.length} pages (${ - Buffer.byteLength(JSON.stringify(index)) / 1024 - } KB)` -); +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/markdoc/nodes/fence.markdoc.js b/markdoc/nodes/fence.markdoc.js index f154ba3..8595669 100644 --- a/markdoc/nodes/fence.markdoc.js +++ b/markdoc/nodes/fence.markdoc.js @@ -1,3 +1,12 @@ +import { Tag } from "@markdoc/markdoc"; + +function getFenceLanguage(language) { + return String(language || "") + .trim() + .toLowerCase() + .split(/\s+/)[0]; +} + const fence = { render: "CodeBlock", attributes: { @@ -14,6 +23,17 @@ const fence = { default: true, }, }, + transform(node, config) { + const attributes = node.transformAttributes(config); + + if (getFenceLanguage(attributes.language) === "mermaid") { + return new Tag("MermaidDiagram", { + chart: attributes.content, + }); + } + + return new Tag("CodeBlock", attributes); + }, }; export default fence; diff --git a/markdoc/tags/index.js b/markdoc/tags/index.js index c494cae..00e6429 100644 --- a/markdoc/tags/index.js +++ b/markdoc/tags/index.js @@ -5,6 +5,7 @@ import figure from './figure.markdoc'; import cardLink from './card-link.markdoc'; import cardGrid from './card-grid.markdoc'; import heroBanner from './hero-banner.markdoc'; +import remoteDoc from './remote-doc.markdoc'; export default { callout, @@ -14,4 +15,5 @@ export default { 'card-link': cardLink, 'card-grid': cardGrid, 'hero-banner': heroBanner, + 'remote-doc': remoteDoc, }; diff --git a/markdoc/tags/remote-doc.markdoc.js b/markdoc/tags/remote-doc.markdoc.js new file mode 100644 index 0000000..630d2d7 --- /dev/null +++ b/markdoc/tags/remote-doc.markdoc.js @@ -0,0 +1,12 @@ +/** + * Markdoc tag for rendering canonical Markdown from a Hypercerts GitHub repo at browser runtime. + */ +module.exports = { + render: 'RemoteMarkdown', + attributes: { + source: { + type: String, + required: true, + }, + }, +}; diff --git a/package-lock.json b/package-lock.json index ceb1af4..6bc12aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,12 +12,38 @@ "@markdoc/next.js": "^0.5.0", "@vercel/analytics": "^1.6.1", "flexsearch": "^0.8.212", + "mermaid": "^11.15.0", "next": "^16.1.6", "prism-react-renderer": "^2.4.1", "react": "^19.2.4", "react-dom": "^19.2.4" } }, + "node_modules/@antfu/install-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-1.1.0.tgz", + "integrity": "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==", + "license": "MIT", + "dependencies": { + "package-manager-detector": "^1.3.0", + "tinyexec": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@braintree/sanitize-url": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-7.1.2.tgz", + "integrity": "sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA==", + "license": "MIT" + }, + "node_modules/@chevrotain/types": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/@chevrotain/types/-/types-11.1.2.tgz", + "integrity": "sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw==", + "license": "Apache-2.0" + }, "node_modules/@emnapi/runtime": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", @@ -28,6 +54,23 @@ "tslib": "^2.4.0" } }, + "node_modules/@iconify/types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@iconify/types/-/types-2.0.0.tgz", + "integrity": "sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==", + "license": "MIT" + }, + "node_modules/@iconify/utils": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@iconify/utils/-/utils-3.1.3.tgz", + "integrity": "sha512-LPKOXPn/zV+zis1oOfGWogaXVpqUybF3ZS6SCZIsz8vg0ivVp9+fVqyYB7xq0aiST/VhUQYGO1qo6uoYSiEJqw==", + "license": "MIT", + "dependencies": { + "@antfu/install-pkg": "^1.1.0", + "@iconify/types": "^2.0.0", + "import-meta-resolve": "^4.2.0" + } + }, "node_modules/@img/colour": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", @@ -533,6 +576,15 @@ "react": "*" } }, + "node_modules/@mermaid-js/parser": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@mermaid-js/parser/-/parser-1.1.1.tgz", + "integrity": "sha512-VuHdsYMK1bT6X2JbcAaWAhugTRvRBRyuZgd+c22swUeI9g/ntaxF7CY7dYarhZovofCbUNO0G7JesfmNtjYOCw==", + "license": "MIT", + "dependencies": { + "@chevrotain/types": "~11.1.1" + } + }, "node_modules/@next/env": { "version": "16.1.6", "resolved": "https://registry.npmjs.org/@next/env/-/env-16.1.6.tgz", @@ -676,6 +728,265 @@ "tslib": "^2.8.0" } }, + "node_modules/@types/d3": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", + "integrity": "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/d3-axis": "*", + "@types/d3-brush": "*", + "@types/d3-chord": "*", + "@types/d3-color": "*", + "@types/d3-contour": "*", + "@types/d3-delaunay": "*", + "@types/d3-dispatch": "*", + "@types/d3-drag": "*", + "@types/d3-dsv": "*", + "@types/d3-ease": "*", + "@types/d3-fetch": "*", + "@types/d3-force": "*", + "@types/d3-format": "*", + "@types/d3-geo": "*", + "@types/d3-hierarchy": "*", + "@types/d3-interpolate": "*", + "@types/d3-path": "*", + "@types/d3-polygon": "*", + "@types/d3-quadtree": "*", + "@types/d3-random": "*", + "@types/d3-scale": "*", + "@types/d3-scale-chromatic": "*", + "@types/d3-selection": "*", + "@types/d3-shape": "*", + "@types/d3-time": "*", + "@types/d3-time-format": "*", + "@types/d3-timer": "*", + "@types/d3-transition": "*", + "@types/d3-zoom": "*" + } + }, + "node_modules/@types/d3-array": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz", + "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", + "license": "MIT" + }, + "node_modules/@types/d3-axis": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.6.tgz", + "integrity": "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-brush": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.6.tgz", + "integrity": "sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-chord": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.6.tgz", + "integrity": "sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg==", + "license": "MIT" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "license": "MIT" + }, + "node_modules/@types/d3-contour": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.6.tgz", + "integrity": "sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw==", + "license": "MIT" + }, + "node_modules/@types/d3-dispatch": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.7.tgz", + "integrity": "sha512-5o9OIAdKkhN1QItV2oqaE5KMIiXAvDWBDPrD85e58Qlz1c1kI/J0NcqbEG88CoTwJrYe7ntUCVfeUl2UJKbWgA==", + "license": "MIT" + }, + "node_modules/@types/d3-drag": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.7.tgz", + "integrity": "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-dsv": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.7.tgz", + "integrity": "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==", + "license": "MIT" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "license": "MIT" + }, + "node_modules/@types/d3-fetch": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.7.tgz", + "integrity": "sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA==", + "license": "MIT", + "dependencies": { + "@types/d3-dsv": "*" + } + }, + "node_modules/@types/d3-force": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.10.tgz", + "integrity": "sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw==", + "license": "MIT" + }, + "node_modules/@types/d3-format": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.4.tgz", + "integrity": "sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g==", + "license": "MIT" + }, + "node_modules/@types/d3-geo": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.1.0.tgz", + "integrity": "sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ==", + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-hierarchy": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.7.tgz", + "integrity": "sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg==", + "license": "MIT" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "license": "MIT", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz", + "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", + "license": "MIT" + }, + "node_modules/@types/d3-polygon": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.2.tgz", + "integrity": "sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA==", + "license": "MIT" + }, + "node_modules/@types/d3-quadtree": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.6.tgz", + "integrity": "sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg==", + "license": "MIT" + }, + "node_modules/@types/d3-random": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.3.tgz", + "integrity": "sha512-Imagg1vJ3y76Y2ea0871wpabqp613+8/r0mCLEBfdtqC7xMSfj9idOnmBYyMoULfHePJyxMAw3nWhJxzc+LFwQ==", + "license": "MIT" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz", + "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", + "license": "MIT", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-iWMJgwkK7yTRmWqRB5plb1kadXyQ5Sj8V/zYlFGMUBbIPKQScw+Dku9cAAMgJG+z5GYDoMjWGLVOvjghDEFnKQ==", + "license": "MIT" + }, + "node_modules/@types/d3-selection": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.11.tgz", + "integrity": "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==", + "license": "MIT" + }, + "node_modules/@types/d3-shape": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.8.tgz", + "integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==", + "license": "MIT", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz", + "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", + "license": "MIT" + }, + "node_modules/@types/d3-time-format": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.3.tgz", + "integrity": "sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg==", + "license": "MIT" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "license": "MIT" + }, + "node_modules/@types/d3-transition": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.9.tgz", + "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-zoom": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.8.tgz", + "integrity": "sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==", + "license": "MIT", + "dependencies": { + "@types/d3-interpolate": "*", + "@types/d3-selection": "*" + } + }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "license": "MIT" + }, "node_modules/@types/linkify-it": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-3.0.5.tgz", @@ -707,6 +1018,23 @@ "integrity": "sha512-AUZTa7hQ2KY5L7AmtSiqxlhWxb4ina0yd8hNbl4TWuqnv/pFP0nDMb3YrfSBf4hJVGLh2YEIBfKaBW/9UEl6IQ==", "license": "MIT" }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT", + "optional": true + }, + "node_modules/@upsetjs/venn.js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@upsetjs/venn.js/-/venn.js-2.0.0.tgz", + "integrity": "sha512-WbBhLrooyePuQ1VZxrJjtLvTc4NVfpOyKx0sKqioq9bX1C1m7Jgykkn8gLrtwumBioXIqam8DLxp88Adbue6Hw==", + "license": "MIT", + "optionalDependencies": { + "d3-selection": "^3.0.0", + "d3-transition": "^3.0.1" + } + }, "node_modules/@vercel/analytics": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-1.6.1.tgz", @@ -795,6 +1123,538 @@ "node": ">=6" } }, + "node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/cose-base": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-1.0.3.tgz", + "integrity": "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==", + "license": "MIT", + "dependencies": { + "layout-base": "^1.0.0" + } + }, + "node_modules/cytoscape": { + "version": "3.34.0", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.34.0.tgz", + "integrity": "sha512-62rNSrioXw93uliKFBwjukeQyeWwH2PqDrTac31r2P6464u3AUvTk0xS4LVvT251g7IgkFunrI48ZEZGjywSOg==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/cytoscape-cose-bilkent": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cytoscape-cose-bilkent/-/cytoscape-cose-bilkent-4.1.0.tgz", + "integrity": "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^1.0.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cytoscape-fcose/-/cytoscape-fcose-2.2.0.tgz", + "integrity": "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^2.2.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/cose-base": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-2.2.0.tgz", + "integrity": "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==", + "license": "MIT", + "dependencies": { + "layout-base": "^2.0.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/layout-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz", + "integrity": "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==", + "license": "MIT" + }, + "node_modules/d3": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", + "integrity": "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==", + "license": "ISC", + "dependencies": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "license": "ISC", + "dependencies": { + "d3-path": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "license": "ISC", + "dependencies": { + "d3-array": "^3.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "license": "ISC", + "dependencies": { + "delaunator": "5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "license": "ISC", + "dependencies": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json.js", + "csv2tsv": "bin/dsv2dsv.js", + "dsv2dsv": "bin/dsv2dsv.js", + "dsv2json": "bin/dsv2json.js", + "json2csv": "bin/json2dsv.js", + "json2dsv": "bin/json2dsv.js", + "json2tsv": "bin/json2dsv.js", + "tsv2csv": "bin/dsv2dsv.js", + "tsv2json": "bin/dsv2json.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-fetch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", + "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", + "license": "ISC", + "dependencies": { + "d3-dsv": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", + "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2.5.0 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "license": "BSD-3-Clause", + "dependencies": { + "internmap": "^1.0.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-sankey/node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-path": "1" + } + }, + "node_modules/d3-sankey/node_modules/internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==", + "license": "ISC" + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/dagre-d3-es": { + "version": "7.0.14", + "resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.14.tgz", + "integrity": "sha512-P4rFMVq9ESWqmOgK+dlXvOtLwYg0i7u0HBGJER0LZDJT2VHIPAMZ/riPxqJceWMStH5+E61QxFra9kIS3AqdMg==", + "license": "MIT", + "dependencies": { + "d3": "^7.9.0", + "lodash-es": "^4.17.21" + } + }, + "node_modules/dayjs": { + "version": "1.11.21", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.21.tgz", + "integrity": "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==", + "license": "MIT" + }, + "node_modules/delaunator": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.1.0.tgz", + "integrity": "sha512-AGrQ4QSgssa1NGmWmLPqN5NY2KajF5MqxetNEO+o0n3ZwZZeTmt7bBnvzHWrmkZFxGgr4HdyFgelzgi06otLuQ==", + "license": "ISC", + "dependencies": { + "robust-predicates": "^3.0.2" + } + }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -805,6 +1665,25 @@ "node": ">=8" } }, + "node_modules/dompurify": { + "version": "3.4.9", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.9.tgz", + "integrity": "sha512-4dPSRMRDqHvs0V4YDFCsaIZo4if5u0xM+llyxiM2fwuZFdKArUBAF3VtI2+n8NKg9P870WMdYk0UhqQNoWXbfQ==", + "license": "(MPL-2.0 OR Apache-2.0)", + "optionalDependencies": { + "@types/trusted-types": "^2.0.7" + } + }, + "node_modules/es-toolkit": { + "version": "1.47.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.47.0.tgz", + "integrity": "sha512-n1GuoD0WEQZMBk5tttoZSqwgyLx01oqa5XsBmCHwPyNe1S9jPBEmtR2pSgp2kJuWE3ciFZ6yRHmY4pM4C3OOkw==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks" + ] + }, "node_modules/flexsearch": { "version": "0.8.212", "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.8.212.tgz", @@ -833,6 +1712,43 @@ ], "license": "Apache-2.0" }, + "node_modules/hachure-fill": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/hachure-fill/-/hachure-fill-0.5.2.tgz", + "integrity": "sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==", + "license": "MIT" + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/import-meta-resolve": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz", + "integrity": "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -845,6 +1761,89 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/katex": { + "version": "0.16.47", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.47.tgz", + "integrity": "sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg==", + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, + "node_modules/katex/node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/khroma": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/khroma/-/khroma-2.1.0.tgz", + "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==" + }, + "node_modules/layout-base": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-1.0.2.tgz", + "integrity": "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==", + "license": "MIT" + }, + "node_modules/lodash-es": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", + "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", + "license": "MIT" + }, + "node_modules/marked": { + "version": "16.4.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-16.4.2.tgz", + "integrity": "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/mermaid": { + "version": "11.15.0", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.15.0.tgz", + "integrity": "sha512-pTMbcf3rWdtLiYGpmoTjHEpeY8seiy6sR+9nD7LOs8KfUbHE4lOUAprTRqRAcWSQ6MQpdX+YEsxShtGsINtPtw==", + "license": "MIT", + "dependencies": { + "@braintree/sanitize-url": "^7.1.1", + "@iconify/utils": "^3.0.2", + "@mermaid-js/parser": "^1.1.1", + "@types/d3": "^7.4.3", + "@upsetjs/venn.js": "^2.0.0", + "cytoscape": "^3.33.1", + "cytoscape-cose-bilkent": "^4.1.0", + "cytoscape-fcose": "^2.2.0", + "d3": "^7.9.0", + "d3-sankey": "^0.12.3", + "dagre-d3-es": "7.0.14", + "dayjs": "^1.11.19", + "dompurify": "^3.3.1", + "es-toolkit": "^1.45.1", + "katex": "^0.16.25", + "khroma": "^2.1.0", + "marked": "^16.3.0", + "roughjs": "^4.6.6", + "stylis": "^4.3.6", + "ts-dedent": "^2.2.0", + "uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0" + } + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -916,12 +1915,40 @@ } } }, + "node_modules/package-manager-detector": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.6.0.tgz", + "integrity": "sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA==", + "license": "MIT" + }, + "node_modules/path-data-parser": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/path-data-parser/-/path-data-parser-0.1.0.tgz", + "integrity": "sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w==", + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", "license": "ISC" }, + "node_modules/points-on-curve": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", + "integrity": "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A==", + "license": "MIT" + }, + "node_modules/points-on-path": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/points-on-path/-/points-on-path-0.2.1.tgz", + "integrity": "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g==", + "license": "MIT", + "dependencies": { + "path-data-parser": "0.1.0", + "points-on-curve": "0.2.0" + } + }, "node_modules/postcss": { "version": "8.4.31", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", @@ -984,6 +2011,36 @@ "react": "^19.2.4" } }, + "node_modules/robust-predicates": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.3.tgz", + "integrity": "sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA==", + "license": "Unlicense" + }, + "node_modules/roughjs": { + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/roughjs/-/roughjs-4.6.6.tgz", + "integrity": "sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ==", + "license": "MIT", + "dependencies": { + "hachure-fill": "^0.5.2", + "path-data-parser": "^0.1.0", + "points-on-curve": "^0.2.0", + "points-on-path": "^0.2.1" + } + }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "license": "BSD-3-Clause" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, "node_modules/scheduler": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", @@ -1080,11 +2137,48 @@ } } }, + "node_modules/stylis": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.4.0.tgz", + "integrity": "sha512-5Z9ZpRzfuH6l/UAvCPAPUo3665Nk2wLaZU3x+TLHKVzIz33+sbJqbtrYoC3KD4/uVOr2Zp+L0LySezP9OHV9yA==", + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/ts-dedent": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.3.0.tgz", + "integrity": "sha512-JfJeIHke7y2egdGGgRAvpCwYFUsHlM2gPcrVOxFkznt/4uzQ7HFmvE63iFHVLBJNDuyDOQgijDK/tXH/f6Msjg==", + "license": "MIT", + "engines": { + "node": ">=6.10" + } + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" + }, + "node_modules/uuid": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.0.tgz", + "integrity": "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } } } } diff --git a/package.json b/package.json index da60b12..8c8d29e 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "@markdoc/next.js": "^0.5.0", "@vercel/analytics": "^1.6.1", "flexsearch": "^0.8.212", + "mermaid": "^11.15.0", "next": "^16.1.6", "prism-react-renderer": "^2.4.1", "react": "^19.2.4", diff --git a/pages/_app.js b/pages/_app.js index 3422f00..6c6168f 100644 --- a/pages/_app.js +++ b/pages/_app.js @@ -11,6 +11,8 @@ import { Link } from '../components/Link'; import { DotPattern } from '../components/DotPattern'; import { HeroBanner } from '../components/HeroBanner'; import { CardGrid } from '../components/CardGrid'; +import { RemoteMarkdown } from '../components/RemoteMarkdown'; +import { MermaidDiagram } from '../components/MermaidDiagram'; import { Analytics } from '@vercel/analytics/next'; const components = { @@ -26,6 +28,9 @@ const components = { DotPattern, HeroBanner, CardGrid, + RemoteMarkdown, + RemoteDoc: RemoteMarkdown, + MermaidDiagram, }; export default function App({ Component, pageProps }) { diff --git a/pages/architecture/epds.md b/pages/architecture/epds.md index a2cfb61..f2d5159 100644 --- a/pages/architecture/epds.md +++ b/pages/architecture/epds.md @@ -1,335 +1,15 @@ --- title: ePDS (extended PDS) description: How the ePDS adds email/OTP login on top of AT Protocol without changing the standard OAuth flow for apps. +rawUrl: "https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/tutorial.md" --- -# ePDS (extended PDS) +{% remote-doc source="https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md" %} -The ePDS adds email-based, passwordless sign-in on top of a standard AT Protocol PDS. Users enter their email, receive a one-time code, and end up with a normal AT Protocol session tied to a DID. +# ePDS docs unavailable -Certified operates production, staging, and test ePDS instances. See [Certified services](/reference/certified-pdss) for the current hostnames and guidance on which to use in which scenario. +The canonical ePDS docs could not be loaded from GitHub or the build-time raw cache. -For applications, the important part is that ePDS still finishes by issuing a standard AT Protocol authorization code. In practice, this means you can integrate it with [`@atproto/oauth-client-node`](https://github.com/bluesky-social/atproto/tree/main/packages/oauth/oauth-client-node). +View the source directly: https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md -## System overview - -```text -Client App - -> starts AT Protocol OAuth against the PDS - -PDS Core - -> remains the OAuth issuer and token endpoint - -> advertises the Auth Service as the authorization endpoint - -Auth Service - -> collects the user's email or OTP - -> verifies the user - -> returns control to PDS Core via signed callback - -PDS Core - -> issues a normal authorization code - -Client App - -> exchanges the code for tokens -``` - -The PDS remains the OAuth issuer and token endpoint. The main difference is that the authorization step happens on the ePDS Auth Service, which handles the email and OTP flow before returning control to the PDS. - -## Integrating with `@atproto/oauth-client-node` - -ePDS works with the standard AT Protocol OAuth client libraries. The main ePDS-specific behavior is how you shape the authorization URL before redirecting the user. - -### Flow 1: your app collects the email - -In Flow 1, your app has its own email field. Start OAuth normally, then add `login_hint=` to the authorization URL before redirecting the user. - -```ts -import { NodeOAuthClient } from '@atproto/oauth-client-node' - -const oauthClient = new NodeOAuthClient({ - clientMetadata: { - client_id: 'https://yourapp.example.com/client-metadata.json', - client_name: 'Your App', - client_uri: 'https://yourapp.example.com', - redirect_uris: ['https://yourapp.example.com/api/oauth/callback'], - scope: 'atproto transition:generic', - grant_types: ['authorization_code', 'refresh_token'], - response_types: ['code'], - token_endpoint_auth_method: 'none', - dpop_bound_access_tokens: true, - }, - stateStore, - sessionStore, -}) - -const url = await oauthClient.authorize('alice.certified.one', { - scope: 'atproto transition:generic', -}) - -// ePDS-specific customization happens here. -const authUrl = new URL(url) -authUrl.searchParams.set('login_hint', email) -authUrl.searchParams.set('epds_handle_mode', 'picker-with-random') - -return authUrl.toString() -``` - -{% callout type="warning" %} -Do not put an email address into the PAR body as `login_hint`. For ePDS, add `login_hint` to the authorization URL instead. -{% /callout %} - -With `login_hint` set, the user lands directly on the OTP entry step instead of first seeing an email form on ePDS. - -### Flow 2: ePDS collects the email - -In Flow 2, your app just shows a "Sign in" button. Start OAuth normally and redirect the user to the authorization URL without `login_hint`. - -```ts -import { NodeOAuthClient } from '@atproto/oauth-client-node' - -const oauthClient = new NodeOAuthClient({ - clientMetadata: { - client_id: 'https://yourapp.example.com/client-metadata.json', - client_name: 'Your App', - client_uri: 'https://yourapp.example.com', - redirect_uris: ['https://yourapp.example.com/api/oauth/callback'], - scope: 'atproto transition:generic', - grant_types: ['authorization_code', 'refresh_token'], - response_types: ['code'], - token_endpoint_auth_method: 'none', - dpop_bound_access_tokens: true, - }, - stateStore, - sessionStore, -}) - -const url = await oauthClient.authorize('alice.certified.one', { - scope: 'atproto transition:generic', -}) - -const authUrl = new URL(url) -authUrl.searchParams.set('epds_handle_mode', 'picker') - -return authUrl.toString() -``` - -Without `login_hint`, ePDS renders its own email form and takes the user through the rest of the OTP flow. - -### Callback handling - -Callback handling stays standard. Once the user finishes on ePDS, your callback handler receives a normal authorization code and hands it back to `oauth-client-node`. - -```ts -const result = await oauthClient.callback(params) - -const session = result.session -const did = session.did -``` - -## Handle modes - -Handle mode controls what happens when a brand new user needs a handle during signup. - -| Mode | Behavior | -|------|----------| -| `picker-with-random` | Show the handle picker with a "Generate random" option. | -| `picker` | Show the handle picker without a random option. | -| `random` | Skip the picker and assign a random handle automatically. | - -Handle mode is resolved in this order: - -1. `epds_handle_mode` query param on the authorization URL -2. `epds_handle_mode` in client metadata -3. The ePDS instance default (`EPDS_DEFAULT_HANDLE_MODE`) - -This only affects new account creation. Existing users keep their current handle and skip this step. - -## Client metadata - -Your client metadata file is a public JSON document served over HTTPS. Its URL is also your `client_id`. - -### Bare-bones example - -```json -{ - "client_id": "https://yourapp.example.com/client-metadata.json", - "client_name": "Your App", - "client_uri": "https://yourapp.example.com", - "redirect_uris": ["https://yourapp.example.com/api/oauth/callback"], - "scope": "atproto transition:generic", - "grant_types": ["authorization_code", "refresh_token"], - "response_types": ["code"], - "token_endpoint_auth_method": "none", - "dpop_bound_access_tokens": true -} -``` - -### Full config example - -```json -{ - "client_id": "https://yourapp.example.com/client-metadata.json", - "client_name": "Your App", - "client_uri": "https://yourapp.example.com", - "logo_uri": "https://yourapp.example.com/logo.png", - "redirect_uris": ["https://yourapp.example.com/api/oauth/callback"], - "scope": "atproto transition:generic", - "grant_types": ["authorization_code", "refresh_token"], - "response_types": ["code"], - "token_endpoint_auth_method": "none", - "dpop_bound_access_tokens": true, - "brand_color": "#0f172a", - "background_color": "#ffffff", - "email_template_uri": "https://yourapp.example.com/email-template.html", - "email_subject_template": "{{code}} - Your {{app_name}} code", - "branding": { - "css": "body { background: #0f172a; color: #e2e8f0; }" - }, - "epds_handle_mode": "picker-with-random" -} -``` - -The extra branding fields customize the hosted login and email experience. `epds_handle_mode` sets your preferred handle mode for new users unless you override it on the authorization URL. - -## Branding and customization - -### How branding works - -ePDS reads branding settings from your app's `client-metadata.json`, using the OAuth `client_id` to look it up. Standard metadata fields like `logo_uri`, `brand_color`, `background_color`, `email_template_uri`, and `email_subject_template` customize the hosted login and email experience. - -Trusted clients can go further by adding custom CSS in client metadata under `branding.css`: - -```json -{ - "branding": { - "css": "body { background: #0f172a; color: #e2e8f0; }" - } -} -``` - -When the client is trusted, ePDS injects that CSS into its hosted auth pages and the stock consent page. - -{% callout type="warning" %} -Trust is checked against the exact `client_id`. - -The `client_id` you send during OAuth, the `client_id` inside `client-metadata.json`, and the entry in `PDS_OAUTH_TRUSTED_CLIENTS` must all be identical. - -For example, if your client metadata says `"client_id": "https://hypercerts-scaffold.vercel.app/client-metadata.json"`, then `PDS_OAUTH_TRUSTED_CLIENTS` must contain `https://hypercerts-scaffold.vercel.app/client-metadata.json` — not just `https://hypercerts-scaffold.vercel.app`. See the [Scaffold Starter App](/tools/scaffold) for a concrete example of a client serving metadata from `/client-metadata.json`. -{% /callout %} - -### Client metadata branding fields - -These fields are the main branding controls exposed through client metadata: - -| Field | What it affects | -|------|------------------| -| `logo_uri` | App logo shown in hosted auth and email flows | -| `brand_color` | Primary brand color used by hosted screens | -| `background_color` | Background color for hosted screens | -| `email_template_uri` | Custom HTML template for OTP emails | -| `email_subject_template` | Subject line template for OTP emails | -| `branding.css` | Custom CSS for trusted clients | - -### CSS injection for trusted clients - -Custom CSS is only applied for clients whose exact `client_id` appears in `PDS_OAUTH_TRUSTED_CLIENTS`. When present, ePDS injects a `` tag closure, and updates the page's CSP `style-src` directive with a SHA-256 hash for the injected stylesheet. - -This gives operators a safety boundary: untrusted clients never get CSS injection, even if their metadata contains branding CSS. - -### Where branding appears - -The send-OTP and initial-OTP screens are two states of the same auth-service route: `https://auth.epds1.test.certified.app/oauth/authorize`. - -| Surface | URL | Supports branding | -|---|---|---| -| Send OTP | `https://auth.epds1.test.certified.app/oauth/authorize` | Metadata fields + trusted-client CSS | -| Initial OTP | `https://auth.epds1.test.certified.app/oauth/authorize` | Metadata fields + trusted-client CSS | -| Choose handle | `https://auth.epds1.test.certified.app/auth/choose-handle` | Metadata fields + trusted-client CSS | -| Recovery | `https://auth.epds1.test.certified.app/auth/recover` | Metadata fields + trusted-client CSS | -| Consent page | `https://epds1.test.certified.app/oauth/authorize` | Trusted-client CSS | - -### Examples - -#### Send OTP - -{% columns %} -{% column %} -Stock - -![Stock send OTP screen](/images/epds/send-otp-stock.png) -{% /column %} -{% column %} -CSS injected - -![CSS-injected send OTP screen](/images/epds/send-otp-css-injected.png) -{% /column %} -{% /columns %} - -#### Initial OTP - -{% columns %} -{% column %} -Stock - -![Stock initial OTP screen](/images/epds/initial-otp-stock.png) -{% /column %} -{% column %} -CSS injected - -![CSS-injected initial OTP screen](/images/epds/initial-otp-css-injected.png) -{% /column %} -{% /columns %} - -#### Choose handle - -{% columns %} -{% column %} -Stock - -![Stock choose handle screen](/images/epds/choose-handle-stock.png) -{% /column %} -{% column %} -CSS injected - -![CSS-injected choose handle screen](/images/epds/choose-handle-css-injected.png) -{% /column %} -{% /columns %} - -#### Consent page - -{% columns %} -{% column %} -Stock - -![Stock consent page](/images/epds/consent-page-stock.png) -{% /column %} -{% column %} -CSS injected - -![CSS-injected consent page](/images/epds/consent-page-css-injected.png) -{% /column %} -{% /columns %} - -#### Recovery - -{% columns %} -{% column %} -Stock - -![Stock recovery screen](/images/epds/recovery-stock.png) -{% /column %} -{% column %} -CSS injected - -![CSS-injected recovery screen](/images/epds/recovery-css-injected.png) -{% /column %} -{% /columns %} - -## Further reading - -- [Account & Identity Setup](/architecture/account-and-identity) -- [Certified PDSs](/reference/certified-pdss) — the production, staging, and test ePDS instances Certified operates -- [Certified Group Service (CGS)](/architecture/certified-group-service) — a governance layer that sits in front of a PDS to support multi-identity, role-based repo management -- [Scaffold Starter App](/tools/scaffold) -- [ePDS repository](https://github.com/hypercerts-org/ePDS) -- Install the ePDS agent skill with `npx skills add hypercerts-org/ePDS --skill epds-login` +{% /remote-doc %} diff --git a/styles/globals.css b/styles/globals.css index 2ac886c..4c4c3c2 100644 --- a/styles/globals.css +++ b/styles/globals.css @@ -1068,6 +1068,11 @@ a.sidebar-link-active:hover { padding-left: 24px; } +.toc-link-h4 { + font-size: 12px; + padding-left: 40px; +} + /* ===== Last Updated ===== */ .page-tools { display: flex; @@ -1134,6 +1139,27 @@ a.sidebar-link-active:hover { font-style: italic; color: var(--color-text-secondary); } + +.remote-doc-status { + margin-bottom: var(--space-6); + padding: var(--space-3) var(--space-4); + border: 1px solid var(--color-border); + border-radius: var(--radius-md); + background: var(--color-bg-subtle); + color: var(--color-text-secondary); + font-size: 0.875rem; + line-height: 1.5; +} + +.remote-doc-status--error { + border-color: var(--color-warning); + background: var(--color-warning-bg); + color: var(--color-text-primary); +} + +.remote-doc-status--error strong { + color: var(--color-text-heading); +} /* ===== Pagination ===== */ .pagination { display: flex; @@ -1414,6 +1440,36 @@ a.sidebar-link-active:hover { color: #d6deeb; } +.mermaid-diagram { + margin: var(--space-6) 0; + padding: var(--space-4); + border: 1px solid var(--color-border); + border-radius: var(--radius-lg); + background: var(--color-bg-subtle); + overflow-x: auto; +} + +.mermaid-diagram svg { + display: block; + max-width: 100%; + height: auto; + margin: 0 auto; +} + +.mermaid-diagram--loading { + color: var(--color-text-secondary); + font-size: 0.875rem; +} + +.mermaid-diagram--error { + border-color: var(--color-warning); + background: var(--color-warning-bg); +} + +.mermaid-diagram--error p { + margin-bottom: var(--space-4); +} + .layout-content code { font-family: var(--font-mono); font-size: 0.875em; From 6f84154b37b6a6ad3e1e17dba062e17af1abd2d5 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Thu, 11 Jun 2026 16:23:20 +0600 Subject: [PATCH 02/16] docs-refresh: add registry-based external docs deploy check --- .github/workflows/docs-refresh-pr-dry-run.yml | 103 ++++++++ .github/workflows/docs-refresh.yml | 121 +++++++++ .gitignore | 4 + components/CopyRawButton.js | 43 +++- components/RemoteMarkdown.js | 105 ++++++-- docs-sources.yml | 8 + docs/remote-markdown.md | 55 ++++- lib/compare-docs-fingerprint.js | 46 ++++ lib/external-docs.js | 233 ++++++++++++++++++ lib/generate-docs-fingerprint.js | 159 ++++++++++++ lib/generate-external-docs-manifest.js | 27 ++ lib/generate-raw-pages.js | 28 +-- lib/generate-search-index.js | 41 ++- markdoc/tags/remote-doc.markdoc.js | 3 +- package-lock.json | 1 + package.json | 8 +- pages/architecture/epds.md | 4 +- 17 files changed, 919 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/docs-refresh-pr-dry-run.yml create mode 100644 .github/workflows/docs-refresh.yml create mode 100644 docs-sources.yml create mode 100644 lib/compare-docs-fingerprint.js create mode 100644 lib/external-docs.js create mode 100644 lib/generate-docs-fingerprint.js create mode 100644 lib/generate-external-docs-manifest.js diff --git a/.github/workflows/docs-refresh-pr-dry-run.yml b/.github/workflows/docs-refresh-pr-dry-run.yml new file mode 100644 index 0000000..4b8eb56 --- /dev/null +++ b/.github/workflows/docs-refresh-pr-dry-run.yml @@ -0,0 +1,103 @@ +name: External docs refresh dry run (temporary) + +on: + pull_request: + branches: + - main + paths: + - '.github/workflows/docs-refresh*.yml' + - 'docs-sources.yml' + - 'docs/remote-markdown.md' + - 'lib/external-docs.js' + - 'lib/generate-docs-fingerprint.js' + - 'lib/generate-external-docs-manifest.js' + - 'lib/compare-docs-fingerprint.js' + - 'package.json' + - 'package-lock.json' + +permissions: + contents: read + +concurrency: + group: external-docs-refresh-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + dry-run: + name: Compare external docs fingerprints without deploying + runs-on: ubuntu-latest + + steps: + - name: Checkout docs repo + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Validate external docs manifest generation + run: npm run generate:external-docs + + - name: Generate current docs fingerprint + run: npm run docs:fingerprint -- --output /tmp/current-docs-fingerprint.json + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: Download deployed docs fingerprint + env: + DOCS_FINGERPRINT_URL: https://docs.hypercerts.org/docs-fingerprint.json + run: | + set +e + http_code=$(curl -sS -L -w "%{http_code}" \ + -o /tmp/deployed-docs-fingerprint.json \ + "$DOCS_FINGERPRINT_URL") + curl_status=$? + set -e + + if [ "$curl_status" -ne 0 ]; then + echo "::error::Failed to download deployed docs fingerprint from $DOCS_FINGERPRINT_URL. Dry-run cannot compare against deployed state." + exit 1 + fi + + case "$http_code" in + 200) + ;; + 404) + echo '{}' > /tmp/deployed-docs-fingerprint.json + ;; + *) + echo "::error::Unexpected HTTP $http_code while downloading $DOCS_FINGERPRINT_URL. Dry-run cannot compare against deployed state." + exit 1 + ;; + esac + + - name: Compare fingerprints + id: diff + run: | + node lib/compare-docs-fingerprint.js \ + /tmp/current-docs-fingerprint.json \ + /tmp/deployed-docs-fingerprint.json >> "$GITHUB_OUTPUT" + + - name: Report dry-run result + run: | + if [ "${{ steps.diff.outputs.changed }}" = "true" ]; then + echo "::notice::External docs fingerprint differs from the deployed site. The production workflow would trigger the Vercel deploy hook." + else + echo "::notice::External docs fingerprint matches the deployed site. The production workflow would do nothing." + fi + + { + echo "### External docs refresh dry run" + echo "" + echo "This temporary PR-only workflow never calls the Vercel deploy hook." + echo "" + echo "Changed: ${{ steps.diff.outputs.changed || 'unknown' }}" + echo "Reason: ${{ steps.diff.outputs.reason || 'not computed' }}" + echo "Current: ${{ steps.diff.outputs.current_fingerprint || 'n/a' }}" + echo "Deployed: ${{ steps.diff.outputs.deployed_fingerprint || 'n/a' }}" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/docs-refresh.yml b/.github/workflows/docs-refresh.yml new file mode 100644 index 0000000..f461533 --- /dev/null +++ b/.github/workflows/docs-refresh.yml @@ -0,0 +1,121 @@ +name: Refresh external docs + +on: + schedule: + - cron: '17 * * * *' + workflow_dispatch: + inputs: + dry_run: + description: 'Compare fingerprints without posting to the Vercel deploy hook.' + required: false + default: true + type: boolean + force_deploy: + description: 'Mark the run as changed even when the deployed fingerprint matches.' + required: false + default: false + type: boolean + +permissions: + contents: read + +concurrency: + group: docs-refresh + cancel-in-progress: false + +jobs: + check-docs: + name: Check external docs fingerprint + runs-on: ubuntu-latest + + steps: + - name: Checkout docs repo + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Generate current docs fingerprint + run: npm run docs:fingerprint -- --output current-docs-fingerprint.json + env: + DOCS_SOURCE_TOKEN: ${{ secrets.DOCS_SOURCE_TOKEN }} + GITHUB_TOKEN: ${{ github.token }} + + - name: Download deployed docs fingerprint + env: + DOCS_FINGERPRINT_URL: https://docs.hypercerts.org/docs-fingerprint.json + run: | + set +e + http_code=$(curl -sS -L -w "%{http_code}" \ + -o deployed-docs-fingerprint.json \ + "$DOCS_FINGERPRINT_URL") + curl_status=$? + set -e + + if [ "$curl_status" -ne 0 ]; then + echo "::error::Failed to download deployed docs fingerprint from $DOCS_FINGERPRINT_URL. Refusing to deploy on an unknown diff." + exit 1 + fi + + case "$http_code" in + 200) + ;; + 404) + echo '{}' > deployed-docs-fingerprint.json + ;; + *) + echo "::error::Unexpected HTTP $http_code while downloading $DOCS_FINGERPRINT_URL. Refusing to deploy on an unknown diff." + exit 1 + ;; + esac + + - name: Check if external docs changed + id: diff + env: + FORCE_DEPLOY: ${{ github.event_name == 'workflow_dispatch' && inputs.force_deploy == true }} + run: | + node lib/compare-docs-fingerprint.js \ + current-docs-fingerprint.json \ + deployed-docs-fingerprint.json >> "$GITHUB_OUTPUT" + + if [ "$FORCE_DEPLOY" = "true" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "reason=forced by workflow_dispatch input" >> "$GITHUB_OUTPUT" + fi + + - name: Trigger Vercel deployment + if: steps.diff.outputs.changed == 'true' + env: + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run == true }} + VERCEL_DEPLOY_HOOK_URL: ${{ secrets.VERCEL_DEPLOY_HOOK_URL }} + run: | + if [ "$DRY_RUN" = "true" ]; then + echo "workflow_dispatch dry_run=true; skipping Vercel deploy hook." + exit 0 + fi + + if [ -z "$VERCEL_DEPLOY_HOOK_URL" ]; then + echo "::error::VERCEL_DEPLOY_HOOK_URL is required when external docs have changed. Create a Vercel Deploy Hook for the production branch and store its URL as a GitHub Actions secret." + exit 1 + fi + + curl -fsS -X POST "$VERCEL_DEPLOY_HOOK_URL" + + - name: Write summary + if: always() + run: | + { + echo "### External docs refresh" + echo "" + echo "Changed: ${{ steps.diff.outputs.changed || 'unknown' }}" + echo "Reason: ${{ steps.diff.outputs.reason || 'not computed' }}" + echo "Dry run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run == true }}" + echo "Current: ${{ steps.diff.outputs.current_fingerprint || 'n/a' }}" + echo "Deployed: ${{ steps.diff.outputs.deployed_fingerprint || 'n/a' }}" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 2c8a0f4..58647af 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,10 @@ out/ # Generated at build time by lib/generate-*.js — no need to track public/raw/ +public/external-docs.json +public/docs-fingerprint.json public/search-index.json public/sitemap.xml +current-docs-fingerprint.json +deployed-docs-fingerprint.json lib/lastUpdated.json diff --git a/components/CopyRawButton.js b/components/CopyRawButton.js index 76f3b13..aa946c1 100644 --- a/components/CopyRawButton.js +++ b/components/CopyRawButton.js @@ -1,22 +1,59 @@ -import { useState } from 'react'; +import { useEffect, useState } from 'react'; import { useRouter } from 'next/router'; +const EXTERNAL_DOCS_MANIFEST_URL = '/external-docs.json'; + function getGeneratedRawUrl(currentPath) { if (currentPath === '/') return '/raw/index.md'; return `/raw${currentPath}.md`; } +async function getExternalDocRawUrl(externalDoc, signal) { + const response = await fetch(EXTERNAL_DOCS_MANIFEST_URL, { + cache: 'no-store', + signal, + }); + + if (!response.ok) { + throw new Error(`Could not load ${EXTERNAL_DOCS_MANIFEST_URL}. Run npm run generate:external-docs before starting the docs site.`); + } + + const manifest = await response.json(); + const rawUrl = manifest?.sources?.[externalDoc]?.rawUrl; + if (!rawUrl) { + throw new Error(`External docs source "${externalDoc}" does not define a raw Markdown URL.`); + } + + return rawUrl; +} + /** * Render page-level actions for copying or viewing the Markdown source for the current docs page. - * Pages that render canonical Markdown from another repository can set `rawUrl` in frontmatter so these actions use that source instead of the generated local fallback. + * Pages that render canonical Markdown from another repository can set `externalDoc` in frontmatter so these actions use the registry source instead of the generated local fallback. */ export function CopyRawButton({ frontmatter }) { const [copied, setCopied] = useState(false); const [copyError, setCopyError] = useState(false); const [isCopying, setIsCopying] = useState(false); + const [externalRawUrl, setExternalRawUrl] = useState(null); const router = useRouter(); const currentPath = router.asPath.split('#')[0].split('?')[0] || '/'; - const rawUrl = frontmatter?.rawUrl || getGeneratedRawUrl(currentPath); + const generatedRawUrl = getGeneratedRawUrl(currentPath); + const rawUrl = frontmatter?.rawUrl || externalRawUrl || generatedRawUrl; + + useEffect(() => { + if (!frontmatter?.externalDoc || frontmatter?.rawUrl) { + setExternalRawUrl(null); + return undefined; + } + + const controller = new AbortController(); + getExternalDocRawUrl(frontmatter.externalDoc, controller.signal) + .then(setExternalRawUrl) + .catch(() => setExternalRawUrl(null)); + + return () => controller.abort(); + }, [frontmatter?.externalDoc, frontmatter?.rawUrl]); const handleCopy = async () => { setIsCopying(true); diff --git a/components/RemoteMarkdown.js b/components/RemoteMarkdown.js index 94b6592..944cd63 100644 --- a/components/RemoteMarkdown.js +++ b/components/RemoteMarkdown.js @@ -17,6 +17,8 @@ import { CardGrid } from './CardGrid'; import { MermaidDiagram } from './MermaidDiagram'; const SOURCE_ORG = 'hypercerts-org'; +const EXTERNAL_DOCS_MANIFEST_URL = '/external-docs.json'; +let externalDocsManifestPromise = null; const RemoteMarkdownContext = React.createContext(null); const markdocConfig = { @@ -68,6 +70,75 @@ function resolveGitHubMarkdownSource(source) { throw new Error('Remote docs can only be loaded from github.com or raw.githubusercontent.com.'); } +function isHttpUrl(source) { + try { + const url = new URL(source); + return url.protocol === 'https:' || url.protocol === 'http:'; + } catch { + return false; + } +} + +function throwIfAborted(signal) { + if (!signal?.aborted) return; + const error = new Error('Aborted'); + error.name = 'AbortError'; + throw error; +} + +async function loadExternalDocsManifest() { + if (!externalDocsManifestPromise) { + externalDocsManifestPromise = fetch(EXTERNAL_DOCS_MANIFEST_URL, { + cache: 'no-store', + }) + .then((response) => { + if (!response.ok) { + throw new Error(`${EXTERNAL_DOCS_MANIFEST_URL} returned ${response.status} ${response.statusText || 'without an external docs manifest'}. Run npm run generate:external-docs before starting the docs site.`); + } + return response.json(); + }) + .catch((error) => { + externalDocsManifestPromise = null; + throw error; + }); + } + + return externalDocsManifestPromise; +} + +function resolveRegisteredMarkdownSource(source, manifest) { + const id = String(source || '').trim(); + if (!/^[a-z0-9][a-z0-9-]*$/.test(id)) { + throw new Error('Remote doc source must be a GitHub URL or a docs-sources.yml id like "epds".'); + } + + const entry = manifest?.sources?.[id]; + if (!entry) { + throw new Error(`No external docs source "${id}" was found in ${EXTERNAL_DOCS_MANIFEST_URL}. Add it to docs-sources.yml and rebuild.`); + } + + const [owner, repo] = String(entry.repo || '').split('/'); + if (owner?.toLowerCase() !== SOURCE_ORG || !repo || !entry.branch || !entry.filePath || !entry.rawUrl || !entry.sourceUrl) { + throw new Error(`External docs source "${id}" is incomplete. Set repo, branch, docsPath, and entrypoint in docs-sources.yml, then rebuild.`); + } + + return { + sourceUrl: entry.sourceUrl, + rawUrl: entry.rawUrl, + owner, + repo, + ref: entry.branch, + filePath: entry.filePath, + }; +} + +async function resolveMarkdownSource(source, signal) { + if (isHttpUrl(source)) return resolveGitHubMarkdownSource(source); + const manifest = await loadExternalDocsManifest(); + throwIfAborted(signal); + return resolveRegisteredMarkdownSource(source, manifest); +} + /** * Remove optional YAML frontmatter before parsing remote Markdown with Markdoc. */ @@ -160,31 +231,31 @@ export function RemoteMarkdown({ source, children }) { const [markdown, setMarkdown] = useState(null); const [error, setError] = useState(null); const [isLoading, setIsLoading] = useState(true); - const sourceState = useMemo(() => { - try { - return { sourceInfo: resolveGitHubMarkdownSource(source), sourceError: null }; - } catch (err) { - return { sourceInfo: null, sourceError: err }; - } - }, [source]); - const { sourceInfo, sourceError } = sourceState; + const [sourceInfo, setSourceInfo] = useState(null); useEffect(() => { - if (!sourceInfo) { - setIsLoading(false); - return undefined; - } - const controller = new AbortController(); setMarkdown(null); setError(null); + setSourceInfo(null); setIsLoading(true); async function loadRemoteMarkdown() { + let nextSourceInfo; + + try { + nextSourceInfo = await resolveMarkdownSource(source, controller.signal); + setSourceInfo(nextSourceInfo); + } catch (err) { + if (err.name === 'AbortError') return; + setError(err); + return; + } + let githubError = null; try { - const nextMarkdown = await fetchMarkdown(sourceInfo.rawUrl, controller.signal, 'GitHub'); + const nextMarkdown = await fetchMarkdown(nextSourceInfo.rawUrl, controller.signal, 'GitHub'); transformMarkdown(nextMarkdown); setMarkdown(nextMarkdown); return; @@ -210,7 +281,7 @@ export function RemoteMarkdown({ source, children }) { }); return () => controller.abort(); - }, [rawCacheUrl, sourceInfo]); + }, [rawCacheUrl, source]); const renderedState = useMemo(() => { if (!markdown || !sourceInfo) return { renderedContent: null, renderError: null }; @@ -228,7 +299,7 @@ export function RemoteMarkdown({ source, children }) { } }, [markdown, sourceInfo]); const { renderedContent, renderError } = renderedState; - const displayError = error || sourceError || renderError; + const displayError = error || renderError; useEffect(() => { if (!renderedContent) return undefined; @@ -260,7 +331,7 @@ export function RemoteMarkdown({ source, children }) { <>
Could not load the canonical docs.{' '} - The browser could not load the live GitHub raw file or the build-time raw cache. Showing the local fallback below. Try refreshing, or edit{' '} + The browser could not load the registered source, the live GitHub raw file, or the build-time raw cache. Showing the local fallback below. Try refreshing, or edit{' '} {sourceInfo ? ( the source file ) : ( diff --git a/docs-sources.yml b/docs-sources.yml new file mode 100644 index 0000000..8299b40 --- /dev/null +++ b/docs-sources.yml @@ -0,0 +1,8 @@ +sources: + - id: epds + title: ePDS + repo: hypercerts-org/ePDS + branch: main + docsPath: docs + entrypoint: tutorial.md + routeBase: /architecture/epds diff --git a/docs/remote-markdown.md b/docs/remote-markdown.md index 1139217..3c0acfe 100644 --- a/docs/remote-markdown.md +++ b/docs/remote-markdown.md @@ -2,12 +2,28 @@ Use the `{% remote-doc %}` Markdoc tag when a page should render canonical Markdown from a Hypercerts service repository without copying that Markdown into this documentation repo. +Register the source once in `docs-sources.yml`: + +```yaml +sources: + - id: epds + title: ePDS + repo: hypercerts-org/ePDS + branch: main + docsPath: docs + entrypoint: tutorial.md + routeBase: /architecture/epds +``` + +Then reference that registry id from the page: + ```md --- -rawUrl: "https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/tutorial.md" +title: ePDS (extended PDS) +externalDoc: epds --- -{% remote-doc source="https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md" %} +{% remote-doc source="epds" %} A short unavailable-state fallback goes here. Keep it brief so the documentation repo does not become a second source of truth. @@ -17,17 +33,42 @@ A short unavailable-state fallback goes here. Keep it brief so the documentation ## How it works - The site remains a static Next.js export. -- The browser fetches the source file from `raw.githubusercontent.com` at runtime first. +- `npm run generate:external-docs` reads `docs-sources.yml` and writes `/external-docs.json` for browser components. +- The browser resolves `{% remote-doc source="epds" %}` through `/external-docs.json`, then fetches the source file from `raw.githubusercontent.com` at runtime first. - If the live GitHub fetch fails, the browser falls back to the build-time copy in `/raw`. +- Build-time search and raw-page generation use `externalDoc` frontmatter, so search and `/raw` use the canonical Markdown instead of the local fallback. - The fetched Markdown is parsed with the same Markdoc tags and nodes used by local pages. - Fenced `mermaid` diagrams render as SVGs in the browser through the Mermaid npm package. - Relative links in the remote Markdown point back to the source GitHub repository. -- `Copy raw` and `View raw` use the page's `rawUrl` frontmatter when it is set. +- `Copy raw` and `View raw` use the page's registered `externalDoc` source when it is set. - The wrapped local Markdown is the last-resort fallback content. Keep it to a short unavailable-state message, not a copy of the canonical docs. +## Scheduled refresh and deploys + +`npm run docs:fingerprint` reads `docs-sources.yml`, fetches GitHub tree metadata for each registered `docsPath`, and writes `public/docs-fingerprint.json` during the static build. The generated file includes a stable `combinedFingerprint`; timestamps are ignored by the comparison script. + +`.github/workflows/docs-refresh.yml` runs hourly and through `workflow_dispatch`: + +1. Generate the current external-docs fingerprint. +2. Download `https://docs.hypercerts.org/docs-fingerprint.json` from the deployed site. +3. Compare only `combinedFingerprint`. +4. If it changed, `POST` to the configured Vercel Deploy Hook. + +Manual `workflow_dispatch` runs default to `dry_run: true`, which compares fingerprints without calling the Vercel hook. Set `dry_run: false` when you want the manual run to deploy. GitHub only accepts `workflow_dispatch` and scheduled runs once the workflow file exists on the default branch. The workflow only treats a deployed-fingerprint `404` as missing first-run state; transient download failures fail the workflow instead of deploying on an unknown diff. + +`.github/workflows/docs-refresh-pr-dry-run.yml` is a temporary PR-only check for this rollout. It runs the same manifest and fingerprint comparison on pull requests, writes a summary, and never calls the Vercel deploy hook. + +Required GitHub Actions secret: + +- `VERCEL_DEPLOY_HOOK_URL` — the Vercel Deploy Hook URL for the production docs branch. + +Optional GitHub Actions secret: + +- `DOCS_SOURCE_TOKEN` — a GitHub token with read access to source repos. Public repos can use the workflow `GITHUB_TOKEN`, but this avoids API rate limits and is required if a source repo becomes private. + ## Constraints - Sources must be in `hypercerts-org` GitHub repositories. -- The current implementation is browser-runtime fetching, not server-side rendering. -- Build-time search and raw-page generation fetch `rawUrl`, so search and `/raw` use the canonical Markdown instead of the local fallback. -- If runtime docs become permanent for many pages, add build-time indexing for the remote sources too. +- Registry ids must be lowercase, for example `epds` or `certified-group-service`. +- A source needs `entrypoint` when `docsPath` points at a directory and a page renders it through `{% remote-doc %}`. +- The current page renderer is browser-runtime fetching, not server-side rendering. diff --git a/lib/compare-docs-fingerprint.js b/lib/compare-docs-fingerprint.js new file mode 100644 index 0000000..c4f7104 --- /dev/null +++ b/lib/compare-docs-fingerprint.js @@ -0,0 +1,46 @@ +const { readFileSync } = require('fs'); + +/** + * Read a generated docs-fingerprint.json file and return its combined fingerprint. + */ +function readCombinedFingerprint(path, label) { + let parsed; + try { + parsed = JSON.parse(readFileSync(path, 'utf8')); + } catch (error) { + throw new Error(`Unable to read ${label} fingerprint at ${path}: ${error.message}`); + } + + return typeof parsed.combinedFingerprint === 'string' ? parsed.combinedFingerprint : ''; +} + +function main() { + const [currentPath, deployedPath] = process.argv.slice(2); + if (!currentPath || !deployedPath) { + throw new Error('Usage: node lib/compare-docs-fingerprint.js '); + } + + const current = readCombinedFingerprint(currentPath, 'current'); + if (!current) { + throw new Error(`Current fingerprint file ${currentPath} does not contain combinedFingerprint.`); + } + + const deployed = readCombinedFingerprint(deployedPath, 'deployed'); + const changed = current !== deployed; + + console.error(changed + ? `External docs changed: deployed=${deployed || ''} current=${current}` + : `External docs unchanged: ${current}`); + + console.log(`changed=${changed ? 'true' : 'false'}`); + console.log(`current_fingerprint=${current}`); + console.log(`deployed_fingerprint=${deployed}`); + console.log(`reason=${changed ? 'combined fingerprint differs' : 'combined fingerprint matches deployed site'}`); +} + +try { + main(); +} catch (error) { + console.error(error.message); + process.exit(1); +} diff --git a/lib/external-docs.js b/lib/external-docs.js new file mode 100644 index 0000000..98809e5 --- /dev/null +++ b/lib/external-docs.js @@ -0,0 +1,233 @@ +const { readFileSync } = require('fs'); +const { join, posix } = require('path'); +const yaml = require('js-yaml'); + +const SOURCE_ORG = 'hypercerts-org'; +const REGISTRY_PATH = join(__dirname, '..', 'docs-sources.yml'); +const MARKDOWN_EXTENSIONS = /\.(md|mdoc|mdx)$/i; + +/** + * Return true when a registry path points at a Markdown file instead of a docs directory. + */ +function isMarkdownFilePath(value) { + return MARKDOWN_EXTENSIONS.test(value); +} + +/** + * Encode a GitHub path without collapsing its slash-separated path segments. + */ +function encodeGitHubPath(value) { + return value.split('/').map(encodeURIComponent).join('/'); +} + +/** + * Normalize a docs-sources.yml path and reject absolute paths or parent traversal. + */ +function normalizeRegistryPath(value, fieldName) { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a non-empty relative path.`); + } + + const normalized = value.trim().replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/$/, ''); + const parts = normalized.split('/'); + + if (normalized.startsWith('/') || parts.includes('..') || parts.includes('')) { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a relative path without empty segments or "..".`); + } + + return normalized; +} + +/** + * Parse the YAML frontmatter block from a Markdown file. + */ +function parseMarkdownFrontmatter(markdown, label = 'Markdown file') { + const match = markdown.match(/^---\n([\s\S]*?)\n---\n?/); + if (!match) return {}; + + try { + return yaml.load(match[1]) || {}; + } catch (error) { + throw new Error(`Invalid frontmatter in ${label}: ${error.message}`); + } +} + +/** + * Build the raw.githubusercontent.com URL for a registered external docs file. + */ +function buildRawGitHubUrl(source, filePath) { + return `https://raw.githubusercontent.com/${source.owner}/${source.repoName}/${encodeURIComponent(source.branch)}/${encodeGitHubPath(filePath)}`; +} + +/** + * Build the GitHub browser URL for a registered external docs path. + */ +function buildGitHubSourceUrl(source, filePath, mode = 'blob') { + return `https://github.com/${source.owner}/${source.repoName}/${mode}/${encodeURIComponent(source.branch)}/${encodeGitHubPath(filePath)}`; +} + +/** + * Return the single Markdown file that a remote-doc page should render, when configured. + */ +function getEntrypointPath(source) { + if (source.entrypoint) return posix.join(source.docsPath, source.entrypoint); + if (isMarkdownFilePath(source.docsPath)) return source.docsPath; + return null; +} + +/** + * Validate and normalize one source entry from docs-sources.yml. + */ +function normalizeSource(rawSource, index) { + const label = `sources[${index}]`; + + if (!rawSource || typeof rawSource !== 'object' || Array.isArray(rawSource)) { + throw new Error(`Invalid docs-sources.yml: ${label} must be an object.`); + } + + const id = rawSource.id; + if (typeof id !== 'string' || !/^[a-z0-9][a-z0-9-]*$/.test(id)) { + throw new Error(`Invalid docs-sources.yml: ${label}.id must be a lowercase id like "epds" or "certified-group-service".`); + } + + const title = rawSource.title; + if (typeof title !== 'string' || title.trim() === '') { + throw new Error(`Invalid docs-sources.yml: source "${id}" must set a human-readable title.`); + } + + const repo = rawSource.repo; + if (typeof repo !== 'string' || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) { + throw new Error(`Invalid docs-sources.yml: source "${id}" repo must look like "hypercerts-org/ePDS".`); + } + + const [owner, repoName] = repo.split('/'); + if (owner.toLowerCase() !== SOURCE_ORG) { + throw new Error(`Invalid docs-sources.yml: source "${id}" must use a ${SOURCE_ORG} repository so browser-rendered docs cannot load arbitrary hosts.`); + } + + const branch = rawSource.branch; + if (typeof branch !== 'string' || branch.trim() === '') { + throw new Error(`Invalid docs-sources.yml: source "${id}" must set the branch to read, for example "main".`); + } + + const docsPath = normalizeRegistryPath(rawSource.docsPath, `source "${id}" docsPath`); + const entrypoint = rawSource.entrypoint + ? normalizeRegistryPath(rawSource.entrypoint, `source "${id}" entrypoint`) + : ''; + + const routeBase = rawSource.routeBase || rawSource.route || ''; + if (routeBase && (typeof routeBase !== 'string' || !routeBase.startsWith('/'))) { + throw new Error(`Invalid docs-sources.yml: source "${id}" routeBase must start with "/".`); + } + + const source = { + id, + title: title.trim(), + repo, + owner, + repoName, + branch: branch.trim(), + docsPath, + entrypoint, + routeBase, + }; + + const entrypointPath = getEntrypointPath(source); + return { + ...source, + entrypointPath, + rawUrl: entrypointPath ? buildRawGitHubUrl(source, entrypointPath) : '', + sourceUrl: entrypointPath + ? buildGitHubSourceUrl(source, entrypointPath, 'blob') + : buildGitHubSourceUrl(source, docsPath, 'tree'), + }; +} + +/** + * Load and validate the central external docs registry. + */ +function loadExternalDocSources(registryPath = REGISTRY_PATH) { + let document; + try { + document = yaml.load(readFileSync(registryPath, 'utf8')) || {}; + } catch (error) { + throw new Error(`Unable to read docs source registry at ${registryPath}: ${error.message}`); + } + + if (!Array.isArray(document.sources)) { + throw new Error('Invalid docs-sources.yml: expected a top-level "sources" array.'); + } + + const seen = new Set(); + return document.sources.map((source, index) => { + const normalized = normalizeSource(source, index); + if (seen.has(normalized.id)) { + throw new Error(`Invalid docs-sources.yml: duplicate source id "${normalized.id}".`); + } + seen.add(normalized.id); + return normalized; + }); +} + +/** + * Find one registered external docs source by id. + */ +function findExternalDocSource(id, sources = loadExternalDocSources()) { + return sources.find((source) => source.id === id) || null; +} + +/** + * Convert a registry source into the public manifest consumed by browser components. + */ +function sourceToPublicManifestEntry(source) { + return { + id: source.id, + title: source.title, + repo: source.repo, + branch: source.branch, + docsPath: source.docsPath, + entrypoint: source.entrypoint || undefined, + routeBase: source.routeBase || undefined, + sourceUrl: source.sourceUrl, + rawUrl: source.rawUrl || undefined, + filePath: source.entrypointPath || undefined, + }; +} + +/** + * Resolve a page frontmatter declaration to the raw Markdown URL that build scripts should fetch. + */ +function resolveFrontmatterRawSource(frontmatter, sources = loadExternalDocSources()) { + if (frontmatter.externalDoc) { + const id = String(frontmatter.externalDoc); + const source = findExternalDocSource(id, sources); + if (!source) { + throw new Error(`Unknown externalDoc "${id}". Add it to docs-sources.yml or remove the externalDoc frontmatter.`); + } + if (!source.rawUrl) { + throw new Error(`External doc "${id}" does not define a renderable Markdown file. Set entrypoint in docs-sources.yml or point docsPath at a Markdown file.`); + } + return { rawUrl: source.rawUrl, label: `externalDoc "${id}"` }; + } + + if (frontmatter.rawUrl) { + return { rawUrl: String(frontmatter.rawUrl), label: 'rawUrl frontmatter' }; + } + + return null; +} + +module.exports = { + REGISTRY_PATH, + SOURCE_ORG, + buildGitHubSourceUrl, + buildRawGitHubUrl, + encodeGitHubPath, + findExternalDocSource, + getEntrypointPath, + isMarkdownFilePath, + loadExternalDocSources, + parseMarkdownFrontmatter, + resolveFrontmatterRawSource, + sourceToPublicManifestEntry, +}; diff --git a/lib/generate-docs-fingerprint.js b/lib/generate-docs-fingerprint.js new file mode 100644 index 0000000..2028953 --- /dev/null +++ b/lib/generate-docs-fingerprint.js @@ -0,0 +1,159 @@ +const crypto = require('crypto'); +const { mkdirSync, writeFileSync } = require('fs'); +const { dirname, join } = require('path'); +const { loadExternalDocSources } = require('./external-docs'); + +const DEFAULT_OUTPUT = join(__dirname, '..', 'public', 'docs-fingerprint.json'); +const GITHUB_API_VERSION = '2022-11-28'; + +/** + * Serialize values with stable object-key ordering so hashes do not depend on construction order. + */ +function stableStringify(value) { + if (Array.isArray(value)) { + return `[${value.map(stableStringify).join(',')}]`; + } + + if (value && typeof value === 'object') { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`) + .join(',')}}`; + } + + return JSON.stringify(value); +} + +/** + * Compute a sha256 digest with the prefix used in generated fingerprint files. + */ +function sha256(value) { + return `sha256:${crypto.createHash('sha256').update(value).digest('hex')}`; +} + +/** + * Read a --output value from the CLI arguments. + */ +function getOutputPath(argv) { + const outputIndex = argv.findIndex((arg) => arg === '--output' || arg === '-o'); + if (outputIndex !== -1) { + const output = argv[outputIndex + 1]; + if (!output) throw new Error('Missing value after --output.'); + return output; + } + + const inlineOutput = argv.find((arg) => arg.startsWith('--output=')); + if (inlineOutput) return inlineOutput.slice('--output='.length); + + return DEFAULT_OUTPUT; +} + +/** + * Fetch JSON from the GitHub REST API with optional authentication for private repos or rate limits. + */ +async function fetchGitHubJson(url, token) { + const headers = { + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + 'User-Agent': 'hypercerts-docs-refresh', + }; + + if (token) headers.Authorization = `Bearer ${token}`; + + const response = await fetch(url, { headers }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`${url} returned ${response.status} ${response.statusText || ''}. Check repo, branch, docsPath, and DOCS_SOURCE_TOKEN. ${body}`.trim()); + } + + return response.json(); +} + +/** + * Fetch the Git tree for a source branch and return blob entries under docsPath. + */ +async function fetchSourceEntries(source, token) { + const treeUrl = `https://api.github.com/repos/${source.owner}/${source.repoName}/git/trees/${encodeURIComponent(source.branch)}?recursive=1`; + const tree = await fetchGitHubJson(treeUrl, token); + + if (tree.truncated) { + throw new Error(`GitHub returned a truncated tree for ${source.repo}@${source.branch}. Narrow source "${source.id}" docsPath or split it into smaller sources before fingerprinting.`); + } + + const root = source.docsPath.replace(/\/$/, ''); + const prefix = `${root}/`; + const files = (tree.tree || []) + .filter((entry) => entry.type === 'blob' && (entry.path === root || entry.path.startsWith(prefix))) + .map((entry) => ({ + path: entry.path, + sha: entry.sha, + size: entry.size || 0, + })) + .sort((a, b) => a.path.localeCompare(b.path)); + + if (files.length === 0) { + throw new Error(`No files found for source "${source.id}" at ${source.repo}@${source.branch}:${source.docsPath}. Check docs-sources.yml.`); + } + + return files; +} + +/** + * Compute the per-source and combined fingerprints for all registered external docs. + */ +async function generateFingerprint() { + const sources = loadExternalDocSources(); + const token = process.env.DOCS_SOURCE_TOKEN || process.env.GITHUB_TOKEN || process.env.GH_TOKEN || ''; + const outputSources = {}; + const stableSources = []; + + for (const source of sources) { + const files = await fetchSourceEntries(source, token); + const stableSource = { + id: source.id, + repo: source.repo, + branch: source.branch, + docsPath: source.docsPath, + entrypoint: source.entrypoint || '', + routeBase: source.routeBase || '', + files, + }; + const fingerprint = sha256(stableStringify(stableSource)); + + outputSources[source.id] = { + title: source.title, + repo: source.repo, + branch: source.branch, + docsPath: source.docsPath, + entrypoint: source.entrypoint || undefined, + routeBase: source.routeBase || undefined, + fileCount: files.length, + fingerprint, + files, + }; + stableSources.push({ ...stableSource, fingerprint }); + } + + const combinedFingerprint = sha256(stableStringify({ schemaVersion: 1, sources: stableSources })); + + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + sources: outputSources, + combinedFingerprint, + }; +} + +async function main() { + const output = getOutputPath(process.argv.slice(2)); + const fingerprint = await generateFingerprint(); + + mkdirSync(dirname(output), { recursive: true }); + writeFileSync(output, `${JSON.stringify(fingerprint, null, 2)}\n`); + console.log(`Generated docs fingerprint ${fingerprint.combinedFingerprint} → ${output}`); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/lib/generate-external-docs-manifest.js b/lib/generate-external-docs-manifest.js new file mode 100644 index 0000000..68e8743 --- /dev/null +++ b/lib/generate-external-docs-manifest.js @@ -0,0 +1,27 @@ +const { mkdirSync, writeFileSync } = require('fs'); +const { dirname, join } = require('path'); +const { + loadExternalDocSources, + sourceToPublicManifestEntry, +} = require('./external-docs'); + +const OUTPUT = join(__dirname, '..', 'public', 'external-docs.json'); + +/** + * Generate the browser-readable manifest used to resolve remote-doc registry ids. + */ +function generateExternalDocsManifest() { + const sources = loadExternalDocSources(); + const manifest = { + schemaVersion: 1, + sources: Object.fromEntries( + sources.map((source) => [source.id, sourceToPublicManifestEntry(source)]) + ), + }; + + mkdirSync(dirname(OUTPUT), { recursive: true }); + writeFileSync(OUTPUT, `${JSON.stringify(manifest, null, 2)}\n`); + console.log(`Generated external docs manifest for ${sources.length} source${sources.length === 1 ? '' : 's'}`); +} + +generateExternalDocsManifest(); diff --git a/lib/generate-raw-pages.js b/lib/generate-raw-pages.js index a3ed888..8e1d68b 100644 --- a/lib/generate-raw-pages.js +++ b/lib/generate-raw-pages.js @@ -7,6 +7,11 @@ const { mkdirSync, } = require('fs'); const { dirname, join, relative } = require('path'); +const { + loadExternalDocSources, + parseMarkdownFrontmatter, + resolveFrontmatterRawSource, +} = require('./external-docs'); const PAGES_DIR = join(__dirname, '..', 'pages'); const OUTPUT_DIR = join(__dirname, '..', 'public', 'raw'); @@ -33,23 +38,17 @@ function getRawOutputPath(filePath) { return join(OUTPUT_DIR, outputRel); } -function getFrontmatterRawUrl(markdown) { - const match = markdown.match(/^---\n([\s\S]*?)\n---\n/); - if (!match) return null; - - const rawUrlMatch = match[1].match(/^rawUrl:\s*["']?([^"'\n]+)["']?\s*$/m); - return rawUrlMatch?.[1] || null; -} - -async function getRawMarkdown(file) { +async function getRawMarkdown(file, sources) { const localMarkdown = readFileSync(file, 'utf-8'); - const rawUrl = getFrontmatterRawUrl(localMarkdown); + const pagePath = relative(PAGES_DIR, file); + const frontmatter = parseMarkdownFrontmatter(localMarkdown, pagePath); + const remoteSource = resolveFrontmatterRawSource(frontmatter, sources); - if (!rawUrl) return localMarkdown; + if (!remoteSource) return localMarkdown; - const response = await fetch(rawUrl, { cache: 'no-store' }); + const response = await fetch(remoteSource.rawUrl, { cache: 'no-store' }); if (!response.ok) { - throw new Error(`Failed to fetch rawUrl for ${relative(PAGES_DIR, file)}: ${rawUrl} returned ${response.status} ${response.statusText || ''}`.trim()); + throw new Error(`Failed to fetch ${remoteSource.label} for ${pagePath}: ${remoteSource.rawUrl} returned ${response.status} ${response.statusText || ''}`.trim()); } return response.text(); @@ -60,11 +59,12 @@ async function main() { mkdirSync(OUTPUT_DIR, { recursive: true }); const files = walkDir(PAGES_DIR); + const sources = loadExternalDocSources(); for (const file of files) { const outputPath = getRawOutputPath(file); mkdirSync(dirname(outputPath), { recursive: true }); - writeFileSync(outputPath, await getRawMarkdown(file)); + writeFileSync(outputPath, await getRawMarkdown(file, sources)); } console.log(`Generated raw markdown files for ${files.length} pages`); diff --git a/lib/generate-search-index.js b/lib/generate-search-index.js index 45377b2..45be89d 100644 --- a/lib/generate-search-index.js +++ b/lib/generate-search-index.js @@ -1,5 +1,10 @@ const { readdirSync, statSync, readFileSync, writeFileSync } = require("fs"); const { join, relative } = require("path"); +const { + loadExternalDocSources, + parseMarkdownFrontmatter, + resolveFrontmatterRawSource, +} = require("./external-docs"); const PAGES_DIR = join(__dirname, "..", "pages"); const OUTPUT = join(__dirname, "..", "public", "search-index.json"); @@ -18,24 +23,8 @@ function walkDir(dir) { return results; } -function stripYamlValue(value) { - return value.trim().replace(/^['"]|['"]$/g, ""); -} - -function extractFrontmatter(content) { - const fmMatch = content.match(/^---\n([\s\S]*?)\n---/); - if (!fmMatch) return { title: "", description: "", rawUrl: "" }; - - const frontmatter = fmMatch[1]; - const titleMatch = frontmatter.match(/^title:\s*(.+)$/m); - const descMatch = frontmatter.match(/^description:\s*(.+)$/m); - const rawUrlMatch = frontmatter.match(/^rawUrl:\s*(.+)$/m); - - return { - title: titleMatch ? stripYamlValue(titleMatch[1]) : "", - description: descMatch ? stripYamlValue(descMatch[1]) : "", - rawUrl: rawUrlMatch ? stripYamlValue(rawUrlMatch[1]) : "", - }; +function getStringFrontmatterValue(frontmatter, key) { + return frontmatter[key] ? String(frontmatter[key]) : ""; } function extractHeadings(content) { @@ -106,12 +95,12 @@ function getSection(path) { return "Other"; } -async function getIndexContent(file, localContent, rawUrl) { - if (!rawUrl) return localContent; +async function getIndexContent(file, localContent, remoteSource) { + if (!remoteSource) return localContent; - const response = await fetch(rawUrl, { cache: "no-store" }); + const response = await fetch(remoteSource.rawUrl, { cache: "no-store" }); if (!response.ok) { - throw new Error(`Failed to fetch rawUrl for search index ${relative(PAGES_DIR, file)}: ${rawUrl} returned ${response.status} ${response.statusText || ""}`.trim()); + throw new Error(`Failed to fetch ${remoteSource.label} for search index ${relative(PAGES_DIR, file)}: ${remoteSource.rawUrl} returned ${response.status} ${response.statusText || ""}`.trim()); } return response.text(); @@ -119,6 +108,7 @@ async function getIndexContent(file, localContent, rawUrl) { async function main() { const files = walkDir(PAGES_DIR); + const sources = loadExternalDocSources(); const index = []; for (const file of files) { @@ -126,8 +116,11 @@ async function main() { const rel = "/" + relative(PAGES_DIR, file).replace(/\.md$/, ""); const path = rel === "/index" ? "/" : rel; - const { title, description, rawUrl } = extractFrontmatter(localContent); - const content = await getIndexContent(file, localContent, rawUrl); + const frontmatter = parseMarkdownFrontmatter(localContent, relative(PAGES_DIR, file)); + const title = getStringFrontmatterValue(frontmatter, "title"); + const description = getStringFrontmatterValue(frontmatter, "description"); + const remoteSource = resolveFrontmatterRawSource(frontmatter, sources); + const content = await getIndexContent(file, localContent, remoteSource); const headings = extractHeadings(content); const section = getSection(path); diff --git a/markdoc/tags/remote-doc.markdoc.js b/markdoc/tags/remote-doc.markdoc.js index 630d2d7..b4403e5 100644 --- a/markdoc/tags/remote-doc.markdoc.js +++ b/markdoc/tags/remote-doc.markdoc.js @@ -1,5 +1,6 @@ /** - * Markdoc tag for rendering canonical Markdown from a Hypercerts GitHub repo at browser runtime. + * Markdoc tag for rendering canonical Markdown from a registered Hypercerts GitHub repo at browser runtime. + * The source can be a docs-sources.yml id such as "epds" or a direct hypercerts-org GitHub Markdown URL. */ module.exports = { render: 'RemoteMarkdown', diff --git a/package-lock.json b/package-lock.json index 6bc12aa..a739621 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@markdoc/next.js": "^0.5.0", "@vercel/analytics": "^1.6.1", "flexsearch": "^0.8.212", + "js-yaml": "^4.1.1", "mermaid": "^11.15.0", "next": "^16.1.6", "prism-react-renderer": "^2.4.1", diff --git a/package.json b/package.json index 8c8d29e..def3778 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,11 @@ "private": true, "description": "Hypercerts Protocol Documentation", "scripts": { - "dev": "node lib/generate-search-index.js && node lib/generate-last-updated.js && node lib/generate-sitemap.js && node lib/generate-raw-pages.js && next dev --webpack", - "build": "node lib/generate-search-index.js && node lib/generate-last-updated.js && node lib/generate-sitemap.js && node lib/generate-raw-pages.js && next build --webpack", + "generate:external-docs": "node lib/generate-external-docs-manifest.js", + "docs:fingerprint": "node lib/generate-docs-fingerprint.js", + "generate": "npm run generate:external-docs && node lib/generate-search-index.js && node lib/generate-last-updated.js && node lib/generate-sitemap.js && node lib/generate-raw-pages.js && npm run docs:fingerprint", + "dev": "npm run generate && next dev --webpack", + "build": "npm run generate && next build --webpack", "start": "next start" }, "dependencies": { @@ -13,6 +16,7 @@ "@markdoc/next.js": "^0.5.0", "@vercel/analytics": "^1.6.1", "flexsearch": "^0.8.212", + "js-yaml": "^4.1.1", "mermaid": "^11.15.0", "next": "^16.1.6", "prism-react-renderer": "^2.4.1", diff --git a/pages/architecture/epds.md b/pages/architecture/epds.md index f2d5159..172fa24 100644 --- a/pages/architecture/epds.md +++ b/pages/architecture/epds.md @@ -1,10 +1,10 @@ --- title: ePDS (extended PDS) description: How the ePDS adds email/OTP login on top of AT Protocol without changing the standard OAuth flow for apps. -rawUrl: "https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/tutorial.md" +externalDoc: epds --- -{% remote-doc source="https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md" %} +{% remote-doc source="epds" %} # ePDS docs unavailable From 44e7e8c42ce0ac53084b31d21d3bf76d3098c095 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Fri, 12 Jun 2026 12:28:13 +0600 Subject: [PATCH 03/16] docs-refresh: make deployed fingerprint URL configurable --- .github/workflows/docs-refresh-pr-dry-run.yml | 2 +- .github/workflows/docs-refresh.yml | 2 +- docs/remote-markdown.md | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs-refresh-pr-dry-run.yml b/.github/workflows/docs-refresh-pr-dry-run.yml index 4b8eb56..a5591e2 100644 --- a/.github/workflows/docs-refresh-pr-dry-run.yml +++ b/.github/workflows/docs-refresh-pr-dry-run.yml @@ -50,7 +50,7 @@ jobs: - name: Download deployed docs fingerprint env: - DOCS_FINGERPRINT_URL: https://docs.hypercerts.org/docs-fingerprint.json + DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://docs.hypercerts.org/docs-fingerprint.json' }} run: | set +e http_code=$(curl -sS -L -w "%{http_code}" \ diff --git a/.github/workflows/docs-refresh.yml b/.github/workflows/docs-refresh.yml index f461533..229cdf1 100644 --- a/.github/workflows/docs-refresh.yml +++ b/.github/workflows/docs-refresh.yml @@ -49,7 +49,7 @@ jobs: - name: Download deployed docs fingerprint env: - DOCS_FINGERPRINT_URL: https://docs.hypercerts.org/docs-fingerprint.json + DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://docs.hypercerts.org/docs-fingerprint.json' }} run: | set +e http_code=$(curl -sS -L -w "%{http_code}" \ diff --git a/docs/remote-markdown.md b/docs/remote-markdown.md index 3c0acfe..1073d39 100644 --- a/docs/remote-markdown.md +++ b/docs/remote-markdown.md @@ -50,7 +50,7 @@ A short unavailable-state fallback goes here. Keep it brief so the documentation `.github/workflows/docs-refresh.yml` runs hourly and through `workflow_dispatch`: 1. Generate the current external-docs fingerprint. -2. Download `https://docs.hypercerts.org/docs-fingerprint.json` from the deployed site. +2. Download the deployed fingerprint from `DOCS_FINGERPRINT_URL`, defaulting to `https://docs.hypercerts.org/docs-fingerprint.json`. 3. Compare only `combinedFingerprint`. 4. If it changed, `POST` to the configured Vercel Deploy Hook. @@ -62,6 +62,10 @@ Required GitHub Actions secret: - `VERCEL_DEPLOY_HOOK_URL` — the Vercel Deploy Hook URL for the production docs branch. +Optional GitHub Actions variable: + +- `DOCS_FINGERPRINT_URL` — the deployed site fingerprint URL. Forks and staging deployments should set this to their own Vercel URL, for example `https://your-test-docs.vercel.app/docs-fingerprint.json`. + Optional GitHub Actions secret: - `DOCS_SOURCE_TOKEN` — a GitHub token with read access to source repos. Public repos can use the workflow `GITHUB_TOKEN`, but this avoids API rate limits and is required if a source repo becomes private. From 4e2a996cbfe01fd3b5f42ffc054b152721665171 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Fri, 12 Jun 2026 13:03:07 +0600 Subject: [PATCH 04/16] docs-refresh: default fingerprint URL to fork deployment --- .github/workflows/docs-refresh-pr-dry-run.yml | 2 +- .github/workflows/docs-refresh.yml | 2 +- docs/remote-markdown.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs-refresh-pr-dry-run.yml b/.github/workflows/docs-refresh-pr-dry-run.yml index a5591e2..c4f0eb8 100644 --- a/.github/workflows/docs-refresh-pr-dry-run.yml +++ b/.github/workflows/docs-refresh-pr-dry-run.yml @@ -50,7 +50,7 @@ jobs: - name: Download deployed docs fingerprint env: - DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://docs.hypercerts.org/docs-fingerprint.json' }} + DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://documentation-zeta-weld.vercel.app/docs-fingerprint.json' }} run: | set +e http_code=$(curl -sS -L -w "%{http_code}" \ diff --git a/.github/workflows/docs-refresh.yml b/.github/workflows/docs-refresh.yml index 229cdf1..d21acbf 100644 --- a/.github/workflows/docs-refresh.yml +++ b/.github/workflows/docs-refresh.yml @@ -49,7 +49,7 @@ jobs: - name: Download deployed docs fingerprint env: - DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://docs.hypercerts.org/docs-fingerprint.json' }} + DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://documentation-zeta-weld.vercel.app/docs-fingerprint.json' }} run: | set +e http_code=$(curl -sS -L -w "%{http_code}" \ diff --git a/docs/remote-markdown.md b/docs/remote-markdown.md index 1073d39..1c6e9aa 100644 --- a/docs/remote-markdown.md +++ b/docs/remote-markdown.md @@ -50,7 +50,7 @@ A short unavailable-state fallback goes here. Keep it brief so the documentation `.github/workflows/docs-refresh.yml` runs hourly and through `workflow_dispatch`: 1. Generate the current external-docs fingerprint. -2. Download the deployed fingerprint from `DOCS_FINGERPRINT_URL`, defaulting to `https://docs.hypercerts.org/docs-fingerprint.json`. +2. Download the deployed fingerprint from `DOCS_FINGERPRINT_URL`, defaulting to `https://documentation-zeta-weld.vercel.app/docs-fingerprint.json`. 3. Compare only `combinedFingerprint`. 4. If it changed, `POST` to the configured Vercel Deploy Hook. From 122fccf172f74aef1580527a0e36cb7d30916ed9 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Fri, 12 Jun 2026 13:08:04 +0600 Subject: [PATCH 05/16] docs-refresh: add hyperindex test source --- docs-sources.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs-sources.yml b/docs-sources.yml index 8299b40..837050a 100644 --- a/docs-sources.yml +++ b/docs-sources.yml @@ -6,3 +6,10 @@ sources: docsPath: docs entrypoint: tutorial.md routeBase: /architecture/epds + + - id: hyperindex-test + title: Hyperindex test source + repo: hypercerts-org/hyperindex + branch: main + docsPath: docs + routeBase: /tools/hyperindex From 07e15fee5b4d3e917419ce3dcbdbc060347821ca Mon Sep 17 00:00:00 2001 From: kzoeps Date: Fri, 12 Jun 2026 13:14:36 +0600 Subject: [PATCH 06/16] docs-refresh: point hyperindex test source at skills update --- docs-sources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-sources.yml b/docs-sources.yml index 837050a..5cfc5c4 100644 --- a/docs-sources.yml +++ b/docs-sources.yml @@ -10,6 +10,6 @@ sources: - id: hyperindex-test title: Hyperindex test source repo: hypercerts-org/hyperindex - branch: main + branch: skills-update docsPath: docs routeBase: /tools/hyperindex From d29eda2e12453e7054cec2610f28ba138d7c5a11 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Fri, 12 Jun 2026 13:16:29 +0600 Subject: [PATCH 07/16] docs-refresh: target hyperindex test markdown --- docs-sources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-sources.yml b/docs-sources.yml index 5cfc5c4..cd27abc 100644 --- a/docs-sources.yml +++ b/docs-sources.yml @@ -11,5 +11,5 @@ sources: title: Hyperindex test source repo: hypercerts-org/hyperindex branch: skills-update - docsPath: docs + docsPath: docs/hyperindex.md routeBase: /tools/hyperindex From db9a90fbe540bc9dbb70cbb705a0587b0f08c2c2 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Fri, 12 Jun 2026 13:27:16 +0600 Subject: [PATCH 08/16] docs-refresh: allow gainforest hyperindex source --- components/RemoteMarkdown.js | 18 +++++++++++------- docs-sources.yml | 2 +- docs/remote-markdown.md | 2 +- lib/external-docs.js | 18 +++++++++++++----- markdoc/tags/remote-doc.markdoc.js | 4 ++-- 5 files changed, 28 insertions(+), 16 deletions(-) diff --git a/components/RemoteMarkdown.js b/components/RemoteMarkdown.js index 944cd63..6249b14 100644 --- a/components/RemoteMarkdown.js +++ b/components/RemoteMarkdown.js @@ -16,7 +16,7 @@ import { HeroBanner } from './HeroBanner'; import { CardGrid } from './CardGrid'; import { MermaidDiagram } from './MermaidDiagram'; -const SOURCE_ORG = 'hypercerts-org'; +const ALLOWED_SOURCE_ORGS = ['hypercerts-org', 'gainforest']; const EXTERNAL_DOCS_MANIFEST_URL = '/external-docs.json'; let externalDocsManifestPromise = null; const RemoteMarkdownContext = React.createContext(null); @@ -26,17 +26,21 @@ const markdocConfig = { nodes: { fence, heading, link }, }; +function isAllowedSourceOwner(owner) { + return ALLOWED_SOURCE_ORGS.includes(String(owner || '').toLowerCase()); +} + /** * Convert an allowed GitHub Markdown URL into the raw URL used by the browser fetch. - * Only hypercerts-org GitHub sources are allowed so docs pages cannot become an open proxy. + * Only approved GitHub owners are allowed so docs pages cannot become an open proxy. */ function resolveGitHubMarkdownSource(source) { const url = new URL(source); if (url.hostname === 'github.com') { const [, owner, repo, marker, ref, ...fileParts] = url.pathname.split('/'); - if (owner?.toLowerCase() !== SOURCE_ORG || marker !== 'blob' || !repo || !ref || fileParts.length === 0) { - throw new Error('Remote docs must use a hypercerts-org GitHub blob URL, for example https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md.'); + if (!isAllowedSourceOwner(owner) || marker !== 'blob' || !repo || !ref || fileParts.length === 0) { + throw new Error(`Remote docs must use a GitHub blob URL under one of these owners: ${ALLOWED_SOURCE_ORGS.join(', ')}.`); } const filePath = fileParts.join('/'); @@ -52,8 +56,8 @@ function resolveGitHubMarkdownSource(source) { if (url.hostname === 'raw.githubusercontent.com') { const [, owner, repo, ref, ...fileParts] = url.pathname.split('/'); - if (owner?.toLowerCase() !== SOURCE_ORG || !repo || !ref || fileParts.length === 0) { - throw new Error('Remote docs must use a raw.githubusercontent.com URL under hypercerts-org.'); + if (!isAllowedSourceOwner(owner) || !repo || !ref || fileParts.length === 0) { + throw new Error(`Remote docs must use a raw.githubusercontent.com URL under one of these owners: ${ALLOWED_SOURCE_ORGS.join(', ')}.`); } const filePath = fileParts.join('/'); @@ -118,7 +122,7 @@ function resolveRegisteredMarkdownSource(source, manifest) { } const [owner, repo] = String(entry.repo || '').split('/'); - if (owner?.toLowerCase() !== SOURCE_ORG || !repo || !entry.branch || !entry.filePath || !entry.rawUrl || !entry.sourceUrl) { + if (!isAllowedSourceOwner(owner) || !repo || !entry.branch || !entry.filePath || !entry.rawUrl || !entry.sourceUrl) { throw new Error(`External docs source "${id}" is incomplete. Set repo, branch, docsPath, and entrypoint in docs-sources.yml, then rebuild.`); } diff --git a/docs-sources.yml b/docs-sources.yml index cd27abc..d6b64f4 100644 --- a/docs-sources.yml +++ b/docs-sources.yml @@ -9,7 +9,7 @@ sources: - id: hyperindex-test title: Hyperindex test source - repo: hypercerts-org/hyperindex + repo: gainforest/hyperindex branch: skills-update docsPath: docs/hyperindex.md routeBase: /tools/hyperindex diff --git a/docs/remote-markdown.md b/docs/remote-markdown.md index 1c6e9aa..dbeaf01 100644 --- a/docs/remote-markdown.md +++ b/docs/remote-markdown.md @@ -72,7 +72,7 @@ Optional GitHub Actions secret: ## Constraints -- Sources must be in `hypercerts-org` GitHub repositories. +- Sources must be in approved GitHub owners: `hypercerts-org` or `gainforest`. - Registry ids must be lowercase, for example `epds` or `certified-group-service`. - A source needs `entrypoint` when `docsPath` points at a directory and a page renders it through `{% remote-doc %}`. - The current page renderer is browser-runtime fetching, not server-side rendering. diff --git a/lib/external-docs.js b/lib/external-docs.js index 98809e5..c574624 100644 --- a/lib/external-docs.js +++ b/lib/external-docs.js @@ -2,10 +2,17 @@ const { readFileSync } = require('fs'); const { join, posix } = require('path'); const yaml = require('js-yaml'); -const SOURCE_ORG = 'hypercerts-org'; +const ALLOWED_SOURCE_ORGS = ['hypercerts-org', 'gainforest']; const REGISTRY_PATH = join(__dirname, '..', 'docs-sources.yml'); const MARKDOWN_EXTENSIONS = /\.(md|mdoc|mdx)$/i; +/** + * Return true when a GitHub repository owner is approved for browser-rendered remote docs. + */ +function isAllowedSourceOwner(owner) { + return ALLOWED_SOURCE_ORGS.includes(String(owner || '').toLowerCase()); +} + /** * Return true when a registry path points at a Markdown file instead of a docs directory. */ @@ -97,12 +104,12 @@ function normalizeSource(rawSource, index) { const repo = rawSource.repo; if (typeof repo !== 'string' || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) { - throw new Error(`Invalid docs-sources.yml: source "${id}" repo must look like "hypercerts-org/ePDS".`); + throw new Error(`Invalid docs-sources.yml: source "${id}" repo must look like "owner/repo".`); } const [owner, repoName] = repo.split('/'); - if (owner.toLowerCase() !== SOURCE_ORG) { - throw new Error(`Invalid docs-sources.yml: source "${id}" must use a ${SOURCE_ORG} repository so browser-rendered docs cannot load arbitrary hosts.`); + if (!isAllowedSourceOwner(owner)) { + throw new Error(`Invalid docs-sources.yml: source "${id}" must use one of these GitHub owners so browser-rendered docs cannot load arbitrary hosts: ${ALLOWED_SOURCE_ORGS.join(', ')}.`); } const branch = rawSource.branch; @@ -219,7 +226,8 @@ function resolveFrontmatterRawSource(frontmatter, sources = loadExternalDocSourc module.exports = { REGISTRY_PATH, - SOURCE_ORG, + ALLOWED_SOURCE_ORGS, + isAllowedSourceOwner, buildGitHubSourceUrl, buildRawGitHubUrl, encodeGitHubPath, diff --git a/markdoc/tags/remote-doc.markdoc.js b/markdoc/tags/remote-doc.markdoc.js index b4403e5..978da1d 100644 --- a/markdoc/tags/remote-doc.markdoc.js +++ b/markdoc/tags/remote-doc.markdoc.js @@ -1,6 +1,6 @@ /** - * Markdoc tag for rendering canonical Markdown from a registered Hypercerts GitHub repo at browser runtime. - * The source can be a docs-sources.yml id such as "epds" or a direct hypercerts-org GitHub Markdown URL. + * Markdoc tag for rendering canonical Markdown from a registered GitHub repo at browser runtime. + * The source can be a docs-sources.yml id such as "epds" or a direct approved GitHub Markdown URL. */ module.exports = { render: 'RemoteMarkdown', From e150893482b43fc999335a6e8898855c7aa20b3f Mon Sep 17 00:00:00 2001 From: kzoeps Date: Fri, 12 Jun 2026 13:49:28 +0600 Subject: [PATCH 09/16] docs-refresh: support explicit raw doc sources --- docs-sources.yml | 11 +-- docs/remote-markdown.md | 10 +-- lib/external-docs.js | 134 +++++++++++++++++++++++++++---- lib/generate-docs-fingerprint.js | 43 +++++++++- 4 files changed, 168 insertions(+), 30 deletions(-) diff --git a/docs-sources.yml b/docs-sources.yml index d6b64f4..1c8bea6 100644 --- a/docs-sources.yml +++ b/docs-sources.yml @@ -1,15 +1,12 @@ sources: - id: epds title: ePDS - repo: hypercerts-org/ePDS - branch: main - docsPath: docs - entrypoint: tutorial.md + rawUrl: https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/tutorial.md + sourceUrl: https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md routeBase: /architecture/epds - id: hyperindex-test title: Hyperindex test source - repo: gainforest/hyperindex - branch: skills-update - docsPath: docs/hyperindex.md + rawUrl: https://raw.githubusercontent.com/gainforest/hyperindex/skills-update/docs/hyperindex.md + sourceUrl: https://github.com/gainforest/hyperindex/blob/skills-update/docs/hyperindex.md routeBase: /tools/hyperindex diff --git a/docs/remote-markdown.md b/docs/remote-markdown.md index dbeaf01..3baf169 100644 --- a/docs/remote-markdown.md +++ b/docs/remote-markdown.md @@ -8,10 +8,8 @@ Register the source once in `docs-sources.yml`: sources: - id: epds title: ePDS - repo: hypercerts-org/ePDS - branch: main - docsPath: docs - entrypoint: tutorial.md + rawUrl: https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/tutorial.md + sourceUrl: https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md routeBase: /architecture/epds ``` @@ -45,7 +43,7 @@ A short unavailable-state fallback goes here. Keep it brief so the documentation ## Scheduled refresh and deploys -`npm run docs:fingerprint` reads `docs-sources.yml`, fetches GitHub tree metadata for each registered `docsPath`, and writes `public/docs-fingerprint.json` during the static build. The generated file includes a stable `combinedFingerprint`; timestamps are ignored by the comparison script. +`npm run docs:fingerprint` reads `docs-sources.yml`, fetches each registered `rawUrl`, hashes the content, and writes `public/docs-fingerprint.json` during the static build. Directory-style sources without `rawUrl` can still use `repo`, `branch`, and `docsPath` to fingerprint GitHub tree metadata. The generated file includes a stable `combinedFingerprint`; timestamps are ignored by the comparison script. `.github/workflows/docs-refresh.yml` runs hourly and through `workflow_dispatch`: @@ -74,5 +72,5 @@ Optional GitHub Actions secret: - Sources must be in approved GitHub owners: `hypercerts-org` or `gainforest`. - Registry ids must be lowercase, for example `epds` or `certified-group-service`. -- A source needs `entrypoint` when `docsPath` points at a directory and a page renders it through `{% remote-doc %}`. +- Prefer explicit `rawUrl` and `sourceUrl` values for page-level remote docs. A source only needs `entrypoint` when `docsPath` points at a directory and a page renders it through `{% remote-doc %}`. - The current page renderer is browser-runtime fetching, not server-side rendering. diff --git a/lib/external-docs.js b/lib/external-docs.js index c574624..ffdfdc3 100644 --- a/lib/external-docs.js +++ b/lib/external-docs.js @@ -59,6 +59,81 @@ function parseMarkdownFrontmatter(markdown, label = 'Markdown file') { } } +/** + * Parse and validate a raw.githubusercontent.com Markdown URL from docs-sources.yml. + */ +function parseRawGitHubMarkdownUrl(value, fieldName) { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a non-empty raw.githubusercontent.com URL.`); + } + + let url; + try { + url = new URL(value.trim()); + } catch { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a valid URL.`); + } + + if (url.protocol !== 'https:' || url.hostname !== 'raw.githubusercontent.com') { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must use https://raw.githubusercontent.com/...`); + } + + const [, owner, repoName, branch, ...fileParts] = url.pathname.split('/'); + if (!isAllowedSourceOwner(owner) || !repoName || !branch || fileParts.length === 0) { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must point at a Markdown file under one of these owners: ${ALLOWED_SOURCE_ORGS.join(', ')}.`); + } + + const filePath = normalizeRegistryPath(fileParts.join('/'), fieldName); + if (!isMarkdownFilePath(filePath)) { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must point at a Markdown file.`); + } + + return { + rawUrl: url.toString(), + owner, + repoName, + branch, + filePath, + repo: `${owner}/${repoName}`, + sourceUrl: `https://github.com/${owner}/${repoName}/blob/${encodeURIComponent(branch)}/${encodeGitHubPath(filePath)}`, + }; +} + +/** + * Parse and validate a github.com blob URL from docs-sources.yml. + */ +function parseGitHubSourceUrl(value, fieldName) { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a non-empty github.com blob URL.`); + } + + let url; + try { + url = new URL(value.trim()); + } catch { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a valid URL.`); + } + + if (url.protocol !== 'https:' || url.hostname !== 'github.com') { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must use https://github.com/...`); + } + + const [, owner, repoName, marker, branch, ...fileParts] = url.pathname.split('/'); + if (!isAllowedSourceOwner(owner) || marker !== 'blob' || !repoName || !branch || fileParts.length === 0) { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a GitHub blob URL under one of these owners: ${ALLOWED_SOURCE_ORGS.join(', ')}.`); + } + + const filePath = normalizeRegistryPath(fileParts.join('/'), fieldName); + return { + sourceUrl: url.toString(), + owner, + repoName, + branch, + filePath, + repo: `${owner}/${repoName}`, + }; +} + /** * Build the raw.githubusercontent.com URL for a registered external docs file. */ @@ -82,6 +157,13 @@ function getEntrypointPath(source) { return null; } +function assertConsistentSourceField(source, inferred, key, fieldName) { + if (!source[key] || !inferred?.[key]) return; + if (source[key].toLowerCase() !== inferred[key].toLowerCase()) { + throw new Error(`Invalid docs-sources.yml: ${fieldName} does not match ${key} "${source[key]}".`); + } +} + /** * Validate and normalize one source entry from docs-sources.yml. */ @@ -102,9 +184,16 @@ function normalizeSource(rawSource, index) { throw new Error(`Invalid docs-sources.yml: source "${id}" must set a human-readable title.`); } - const repo = rawSource.repo; + const rawInfo = rawSource.rawUrl + ? parseRawGitHubMarkdownUrl(rawSource.rawUrl, `source "${id}" rawUrl`) + : null; + const sourceInfo = rawSource.sourceUrl + ? parseGitHubSourceUrl(rawSource.sourceUrl, `source "${id}" sourceUrl`) + : null; + + const repo = rawSource.repo || rawInfo?.repo || sourceInfo?.repo; if (typeof repo !== 'string' || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) { - throw new Error(`Invalid docs-sources.yml: source "${id}" repo must look like "owner/repo".`); + throw new Error(`Invalid docs-sources.yml: source "${id}" must set repo like "owner/repo" or provide rawUrl.`); } const [owner, repoName] = repo.split('/'); @@ -112,21 +201,22 @@ function normalizeSource(rawSource, index) { throw new Error(`Invalid docs-sources.yml: source "${id}" must use one of these GitHub owners so browser-rendered docs cannot load arbitrary hosts: ${ALLOWED_SOURCE_ORGS.join(', ')}.`); } - const branch = rawSource.branch; + const branch = rawSource.branch || rawInfo?.branch || sourceInfo?.branch; if (typeof branch !== 'string' || branch.trim() === '') { - throw new Error(`Invalid docs-sources.yml: source "${id}" must set the branch to read, for example "main".`); + throw new Error(`Invalid docs-sources.yml: source "${id}" must set branch or provide rawUrl.`); + } + + const docsPath = rawSource.docsPath + ? normalizeRegistryPath(rawSource.docsPath, `source "${id}" docsPath`) + : rawInfo?.filePath || sourceInfo?.filePath; + if (!docsPath) { + throw new Error(`Invalid docs-sources.yml: source "${id}" must set docsPath or provide rawUrl.`); } - const docsPath = normalizeRegistryPath(rawSource.docsPath, `source "${id}" docsPath`); const entrypoint = rawSource.entrypoint ? normalizeRegistryPath(rawSource.entrypoint, `source "${id}" entrypoint`) : ''; - const routeBase = rawSource.routeBase || rawSource.route || ''; - if (routeBase && (typeof routeBase !== 'string' || !routeBase.startsWith('/'))) { - throw new Error(`Invalid docs-sources.yml: source "${id}" routeBase must start with "/".`); - } - const source = { id, title: title.trim(), @@ -136,17 +226,27 @@ function normalizeSource(rawSource, index) { branch: branch.trim(), docsPath, entrypoint, - routeBase, + routeBase: rawSource.routeBase || rawSource.route || '', }; - const entrypointPath = getEntrypointPath(source); + assertConsistentSourceField(source, rawInfo, 'repo', `source "${id}" rawUrl`); + assertConsistentSourceField(source, rawInfo, 'branch', `source "${id}" rawUrl`); + assertConsistentSourceField(source, sourceInfo, 'repo', `source "${id}" sourceUrl`); + assertConsistentSourceField(source, sourceInfo, 'branch', `source "${id}" sourceUrl`); + + if (source.routeBase && (typeof source.routeBase !== 'string' || !source.routeBase.startsWith('/'))) { + throw new Error(`Invalid docs-sources.yml: source "${id}" routeBase must start with "/".`); + } + + const entrypointPath = rawInfo?.filePath || sourceInfo?.filePath || getEntrypointPath(source); return { ...source, entrypointPath, - rawUrl: entrypointPath ? buildRawGitHubUrl(source, entrypointPath) : '', - sourceUrl: entrypointPath + rawUrl: rawInfo?.rawUrl || (entrypointPath ? buildRawGitHubUrl(source, entrypointPath) : ''), + sourceUrl: rawSource.sourceUrl || rawInfo?.sourceUrl || (entrypointPath ? buildGitHubSourceUrl(source, entrypointPath, 'blob') - : buildGitHubSourceUrl(source, docsPath, 'tree'), + : buildGitHubSourceUrl(source, docsPath, 'tree')), + fingerprintMode: rawInfo ? 'rawUrl' : 'gitTree', }; } @@ -212,7 +312,7 @@ function resolveFrontmatterRawSource(frontmatter, sources = loadExternalDocSourc throw new Error(`Unknown externalDoc "${id}". Add it to docs-sources.yml or remove the externalDoc frontmatter.`); } if (!source.rawUrl) { - throw new Error(`External doc "${id}" does not define a renderable Markdown file. Set entrypoint in docs-sources.yml or point docsPath at a Markdown file.`); + throw new Error(`External doc "${id}" does not define a renderable Markdown file. Set rawUrl in docs-sources.yml or point docsPath at a Markdown file.`); } return { rawUrl: source.rawUrl, label: `externalDoc "${id}"` }; } @@ -235,7 +335,9 @@ module.exports = { getEntrypointPath, isMarkdownFilePath, loadExternalDocSources, + parseGitHubSourceUrl, parseMarkdownFrontmatter, + parseRawGitHubMarkdownUrl, resolveFrontmatterRawSource, sourceToPublicManifestEntry, }; diff --git a/lib/generate-docs-fingerprint.js b/lib/generate-docs-fingerprint.js index 2028953..2857a8d 100644 --- a/lib/generate-docs-fingerprint.js +++ b/lib/generate-docs-fingerprint.js @@ -69,10 +69,34 @@ async function fetchGitHubJson(url, token) { return response.json(); } +/** + * Fetch a registry rawUrl and return one content-hash entry for fingerprinting. + */ +async function fetchRawUrlEntry(source, token) { + const headers = { + 'User-Agent': 'hypercerts-docs-refresh', + }; + + if (token) headers.Authorization = `Bearer ${token}`; + + const response = await fetch(source.rawUrl, { headers, cache: 'no-store' }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`${source.rawUrl} returned ${response.status} ${response.statusText || ''}. Check rawUrl and DOCS_SOURCE_TOKEN. ${body}`.trim()); + } + + const content = await response.text(); + return [{ + path: source.entrypointPath || source.docsPath || source.rawUrl, + sha: sha256(content), + size: Buffer.byteLength(content), + }]; +} + /** * Fetch the Git tree for a source branch and return blob entries under docsPath. */ -async function fetchSourceEntries(source, token) { +async function fetchGitTreeEntries(source, token) { const treeUrl = `https://api.github.com/repos/${source.owner}/${source.repoName}/git/trees/${encodeURIComponent(source.branch)}?recursive=1`; const tree = await fetchGitHubJson(treeUrl, token); @@ -98,6 +122,17 @@ async function fetchSourceEntries(source, token) { return files; } +/** + * Fetch the fingerprint entries for one source. + */ +async function fetchSourceEntries(source, token) { + if (source.fingerprintMode === 'rawUrl') { + return fetchRawUrlEntry(source, token); + } + + return fetchGitTreeEntries(source, token); +} + /** * Compute the per-source and combined fingerprints for all registered external docs. */ @@ -116,6 +151,9 @@ async function generateFingerprint() { docsPath: source.docsPath, entrypoint: source.entrypoint || '', routeBase: source.routeBase || '', + rawUrl: source.rawUrl || '', + sourceUrl: source.sourceUrl || '', + fingerprintMode: source.fingerprintMode || 'gitTree', files, }; const fingerprint = sha256(stableStringify(stableSource)); @@ -127,6 +165,9 @@ async function generateFingerprint() { docsPath: source.docsPath, entrypoint: source.entrypoint || undefined, routeBase: source.routeBase || undefined, + rawUrl: source.rawUrl || undefined, + sourceUrl: source.sourceUrl || undefined, + fingerprintMode: source.fingerprintMode || 'gitTree', fileCount: files.length, fingerprint, files, From 34f79db12321072e5049f91833cac55d6905901b Mon Sep 17 00:00:00 2001 From: kzoeps Date: Fri, 12 Jun 2026 14:07:47 +0600 Subject: [PATCH 10/16] docs-refresh: use hyperindex as remote test source --- docs-sources.yml | 6 ------ pages/architecture/epds.md | 4 ++-- pages/tools/hyperindex.md | 9 +++++++++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs-sources.yml b/docs-sources.yml index 1c8bea6..87096f5 100644 --- a/docs-sources.yml +++ b/docs-sources.yml @@ -1,10 +1,4 @@ sources: - - id: epds - title: ePDS - rawUrl: https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/tutorial.md - sourceUrl: https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md - routeBase: /architecture/epds - - id: hyperindex-test title: Hyperindex test source rawUrl: https://raw.githubusercontent.com/gainforest/hyperindex/skills-update/docs/hyperindex.md diff --git a/pages/architecture/epds.md b/pages/architecture/epds.md index 172fa24..f2d5159 100644 --- a/pages/architecture/epds.md +++ b/pages/architecture/epds.md @@ -1,10 +1,10 @@ --- title: ePDS (extended PDS) description: How the ePDS adds email/OTP login on top of AT Protocol without changing the standard OAuth flow for apps. -externalDoc: epds +rawUrl: "https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/tutorial.md" --- -{% remote-doc source="epds" %} +{% remote-doc source="https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md" %} # ePDS docs unavailable diff --git a/pages/tools/hyperindex.md b/pages/tools/hyperindex.md index c2c7222..77ecbc3 100644 --- a/pages/tools/hyperindex.md +++ b/pages/tools/hyperindex.md @@ -1,8 +1,15 @@ --- title: Hyperindex description: A Go ATProto indexer that indexes hypercert records and exposes them via GraphQL. +externalDoc: hyperindex-test --- +{% remote-doc source="hyperindex-test" %} + +# Hyperindex docs unavailable + +The canonical Hyperindex docs could not be loaded from GitHub or the build-time raw cache. Showing the local fallback below. + # Hyperindex Hyperindex (`hi`) is a Go AT Protocol AppView server that indexes records and exposes them via GraphQL. Use it to: @@ -333,3 +340,5 @@ docker compose up --build - [GitHub repository](https://github.com/gainforest/hyperindex) — upstream GainForest repository that the Certified indexer is forked from - [Certified Services](/reference/certified-services#indexers) — current public indexer endpoints - [Building on Hypercerts](/getting-started/building-on-hypercerts) — integration patterns for platforms and tools + +{% /remote-doc %} From 62292779357f78b80d549d30342a5976b910f452 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Fri, 12 Jun 2026 14:22:46 +0600 Subject: [PATCH 11/16] docs-refresh: render remote docs at build time --- .gitignore | 1 + components/CopyRawButton.js | 41 +--- components/RemoteMarkdown.js | 255 +++---------------------- docs/remote-markdown.md | 10 +- lib/generate-external-docs-manifest.js | 155 ++++++++++++++- 5 files changed, 186 insertions(+), 276 deletions(-) diff --git a/.gitignore b/.gitignore index 58647af..77eaaea 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ public/search-index.json public/sitemap.xml current-docs-fingerprint.json deployed-docs-fingerprint.json +lib/external-docs-content.json lib/lastUpdated.json diff --git a/components/CopyRawButton.js b/components/CopyRawButton.js index aa946c1..34ebe45 100644 --- a/components/CopyRawButton.js +++ b/components/CopyRawButton.js @@ -1,30 +1,15 @@ -import { useEffect, useState } from 'react'; +import { useState } from 'react'; import { useRouter } from 'next/router'; - -const EXTERNAL_DOCS_MANIFEST_URL = '/external-docs.json'; +import externalDocsContent from '../lib/external-docs-content.json'; function getGeneratedRawUrl(currentPath) { if (currentPath === '/') return '/raw/index.md'; return `/raw${currentPath}.md`; } -async function getExternalDocRawUrl(externalDoc, signal) { - const response = await fetch(EXTERNAL_DOCS_MANIFEST_URL, { - cache: 'no-store', - signal, - }); - - if (!response.ok) { - throw new Error(`Could not load ${EXTERNAL_DOCS_MANIFEST_URL}. Run npm run generate:external-docs before starting the docs site.`); - } - - const manifest = await response.json(); - const rawUrl = manifest?.sources?.[externalDoc]?.rawUrl; - if (!rawUrl) { - throw new Error(`External docs source "${externalDoc}" does not define a raw Markdown URL.`); - } - - return rawUrl; +function getExternalDocRawUrl(externalDoc) { + const entryId = externalDocsContent.sources?.[externalDoc]; + return entryId ? externalDocsContent.entries?.[entryId]?.rawUrl : null; } /** @@ -35,26 +20,12 @@ export function CopyRawButton({ frontmatter }) { const [copied, setCopied] = useState(false); const [copyError, setCopyError] = useState(false); const [isCopying, setIsCopying] = useState(false); - const [externalRawUrl, setExternalRawUrl] = useState(null); const router = useRouter(); const currentPath = router.asPath.split('#')[0].split('?')[0] || '/'; const generatedRawUrl = getGeneratedRawUrl(currentPath); + const externalRawUrl = frontmatter?.externalDoc ? getExternalDocRawUrl(frontmatter.externalDoc) : null; const rawUrl = frontmatter?.rawUrl || externalRawUrl || generatedRawUrl; - useEffect(() => { - if (!frontmatter?.externalDoc || frontmatter?.rawUrl) { - setExternalRawUrl(null); - return undefined; - } - - const controller = new AbortController(); - getExternalDocRawUrl(frontmatter.externalDoc, controller.signal) - .then(setExternalRawUrl) - .catch(() => setExternalRawUrl(null)); - - return () => controller.abort(); - }, [frontmatter?.externalDoc, frontmatter?.rawUrl]); - const handleCopy = async () => { setIsCopying(true); diff --git a/components/RemoteMarkdown.js b/components/RemoteMarkdown.js index 6249b14..6e58ad1 100644 --- a/components/RemoteMarkdown.js +++ b/components/RemoteMarkdown.js @@ -1,6 +1,6 @@ -import React, { useEffect, useMemo, useState, useContext } from 'react'; -import { useRouter } from 'next/router'; +import React, { useContext, useMemo } from 'react'; import Markdoc from '@markdoc/markdoc'; +import externalDocsContent from '../lib/external-docs-content.json'; import tags from '../markdoc/tags'; import { fence, heading, link } from '../markdoc/nodes'; import { Callout } from './Callout'; @@ -16,9 +16,6 @@ import { HeroBanner } from './HeroBanner'; import { CardGrid } from './CardGrid'; import { MermaidDiagram } from './MermaidDiagram'; -const ALLOWED_SOURCE_ORGS = ['hypercerts-org', 'gainforest']; -const EXTERNAL_DOCS_MANIFEST_URL = '/external-docs.json'; -let externalDocsManifestPromise = null; const RemoteMarkdownContext = React.createContext(null); const markdocConfig = { @@ -26,123 +23,6 @@ const markdocConfig = { nodes: { fence, heading, link }, }; -function isAllowedSourceOwner(owner) { - return ALLOWED_SOURCE_ORGS.includes(String(owner || '').toLowerCase()); -} - -/** - * Convert an allowed GitHub Markdown URL into the raw URL used by the browser fetch. - * Only approved GitHub owners are allowed so docs pages cannot become an open proxy. - */ -function resolveGitHubMarkdownSource(source) { - const url = new URL(source); - - if (url.hostname === 'github.com') { - const [, owner, repo, marker, ref, ...fileParts] = url.pathname.split('/'); - if (!isAllowedSourceOwner(owner) || marker !== 'blob' || !repo || !ref || fileParts.length === 0) { - throw new Error(`Remote docs must use a GitHub blob URL under one of these owners: ${ALLOWED_SOURCE_ORGS.join(', ')}.`); - } - - const filePath = fileParts.join('/'); - return { - sourceUrl: url.toString(), - rawUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${filePath}`, - owner, - repo, - ref, - filePath, - }; - } - - if (url.hostname === 'raw.githubusercontent.com') { - const [, owner, repo, ref, ...fileParts] = url.pathname.split('/'); - if (!isAllowedSourceOwner(owner) || !repo || !ref || fileParts.length === 0) { - throw new Error(`Remote docs must use a raw.githubusercontent.com URL under one of these owners: ${ALLOWED_SOURCE_ORGS.join(', ')}.`); - } - - const filePath = fileParts.join('/'); - return { - sourceUrl: `https://github.com/${owner}/${repo}/blob/${ref}/${filePath}`, - rawUrl: url.toString(), - owner, - repo, - ref, - filePath, - }; - } - - throw new Error('Remote docs can only be loaded from github.com or raw.githubusercontent.com.'); -} - -function isHttpUrl(source) { - try { - const url = new URL(source); - return url.protocol === 'https:' || url.protocol === 'http:'; - } catch { - return false; - } -} - -function throwIfAborted(signal) { - if (!signal?.aborted) return; - const error = new Error('Aborted'); - error.name = 'AbortError'; - throw error; -} - -async function loadExternalDocsManifest() { - if (!externalDocsManifestPromise) { - externalDocsManifestPromise = fetch(EXTERNAL_DOCS_MANIFEST_URL, { - cache: 'no-store', - }) - .then((response) => { - if (!response.ok) { - throw new Error(`${EXTERNAL_DOCS_MANIFEST_URL} returned ${response.status} ${response.statusText || 'without an external docs manifest'}. Run npm run generate:external-docs before starting the docs site.`); - } - return response.json(); - }) - .catch((error) => { - externalDocsManifestPromise = null; - throw error; - }); - } - - return externalDocsManifestPromise; -} - -function resolveRegisteredMarkdownSource(source, manifest) { - const id = String(source || '').trim(); - if (!/^[a-z0-9][a-z0-9-]*$/.test(id)) { - throw new Error('Remote doc source must be a GitHub URL or a docs-sources.yml id like "epds".'); - } - - const entry = manifest?.sources?.[id]; - if (!entry) { - throw new Error(`No external docs source "${id}" was found in ${EXTERNAL_DOCS_MANIFEST_URL}. Add it to docs-sources.yml and rebuild.`); - } - - const [owner, repo] = String(entry.repo || '').split('/'); - if (!isAllowedSourceOwner(owner) || !repo || !entry.branch || !entry.filePath || !entry.rawUrl || !entry.sourceUrl) { - throw new Error(`External docs source "${id}" is incomplete. Set repo, branch, docsPath, and entrypoint in docs-sources.yml, then rebuild.`); - } - - return { - sourceUrl: entry.sourceUrl, - rawUrl: entry.rawUrl, - owner, - repo, - ref: entry.branch, - filePath: entry.filePath, - }; -} - -async function resolveMarkdownSource(source, signal) { - if (isHttpUrl(source)) return resolveGitHubMarkdownSource(source); - const manifest = await loadExternalDocsManifest(); - throwIfAborted(signal); - return resolveRegisteredMarkdownSource(source, manifest); -} - /** * Remove optional YAML frontmatter before parsing remote Markdown with Markdoc. */ @@ -164,7 +44,7 @@ function resolveRemoteHref(href, source) { } const sourceDir = source.filePath.split('/').slice(0, -1).join('/'); - const basePath = `/${source.owner}/${source.repo}/blob/${source.ref}/${sourceDir ? `${sourceDir}/` : ''}`; + const basePath = `/${source.owner}/${source.repoName}/blob/${source.branch}/${sourceDir ? `${sourceDir}/` : ''}`; const resolved = new URL(href, `https://github.com${basePath}`); const [, owner, repo, , ref, ...repoPathParts] = resolved.pathname.split('/'); const repoPath = repoPathParts.join('/'); @@ -201,97 +81,34 @@ const remoteMarkdocComponents = { MermaidDiagram, }; -function getRawCacheUrl(currentPath) { - if (currentPath === '/') return '/raw/index.md'; - return `/raw${currentPath.replace(/\.html$/, '')}.md`; -} - -async function fetchMarkdown(url, signal, errorPrefix) { - const response = await fetch(url, { - cache: 'no-store', - signal, - }); - - if (!response.ok) { - throw new Error(`${errorPrefix} returned ${response.status} ${response.statusText || 'without the Markdown file'}.`); - } - - return response.text(); -} - function transformMarkdown(markdown) { const ast = Markdoc.parse(stripFrontmatter(markdown)); return Markdoc.transform(ast, markdocConfig); } +function getBuildTimeEntry(source) { + const id = externalDocsContent.sources?.[source] || source; + const mappedId = externalDocsContent.bySourceUrl?.[source] || externalDocsContent.byRawUrl?.[source] || id; + return externalDocsContent.entries?.[mappedId] || null; +} + /** - * Runtime Markdoc renderer for Markdown files that live in Hypercerts service repos. - * It tries the live GitHub raw source first, then the build-time `/raw` cache, then the wrapped local fallback. + * Build-time Markdoc renderer for Markdown files that live in registered source repos. + * `npm run generate:external-docs` fetches the raw Markdown before `next build`, so static exports contain the rendered HTML and do not fetch GitHub in the browser. */ export function RemoteMarkdown({ source, children }) { - const router = useRouter(); - const currentPath = router.asPath.split('#')[0].split('?')[0] || '/'; - const rawCacheUrl = getRawCacheUrl(currentPath); - const [markdown, setMarkdown] = useState(null); - const [error, setError] = useState(null); - const [isLoading, setIsLoading] = useState(true); - const [sourceInfo, setSourceInfo] = useState(null); - - useEffect(() => { - const controller = new AbortController(); - setMarkdown(null); - setError(null); - setSourceInfo(null); - setIsLoading(true); - - async function loadRemoteMarkdown() { - let nextSourceInfo; - - try { - nextSourceInfo = await resolveMarkdownSource(source, controller.signal); - setSourceInfo(nextSourceInfo); - } catch (err) { - if (err.name === 'AbortError') return; - setError(err); - return; - } - - let githubError = null; - - try { - const nextMarkdown = await fetchMarkdown(nextSourceInfo.rawUrl, controller.signal, 'GitHub'); - transformMarkdown(nextMarkdown); - setMarkdown(nextMarkdown); - return; - } catch (err) { - if (err.name === 'AbortError') return; - githubError = err; - } - - try { - const cachedMarkdown = await fetchMarkdown(rawCacheUrl, controller.signal, 'Build-time raw cache'); - transformMarkdown(cachedMarkdown); - setMarkdown(cachedMarkdown); - } catch (err) { - if (err.name === 'AbortError') return; - setError(new Error(`${githubError.message} The build-time raw cache also failed: ${err.message}`)); - } finally { - setIsLoading(false); - } - } - - loadRemoteMarkdown().finally(() => { - if (!controller.signal.aborted) setIsLoading(false); - }); - - return () => controller.abort(); - }, [rawCacheUrl, source]); + const entry = getBuildTimeEntry(source); const renderedState = useMemo(() => { - if (!markdown || !sourceInfo) return { renderedContent: null, renderError: null }; + if (!entry?.markdown) { + return { + renderedContent: null, + renderError: new Error(`No build-time Markdown was generated for remote docs source "${source}". Add it to docs-sources.yml or set rawUrl frontmatter, then run npm run generate:external-docs.`), + }; + } try { - const content = transformMarkdown(markdown); + const content = transformMarkdown(entry.markdown); return { renderedContent: Markdoc.renderers.react(content, React, { components: remoteMarkdocComponents, @@ -301,47 +118,29 @@ export function RemoteMarkdown({ source, children }) { } catch (err) { return { renderedContent: null, renderError: err }; } - }, [markdown, sourceInfo]); - const { renderedContent, renderError } = renderedState; - const displayError = error || renderError; + }, [entry, source]); - useEffect(() => { - if (!renderedContent) return undefined; - - const frame = window.requestAnimationFrame(() => { - window.dispatchEvent(new CustomEvent('remote-docs:loaded')); - }); - - return () => window.cancelAnimationFrame(frame); - }, [renderedContent]); + const { renderedContent, renderError } = renderedState; if (renderedContent) { return ( - + {renderedContent} ); } - if (isLoading && !displayError) { - return ( -
- Loading the canonical docs from GitHub… -
- ); - } - return ( <>
- Could not load the canonical docs.{' '} - The browser could not load the registered source, the live GitHub raw file, or the build-time raw cache. Showing the local fallback below. Try refreshing, or edit{' '} - {sourceInfo ? ( - the source file + Could not render the canonical docs.{' '} + The static build did not include renderable Markdown for this source. Showing the local fallback below. Try rebuilding the site, or edit{' '} + {entry?.sourceUrl ? ( + the source file ) : ( 'the configured source URL' )}{' '} - if the URL is wrong. Details: {displayError?.message || 'No renderable Markdown source was available.'} + if the URL is wrong. Details: {renderError?.message || 'No renderable Markdown source was available.'}
{children} diff --git a/docs/remote-markdown.md b/docs/remote-markdown.md index 3baf169..3248a5f 100644 --- a/docs/remote-markdown.md +++ b/docs/remote-markdown.md @@ -31,11 +31,11 @@ A short unavailable-state fallback goes here. Keep it brief so the documentation ## How it works - The site remains a static Next.js export. -- `npm run generate:external-docs` reads `docs-sources.yml` and writes `/external-docs.json` for browser components. -- The browser resolves `{% remote-doc source="epds" %}` through `/external-docs.json`, then fetches the source file from `raw.githubusercontent.com` at runtime first. -- If the live GitHub fetch fails, the browser falls back to the build-time copy in `/raw`. +- `npm run generate:external-docs` reads `docs-sources.yml`, fetches registered raw Markdown, writes `/external-docs.json`, and writes build-time content to `lib/external-docs-content.json`. +- `{% remote-doc source="epds" %}` resolves against that generated build-time content during `next build`. +- The exported page contains the rendered remote Markdown HTML; the browser does not fetch GitHub to display the docs. - Build-time search and raw-page generation use `externalDoc` frontmatter, so search and `/raw` use the canonical Markdown instead of the local fallback. -- The fetched Markdown is parsed with the same Markdoc tags and nodes used by local pages. +- The generated Markdown is parsed with the same Markdoc tags and nodes used by local pages. - Fenced `mermaid` diagrams render as SVGs in the browser through the Mermaid npm package. - Relative links in the remote Markdown point back to the source GitHub repository. - `Copy raw` and `View raw` use the page's registered `externalDoc` source when it is set. @@ -73,4 +73,4 @@ Optional GitHub Actions secret: - Sources must be in approved GitHub owners: `hypercerts-org` or `gainforest`. - Registry ids must be lowercase, for example `epds` or `certified-group-service`. - Prefer explicit `rawUrl` and `sourceUrl` values for page-level remote docs. A source only needs `entrypoint` when `docsPath` points at a directory and a page renders it through `{% remote-doc %}`. -- The current page renderer is browser-runtime fetching, not server-side rendering. +- Registry-backed remote docs render during the static build. If generated build-time content is missing, the page shows the wrapped local fallback. diff --git a/lib/generate-external-docs-manifest.js b/lib/generate-external-docs-manifest.js index 68e8743..a001987 100644 --- a/lib/generate-external-docs-manifest.js +++ b/lib/generate-external-docs-manifest.js @@ -1,16 +1,126 @@ -const { mkdirSync, writeFileSync } = require('fs'); -const { dirname, join } = require('path'); +const crypto = require('crypto'); +const { + mkdirSync, + readFileSync, + readdirSync, + statSync, + writeFileSync, +} = require('fs'); +const { dirname, join, relative } = require('path'); const { loadExternalDocSources, + parseMarkdownFrontmatter, + parseRawGitHubMarkdownUrl, sourceToPublicManifestEntry, } = require('./external-docs'); -const OUTPUT = join(__dirname, '..', 'public', 'external-docs.json'); +const MANIFEST_OUTPUT = join(__dirname, '..', 'public', 'external-docs.json'); +const CONTENT_OUTPUT = join(__dirname, 'external-docs-content.json'); +const PAGES_DIR = join(__dirname, '..', 'pages'); + +/** + * Compute a short stable identifier for raw URL sources that are declared directly in page frontmatter. + */ +function hashId(value) { + return crypto.createHash('sha256').update(value).digest('hex').slice(0, 12); +} + +function walkDir(dir) { + const results = []; + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) { + results.push(...walkDir(full)); + } else if (full.endsWith('.md')) { + results.push(full); + } + } + return results; +} + +/** + * Fetch one Markdown source during the static build so pages can render without browser fetches. + */ +async function fetchMarkdown(rawUrl) { + const headers = { + 'User-Agent': 'hypercerts-docs-build', + }; + const token = process.env.DOCS_SOURCE_TOKEN || process.env.GITHUB_TOKEN || process.env.GH_TOKEN || ''; + if (token) headers.Authorization = `Bearer ${token}`; + + const response = await fetch(rawUrl, { headers, cache: 'no-store' }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`${rawUrl} returned ${response.status} ${response.statusText || ''}. Check docs-sources.yml, page rawUrl frontmatter, and DOCS_SOURCE_TOKEN. ${body}`.trim()); + } + + return response.text(); +} + +function entryFromRegistrySource(source) { + return { + id: source.id, + registryId: source.id, + title: source.title, + repo: source.repo, + owner: source.owner, + repoName: source.repoName, + branch: source.branch, + docsPath: source.docsPath, + entrypoint: source.entrypoint || undefined, + routeBase: source.routeBase || undefined, + sourceUrl: source.sourceUrl, + rawUrl: source.rawUrl, + filePath: source.entrypointPath || source.docsPath, + }; +} + +function entryFromRawFrontmatter(file, frontmatter) { + if (!frontmatter.rawUrl) return null; + + const rawInfo = parseRawGitHubMarkdownUrl(frontmatter.rawUrl, `${relative(PAGES_DIR, file)} rawUrl`); + return { + id: `raw:${hashId(rawInfo.rawUrl)}`, + title: frontmatter.title ? String(frontmatter.title) : rawInfo.filePath, + repo: rawInfo.repo, + owner: rawInfo.owner, + repoName: rawInfo.repoName, + branch: rawInfo.branch, + docsPath: rawInfo.filePath, + routeBase: undefined, + sourceUrl: rawInfo.sourceUrl, + rawUrl: rawInfo.rawUrl, + filePath: rawInfo.filePath, + }; +} + +function addContentEntry(content, entry, sourceKey) { + if (!entry?.rawUrl) return; + + const existingId = content.byRawUrl[entry.rawUrl]; + const id = existingId || entry.id; + + if (!content.entries[id]) { + content.entries[id] = { ...entry, id }; + content.byRawUrl[entry.rawUrl] = id; + content.bySourceUrl[entry.sourceUrl] = id; + } + + if (sourceKey) { + content.sources[sourceKey] = id; + } +} + +async function hydrateContentEntries(content) { + for (const entry of Object.values(content.entries)) { + entry.markdown = await fetchMarkdown(entry.rawUrl); + } +} /** - * Generate the browser-readable manifest used to resolve remote-doc registry ids. + * Generate the browser-readable manifest and build-time Markdown content used by remote docs pages. */ -function generateExternalDocsManifest() { +async function generateExternalDocs() { const sources = loadExternalDocSources(); const manifest = { schemaVersion: 1, @@ -19,9 +129,38 @@ function generateExternalDocsManifest() { ), }; - mkdirSync(dirname(OUTPUT), { recursive: true }); - writeFileSync(OUTPUT, `${JSON.stringify(manifest, null, 2)}\n`); + const content = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + entries: {}, + sources: {}, + byRawUrl: {}, + bySourceUrl: {}, + }; + + for (const source of sources) { + addContentEntry(content, entryFromRegistrySource(source), source.id); + } + + for (const file of walkDir(PAGES_DIR)) { + const markdown = readFileSync(file, 'utf8'); + const frontmatter = parseMarkdownFrontmatter(markdown, relative(PAGES_DIR, file)); + addContentEntry(content, entryFromRawFrontmatter(file, frontmatter)); + } + + await hydrateContentEntries(content); + + mkdirSync(dirname(MANIFEST_OUTPUT), { recursive: true }); + writeFileSync(MANIFEST_OUTPUT, `${JSON.stringify(manifest, null, 2)}\n`); + + mkdirSync(dirname(CONTENT_OUTPUT), { recursive: true }); + writeFileSync(CONTENT_OUTPUT, `${JSON.stringify(content, null, 2)}\n`); + console.log(`Generated external docs manifest for ${sources.length} source${sources.length === 1 ? '' : 's'}`); + console.log(`Generated build-time external docs content for ${Object.keys(content.entries).length} Markdown source${Object.keys(content.entries).length === 1 ? '' : 's'}`); } -generateExternalDocsManifest(); +generateExternalDocs().catch((error) => { + console.error(error); + process.exit(1); +}); From 0e8dd5574ff62b19732bacf03667fc646cc11eea Mon Sep 17 00:00:00 2001 From: kzoeps Date: Fri, 26 Jun 2026 16:55:39 +0200 Subject: [PATCH 12/16] tools: document Hypercerts agent skills --- lib/navigation.js | 1 + pages/tools/hypercerts-agent-skills.md | 51 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 pages/tools/hypercerts-agent-skills.md diff --git a/lib/navigation.js b/lib/navigation.js index 36a63fd..b208671 100644 --- a/lib/navigation.js +++ b/lib/navigation.js @@ -53,6 +53,7 @@ export const navigation = [ children: [ { title: "Scaffold Starter App", path: "/tools/scaffold" }, { title: "Hypercerts CLI", path: "/tools/hypercerts-cli" }, + { title: "Agent Skills", path: "/tools/hypercerts-agent-skills" }, { title: "Hyperindex", path: "/tools/hyperindex" }, { title: "Hyperscan", path: "/tools/hyperscan" }, { title: "Hyperboards", path: "/tools/hyperboards" }, diff --git a/pages/tools/hypercerts-agent-skills.md b/pages/tools/hypercerts-agent-skills.md new file mode 100644 index 0000000..194f69b --- /dev/null +++ b/pages/tools/hypercerts-agent-skills.md @@ -0,0 +1,51 @@ +--- +title: Hypercerts Agent Skills +description: Agent skills that help coding agents work across the Hypercerts stack. +--- + +# Hypercerts Agent Skills + +Hypercerts Agent Skills help coding agents work across the Hypercerts stack. They are intended for agents and agent harnesses, not as a runtime dependency for Hypercerts applications. + +The main entry point is the [`hypercerts`](https://github.com/hypercerts-org/skills/tree/main/skills/hypercerts) meta-skill. It is a small catalog skill that helps an agent choose and install the right focused skill for a task. + +## Install + +Install the meta-skill from the [Hypercerts skills repository](https://github.com/hypercerts-org/skills): + +```bash +npx skills add hypercerts-org/skills --skill hypercerts +``` + +List the skills available in the repository: + +```bash +npx skills add hypercerts-org/skills --list +``` + +## What the meta-skill does + +The `hypercerts` meta-skill keeps the first step small: + +- Choose the smallest focused skill for the task. +- Install that focused skill from its source repository. +- Read the focused skill's own `SKILL.md` instructions. +- Continue the task using the focused skill instead of keeping all workflow guidance in one large skill. + +This keeps task-specific instructions close to the projects they describe while still giving agents one Hypercerts entry point. + +## Focused skills + +| Task area | Focused skill | +| --- | --- | +| ePDS login, AT Protocol OAuth, OTP login, email-first auth, PAR, PKCE, and DPoP flows | `epds-login` | +| Hyperindex GraphQL queries, hosted endpoints, filtering, pagination, sorting, and consumer query workflows | `hyperindex` | +| Certified Group Service app development, group-owned records, member and role management, blob uploads, and API keys | `app-development-with-cgs` | +| Certified Organization Labeler / OrgLabeler labels for filtering certified actors and hiding likely test data | `orglabeler` | +| Hypercerts lexicons, generated TypeScript types, validators, and reading or writing Hypercerts records | `building-with-hypercerts-lexicons` | + +See the repository's [skill map](https://github.com/hypercerts-org/skills/blob/main/skills/hypercerts/references/skill-map.md) for the current source repositories and fallback install URLs. + +{% callout type="note" %} +Agent Skills do not define a portable way for one skill to invoke another skill directly. The meta-skill points the agent to the right focused skill and install path; the agent then reads and follows that focused skill's own instructions. +{% /callout %} From 1887c03fb4bcc92bd66a73c1e5a5a48c16b5cef3 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Tue, 14 Jul 2026 14:47:17 +0200 Subject: [PATCH 13/16] external-docs: simplify build-time source rendering --- .github/workflows/docs-refresh-pr-dry-run.yml | 25 +- .github/workflows/docs-refresh.yml | 6 +- .gitignore | 1 - components/AnnouncementBanner.js | 4 +- components/CopyRawButton.js | 16 +- components/Layout.js | 2 +- components/RemoteMarkdown.js | 148 -------- components/TableOfContents.js | 50 +-- docs-sources.yml | 12 +- docs/remote-markdown.md | 87 ++--- lib/external-doc-links.js | 47 +++ lib/external-doc-page.js | 45 +++ lib/external-docs-loader.js | 45 +++ lib/external-docs-snapshot.js | 166 +++++++++ lib/external-docs.js | 339 +++++------------- lib/generate-docs-fingerprint.js | 199 +++------- lib/generate-external-docs-manifest.js | 160 +-------- lib/generate-last-updated.js | 14 +- lib/generate-raw-pages.js | 21 +- lib/generate-search-index.js | 25 +- markdoc/nodes/document.markdoc.js | 37 ++ markdoc/nodes/image.markdoc.js | 18 + markdoc/nodes/index.js | 2 + markdoc/nodes/link.markdoc.js | 19 +- markdoc/tags/br.markdoc.js | 4 + markdoc/tags/index.js | 4 +- markdoc/tags/remote-doc.markdoc.js | 13 - next.config.js | 8 + package.json | 3 +- pages/_app.js | 3 - pages/architecture/epds.md | 12 +- pages/lexicons/hypercerts-lexicons/index.md | 2 +- pages/tools/hyperindex.md | 339 ------------------ styles/globals.css | 20 -- test/external-doc-links.test.js | 37 ++ test/external-docs-snapshot.test.js | 119 ++++++ test/external-docs.test.js | 218 +++++++++++ 37 files changed, 1030 insertions(+), 1240 deletions(-) delete mode 100644 components/RemoteMarkdown.js create mode 100644 lib/external-doc-links.js create mode 100644 lib/external-doc-page.js create mode 100644 lib/external-docs-loader.js create mode 100644 lib/external-docs-snapshot.js create mode 100644 markdoc/nodes/document.markdoc.js create mode 100644 markdoc/nodes/image.markdoc.js create mode 100644 markdoc/tags/br.markdoc.js delete mode 100644 markdoc/tags/remote-doc.markdoc.js create mode 100644 test/external-doc-links.test.js create mode 100644 test/external-docs-snapshot.test.js create mode 100644 test/external-docs.test.js diff --git a/.github/workflows/docs-refresh-pr-dry-run.yml b/.github/workflows/docs-refresh-pr-dry-run.yml index c4f0eb8..b4de95b 100644 --- a/.github/workflows/docs-refresh-pr-dry-run.yml +++ b/.github/workflows/docs-refresh-pr-dry-run.yml @@ -8,10 +8,17 @@ on: - '.github/workflows/docs-refresh*.yml' - 'docs-sources.yml' - 'docs/remote-markdown.md' - - 'lib/external-docs.js' + - 'components/**' + - 'lib/external-docs*.js' - 'lib/generate-docs-fingerprint.js' - 'lib/generate-external-docs-manifest.js' + - 'lib/generate-raw-pages.js' + - 'lib/generate-search-index.js' - 'lib/compare-docs-fingerprint.js' + - 'markdoc/**' + - 'next.config.js' + - 'pages/**' + - 'test/**' - 'package.json' - 'package-lock.json' @@ -29,10 +36,10 @@ jobs: steps: - name: Checkout docs repo - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.2 - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v6.4.0 with: node-version: 22 cache: npm @@ -40,17 +47,17 @@ jobs: - name: Install dependencies run: npm ci - - name: Validate external docs manifest generation - run: npm run generate:external-docs + - name: Run tests + run: npm test - - name: Generate current docs fingerprint - run: npm run docs:fingerprint -- --output /tmp/current-docs-fingerprint.json + - name: Build static documentation and fingerprint + run: npm run build env: GITHUB_TOKEN: ${{ github.token }} - name: Download deployed docs fingerprint env: - DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://documentation-zeta-weld.vercel.app/docs-fingerprint.json' }} + DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://staging.docs.hypercerts.org/docs-fingerprint.json' }} run: | set +e http_code=$(curl -sS -L -w "%{http_code}" \ @@ -80,7 +87,7 @@ jobs: id: diff run: | node lib/compare-docs-fingerprint.js \ - /tmp/current-docs-fingerprint.json \ + public/docs-fingerprint.json \ /tmp/deployed-docs-fingerprint.json >> "$GITHUB_OUTPUT" - name: Report dry-run result diff --git a/.github/workflows/docs-refresh.yml b/.github/workflows/docs-refresh.yml index d21acbf..6f48676 100644 --- a/.github/workflows/docs-refresh.yml +++ b/.github/workflows/docs-refresh.yml @@ -30,10 +30,10 @@ jobs: steps: - name: Checkout docs repo - uses: actions/checkout@v4 + uses: actions/checkout@v6.0.2 - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v6.4.0 with: node-version: 22 cache: npm @@ -49,7 +49,7 @@ jobs: - name: Download deployed docs fingerprint env: - DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://documentation-zeta-weld.vercel.app/docs-fingerprint.json' }} + DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://staging.docs.hypercerts.org/docs-fingerprint.json' }} run: | set +e http_code=$(curl -sS -L -w "%{http_code}" \ diff --git a/.gitignore b/.gitignore index 77eaaea..06937ba 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ out/ # Generated at build time by lib/generate-*.js — no need to track public/raw/ -public/external-docs.json public/docs-fingerprint.json public/search-index.json public/sitemap.xml diff --git a/components/AnnouncementBanner.js b/components/AnnouncementBanner.js index a8d3937..b0155d4 100644 --- a/components/AnnouncementBanner.js +++ b/components/AnnouncementBanner.js @@ -1,7 +1,7 @@ import { useState, useEffect } from 'react'; import Link from 'next/link'; -const BANNER_ID = 'july-2026-community-call'; +const BANNER_ID = 'july-14-2026-community-call'; export function AnnouncementBanner() { const [dismissed, setDismissed] = useState(true); @@ -21,7 +21,7 @@ export function AnnouncementBanner() { return (
- July 7, 2026 + July 14, 2026 Join our July community call. Updates, demos and open Q&A:{' '} diff --git a/components/CopyRawButton.js b/components/CopyRawButton.js index 34ebe45..42003b9 100644 --- a/components/CopyRawButton.js +++ b/components/CopyRawButton.js @@ -1,30 +1,22 @@ import { useState } from 'react'; import { useRouter } from 'next/router'; -import externalDocsContent from '../lib/external-docs-content.json'; function getGeneratedRawUrl(currentPath) { if (currentPath === '/') return '/raw/index.md'; return `/raw${currentPath}.md`; } -function getExternalDocRawUrl(externalDoc) { - const entryId = externalDocsContent.sources?.[externalDoc]; - return entryId ? externalDocsContent.entries?.[entryId]?.rawUrl : null; -} - /** - * Render page-level actions for copying or viewing the Markdown source for the current docs page. - * Pages that render canonical Markdown from another repository can set `externalDoc` in frontmatter so these actions use the registry source instead of the generated local fallback. + * Render page-level actions for copying or viewing the generated Markdown for the current docs page. + * External docs are included in the same local /raw output as ordinary pages. */ -export function CopyRawButton({ frontmatter }) { +export function CopyRawButton() { const [copied, setCopied] = useState(false); const [copyError, setCopyError] = useState(false); const [isCopying, setIsCopying] = useState(false); const router = useRouter(); const currentPath = router.asPath.split('#')[0].split('?')[0] || '/'; - const generatedRawUrl = getGeneratedRawUrl(currentPath); - const externalRawUrl = frontmatter?.externalDoc ? getExternalDocRawUrl(frontmatter.externalDoc) : null; - const rawUrl = frontmatter?.rawUrl || externalRawUrl || generatedRawUrl; + const rawUrl = getGeneratedRawUrl(currentPath); const handleCopy = async () => { setIsCopying(true); diff --git a/components/Layout.js b/components/Layout.js index 39601ec..a5e0dc4 100644 --- a/components/Layout.js +++ b/components/Layout.js @@ -201,7 +201,7 @@ export default function Layout({ children, frontmatter }) {
- {frontmatter && } + {frontmatter && }
{children}
diff --git a/components/RemoteMarkdown.js b/components/RemoteMarkdown.js deleted file mode 100644 index 6e58ad1..0000000 --- a/components/RemoteMarkdown.js +++ /dev/null @@ -1,148 +0,0 @@ -import React, { useContext, useMemo } from 'react'; -import Markdoc from '@markdoc/markdoc'; -import externalDocsContent from '../lib/external-docs-content.json'; -import tags from '../markdoc/tags'; -import { fence, heading, link } from '../markdoc/nodes'; -import { Callout } from './Callout'; -import { Columns } from './Columns'; -import { Column } from './Column'; -import { Figure } from './Figure'; -import { Heading } from './Heading'; -import { CardLink } from './CardLink'; -import { CodeBlock } from './CodeBlock'; -import { Link } from './Link'; -import { DotPattern } from './DotPattern'; -import { HeroBanner } from './HeroBanner'; -import { CardGrid } from './CardGrid'; -import { MermaidDiagram } from './MermaidDiagram'; - -const RemoteMarkdownContext = React.createContext(null); - -const markdocConfig = { - tags, - nodes: { fence, heading, link }, -}; - -/** - * Remove optional YAML frontmatter before parsing remote Markdown with Markdoc. - */ -function stripFrontmatter(markdown) { - if (!markdown.startsWith('---\n')) return markdown; - - const end = markdown.indexOf('\n---\n', 4); - if (end === -1) return markdown; - - return markdown.slice(end + 5).trimStart(); -} - -/** - * Resolve relative links in remotely-rendered docs back to the source GitHub repo. - */ -function resolveRemoteHref(href, source) { - if (!href || href.startsWith('#') || href.startsWith('/') || /^[a-z][a-z0-9+.-]*:/i.test(href)) { - return href; - } - - const sourceDir = source.filePath.split('/').slice(0, -1).join('/'); - const basePath = `/${source.owner}/${source.repoName}/blob/${source.branch}/${sourceDir ? `${sourceDir}/` : ''}`; - const resolved = new URL(href, `https://github.com${basePath}`); - const [, owner, repo, , ref, ...repoPathParts] = resolved.pathname.split('/'); - const repoPath = repoPathParts.join('/'); - const lastSegment = repoPathParts[repoPathParts.length - 1] || ''; - const mode = /\.[a-z0-9]+$/i.test(lastSegment) ? 'blob' : 'tree'; - - return `https://github.com/${owner}/${repo}/${mode}/${ref}/${repoPath}${resolved.hash}`; -} - -function RemoteLink({ href, children, ...props }) { - const source = useContext(RemoteMarkdownContext); - const resolvedHref = source ? resolveRemoteHref(href, source) : href; - - return ( - - {children} - - ); -} - -const remoteMarkdocComponents = { - Callout, - Columns, - Column, - Figure, - Heading, - CardLink, - CodeBlock, - Fence: CodeBlock, - Link: RemoteLink, - DotPattern, - HeroBanner, - CardGrid, - MermaidDiagram, -}; - -function transformMarkdown(markdown) { - const ast = Markdoc.parse(stripFrontmatter(markdown)); - return Markdoc.transform(ast, markdocConfig); -} - -function getBuildTimeEntry(source) { - const id = externalDocsContent.sources?.[source] || source; - const mappedId = externalDocsContent.bySourceUrl?.[source] || externalDocsContent.byRawUrl?.[source] || id; - return externalDocsContent.entries?.[mappedId] || null; -} - -/** - * Build-time Markdoc renderer for Markdown files that live in registered source repos. - * `npm run generate:external-docs` fetches the raw Markdown before `next build`, so static exports contain the rendered HTML and do not fetch GitHub in the browser. - */ -export function RemoteMarkdown({ source, children }) { - const entry = getBuildTimeEntry(source); - - const renderedState = useMemo(() => { - if (!entry?.markdown) { - return { - renderedContent: null, - renderError: new Error(`No build-time Markdown was generated for remote docs source "${source}". Add it to docs-sources.yml or set rawUrl frontmatter, then run npm run generate:external-docs.`), - }; - } - - try { - const content = transformMarkdown(entry.markdown); - return { - renderedContent: Markdoc.renderers.react(content, React, { - components: remoteMarkdocComponents, - }), - renderError: null, - }; - } catch (err) { - return { renderedContent: null, renderError: err }; - } - }, [entry, source]); - - const { renderedContent, renderError } = renderedState; - - if (renderedContent) { - return ( - - {renderedContent} - - ); - } - - return ( - <> -
- Could not render the canonical docs.{' '} - The static build did not include renderable Markdown for this source. Showing the local fallback below. Try rebuilding the site, or edit{' '} - {entry?.sourceUrl ? ( - the source file - ) : ( - 'the configured source URL' - )}{' '} - if the URL is wrong. Details: {renderError?.message || 'No renderable Markdown source was available.'} -
- {children} - - ); -} diff --git a/components/TableOfContents.js b/components/TableOfContents.js index 71ca2ac..f5b2a77 100644 --- a/components/TableOfContents.js +++ b/components/TableOfContents.js @@ -8,44 +8,30 @@ export function TableOfContents() { const router = useRouter(); const currentPath = router.asPath.split("#")[0].split("?")[0]; - // Extract H2 and H3 headings from the DOM after render and after runtime docs load. + // External docs are part of the static page, so headings only need collecting after route changes. useEffect(() => { - if (typeof window === "undefined") return undefined; const article = document.querySelector(".layout-content article"); if (!article) { setHeadings([]); - return undefined; + return; } - const collectHeadings = () => { - const elements = article.querySelectorAll("h2, h3, h4"); - const items = Array.from(elements).map((el) => { - if (!el.id) { - el.id = el.textContent - .toLowerCase() - .replace(/[^a-z0-9]+/g, "-") - .replace(/(^-|-$)/g, ""); - } - return { - id: el.id, - text: el.textContent, - level: Number(el.tagName.slice(1)), - }; - }); - setHeadings(items); - setActiveId(""); - }; - - collectHeadings(); - - const observer = new MutationObserver(collectHeadings); - observer.observe(article, { childList: true, subtree: true }); - window.addEventListener("remote-docs:loaded", collectHeadings); - - return () => { - observer.disconnect(); - window.removeEventListener("remote-docs:loaded", collectHeadings); - }; + const elements = article.querySelectorAll("h2, h3, h4"); + const items = Array.from(elements).map((el) => { + if (!el.id) { + el.id = el.textContent + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/(^-|-$)/g, ""); + } + return { + id: el.id, + text: el.textContent, + level: Number(el.tagName.slice(1)), + }; + }); + setHeadings(items); + setActiveId(""); }, [currentPath]); // Scroll spy using scroll position diff --git a/docs-sources.yml b/docs-sources.yml index 87096f5..f8f37cf 100644 --- a/docs-sources.yml +++ b/docs-sources.yml @@ -1,6 +1,12 @@ sources: + - id: epds + title: ePDS + repo: hypercerts-org/ePDS + ref: main + path: docs/tutorial.md + - id: hyperindex-test title: Hyperindex test source - rawUrl: https://raw.githubusercontent.com/gainforest/hyperindex/skills-update/docs/hyperindex.md - sourceUrl: https://github.com/gainforest/hyperindex/blob/skills-update/docs/hyperindex.md - routeBase: /tools/hyperindex + repo: gainforest/hyperindex + ref: skills-update + path: docs/hyperindex.md diff --git a/docs/remote-markdown.md b/docs/remote-markdown.md index 3248a5f..f62bd14 100644 --- a/docs/remote-markdown.md +++ b/docs/remote-markdown.md @@ -1,76 +1,65 @@ -# Runtime remote Markdown +# Build-time external documentation -Use the `{% remote-doc %}` Markdoc tag when a page should render canonical Markdown from a Hypercerts service repository without copying that Markdown into this documentation repo. +Use an external documentation page when one Markdown file in a service repository is the canonical source for a route on this site. External files are fetched before the static build; browsers never fetch the page Markdown. -Register the source once in `docs-sources.yml`: +## Register a source + +Add the file to `docs-sources.yml`: ```yaml sources: - id: epds title: ePDS - rawUrl: https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/tutorial.md - sourceUrl: https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md - routeBase: /architecture/epds + repo: hypercerts-org/ePDS + ref: main + path: docs/tutorial.md ``` -Then reference that registry id from the page: +- `id` is the stable lowercase identifier used by pages. +- `title` identifies the source in generated fingerprint metadata. +- `repo` is the GitHub `owner/repository` pair. +- `ref` is the branch, tag, or commit to fetch. +- `path` is one `.md`, `.mdoc`, or `.mdx` file in that repository. + +GitHub API request and browser URLs are derived internally. Do not add URLs, directory paths, or separate entrypoints to the registry. + +## Create the page + +Set `externalDoc` in a frontmatter-only page: ```md --- title: ePDS (extended PDS) +description: How to integrate applications with ePDS login. externalDoc: epds --- - -{% remote-doc source="epds" %} - -A short unavailable-state fallback goes here. Keep it brief so the documentation repo does not become a second source of truth. - -{% /remote-doc %} ``` -## How it works - -- The site remains a static Next.js export. -- `npm run generate:external-docs` reads `docs-sources.yml`, fetches registered raw Markdown, writes `/external-docs.json`, and writes build-time content to `lib/external-docs-content.json`. -- `{% remote-doc source="epds" %}` resolves against that generated build-time content during `next build`. -- The exported page contains the rendered remote Markdown HTML; the browser does not fetch GitHub to display the docs. -- Build-time search and raw-page generation use `externalDoc` frontmatter, so search and `/raw` use the canonical Markdown instead of the local fallback. -- The generated Markdown is parsed with the same Markdoc tags and nodes used by local pages. -- Fenced `mermaid` diagrams render as SVGs in the browser through the Mermaid npm package. -- Relative links in the remote Markdown point back to the source GitHub repository. -- `Copy raw` and `View raw` use the page's registered `externalDoc` source when it is set. -- The wrapped local Markdown is the last-resort fallback content. Keep it to a short unavailable-state message, not a copy of the canonical docs. - -## Scheduled refresh and deploys - -`npm run docs:fingerprint` reads `docs-sources.yml`, fetches each registered `rawUrl`, hashes the content, and writes `public/docs-fingerprint.json` during the static build. Directory-style sources without `rawUrl` can still use `repo`, `branch`, and `docsPath` to fingerprint GitHub tree metadata. The generated file includes a stable `combinedFingerprint`; timestamps are ignored by the comparison script. - -`.github/workflows/docs-refresh.yml` runs hourly and through `workflow_dispatch`: - -1. Generate the current external-docs fingerprint. -2. Download the deployed fingerprint from `DOCS_FINGERPRINT_URL`, defaulting to `https://documentation-zeta-weld.vercel.app/docs-fingerprint.json`. -3. Compare only `combinedFingerprint`. -4. If it changed, `POST` to the configured Vercel Deploy Hook. +Do not add a local Markdown body. The registered file is the only page body, which prevents stale fallback content from diverging from rendering, search, or `/raw` exports. -Manual `workflow_dispatch` runs default to `dry_run: true`, which compares fingerprints without calling the Vercel hook. Set `dry_run: false` when you want the manual run to deploy. GitHub only accepts `workflow_dispatch` and scheduled runs once the workflow file exists on the default branch. The workflow only treats a deployed-fingerprint `404` as missing first-run state; transient download failures fail the workflow instead of deploying on an unknown diff. +## Build behavior -`.github/workflows/docs-refresh-pr-dry-run.yml` is a temporary PR-only check for this rollout. It runs the same manifest and fingerprint comparison on pull requests, writes a summary, and never calls the Vercel deploy hook. +`npm run generate:external-docs` fetches every registered file once through the GitHub contents API and writes `lib/external-docs-content.json`. The static build then uses that immutable snapshot for: -Required GitHub Actions secret: +- Markdoc page rendering; +- search indexing; +- local `/raw` page exports; +- last-updated metadata; +- deployed external-docs fingerprints. -- `VERCEL_DEPLOY_HOOK_URL` — the Vercel Deploy Hook URL for the production docs branch. +External Markdown is parsed with the same Markdoc configuration as local pages. Relative links point to the source repository, relative images use public GitHub content URLs, and fenced `mermaid` diagrams render through the Mermaid component. Extensionless relative paths are treated as directories; link to extensionless files such as `LICENSE`, `Dockerfile`, or `Makefile` with an absolute GitHub URL. -Optional GitHub Actions variable: +A missing source, failed content request, empty file, local fallback body, or invalid Markdoc in an external page fails the build with an actionable error. The existing deployment remains online instead of publishing stale or inconsistent content. Source commit timestamps are informational and may be omitted when GitHub cannot provide them. -- `DOCS_FINGERPRINT_URL` — the deployed site fingerprint URL. Forks and staging deployments should set this to their own Vercel URL, for example `https://your-test-docs.vercel.app/docs-fingerprint.json`. +## Refresh workflow -Optional GitHub Actions secret: +`.github/workflows/docs-refresh.yml` runs hourly and can also be dispatched manually. It fetches the registered files, compares their combined fingerprint with the deployed site, and calls the configured Vercel deploy hook when they differ. Manual runs default to dry-run mode. -- `DOCS_SOURCE_TOKEN` — a GitHub token with read access to source repos. Public repos can use the workflow `GITHUB_TOKEN`, but this avoids API rate limits and is required if a source repo becomes private. +`.github/workflows/docs-refresh-pr-dry-run.yml` builds and compares fingerprints on relevant pull requests without calling a deploy hook. -## Constraints +Configuration: -- Sources must be in approved GitHub owners: `hypercerts-org` or `gainforest`. -- Registry ids must be lowercase, for example `epds` or `certified-group-service`. -- Prefer explicit `rawUrl` and `sourceUrl` values for page-level remote docs. A source only needs `entrypoint` when `docsPath` points at a directory and a page renders it through `{% remote-doc %}`. -- Registry-backed remote docs render during the static build. If generated build-time content is missing, the page shows the wrapped local fallback. +- `VERCEL_DEPLOY_HOOK_URL` is required before deployment is enabled. +- `DOCS_FINGERPRINT_URL` optionally selects the deployed fingerprint to compare. +- `DOCS_SOURCE_TOKEN` optionally grants source-repository access and additional GitHub API capacity. It is required for private repositories. +- `DOCS_ALLOWED_SOURCE_ORGS` optionally overrides the comma-separated trusted-owner allowlist. It defaults to `hypercerts-org,gainforest`. diff --git a/lib/external-doc-links.js b/lib/external-doc-links.js new file mode 100644 index 0000000..8d2a495 --- /dev/null +++ b/lib/external-doc-links.js @@ -0,0 +1,47 @@ +function isAbsoluteOrSiteHref(href) { + return !href + || href.startsWith('#') + || href.startsWith('/') + || /^[a-z][a-z0-9+.-]*:/i.test(href); +} + +function getSourceLocation(source) { + const [owner, repoName] = source.repo.split('/'); + const sourceDirectory = source.path.split('/').slice(0, -1).join('/'); + return { owner, repoName, sourceDirectory }; +} + +/** + * Resolve a relative external-document link to the corresponding GitHub blob or tree URL. + * For example, ../packages/demo from docs/tutorial.md becomes the repository's packages/demo tree. + */ +function resolveExternalDocHref(href, source) { + if (isAbsoluteOrSiteHref(href)) return href; + + const { owner, repoName, sourceDirectory } = getSourceLocation(source); + const encodedRef = encodeURIComponent(source.ref); + const basePath = `/${owner}/${repoName}/blob/${encodedRef}/${sourceDirectory ? `${sourceDirectory}/` : ''}`; + const resolved = new URL(href, `https://github.com${basePath}`); + const [, resolvedOwner, resolvedRepo, , ref, ...repoPathParts] = resolved.pathname.split('/'); + const repoPath = repoPathParts.join('/'); + const lastSegment = repoPathParts[repoPathParts.length - 1] || ''; + const mode = /\.[a-z0-9]+$/i.test(lastSegment) ? 'blob' : 'tree'; + + return `https://github.com/${resolvedOwner}/${resolvedRepo}/${mode}/${ref}/${repoPath}${resolved.search}${resolved.hash}`; +} + +/** + * Resolve a relative external-document image to public GitHub content that browsers can display. + */ +function resolveExternalDocImageSrc(src, source) { + if (isAbsoluteOrSiteHref(src)) return src; + + const { owner, repoName, sourceDirectory } = getSourceLocation(source); + const base = `https://raw.githubusercontent.com/${owner}/${repoName}/${encodeURIComponent(source.ref)}/${sourceDirectory ? `${sourceDirectory}/` : ''}`; + return new URL(src, base).toString(); +} + +module.exports = { + resolveExternalDocHref, + resolveExternalDocImageSrc, +}; diff --git a/lib/external-doc-page.js b/lib/external-doc-page.js new file mode 100644 index 0000000..5b2f38b --- /dev/null +++ b/lib/external-doc-page.js @@ -0,0 +1,45 @@ +/** + * Resolve one externalDoc id to the exact Markdown snapshot used by the build. + */ +function resolveExternalDocSnapshot(externalDoc, content) { + if (typeof externalDoc !== 'string' || !/^[a-z0-9][a-z0-9-]*$/.test(externalDoc)) { + throw new Error('externalDoc must be a lowercase registry id such as "epds".'); + } + + const snapshot = content?.sources?.[externalDoc]; + if (!snapshot) { + throw new Error(`Unknown externalDoc "${externalDoc}". Add it to docs-sources.yml and run npm run generate:external-docs.`); + } + if (typeof snapshot.markdown !== 'string' || snapshot.markdown.trim() === '') { + throw new Error(`External doc "${externalDoc}" has no generated Markdown. Run npm run generate:external-docs and check the registered repo, ref, and path.`); + } + + return snapshot; +} + +function getMarkdownBody(markdown) { + const frontmatter = markdown.match(/^---\n[\s\S]*?\n---\n?/); + return frontmatter ? markdown.slice(frontmatter[0].length) : markdown; +} + +/** + * Resolve the Markdown and source metadata for one page. + * Pages with externalDoc must contain frontmatter only so stale local fallback content cannot diverge from the build snapshot. + */ +function resolvePageDocument(frontmatter, localMarkdown, content, label = 'Markdown page') { + if (!Object.prototype.hasOwnProperty.call(frontmatter, 'externalDoc')) { + return { markdown: localMarkdown, externalDoc: null }; + } + + if (getMarkdownBody(localMarkdown).trim() !== '') { + throw new Error(`${label} sets externalDoc and must not contain a local Markdown body. Remove the stale fallback content; external source failures stop the build.`); + } + + const externalDoc = resolveExternalDocSnapshot(frontmatter.externalDoc, content); + return { markdown: externalDoc.markdown, externalDoc }; +} + +module.exports = { + resolveExternalDocSnapshot, + resolvePageDocument, +}; diff --git a/lib/external-docs-loader.js b/lib/external-docs-loader.js new file mode 100644 index 0000000..9d5531a --- /dev/null +++ b/lib/external-docs-loader.js @@ -0,0 +1,45 @@ +const yaml = require('js-yaml'); +const { + CONTENT_PATH, + loadExternalDocsContent, + parseMarkdownFrontmatter, + resolvePageDocument, +} = require('./external-docs'); + +function stripMarkdownFrontmatter(markdown) { + const match = markdown.match(/^---\n[\s\S]*?\n---\n?/); + return match ? markdown.slice(match[0].length) : markdown; +} + +/** + * Compile a frontmatter-only external page from its immutable Markdown snapshot. + * Generated source metadata lets Markdoc resolve relative links without exposing registry URLs or the full snapshot to shared bundles. + */ +function compileExternalDocPage(frontmatter, source, content, label) { + const page = resolvePageDocument(frontmatter, source, content, label); + const { id, repo, ref, path } = page.externalDoc; + const compiledFrontmatter = { + ...frontmatter, + __externalDocSource: { id, repo, ref, path }, + }; + + return `---\n${yaml.dump(compiledFrontmatter, { lineWidth: -1, noRefs: true }).trimEnd()}\n---\n\n${stripMarkdownFrontmatter(page.markdown).trimStart()}`; +} + +/** + * Replace a frontmatter-only externalDoc page with its generated Markdown snapshot before Markdoc parses it. + * The injected source metadata is consumed during transformation and is never written back to the page file. + */ +function externalDocsLoader(source) { + const frontmatter = parseMarkdownFrontmatter(source, this.resourcePath); + if (!Object.prototype.hasOwnProperty.call(frontmatter, 'externalDoc')) { + return source; + } + + const content = loadExternalDocsContent(); + this.addDependency(CONTENT_PATH); + return compileExternalDocPage(frontmatter, source, content, this.resourcePath); +} + +module.exports = externalDocsLoader; +module.exports.compileExternalDocPage = compileExternalDocPage; diff --git a/lib/external-docs-snapshot.js b/lib/external-docs-snapshot.js new file mode 100644 index 0000000..072fdb0 --- /dev/null +++ b/lib/external-docs-snapshot.js @@ -0,0 +1,166 @@ +const crypto = require('crypto'); +const { encodeGitHubPath } = require('./external-docs'); + +const GITHUB_API_VERSION = '2022-11-28'; + +/** + * Serialize a value with stable object-key ordering so fingerprints do not depend on construction order. + */ +function stableStringify(value) { + if (Array.isArray(value)) { + return `[${value.map(stableStringify).join(',')}]`; + } + + if (value && typeof value === 'object') { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`) + .join(',')}}`; + } + + return JSON.stringify(value); +} + +/** + * Compute a SHA-256 digest with the prefix used by generated fingerprint files. + */ +function sha256(value) { + return `sha256:${crypto.createHash('sha256').update(value).digest('hex')}`; +} + +function getGitHubHeaders(token, accept, userAgent) { + const headers = { + Accept: accept, + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + 'User-Agent': userAgent, + }; + if (token) headers.Authorization = `Bearer ${token}`; + return headers; +} + +function getRepoParts(source) { + const [owner, repoName] = source.repo.split('/'); + return { owner, repoName }; +} + +async function fetchGitHubResponse(url, token, accept, userAgent) { + const response = await fetch(url, { + headers: getGitHubHeaders(token, accept, userAgent), + cache: 'no-store', + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`${url} returned ${response.status} ${response.statusText || ''}. Check repo, ref, path, and DOCS_SOURCE_TOKEN. ${body}`.trim()); + } + return response; +} + +async function fetchSourceMarkdown(source, token) { + const { owner, repoName } = getRepoParts(source); + const url = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/contents/${encodeGitHubPath(source.path)}?ref=${encodeURIComponent(source.ref)}`; + const response = await fetchGitHubResponse( + url, + token, + 'application/vnd.github.raw+json', + 'hypercerts-docs-build', + ); + const markdown = await response.text(); + if (markdown.trim() === '') { + throw new Error(`External doc "${source.id}" at ${source.repo}@${source.ref}:${source.path} is empty. Add Markdown content or remove the source from docs-sources.yml.`); + } + return markdown; +} + +async function fetchSourceUpdatedAt(source, token) { + const { owner, repoName } = getRepoParts(source); + const url = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/commits?sha=${encodeURIComponent(source.ref)}&path=${encodeURIComponent(source.path)}&per_page=1`; + + try { + const response = await fetchGitHubResponse( + url, + token, + 'application/vnd.github+json', + 'hypercerts-docs-build', + ); + const commits = await response.json(); + return commits[0]?.commit?.committer?.date || commits[0]?.commit?.author?.date || null; + } catch { + return null; + } +} + +/** + * Fetch one immutable Markdown snapshot used by rendering, search, raw exports, and fingerprinting. + */ +async function collectSourceSnapshot(source, token = '') { + const [markdown, updatedAt] = await Promise.all([ + fetchSourceMarkdown(source, token), + fetchSourceUpdatedAt(source, token), + ]); + + return { + ...source, + updatedAt, + size: Buffer.byteLength(markdown), + contentHash: sha256(markdown), + markdown, + }; +} + +/** + * Fetch all registered sources concurrently and preserve registry order in the returned snapshots. + */ +async function collectExternalDocSnapshots(sources, token = '') { + return Promise.all(sources.map((source) => collectSourceSnapshot(source, token))); +} + +function stableSnapshotSource(snapshot) { + return { + id: snapshot.id, + title: snapshot.title, + repo: snapshot.repo, + ref: snapshot.ref, + path: snapshot.path, + contentHash: snapshot.contentHash, + }; +} + +/** + * Build per-source and combined fingerprints from the exact Markdown snapshots used by the site build. + */ +function buildFingerprintDocument(snapshots) { + const outputSources = {}; + const stableSources = []; + const sortedSnapshots = [...snapshots].sort((a, b) => a.id.localeCompare(b.id)); + + for (const snapshot of sortedSnapshots) { + const stableSource = stableSnapshotSource(snapshot); + const fingerprint = sha256(stableStringify(stableSource)); + outputSources[snapshot.id] = { + title: snapshot.title, + repo: snapshot.repo, + ref: snapshot.ref, + path: snapshot.path, + updatedAt: snapshot.updatedAt || undefined, + size: snapshot.size, + contentHash: snapshot.contentHash, + fingerprint, + }; + stableSources.push({ ...stableSource, fingerprint }); + } + + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + sources: outputSources, + combinedFingerprint: sha256(stableStringify({ schemaVersion: 1, sources: stableSources })), + }; +} + +module.exports = { + buildFingerprintDocument, + collectExternalDocSnapshots, + collectSourceSnapshot, + sha256, + stableStringify, +}; diff --git a/lib/external-docs.js b/lib/external-docs.js index ffdfdc3..e25e9de 100644 --- a/lib/external-docs.js +++ b/lib/external-docs.js @@ -1,20 +1,43 @@ const { readFileSync } = require('fs'); -const { join, posix } = require('path'); +const { join } = require('path'); const yaml = require('js-yaml'); - -const ALLOWED_SOURCE_ORGS = ['hypercerts-org', 'gainforest']; +const { + resolveExternalDocSnapshot, + resolvePageDocument, +} = require('./external-doc-page'); + +/** Default GitHub organizations trusted as external documentation sources. */ +const DEFAULT_ALLOWED_SOURCE_ORGS = ['hypercerts-org', 'gainforest']; +/** Absolute path to the external documentation source registry. */ const REGISTRY_PATH = join(__dirname, '..', 'docs-sources.yml'); +/** Absolute path to the generated build-time external documentation snapshot. */ +const CONTENT_PATH = join(__dirname, 'external-docs-content.json'); const MARKDOWN_EXTENSIONS = /\.(md|mdoc|mdx)$/i; +const SOURCE_FIELDS = new Set(['id', 'title', 'repo', 'ref', 'path']); /** - * Return true when a GitHub repository owner is approved for browser-rendered remote docs. + * Read the trusted GitHub organization allowlist, falling back to project defaults. */ -function isAllowedSourceOwner(owner) { - return ALLOWED_SOURCE_ORGS.includes(String(owner || '').toLowerCase()); +function getAllowedSourceOrgs(env = process.env) { + const configured = env.DOCS_ALLOWED_SOURCE_ORGS; + if (typeof configured !== 'string' || configured.trim() === '') { + return [...DEFAULT_ALLOWED_SOURCE_ORGS]; + } + + const organizations = [...new Set(configured + .split(',') + .map((owner) => owner.trim().toLowerCase()) + .filter(Boolean))]; + + if (organizations.length === 0 || organizations.some((owner) => !/^[a-z0-9_.-]+$/.test(owner))) { + throw new Error('DOCS_ALLOWED_SOURCE_ORGS must be a comma-separated list of GitHub organization names.'); + } + + return organizations; } /** - * Return true when a registry path points at a Markdown file instead of a docs directory. + * Return true when a registry path points at a supported Markdown file. */ function isMarkdownFilePath(value) { return MARKDOWN_EXTENSIONS.test(value); @@ -27,233 +50,73 @@ function encodeGitHubPath(value) { return value.split('/').map(encodeURIComponent).join('/'); } -/** - * Normalize a docs-sources.yml path and reject absolute paths or parent traversal. - */ function normalizeRegistryPath(value, fieldName) { if (typeof value !== 'string' || value.trim() === '') { - throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a non-empty relative path.`); + throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a non-empty relative Markdown path.`); } - const normalized = value.trim().replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/$/, ''); - const parts = normalized.split('/'); + if (value.includes('\\')) { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must use forward slashes.`); + } + const normalized = value.trim().replace(/^\.\//, '').replace(/\/$/, ''); + const parts = normalized.split('/'); if (normalized.startsWith('/') || parts.includes('..') || parts.includes('')) { throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a relative path without empty segments or "..".`); } - - return normalized; -} - -/** - * Parse the YAML frontmatter block from a Markdown file. - */ -function parseMarkdownFrontmatter(markdown, label = 'Markdown file') { - const match = markdown.match(/^---\n([\s\S]*?)\n---\n?/); - if (!match) return {}; - - try { - return yaml.load(match[1]) || {}; - } catch (error) { - throw new Error(`Invalid frontmatter in ${label}: ${error.message}`); - } -} - -/** - * Parse and validate a raw.githubusercontent.com Markdown URL from docs-sources.yml. - */ -function parseRawGitHubMarkdownUrl(value, fieldName) { - if (typeof value !== 'string' || value.trim() === '') { - throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a non-empty raw.githubusercontent.com URL.`); - } - - let url; - try { - url = new URL(value.trim()); - } catch { - throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a valid URL.`); - } - - if (url.protocol !== 'https:' || url.hostname !== 'raw.githubusercontent.com') { - throw new Error(`Invalid docs-sources.yml: ${fieldName} must use https://raw.githubusercontent.com/...`); - } - - const [, owner, repoName, branch, ...fileParts] = url.pathname.split('/'); - if (!isAllowedSourceOwner(owner) || !repoName || !branch || fileParts.length === 0) { - throw new Error(`Invalid docs-sources.yml: ${fieldName} must point at a Markdown file under one of these owners: ${ALLOWED_SOURCE_ORGS.join(', ')}.`); - } - - const filePath = normalizeRegistryPath(fileParts.join('/'), fieldName); - if (!isMarkdownFilePath(filePath)) { - throw new Error(`Invalid docs-sources.yml: ${fieldName} must point at a Markdown file.`); - } - - return { - rawUrl: url.toString(), - owner, - repoName, - branch, - filePath, - repo: `${owner}/${repoName}`, - sourceUrl: `https://github.com/${owner}/${repoName}/blob/${encodeURIComponent(branch)}/${encodeGitHubPath(filePath)}`, - }; -} - -/** - * Parse and validate a github.com blob URL from docs-sources.yml. - */ -function parseGitHubSourceUrl(value, fieldName) { - if (typeof value !== 'string' || value.trim() === '') { - throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a non-empty github.com blob URL.`); - } - - let url; - try { - url = new URL(value.trim()); - } catch { - throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a valid URL.`); - } - - if (url.protocol !== 'https:' || url.hostname !== 'github.com') { - throw new Error(`Invalid docs-sources.yml: ${fieldName} must use https://github.com/...`); - } - - const [, owner, repoName, marker, branch, ...fileParts] = url.pathname.split('/'); - if (!isAllowedSourceOwner(owner) || marker !== 'blob' || !repoName || !branch || fileParts.length === 0) { - throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a GitHub blob URL under one of these owners: ${ALLOWED_SOURCE_ORGS.join(', ')}.`); + if (!isMarkdownFilePath(normalized)) { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must point to a .md, .mdoc, or .mdx file.`); } - const filePath = normalizeRegistryPath(fileParts.join('/'), fieldName); - return { - sourceUrl: url.toString(), - owner, - repoName, - branch, - filePath, - repo: `${owner}/${repoName}`, - }; -} - -/** - * Build the raw.githubusercontent.com URL for a registered external docs file. - */ -function buildRawGitHubUrl(source, filePath) { - return `https://raw.githubusercontent.com/${source.owner}/${source.repoName}/${encodeURIComponent(source.branch)}/${encodeGitHubPath(filePath)}`; -} - -/** - * Build the GitHub browser URL for a registered external docs path. - */ -function buildGitHubSourceUrl(source, filePath, mode = 'blob') { - return `https://github.com/${source.owner}/${source.repoName}/${mode}/${encodeURIComponent(source.branch)}/${encodeGitHubPath(filePath)}`; -} - -/** - * Return the single Markdown file that a remote-doc page should render, when configured. - */ -function getEntrypointPath(source) { - if (source.entrypoint) return posix.join(source.docsPath, source.entrypoint); - if (isMarkdownFilePath(source.docsPath)) return source.docsPath; - return null; -} - -function assertConsistentSourceField(source, inferred, key, fieldName) { - if (!source[key] || !inferred?.[key]) return; - if (source[key].toLowerCase() !== inferred[key].toLowerCase()) { - throw new Error(`Invalid docs-sources.yml: ${fieldName} does not match ${key} "${source[key]}".`); - } + return normalized; } -/** - * Validate and normalize one source entry from docs-sources.yml. - */ -function normalizeSource(rawSource, index) { +function normalizeSource(rawSource, index, allowedOrgs) { const label = `sources[${index}]`; - if (!rawSource || typeof rawSource !== 'object' || Array.isArray(rawSource)) { throw new Error(`Invalid docs-sources.yml: ${label} must be an object.`); } - const id = rawSource.id; + const unsupportedFields = Object.keys(rawSource).filter((field) => !SOURCE_FIELDS.has(field)); + if (unsupportedFields.length > 0) { + throw new Error(`Invalid docs-sources.yml: ${label} contains unsupported field${unsupportedFields.length === 1 ? '' : 's'} ${unsupportedFields.map((field) => `"${field}"`).join(', ')}. Use only id, title, repo, ref, and path.`); + } + + const { id, title, repo, ref } = rawSource; if (typeof id !== 'string' || !/^[a-z0-9][a-z0-9-]*$/.test(id)) { throw new Error(`Invalid docs-sources.yml: ${label}.id must be a lowercase id like "epds" or "certified-group-service".`); } - - const title = rawSource.title; if (typeof title !== 'string' || title.trim() === '') { throw new Error(`Invalid docs-sources.yml: source "${id}" must set a human-readable title.`); } - - const rawInfo = rawSource.rawUrl - ? parseRawGitHubMarkdownUrl(rawSource.rawUrl, `source "${id}" rawUrl`) - : null; - const sourceInfo = rawSource.sourceUrl - ? parseGitHubSourceUrl(rawSource.sourceUrl, `source "${id}" sourceUrl`) - : null; - - const repo = rawSource.repo || rawInfo?.repo || sourceInfo?.repo; if (typeof repo !== 'string' || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) { - throw new Error(`Invalid docs-sources.yml: source "${id}" must set repo like "owner/repo" or provide rawUrl.`); + throw new Error(`Invalid docs-sources.yml: source "${id}" repo must look like "owner/repository".`); } - const [owner, repoName] = repo.split('/'); - if (!isAllowedSourceOwner(owner)) { - throw new Error(`Invalid docs-sources.yml: source "${id}" must use one of these GitHub owners so browser-rendered docs cannot load arbitrary hosts: ${ALLOWED_SOURCE_ORGS.join(', ')}.`); + const [owner] = repo.split('/'); + if (!allowedOrgs.includes(owner.toLowerCase())) { + throw new Error(`Invalid docs-sources.yml: source "${id}" must use one of these trusted GitHub owners: ${allowedOrgs.join(', ')}.`); } - - const branch = rawSource.branch || rawInfo?.branch || sourceInfo?.branch; - if (typeof branch !== 'string' || branch.trim() === '') { - throw new Error(`Invalid docs-sources.yml: source "${id}" must set branch or provide rawUrl.`); + if (typeof ref !== 'string' || ref.trim() === '') { + throw new Error(`Invalid docs-sources.yml: source "${id}" must set a branch, tag, or commit in ref.`); } - - const docsPath = rawSource.docsPath - ? normalizeRegistryPath(rawSource.docsPath, `source "${id}" docsPath`) - : rawInfo?.filePath || sourceInfo?.filePath; - if (!docsPath) { - throw new Error(`Invalid docs-sources.yml: source "${id}" must set docsPath or provide rawUrl.`); + if (/[\u0000-\u001f\u007f]/.test(ref)) { + throw new Error(`Invalid docs-sources.yml: source "${id}" ref must not contain control characters.`); } - const entrypoint = rawSource.entrypoint - ? normalizeRegistryPath(rawSource.entrypoint, `source "${id}" entrypoint`) - : ''; - - const source = { + return { id, title: title.trim(), repo, - owner, - repoName, - branch: branch.trim(), - docsPath, - entrypoint, - routeBase: rawSource.routeBase || rawSource.route || '', - }; - - assertConsistentSourceField(source, rawInfo, 'repo', `source "${id}" rawUrl`); - assertConsistentSourceField(source, rawInfo, 'branch', `source "${id}" rawUrl`); - assertConsistentSourceField(source, sourceInfo, 'repo', `source "${id}" sourceUrl`); - assertConsistentSourceField(source, sourceInfo, 'branch', `source "${id}" sourceUrl`); - - if (source.routeBase && (typeof source.routeBase !== 'string' || !source.routeBase.startsWith('/'))) { - throw new Error(`Invalid docs-sources.yml: source "${id}" routeBase must start with "/".`); - } - - const entrypointPath = rawInfo?.filePath || sourceInfo?.filePath || getEntrypointPath(source); - return { - ...source, - entrypointPath, - rawUrl: rawInfo?.rawUrl || (entrypointPath ? buildRawGitHubUrl(source, entrypointPath) : ''), - sourceUrl: rawSource.sourceUrl || rawInfo?.sourceUrl || (entrypointPath - ? buildGitHubSourceUrl(source, entrypointPath, 'blob') - : buildGitHubSourceUrl(source, docsPath, 'tree')), - fingerprintMode: rawInfo ? 'rawUrl' : 'gitTree', + ref: ref.trim(), + path: normalizeRegistryPath(rawSource.path, `source "${id}" path`), }; } /** - * Load and validate the central external docs registry. + * Load and validate the build-time external documentation registry. */ -function loadExternalDocSources(registryPath = REGISTRY_PATH) { +function loadExternalDocSources(registryPath = REGISTRY_PATH, env = process.env) { let document; try { document = yaml.load(readFileSync(registryPath, 'utf8')) || {}; @@ -265,79 +128,57 @@ function loadExternalDocSources(registryPath = REGISTRY_PATH) { throw new Error('Invalid docs-sources.yml: expected a top-level "sources" array.'); } + const allowedOrgs = getAllowedSourceOrgs(env); const seen = new Set(); - return document.sources.map((source, index) => { - const normalized = normalizeSource(source, index); - if (seen.has(normalized.id)) { - throw new Error(`Invalid docs-sources.yml: duplicate source id "${normalized.id}".`); + return document.sources.map((rawSource, index) => { + const source = normalizeSource(rawSource, index, allowedOrgs); + if (seen.has(source.id)) { + throw new Error(`Invalid docs-sources.yml: duplicate source id "${source.id}".`); } - seen.add(normalized.id); - return normalized; + seen.add(source.id); + return source; }); } /** - * Find one registered external docs source by id. + * Parse the YAML frontmatter block from a Markdown file. */ -function findExternalDocSource(id, sources = loadExternalDocSources()) { - return sources.find((source) => source.id === id) || null; -} +function parseMarkdownFrontmatter(markdown, label = 'Markdown file') { + const match = markdown.match(/^---\n([\s\S]*?)\n---\n?/); + if (!match) return {}; -/** - * Convert a registry source into the public manifest consumed by browser components. - */ -function sourceToPublicManifestEntry(source) { - return { - id: source.id, - title: source.title, - repo: source.repo, - branch: source.branch, - docsPath: source.docsPath, - entrypoint: source.entrypoint || undefined, - routeBase: source.routeBase || undefined, - sourceUrl: source.sourceUrl, - rawUrl: source.rawUrl || undefined, - filePath: source.entrypointPath || undefined, - }; + try { + return yaml.load(match[1]) || {}; + } catch (error) { + throw new Error(`Invalid frontmatter in ${label}: ${error.message}`); + } } /** - * Resolve a page frontmatter declaration to the raw Markdown URL that build scripts should fetch. + * Load the immutable external documentation snapshot generated before the site build. */ -function resolveFrontmatterRawSource(frontmatter, sources = loadExternalDocSources()) { - if (frontmatter.externalDoc) { - const id = String(frontmatter.externalDoc); - const source = findExternalDocSource(id, sources); - if (!source) { - throw new Error(`Unknown externalDoc "${id}". Add it to docs-sources.yml or remove the externalDoc frontmatter.`); - } - if (!source.rawUrl) { - throw new Error(`External doc "${id}" does not define a renderable Markdown file. Set rawUrl in docs-sources.yml or point docsPath at a Markdown file.`); +function loadExternalDocsContent(contentPath = CONTENT_PATH) { + try { + const content = JSON.parse(readFileSync(contentPath, 'utf8')); + if (!content.sources || typeof content.sources !== 'object' || Array.isArray(content.sources)) { + throw new Error('expected a top-level "sources" object'); } - return { rawUrl: source.rawUrl, label: `externalDoc "${id}"` }; - } - - if (frontmatter.rawUrl) { - return { rawUrl: String(frontmatter.rawUrl), label: 'rawUrl frontmatter' }; + return content; + } catch (error) { + throw new Error(`Unable to read generated external docs content at ${contentPath}: ${error.message}. Run npm run generate:external-docs first.`); } - - return null; } module.exports = { + CONTENT_PATH, + DEFAULT_ALLOWED_SOURCE_ORGS, REGISTRY_PATH, - ALLOWED_SOURCE_ORGS, - isAllowedSourceOwner, - buildGitHubSourceUrl, - buildRawGitHubUrl, encodeGitHubPath, - findExternalDocSource, - getEntrypointPath, + getAllowedSourceOrgs, isMarkdownFilePath, loadExternalDocSources, - parseGitHubSourceUrl, + loadExternalDocsContent, parseMarkdownFrontmatter, - parseRawGitHubMarkdownUrl, - resolveFrontmatterRawSource, - sourceToPublicManifestEntry, + resolveExternalDocSnapshot, + resolvePageDocument, }; diff --git a/lib/generate-docs-fingerprint.js b/lib/generate-docs-fingerprint.js index 2857a8d..13306b0 100644 --- a/lib/generate-docs-fingerprint.js +++ b/lib/generate-docs-fingerprint.js @@ -1,193 +1,78 @@ -const crypto = require('crypto'); -const { mkdirSync, writeFileSync } = require('fs'); +const { mkdirSync, readFileSync, writeFileSync } = require('fs'); const { dirname, join } = require('path'); const { loadExternalDocSources } = require('./external-docs'); +const { + buildFingerprintDocument, + collectExternalDocSnapshots, +} = require('./external-docs-snapshot'); const DEFAULT_OUTPUT = join(__dirname, '..', 'public', 'docs-fingerprint.json'); -const GITHUB_API_VERSION = '2022-11-28'; /** - * Serialize values with stable object-key ordering so hashes do not depend on construction order. + * Fingerprinting process: + * + * 1. A site build fetches every registered Markdown file into one immutable snapshot. + * Rendering, search, raw exports, and this script all consume that same snapshot. + * 2. Scheduled refresh checks have no build snapshot, so they fetch the registered files directly. + * 3. Each source fingerprint includes its registry identity and SHA-256 content hash. + * 4. Per-source fingerprints are sorted by id and combined into one stable fingerprint. + * 5. Generation and source-update timestamps are informational and never affect comparisons. */ -function stableStringify(value) { - if (Array.isArray(value)) { - return `[${value.map(stableStringify).join(',')}]`; - } - if (value && typeof value === 'object') { - return `{${Object.keys(value) - .sort() - .map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`) - .join(',')}}`; +function getArgumentValue(argv, longName, shortName) { + const index = argv.findIndex((arg) => arg === longName || arg === shortName); + if (index !== -1) { + const value = argv[index + 1]; + if (!value) throw new Error(`Missing value after ${argv[index]}.`); + return value; } - return JSON.stringify(value); -} - -/** - * Compute a sha256 digest with the prefix used in generated fingerprint files. - */ -function sha256(value) { - return `sha256:${crypto.createHash('sha256').update(value).digest('hex')}`; + const inline = argv.find((arg) => arg.startsWith(`${longName}=`)); + return inline ? inline.slice(longName.length + 1) : null; } /** - * Read a --output value from the CLI arguments. + * Read the fingerprint output path from CLI arguments. */ function getOutputPath(argv) { - const outputIndex = argv.findIndex((arg) => arg === '--output' || arg === '-o'); - if (outputIndex !== -1) { - const output = argv[outputIndex + 1]; - if (!output) throw new Error('Missing value after --output.'); - return output; - } - - const inlineOutput = argv.find((arg) => arg.startsWith('--output=')); - if (inlineOutput) return inlineOutput.slice('--output='.length); - - return DEFAULT_OUTPUT; -} - -/** - * Fetch JSON from the GitHub REST API with optional authentication for private repos or rate limits. - */ -async function fetchGitHubJson(url, token) { - const headers = { - Accept: 'application/vnd.github+json', - 'X-GitHub-Api-Version': GITHUB_API_VERSION, - 'User-Agent': 'hypercerts-docs-refresh', - }; - - if (token) headers.Authorization = `Bearer ${token}`; - - const response = await fetch(url, { headers }); - if (!response.ok) { - const body = await response.text().catch(() => ''); - throw new Error(`${url} returned ${response.status} ${response.statusText || ''}. Check repo, branch, docsPath, and DOCS_SOURCE_TOKEN. ${body}`.trim()); - } - - return response.json(); + return getArgumentValue(argv, '--output', '-o') || DEFAULT_OUTPUT; } /** - * Fetch a registry rawUrl and return one content-hash entry for fingerprinting. + * Read an optional generated snapshot path from CLI arguments. */ -async function fetchRawUrlEntry(source, token) { - const headers = { - 'User-Agent': 'hypercerts-docs-refresh', - }; - - if (token) headers.Authorization = `Bearer ${token}`; - - const response = await fetch(source.rawUrl, { headers, cache: 'no-store' }); - if (!response.ok) { - const body = await response.text().catch(() => ''); - throw new Error(`${source.rawUrl} returned ${response.status} ${response.statusText || ''}. Check rawUrl and DOCS_SOURCE_TOKEN. ${body}`.trim()); - } - - const content = await response.text(); - return [{ - path: source.entrypointPath || source.docsPath || source.rawUrl, - sha: sha256(content), - size: Buffer.byteLength(content), - }]; +function getContentPath(argv) { + return getArgumentValue(argv, '--content', '-c'); } -/** - * Fetch the Git tree for a source branch and return blob entries under docsPath. - */ -async function fetchGitTreeEntries(source, token) { - const treeUrl = `https://api.github.com/repos/${source.owner}/${source.repoName}/git/trees/${encodeURIComponent(source.branch)}?recursive=1`; - const tree = await fetchGitHubJson(treeUrl, token); - - if (tree.truncated) { - throw new Error(`GitHub returned a truncated tree for ${source.repo}@${source.branch}. Narrow source "${source.id}" docsPath or split it into smaller sources before fingerprinting.`); +function loadSnapshotsFromContent(contentPath) { + let content; + try { + content = JSON.parse(readFileSync(contentPath, 'utf8')); + } catch (error) { + throw new Error(`Unable to read external docs snapshot at ${contentPath}: ${error.message}. Run npm run generate:external-docs first.`); } - const root = source.docsPath.replace(/\/$/, ''); - const prefix = `${root}/`; - const files = (tree.tree || []) - .filter((entry) => entry.type === 'blob' && (entry.path === root || entry.path.startsWith(prefix))) - .map((entry) => ({ - path: entry.path, - sha: entry.sha, - size: entry.size || 0, - })) - .sort((a, b) => a.path.localeCompare(b.path)); - - if (files.length === 0) { - throw new Error(`No files found for source "${source.id}" at ${source.repo}@${source.branch}:${source.docsPath}. Check docs-sources.yml.`); + if (!content.sources || typeof content.sources !== 'object' || Array.isArray(content.sources)) { + throw new Error(`External docs snapshot at ${contentPath} must contain a "sources" object.`); } - return files; + return Object.values(content.sources); } -/** - * Fetch the fingerprint entries for one source. - */ -async function fetchSourceEntries(source, token) { - if (source.fingerprintMode === 'rawUrl') { - return fetchRawUrlEntry(source, token); - } - - return fetchGitTreeEntries(source, token); -} +async function getSnapshots(contentPath) { + if (contentPath) return loadSnapshotsFromContent(contentPath); -/** - * Compute the per-source and combined fingerprints for all registered external docs. - */ -async function generateFingerprint() { const sources = loadExternalDocSources(); const token = process.env.DOCS_SOURCE_TOKEN || process.env.GITHUB_TOKEN || process.env.GH_TOKEN || ''; - const outputSources = {}; - const stableSources = []; - - for (const source of sources) { - const files = await fetchSourceEntries(source, token); - const stableSource = { - id: source.id, - repo: source.repo, - branch: source.branch, - docsPath: source.docsPath, - entrypoint: source.entrypoint || '', - routeBase: source.routeBase || '', - rawUrl: source.rawUrl || '', - sourceUrl: source.sourceUrl || '', - fingerprintMode: source.fingerprintMode || 'gitTree', - files, - }; - const fingerprint = sha256(stableStringify(stableSource)); - - outputSources[source.id] = { - title: source.title, - repo: source.repo, - branch: source.branch, - docsPath: source.docsPath, - entrypoint: source.entrypoint || undefined, - routeBase: source.routeBase || undefined, - rawUrl: source.rawUrl || undefined, - sourceUrl: source.sourceUrl || undefined, - fingerprintMode: source.fingerprintMode || 'gitTree', - fileCount: files.length, - fingerprint, - files, - }; - stableSources.push({ ...stableSource, fingerprint }); - } - - const combinedFingerprint = sha256(stableStringify({ schemaVersion: 1, sources: stableSources })); - - return { - schemaVersion: 1, - generatedAt: new Date().toISOString(), - sources: outputSources, - combinedFingerprint, - }; + return collectExternalDocSnapshots(sources, token); } async function main() { - const output = getOutputPath(process.argv.slice(2)); - const fingerprint = await generateFingerprint(); + const argv = process.argv.slice(2); + const output = getOutputPath(argv); + const snapshots = await getSnapshots(getContentPath(argv)); + const fingerprint = buildFingerprintDocument(snapshots); mkdirSync(dirname(output), { recursive: true }); writeFileSync(output, `${JSON.stringify(fingerprint, null, 2)}\n`); diff --git a/lib/generate-external-docs-manifest.js b/lib/generate-external-docs-manifest.js index a001987..c02a608 100644 --- a/lib/generate-external-docs-manifest.js +++ b/lib/generate-external-docs-manifest.js @@ -1,163 +1,25 @@ -const crypto = require('crypto'); -const { - mkdirSync, - readFileSync, - readdirSync, - statSync, - writeFileSync, -} = require('fs'); -const { dirname, join, relative } = require('path'); -const { - loadExternalDocSources, - parseMarkdownFrontmatter, - parseRawGitHubMarkdownUrl, - sourceToPublicManifestEntry, -} = require('./external-docs'); - -const MANIFEST_OUTPUT = join(__dirname, '..', 'public', 'external-docs.json'); -const CONTENT_OUTPUT = join(__dirname, 'external-docs-content.json'); -const PAGES_DIR = join(__dirname, '..', 'pages'); - -/** - * Compute a short stable identifier for raw URL sources that are declared directly in page frontmatter. - */ -function hashId(value) { - return crypto.createHash('sha256').update(value).digest('hex').slice(0, 12); -} - -function walkDir(dir) { - const results = []; - for (const entry of readdirSync(dir)) { - const full = join(dir, entry); - if (statSync(full).isDirectory()) { - results.push(...walkDir(full)); - } else if (full.endsWith('.md')) { - results.push(full); - } - } - return results; -} +const { mkdirSync, writeFileSync } = require('fs'); +const { dirname } = require('path'); +const { CONTENT_PATH, loadExternalDocSources } = require('./external-docs'); +const { collectExternalDocSnapshots } = require('./external-docs-snapshot'); /** - * Fetch one Markdown source during the static build so pages can render without browser fetches. - */ -async function fetchMarkdown(rawUrl) { - const headers = { - 'User-Agent': 'hypercerts-docs-build', - }; - const token = process.env.DOCS_SOURCE_TOKEN || process.env.GITHUB_TOKEN || process.env.GH_TOKEN || ''; - if (token) headers.Authorization = `Bearer ${token}`; - - const response = await fetch(rawUrl, { headers, cache: 'no-store' }); - if (!response.ok) { - const body = await response.text().catch(() => ''); - throw new Error(`${rawUrl} returned ${response.status} ${response.statusText || ''}. Check docs-sources.yml, page rawUrl frontmatter, and DOCS_SOURCE_TOKEN. ${body}`.trim()); - } - - return response.text(); -} - -function entryFromRegistrySource(source) { - return { - id: source.id, - registryId: source.id, - title: source.title, - repo: source.repo, - owner: source.owner, - repoName: source.repoName, - branch: source.branch, - docsPath: source.docsPath, - entrypoint: source.entrypoint || undefined, - routeBase: source.routeBase || undefined, - sourceUrl: source.sourceUrl, - rawUrl: source.rawUrl, - filePath: source.entrypointPath || source.docsPath, - }; -} - -function entryFromRawFrontmatter(file, frontmatter) { - if (!frontmatter.rawUrl) return null; - - const rawInfo = parseRawGitHubMarkdownUrl(frontmatter.rawUrl, `${relative(PAGES_DIR, file)} rawUrl`); - return { - id: `raw:${hashId(rawInfo.rawUrl)}`, - title: frontmatter.title ? String(frontmatter.title) : rawInfo.filePath, - repo: rawInfo.repo, - owner: rawInfo.owner, - repoName: rawInfo.repoName, - branch: rawInfo.branch, - docsPath: rawInfo.filePath, - routeBase: undefined, - sourceUrl: rawInfo.sourceUrl, - rawUrl: rawInfo.rawUrl, - filePath: rawInfo.filePath, - }; -} - -function addContentEntry(content, entry, sourceKey) { - if (!entry?.rawUrl) return; - - const existingId = content.byRawUrl[entry.rawUrl]; - const id = existingId || entry.id; - - if (!content.entries[id]) { - content.entries[id] = { ...entry, id }; - content.byRawUrl[entry.rawUrl] = id; - content.bySourceUrl[entry.sourceUrl] = id; - } - - if (sourceKey) { - content.sources[sourceKey] = id; - } -} - -async function hydrateContentEntries(content) { - for (const entry of Object.values(content.entries)) { - entry.markdown = await fetchMarkdown(entry.rawUrl); - } -} - -/** - * Generate the browser-readable manifest and build-time Markdown content used by remote docs pages. + * Fetch every registered Markdown file once and write the immutable build snapshot. */ async function generateExternalDocs() { const sources = loadExternalDocSources(); - const manifest = { - schemaVersion: 1, - sources: Object.fromEntries( - sources.map((source) => [source.id, sourceToPublicManifestEntry(source)]) - ), - }; - + const token = process.env.DOCS_SOURCE_TOKEN || process.env.GITHUB_TOKEN || process.env.GH_TOKEN || ''; + const snapshots = await collectExternalDocSnapshots(sources, token); const content = { schemaVersion: 1, generatedAt: new Date().toISOString(), - entries: {}, - sources: {}, - byRawUrl: {}, - bySourceUrl: {}, + sources: Object.fromEntries(snapshots.map((snapshot) => [snapshot.id, snapshot])), }; - for (const source of sources) { - addContentEntry(content, entryFromRegistrySource(source), source.id); - } - - for (const file of walkDir(PAGES_DIR)) { - const markdown = readFileSync(file, 'utf8'); - const frontmatter = parseMarkdownFrontmatter(markdown, relative(PAGES_DIR, file)); - addContentEntry(content, entryFromRawFrontmatter(file, frontmatter)); - } - - await hydrateContentEntries(content); - - mkdirSync(dirname(MANIFEST_OUTPUT), { recursive: true }); - writeFileSync(MANIFEST_OUTPUT, `${JSON.stringify(manifest, null, 2)}\n`); - - mkdirSync(dirname(CONTENT_OUTPUT), { recursive: true }); - writeFileSync(CONTENT_OUTPUT, `${JSON.stringify(content, null, 2)}\n`); + mkdirSync(dirname(CONTENT_PATH), { recursive: true }); + writeFileSync(CONTENT_PATH, `${JSON.stringify(content, null, 2)}\n`); - console.log(`Generated external docs manifest for ${sources.length} source${sources.length === 1 ? '' : 's'}`); - console.log(`Generated build-time external docs content for ${Object.keys(content.entries).length} Markdown source${Object.keys(content.entries).length === 1 ? '' : 's'}`); + console.log(`Generated build-time snapshots for ${snapshots.length} external Markdown source${snapshots.length === 1 ? '' : 's'}`); } generateExternalDocs().catch((error) => { diff --git a/lib/generate-last-updated.js b/lib/generate-last-updated.js index d158485..71e008d 100644 --- a/lib/generate-last-updated.js +++ b/lib/generate-last-updated.js @@ -1,6 +1,11 @@ const { execSync } = require("child_process"); -const { readdirSync, statSync, writeFileSync } = require("fs"); +const { readFileSync, readdirSync, statSync, writeFileSync } = require("fs"); const { join, relative } = require("path"); +const { + loadExternalDocsContent, + parseMarkdownFrontmatter, + resolvePageDocument, +} = require("./external-docs"); const PAGES_DIR = join(__dirname, "..", "pages"); const OUTPUT = join(__dirname, "lastUpdated.json"); @@ -30,12 +35,17 @@ function getLastUpdated(filePath) { } const files = walkDir(PAGES_DIR); +const externalContent = loadExternalDocsContent(); const map = {}; for (const file of files) { const rel = "/" + relative(PAGES_DIR, file).replace(/\.md$/, ""); const route = rel === "/index" ? "/" : rel; - const date = getLastUpdated(file); + const pagePath = relative(PAGES_DIR, file); + const markdown = readFileSync(file, "utf8"); + const frontmatter = parseMarkdownFrontmatter(markdown, pagePath); + const page = resolvePageDocument(frontmatter, markdown, externalContent, pagePath); + const date = page.externalDoc ? page.externalDoc.updatedAt : getLastUpdated(file); if (date) { map[route] = date; } diff --git a/lib/generate-raw-pages.js b/lib/generate-raw-pages.js index 8e1d68b..b66bb3b 100644 --- a/lib/generate-raw-pages.js +++ b/lib/generate-raw-pages.js @@ -8,9 +8,9 @@ const { } = require('fs'); const { dirname, join, relative } = require('path'); const { - loadExternalDocSources, + loadExternalDocsContent, parseMarkdownFrontmatter, - resolveFrontmatterRawSource, + resolvePageDocument, } = require('./external-docs'); const PAGES_DIR = join(__dirname, '..', 'pages'); @@ -38,20 +38,11 @@ function getRawOutputPath(filePath) { return join(OUTPUT_DIR, outputRel); } -async function getRawMarkdown(file, sources) { +function getRawMarkdown(file, externalContent) { const localMarkdown = readFileSync(file, 'utf-8'); const pagePath = relative(PAGES_DIR, file); const frontmatter = parseMarkdownFrontmatter(localMarkdown, pagePath); - const remoteSource = resolveFrontmatterRawSource(frontmatter, sources); - - if (!remoteSource) return localMarkdown; - - const response = await fetch(remoteSource.rawUrl, { cache: 'no-store' }); - if (!response.ok) { - throw new Error(`Failed to fetch ${remoteSource.label} for ${pagePath}: ${remoteSource.rawUrl} returned ${response.status} ${response.statusText || ''}`.trim()); - } - - return response.text(); + return resolvePageDocument(frontmatter, localMarkdown, externalContent, pagePath).markdown; } async function main() { @@ -59,12 +50,12 @@ async function main() { mkdirSync(OUTPUT_DIR, { recursive: true }); const files = walkDir(PAGES_DIR); - const sources = loadExternalDocSources(); + const externalContent = loadExternalDocsContent(); for (const file of files) { const outputPath = getRawOutputPath(file); mkdirSync(dirname(outputPath), { recursive: true }); - writeFileSync(outputPath, await getRawMarkdown(file, sources)); + writeFileSync(outputPath, getRawMarkdown(file, externalContent)); } console.log(`Generated raw markdown files for ${files.length} pages`); diff --git a/lib/generate-search-index.js b/lib/generate-search-index.js index 45be89d..2103fc5 100644 --- a/lib/generate-search-index.js +++ b/lib/generate-search-index.js @@ -1,9 +1,9 @@ const { readdirSync, statSync, readFileSync, writeFileSync } = require("fs"); const { join, relative } = require("path"); const { - loadExternalDocSources, + loadExternalDocsContent, parseMarkdownFrontmatter, - resolveFrontmatterRawSource, + resolvePageDocument, } = require("./external-docs"); const PAGES_DIR = join(__dirname, "..", "pages"); @@ -95,20 +95,9 @@ function getSection(path) { return "Other"; } -async function getIndexContent(file, localContent, remoteSource) { - if (!remoteSource) return localContent; - - const response = await fetch(remoteSource.rawUrl, { cache: "no-store" }); - if (!response.ok) { - throw new Error(`Failed to fetch ${remoteSource.label} for search index ${relative(PAGES_DIR, file)}: ${remoteSource.rawUrl} returned ${response.status} ${response.statusText || ""}`.trim()); - } - - return response.text(); -} - async function main() { const files = walkDir(PAGES_DIR); - const sources = loadExternalDocSources(); + const externalContent = loadExternalDocsContent(); const index = []; for (const file of files) { @@ -119,8 +108,12 @@ async function main() { const frontmatter = parseMarkdownFrontmatter(localContent, relative(PAGES_DIR, file)); const title = getStringFrontmatterValue(frontmatter, "title"); const description = getStringFrontmatterValue(frontmatter, "description"); - const remoteSource = resolveFrontmatterRawSource(frontmatter, sources); - const content = await getIndexContent(file, localContent, remoteSource); + const { markdown: content } = resolvePageDocument( + frontmatter, + localContent, + externalContent, + relative(PAGES_DIR, file), + ); const headings = extractHeadings(content); const section = getSection(path); diff --git a/markdoc/nodes/document.markdoc.js b/markdoc/nodes/document.markdoc.js new file mode 100644 index 0000000..a0058b4 --- /dev/null +++ b/markdoc/nodes/document.markdoc.js @@ -0,0 +1,37 @@ +import Markdoc, { Tag, nodes } from '@markdoc/markdoc'; + +function formatValidationError(validation) { + const line = validation.lines?.[0]; + const location = Number.isInteger(line) ? `line ${line + 1}: ` : ''; + return `${location}${validation.error.message}`; +} + +const document = { + ...nodes.document, + transform(node, config) { + const frontmatter = config.variables?.markdoc?.frontmatter || {}; + const source = frontmatter.__externalDocSource; + if (!source) { + return new Tag('article', {}, node.transformChildren(config)); + } + + const validationErrors = Markdoc.validate(node, config) + .filter((validation) => validation.error.level !== 'warning'); + if (validationErrors.length > 0) { + const details = validationErrors.map(formatValidationError).join('; '); + throw new Error(`External doc "${source.id}" contains invalid Markdoc. ${details}`); + } + + const externalConfig = { + ...config, + variables: { + ...config.variables, + externalDocSource: source, + }, + }; + + return new Tag('article', {}, node.transformChildren(externalConfig)); + }, +}; + +export default document; diff --git a/markdoc/nodes/image.markdoc.js b/markdoc/nodes/image.markdoc.js new file mode 100644 index 0000000..190162a --- /dev/null +++ b/markdoc/nodes/image.markdoc.js @@ -0,0 +1,18 @@ +import { Tag, nodes } from '@markdoc/markdoc'; +import externalDocLinks from '../../lib/external-doc-links'; + +const { resolveExternalDocImageSrc } = externalDocLinks; + +const image = { + ...nodes.image, + transform(node, config) { + const attributes = node.transformAttributes(config); + const source = config.variables?.externalDocSource; + return new Tag('img', { + ...attributes, + src: source ? resolveExternalDocImageSrc(attributes.src, source) : attributes.src, + }); + }, +}; + +export default image; diff --git a/markdoc/nodes/index.js b/markdoc/nodes/index.js index fe064ec..3f0657a 100644 --- a/markdoc/nodes/index.js +++ b/markdoc/nodes/index.js @@ -1,3 +1,5 @@ +export { default as document } from './document.markdoc'; export { default as heading } from './heading.markdoc'; export { default as fence } from './fence.markdoc'; +export { default as image } from './image.markdoc'; export { default as link } from './link.markdoc'; diff --git a/markdoc/nodes/link.markdoc.js b/markdoc/nodes/link.markdoc.js index 8cccbc2..740eedb 100644 --- a/markdoc/nodes/link.markdoc.js +++ b/markdoc/nodes/link.markdoc.js @@ -1,8 +1,23 @@ -import { nodes } from "@markdoc/markdoc"; +import { Tag, nodes } from '@markdoc/markdoc'; +import externalDocLinks from '../../lib/external-doc-links'; + +const { resolveExternalDocHref } = externalDocLinks; const link = { ...nodes.link, - render: "Link", + render: 'Link', + transform(node, config) { + const attributes = node.transformAttributes(config); + const source = config.variables?.externalDocSource; + return new Tag( + 'Link', + { + ...attributes, + href: source ? resolveExternalDocHref(attributes.href, source) : attributes.href, + }, + node.transformChildren(config), + ); + }, }; export default link; diff --git a/markdoc/tags/br.markdoc.js b/markdoc/tags/br.markdoc.js new file mode 100644 index 0000000..c76632d --- /dev/null +++ b/markdoc/tags/br.markdoc.js @@ -0,0 +1,4 @@ +module.exports = { + render: 'br', + selfClosing: true, +}; diff --git a/markdoc/tags/index.js b/markdoc/tags/index.js index 00e6429..9610e46 100644 --- a/markdoc/tags/index.js +++ b/markdoc/tags/index.js @@ -5,9 +5,10 @@ import figure from './figure.markdoc'; import cardLink from './card-link.markdoc'; import cardGrid from './card-grid.markdoc'; import heroBanner from './hero-banner.markdoc'; -import remoteDoc from './remote-doc.markdoc'; +import br from './br.markdoc'; export default { + br, callout, columns, column, @@ -15,5 +16,4 @@ export default { 'card-link': cardLink, 'card-grid': cardGrid, 'hero-banner': heroBanner, - 'remote-doc': remoteDoc, }; diff --git a/markdoc/tags/remote-doc.markdoc.js b/markdoc/tags/remote-doc.markdoc.js deleted file mode 100644 index 978da1d..0000000 --- a/markdoc/tags/remote-doc.markdoc.js +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Markdoc tag for rendering canonical Markdown from a registered GitHub repo at browser runtime. - * The source can be a docs-sources.yml id such as "epds" or a direct approved GitHub Markdown URL. - */ -module.exports = { - render: 'RemoteMarkdown', - attributes: { - source: { - type: String, - required: true, - }, - }, -}; diff --git a/next.config.js b/next.config.js index 4df4325..4113d48 100644 --- a/next.config.js +++ b/next.config.js @@ -3,4 +3,12 @@ const withMarkdoc = require('@markdoc/next.js'); module.exports = withMarkdoc({ mode: 'static' })({ output: 'export', pageExtensions: ['md', 'mdoc', 'js', 'jsx', 'ts', 'tsx'], + webpack(config) { + config.module.rules.push({ + test: /\.(md|mdoc)$/, + enforce: 'pre', + use: [require.resolve('./lib/external-docs-loader')], + }); + return config; + }, }); diff --git a/package.json b/package.json index def3778..12b6e9a 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,10 @@ "private": true, "description": "Hypercerts Protocol Documentation", "scripts": { + "test": "node --test", "generate:external-docs": "node lib/generate-external-docs-manifest.js", "docs:fingerprint": "node lib/generate-docs-fingerprint.js", - "generate": "npm run generate:external-docs && node lib/generate-search-index.js && node lib/generate-last-updated.js && node lib/generate-sitemap.js && node lib/generate-raw-pages.js && npm run docs:fingerprint", + "generate": "npm run generate:external-docs && node lib/generate-search-index.js && node lib/generate-last-updated.js && node lib/generate-sitemap.js && node lib/generate-raw-pages.js && npm run docs:fingerprint -- --content lib/external-docs-content.json", "dev": "npm run generate && next dev --webpack", "build": "npm run generate && next build --webpack", "start": "next start" diff --git a/pages/_app.js b/pages/_app.js index 6c6168f..281d55f 100644 --- a/pages/_app.js +++ b/pages/_app.js @@ -11,7 +11,6 @@ import { Link } from '../components/Link'; import { DotPattern } from '../components/DotPattern'; import { HeroBanner } from '../components/HeroBanner'; import { CardGrid } from '../components/CardGrid'; -import { RemoteMarkdown } from '../components/RemoteMarkdown'; import { MermaidDiagram } from '../components/MermaidDiagram'; import { Analytics } from '@vercel/analytics/next'; @@ -28,8 +27,6 @@ const components = { DotPattern, HeroBanner, CardGrid, - RemoteMarkdown, - RemoteDoc: RemoteMarkdown, MermaidDiagram, }; diff --git a/pages/architecture/epds.md b/pages/architecture/epds.md index f2d5159..3d5e77c 100644 --- a/pages/architecture/epds.md +++ b/pages/architecture/epds.md @@ -1,15 +1,5 @@ --- title: ePDS (extended PDS) description: How the ePDS adds email/OTP login on top of AT Protocol without changing the standard OAuth flow for apps. -rawUrl: "https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/tutorial.md" +externalDoc: epds --- - -{% remote-doc source="https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md" %} - -# ePDS docs unavailable - -The canonical ePDS docs could not be loaded from GitHub or the build-time raw cache. - -View the source directly: https://github.com/hypercerts-org/ePDS/blob/main/docs/tutorial.md - -{% /remote-doc %} diff --git a/pages/lexicons/hypercerts-lexicons/index.md b/pages/lexicons/hypercerts-lexicons/index.md index 1d0a84b..3fb4dbc 100644 --- a/pages/lexicons/hypercerts-lexicons/index.md +++ b/pages/lexicons/hypercerts-lexicons/index.md @@ -10,7 +10,7 @@ These lexicons define the record types for tracking impact work in the `org.hype | Lexicon | NSID | Description | |---------|------|-------------| | **[Activity Claim](/lexicons/hypercerts-lexicons/activity-claim)** | `org.hypercerts.claim.activity` | The core hypercert record — who did what, when, where | -| **[Contribution](/lexicons/hypercerts-lexicons/contribution)** | `org.hypercerts.claim.contributorInformation`
`org.hypercerts.claim.contribution` | Contributor identity and contribution details (two lexicons) | +| **[Contribution](/lexicons/hypercerts-lexicons/contribution)** | `org.hypercerts.claim.contributorInformation`{% br /%}`org.hypercerts.claim.contribution` | Contributor identity and contribution details (two lexicons) | | **[Attachment](/lexicons/hypercerts-lexicons/attachment)** | `org.hypercerts.context.attachment` | Supporting documentation — URLs, files, IPFS links | | **[Measurement](/lexicons/hypercerts-lexicons/measurement)** | `org.hypercerts.context.measurement` | Quantitative data attached to a claim | | **[Evaluation](/lexicons/hypercerts-lexicons/evaluation)** | `org.hypercerts.context.evaluation` | Third-party assessment of a claim | diff --git a/pages/tools/hyperindex.md b/pages/tools/hyperindex.md index 77ecbc3..40a83e3 100644 --- a/pages/tools/hyperindex.md +++ b/pages/tools/hyperindex.md @@ -3,342 +3,3 @@ title: Hyperindex description: A Go ATProto indexer that indexes hypercert records and exposes them via GraphQL. externalDoc: hyperindex-test --- - -{% remote-doc source="hyperindex-test" %} - -# Hyperindex docs unavailable - -The canonical Hyperindex docs could not be loaded from GitHub or the build-time raw cache. Showing the local fallback below. - -# Hyperindex - -Hyperindex (`hi`) is a Go AT Protocol AppView server that indexes records and exposes them via GraphQL. Use it to: - -- Index all hypercert-related records from the ATProto network in real time -- Query indexed data through a typed GraphQL API -- Backfill historical records from any user or the entire network -- Run your own indexer for full control over data availability and query performance - -Built in Go. Hyperindex was originally built by our friends at [GainForest](https://gainforest.earth); the Certified indexer is forked from [github.com/gainforest/hyperindex](https://github.com/gainforest/hyperindex). Seeing the GainForest GitHub organization or GainForest-hosted Hyperindex references in related tooling is expected. Tap reference implementation: [github.com/bluesky-social](https://github.com/bluesky-social/indigo/tree/main/cmd/tap). - -Hosted production and staging endpoints: [Certified Services](/reference/certified-services#indexers). - -## Why indexers & discovery - -AT Protocol is federated, so hypercert records are distributed across many PDSs instead of living in a single database. If an app wants to discover records across users and organizations, it needs a way to aggregate that network data into one queryable view. - -Indexers handle that job. They consume network events, fetch and parse records by lexicon, normalize them into query-ready storage, and expose APIs for search, filtering, and aggregation. - -Hyperindex is the reference indexer used in this ecosystem. - -If you want to inspect indexers running across the broader ecosystem, use [Hyperscan](/tools/hyperscan). - -## How it works - -Hyperindex is **Tap-first** (recommended). Tap handles ingestion, and Hyperindex consumes Tap events, stores records, and exposes them via GraphQL. - -```text -ATProto Relay ──→ Tap ──→ Hyperindex Consumer ──→ Records DB ──→ GraphQL API - │ - Activity Log ──→ Admin Dashboard -``` - -Jetstream mode still exists as a legacy/non-Tap mode, but Tap is the preferred setup. - -## Query via GraphQL - -Access your indexed data at `/graphql`. For the Certified-hosted production indexer, use: - -- GraphQL API: [`https://api.indexer.hypercerts.dev/graphql`](https://api.indexer.hypercerts.dev/graphql) -- GraphiQL explorer: [`https://api.indexer.hypercerts.dev/graphiql`](https://api.indexer.hypercerts.dev/graphiql) -- WebSocket subscriptions: `wss://api.indexer.hypercerts.dev/graphql/ws` - -The GraphQL API supports standard introspection on the same `/graphql` endpoint. GraphQL clients and code generators can point at that URL directly, for example: - -```yaml -schema: https://api.indexer.hypercerts.dev/graphql -``` - -If a tool expects a local schema file, export one with: - -```bash -npx -y get-graphql-schema https://api.indexer.hypercerts.dev/graphql > schema.graphql -npx -y get-graphql-schema https://api.indexer.hypercerts.dev/graphql --json > schema.json -``` - -If Node prints a `punycode` deprecation warning, it comes from the CLI's dependencies; the schema file is still written. - -```graphql -# Query records by collection -query { - records(collection: "org.hypercerts.claim.activity") { - edges { - node { - uri - did - value - } - } - } -} - -# With typed queries (when lexicon schemas are loaded) -query { - orgHypercertsClaimActivity(first: 10) { - edges { - node { - uri - workScope - startDate - createdAt - } - } - } -} - -# With typed filter queries (title contains "Hypercerts") -query { - orgHypercertsClaimActivity( - first: 10 - where: { title: { contains: "Hypercerts" } } - ) { - edges { - node { - uri - title - createdAt - } - } - } -} -``` - -## Quick start - -For local development with default settings: - -```bash -git clone https://github.com/gainforest/hyperindex.git -cd hyperindex -cp .env.example .env -go run ./cmd/hyperindex -``` - -Open [http://localhost:8080/graphiql/admin](http://localhost:8080/graphiql/admin) to access the admin interface. - -## Register lexicons - -Lexicons define the AT Protocol record types you want to index. You can register them via: - -1. Admin GraphQL API at `/graphiql/admin` -2. Client admin UI at `https:///lexicons` (you must log in with an admin DID) - -```graphql -mutation { - uploadLexicons(files: [...]) # Upload lexicon JSON files -} -``` - -Or place lexicon JSON files in a directory and set the `LEXICON_DIR` environment variable. - -For hypercerts, you would register the `org.hypercerts.claim.*` lexicons — see [Introduction to Lexicons](/lexicons/introduction-to-lexicons) for the full list. - -## Endpoints - -| Endpoint | Description | -|---|---| -| `/graphql` | Public GraphQL API. Supports standard GraphQL introspection for schema and codegen tools. | -| `/graphql/ws` | GraphQL subscriptions (WebSocket) | -| `/admin/graphql` | Admin GraphQL API | -| `/graphiql` | Browser-based GraphiQL explorer for the public API | -| `/graphiql/admin` | GraphQL playground (admin API) | -| `/health` | Health check | -| `/stats` | Server statistics | - -## Deployment configuration - -Use this section when deploying Hyperindex (backend, Tap, and client). It lists the environment variables you should set first for a reliable initial deployment, followed by optional variables for advanced tuning. - -### Baseline deployment variables - -These are the variables you should set first to get a stable deployment running. - -Note: some managed platforms (including Railway) may auto-provision a subset of variables. - -### Hyperindex backend - -| Variable | Example | What it is for | -|---|---|---| -| `HOST` | `0.0.0.0` | Makes the app reachable in container runtime | -| `PORT` | `8080` | App port | -| `DATABASE_URL` | `sqlite:/data/hypergoat.db` | Main indexed-records database | -| `EXTERNAL_BASE_URL` | `https://hyperindex.example.com` | Public backend URL used by frontend/admin flows and GraphiQL links | -| `SECRET_KEY_BASE` | `` | Session/signing secret | -| `ADMIN_DIDS` | `did:plc:...` | DIDs with admin privileges | -| `ADMIN_API_KEY` | `` | Required at startup. Shared secret for trusted admin proxy requests; must exactly match client `HYPERINDEX_ADMIN_API_KEY` | -| `TAP_ENABLED` | `true` | Enables Tap mode | -| `TAP_URL` | `ws://tap.railway.internal:2480` | Tap websocket endpoint | -| `TAP_ADMIN_PASSWORD` | `` | Tap admin auth secret | - -### Tap service - -| Variable | Example | What it is for | -|---|---|---| -| `TAP_DATABASE_URL` | `sqlite:///data/tap.db` | Persists Tap cursor/state (self-managed; Railway autoconfigures) | -| `TAP_ADMIN_PASSWORD` | `` | Protects Tap admin routes | -| `TAP_COLLECTION_FILTERS` | `app.certified.*,org.hypercerts.*` | Filters ingested record collections | -| `TAP_SIGNAL_COLLECTION` | `app.certified.actor.profile` | Signal collection for repo discovery | - -### Client (Next.js) - -| Variable | Example | What it is for | -|---|---|---| -| `NEXT_PUBLIC_HYPERINDEX_URL` | `https://hyperindex.example.com` | Browser-side URL of your Hyperindex backend | -| `HYPERINDEX_URL` | `https://hyperindex.example.com` | Server-side URL of your Hyperindex backend (used by Next API proxy routes). If unset, it falls back to `NEXT_PUBLIC_HYPERINDEX_URL` | -| `HYPERINDEX_ADMIN_API_KEY` | `` | Server-side only. Used by the Next.js admin proxy; must exactly match backend `ADMIN_API_KEY` | -| `NEXT_PUBLIC_CLIENT_URL` | `https://hyperindex-frontend.example.com` | Client frontend URL used for OAuth client metadata and auth redirects | -| `COOKIE_SECRET` | `` | Session encryption | -| `ATPROTO_JWK_PRIVATE` | `` | Confidential OAuth signing key | - -Hyperindex normalizes both `NEXT_PUBLIC_HYPERINDEX_URL` and `HYPERINDEX_URL`: it trims surrounding whitespace, removes trailing slashes, and prepends `https://` when no scheme is provided. - -### Admin API key pairing - -When deploying the backend with the Next.js client, set the same random secret in both services: - -```bash -# Hyperindex backend -ADMIN_API_KEY= - -# Next.js client -HYPERINDEX_ADMIN_API_KEY= -``` - -The backend requires `ADMIN_API_KEY` at startup. It must be at least 16 characters and must not include leading or trailing whitespace. The client keeps `HYPERINDEX_ADMIN_API_KEY` server-side only and uses it when proxying admin GraphQL requests to `/admin/graphql`. - -This key does **not** grant admin rights by itself. The signed-in user's DID must still be listed in backend `ADMIN_DIDS`; the matching key only lets the backend trust the client's `X-User-DID` header. - -### Optional variables - -Set these only when needed. - -### Backend -- `ALLOWED_ORIGINS` -- `TAP_DISABLE_ACKS` - -> `TAP_DISABLE_ACKS` is configured on the **backend/indexer** service. -> In some deployments, ACK mode (`TAP_DISABLE_ACKS=false`) can cause repeated websocket disconnect loops (for example: `connection reset by peer`, `close 1006`, frequent reconnect backoff). -> If you see that pattern, set `TAP_DISABLE_ACKS=true` on the **Hyperindex backend** to stabilize ingestion first, then investigate Tap resource/config compatibility before re-enabling ACK mode. - -### Tap -- `TAP_FULL_NETWORK` -- `TAP_FIREHOSE_PARALLELISM` -- `TAP_RESYNC_PARALLELISM` -- `TAP_OUTBOX_PARALLELISM` -- `TAP_MAX_DB_CONNS` -- `TAP_OUTBOX_CAPACITY` -- `TAP_NO_REPLAY` -- `TAP_REPO_FETCH_TIMEOUT` - -> `TAP_FULL_NETWORK=true` enables full-network tracking and triggers a broad historical backfill across discoverable repos. -> This can materially increase ingestion load, network requests, and storage use. - -### Client -- Additional auth/provider-specific settings depending on deployment model - -## Common pitfalls - -- **Wrong variable on wrong service** - - `TAP_COLLECTION_FILTERS`, `TAP_SIGNAL_COLLECTION`, `TAP_FULL_NETWORK` belong to **Tap** - - `TAP_DISABLE_ACKS`, `TAP_ENABLED`, `TAP_URL` belong to **Hyperindex backend** - -- **Client works but admin requests fail** - - `HYPERINDEX_URL` is missing on the client deployment - - `NEXT_PUBLIC_HYPERINDEX_URL` alone is not enough for server-side proxy routes - - `HYPERINDEX_ADMIN_API_KEY` is missing on the client deployment - - `HYPERINDEX_ADMIN_API_KEY` does not exactly match backend `ADMIN_API_KEY` - - `EXTERNAL_BASE_URL` does not match the backend's public URL - -- **`admin privileges required` while logged in** - - Logged-in DID is not present in backend `ADMIN_DIDS` - - The client and backend admin API keys do not match, so the backend ignores the proxied `X-User-DID` header - - You rotated `ADMIN_API_KEY` or `HYPERINDEX_ADMIN_API_KEY` on only one service - -- **Trailing slash URL issues** - - `NEXT_PUBLIC_CLIENT_URL` must **not** include a trailing slash. - - Use: - - `https://hyperindex-frontend.example.com` ✅ - - `https://hyperindex-frontend.example.com/` ❌ - - A trailing slash can cause OAuth client metadata lookup errors (for example: `client_metadata not found`). - -- **Healthcheck confusion** - - Backend healthcheck should be `/health` - - Frontend usually uses `/` unless you explicitly add a `/health` route - -## Deploy on Railway - -### 1) Deploy Hyperindex backend - -1. Create a Railway service from the repository -2. Attach a persistent volume mounted to `/data` -3. Set healthcheck path to `/health` -4. Add backend variables from the baseline list above -5. Deploy - -Use `PORT=8080` for the Hyperindex service. For public URL variables (for example `EXTERNAL_BASE_URL`, `NEXT_PUBLIC_HYPERINDEX_URL`, `HYPERINDEX_URL`), use the Railway-generated HTTPS domain (typically without appending `:8080`). - -### 2) Deploy Tap - -1. Create a Railway service from image: - `ghcr.io/bluesky-social/indigo/tap:latest` (or a pinned tag) -2. Attach a persistent volume mounted to `/data` -3. Add Tap variables from the baseline list above, but **do not manually set `TAP_DATABASE_URL` on Railway** (Railway autoconfigures this for the service) -4. Deploy - -See the official ATProto Tap Railway guide: [RAILWAY_DEPLOY.md](https://github.com/bluesky-social/indigo/blob/main/cmd/tap/RAILWAY_DEPLOY.md). - -### 3) Connect backend to Tap - -Set on backend service: - -- `TAP_ENABLED=true` -- `TAP_URL=ws://:2480` -- `TAP_ADMIN_PASSWORD=` - -Redeploy backend after updating these values. - -### 4) Deploy client (Next.js) - -Deploy client on Railway/Vercel/etc and set: - -- `NEXT_PUBLIC_HYPERINDEX_URL=` -- `HYPERINDEX_URL=` -- auth/session vars (`NEXT_PUBLIC_CLIENT_URL`, `COOKIE_SECRET`, `ATPROTO_JWK_PRIVATE`) as needed - -### Railway-specific DB path notes - -For mounted volumes at `/data`: - -**Railway** - -- Backend SQLite: `DATABASE_URL=sqlite:/data/hypergoat.db` -- Tap SQLite: do not manually set `TAP_DATABASE_URL` (it is autoconfigured) - -**Non-Railway/self-managed** - -- Backend SQLite: `DATABASE_URL=sqlite:/data/hypergoat.db` -- Tap SQLite: `TAP_DATABASE_URL=sqlite:///data/tap.db` - -## Running with Docker - -```bash -docker compose up --build -``` - -## Learn more - -- [GitHub repository](https://github.com/gainforest/hyperindex) — upstream GainForest repository that the Certified indexer is forked from -- [Certified Services](/reference/certified-services#indexers) — current public indexer endpoints -- [Building on Hypercerts](/getting-started/building-on-hypercerts) — integration patterns for platforms and tools - -{% /remote-doc %} diff --git a/styles/globals.css b/styles/globals.css index 4c4c3c2..446ce42 100644 --- a/styles/globals.css +++ b/styles/globals.css @@ -1140,26 +1140,6 @@ a.sidebar-link-active:hover { color: var(--color-text-secondary); } -.remote-doc-status { - margin-bottom: var(--space-6); - padding: var(--space-3) var(--space-4); - border: 1px solid var(--color-border); - border-radius: var(--radius-md); - background: var(--color-bg-subtle); - color: var(--color-text-secondary); - font-size: 0.875rem; - line-height: 1.5; -} - -.remote-doc-status--error { - border-color: var(--color-warning); - background: var(--color-warning-bg); - color: var(--color-text-primary); -} - -.remote-doc-status--error strong { - color: var(--color-text-heading); -} /* ===== Pagination ===== */ .pagination { display: flex; diff --git a/test/external-doc-links.test.js b/test/external-doc-links.test.js new file mode 100644 index 0000000..6a59bc1 --- /dev/null +++ b/test/external-doc-links.test.js @@ -0,0 +1,37 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { + resolveExternalDocHref, + resolveExternalDocImageSrc, +} = require('../lib/external-doc-links'); + +const source = { + repo: 'hypercerts-org/ePDS', + ref: 'main', + path: 'docs/tutorial.md', +}; + +test('leaves anchors, site paths, and absolute URLs unchanged', () => { + for (const href of ['#section', '/reference/faq', 'https://example.com/docs', 'mailto:docs@example.com']) { + assert.equal(resolveExternalDocHref(href, source), href); + } +}); + +test('resolves relative files and directories to GitHub', () => { + assert.equal( + resolveExternalDocHref('./other.md?plain=1#section', source), + 'https://github.com/hypercerts-org/ePDS/blob/main/docs/other.md?plain=1#section', + ); + assert.equal( + resolveExternalDocHref('../packages/demo', source), + 'https://github.com/hypercerts-org/ePDS/tree/main/packages/demo', + ); +}); + +test('resolves relative images to raw GitHub content', () => { + assert.equal( + resolveExternalDocImageSrc('./images/architecture.png', source), + 'https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/images/architecture.png', + ); + assert.equal(resolveExternalDocImageSrc('/images/local.png', source), '/images/local.png'); +}); diff --git a/test/external-docs-snapshot.test.js b/test/external-docs-snapshot.test.js new file mode 100644 index 0000000..d38fe3a --- /dev/null +++ b/test/external-docs-snapshot.test.js @@ -0,0 +1,119 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { + buildFingerprintDocument, + collectSourceSnapshot, + sha256, +} = require('../lib/external-docs-snapshot'); + +function response({ ok = true, status = 200, statusText = 'OK', text = '', json }) { + return { + ok, + status, + statusText, + text: async () => text, + json: async () => json, + }; +} + +const source = { + id: 'epds', + title: 'ePDS', + repo: 'hypercerts-org/ePDS', + ref: 'main', + path: 'docs/tutorial.md', +}; + +test('build fingerprint uses the exact Markdown snapshot fetched for rendering', async (context) => { + const originalFetch = global.fetch; + context.after(() => { global.fetch = originalFetch; }); + let upstreamMarkdown = '# Version A'; + let contentFetches = 0; + + global.fetch = async (url) => { + if (String(url).includes('/commits?')) { + return response({ json: [{ commit: { committer: { date: '2026-07-13T00:00:00Z' } } }] }); + } + contentFetches += 1; + return response({ text: upstreamMarkdown }); + }; + + const snapshot = await collectSourceSnapshot(source); + upstreamMarkdown = '# Version B'; + const fingerprint = buildFingerprintDocument([snapshot]); + + assert.equal(contentFetches, 1); + assert.equal(snapshot.markdown, '# Version A'); + assert.equal(snapshot.contentHash, sha256('# Version A')); + assert.notEqual(snapshot.contentHash, sha256(upstreamMarkdown)); + assert.equal(snapshot.updatedAt, '2026-07-13T00:00:00Z'); + assert.equal(fingerprint.sources.epds.contentHash, sha256('# Version A')); +}); + +test('fails when the registered Markdown file cannot be fetched', async (context) => { + const originalFetch = global.fetch; + context.after(() => { global.fetch = originalFetch; }); + + global.fetch = async (url) => { + if (String(url).includes('/commits?')) { + return response({ json: [] }); + } + return response({ ok: false, status: 404, statusText: 'Not Found', text: 'missing' }); + }; + + await assert.rejects( + () => collectSourceSnapshot(source), + /returned 404 Not Found.*Check repo, ref, path, and DOCS_SOURCE_TOKEN/, + ); +}); + +test('fails when the registered Markdown file is empty', async (context) => { + const originalFetch = global.fetch; + context.after(() => { global.fetch = originalFetch; }); + + global.fetch = async (url) => { + if (String(url).includes('/commits?')) { + return response({ json: [] }); + } + return response({ text: ' \n' }); + }; + + await assert.rejects(() => collectSourceSnapshot(source), /External doc "epds".*is empty/); +}); + +test('informational timestamps do not affect combined fingerprints', () => { + const base = { + ...source, + size: 6, + contentHash: sha256('# Docs'), + markdown: '# Docs', + }; + + const first = buildFingerprintDocument([{ ...base, updatedAt: '2026-01-01T00:00:00Z' }]); + const second = buildFingerprintDocument([{ ...base, updatedAt: '2026-07-01T00:00:00Z' }]); + assert.equal(first.combinedFingerprint, second.combinedFingerprint); +}); + +test('registry ordering does not affect the combined fingerprint', () => { + const first = { + ...source, + size: 3, + contentHash: sha256('# A'), + markdown: '# A', + }; + const second = { + id: 'hyperindex', + title: 'Hyperindex', + repo: 'gainforest/hyperindex', + ref: 'main', + path: 'docs/hyperindex.md', + size: 3, + contentHash: sha256('# B'), + markdown: '# B', + }; + + assert.equal( + buildFingerprintDocument([first, second]).combinedFingerprint, + buildFingerprintDocument([second, first]).combinedFingerprint, + ); +}); diff --git a/test/external-docs.test.js b/test/external-docs.test.js new file mode 100644 index 0000000..520451f --- /dev/null +++ b/test/external-docs.test.js @@ -0,0 +1,218 @@ +const assert = require('node:assert/strict'); +const { mkdtempSync, rmSync, writeFileSync } = require('node:fs'); +const { tmpdir } = require('node:os'); +const { join } = require('node:path'); +const test = require('node:test'); +const { + DEFAULT_ALLOWED_SOURCE_ORGS, + getAllowedSourceOrgs, + loadExternalDocSources, + loadExternalDocsContent, + parseMarkdownFrontmatter, + resolveExternalDocSnapshot, + resolvePageDocument, +} = require('../lib/external-docs'); +const { compileExternalDocPage } = require('../lib/external-docs-loader'); + +function withTempFile(name, contents, run) { + const directory = mkdtempSync(join(tmpdir(), 'external-docs-test-')); + const path = join(directory, name); + writeFileSync(path, contents); + + try { + return run(path); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +function loadRegistry(contents, env = {}) { + return withTempFile('docs-sources.yml', contents, (path) => loadExternalDocSources(path, env)); +} + +const registry = `sources: + - id: epds + title: ePDS + repo: hypercerts-org/ePDS + ref: main + path: docs/tutorial.md +`; + +const snapshot = { + id: 'epds', + title: 'ePDS', + repo: 'hypercerts-org/ePDS', + ref: 'main', + path: 'docs/tutorial.md', + markdown: '# Canonical ePDS docs', +}; +const content = { sources: { epds: snapshot } }; + +test('uses default source organizations when the environment override is absent or blank', () => { + assert.deepEqual(getAllowedSourceOrgs({}), DEFAULT_ALLOWED_SOURCE_ORGS); + assert.deepEqual(getAllowedSourceOrgs({ DOCS_ALLOWED_SOURCE_ORGS: ' ' }), DEFAULT_ALLOWED_SOURCE_ORGS); +}); + +test('normalizes and deduplicates configured source organizations', () => { + assert.deepEqual( + getAllowedSourceOrgs({ DOCS_ALLOWED_SOURCE_ORGS: ' Example-Org,HYPERCERTS-ORG,example-org ' }), + ['example-org', 'hypercerts-org'], + ); + assert.throws( + () => getAllowedSourceOrgs({ DOCS_ALLOWED_SOURCE_ORGS: 'valid owner,other' }), + /comma-separated list/, + ); +}); + +test('loads the single-file source schema', () => { + assert.deepEqual(loadRegistry(registry), [{ + id: 'epds', + title: 'ePDS', + repo: 'hypercerts-org/ePDS', + ref: 'main', + path: 'docs/tutorial.md', + }]); +}); + +for (const extension of ['md', 'mdoc', 'mdx']) { + test(`accepts .${extension} source files`, () => { + const [source] = loadRegistry(`sources: + - id: docs + title: Docs + repo: hypercerts-org/docs + ref: main + path: guide.${extension} +`); + assert.equal(source.path, `guide.${extension}`); + }); +} + +test('rejects legacy URL and directory source fields', () => { + for (const field of ['rawUrl', 'sourceUrl', 'docsPath', 'entrypoint', 'fingerprintMode']) { + assert.throws(() => loadRegistry(`sources: + - id: docs + title: Docs + repo: hypercerts-org/docs + ref: main + path: guide.md + ${field}: legacy +`), new RegExp(`unsupported field.*${field}`)); + } +}); + +test('validates registry shape, source identity, repository, ref, path, and duplicates', () => { + const invalidRegistries = [ + '{}', + 'sources: {}', + 'sources:\n - id: Bad_ID\n title: Bad\n repo: hypercerts-org/docs\n ref: main\n path: guide.md', + 'sources:\n - id: docs\n repo: hypercerts-org/docs\n ref: main\n path: guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: invalid\n ref: main\n path: guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: unapproved/docs\n ref: main\n path: guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: ""\n path: guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: main\n path: guide.txt', + 'sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: main\n path: ../guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: main\n path: /guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: main\n path: docs//guide.md', + "sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: main\n path: 'docs\\\\guide.md'", + 'sources:\n - id: docs\n title: One\n repo: hypercerts-org/docs\n ref: main\n path: one.md\n - id: docs\n title: Two\n repo: hypercerts-org/docs\n ref: main\n path: two.md', + ]; + + for (const value of invalidRegistries) { + assert.throws(() => loadRegistry(value), /Invalid docs-sources.yml/); + } +}); + +test('parses absent and valid frontmatter', () => { + assert.deepEqual(parseMarkdownFrontmatter('# Heading'), {}); + assert.deepEqual( + parseMarkdownFrontmatter('---\ntitle: Example\nexternalDoc: epds\n---\n'), + { title: 'Example', externalDoc: 'epds' }, + ); +}); + +test('includes the file label in malformed frontmatter errors', () => { + assert.throws( + () => parseMarkdownFrontmatter('---\ntitle: [broken\n---\n', 'pages/broken.md'), + /Invalid frontmatter in pages\/broken.md/, + ); +}); + +test('loads generated external content and reports missing or malformed files', () => { + withTempFile('content.json', JSON.stringify(content), (path) => { + assert.equal(loadExternalDocsContent(path).sources.epds.markdown, '# Canonical ePDS docs'); + }); + for (const malformed of ['{broken', '{}', '{"sources":[]}']) { + withTempFile('content.json', malformed, (path) => { + assert.throws(() => loadExternalDocsContent(path), /Run npm run generate:external-docs first/); + }); + } + assert.throws( + () => loadExternalDocsContent(join(tmpdir(), 'missing-external-docs-content.json')), + /Run npm run generate:external-docs first/, + ); +}); + +test('resolves externalDoc through the generated build snapshot', () => { + assert.equal(resolveExternalDocSnapshot('epds', content), snapshot); + assert.throws(() => resolveExternalDocSnapshot('missing', content), /Unknown externalDoc "missing"/); + assert.throws(() => resolveExternalDocSnapshot(42, content), /externalDoc must be a lowercase registry id/); + assert.throws( + () => resolveExternalDocSnapshot('epds', { sources: { epds: { markdown: ' ' } } }), + /has no generated Markdown/, + ); +}); + +test('uses external Markdown for a frontmatter-only external page', () => { + const localMarkdown = '---\ntitle: ePDS\nexternalDoc: epds\n---\n'; + assert.deepEqual( + resolvePageDocument({ title: 'ePDS', externalDoc: 'epds' }, localMarkdown, content, 'pages/epds.md'), + { markdown: '# Canonical ePDS docs', externalDoc: snapshot }, + ); +}); + +test('compiles external Markdown with local frontmatter and generated source metadata', () => { + const localMarkdown = '---\ntitle: Local title\nexternalDoc: epds\n---\n'; + const externalContent = { + sources: { + epds: { + ...snapshot, + markdown: '---\ntitle: Upstream title\n---\n# Canonical body', + }, + }, + }; + const compiled = compileExternalDocPage( + { title: 'Local title', externalDoc: 'epds' }, + localMarkdown, + externalContent, + 'pages/epds.md', + ); + + assert.deepEqual(parseMarkdownFrontmatter(compiled), { + title: 'Local title', + externalDoc: 'epds', + __externalDocSource: { + id: 'epds', + repo: 'hypercerts-org/ePDS', + ref: 'main', + path: 'docs/tutorial.md', + }, + }); + assert.match(compiled, /\n# Canonical body$/); + assert.doesNotMatch(compiled, /Upstream title/); +}); + +test('leaves ordinary local pages unchanged', () => { + const localMarkdown = '---\ntitle: Local\n---\n# Local page'; + assert.deepEqual( + resolvePageDocument({ title: 'Local' }, localMarkdown, content), + { markdown: localMarkdown, externalDoc: null }, + ); +}); + +test('rejects stale local fallback content on external pages', () => { + const localMarkdown = '---\ntitle: ePDS\nexternalDoc: epds\n---\n# Stale fallback'; + assert.throws( + () => resolvePageDocument({ title: 'ePDS', externalDoc: 'epds' }, localMarkdown, content, 'pages/epds.md'), + /pages\/epds.md sets externalDoc and must not contain a local Markdown body/, + ); +}); From 0b23a4f16f8492e17cef53dadb77177cabfaeac6 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Tue, 14 Jul 2026 15:44:26 +0200 Subject: [PATCH 14/16] github-actions: replace refresh dry run with docs CI --- .github/workflows/docs-ci.yml | 56 +++++++++ .github/workflows/docs-refresh-pr-dry-run.yml | 110 ------------------ 2 files changed, 56 insertions(+), 110 deletions(-) create mode 100644 .github/workflows/docs-ci.yml delete mode 100644 .github/workflows/docs-refresh-pr-dry-run.yml diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml new file mode 100644 index 0000000..c7a7bac --- /dev/null +++ b/.github/workflows/docs-ci.yml @@ -0,0 +1,56 @@ +name: Docs CI + +on: + pull_request: + branches: + - main + paths: + - '.github/workflows/docs*.yml' + - 'docs-sources.yml' + - 'docs/remote-markdown.md' + - 'components/**' + - 'lib/external-docs*.js' + - 'lib/generate-docs-fingerprint.js' + - 'lib/generate-external-docs-manifest.js' + - 'lib/generate-raw-pages.js' + - 'lib/generate-search-index.js' + - 'lib/compare-docs-fingerprint.js' + - 'markdoc/**' + - 'next.config.js' + - 'pages/**' + - 'test/**' + - 'package.json' + - 'package-lock.json' + +permissions: + contents: read + +concurrency: + group: docs-ci-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + checks: + name: Test and build documentation + runs-on: ubuntu-latest + + steps: + - name: Checkout docs repo + uses: actions/checkout@v6.0.2 + + - name: Setup Node + uses: actions/setup-node@v6.4.0 + with: + node-version: 22 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Run tests + run: npm test + + - name: Build static documentation + run: npm run build + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/docs-refresh-pr-dry-run.yml b/.github/workflows/docs-refresh-pr-dry-run.yml deleted file mode 100644 index b4de95b..0000000 --- a/.github/workflows/docs-refresh-pr-dry-run.yml +++ /dev/null @@ -1,110 +0,0 @@ -name: External docs refresh dry run (temporary) - -on: - pull_request: - branches: - - main - paths: - - '.github/workflows/docs-refresh*.yml' - - 'docs-sources.yml' - - 'docs/remote-markdown.md' - - 'components/**' - - 'lib/external-docs*.js' - - 'lib/generate-docs-fingerprint.js' - - 'lib/generate-external-docs-manifest.js' - - 'lib/generate-raw-pages.js' - - 'lib/generate-search-index.js' - - 'lib/compare-docs-fingerprint.js' - - 'markdoc/**' - - 'next.config.js' - - 'pages/**' - - 'test/**' - - 'package.json' - - 'package-lock.json' - -permissions: - contents: read - -concurrency: - group: external-docs-refresh-pr-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - dry-run: - name: Compare external docs fingerprints without deploying - runs-on: ubuntu-latest - - steps: - - name: Checkout docs repo - uses: actions/checkout@v6.0.2 - - - name: Setup Node - uses: actions/setup-node@v6.4.0 - with: - node-version: 22 - cache: npm - - - name: Install dependencies - run: npm ci - - - name: Run tests - run: npm test - - - name: Build static documentation and fingerprint - run: npm run build - env: - GITHUB_TOKEN: ${{ github.token }} - - - name: Download deployed docs fingerprint - env: - DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://staging.docs.hypercerts.org/docs-fingerprint.json' }} - run: | - set +e - http_code=$(curl -sS -L -w "%{http_code}" \ - -o /tmp/deployed-docs-fingerprint.json \ - "$DOCS_FINGERPRINT_URL") - curl_status=$? - set -e - - if [ "$curl_status" -ne 0 ]; then - echo "::error::Failed to download deployed docs fingerprint from $DOCS_FINGERPRINT_URL. Dry-run cannot compare against deployed state." - exit 1 - fi - - case "$http_code" in - 200) - ;; - 404) - echo '{}' > /tmp/deployed-docs-fingerprint.json - ;; - *) - echo "::error::Unexpected HTTP $http_code while downloading $DOCS_FINGERPRINT_URL. Dry-run cannot compare against deployed state." - exit 1 - ;; - esac - - - name: Compare fingerprints - id: diff - run: | - node lib/compare-docs-fingerprint.js \ - public/docs-fingerprint.json \ - /tmp/deployed-docs-fingerprint.json >> "$GITHUB_OUTPUT" - - - name: Report dry-run result - run: | - if [ "${{ steps.diff.outputs.changed }}" = "true" ]; then - echo "::notice::External docs fingerprint differs from the deployed site. The production workflow would trigger the Vercel deploy hook." - else - echo "::notice::External docs fingerprint matches the deployed site. The production workflow would do nothing." - fi - - { - echo "### External docs refresh dry run" - echo "" - echo "This temporary PR-only workflow never calls the Vercel deploy hook." - echo "" - echo "Changed: ${{ steps.diff.outputs.changed || 'unknown' }}" - echo "Reason: ${{ steps.diff.outputs.reason || 'not computed' }}" - echo "Current: ${{ steps.diff.outputs.current_fingerprint || 'n/a' }}" - echo "Deployed: ${{ steps.diff.outputs.deployed_fingerprint || 'n/a' }}" - } >> "$GITHUB_STEP_SUMMARY" From bd4d4c7726b2a4080c490b9aaf2f6cc563a0874d Mon Sep 17 00:00:00 2001 From: kzoeps Date: Tue, 14 Jul 2026 16:08:22 +0200 Subject: [PATCH 15/16] external-docs: address confirmed review findings --- .github/workflows/docs-ci.yml | 2 ++ .github/workflows/docs-refresh.yml | 18 +++++++++++++----- components/MermaidDiagram.js | 10 ++++++---- lib/external-doc-page.js | 2 +- lib/external-docs-loader.js | 2 +- lib/external-docs.js | 2 +- test/external-docs.test.js | 29 ++++++++++++++++++++++++++++- 7 files changed, 52 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index c7a7bac..4260c4a 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -37,6 +37,8 @@ jobs: steps: - name: Checkout docs repo uses: actions/checkout@v6.0.2 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@v6.4.0 diff --git a/.github/workflows/docs-refresh.yml b/.github/workflows/docs-refresh.yml index 6f48676..66deb82 100644 --- a/.github/workflows/docs-refresh.yml +++ b/.github/workflows/docs-refresh.yml @@ -31,6 +31,8 @@ jobs: steps: - name: Checkout docs repo uses: actions/checkout@v6.0.2 + with: + persist-credentials: false - name: Setup Node uses: actions/setup-node@v6.4.0 @@ -109,13 +111,19 @@ jobs: - name: Write summary if: always() + env: + DIFF_CHANGED: ${{ steps.diff.outputs.changed || 'unknown' }} + DIFF_REASON: ${{ steps.diff.outputs.reason || 'not computed' }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run == true }} + CURRENT_FINGERPRINT: ${{ steps.diff.outputs.current_fingerprint || 'n/a' }} + DEPLOYED_FINGERPRINT: ${{ steps.diff.outputs.deployed_fingerprint || 'n/a' }} run: | { echo "### External docs refresh" echo "" - echo "Changed: ${{ steps.diff.outputs.changed || 'unknown' }}" - echo "Reason: ${{ steps.diff.outputs.reason || 'not computed' }}" - echo "Dry run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run == true }}" - echo "Current: ${{ steps.diff.outputs.current_fingerprint || 'n/a' }}" - echo "Deployed: ${{ steps.diff.outputs.deployed_fingerprint || 'n/a' }}" + echo "Changed: $DIFF_CHANGED" + echo "Reason: $DIFF_REASON" + echo "Dry run: $DRY_RUN" + echo "Current: $CURRENT_FINGERPRINT" + echo "Deployed: $DEPLOYED_FINGERPRINT" } >> "$GITHUB_STEP_SUMMARY" diff --git a/components/MermaidDiagram.js b/components/MermaidDiagram.js index 0809191..aea38b2 100644 --- a/components/MermaidDiagram.js +++ b/components/MermaidDiagram.js @@ -6,10 +6,12 @@ let diagramIdCounter = 0; function getMermaid() { if (!mermaidModulePromise) { - mermaidModulePromise = import('mermaid').then((module) => { - const mermaid = module.default || module; - return mermaid; - }); + mermaidModulePromise = import('mermaid') + .then((module) => module.default || module) + .catch((error) => { + mermaidModulePromise = undefined; + throw error; + }); } return mermaidModulePromise; diff --git a/lib/external-doc-page.js b/lib/external-doc-page.js index 5b2f38b..bae0f98 100644 --- a/lib/external-doc-page.js +++ b/lib/external-doc-page.js @@ -18,7 +18,7 @@ function resolveExternalDocSnapshot(externalDoc, content) { } function getMarkdownBody(markdown) { - const frontmatter = markdown.match(/^---\n[\s\S]*?\n---\n?/); + const frontmatter = markdown.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/); return frontmatter ? markdown.slice(frontmatter[0].length) : markdown; } diff --git a/lib/external-docs-loader.js b/lib/external-docs-loader.js index 9d5531a..5523281 100644 --- a/lib/external-docs-loader.js +++ b/lib/external-docs-loader.js @@ -7,7 +7,7 @@ const { } = require('./external-docs'); function stripMarkdownFrontmatter(markdown) { - const match = markdown.match(/^---\n[\s\S]*?\n---\n?/); + const match = markdown.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/); return match ? markdown.slice(match[0].length) : markdown; } diff --git a/lib/external-docs.js b/lib/external-docs.js index e25e9de..f02b84f 100644 --- a/lib/external-docs.js +++ b/lib/external-docs.js @@ -144,7 +144,7 @@ function loadExternalDocSources(registryPath = REGISTRY_PATH, env = process.env) * Parse the YAML frontmatter block from a Markdown file. */ function parseMarkdownFrontmatter(markdown, label = 'Markdown file') { - const match = markdown.match(/^---\n([\s\S]*?)\n---\n?/); + const match = markdown.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/); if (!match) return {}; try { diff --git a/test/external-docs.test.js b/test/external-docs.test.js index 520451f..3de898c 100644 --- a/test/external-docs.test.js +++ b/test/external-docs.test.js @@ -122,12 +122,16 @@ test('validates registry shape, source identity, repository, ref, path, and dupl } }); -test('parses absent and valid frontmatter', () => { +test('parses absent, LF, and CRLF frontmatter', () => { assert.deepEqual(parseMarkdownFrontmatter('# Heading'), {}); assert.deepEqual( parseMarkdownFrontmatter('---\ntitle: Example\nexternalDoc: epds\n---\n'), { title: 'Example', externalDoc: 'epds' }, ); + assert.deepEqual( + parseMarkdownFrontmatter('---\r\ntitle: Example\r\nexternalDoc: epds\r\n---\r\n'), + { title: 'Example', externalDoc: 'epds' }, + ); }); test('includes the file label in malformed frontmatter errors', () => { @@ -201,6 +205,29 @@ test('compiles external Markdown with local frontmatter and generated source met assert.doesNotMatch(compiled, /Upstream title/); }); +test('compiles CRLF external pages and strips CRLF snapshot frontmatter', () => { + const localMarkdown = '---\r\ntitle: Local title\r\nexternalDoc: epds\r\n---\r\n'; + const externalContent = { + sources: { + epds: { + ...snapshot, + markdown: '---\r\ntitle: Upstream title\r\n---\r\n# Canonical body', + }, + }, + }; + const frontmatter = parseMarkdownFrontmatter(localMarkdown); + const compiled = compileExternalDocPage( + frontmatter, + localMarkdown, + externalContent, + 'pages/epds.md', + ); + + assert.equal(frontmatter.externalDoc, 'epds'); + assert.match(compiled, /\n# Canonical body$/); + assert.doesNotMatch(compiled, /Upstream title/); +}); + test('leaves ordinary local pages unchanged', () => { const localMarkdown = '---\ntitle: Local\n---\n# Local page'; assert.deepEqual( From 37496062632ae142d83289831625a4f1a252812b Mon Sep 17 00:00:00 2001 From: kzoeps Date: Tue, 14 Jul 2026 16:16:19 +0200 Subject: [PATCH 16/16] external-docs: document new functions and constants --- components/CopyRawButton.js | 3 +++ components/MermaidDiagram.js | 15 +++++++++++++++ lib/compare-docs-fingerprint.js | 3 +++ lib/external-doc-links.js | 6 ++++++ lib/external-doc-page.js | 3 +++ lib/external-docs-loader.js | 3 +++ lib/external-docs-snapshot.js | 19 +++++++++++++++++++ lib/external-docs.js | 8 ++++++++ lib/generate-docs-fingerprint.js | 13 +++++++++++++ lib/generate-last-updated.js | 1 + lib/generate-raw-pages.js | 6 ++++++ lib/generate-search-index.js | 6 ++++++ markdoc/nodes/document.markdoc.js | 7 +++++++ markdoc/nodes/fence.markdoc.js | 6 ++++++ markdoc/nodes/image.markdoc.js | 4 ++++ markdoc/nodes/link.markdoc.js | 3 +++ markdoc/tags/br.markdoc.js | 1 + next.config.js | 3 +++ 18 files changed, 110 insertions(+) diff --git a/components/CopyRawButton.js b/components/CopyRawButton.js index 42003b9..801c362 100644 --- a/components/CopyRawButton.js +++ b/components/CopyRawButton.js @@ -1,6 +1,9 @@ import { useState } from 'react'; import { useRouter } from 'next/router'; +/** + * Map a documentation route to the generated local Markdown artifact used by page actions. + */ function getGeneratedRawUrl(currentPath) { if (currentPath === '/') return '/raw/index.md'; return `/raw${currentPath}.md`; diff --git a/components/MermaidDiagram.js b/components/MermaidDiagram.js index aea38b2..9c3ceec 100644 --- a/components/MermaidDiagram.js +++ b/components/MermaidDiagram.js @@ -1,9 +1,14 @@ import { useEffect, useMemo, useState } from 'react'; import { CodeBlock } from './CodeBlock'; +/** Cached Mermaid module load shared by every diagram on the current page. */ let mermaidModulePromise; +/** Monotonic counter used to give Mermaid render targets unique DOM identifiers. */ let diagramIdCounter = 0; +/** + * Load Mermaid once per browser session while allowing a failed chunk load to be retried. + */ function getMermaid() { if (!mermaidModulePromise) { mermaidModulePromise = import('mermaid') @@ -17,11 +22,17 @@ function getMermaid() { return mermaidModulePromise; } +/** + * Allocate a unique identifier for one Mermaid render target. + */ function getDiagramId() { diagramIdCounter += 1; return `mermaid-diagram-${diagramIdCounter}`; } +/** + * Select the Mermaid theme that matches the current documentation color scheme. + */ function getPreferredMermaidTheme() { if (typeof document !== 'undefined' && document.documentElement.classList.contains('dark')) { return 'dark'; @@ -43,6 +54,7 @@ export function MermaidDiagram({ chart, children }) { const [theme, setTheme] = useState('neutral'); useEffect(() => { + /** Synchronize the diagram theme when the page color-scheme class changes. */ const updateTheme = () => setTheme(getPreferredMermaidTheme()); updateTheme(); @@ -64,6 +76,9 @@ export function MermaidDiagram({ chart, children }) { setError(null); setSvg(''); + /** + * Render the current source with Mermaid unless this React effect has been cancelled. + */ async function renderDiagram() { try { const mermaid = await getMermaid(); diff --git a/lib/compare-docs-fingerprint.js b/lib/compare-docs-fingerprint.js index c4f7104..5299cc8 100644 --- a/lib/compare-docs-fingerprint.js +++ b/lib/compare-docs-fingerprint.js @@ -14,6 +14,9 @@ function readCombinedFingerprint(path, label) { return typeof parsed.combinedFingerprint === 'string' ? parsed.combinedFingerprint : ''; } +/** + * Compare current and deployed fingerprint files and emit GitHub Actions step outputs. + */ function main() { const [currentPath, deployedPath] = process.argv.slice(2); if (!currentPath || !deployedPath) { diff --git a/lib/external-doc-links.js b/lib/external-doc-links.js index 8d2a495..658d0bd 100644 --- a/lib/external-doc-links.js +++ b/lib/external-doc-links.js @@ -1,3 +1,6 @@ +/** + * Return true when a link should remain unchanged instead of being resolved against an external repository. + */ function isAbsoluteOrSiteHref(href) { return !href || href.startsWith('#') @@ -5,6 +8,9 @@ function isAbsoluteOrSiteHref(href) { || /^[a-z][a-z0-9+.-]*:/i.test(href); } +/** + * Split external source metadata into repository coordinates and the Markdown file's containing directory. + */ function getSourceLocation(source) { const [owner, repoName] = source.repo.split('/'); const sourceDirectory = source.path.split('/').slice(0, -1).join('/'); diff --git a/lib/external-doc-page.js b/lib/external-doc-page.js index bae0f98..f61b8c4 100644 --- a/lib/external-doc-page.js +++ b/lib/external-doc-page.js @@ -17,6 +17,9 @@ function resolveExternalDocSnapshot(externalDoc, content) { return snapshot; } +/** + * Remove an optional LF- or CRLF-delimited frontmatter block from Markdown content. + */ function getMarkdownBody(markdown) { const frontmatter = markdown.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/); return frontmatter ? markdown.slice(frontmatter[0].length) : markdown; diff --git a/lib/external-docs-loader.js b/lib/external-docs-loader.js index 5523281..4b959ee 100644 --- a/lib/external-docs-loader.js +++ b/lib/external-docs-loader.js @@ -6,6 +6,9 @@ const { resolvePageDocument, } = require('./external-docs'); +/** + * Strip upstream frontmatter so local page metadata remains authoritative after compilation. + */ function stripMarkdownFrontmatter(markdown) { const match = markdown.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/); return match ? markdown.slice(match[0].length) : markdown; diff --git a/lib/external-docs-snapshot.js b/lib/external-docs-snapshot.js index 072fdb0..cb001d0 100644 --- a/lib/external-docs-snapshot.js +++ b/lib/external-docs-snapshot.js @@ -1,6 +1,7 @@ const crypto = require('crypto'); const { encodeGitHubPath } = require('./external-docs'); +/** GitHub REST API version used for deterministic source and commit requests. */ const GITHUB_API_VERSION = '2022-11-28'; /** @@ -28,6 +29,9 @@ function sha256(value) { return `sha256:${crypto.createHash('sha256').update(value).digest('hex')}`; } +/** + * Build GitHub API headers with optional bearer authentication and an explicit response media type. + */ function getGitHubHeaders(token, accept, userAgent) { const headers = { Accept: accept, @@ -38,11 +42,17 @@ function getGitHubHeaders(token, accept, userAgent) { return headers; } +/** + * Split a validated external source repository into its GitHub owner and repository name. + */ function getRepoParts(source) { const [owner, repoName] = source.repo.split('/'); return { owner, repoName }; } +/** + * Fetch a GitHub API resource and convert non-success responses into actionable source errors. + */ async function fetchGitHubResponse(url, token, accept, userAgent) { const response = await fetch(url, { headers: getGitHubHeaders(token, accept, userAgent), @@ -55,6 +65,9 @@ async function fetchGitHubResponse(url, token, accept, userAgent) { return response; } +/** + * Fetch the raw Markdown file registered by an external documentation source. + */ async function fetchSourceMarkdown(source, token) { const { owner, repoName } = getRepoParts(source); const url = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/contents/${encodeGitHubPath(source.path)}?ref=${encodeURIComponent(source.ref)}`; @@ -71,6 +84,9 @@ async function fetchSourceMarkdown(source, token) { return markdown; } +/** + * Read the latest source-file commit timestamp, returning null when commit metadata is unavailable. + */ async function fetchSourceUpdatedAt(source, token) { const { owner, repoName } = getRepoParts(source); const url = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/commits?sha=${encodeURIComponent(source.ref)}&path=${encodeURIComponent(source.path)}&per_page=1`; @@ -114,6 +130,9 @@ async function collectExternalDocSnapshots(sources, token = '') { return Promise.all(sources.map((source) => collectSourceSnapshot(source, token))); } +/** + * Select only source identity and content fields that are allowed to affect a fingerprint. + */ function stableSnapshotSource(snapshot) { return { id: snapshot.id, diff --git a/lib/external-docs.js b/lib/external-docs.js index f02b84f..9669f03 100644 --- a/lib/external-docs.js +++ b/lib/external-docs.js @@ -12,7 +12,9 @@ const DEFAULT_ALLOWED_SOURCE_ORGS = ['hypercerts-org', 'gainforest']; const REGISTRY_PATH = join(__dirname, '..', 'docs-sources.yml'); /** Absolute path to the generated build-time external documentation snapshot. */ const CONTENT_PATH = join(__dirname, 'external-docs-content.json'); +/** File extensions accepted for registered external Markdown sources. */ const MARKDOWN_EXTENSIONS = /\.(md|mdoc|mdx)$/i; +/** Registry keys allowed on each external documentation source. */ const SOURCE_FIELDS = new Set(['id', 'title', 'repo', 'ref', 'path']); /** @@ -50,6 +52,9 @@ function encodeGitHubPath(value) { return value.split('/').map(encodeURIComponent).join('/'); } +/** + * Normalize and validate one registry path without allowing absolute or parent-relative traversal. + */ function normalizeRegistryPath(value, fieldName) { if (typeof value !== 'string' || value.trim() === '') { throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a non-empty relative Markdown path.`); @@ -71,6 +76,9 @@ function normalizeRegistryPath(value, fieldName) { return normalized; } +/** + * Validate one raw registry entry and return the normalized source consumed by snapshot generation. + */ function normalizeSource(rawSource, index, allowedOrgs) { const label = `sources[${index}]`; if (!rawSource || typeof rawSource !== 'object' || Array.isArray(rawSource)) { diff --git a/lib/generate-docs-fingerprint.js b/lib/generate-docs-fingerprint.js index 13306b0..a053fed 100644 --- a/lib/generate-docs-fingerprint.js +++ b/lib/generate-docs-fingerprint.js @@ -6,6 +6,7 @@ const { collectExternalDocSnapshots, } = require('./external-docs-snapshot'); +/** Default public artifact written when the CLI receives no explicit output path. */ const DEFAULT_OUTPUT = join(__dirname, '..', 'public', 'docs-fingerprint.json'); /** @@ -19,6 +20,9 @@ const DEFAULT_OUTPUT = join(__dirname, '..', 'public', 'docs-fingerprint.json'); * 5. Generation and source-update timestamps are informational and never affect comparisons. */ +/** + * Read a CLI option from separate or --name=value argument syntax. + */ function getArgumentValue(argv, longName, shortName) { const index = argv.findIndex((arg) => arg === longName || arg === shortName); if (index !== -1) { @@ -45,6 +49,9 @@ function getContentPath(argv) { return getArgumentValue(argv, '--content', '-c'); } +/** + * Load snapshots from a generated content file and reject malformed source collections. + */ function loadSnapshotsFromContent(contentPath) { let content; try { @@ -60,6 +67,9 @@ function loadSnapshotsFromContent(contentPath) { return Object.values(content.sources); } +/** + * Use an existing build snapshot when provided, otherwise fetch every registered source. + */ async function getSnapshots(contentPath) { if (contentPath) return loadSnapshotsFromContent(contentPath); @@ -68,6 +78,9 @@ async function getSnapshots(contentPath) { return collectExternalDocSnapshots(sources, token); } +/** + * Generate and write the combined external-documentation fingerprint artifact. + */ async function main() { const argv = process.argv.slice(2); const output = getOutputPath(argv); diff --git a/lib/generate-last-updated.js b/lib/generate-last-updated.js index 71e008d..0f10073 100644 --- a/lib/generate-last-updated.js +++ b/lib/generate-last-updated.js @@ -35,6 +35,7 @@ function getLastUpdated(filePath) { } const files = walkDir(PAGES_DIR); +/** Generated snapshot used to prefer upstream commit timestamps for external pages. */ const externalContent = loadExternalDocsContent(); const map = {}; diff --git a/lib/generate-raw-pages.js b/lib/generate-raw-pages.js index b66bb3b..2acb986 100644 --- a/lib/generate-raw-pages.js +++ b/lib/generate-raw-pages.js @@ -38,6 +38,9 @@ function getRawOutputPath(filePath) { return join(OUTPUT_DIR, outputRel); } +/** + * Resolve one page to the exact Markdown that should be published in the raw artifact tree. + */ function getRawMarkdown(file, externalContent) { const localMarkdown = readFileSync(file, 'utf-8'); const pagePath = relative(PAGES_DIR, file); @@ -45,6 +48,9 @@ function getRawMarkdown(file, externalContent) { return resolvePageDocument(frontmatter, localMarkdown, externalContent, pagePath).markdown; } +/** + * Regenerate raw Markdown artifacts for every documentation page. + */ async function main() { rmSync(OUTPUT_DIR, { recursive: true, force: true }); mkdirSync(OUTPUT_DIR, { recursive: true }); diff --git a/lib/generate-search-index.js b/lib/generate-search-index.js index 2103fc5..a8c1279 100644 --- a/lib/generate-search-index.js +++ b/lib/generate-search-index.js @@ -23,6 +23,9 @@ function walkDir(dir) { return results; } +/** + * Read a frontmatter field as a string while treating absent and falsy values as empty metadata. + */ function getStringFrontmatterValue(frontmatter, key) { return frontmatter[key] ? String(frontmatter[key]) : ""; } @@ -95,6 +98,9 @@ function getSection(path) { return "Other"; } +/** + * Build the search index from resolved local and external page Markdown. + */ async function main() { const files = walkDir(PAGES_DIR); const externalContent = loadExternalDocsContent(); diff --git a/markdoc/nodes/document.markdoc.js b/markdoc/nodes/document.markdoc.js index a0058b4..22e393f 100644 --- a/markdoc/nodes/document.markdoc.js +++ b/markdoc/nodes/document.markdoc.js @@ -1,13 +1,20 @@ import Markdoc, { Tag, nodes } from '@markdoc/markdoc'; +/** + * Format a Markdoc validation result with its one-based source line when available. + */ function formatValidationError(validation) { const line = validation.lines?.[0]; const location = Number.isInteger(line) ? `line ${line + 1}: ` : ''; return `${location}${validation.error.message}`; } +/** Markdoc document node that validates external pages and provides source context to child nodes. */ const document = { ...nodes.document, + /** + * Transform a document while rejecting invalid external Markdoc and preserving normal local rendering. + */ transform(node, config) { const frontmatter = config.variables?.markdoc?.frontmatter || {}; const source = frontmatter.__externalDocSource; diff --git a/markdoc/nodes/fence.markdoc.js b/markdoc/nodes/fence.markdoc.js index 8595669..c8c298e 100644 --- a/markdoc/nodes/fence.markdoc.js +++ b/markdoc/nodes/fence.markdoc.js @@ -1,5 +1,8 @@ import { Tag } from "@markdoc/markdoc"; +/** + * Normalize a fence language declaration to the first lowercase language token. + */ function getFenceLanguage(language) { return String(language || "") .trim() @@ -23,6 +26,9 @@ const fence = { default: true, }, }, + /** + * Route Mermaid fences to the diagram component and preserve the normal code-block path for other languages. + */ transform(node, config) { const attributes = node.transformAttributes(config); diff --git a/markdoc/nodes/image.markdoc.js b/markdoc/nodes/image.markdoc.js index 190162a..dd49590 100644 --- a/markdoc/nodes/image.markdoc.js +++ b/markdoc/nodes/image.markdoc.js @@ -3,8 +3,12 @@ import externalDocLinks from '../../lib/external-doc-links'; const { resolveExternalDocImageSrc } = externalDocLinks; +/** Markdoc image node that resolves relative images from external documentation repositories. */ const image = { ...nodes.image, + /** + * Rewrite external relative image sources while leaving local and absolute sources unchanged. + */ transform(node, config) { const attributes = node.transformAttributes(config); const source = config.variables?.externalDocSource; diff --git a/markdoc/nodes/link.markdoc.js b/markdoc/nodes/link.markdoc.js index 740eedb..16d3be1 100644 --- a/markdoc/nodes/link.markdoc.js +++ b/markdoc/nodes/link.markdoc.js @@ -6,6 +6,9 @@ const { resolveExternalDocHref } = externalDocLinks; const link = { ...nodes.link, render: 'Link', + /** + * Rewrite external relative links to GitHub while preserving standard local-page link rendering. + */ transform(node, config) { const attributes = node.transformAttributes(config); const source = config.variables?.externalDocSource; diff --git a/markdoc/tags/br.markdoc.js b/markdoc/tags/br.markdoc.js index c76632d..07d0d2a 100644 --- a/markdoc/tags/br.markdoc.js +++ b/markdoc/tags/br.markdoc.js @@ -1,3 +1,4 @@ +/** Markdoc configuration for rendering explicit self-closing line-break tags. */ module.exports = { render: 'br', selfClosing: true, diff --git a/next.config.js b/next.config.js index 4113d48..52fb133 100644 --- a/next.config.js +++ b/next.config.js @@ -3,6 +3,9 @@ const withMarkdoc = require('@markdoc/next.js'); module.exports = withMarkdoc({ mode: 'static' })({ output: 'export', pageExtensions: ['md', 'mdoc', 'js', 'jsx', 'ts', 'tsx'], + /** + * Preprocess Markdown pages with external-document snapshots before Markdoc compilation. + */ webpack(config) { config.module.rules.push({ test: /\.(md|mdoc)$/,