diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml new file mode 100644 index 0000000..4260c4a --- /dev/null +++ b/.github/workflows/docs-ci.yml @@ -0,0 +1,58 @@ +name: Docs CI + +on: + pull_request: + branches: + - main + paths: + - '.github/workflows/docs*.yml' + - 'docs-sources.yml' + - 'docs/remote-markdown.md' + - 'components/**' + - 'lib/external-docs*.js' + - 'lib/generate-docs-fingerprint.js' + - 'lib/generate-external-docs-manifest.js' + - 'lib/generate-raw-pages.js' + - 'lib/generate-search-index.js' + - 'lib/compare-docs-fingerprint.js' + - 'markdoc/**' + - 'next.config.js' + - 'pages/**' + - 'test/**' + - 'package.json' + - 'package-lock.json' + +permissions: + contents: read + +concurrency: + group: docs-ci-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + checks: + name: Test and build documentation + runs-on: ubuntu-latest + + steps: + - name: Checkout docs repo + uses: actions/checkout@v6.0.2 + with: + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@v6.4.0 + with: + node-version: 22 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Run tests + run: npm test + + - name: Build static documentation + run: npm run build + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/docs-refresh.yml b/.github/workflows/docs-refresh.yml new file mode 100644 index 0000000..66deb82 --- /dev/null +++ b/.github/workflows/docs-refresh.yml @@ -0,0 +1,129 @@ +name: Refresh external docs + +on: + schedule: + - cron: '17 * * * *' + workflow_dispatch: + inputs: + dry_run: + description: 'Compare fingerprints without posting to the Vercel deploy hook.' + required: false + default: true + type: boolean + force_deploy: + description: 'Mark the run as changed even when the deployed fingerprint matches.' + required: false + default: false + type: boolean + +permissions: + contents: read + +concurrency: + group: docs-refresh + cancel-in-progress: false + +jobs: + check-docs: + name: Check external docs fingerprint + runs-on: ubuntu-latest + + steps: + - name: Checkout docs repo + uses: actions/checkout@v6.0.2 + with: + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@v6.4.0 + with: + node-version: 22 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Generate current docs fingerprint + run: npm run docs:fingerprint -- --output current-docs-fingerprint.json + env: + DOCS_SOURCE_TOKEN: ${{ secrets.DOCS_SOURCE_TOKEN }} + GITHUB_TOKEN: ${{ github.token }} + + - name: Download deployed docs fingerprint + env: + DOCS_FINGERPRINT_URL: ${{ vars.DOCS_FINGERPRINT_URL || 'https://staging.docs.hypercerts.org/docs-fingerprint.json' }} + run: | + set +e + http_code=$(curl -sS -L -w "%{http_code}" \ + -o deployed-docs-fingerprint.json \ + "$DOCS_FINGERPRINT_URL") + curl_status=$? + set -e + + if [ "$curl_status" -ne 0 ]; then + echo "::error::Failed to download deployed docs fingerprint from $DOCS_FINGERPRINT_URL. Refusing to deploy on an unknown diff." + exit 1 + fi + + case "$http_code" in + 200) + ;; + 404) + echo '{}' > deployed-docs-fingerprint.json + ;; + *) + echo "::error::Unexpected HTTP $http_code while downloading $DOCS_FINGERPRINT_URL. Refusing to deploy on an unknown diff." + exit 1 + ;; + esac + + - name: Check if external docs changed + id: diff + env: + FORCE_DEPLOY: ${{ github.event_name == 'workflow_dispatch' && inputs.force_deploy == true }} + run: | + node lib/compare-docs-fingerprint.js \ + current-docs-fingerprint.json \ + deployed-docs-fingerprint.json >> "$GITHUB_OUTPUT" + + if [ "$FORCE_DEPLOY" = "true" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "reason=forced by workflow_dispatch input" >> "$GITHUB_OUTPUT" + fi + + - name: Trigger Vercel deployment + if: steps.diff.outputs.changed == 'true' + env: + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run == true }} + VERCEL_DEPLOY_HOOK_URL: ${{ secrets.VERCEL_DEPLOY_HOOK_URL }} + run: | + if [ "$DRY_RUN" = "true" ]; then + echo "workflow_dispatch dry_run=true; skipping Vercel deploy hook." + exit 0 + fi + + if [ -z "$VERCEL_DEPLOY_HOOK_URL" ]; then + echo "::error::VERCEL_DEPLOY_HOOK_URL is required when external docs have changed. Create a Vercel Deploy Hook for the production branch and store its URL as a GitHub Actions secret." + exit 1 + fi + + curl -fsS -X POST "$VERCEL_DEPLOY_HOOK_URL" + + - name: Write summary + if: always() + env: + DIFF_CHANGED: ${{ steps.diff.outputs.changed || 'unknown' }} + DIFF_REASON: ${{ steps.diff.outputs.reason || 'not computed' }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run == true }} + CURRENT_FINGERPRINT: ${{ steps.diff.outputs.current_fingerprint || 'n/a' }} + DEPLOYED_FINGERPRINT: ${{ steps.diff.outputs.deployed_fingerprint || 'n/a' }} + run: | + { + echo "### External docs refresh" + echo "" + echo "Changed: $DIFF_CHANGED" + echo "Reason: $DIFF_REASON" + echo "Dry run: $DRY_RUN" + echo "Current: $CURRENT_FINGERPRINT" + echo "Deployed: $DEPLOYED_FINGERPRINT" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 2c8a0f4..06937ba 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,10 @@ out/ # Generated at build time by lib/generate-*.js — no need to track public/raw/ +public/docs-fingerprint.json public/search-index.json public/sitemap.xml +current-docs-fingerprint.json +deployed-docs-fingerprint.json +lib/external-docs-content.json lib/lastUpdated.json diff --git a/components/CopyRawButton.js b/components/CopyRawButton.js index b8ec7b3..801c362 100644 --- a/components/CopyRawButton.js +++ b/components/CopyRawButton.js @@ -1,18 +1,25 @@ import { useState } from 'react'; import { useRouter } from 'next/router'; -function getRawUrl(currentPath) { +/** + * Map a documentation route to the generated local Markdown artifact used by page actions. + */ +function getGeneratedRawUrl(currentPath) { if (currentPath === '/') return '/raw/index.md'; return `/raw${currentPath}.md`; } +/** + * Render page-level actions for copying or viewing the generated Markdown for the current docs page. + * External docs are included in the same local /raw output as ordinary pages. + */ export function CopyRawButton() { const [copied, setCopied] = useState(false); const [copyError, setCopyError] = useState(false); const [isCopying, setIsCopying] = useState(false); const router = useRouter(); const currentPath = router.asPath.split('#')[0].split('?')[0] || '/'; - const rawUrl = getRawUrl(currentPath); + const rawUrl = getGeneratedRawUrl(currentPath); const handleCopy = async () => { setIsCopying(true); diff --git a/components/MermaidDiagram.js b/components/MermaidDiagram.js new file mode 100644 index 0000000..9c3ceec --- /dev/null +++ b/components/MermaidDiagram.js @@ -0,0 +1,133 @@ +import { useEffect, useMemo, useState } from 'react'; +import { CodeBlock } from './CodeBlock'; + +/** Cached Mermaid module load shared by every diagram on the current page. */ +let mermaidModulePromise; +/** Monotonic counter used to give Mermaid render targets unique DOM identifiers. */ +let diagramIdCounter = 0; + +/** + * Load Mermaid once per browser session while allowing a failed chunk load to be retried. + */ +function getMermaid() { + if (!mermaidModulePromise) { + mermaidModulePromise = import('mermaid') + .then((module) => module.default || module) + .catch((error) => { + mermaidModulePromise = undefined; + throw error; + }); + } + + return mermaidModulePromise; +} + +/** + * Allocate a unique identifier for one Mermaid render target. + */ +function getDiagramId() { + diagramIdCounter += 1; + return `mermaid-diagram-${diagramIdCounter}`; +} + +/** + * Select the Mermaid theme that matches the current documentation color scheme. + */ +function getPreferredMermaidTheme() { + if (typeof document !== 'undefined' && document.documentElement.classList.contains('dark')) { + return 'dark'; + } + + return 'neutral'; +} + +/** + * Render a Mermaid fenced code block as an SVG diagram in the browser. + * Use this for Markdown fences with `mermaid` as the language; invalid diagrams fall back to copyable code with an actionable syntax error. + */ +export function MermaidDiagram({ chart, children }) { + const source = (chart || children || '').trim(); + const diagramId = useMemo(getDiagramId, []); + const [svg, setSvg] = useState(''); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(Boolean(source)); + const [theme, setTheme] = useState('neutral'); + + useEffect(() => { + /** Synchronize the diagram theme when the page color-scheme class changes. */ + const updateTheme = () => setTheme(getPreferredMermaidTheme()); + updateTheme(); + + const observer = new MutationObserver(updateTheme); + observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class'] }); + + return () => observer.disconnect(); + }, []); + + useEffect(() => { + if (!source) { + setLoading(false); + setError(new Error('No Mermaid source was provided. Add diagram text inside the mermaid code fence.')); + return undefined; + } + + let cancelled = false; + setLoading(true); + setError(null); + setSvg(''); + + /** + * Render the current source with Mermaid unless this React effect has been cancelled. + */ + async function renderDiagram() { + try { + const mermaid = await getMermaid(); + mermaid.initialize({ + startOnLoad: false, + securityLevel: 'strict', + theme, + }); + const result = await mermaid.render(diagramId, source); + if (cancelled) return; + setSvg(result.svg); + } catch (err) { + if (cancelled) return; + setError(err); + } finally { + if (!cancelled) setLoading(false); + } + } + + renderDiagram(); + + return () => { + cancelled = true; + }; + }, [diagramId, source, theme]); + + if (error) { + return ( +
+

+ Could not render Mermaid diagram. Check the diagram syntax in the source Markdown and reload the page. Details: {error.message} +

+ +
+ ); + } + + if (loading) { + return ( +
+ Rendering Mermaid diagram… +
+ ); + } + + return ( +
+ ); +} diff --git a/components/TableOfContents.js b/components/TableOfContents.js index 21f78ba..f5b2a77 100644 --- a/components/TableOfContents.js +++ b/components/TableOfContents.js @@ -8,16 +8,15 @@ export function TableOfContents() { const router = useRouter(); const currentPath = router.asPath.split("#")[0].split("?")[0]; - // Extract H2 and H3 headings from the DOM after render + // External docs are part of the static page, so headings only need collecting after route changes. useEffect(() => { - if (typeof window === "undefined") return; const article = document.querySelector(".layout-content article"); if (!article) { setHeadings([]); return; } - const elements = article.querySelectorAll("h2, h3"); + const elements = article.querySelectorAll("h2, h3, h4"); const items = Array.from(elements).map((el) => { if (!el.id) { el.id = el.textContent @@ -28,7 +27,7 @@ export function TableOfContents() { return { id: el.id, text: el.textContent, - level: el.tagName === "H3" ? 3 : 2, + level: Number(el.tagName.slice(1)), }; }); setHeadings(items); @@ -94,8 +93,8 @@ export function TableOfContents() { { e.preventDefault(); const target = document.getElementById(id); diff --git a/docs-sources.yml b/docs-sources.yml new file mode 100644 index 0000000..f8f37cf --- /dev/null +++ b/docs-sources.yml @@ -0,0 +1,12 @@ +sources: + - id: epds + title: ePDS + repo: hypercerts-org/ePDS + ref: main + path: docs/tutorial.md + + - id: hyperindex-test + title: Hyperindex test source + repo: gainforest/hyperindex + ref: skills-update + path: docs/hyperindex.md diff --git a/docs/remote-markdown.md b/docs/remote-markdown.md new file mode 100644 index 0000000..f62bd14 --- /dev/null +++ b/docs/remote-markdown.md @@ -0,0 +1,65 @@ +# Build-time external documentation + +Use an external documentation page when one Markdown file in a service repository is the canonical source for a route on this site. External files are fetched before the static build; browsers never fetch the page Markdown. + +## Register a source + +Add the file to `docs-sources.yml`: + +```yaml +sources: + - id: epds + title: ePDS + repo: hypercerts-org/ePDS + ref: main + path: docs/tutorial.md +``` + +- `id` is the stable lowercase identifier used by pages. +- `title` identifies the source in generated fingerprint metadata. +- `repo` is the GitHub `owner/repository` pair. +- `ref` is the branch, tag, or commit to fetch. +- `path` is one `.md`, `.mdoc`, or `.mdx` file in that repository. + +GitHub API request and browser URLs are derived internally. Do not add URLs, directory paths, or separate entrypoints to the registry. + +## Create the page + +Set `externalDoc` in a frontmatter-only page: + +```md +--- +title: ePDS (extended PDS) +description: How to integrate applications with ePDS login. +externalDoc: epds +--- +``` + +Do not add a local Markdown body. The registered file is the only page body, which prevents stale fallback content from diverging from rendering, search, or `/raw` exports. + +## Build behavior + +`npm run generate:external-docs` fetches every registered file once through the GitHub contents API and writes `lib/external-docs-content.json`. The static build then uses that immutable snapshot for: + +- Markdoc page rendering; +- search indexing; +- local `/raw` page exports; +- last-updated metadata; +- deployed external-docs fingerprints. + +External Markdown is parsed with the same Markdoc configuration as local pages. Relative links point to the source repository, relative images use public GitHub content URLs, and fenced `mermaid` diagrams render through the Mermaid component. Extensionless relative paths are treated as directories; link to extensionless files such as `LICENSE`, `Dockerfile`, or `Makefile` with an absolute GitHub URL. + +A missing source, failed content request, empty file, local fallback body, or invalid Markdoc in an external page fails the build with an actionable error. The existing deployment remains online instead of publishing stale or inconsistent content. Source commit timestamps are informational and may be omitted when GitHub cannot provide them. + +## Refresh workflow + +`.github/workflows/docs-refresh.yml` runs hourly and can also be dispatched manually. It fetches the registered files, compares their combined fingerprint with the deployed site, and calls the configured Vercel deploy hook when they differ. Manual runs default to dry-run mode. + +`.github/workflows/docs-refresh-pr-dry-run.yml` builds and compares fingerprints on relevant pull requests without calling a deploy hook. + +Configuration: + +- `VERCEL_DEPLOY_HOOK_URL` is required before deployment is enabled. +- `DOCS_FINGERPRINT_URL` optionally selects the deployed fingerprint to compare. +- `DOCS_SOURCE_TOKEN` optionally grants source-repository access and additional GitHub API capacity. It is required for private repositories. +- `DOCS_ALLOWED_SOURCE_ORGS` optionally overrides the comma-separated trusted-owner allowlist. It defaults to `hypercerts-org,gainforest`. diff --git a/lib/compare-docs-fingerprint.js b/lib/compare-docs-fingerprint.js new file mode 100644 index 0000000..5299cc8 --- /dev/null +++ b/lib/compare-docs-fingerprint.js @@ -0,0 +1,49 @@ +const { readFileSync } = require('fs'); + +/** + * Read a generated docs-fingerprint.json file and return its combined fingerprint. + */ +function readCombinedFingerprint(path, label) { + let parsed; + try { + parsed = JSON.parse(readFileSync(path, 'utf8')); + } catch (error) { + throw new Error(`Unable to read ${label} fingerprint at ${path}: ${error.message}`); + } + + return typeof parsed.combinedFingerprint === 'string' ? parsed.combinedFingerprint : ''; +} + +/** + * Compare current and deployed fingerprint files and emit GitHub Actions step outputs. + */ +function main() { + const [currentPath, deployedPath] = process.argv.slice(2); + if (!currentPath || !deployedPath) { + throw new Error('Usage: node lib/compare-docs-fingerprint.js '); + } + + const current = readCombinedFingerprint(currentPath, 'current'); + if (!current) { + throw new Error(`Current fingerprint file ${currentPath} does not contain combinedFingerprint.`); + } + + const deployed = readCombinedFingerprint(deployedPath, 'deployed'); + const changed = current !== deployed; + + console.error(changed + ? `External docs changed: deployed=${deployed || ''} current=${current}` + : `External docs unchanged: ${current}`); + + console.log(`changed=${changed ? 'true' : 'false'}`); + console.log(`current_fingerprint=${current}`); + console.log(`deployed_fingerprint=${deployed}`); + console.log(`reason=${changed ? 'combined fingerprint differs' : 'combined fingerprint matches deployed site'}`); +} + +try { + main(); +} catch (error) { + console.error(error.message); + process.exit(1); +} diff --git a/lib/external-doc-links.js b/lib/external-doc-links.js new file mode 100644 index 0000000..658d0bd --- /dev/null +++ b/lib/external-doc-links.js @@ -0,0 +1,53 @@ +/** + * Return true when a link should remain unchanged instead of being resolved against an external repository. + */ +function isAbsoluteOrSiteHref(href) { + return !href + || href.startsWith('#') + || href.startsWith('/') + || /^[a-z][a-z0-9+.-]*:/i.test(href); +} + +/** + * Split external source metadata into repository coordinates and the Markdown file's containing directory. + */ +function getSourceLocation(source) { + const [owner, repoName] = source.repo.split('/'); + const sourceDirectory = source.path.split('/').slice(0, -1).join('/'); + return { owner, repoName, sourceDirectory }; +} + +/** + * Resolve a relative external-document link to the corresponding GitHub blob or tree URL. + * For example, ../packages/demo from docs/tutorial.md becomes the repository's packages/demo tree. + */ +function resolveExternalDocHref(href, source) { + if (isAbsoluteOrSiteHref(href)) return href; + + const { owner, repoName, sourceDirectory } = getSourceLocation(source); + const encodedRef = encodeURIComponent(source.ref); + const basePath = `/${owner}/${repoName}/blob/${encodedRef}/${sourceDirectory ? `${sourceDirectory}/` : ''}`; + const resolved = new URL(href, `https://github.com${basePath}`); + const [, resolvedOwner, resolvedRepo, , ref, ...repoPathParts] = resolved.pathname.split('/'); + const repoPath = repoPathParts.join('/'); + const lastSegment = repoPathParts[repoPathParts.length - 1] || ''; + const mode = /\.[a-z0-9]+$/i.test(lastSegment) ? 'blob' : 'tree'; + + return `https://github.com/${resolvedOwner}/${resolvedRepo}/${mode}/${ref}/${repoPath}${resolved.search}${resolved.hash}`; +} + +/** + * Resolve a relative external-document image to public GitHub content that browsers can display. + */ +function resolveExternalDocImageSrc(src, source) { + if (isAbsoluteOrSiteHref(src)) return src; + + const { owner, repoName, sourceDirectory } = getSourceLocation(source); + const base = `https://raw.githubusercontent.com/${owner}/${repoName}/${encodeURIComponent(source.ref)}/${sourceDirectory ? `${sourceDirectory}/` : ''}`; + return new URL(src, base).toString(); +} + +module.exports = { + resolveExternalDocHref, + resolveExternalDocImageSrc, +}; diff --git a/lib/external-doc-page.js b/lib/external-doc-page.js new file mode 100644 index 0000000..f61b8c4 --- /dev/null +++ b/lib/external-doc-page.js @@ -0,0 +1,48 @@ +/** + * Resolve one externalDoc id to the exact Markdown snapshot used by the build. + */ +function resolveExternalDocSnapshot(externalDoc, content) { + if (typeof externalDoc !== 'string' || !/^[a-z0-9][a-z0-9-]*$/.test(externalDoc)) { + throw new Error('externalDoc must be a lowercase registry id such as "epds".'); + } + + const snapshot = content?.sources?.[externalDoc]; + if (!snapshot) { + throw new Error(`Unknown externalDoc "${externalDoc}". Add it to docs-sources.yml and run npm run generate:external-docs.`); + } + if (typeof snapshot.markdown !== 'string' || snapshot.markdown.trim() === '') { + throw new Error(`External doc "${externalDoc}" has no generated Markdown. Run npm run generate:external-docs and check the registered repo, ref, and path.`); + } + + return snapshot; +} + +/** + * Remove an optional LF- or CRLF-delimited frontmatter block from Markdown content. + */ +function getMarkdownBody(markdown) { + const frontmatter = markdown.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/); + return frontmatter ? markdown.slice(frontmatter[0].length) : markdown; +} + +/** + * Resolve the Markdown and source metadata for one page. + * Pages with externalDoc must contain frontmatter only so stale local fallback content cannot diverge from the build snapshot. + */ +function resolvePageDocument(frontmatter, localMarkdown, content, label = 'Markdown page') { + if (!Object.prototype.hasOwnProperty.call(frontmatter, 'externalDoc')) { + return { markdown: localMarkdown, externalDoc: null }; + } + + if (getMarkdownBody(localMarkdown).trim() !== '') { + throw new Error(`${label} sets externalDoc and must not contain a local Markdown body. Remove the stale fallback content; external source failures stop the build.`); + } + + const externalDoc = resolveExternalDocSnapshot(frontmatter.externalDoc, content); + return { markdown: externalDoc.markdown, externalDoc }; +} + +module.exports = { + resolveExternalDocSnapshot, + resolvePageDocument, +}; diff --git a/lib/external-docs-loader.js b/lib/external-docs-loader.js new file mode 100644 index 0000000..4b959ee --- /dev/null +++ b/lib/external-docs-loader.js @@ -0,0 +1,48 @@ +const yaml = require('js-yaml'); +const { + CONTENT_PATH, + loadExternalDocsContent, + parseMarkdownFrontmatter, + resolvePageDocument, +} = require('./external-docs'); + +/** + * Strip upstream frontmatter so local page metadata remains authoritative after compilation. + */ +function stripMarkdownFrontmatter(markdown) { + const match = markdown.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/); + return match ? markdown.slice(match[0].length) : markdown; +} + +/** + * Compile a frontmatter-only external page from its immutable Markdown snapshot. + * Generated source metadata lets Markdoc resolve relative links without exposing registry URLs or the full snapshot to shared bundles. + */ +function compileExternalDocPage(frontmatter, source, content, label) { + const page = resolvePageDocument(frontmatter, source, content, label); + const { id, repo, ref, path } = page.externalDoc; + const compiledFrontmatter = { + ...frontmatter, + __externalDocSource: { id, repo, ref, path }, + }; + + return `---\n${yaml.dump(compiledFrontmatter, { lineWidth: -1, noRefs: true }).trimEnd()}\n---\n\n${stripMarkdownFrontmatter(page.markdown).trimStart()}`; +} + +/** + * Replace a frontmatter-only externalDoc page with its generated Markdown snapshot before Markdoc parses it. + * The injected source metadata is consumed during transformation and is never written back to the page file. + */ +function externalDocsLoader(source) { + const frontmatter = parseMarkdownFrontmatter(source, this.resourcePath); + if (!Object.prototype.hasOwnProperty.call(frontmatter, 'externalDoc')) { + return source; + } + + const content = loadExternalDocsContent(); + this.addDependency(CONTENT_PATH); + return compileExternalDocPage(frontmatter, source, content, this.resourcePath); +} + +module.exports = externalDocsLoader; +module.exports.compileExternalDocPage = compileExternalDocPage; diff --git a/lib/external-docs-snapshot.js b/lib/external-docs-snapshot.js new file mode 100644 index 0000000..cb001d0 --- /dev/null +++ b/lib/external-docs-snapshot.js @@ -0,0 +1,185 @@ +const crypto = require('crypto'); +const { encodeGitHubPath } = require('./external-docs'); + +/** GitHub REST API version used for deterministic source and commit requests. */ +const GITHUB_API_VERSION = '2022-11-28'; + +/** + * Serialize a value with stable object-key ordering so fingerprints do not depend on construction order. + */ +function stableStringify(value) { + if (Array.isArray(value)) { + return `[${value.map(stableStringify).join(',')}]`; + } + + if (value && typeof value === 'object') { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`) + .join(',')}}`; + } + + return JSON.stringify(value); +} + +/** + * Compute a SHA-256 digest with the prefix used by generated fingerprint files. + */ +function sha256(value) { + return `sha256:${crypto.createHash('sha256').update(value).digest('hex')}`; +} + +/** + * Build GitHub API headers with optional bearer authentication and an explicit response media type. + */ +function getGitHubHeaders(token, accept, userAgent) { + const headers = { + Accept: accept, + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + 'User-Agent': userAgent, + }; + if (token) headers.Authorization = `Bearer ${token}`; + return headers; +} + +/** + * Split a validated external source repository into its GitHub owner and repository name. + */ +function getRepoParts(source) { + const [owner, repoName] = source.repo.split('/'); + return { owner, repoName }; +} + +/** + * Fetch a GitHub API resource and convert non-success responses into actionable source errors. + */ +async function fetchGitHubResponse(url, token, accept, userAgent) { + const response = await fetch(url, { + headers: getGitHubHeaders(token, accept, userAgent), + cache: 'no-store', + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`${url} returned ${response.status} ${response.statusText || ''}. Check repo, ref, path, and DOCS_SOURCE_TOKEN. ${body}`.trim()); + } + return response; +} + +/** + * Fetch the raw Markdown file registered by an external documentation source. + */ +async function fetchSourceMarkdown(source, token) { + const { owner, repoName } = getRepoParts(source); + const url = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/contents/${encodeGitHubPath(source.path)}?ref=${encodeURIComponent(source.ref)}`; + const response = await fetchGitHubResponse( + url, + token, + 'application/vnd.github.raw+json', + 'hypercerts-docs-build', + ); + const markdown = await response.text(); + if (markdown.trim() === '') { + throw new Error(`External doc "${source.id}" at ${source.repo}@${source.ref}:${source.path} is empty. Add Markdown content or remove the source from docs-sources.yml.`); + } + return markdown; +} + +/** + * Read the latest source-file commit timestamp, returning null when commit metadata is unavailable. + */ +async function fetchSourceUpdatedAt(source, token) { + const { owner, repoName } = getRepoParts(source); + const url = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/commits?sha=${encodeURIComponent(source.ref)}&path=${encodeURIComponent(source.path)}&per_page=1`; + + try { + const response = await fetchGitHubResponse( + url, + token, + 'application/vnd.github+json', + 'hypercerts-docs-build', + ); + const commits = await response.json(); + return commits[0]?.commit?.committer?.date || commits[0]?.commit?.author?.date || null; + } catch { + return null; + } +} + +/** + * Fetch one immutable Markdown snapshot used by rendering, search, raw exports, and fingerprinting. + */ +async function collectSourceSnapshot(source, token = '') { + const [markdown, updatedAt] = await Promise.all([ + fetchSourceMarkdown(source, token), + fetchSourceUpdatedAt(source, token), + ]); + + return { + ...source, + updatedAt, + size: Buffer.byteLength(markdown), + contentHash: sha256(markdown), + markdown, + }; +} + +/** + * Fetch all registered sources concurrently and preserve registry order in the returned snapshots. + */ +async function collectExternalDocSnapshots(sources, token = '') { + return Promise.all(sources.map((source) => collectSourceSnapshot(source, token))); +} + +/** + * Select only source identity and content fields that are allowed to affect a fingerprint. + */ +function stableSnapshotSource(snapshot) { + return { + id: snapshot.id, + title: snapshot.title, + repo: snapshot.repo, + ref: snapshot.ref, + path: snapshot.path, + contentHash: snapshot.contentHash, + }; +} + +/** + * Build per-source and combined fingerprints from the exact Markdown snapshots used by the site build. + */ +function buildFingerprintDocument(snapshots) { + const outputSources = {}; + const stableSources = []; + const sortedSnapshots = [...snapshots].sort((a, b) => a.id.localeCompare(b.id)); + + for (const snapshot of sortedSnapshots) { + const stableSource = stableSnapshotSource(snapshot); + const fingerprint = sha256(stableStringify(stableSource)); + outputSources[snapshot.id] = { + title: snapshot.title, + repo: snapshot.repo, + ref: snapshot.ref, + path: snapshot.path, + updatedAt: snapshot.updatedAt || undefined, + size: snapshot.size, + contentHash: snapshot.contentHash, + fingerprint, + }; + stableSources.push({ ...stableSource, fingerprint }); + } + + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + sources: outputSources, + combinedFingerprint: sha256(stableStringify({ schemaVersion: 1, sources: stableSources })), + }; +} + +module.exports = { + buildFingerprintDocument, + collectExternalDocSnapshots, + collectSourceSnapshot, + sha256, + stableStringify, +}; diff --git a/lib/external-docs.js b/lib/external-docs.js new file mode 100644 index 0000000..9669f03 --- /dev/null +++ b/lib/external-docs.js @@ -0,0 +1,192 @@ +const { readFileSync } = require('fs'); +const { join } = require('path'); +const yaml = require('js-yaml'); +const { + resolveExternalDocSnapshot, + resolvePageDocument, +} = require('./external-doc-page'); + +/** Default GitHub organizations trusted as external documentation sources. */ +const DEFAULT_ALLOWED_SOURCE_ORGS = ['hypercerts-org', 'gainforest']; +/** Absolute path to the external documentation source registry. */ +const REGISTRY_PATH = join(__dirname, '..', 'docs-sources.yml'); +/** Absolute path to the generated build-time external documentation snapshot. */ +const CONTENT_PATH = join(__dirname, 'external-docs-content.json'); +/** File extensions accepted for registered external Markdown sources. */ +const MARKDOWN_EXTENSIONS = /\.(md|mdoc|mdx)$/i; +/** Registry keys allowed on each external documentation source. */ +const SOURCE_FIELDS = new Set(['id', 'title', 'repo', 'ref', 'path']); + +/** + * Read the trusted GitHub organization allowlist, falling back to project defaults. + */ +function getAllowedSourceOrgs(env = process.env) { + const configured = env.DOCS_ALLOWED_SOURCE_ORGS; + if (typeof configured !== 'string' || configured.trim() === '') { + return [...DEFAULT_ALLOWED_SOURCE_ORGS]; + } + + const organizations = [...new Set(configured + .split(',') + .map((owner) => owner.trim().toLowerCase()) + .filter(Boolean))]; + + if (organizations.length === 0 || organizations.some((owner) => !/^[a-z0-9_.-]+$/.test(owner))) { + throw new Error('DOCS_ALLOWED_SOURCE_ORGS must be a comma-separated list of GitHub organization names.'); + } + + return organizations; +} + +/** + * Return true when a registry path points at a supported Markdown file. + */ +function isMarkdownFilePath(value) { + return MARKDOWN_EXTENSIONS.test(value); +} + +/** + * Encode a GitHub path without collapsing its slash-separated path segments. + */ +function encodeGitHubPath(value) { + return value.split('/').map(encodeURIComponent).join('/'); +} + +/** + * Normalize and validate one registry path without allowing absolute or parent-relative traversal. + */ +function normalizeRegistryPath(value, fieldName) { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a non-empty relative Markdown path.`); + } + + if (value.includes('\\')) { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must use forward slashes.`); + } + + const normalized = value.trim().replace(/^\.\//, '').replace(/\/$/, ''); + const parts = normalized.split('/'); + if (normalized.startsWith('/') || parts.includes('..') || parts.includes('')) { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must be a relative path without empty segments or "..".`); + } + if (!isMarkdownFilePath(normalized)) { + throw new Error(`Invalid docs-sources.yml: ${fieldName} must point to a .md, .mdoc, or .mdx file.`); + } + + return normalized; +} + +/** + * Validate one raw registry entry and return the normalized source consumed by snapshot generation. + */ +function normalizeSource(rawSource, index, allowedOrgs) { + const label = `sources[${index}]`; + if (!rawSource || typeof rawSource !== 'object' || Array.isArray(rawSource)) { + throw new Error(`Invalid docs-sources.yml: ${label} must be an object.`); + } + + const unsupportedFields = Object.keys(rawSource).filter((field) => !SOURCE_FIELDS.has(field)); + if (unsupportedFields.length > 0) { + throw new Error(`Invalid docs-sources.yml: ${label} contains unsupported field${unsupportedFields.length === 1 ? '' : 's'} ${unsupportedFields.map((field) => `"${field}"`).join(', ')}. Use only id, title, repo, ref, and path.`); + } + + const { id, title, repo, ref } = rawSource; + if (typeof id !== 'string' || !/^[a-z0-9][a-z0-9-]*$/.test(id)) { + throw new Error(`Invalid docs-sources.yml: ${label}.id must be a lowercase id like "epds" or "certified-group-service".`); + } + if (typeof title !== 'string' || title.trim() === '') { + throw new Error(`Invalid docs-sources.yml: source "${id}" must set a human-readable title.`); + } + if (typeof repo !== 'string' || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) { + throw new Error(`Invalid docs-sources.yml: source "${id}" repo must look like "owner/repository".`); + } + + const [owner] = repo.split('/'); + if (!allowedOrgs.includes(owner.toLowerCase())) { + throw new Error(`Invalid docs-sources.yml: source "${id}" must use one of these trusted GitHub owners: ${allowedOrgs.join(', ')}.`); + } + if (typeof ref !== 'string' || ref.trim() === '') { + throw new Error(`Invalid docs-sources.yml: source "${id}" must set a branch, tag, or commit in ref.`); + } + if (/[\u0000-\u001f\u007f]/.test(ref)) { + throw new Error(`Invalid docs-sources.yml: source "${id}" ref must not contain control characters.`); + } + + return { + id, + title: title.trim(), + repo, + ref: ref.trim(), + path: normalizeRegistryPath(rawSource.path, `source "${id}" path`), + }; +} + +/** + * Load and validate the build-time external documentation registry. + */ +function loadExternalDocSources(registryPath = REGISTRY_PATH, env = process.env) { + let document; + try { + document = yaml.load(readFileSync(registryPath, 'utf8')) || {}; + } catch (error) { + throw new Error(`Unable to read docs source registry at ${registryPath}: ${error.message}`); + } + + if (!Array.isArray(document.sources)) { + throw new Error('Invalid docs-sources.yml: expected a top-level "sources" array.'); + } + + const allowedOrgs = getAllowedSourceOrgs(env); + const seen = new Set(); + return document.sources.map((rawSource, index) => { + const source = normalizeSource(rawSource, index, allowedOrgs); + if (seen.has(source.id)) { + throw new Error(`Invalid docs-sources.yml: duplicate source id "${source.id}".`); + } + seen.add(source.id); + return source; + }); +} + +/** + * Parse the YAML frontmatter block from a Markdown file. + */ +function parseMarkdownFrontmatter(markdown, label = 'Markdown file') { + const match = markdown.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/); + if (!match) return {}; + + try { + return yaml.load(match[1]) || {}; + } catch (error) { + throw new Error(`Invalid frontmatter in ${label}: ${error.message}`); + } +} + +/** + * Load the immutable external documentation snapshot generated before the site build. + */ +function loadExternalDocsContent(contentPath = CONTENT_PATH) { + try { + const content = JSON.parse(readFileSync(contentPath, 'utf8')); + if (!content.sources || typeof content.sources !== 'object' || Array.isArray(content.sources)) { + throw new Error('expected a top-level "sources" object'); + } + return content; + } catch (error) { + throw new Error(`Unable to read generated external docs content at ${contentPath}: ${error.message}. Run npm run generate:external-docs first.`); + } +} + +module.exports = { + CONTENT_PATH, + DEFAULT_ALLOWED_SOURCE_ORGS, + REGISTRY_PATH, + encodeGitHubPath, + getAllowedSourceOrgs, + isMarkdownFilePath, + loadExternalDocSources, + loadExternalDocsContent, + parseMarkdownFrontmatter, + resolveExternalDocSnapshot, + resolvePageDocument, +}; diff --git a/lib/generate-docs-fingerprint.js b/lib/generate-docs-fingerprint.js new file mode 100644 index 0000000..a053fed --- /dev/null +++ b/lib/generate-docs-fingerprint.js @@ -0,0 +1,98 @@ +const { mkdirSync, readFileSync, writeFileSync } = require('fs'); +const { dirname, join } = require('path'); +const { loadExternalDocSources } = require('./external-docs'); +const { + buildFingerprintDocument, + collectExternalDocSnapshots, +} = require('./external-docs-snapshot'); + +/** Default public artifact written when the CLI receives no explicit output path. */ +const DEFAULT_OUTPUT = join(__dirname, '..', 'public', 'docs-fingerprint.json'); + +/** + * Fingerprinting process: + * + * 1. A site build fetches every registered Markdown file into one immutable snapshot. + * Rendering, search, raw exports, and this script all consume that same snapshot. + * 2. Scheduled refresh checks have no build snapshot, so they fetch the registered files directly. + * 3. Each source fingerprint includes its registry identity and SHA-256 content hash. + * 4. Per-source fingerprints are sorted by id and combined into one stable fingerprint. + * 5. Generation and source-update timestamps are informational and never affect comparisons. + */ + +/** + * Read a CLI option from separate or --name=value argument syntax. + */ +function getArgumentValue(argv, longName, shortName) { + const index = argv.findIndex((arg) => arg === longName || arg === shortName); + if (index !== -1) { + const value = argv[index + 1]; + if (!value) throw new Error(`Missing value after ${argv[index]}.`); + return value; + } + + const inline = argv.find((arg) => arg.startsWith(`${longName}=`)); + return inline ? inline.slice(longName.length + 1) : null; +} + +/** + * Read the fingerprint output path from CLI arguments. + */ +function getOutputPath(argv) { + return getArgumentValue(argv, '--output', '-o') || DEFAULT_OUTPUT; +} + +/** + * Read an optional generated snapshot path from CLI arguments. + */ +function getContentPath(argv) { + return getArgumentValue(argv, '--content', '-c'); +} + +/** + * Load snapshots from a generated content file and reject malformed source collections. + */ +function loadSnapshotsFromContent(contentPath) { + let content; + try { + content = JSON.parse(readFileSync(contentPath, 'utf8')); + } catch (error) { + throw new Error(`Unable to read external docs snapshot at ${contentPath}: ${error.message}. Run npm run generate:external-docs first.`); + } + + if (!content.sources || typeof content.sources !== 'object' || Array.isArray(content.sources)) { + throw new Error(`External docs snapshot at ${contentPath} must contain a "sources" object.`); + } + + return Object.values(content.sources); +} + +/** + * Use an existing build snapshot when provided, otherwise fetch every registered source. + */ +async function getSnapshots(contentPath) { + if (contentPath) return loadSnapshotsFromContent(contentPath); + + const sources = loadExternalDocSources(); + const token = process.env.DOCS_SOURCE_TOKEN || process.env.GITHUB_TOKEN || process.env.GH_TOKEN || ''; + return collectExternalDocSnapshots(sources, token); +} + +/** + * Generate and write the combined external-documentation fingerprint artifact. + */ +async function main() { + const argv = process.argv.slice(2); + const output = getOutputPath(argv); + const snapshots = await getSnapshots(getContentPath(argv)); + const fingerprint = buildFingerprintDocument(snapshots); + + mkdirSync(dirname(output), { recursive: true }); + writeFileSync(output, `${JSON.stringify(fingerprint, null, 2)}\n`); + console.log(`Generated docs fingerprint ${fingerprint.combinedFingerprint} → ${output}`); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/lib/generate-external-docs-manifest.js b/lib/generate-external-docs-manifest.js new file mode 100644 index 0000000..c02a608 --- /dev/null +++ b/lib/generate-external-docs-manifest.js @@ -0,0 +1,28 @@ +const { mkdirSync, writeFileSync } = require('fs'); +const { dirname } = require('path'); +const { CONTENT_PATH, loadExternalDocSources } = require('./external-docs'); +const { collectExternalDocSnapshots } = require('./external-docs-snapshot'); + +/** + * Fetch every registered Markdown file once and write the immutable build snapshot. + */ +async function generateExternalDocs() { + const sources = loadExternalDocSources(); + const token = process.env.DOCS_SOURCE_TOKEN || process.env.GITHUB_TOKEN || process.env.GH_TOKEN || ''; + const snapshots = await collectExternalDocSnapshots(sources, token); + const content = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + sources: Object.fromEntries(snapshots.map((snapshot) => [snapshot.id, snapshot])), + }; + + mkdirSync(dirname(CONTENT_PATH), { recursive: true }); + writeFileSync(CONTENT_PATH, `${JSON.stringify(content, null, 2)}\n`); + + console.log(`Generated build-time snapshots for ${snapshots.length} external Markdown source${snapshots.length === 1 ? '' : 's'}`); +} + +generateExternalDocs().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/lib/generate-last-updated.js b/lib/generate-last-updated.js index d158485..0f10073 100644 --- a/lib/generate-last-updated.js +++ b/lib/generate-last-updated.js @@ -1,6 +1,11 @@ const { execSync } = require("child_process"); -const { readdirSync, statSync, writeFileSync } = require("fs"); +const { readFileSync, readdirSync, statSync, writeFileSync } = require("fs"); const { join, relative } = require("path"); +const { + loadExternalDocsContent, + parseMarkdownFrontmatter, + resolvePageDocument, +} = require("./external-docs"); const PAGES_DIR = join(__dirname, "..", "pages"); const OUTPUT = join(__dirname, "lastUpdated.json"); @@ -30,12 +35,18 @@ function getLastUpdated(filePath) { } const files = walkDir(PAGES_DIR); +/** Generated snapshot used to prefer upstream commit timestamps for external pages. */ +const externalContent = loadExternalDocsContent(); const map = {}; for (const file of files) { const rel = "/" + relative(PAGES_DIR, file).replace(/\.md$/, ""); const route = rel === "/index" ? "/" : rel; - const date = getLastUpdated(file); + const pagePath = relative(PAGES_DIR, file); + const markdown = readFileSync(file, "utf8"); + const frontmatter = parseMarkdownFrontmatter(markdown, pagePath); + const page = resolvePageDocument(frontmatter, markdown, externalContent, pagePath); + const date = page.externalDoc ? page.externalDoc.updatedAt : getLastUpdated(file); if (date) { map[route] = date; } diff --git a/lib/generate-raw-pages.js b/lib/generate-raw-pages.js index bda1cbc..2acb986 100644 --- a/lib/generate-raw-pages.js +++ b/lib/generate-raw-pages.js @@ -7,6 +7,11 @@ const { mkdirSync, } = require('fs'); const { dirname, join, relative } = require('path'); +const { + loadExternalDocsContent, + parseMarkdownFrontmatter, + resolvePageDocument, +} = require('./external-docs'); const PAGES_DIR = join(__dirname, '..', 'pages'); const OUTPUT_DIR = join(__dirname, '..', 'public', 'raw'); @@ -33,15 +38,36 @@ function getRawOutputPath(filePath) { return join(OUTPUT_DIR, outputRel); } -rmSync(OUTPUT_DIR, { recursive: true, force: true }); -mkdirSync(OUTPUT_DIR, { recursive: true }); +/** + * Resolve one page to the exact Markdown that should be published in the raw artifact tree. + */ +function getRawMarkdown(file, externalContent) { + const localMarkdown = readFileSync(file, 'utf-8'); + const pagePath = relative(PAGES_DIR, file); + const frontmatter = parseMarkdownFrontmatter(localMarkdown, pagePath); + return resolvePageDocument(frontmatter, localMarkdown, externalContent, pagePath).markdown; +} + +/** + * Regenerate raw Markdown artifacts for every documentation page. + */ +async function main() { + rmSync(OUTPUT_DIR, { recursive: true, force: true }); + mkdirSync(OUTPUT_DIR, { recursive: true }); -const files = walkDir(PAGES_DIR); + const files = walkDir(PAGES_DIR); + const externalContent = loadExternalDocsContent(); + + for (const file of files) { + const outputPath = getRawOutputPath(file); + mkdirSync(dirname(outputPath), { recursive: true }); + writeFileSync(outputPath, getRawMarkdown(file, externalContent)); + } -for (const file of files) { - const outputPath = getRawOutputPath(file); - mkdirSync(dirname(outputPath), { recursive: true }); - writeFileSync(outputPath, readFileSync(file, 'utf-8')); + console.log(`Generated raw markdown files for ${files.length} pages`); } -console.log(`Generated raw markdown files for ${files.length} pages`); +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/lib/generate-search-index.js b/lib/generate-search-index.js index e6afca6..a8c1279 100644 --- a/lib/generate-search-index.js +++ b/lib/generate-search-index.js @@ -1,5 +1,10 @@ const { readdirSync, statSync, readFileSync, writeFileSync } = require("fs"); const { join, relative } = require("path"); +const { + loadExternalDocsContent, + parseMarkdownFrontmatter, + resolvePageDocument, +} = require("./external-docs"); const PAGES_DIR = join(__dirname, "..", "pages"); const OUTPUT = join(__dirname, "..", "public", "search-index.json"); @@ -18,18 +23,11 @@ function walkDir(dir) { return results; } -function extractFrontmatter(content) { - const fmMatch = content.match(/^---\n([\s\S]*?)\n---/); - if (!fmMatch) return { title: "", description: "" }; - - const frontmatter = fmMatch[1]; - const titleMatch = frontmatter.match(/^title:\s*(.+)$/m); - const descMatch = frontmatter.match(/^description:\s*(.+)$/m); - - return { - title: titleMatch ? titleMatch[1].trim() : "", - description: descMatch ? descMatch[1].trim() : "", - }; +/** + * Read a frontmatter field as a string while treating absent and falsy values as empty metadata. + */ +function getStringFrontmatterValue(frontmatter, key) { + return frontmatter[key] ? String(frontmatter[key]) : ""; } function extractHeadings(content) { @@ -100,40 +98,59 @@ function getSection(path) { return "Other"; } -const files = walkDir(PAGES_DIR); -const index = []; - -for (const file of files) { - const content = readFileSync(file, "utf-8"); - const rel = "/" + relative(PAGES_DIR, file).replace(/\.md$/, ""); - const path = rel === "/index" ? "/" : rel; - - const { title, description } = extractFrontmatter(content); - const headings = extractHeadings(content); - const section = getSection(path); - - // For the home page, only include title (body is mostly card markup) - let body = ""; - if (path !== "/") { - body = stripMarkdown(content); - if (body.length > MAX_BODY_LENGTH) { - body = body.substring(0, MAX_BODY_LENGTH); +/** + * Build the search index from resolved local and external page Markdown. + */ +async function main() { + const files = walkDir(PAGES_DIR); + const externalContent = loadExternalDocsContent(); + const index = []; + + for (const file of files) { + const localContent = readFileSync(file, "utf-8"); + const rel = "/" + relative(PAGES_DIR, file).replace(/\.md$/, ""); + const path = rel === "/index" ? "/" : rel; + + const frontmatter = parseMarkdownFrontmatter(localContent, relative(PAGES_DIR, file)); + const title = getStringFrontmatterValue(frontmatter, "title"); + const description = getStringFrontmatterValue(frontmatter, "description"); + const { markdown: content } = resolvePageDocument( + frontmatter, + localContent, + externalContent, + relative(PAGES_DIR, file), + ); + const headings = extractHeadings(content); + const section = getSection(path); + + // For the home page, only include title (body is mostly card markup) + let body = ""; + if (path !== "/") { + body = stripMarkdown(content); + if (body.length > MAX_BODY_LENGTH) { + body = body.substring(0, MAX_BODY_LENGTH); + } } + + index.push({ + path, + title, + description: description || "", + section, + headings, + body, + }); } - index.push({ - path, - title, - description: description || "", - section, - headings, - body, - }); + writeFileSync(OUTPUT, JSON.stringify(index, null, 2) + "\n"); + console.log( + `Generated search index for ${index.length} pages (${ + Buffer.byteLength(JSON.stringify(index)) / 1024 + } KB)` + ); } -writeFileSync(OUTPUT, JSON.stringify(index, null, 2) + "\n"); -console.log( - `Generated search index for ${index.length} pages (${ - Buffer.byteLength(JSON.stringify(index)) / 1024 - } KB)` -); +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/markdoc/nodes/document.markdoc.js b/markdoc/nodes/document.markdoc.js new file mode 100644 index 0000000..22e393f --- /dev/null +++ b/markdoc/nodes/document.markdoc.js @@ -0,0 +1,44 @@ +import Markdoc, { Tag, nodes } from '@markdoc/markdoc'; + +/** + * Format a Markdoc validation result with its one-based source line when available. + */ +function formatValidationError(validation) { + const line = validation.lines?.[0]; + const location = Number.isInteger(line) ? `line ${line + 1}: ` : ''; + return `${location}${validation.error.message}`; +} + +/** Markdoc document node that validates external pages and provides source context to child nodes. */ +const document = { + ...nodes.document, + /** + * Transform a document while rejecting invalid external Markdoc and preserving normal local rendering. + */ + transform(node, config) { + const frontmatter = config.variables?.markdoc?.frontmatter || {}; + const source = frontmatter.__externalDocSource; + if (!source) { + return new Tag('article', {}, node.transformChildren(config)); + } + + const validationErrors = Markdoc.validate(node, config) + .filter((validation) => validation.error.level !== 'warning'); + if (validationErrors.length > 0) { + const details = validationErrors.map(formatValidationError).join('; '); + throw new Error(`External doc "${source.id}" contains invalid Markdoc. ${details}`); + } + + const externalConfig = { + ...config, + variables: { + ...config.variables, + externalDocSource: source, + }, + }; + + return new Tag('article', {}, node.transformChildren(externalConfig)); + }, +}; + +export default document; diff --git a/markdoc/nodes/fence.markdoc.js b/markdoc/nodes/fence.markdoc.js index f154ba3..c8c298e 100644 --- a/markdoc/nodes/fence.markdoc.js +++ b/markdoc/nodes/fence.markdoc.js @@ -1,3 +1,15 @@ +import { Tag } from "@markdoc/markdoc"; + +/** + * Normalize a fence language declaration to the first lowercase language token. + */ +function getFenceLanguage(language) { + return String(language || "") + .trim() + .toLowerCase() + .split(/\s+/)[0]; +} + const fence = { render: "CodeBlock", attributes: { @@ -14,6 +26,20 @@ const fence = { default: true, }, }, + /** + * Route Mermaid fences to the diagram component and preserve the normal code-block path for other languages. + */ + transform(node, config) { + const attributes = node.transformAttributes(config); + + if (getFenceLanguage(attributes.language) === "mermaid") { + return new Tag("MermaidDiagram", { + chart: attributes.content, + }); + } + + return new Tag("CodeBlock", attributes); + }, }; export default fence; diff --git a/markdoc/nodes/image.markdoc.js b/markdoc/nodes/image.markdoc.js new file mode 100644 index 0000000..dd49590 --- /dev/null +++ b/markdoc/nodes/image.markdoc.js @@ -0,0 +1,22 @@ +import { Tag, nodes } from '@markdoc/markdoc'; +import externalDocLinks from '../../lib/external-doc-links'; + +const { resolveExternalDocImageSrc } = externalDocLinks; + +/** Markdoc image node that resolves relative images from external documentation repositories. */ +const image = { + ...nodes.image, + /** + * Rewrite external relative image sources while leaving local and absolute sources unchanged. + */ + transform(node, config) { + const attributes = node.transformAttributes(config); + const source = config.variables?.externalDocSource; + return new Tag('img', { + ...attributes, + src: source ? resolveExternalDocImageSrc(attributes.src, source) : attributes.src, + }); + }, +}; + +export default image; diff --git a/markdoc/nodes/index.js b/markdoc/nodes/index.js index fe064ec..3f0657a 100644 --- a/markdoc/nodes/index.js +++ b/markdoc/nodes/index.js @@ -1,3 +1,5 @@ +export { default as document } from './document.markdoc'; export { default as heading } from './heading.markdoc'; export { default as fence } from './fence.markdoc'; +export { default as image } from './image.markdoc'; export { default as link } from './link.markdoc'; diff --git a/markdoc/nodes/link.markdoc.js b/markdoc/nodes/link.markdoc.js index 8cccbc2..16d3be1 100644 --- a/markdoc/nodes/link.markdoc.js +++ b/markdoc/nodes/link.markdoc.js @@ -1,8 +1,26 @@ -import { nodes } from "@markdoc/markdoc"; +import { Tag, nodes } from '@markdoc/markdoc'; +import externalDocLinks from '../../lib/external-doc-links'; + +const { resolveExternalDocHref } = externalDocLinks; const link = { ...nodes.link, - render: "Link", + render: 'Link', + /** + * Rewrite external relative links to GitHub while preserving standard local-page link rendering. + */ + transform(node, config) { + const attributes = node.transformAttributes(config); + const source = config.variables?.externalDocSource; + return new Tag( + 'Link', + { + ...attributes, + href: source ? resolveExternalDocHref(attributes.href, source) : attributes.href, + }, + node.transformChildren(config), + ); + }, }; export default link; diff --git a/markdoc/tags/br.markdoc.js b/markdoc/tags/br.markdoc.js index c76632d..07d0d2a 100644 --- a/markdoc/tags/br.markdoc.js +++ b/markdoc/tags/br.markdoc.js @@ -1,3 +1,4 @@ +/** Markdoc configuration for rendering explicit self-closing line-break tags. */ module.exports = { render: 'br', selfClosing: true, diff --git a/next.config.js b/next.config.js index 4df4325..52fb133 100644 --- a/next.config.js +++ b/next.config.js @@ -3,4 +3,15 @@ const withMarkdoc = require('@markdoc/next.js'); module.exports = withMarkdoc({ mode: 'static' })({ output: 'export', pageExtensions: ['md', 'mdoc', 'js', 'jsx', 'ts', 'tsx'], + /** + * Preprocess Markdown pages with external-document snapshots before Markdoc compilation. + */ + webpack(config) { + config.module.rules.push({ + test: /\.(md|mdoc)$/, + enforce: 'pre', + use: [require.resolve('./lib/external-docs-loader')], + }); + return config; + }, }); diff --git a/package-lock.json b/package-lock.json index ceb1af4..a739621 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,12 +12,39 @@ "@markdoc/next.js": "^0.5.0", "@vercel/analytics": "^1.6.1", "flexsearch": "^0.8.212", + "js-yaml": "^4.1.1", + "mermaid": "^11.15.0", "next": "^16.1.6", "prism-react-renderer": "^2.4.1", "react": "^19.2.4", "react-dom": "^19.2.4" } }, + "node_modules/@antfu/install-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-1.1.0.tgz", + "integrity": "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==", + "license": "MIT", + "dependencies": { + "package-manager-detector": "^1.3.0", + "tinyexec": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@braintree/sanitize-url": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-7.1.2.tgz", + "integrity": "sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA==", + "license": "MIT" + }, + "node_modules/@chevrotain/types": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/@chevrotain/types/-/types-11.1.2.tgz", + "integrity": "sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw==", + "license": "Apache-2.0" + }, "node_modules/@emnapi/runtime": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", @@ -28,6 +55,23 @@ "tslib": "^2.4.0" } }, + "node_modules/@iconify/types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@iconify/types/-/types-2.0.0.tgz", + "integrity": "sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==", + "license": "MIT" + }, + "node_modules/@iconify/utils": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@iconify/utils/-/utils-3.1.3.tgz", + "integrity": "sha512-LPKOXPn/zV+zis1oOfGWogaXVpqUybF3ZS6SCZIsz8vg0ivVp9+fVqyYB7xq0aiST/VhUQYGO1qo6uoYSiEJqw==", + "license": "MIT", + "dependencies": { + "@antfu/install-pkg": "^1.1.0", + "@iconify/types": "^2.0.0", + "import-meta-resolve": "^4.2.0" + } + }, "node_modules/@img/colour": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", @@ -533,6 +577,15 @@ "react": "*" } }, + "node_modules/@mermaid-js/parser": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@mermaid-js/parser/-/parser-1.1.1.tgz", + "integrity": "sha512-VuHdsYMK1bT6X2JbcAaWAhugTRvRBRyuZgd+c22swUeI9g/ntaxF7CY7dYarhZovofCbUNO0G7JesfmNtjYOCw==", + "license": "MIT", + "dependencies": { + "@chevrotain/types": "~11.1.1" + } + }, "node_modules/@next/env": { "version": "16.1.6", "resolved": "https://registry.npmjs.org/@next/env/-/env-16.1.6.tgz", @@ -676,6 +729,265 @@ "tslib": "^2.8.0" } }, + "node_modules/@types/d3": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", + "integrity": "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/d3-axis": "*", + "@types/d3-brush": "*", + "@types/d3-chord": "*", + "@types/d3-color": "*", + "@types/d3-contour": "*", + "@types/d3-delaunay": "*", + "@types/d3-dispatch": "*", + "@types/d3-drag": "*", + "@types/d3-dsv": "*", + "@types/d3-ease": "*", + "@types/d3-fetch": "*", + "@types/d3-force": "*", + "@types/d3-format": "*", + "@types/d3-geo": "*", + "@types/d3-hierarchy": "*", + "@types/d3-interpolate": "*", + "@types/d3-path": "*", + "@types/d3-polygon": "*", + "@types/d3-quadtree": "*", + "@types/d3-random": "*", + "@types/d3-scale": "*", + "@types/d3-scale-chromatic": "*", + "@types/d3-selection": "*", + "@types/d3-shape": "*", + "@types/d3-time": "*", + "@types/d3-time-format": "*", + "@types/d3-timer": "*", + "@types/d3-transition": "*", + "@types/d3-zoom": "*" + } + }, + "node_modules/@types/d3-array": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz", + "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", + "license": "MIT" + }, + "node_modules/@types/d3-axis": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.6.tgz", + "integrity": "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-brush": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.6.tgz", + "integrity": "sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-chord": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.6.tgz", + "integrity": "sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg==", + "license": "MIT" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "license": "MIT" + }, + "node_modules/@types/d3-contour": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.6.tgz", + "integrity": "sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw==", + "license": "MIT" + }, + "node_modules/@types/d3-dispatch": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.7.tgz", + "integrity": "sha512-5o9OIAdKkhN1QItV2oqaE5KMIiXAvDWBDPrD85e58Qlz1c1kI/J0NcqbEG88CoTwJrYe7ntUCVfeUl2UJKbWgA==", + "license": "MIT" + }, + "node_modules/@types/d3-drag": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.7.tgz", + "integrity": "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-dsv": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.7.tgz", + "integrity": "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==", + "license": "MIT" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "license": "MIT" + }, + "node_modules/@types/d3-fetch": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.7.tgz", + "integrity": "sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA==", + "license": "MIT", + "dependencies": { + "@types/d3-dsv": "*" + } + }, + "node_modules/@types/d3-force": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.10.tgz", + "integrity": "sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw==", + "license": "MIT" + }, + "node_modules/@types/d3-format": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.4.tgz", + "integrity": "sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g==", + "license": "MIT" + }, + "node_modules/@types/d3-geo": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.1.0.tgz", + "integrity": "sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ==", + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-hierarchy": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.7.tgz", + "integrity": "sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg==", + "license": "MIT" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "license": "MIT", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz", + "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", + "license": "MIT" + }, + "node_modules/@types/d3-polygon": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.2.tgz", + "integrity": "sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA==", + "license": "MIT" + }, + "node_modules/@types/d3-quadtree": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.6.tgz", + "integrity": "sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg==", + "license": "MIT" + }, + "node_modules/@types/d3-random": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.3.tgz", + "integrity": "sha512-Imagg1vJ3y76Y2ea0871wpabqp613+8/r0mCLEBfdtqC7xMSfj9idOnmBYyMoULfHePJyxMAw3nWhJxzc+LFwQ==", + "license": "MIT" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz", + "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", + "license": "MIT", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-iWMJgwkK7yTRmWqRB5plb1kadXyQ5Sj8V/zYlFGMUBbIPKQScw+Dku9cAAMgJG+z5GYDoMjWGLVOvjghDEFnKQ==", + "license": "MIT" + }, + "node_modules/@types/d3-selection": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.11.tgz", + "integrity": "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==", + "license": "MIT" + }, + "node_modules/@types/d3-shape": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.8.tgz", + "integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==", + "license": "MIT", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz", + "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", + "license": "MIT" + }, + "node_modules/@types/d3-time-format": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.3.tgz", + "integrity": "sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg==", + "license": "MIT" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "license": "MIT" + }, + "node_modules/@types/d3-transition": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.9.tgz", + "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-zoom": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.8.tgz", + "integrity": "sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==", + "license": "MIT", + "dependencies": { + "@types/d3-interpolate": "*", + "@types/d3-selection": "*" + } + }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "license": "MIT" + }, "node_modules/@types/linkify-it": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-3.0.5.tgz", @@ -707,6 +1019,23 @@ "integrity": "sha512-AUZTa7hQ2KY5L7AmtSiqxlhWxb4ina0yd8hNbl4TWuqnv/pFP0nDMb3YrfSBf4hJVGLh2YEIBfKaBW/9UEl6IQ==", "license": "MIT" }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT", + "optional": true + }, + "node_modules/@upsetjs/venn.js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@upsetjs/venn.js/-/venn.js-2.0.0.tgz", + "integrity": "sha512-WbBhLrooyePuQ1VZxrJjtLvTc4NVfpOyKx0sKqioq9bX1C1m7Jgykkn8gLrtwumBioXIqam8DLxp88Adbue6Hw==", + "license": "MIT", + "optionalDependencies": { + "d3-selection": "^3.0.0", + "d3-transition": "^3.0.1" + } + }, "node_modules/@vercel/analytics": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-1.6.1.tgz", @@ -795,6 +1124,538 @@ "node": ">=6" } }, + "node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/cose-base": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-1.0.3.tgz", + "integrity": "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==", + "license": "MIT", + "dependencies": { + "layout-base": "^1.0.0" + } + }, + "node_modules/cytoscape": { + "version": "3.34.0", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.34.0.tgz", + "integrity": "sha512-62rNSrioXw93uliKFBwjukeQyeWwH2PqDrTac31r2P6464u3AUvTk0xS4LVvT251g7IgkFunrI48ZEZGjywSOg==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/cytoscape-cose-bilkent": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cytoscape-cose-bilkent/-/cytoscape-cose-bilkent-4.1.0.tgz", + "integrity": "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^1.0.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cytoscape-fcose/-/cytoscape-fcose-2.2.0.tgz", + "integrity": "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^2.2.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/cose-base": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-2.2.0.tgz", + "integrity": "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==", + "license": "MIT", + "dependencies": { + "layout-base": "^2.0.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/layout-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz", + "integrity": "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==", + "license": "MIT" + }, + "node_modules/d3": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", + "integrity": "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==", + "license": "ISC", + "dependencies": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "license": "ISC", + "dependencies": { + "d3-path": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "license": "ISC", + "dependencies": { + "d3-array": "^3.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "license": "ISC", + "dependencies": { + "delaunator": "5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "license": "ISC", + "dependencies": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json.js", + "csv2tsv": "bin/dsv2dsv.js", + "dsv2dsv": "bin/dsv2dsv.js", + "dsv2json": "bin/dsv2json.js", + "json2csv": "bin/json2dsv.js", + "json2dsv": "bin/json2dsv.js", + "json2tsv": "bin/json2dsv.js", + "tsv2csv": "bin/dsv2dsv.js", + "tsv2json": "bin/dsv2json.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-fetch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", + "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", + "license": "ISC", + "dependencies": { + "d3-dsv": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", + "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2.5.0 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "license": "BSD-3-Clause", + "dependencies": { + "internmap": "^1.0.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-sankey/node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-path": "1" + } + }, + "node_modules/d3-sankey/node_modules/internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==", + "license": "ISC" + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/dagre-d3-es": { + "version": "7.0.14", + "resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.14.tgz", + "integrity": "sha512-P4rFMVq9ESWqmOgK+dlXvOtLwYg0i7u0HBGJER0LZDJT2VHIPAMZ/riPxqJceWMStH5+E61QxFra9kIS3AqdMg==", + "license": "MIT", + "dependencies": { + "d3": "^7.9.0", + "lodash-es": "^4.17.21" + } + }, + "node_modules/dayjs": { + "version": "1.11.21", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.21.tgz", + "integrity": "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==", + "license": "MIT" + }, + "node_modules/delaunator": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.1.0.tgz", + "integrity": "sha512-AGrQ4QSgssa1NGmWmLPqN5NY2KajF5MqxetNEO+o0n3ZwZZeTmt7bBnvzHWrmkZFxGgr4HdyFgelzgi06otLuQ==", + "license": "ISC", + "dependencies": { + "robust-predicates": "^3.0.2" + } + }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -805,6 +1666,25 @@ "node": ">=8" } }, + "node_modules/dompurify": { + "version": "3.4.9", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.9.tgz", + "integrity": "sha512-4dPSRMRDqHvs0V4YDFCsaIZo4if5u0xM+llyxiM2fwuZFdKArUBAF3VtI2+n8NKg9P870WMdYk0UhqQNoWXbfQ==", + "license": "(MPL-2.0 OR Apache-2.0)", + "optionalDependencies": { + "@types/trusted-types": "^2.0.7" + } + }, + "node_modules/es-toolkit": { + "version": "1.47.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.47.0.tgz", + "integrity": "sha512-n1GuoD0WEQZMBk5tttoZSqwgyLx01oqa5XsBmCHwPyNe1S9jPBEmtR2pSgp2kJuWE3ciFZ6yRHmY4pM4C3OOkw==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks" + ] + }, "node_modules/flexsearch": { "version": "0.8.212", "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.8.212.tgz", @@ -833,6 +1713,43 @@ ], "license": "Apache-2.0" }, + "node_modules/hachure-fill": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/hachure-fill/-/hachure-fill-0.5.2.tgz", + "integrity": "sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==", + "license": "MIT" + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/import-meta-resolve": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz", + "integrity": "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -845,6 +1762,89 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/katex": { + "version": "0.16.47", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.47.tgz", + "integrity": "sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg==", + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, + "node_modules/katex/node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/khroma": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/khroma/-/khroma-2.1.0.tgz", + "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==" + }, + "node_modules/layout-base": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-1.0.2.tgz", + "integrity": "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==", + "license": "MIT" + }, + "node_modules/lodash-es": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", + "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", + "license": "MIT" + }, + "node_modules/marked": { + "version": "16.4.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-16.4.2.tgz", + "integrity": "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/mermaid": { + "version": "11.15.0", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.15.0.tgz", + "integrity": "sha512-pTMbcf3rWdtLiYGpmoTjHEpeY8seiy6sR+9nD7LOs8KfUbHE4lOUAprTRqRAcWSQ6MQpdX+YEsxShtGsINtPtw==", + "license": "MIT", + "dependencies": { + "@braintree/sanitize-url": "^7.1.1", + "@iconify/utils": "^3.0.2", + "@mermaid-js/parser": "^1.1.1", + "@types/d3": "^7.4.3", + "@upsetjs/venn.js": "^2.0.0", + "cytoscape": "^3.33.1", + "cytoscape-cose-bilkent": "^4.1.0", + "cytoscape-fcose": "^2.2.0", + "d3": "^7.9.0", + "d3-sankey": "^0.12.3", + "dagre-d3-es": "7.0.14", + "dayjs": "^1.11.19", + "dompurify": "^3.3.1", + "es-toolkit": "^1.45.1", + "katex": "^0.16.25", + "khroma": "^2.1.0", + "marked": "^16.3.0", + "roughjs": "^4.6.6", + "stylis": "^4.3.6", + "ts-dedent": "^2.2.0", + "uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0" + } + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -916,12 +1916,40 @@ } } }, + "node_modules/package-manager-detector": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.6.0.tgz", + "integrity": "sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA==", + "license": "MIT" + }, + "node_modules/path-data-parser": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/path-data-parser/-/path-data-parser-0.1.0.tgz", + "integrity": "sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w==", + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", "license": "ISC" }, + "node_modules/points-on-curve": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", + "integrity": "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A==", + "license": "MIT" + }, + "node_modules/points-on-path": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/points-on-path/-/points-on-path-0.2.1.tgz", + "integrity": "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g==", + "license": "MIT", + "dependencies": { + "path-data-parser": "0.1.0", + "points-on-curve": "0.2.0" + } + }, "node_modules/postcss": { "version": "8.4.31", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", @@ -984,6 +2012,36 @@ "react": "^19.2.4" } }, + "node_modules/robust-predicates": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.3.tgz", + "integrity": "sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA==", + "license": "Unlicense" + }, + "node_modules/roughjs": { + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/roughjs/-/roughjs-4.6.6.tgz", + "integrity": "sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ==", + "license": "MIT", + "dependencies": { + "hachure-fill": "^0.5.2", + "path-data-parser": "^0.1.0", + "points-on-curve": "^0.2.0", + "points-on-path": "^0.2.1" + } + }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "license": "BSD-3-Clause" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, "node_modules/scheduler": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", @@ -1080,11 +2138,48 @@ } } }, + "node_modules/stylis": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.4.0.tgz", + "integrity": "sha512-5Z9ZpRzfuH6l/UAvCPAPUo3665Nk2wLaZU3x+TLHKVzIz33+sbJqbtrYoC3KD4/uVOr2Zp+L0LySezP9OHV9yA==", + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/ts-dedent": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.3.0.tgz", + "integrity": "sha512-JfJeIHke7y2egdGGgRAvpCwYFUsHlM2gPcrVOxFkznt/4uzQ7HFmvE63iFHVLBJNDuyDOQgijDK/tXH/f6Msjg==", + "license": "MIT", + "engines": { + "node": ">=6.10" + } + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" + }, + "node_modules/uuid": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.0.tgz", + "integrity": "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } } } } diff --git a/package.json b/package.json index da60b12..12b6e9a 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,12 @@ "private": true, "description": "Hypercerts Protocol Documentation", "scripts": { - "dev": "node lib/generate-search-index.js && node lib/generate-last-updated.js && node lib/generate-sitemap.js && node lib/generate-raw-pages.js && next dev --webpack", - "build": "node lib/generate-search-index.js && node lib/generate-last-updated.js && node lib/generate-sitemap.js && node lib/generate-raw-pages.js && next build --webpack", + "test": "node --test", + "generate:external-docs": "node lib/generate-external-docs-manifest.js", + "docs:fingerprint": "node lib/generate-docs-fingerprint.js", + "generate": "npm run generate:external-docs && node lib/generate-search-index.js && node lib/generate-last-updated.js && node lib/generate-sitemap.js && node lib/generate-raw-pages.js && npm run docs:fingerprint -- --content lib/external-docs-content.json", + "dev": "npm run generate && next dev --webpack", + "build": "npm run generate && next build --webpack", "start": "next start" }, "dependencies": { @@ -13,6 +17,8 @@ "@markdoc/next.js": "^0.5.0", "@vercel/analytics": "^1.6.1", "flexsearch": "^0.8.212", + "js-yaml": "^4.1.1", + "mermaid": "^11.15.0", "next": "^16.1.6", "prism-react-renderer": "^2.4.1", "react": "^19.2.4", diff --git a/pages/_app.js b/pages/_app.js index 3422f00..281d55f 100644 --- a/pages/_app.js +++ b/pages/_app.js @@ -11,6 +11,7 @@ import { Link } from '../components/Link'; import { DotPattern } from '../components/DotPattern'; import { HeroBanner } from '../components/HeroBanner'; import { CardGrid } from '../components/CardGrid'; +import { MermaidDiagram } from '../components/MermaidDiagram'; import { Analytics } from '@vercel/analytics/next'; const components = { @@ -26,6 +27,7 @@ const components = { DotPattern, HeroBanner, CardGrid, + MermaidDiagram, }; export default function App({ Component, pageProps }) { diff --git a/pages/architecture/epds.md b/pages/architecture/epds.md index a2cfb61..3d5e77c 100644 --- a/pages/architecture/epds.md +++ b/pages/architecture/epds.md @@ -1,335 +1,5 @@ --- title: ePDS (extended PDS) description: How the ePDS adds email/OTP login on top of AT Protocol without changing the standard OAuth flow for apps. +externalDoc: epds --- - -# ePDS (extended PDS) - -The ePDS adds email-based, passwordless sign-in on top of a standard AT Protocol PDS. Users enter their email, receive a one-time code, and end up with a normal AT Protocol session tied to a DID. - -Certified operates production, staging, and test ePDS instances. See [Certified services](/reference/certified-pdss) for the current hostnames and guidance on which to use in which scenario. - -For applications, the important part is that ePDS still finishes by issuing a standard AT Protocol authorization code. In practice, this means you can integrate it with [`@atproto/oauth-client-node`](https://github.com/bluesky-social/atproto/tree/main/packages/oauth/oauth-client-node). - -## System overview - -```text -Client App - -> starts AT Protocol OAuth against the PDS - -PDS Core - -> remains the OAuth issuer and token endpoint - -> advertises the Auth Service as the authorization endpoint - -Auth Service - -> collects the user's email or OTP - -> verifies the user - -> returns control to PDS Core via signed callback - -PDS Core - -> issues a normal authorization code - -Client App - -> exchanges the code for tokens -``` - -The PDS remains the OAuth issuer and token endpoint. The main difference is that the authorization step happens on the ePDS Auth Service, which handles the email and OTP flow before returning control to the PDS. - -## Integrating with `@atproto/oauth-client-node` - -ePDS works with the standard AT Protocol OAuth client libraries. The main ePDS-specific behavior is how you shape the authorization URL before redirecting the user. - -### Flow 1: your app collects the email - -In Flow 1, your app has its own email field. Start OAuth normally, then add `login_hint=` to the authorization URL before redirecting the user. - -```ts -import { NodeOAuthClient } from '@atproto/oauth-client-node' - -const oauthClient = new NodeOAuthClient({ - clientMetadata: { - client_id: 'https://yourapp.example.com/client-metadata.json', - client_name: 'Your App', - client_uri: 'https://yourapp.example.com', - redirect_uris: ['https://yourapp.example.com/api/oauth/callback'], - scope: 'atproto transition:generic', - grant_types: ['authorization_code', 'refresh_token'], - response_types: ['code'], - token_endpoint_auth_method: 'none', - dpop_bound_access_tokens: true, - }, - stateStore, - sessionStore, -}) - -const url = await oauthClient.authorize('alice.certified.one', { - scope: 'atproto transition:generic', -}) - -// ePDS-specific customization happens here. -const authUrl = new URL(url) -authUrl.searchParams.set('login_hint', email) -authUrl.searchParams.set('epds_handle_mode', 'picker-with-random') - -return authUrl.toString() -``` - -{% callout type="warning" %} -Do not put an email address into the PAR body as `login_hint`. For ePDS, add `login_hint` to the authorization URL instead. -{% /callout %} - -With `login_hint` set, the user lands directly on the OTP entry step instead of first seeing an email form on ePDS. - -### Flow 2: ePDS collects the email - -In Flow 2, your app just shows a "Sign in" button. Start OAuth normally and redirect the user to the authorization URL without `login_hint`. - -```ts -import { NodeOAuthClient } from '@atproto/oauth-client-node' - -const oauthClient = new NodeOAuthClient({ - clientMetadata: { - client_id: 'https://yourapp.example.com/client-metadata.json', - client_name: 'Your App', - client_uri: 'https://yourapp.example.com', - redirect_uris: ['https://yourapp.example.com/api/oauth/callback'], - scope: 'atproto transition:generic', - grant_types: ['authorization_code', 'refresh_token'], - response_types: ['code'], - token_endpoint_auth_method: 'none', - dpop_bound_access_tokens: true, - }, - stateStore, - sessionStore, -}) - -const url = await oauthClient.authorize('alice.certified.one', { - scope: 'atproto transition:generic', -}) - -const authUrl = new URL(url) -authUrl.searchParams.set('epds_handle_mode', 'picker') - -return authUrl.toString() -``` - -Without `login_hint`, ePDS renders its own email form and takes the user through the rest of the OTP flow. - -### Callback handling - -Callback handling stays standard. Once the user finishes on ePDS, your callback handler receives a normal authorization code and hands it back to `oauth-client-node`. - -```ts -const result = await oauthClient.callback(params) - -const session = result.session -const did = session.did -``` - -## Handle modes - -Handle mode controls what happens when a brand new user needs a handle during signup. - -| Mode | Behavior | -|------|----------| -| `picker-with-random` | Show the handle picker with a "Generate random" option. | -| `picker` | Show the handle picker without a random option. | -| `random` | Skip the picker and assign a random handle automatically. | - -Handle mode is resolved in this order: - -1. `epds_handle_mode` query param on the authorization URL -2. `epds_handle_mode` in client metadata -3. The ePDS instance default (`EPDS_DEFAULT_HANDLE_MODE`) - -This only affects new account creation. Existing users keep their current handle and skip this step. - -## Client metadata - -Your client metadata file is a public JSON document served over HTTPS. Its URL is also your `client_id`. - -### Bare-bones example - -```json -{ - "client_id": "https://yourapp.example.com/client-metadata.json", - "client_name": "Your App", - "client_uri": "https://yourapp.example.com", - "redirect_uris": ["https://yourapp.example.com/api/oauth/callback"], - "scope": "atproto transition:generic", - "grant_types": ["authorization_code", "refresh_token"], - "response_types": ["code"], - "token_endpoint_auth_method": "none", - "dpop_bound_access_tokens": true -} -``` - -### Full config example - -```json -{ - "client_id": "https://yourapp.example.com/client-metadata.json", - "client_name": "Your App", - "client_uri": "https://yourapp.example.com", - "logo_uri": "https://yourapp.example.com/logo.png", - "redirect_uris": ["https://yourapp.example.com/api/oauth/callback"], - "scope": "atproto transition:generic", - "grant_types": ["authorization_code", "refresh_token"], - "response_types": ["code"], - "token_endpoint_auth_method": "none", - "dpop_bound_access_tokens": true, - "brand_color": "#0f172a", - "background_color": "#ffffff", - "email_template_uri": "https://yourapp.example.com/email-template.html", - "email_subject_template": "{{code}} - Your {{app_name}} code", - "branding": { - "css": "body { background: #0f172a; color: #e2e8f0; }" - }, - "epds_handle_mode": "picker-with-random" -} -``` - -The extra branding fields customize the hosted login and email experience. `epds_handle_mode` sets your preferred handle mode for new users unless you override it on the authorization URL. - -## Branding and customization - -### How branding works - -ePDS reads branding settings from your app's `client-metadata.json`, using the OAuth `client_id` to look it up. Standard metadata fields like `logo_uri`, `brand_color`, `background_color`, `email_template_uri`, and `email_subject_template` customize the hosted login and email experience. - -Trusted clients can go further by adding custom CSS in client metadata under `branding.css`: - -```json -{ - "branding": { - "css": "body { background: #0f172a; color: #e2e8f0; }" - } -} -``` - -When the client is trusted, ePDS injects that CSS into its hosted auth pages and the stock consent page. - -{% callout type="warning" %} -Trust is checked against the exact `client_id`. - -The `client_id` you send during OAuth, the `client_id` inside `client-metadata.json`, and the entry in `PDS_OAUTH_TRUSTED_CLIENTS` must all be identical. - -For example, if your client metadata says `"client_id": "https://hypercerts-scaffold.vercel.app/client-metadata.json"`, then `PDS_OAUTH_TRUSTED_CLIENTS` must contain `https://hypercerts-scaffold.vercel.app/client-metadata.json` — not just `https://hypercerts-scaffold.vercel.app`. See the [Scaffold Starter App](/tools/scaffold) for a concrete example of a client serving metadata from `/client-metadata.json`. -{% /callout %} - -### Client metadata branding fields - -These fields are the main branding controls exposed through client metadata: - -| Field | What it affects | -|------|------------------| -| `logo_uri` | App logo shown in hosted auth and email flows | -| `brand_color` | Primary brand color used by hosted screens | -| `background_color` | Background color for hosted screens | -| `email_template_uri` | Custom HTML template for OTP emails | -| `email_subject_template` | Subject line template for OTP emails | -| `branding.css` | Custom CSS for trusted clients | - -### CSS injection for trusted clients - -Custom CSS is only applied for clients whose exact `client_id` appears in `PDS_OAUTH_TRUSTED_CLIENTS`. When present, ePDS injects a `` tag closure, and updates the page's CSP `style-src` directive with a SHA-256 hash for the injected stylesheet. - -This gives operators a safety boundary: untrusted clients never get CSS injection, even if their metadata contains branding CSS. - -### Where branding appears - -The send-OTP and initial-OTP screens are two states of the same auth-service route: `https://auth.epds1.test.certified.app/oauth/authorize`. - -| Surface | URL | Supports branding | -|---|---|---| -| Send OTP | `https://auth.epds1.test.certified.app/oauth/authorize` | Metadata fields + trusted-client CSS | -| Initial OTP | `https://auth.epds1.test.certified.app/oauth/authorize` | Metadata fields + trusted-client CSS | -| Choose handle | `https://auth.epds1.test.certified.app/auth/choose-handle` | Metadata fields + trusted-client CSS | -| Recovery | `https://auth.epds1.test.certified.app/auth/recover` | Metadata fields + trusted-client CSS | -| Consent page | `https://epds1.test.certified.app/oauth/authorize` | Trusted-client CSS | - -### Examples - -#### Send OTP - -{% columns %} -{% column %} -Stock - -![Stock send OTP screen](/images/epds/send-otp-stock.png) -{% /column %} -{% column %} -CSS injected - -![CSS-injected send OTP screen](/images/epds/send-otp-css-injected.png) -{% /column %} -{% /columns %} - -#### Initial OTP - -{% columns %} -{% column %} -Stock - -![Stock initial OTP screen](/images/epds/initial-otp-stock.png) -{% /column %} -{% column %} -CSS injected - -![CSS-injected initial OTP screen](/images/epds/initial-otp-css-injected.png) -{% /column %} -{% /columns %} - -#### Choose handle - -{% columns %} -{% column %} -Stock - -![Stock choose handle screen](/images/epds/choose-handle-stock.png) -{% /column %} -{% column %} -CSS injected - -![CSS-injected choose handle screen](/images/epds/choose-handle-css-injected.png) -{% /column %} -{% /columns %} - -#### Consent page - -{% columns %} -{% column %} -Stock - -![Stock consent page](/images/epds/consent-page-stock.png) -{% /column %} -{% column %} -CSS injected - -![CSS-injected consent page](/images/epds/consent-page-css-injected.png) -{% /column %} -{% /columns %} - -#### Recovery - -{% columns %} -{% column %} -Stock - -![Stock recovery screen](/images/epds/recovery-stock.png) -{% /column %} -{% column %} -CSS injected - -![CSS-injected recovery screen](/images/epds/recovery-css-injected.png) -{% /column %} -{% /columns %} - -## Further reading - -- [Account & Identity Setup](/architecture/account-and-identity) -- [Certified PDSs](/reference/certified-pdss) — the production, staging, and test ePDS instances Certified operates -- [Certified Group Service (CGS)](/architecture/certified-group-service) — a governance layer that sits in front of a PDS to support multi-identity, role-based repo management -- [Scaffold Starter App](/tools/scaffold) -- [ePDS repository](https://github.com/hypercerts-org/ePDS) -- Install the ePDS agent skill with `npx skills add hypercerts-org/ePDS --skill epds-login` diff --git a/pages/tools/hyperindex.md b/pages/tools/hyperindex.md index c2c7222..40a83e3 100644 --- a/pages/tools/hyperindex.md +++ b/pages/tools/hyperindex.md @@ -1,335 +1,5 @@ --- title: Hyperindex description: A Go ATProto indexer that indexes hypercert records and exposes them via GraphQL. +externalDoc: hyperindex-test --- - -# Hyperindex - -Hyperindex (`hi`) is a Go AT Protocol AppView server that indexes records and exposes them via GraphQL. Use it to: - -- Index all hypercert-related records from the ATProto network in real time -- Query indexed data through a typed GraphQL API -- Backfill historical records from any user or the entire network -- Run your own indexer for full control over data availability and query performance - -Built in Go. Hyperindex was originally built by our friends at [GainForest](https://gainforest.earth); the Certified indexer is forked from [github.com/gainforest/hyperindex](https://github.com/gainforest/hyperindex). Seeing the GainForest GitHub organization or GainForest-hosted Hyperindex references in related tooling is expected. Tap reference implementation: [github.com/bluesky-social](https://github.com/bluesky-social/indigo/tree/main/cmd/tap). - -Hosted production and staging endpoints: [Certified Services](/reference/certified-services#indexers). - -## Why indexers & discovery - -AT Protocol is federated, so hypercert records are distributed across many PDSs instead of living in a single database. If an app wants to discover records across users and organizations, it needs a way to aggregate that network data into one queryable view. - -Indexers handle that job. They consume network events, fetch and parse records by lexicon, normalize them into query-ready storage, and expose APIs for search, filtering, and aggregation. - -Hyperindex is the reference indexer used in this ecosystem. - -If you want to inspect indexers running across the broader ecosystem, use [Hyperscan](/tools/hyperscan). - -## How it works - -Hyperindex is **Tap-first** (recommended). Tap handles ingestion, and Hyperindex consumes Tap events, stores records, and exposes them via GraphQL. - -```text -ATProto Relay ──→ Tap ──→ Hyperindex Consumer ──→ Records DB ──→ GraphQL API - │ - Activity Log ──→ Admin Dashboard -``` - -Jetstream mode still exists as a legacy/non-Tap mode, but Tap is the preferred setup. - -## Query via GraphQL - -Access your indexed data at `/graphql`. For the Certified-hosted production indexer, use: - -- GraphQL API: [`https://api.indexer.hypercerts.dev/graphql`](https://api.indexer.hypercerts.dev/graphql) -- GraphiQL explorer: [`https://api.indexer.hypercerts.dev/graphiql`](https://api.indexer.hypercerts.dev/graphiql) -- WebSocket subscriptions: `wss://api.indexer.hypercerts.dev/graphql/ws` - -The GraphQL API supports standard introspection on the same `/graphql` endpoint. GraphQL clients and code generators can point at that URL directly, for example: - -```yaml -schema: https://api.indexer.hypercerts.dev/graphql -``` - -If a tool expects a local schema file, export one with: - -```bash -npx -y get-graphql-schema https://api.indexer.hypercerts.dev/graphql > schema.graphql -npx -y get-graphql-schema https://api.indexer.hypercerts.dev/graphql --json > schema.json -``` - -If Node prints a `punycode` deprecation warning, it comes from the CLI's dependencies; the schema file is still written. - -```graphql -# Query records by collection -query { - records(collection: "org.hypercerts.claim.activity") { - edges { - node { - uri - did - value - } - } - } -} - -# With typed queries (when lexicon schemas are loaded) -query { - orgHypercertsClaimActivity(first: 10) { - edges { - node { - uri - workScope - startDate - createdAt - } - } - } -} - -# With typed filter queries (title contains "Hypercerts") -query { - orgHypercertsClaimActivity( - first: 10 - where: { title: { contains: "Hypercerts" } } - ) { - edges { - node { - uri - title - createdAt - } - } - } -} -``` - -## Quick start - -For local development with default settings: - -```bash -git clone https://github.com/gainforest/hyperindex.git -cd hyperindex -cp .env.example .env -go run ./cmd/hyperindex -``` - -Open [http://localhost:8080/graphiql/admin](http://localhost:8080/graphiql/admin) to access the admin interface. - -## Register lexicons - -Lexicons define the AT Protocol record types you want to index. You can register them via: - -1. Admin GraphQL API at `/graphiql/admin` -2. Client admin UI at `https:///lexicons` (you must log in with an admin DID) - -```graphql -mutation { - uploadLexicons(files: [...]) # Upload lexicon JSON files -} -``` - -Or place lexicon JSON files in a directory and set the `LEXICON_DIR` environment variable. - -For hypercerts, you would register the `org.hypercerts.claim.*` lexicons — see [Introduction to Lexicons](/lexicons/introduction-to-lexicons) for the full list. - -## Endpoints - -| Endpoint | Description | -|---|---| -| `/graphql` | Public GraphQL API. Supports standard GraphQL introspection for schema and codegen tools. | -| `/graphql/ws` | GraphQL subscriptions (WebSocket) | -| `/admin/graphql` | Admin GraphQL API | -| `/graphiql` | Browser-based GraphiQL explorer for the public API | -| `/graphiql/admin` | GraphQL playground (admin API) | -| `/health` | Health check | -| `/stats` | Server statistics | - -## Deployment configuration - -Use this section when deploying Hyperindex (backend, Tap, and client). It lists the environment variables you should set first for a reliable initial deployment, followed by optional variables for advanced tuning. - -### Baseline deployment variables - -These are the variables you should set first to get a stable deployment running. - -Note: some managed platforms (including Railway) may auto-provision a subset of variables. - -### Hyperindex backend - -| Variable | Example | What it is for | -|---|---|---| -| `HOST` | `0.0.0.0` | Makes the app reachable in container runtime | -| `PORT` | `8080` | App port | -| `DATABASE_URL` | `sqlite:/data/hypergoat.db` | Main indexed-records database | -| `EXTERNAL_BASE_URL` | `https://hyperindex.example.com` | Public backend URL used by frontend/admin flows and GraphiQL links | -| `SECRET_KEY_BASE` | `` | Session/signing secret | -| `ADMIN_DIDS` | `did:plc:...` | DIDs with admin privileges | -| `ADMIN_API_KEY` | `` | Required at startup. Shared secret for trusted admin proxy requests; must exactly match client `HYPERINDEX_ADMIN_API_KEY` | -| `TAP_ENABLED` | `true` | Enables Tap mode | -| `TAP_URL` | `ws://tap.railway.internal:2480` | Tap websocket endpoint | -| `TAP_ADMIN_PASSWORD` | `` | Tap admin auth secret | - -### Tap service - -| Variable | Example | What it is for | -|---|---|---| -| `TAP_DATABASE_URL` | `sqlite:///data/tap.db` | Persists Tap cursor/state (self-managed; Railway autoconfigures) | -| `TAP_ADMIN_PASSWORD` | `` | Protects Tap admin routes | -| `TAP_COLLECTION_FILTERS` | `app.certified.*,org.hypercerts.*` | Filters ingested record collections | -| `TAP_SIGNAL_COLLECTION` | `app.certified.actor.profile` | Signal collection for repo discovery | - -### Client (Next.js) - -| Variable | Example | What it is for | -|---|---|---| -| `NEXT_PUBLIC_HYPERINDEX_URL` | `https://hyperindex.example.com` | Browser-side URL of your Hyperindex backend | -| `HYPERINDEX_URL` | `https://hyperindex.example.com` | Server-side URL of your Hyperindex backend (used by Next API proxy routes). If unset, it falls back to `NEXT_PUBLIC_HYPERINDEX_URL` | -| `HYPERINDEX_ADMIN_API_KEY` | `` | Server-side only. Used by the Next.js admin proxy; must exactly match backend `ADMIN_API_KEY` | -| `NEXT_PUBLIC_CLIENT_URL` | `https://hyperindex-frontend.example.com` | Client frontend URL used for OAuth client metadata and auth redirects | -| `COOKIE_SECRET` | `` | Session encryption | -| `ATPROTO_JWK_PRIVATE` | `` | Confidential OAuth signing key | - -Hyperindex normalizes both `NEXT_PUBLIC_HYPERINDEX_URL` and `HYPERINDEX_URL`: it trims surrounding whitespace, removes trailing slashes, and prepends `https://` when no scheme is provided. - -### Admin API key pairing - -When deploying the backend with the Next.js client, set the same random secret in both services: - -```bash -# Hyperindex backend -ADMIN_API_KEY= - -# Next.js client -HYPERINDEX_ADMIN_API_KEY= -``` - -The backend requires `ADMIN_API_KEY` at startup. It must be at least 16 characters and must not include leading or trailing whitespace. The client keeps `HYPERINDEX_ADMIN_API_KEY` server-side only and uses it when proxying admin GraphQL requests to `/admin/graphql`. - -This key does **not** grant admin rights by itself. The signed-in user's DID must still be listed in backend `ADMIN_DIDS`; the matching key only lets the backend trust the client's `X-User-DID` header. - -### Optional variables - -Set these only when needed. - -### Backend -- `ALLOWED_ORIGINS` -- `TAP_DISABLE_ACKS` - -> `TAP_DISABLE_ACKS` is configured on the **backend/indexer** service. -> In some deployments, ACK mode (`TAP_DISABLE_ACKS=false`) can cause repeated websocket disconnect loops (for example: `connection reset by peer`, `close 1006`, frequent reconnect backoff). -> If you see that pattern, set `TAP_DISABLE_ACKS=true` on the **Hyperindex backend** to stabilize ingestion first, then investigate Tap resource/config compatibility before re-enabling ACK mode. - -### Tap -- `TAP_FULL_NETWORK` -- `TAP_FIREHOSE_PARALLELISM` -- `TAP_RESYNC_PARALLELISM` -- `TAP_OUTBOX_PARALLELISM` -- `TAP_MAX_DB_CONNS` -- `TAP_OUTBOX_CAPACITY` -- `TAP_NO_REPLAY` -- `TAP_REPO_FETCH_TIMEOUT` - -> `TAP_FULL_NETWORK=true` enables full-network tracking and triggers a broad historical backfill across discoverable repos. -> This can materially increase ingestion load, network requests, and storage use. - -### Client -- Additional auth/provider-specific settings depending on deployment model - -## Common pitfalls - -- **Wrong variable on wrong service** - - `TAP_COLLECTION_FILTERS`, `TAP_SIGNAL_COLLECTION`, `TAP_FULL_NETWORK` belong to **Tap** - - `TAP_DISABLE_ACKS`, `TAP_ENABLED`, `TAP_URL` belong to **Hyperindex backend** - -- **Client works but admin requests fail** - - `HYPERINDEX_URL` is missing on the client deployment - - `NEXT_PUBLIC_HYPERINDEX_URL` alone is not enough for server-side proxy routes - - `HYPERINDEX_ADMIN_API_KEY` is missing on the client deployment - - `HYPERINDEX_ADMIN_API_KEY` does not exactly match backend `ADMIN_API_KEY` - - `EXTERNAL_BASE_URL` does not match the backend's public URL - -- **`admin privileges required` while logged in** - - Logged-in DID is not present in backend `ADMIN_DIDS` - - The client and backend admin API keys do not match, so the backend ignores the proxied `X-User-DID` header - - You rotated `ADMIN_API_KEY` or `HYPERINDEX_ADMIN_API_KEY` on only one service - -- **Trailing slash URL issues** - - `NEXT_PUBLIC_CLIENT_URL` must **not** include a trailing slash. - - Use: - - `https://hyperindex-frontend.example.com` ✅ - - `https://hyperindex-frontend.example.com/` ❌ - - A trailing slash can cause OAuth client metadata lookup errors (for example: `client_metadata not found`). - -- **Healthcheck confusion** - - Backend healthcheck should be `/health` - - Frontend usually uses `/` unless you explicitly add a `/health` route - -## Deploy on Railway - -### 1) Deploy Hyperindex backend - -1. Create a Railway service from the repository -2. Attach a persistent volume mounted to `/data` -3. Set healthcheck path to `/health` -4. Add backend variables from the baseline list above -5. Deploy - -Use `PORT=8080` for the Hyperindex service. For public URL variables (for example `EXTERNAL_BASE_URL`, `NEXT_PUBLIC_HYPERINDEX_URL`, `HYPERINDEX_URL`), use the Railway-generated HTTPS domain (typically without appending `:8080`). - -### 2) Deploy Tap - -1. Create a Railway service from image: - `ghcr.io/bluesky-social/indigo/tap:latest` (or a pinned tag) -2. Attach a persistent volume mounted to `/data` -3. Add Tap variables from the baseline list above, but **do not manually set `TAP_DATABASE_URL` on Railway** (Railway autoconfigures this for the service) -4. Deploy - -See the official ATProto Tap Railway guide: [RAILWAY_DEPLOY.md](https://github.com/bluesky-social/indigo/blob/main/cmd/tap/RAILWAY_DEPLOY.md). - -### 3) Connect backend to Tap - -Set on backend service: - -- `TAP_ENABLED=true` -- `TAP_URL=ws://:2480` -- `TAP_ADMIN_PASSWORD=` - -Redeploy backend after updating these values. - -### 4) Deploy client (Next.js) - -Deploy client on Railway/Vercel/etc and set: - -- `NEXT_PUBLIC_HYPERINDEX_URL=` -- `HYPERINDEX_URL=` -- auth/session vars (`NEXT_PUBLIC_CLIENT_URL`, `COOKIE_SECRET`, `ATPROTO_JWK_PRIVATE`) as needed - -### Railway-specific DB path notes - -For mounted volumes at `/data`: - -**Railway** - -- Backend SQLite: `DATABASE_URL=sqlite:/data/hypergoat.db` -- Tap SQLite: do not manually set `TAP_DATABASE_URL` (it is autoconfigured) - -**Non-Railway/self-managed** - -- Backend SQLite: `DATABASE_URL=sqlite:/data/hypergoat.db` -- Tap SQLite: `TAP_DATABASE_URL=sqlite:///data/tap.db` - -## Running with Docker - -```bash -docker compose up --build -``` - -## Learn more - -- [GitHub repository](https://github.com/gainforest/hyperindex) — upstream GainForest repository that the Certified indexer is forked from -- [Certified Services](/reference/certified-services#indexers) — current public indexer endpoints -- [Building on Hypercerts](/getting-started/building-on-hypercerts) — integration patterns for platforms and tools diff --git a/styles/globals.css b/styles/globals.css index b021981..0485154 100644 --- a/styles/globals.css +++ b/styles/globals.css @@ -1081,6 +1081,11 @@ a.sidebar-link-active:hover { padding-left: 24px; } +.toc-link-h4 { + font-size: 12px; + padding-left: 40px; +} + /* ===== Last Updated ===== */ .page-tools { display: flex; @@ -1147,6 +1152,7 @@ a.sidebar-link-active:hover { font-style: italic; color: var(--color-text-secondary); } + /* ===== Pagination ===== */ .pagination { display: flex; @@ -1427,6 +1433,36 @@ a.sidebar-link-active:hover { color: #d6deeb; } +.mermaid-diagram { + margin: var(--space-6) 0; + padding: var(--space-4); + border: 1px solid var(--color-border); + border-radius: var(--radius-lg); + background: var(--color-bg-subtle); + overflow-x: auto; +} + +.mermaid-diagram svg { + display: block; + max-width: 100%; + height: auto; + margin: 0 auto; +} + +.mermaid-diagram--loading { + color: var(--color-text-secondary); + font-size: 0.875rem; +} + +.mermaid-diagram--error { + border-color: var(--color-warning); + background: var(--color-warning-bg); +} + +.mermaid-diagram--error p { + margin-bottom: var(--space-4); +} + .layout-content code { font-family: var(--font-mono); font-size: 0.875em; diff --git a/test/external-doc-links.test.js b/test/external-doc-links.test.js new file mode 100644 index 0000000..6a59bc1 --- /dev/null +++ b/test/external-doc-links.test.js @@ -0,0 +1,37 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { + resolveExternalDocHref, + resolveExternalDocImageSrc, +} = require('../lib/external-doc-links'); + +const source = { + repo: 'hypercerts-org/ePDS', + ref: 'main', + path: 'docs/tutorial.md', +}; + +test('leaves anchors, site paths, and absolute URLs unchanged', () => { + for (const href of ['#section', '/reference/faq', 'https://example.com/docs', 'mailto:docs@example.com']) { + assert.equal(resolveExternalDocHref(href, source), href); + } +}); + +test('resolves relative files and directories to GitHub', () => { + assert.equal( + resolveExternalDocHref('./other.md?plain=1#section', source), + 'https://github.com/hypercerts-org/ePDS/blob/main/docs/other.md?plain=1#section', + ); + assert.equal( + resolveExternalDocHref('../packages/demo', source), + 'https://github.com/hypercerts-org/ePDS/tree/main/packages/demo', + ); +}); + +test('resolves relative images to raw GitHub content', () => { + assert.equal( + resolveExternalDocImageSrc('./images/architecture.png', source), + 'https://raw.githubusercontent.com/hypercerts-org/ePDS/main/docs/images/architecture.png', + ); + assert.equal(resolveExternalDocImageSrc('/images/local.png', source), '/images/local.png'); +}); diff --git a/test/external-docs-snapshot.test.js b/test/external-docs-snapshot.test.js new file mode 100644 index 0000000..d38fe3a --- /dev/null +++ b/test/external-docs-snapshot.test.js @@ -0,0 +1,119 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { + buildFingerprintDocument, + collectSourceSnapshot, + sha256, +} = require('../lib/external-docs-snapshot'); + +function response({ ok = true, status = 200, statusText = 'OK', text = '', json }) { + return { + ok, + status, + statusText, + text: async () => text, + json: async () => json, + }; +} + +const source = { + id: 'epds', + title: 'ePDS', + repo: 'hypercerts-org/ePDS', + ref: 'main', + path: 'docs/tutorial.md', +}; + +test('build fingerprint uses the exact Markdown snapshot fetched for rendering', async (context) => { + const originalFetch = global.fetch; + context.after(() => { global.fetch = originalFetch; }); + let upstreamMarkdown = '# Version A'; + let contentFetches = 0; + + global.fetch = async (url) => { + if (String(url).includes('/commits?')) { + return response({ json: [{ commit: { committer: { date: '2026-07-13T00:00:00Z' } } }] }); + } + contentFetches += 1; + return response({ text: upstreamMarkdown }); + }; + + const snapshot = await collectSourceSnapshot(source); + upstreamMarkdown = '# Version B'; + const fingerprint = buildFingerprintDocument([snapshot]); + + assert.equal(contentFetches, 1); + assert.equal(snapshot.markdown, '# Version A'); + assert.equal(snapshot.contentHash, sha256('# Version A')); + assert.notEqual(snapshot.contentHash, sha256(upstreamMarkdown)); + assert.equal(snapshot.updatedAt, '2026-07-13T00:00:00Z'); + assert.equal(fingerprint.sources.epds.contentHash, sha256('# Version A')); +}); + +test('fails when the registered Markdown file cannot be fetched', async (context) => { + const originalFetch = global.fetch; + context.after(() => { global.fetch = originalFetch; }); + + global.fetch = async (url) => { + if (String(url).includes('/commits?')) { + return response({ json: [] }); + } + return response({ ok: false, status: 404, statusText: 'Not Found', text: 'missing' }); + }; + + await assert.rejects( + () => collectSourceSnapshot(source), + /returned 404 Not Found.*Check repo, ref, path, and DOCS_SOURCE_TOKEN/, + ); +}); + +test('fails when the registered Markdown file is empty', async (context) => { + const originalFetch = global.fetch; + context.after(() => { global.fetch = originalFetch; }); + + global.fetch = async (url) => { + if (String(url).includes('/commits?')) { + return response({ json: [] }); + } + return response({ text: ' \n' }); + }; + + await assert.rejects(() => collectSourceSnapshot(source), /External doc "epds".*is empty/); +}); + +test('informational timestamps do not affect combined fingerprints', () => { + const base = { + ...source, + size: 6, + contentHash: sha256('# Docs'), + markdown: '# Docs', + }; + + const first = buildFingerprintDocument([{ ...base, updatedAt: '2026-01-01T00:00:00Z' }]); + const second = buildFingerprintDocument([{ ...base, updatedAt: '2026-07-01T00:00:00Z' }]); + assert.equal(first.combinedFingerprint, second.combinedFingerprint); +}); + +test('registry ordering does not affect the combined fingerprint', () => { + const first = { + ...source, + size: 3, + contentHash: sha256('# A'), + markdown: '# A', + }; + const second = { + id: 'hyperindex', + title: 'Hyperindex', + repo: 'gainforest/hyperindex', + ref: 'main', + path: 'docs/hyperindex.md', + size: 3, + contentHash: sha256('# B'), + markdown: '# B', + }; + + assert.equal( + buildFingerprintDocument([first, second]).combinedFingerprint, + buildFingerprintDocument([second, first]).combinedFingerprint, + ); +}); diff --git a/test/external-docs.test.js b/test/external-docs.test.js new file mode 100644 index 0000000..3de898c --- /dev/null +++ b/test/external-docs.test.js @@ -0,0 +1,245 @@ +const assert = require('node:assert/strict'); +const { mkdtempSync, rmSync, writeFileSync } = require('node:fs'); +const { tmpdir } = require('node:os'); +const { join } = require('node:path'); +const test = require('node:test'); +const { + DEFAULT_ALLOWED_SOURCE_ORGS, + getAllowedSourceOrgs, + loadExternalDocSources, + loadExternalDocsContent, + parseMarkdownFrontmatter, + resolveExternalDocSnapshot, + resolvePageDocument, +} = require('../lib/external-docs'); +const { compileExternalDocPage } = require('../lib/external-docs-loader'); + +function withTempFile(name, contents, run) { + const directory = mkdtempSync(join(tmpdir(), 'external-docs-test-')); + const path = join(directory, name); + writeFileSync(path, contents); + + try { + return run(path); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +function loadRegistry(contents, env = {}) { + return withTempFile('docs-sources.yml', contents, (path) => loadExternalDocSources(path, env)); +} + +const registry = `sources: + - id: epds + title: ePDS + repo: hypercerts-org/ePDS + ref: main + path: docs/tutorial.md +`; + +const snapshot = { + id: 'epds', + title: 'ePDS', + repo: 'hypercerts-org/ePDS', + ref: 'main', + path: 'docs/tutorial.md', + markdown: '# Canonical ePDS docs', +}; +const content = { sources: { epds: snapshot } }; + +test('uses default source organizations when the environment override is absent or blank', () => { + assert.deepEqual(getAllowedSourceOrgs({}), DEFAULT_ALLOWED_SOURCE_ORGS); + assert.deepEqual(getAllowedSourceOrgs({ DOCS_ALLOWED_SOURCE_ORGS: ' ' }), DEFAULT_ALLOWED_SOURCE_ORGS); +}); + +test('normalizes and deduplicates configured source organizations', () => { + assert.deepEqual( + getAllowedSourceOrgs({ DOCS_ALLOWED_SOURCE_ORGS: ' Example-Org,HYPERCERTS-ORG,example-org ' }), + ['example-org', 'hypercerts-org'], + ); + assert.throws( + () => getAllowedSourceOrgs({ DOCS_ALLOWED_SOURCE_ORGS: 'valid owner,other' }), + /comma-separated list/, + ); +}); + +test('loads the single-file source schema', () => { + assert.deepEqual(loadRegistry(registry), [{ + id: 'epds', + title: 'ePDS', + repo: 'hypercerts-org/ePDS', + ref: 'main', + path: 'docs/tutorial.md', + }]); +}); + +for (const extension of ['md', 'mdoc', 'mdx']) { + test(`accepts .${extension} source files`, () => { + const [source] = loadRegistry(`sources: + - id: docs + title: Docs + repo: hypercerts-org/docs + ref: main + path: guide.${extension} +`); + assert.equal(source.path, `guide.${extension}`); + }); +} + +test('rejects legacy URL and directory source fields', () => { + for (const field of ['rawUrl', 'sourceUrl', 'docsPath', 'entrypoint', 'fingerprintMode']) { + assert.throws(() => loadRegistry(`sources: + - id: docs + title: Docs + repo: hypercerts-org/docs + ref: main + path: guide.md + ${field}: legacy +`), new RegExp(`unsupported field.*${field}`)); + } +}); + +test('validates registry shape, source identity, repository, ref, path, and duplicates', () => { + const invalidRegistries = [ + '{}', + 'sources: {}', + 'sources:\n - id: Bad_ID\n title: Bad\n repo: hypercerts-org/docs\n ref: main\n path: guide.md', + 'sources:\n - id: docs\n repo: hypercerts-org/docs\n ref: main\n path: guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: invalid\n ref: main\n path: guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: unapproved/docs\n ref: main\n path: guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: ""\n path: guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: main\n path: guide.txt', + 'sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: main\n path: ../guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: main\n path: /guide.md', + 'sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: main\n path: docs//guide.md', + "sources:\n - id: docs\n title: Docs\n repo: hypercerts-org/docs\n ref: main\n path: 'docs\\\\guide.md'", + 'sources:\n - id: docs\n title: One\n repo: hypercerts-org/docs\n ref: main\n path: one.md\n - id: docs\n title: Two\n repo: hypercerts-org/docs\n ref: main\n path: two.md', + ]; + + for (const value of invalidRegistries) { + assert.throws(() => loadRegistry(value), /Invalid docs-sources.yml/); + } +}); + +test('parses absent, LF, and CRLF frontmatter', () => { + assert.deepEqual(parseMarkdownFrontmatter('# Heading'), {}); + assert.deepEqual( + parseMarkdownFrontmatter('---\ntitle: Example\nexternalDoc: epds\n---\n'), + { title: 'Example', externalDoc: 'epds' }, + ); + assert.deepEqual( + parseMarkdownFrontmatter('---\r\ntitle: Example\r\nexternalDoc: epds\r\n---\r\n'), + { title: 'Example', externalDoc: 'epds' }, + ); +}); + +test('includes the file label in malformed frontmatter errors', () => { + assert.throws( + () => parseMarkdownFrontmatter('---\ntitle: [broken\n---\n', 'pages/broken.md'), + /Invalid frontmatter in pages\/broken.md/, + ); +}); + +test('loads generated external content and reports missing or malformed files', () => { + withTempFile('content.json', JSON.stringify(content), (path) => { + assert.equal(loadExternalDocsContent(path).sources.epds.markdown, '# Canonical ePDS docs'); + }); + for (const malformed of ['{broken', '{}', '{"sources":[]}']) { + withTempFile('content.json', malformed, (path) => { + assert.throws(() => loadExternalDocsContent(path), /Run npm run generate:external-docs first/); + }); + } + assert.throws( + () => loadExternalDocsContent(join(tmpdir(), 'missing-external-docs-content.json')), + /Run npm run generate:external-docs first/, + ); +}); + +test('resolves externalDoc through the generated build snapshot', () => { + assert.equal(resolveExternalDocSnapshot('epds', content), snapshot); + assert.throws(() => resolveExternalDocSnapshot('missing', content), /Unknown externalDoc "missing"/); + assert.throws(() => resolveExternalDocSnapshot(42, content), /externalDoc must be a lowercase registry id/); + assert.throws( + () => resolveExternalDocSnapshot('epds', { sources: { epds: { markdown: ' ' } } }), + /has no generated Markdown/, + ); +}); + +test('uses external Markdown for a frontmatter-only external page', () => { + const localMarkdown = '---\ntitle: ePDS\nexternalDoc: epds\n---\n'; + assert.deepEqual( + resolvePageDocument({ title: 'ePDS', externalDoc: 'epds' }, localMarkdown, content, 'pages/epds.md'), + { markdown: '# Canonical ePDS docs', externalDoc: snapshot }, + ); +}); + +test('compiles external Markdown with local frontmatter and generated source metadata', () => { + const localMarkdown = '---\ntitle: Local title\nexternalDoc: epds\n---\n'; + const externalContent = { + sources: { + epds: { + ...snapshot, + markdown: '---\ntitle: Upstream title\n---\n# Canonical body', + }, + }, + }; + const compiled = compileExternalDocPage( + { title: 'Local title', externalDoc: 'epds' }, + localMarkdown, + externalContent, + 'pages/epds.md', + ); + + assert.deepEqual(parseMarkdownFrontmatter(compiled), { + title: 'Local title', + externalDoc: 'epds', + __externalDocSource: { + id: 'epds', + repo: 'hypercerts-org/ePDS', + ref: 'main', + path: 'docs/tutorial.md', + }, + }); + assert.match(compiled, /\n# Canonical body$/); + assert.doesNotMatch(compiled, /Upstream title/); +}); + +test('compiles CRLF external pages and strips CRLF snapshot frontmatter', () => { + const localMarkdown = '---\r\ntitle: Local title\r\nexternalDoc: epds\r\n---\r\n'; + const externalContent = { + sources: { + epds: { + ...snapshot, + markdown: '---\r\ntitle: Upstream title\r\n---\r\n# Canonical body', + }, + }, + }; + const frontmatter = parseMarkdownFrontmatter(localMarkdown); + const compiled = compileExternalDocPage( + frontmatter, + localMarkdown, + externalContent, + 'pages/epds.md', + ); + + assert.equal(frontmatter.externalDoc, 'epds'); + assert.match(compiled, /\n# Canonical body$/); + assert.doesNotMatch(compiled, /Upstream title/); +}); + +test('leaves ordinary local pages unchanged', () => { + const localMarkdown = '---\ntitle: Local\n---\n# Local page'; + assert.deepEqual( + resolvePageDocument({ title: 'Local' }, localMarkdown, content), + { markdown: localMarkdown, externalDoc: null }, + ); +}); + +test('rejects stale local fallback content on external pages', () => { + const localMarkdown = '---\ntitle: ePDS\nexternalDoc: epds\n---\n# Stale fallback'; + assert.throws( + () => resolvePageDocument({ title: 'ePDS', externalDoc: 'epds' }, localMarkdown, content, 'pages/epds.md'), + /pages\/epds.md sets externalDoc and must not contain a local Markdown body/, + ); +});