From 64c4914e5555596f2143ad29d84a3f0eaa8dcfe9 Mon Sep 17 00:00:00 2001 From: Julia Date: Sun, 22 Nov 2026 01:35:00 +0100 Subject: [PATCH 1/4] feat(docs): add public-manifest.json and publish lint for DevIndex guides (#161) --- .github/workflows/ci.yml | 3 + buildScripts/lintPublicDocs.mjs | 275 ++++++++++++++++++ docs/public-manifest.json | 38 +++ package.json | 1 + .../unit/buildScripts/LintPublicDocs.spec.mjs | 129 ++++++++ 5 files changed, 446 insertions(+) create mode 100644 buildScripts/lintPublicDocs.mjs create mode 100644 docs/public-manifest.json create mode 100644 test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bfd239..bdc6043 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,9 @@ jobs: - name: Assert no derived data is tracked run: npm run check-data-tracking + - name: Lint public documentation manifest + run: npm run lint-public-docs + # `npm ci` above ran the postinstall linker, so this asserts the projection actually happened # rather than re-doing it: both skill surfaces point at the installed package, and no skill # byte is tracked here. An `--ignore-scripts` install resolves the dependency without linking diff --git a/buildScripts/lintPublicDocs.mjs b/buildScripts/lintPublicDocs.mjs new file mode 100644 index 0000000..85c0696 --- /dev/null +++ b/buildScripts/lintPublicDocs.mjs @@ -0,0 +1,275 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import {fileURLToPath} from 'node:url'; + +/** + * @module buildScripts.lintPublicDocs + * @summary Validates `docs/public-manifest.json` against `learn/tree.json` and enforces public-publish hygiene across + * every manifest-admitted Markdown guide (`neomjs/neo` Discussion #20920 `A0 + A1`, `neomjs/devindex#161`). + * + * Default-deny spoke federation requires two guarantees: + * 1. **Structural parity:** Every `learn/tree.json` leaf is explicitly listed in `docs/public-manifest.json` at its + * canonical `/learn/guides/devindex/` route, and every manifest entry resolves to a real tracked `.md` file + * inside `learn/` with no path traversal or duplicate routes. + * 2. **Content hygiene:** Admitted product guides contain zero real host filesystem paths (`/Users//`, + * `/home//`, `C:\Users\\`), zero private-repository identifiers (`neomjs/business`, + * `neomjs/neo-stage`, `neomjs/neo-pro`), and zero broken relative `.md` links outside fenced code blocks, while + * permitting public `@neo-*` maintainer handles used in illustrative examples. + */ + +const + __filename = fileURLToPath(import.meta.url), + REPO_ROOT = path.resolve(path.dirname(__filename), '..'), + DEFAULT_PREFIX = '/learn/guides/devindex/', + EXPECTED_SCHEMA = 'public-manifest.v1', + EXPECTED_SPOKE = 'neomjs/devindex', + // Root-anchored Unix/macOS & Windows user home paths; ignores repo-relative segments like `view/home/MainContainer.mjs` + // and generic placeholders (``, ``, ``, `you`, `user`, `username`). + UNIX_HOST_PATH_RE = /(?:^|[\s`'"(=])(\/(?:Users|home)\/(?![<{]|(?:you|user|username)(?:\/|\b))[A-Za-z0-9._-]+\/[^\s`'")>\]]*)/gm, + WIN_HOST_PATH_RE = /(?:^|[\s`'"(=])([A-Za-z]:\\Users\\(?![<{]|(?:you|user|username)(?:\\|\b))[A-Za-z0-9._-]+\\[^\s`'")>\]]*)/gm, + PRIVATE_REPO_RE = /\bneomjs\/(?:business|neo-stage|neo-pro)\b/gi, + MD_LINK_RE = /\[[^\]]*\]\(([^)#?\s]+\.md)(?:[?#][^)]*)?\)/g, + FENCE_RE = /^ {0,3}(`{3,}|~{3,})/; + +/** + * @summary Strips fenced Markdown code blocks while preserving line numbers for diagnostic reporting. + * @param {String} markdown + * @returns {String} + */ +export function stripFencedCodeBlocks(markdown) { + const + lines = String(markdown).split('\n'), + out = []; + + let fence = null; + + for (const rawLine of lines) { + const + line = rawLine.replace(/\r$/, ''), + match = FENCE_RE.exec(line); + + if (fence) { + if (match && match[1][0] === fence[0] && match[1].length >= fence.length) { + fence = null + } + out.push(''); + continue + } + + if (match) { + fence = match[1]; + out.push(''); + continue + } + + out.push(line) + } + + return out.join('\n') +} + +/** + * @summary Computes the 1-based line number for a character offset in a string. + * @param {String} text + * @param {Number} index + * @returns {Number} + */ +function lineAt(text, index) { + let line = 1; + for (let i = 0; i < index && i < text.length; i++) { + if (text[i] === '\n') line++ + } + return line +} + +/** + * @summary Validates `docs/public-manifest.json`, `learn/tree.json`, and all admitted Markdown files under `root`. + * @param {Object} [options={}] + * @param {String} [options.root=REPO_ROOT] Repository root directory. + * @param {String} [options.manifestRelPath='docs/public-manifest.json'] Manifest path relative to `root`. + * @param {String} [options.routePrefix=DEFAULT_PREFIX] Required canonical URL prefix for routes in this spoke. + * @param {String} [options.expectedSpoke=EXPECTED_SPOKE] Expected `spoke` identifier in the manifest. + * @returns {{entryCount: Number, routes: String[]}} + */ +export function lintPublicDocs({ + root = REPO_ROOT, + manifestRelPath = 'docs/public-manifest.json', + routePrefix = DEFAULT_PREFIX, + expectedSpoke = EXPECTED_SPOKE +} = {}) { + const + errors = [], + manifestPath = path.resolve(root, manifestRelPath); + + if (!fs.existsSync(manifestPath)) { + throw new Error(`[public-docs] Missing manifest: ${manifestRelPath}`) + } + + let manifest; + try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) + } catch (err) { + throw new Error(`[public-docs] Invalid JSON in ${manifestRelPath}: ${err.message}`) + } + + if (manifest?.schema !== EXPECTED_SCHEMA) { + errors.push(`schema must be "${EXPECTED_SCHEMA}" (got ${JSON.stringify(manifest?.schema)})`) + } + if (manifest?.spoke !== expectedSpoke) { + errors.push(`spoke must be "${expectedSpoke}" (got ${JSON.stringify(manifest?.spoke)})`) + } + if (typeof manifest?.treeFile !== 'string' || !manifest.treeFile.trim()) { + errors.push('treeFile must be a non-empty relative path') + } + if (!manifest?.treeMount || typeof manifest.treeMount.id !== 'string' || typeof manifest.treeMount.name !== 'string') { + errors.push('treeMount must declare string id and name fields') + } + if (!Array.isArray(manifest?.entries) || manifest.entries.length === 0) { + errors.push('entries must be a non-empty array') + } + + if (errors.length > 0) { + throw new Error(`[public-docs] ${manifestRelPath} failed header validation:\n - ${errors.join('\n - ')}`) + } + + const treePath = path.resolve(root, manifest.treeFile); + if (!fs.existsSync(treePath)) { + throw new Error(`[public-docs] Missing treeFile declared by manifest: ${manifest.treeFile}`) + } + + const tree = JSON.parse(fs.readFileSync(treePath, 'utf-8')); + if (!Array.isArray(tree?.data)) { + throw new Error(`[public-docs] ${manifest.treeFile} must contain a top-level "data" array`) + } + + const + learnRoot = path.resolve(root, 'learn'), + treeLeafByFile = new Map(); + + for (const node of tree.data) { + if (node.isLeaf !== false) { + const relFile = `learn/${node.id}.md`; + treeLeafByFile.set(relFile, node.id) + } + } + + const + seenFiles = new Set(), + seenRoutes = new Set(); + + for (const [index, entry] of manifest.entries.entries()) { + const label = `entries[${index}]`; + + if (!entry || typeof entry.file !== 'string' || typeof entry.route !== 'string') { + errors.push(`${label}: each entry requires string "file" and "route"`); + continue + } + + const normalizedFile = path.posix.normalize(entry.file); + if (normalizedFile !== entry.file || !entry.file.startsWith('learn/') || !entry.file.endsWith('.md') || entry.file.includes('..')) { + errors.push(`${label}: invalid file path "${entry.file}" (must be normalized "learn/.md")`); + continue + } + + if (seenFiles.has(entry.file)) { + errors.push(`${label}: duplicate file "${entry.file}"`) + } + seenFiles.add(entry.file); + + if (!entry.route.startsWith(routePrefix) || entry.route.endsWith('/') || entry.route.endsWith('.md') || entry.route.includes('..')) { + errors.push(`${label}: invalid route "${entry.route}" (must start with "${routePrefix}" without trailing "/" or ".md")`) + } + + const expectedId = treeLeafByFile.get(entry.file); + if (expectedId && entry.route !== `${routePrefix}${expectedId}`) { + errors.push(`${label}: route "${entry.route}" does not match pre-split tree route "${routePrefix}${expectedId}"`) + } + + if (seenRoutes.has(entry.route)) { + errors.push(`${label}: duplicate route "${entry.route}"`) + } + seenRoutes.add(entry.route); + + const absFile = path.resolve(root, entry.file); + if (!absFile.startsWith(learnRoot + path.sep) || !fs.existsSync(absFile) || !fs.statSync(absFile).isFile()) { + errors.push(`${label}: missing Markdown file "${entry.file}"`); + continue + } + + const + rawMarkdown = fs.readFileSync(absFile, 'utf-8'), + proseOnly = stripFencedCodeBlocks(rawMarkdown); + + for (const [regex, kind] of [ + [UNIX_HOST_PATH_RE, 'host filesystem path'], + [WIN_HOST_PATH_RE, 'Windows host filesystem path'] + ]) { + regex.lastIndex = 0; + for (const match of rawMarkdown.matchAll(regex)) { + const + token = match[1], + line = lineAt(rawMarkdown, match.index + match[0].indexOf(token)); + errors.push(`${entry.file}:${line}: forbidden ${kind} "${token}"`) + } + } + + PRIVATE_REPO_RE.lastIndex = 0; + for (const match of rawMarkdown.matchAll(PRIVATE_REPO_RE)) { + const line = lineAt(rawMarkdown, match.index); + errors.push(`${entry.file}:${line}: forbidden private-repository reference "${match[0]}"`) + } + + MD_LINK_RE.lastIndex = 0; + for (const match of proseOnly.matchAll(MD_LINK_RE)) { + const target = match[1]; + if (/^(?:https?:|mailto:)/i.test(target) || target.startsWith('/')) continue; + + const resolvedTarget = path.resolve(path.dirname(absFile), target); + if (!resolvedTarget.startsWith(learnRoot + path.sep) || !fs.existsSync(resolvedTarget)) { + const line = lineAt(proseOnly, match.index); + errors.push(`${entry.file}:${line}: broken relative Markdown link "${target}"`) + } + } + } + + for (const relFile of treeLeafByFile.keys()) { + if (!seenFiles.has(relFile)) { + errors.push(`${manifest.treeFile}: leaf "${relFile}" is missing from ${manifestRelPath}`) + } + } + + for (const relFile of seenFiles) { + if (!treeLeafByFile.has(relFile)) { + errors.push(`${manifestRelPath}: "${relFile}" is not a leaf in ${manifest.treeFile}`) + } + } + + if (errors.length > 0) { + throw new Error(`[public-docs] Validation failed (${errors.length} issue(s)):\n - ${errors.join('\n - ')}`) + } + + return { + entryCount: manifest.entries.length, + routes : [...seenRoutes] + } +} + +const isMainModule = (() => { + if (!process.argv[1]) return false; + try { + return fs.realpathSync(process.argv[1]) === fs.realpathSync(__filename) + } catch { + return path.resolve(process.argv[1]) === __filename + } +})(); + +if (isMainModule) { + try { + const {entryCount} = lintPublicDocs(); + console.log(`[public-docs] OK — verified ${entryCount} manifest-listed guides against learn/tree.json.`) + } catch (err) { + console.error(err.message); + process.exit(1) + } +} diff --git a/docs/public-manifest.json b/docs/public-manifest.json new file mode 100644 index 0000000..7d04164 --- /dev/null +++ b/docs/public-manifest.json @@ -0,0 +1,38 @@ +{ + "schema" : "public-manifest.v1", + "spoke" : "neomjs/devindex", + "treeFile" : "learn/tree.json", + "treeMount": { + "parentId": "Guides", + "id" : "guides/devindex", + "name" : "DevIndex" + }, + "entries" : [ + {"file": "learn/Introduction.md", "route": "/learn/guides/devindex/Introduction"}, + {"file": "learn/EthicalManifesto.md", "route": "/learn/guides/devindex/EthicalManifesto"}, + {"file": "learn/Methodology.md", "route": "/learn/guides/devindex/Methodology"}, + {"file": "learn/OptIn.md", "route": "/learn/guides/devindex/OptIn"}, + {"file": "learn/OptOut.md", "route": "/learn/guides/devindex/OptOut"}, + {"file": "learn/UserGuide.md", "route": "/learn/guides/devindex/UserGuide"}, + {"file": "learn/personas/TalentScouts.md", "route": "/learn/guides/devindex/personas/TalentScouts"}, + {"file": "learn/personas/DataScientists.md", "route": "/learn/guides/devindex/personas/DataScientists"}, + {"file": "learn/personas/PolicyMakers.md", "route": "/learn/guides/devindex/personas/PolicyMakers"}, + {"file": "learn/data-factory/Intro.md", "route": "/learn/guides/devindex/data-factory/Intro"}, + {"file": "learn/data-factory/Orchestrator.md", "route": "/learn/guides/devindex/data-factory/Orchestrator"}, + {"file": "learn/data-factory/Storage.md", "route": "/learn/guides/devindex/data-factory/Storage"}, + {"file": "learn/data-factory/GitHubAPI.md", "route": "/learn/guides/devindex/data-factory/GitHubAPI"}, + {"file": "learn/data-factory/Engine.md", "route": "/learn/guides/devindex/data-factory/Engine"}, + {"file": "learn/data-factory/Updater.md", "route": "/learn/guides/devindex/data-factory/Updater"}, + {"file": "learn/data-factory/DataEnrichment.md", "route": "/learn/guides/devindex/data-factory/DataEnrichment"}, + {"file": "learn/data-factory/DataHygiene.md", "route": "/learn/guides/devindex/data-factory/DataHygiene"}, + {"file": "learn/data-factory/OptIn.md", "route": "/learn/guides/devindex/data-factory/OptIn"}, + {"file": "learn/data-factory/OptOut.md", "route": "/learn/guides/devindex/data-factory/OptOut"}, + {"file": "learn/Backend.md", "route": "/learn/guides/devindex/Backend"}, + {"file": "learn/frontend/Architecture.md", "route": "/learn/guides/devindex/frontend/Architecture"}, + {"file": "learn/frontend/TheGrid.md", "route": "/learn/guides/devindex/frontend/TheGrid"}, + {"file": "learn/frontend/StateAndControls.md", "route": "/learn/guides/devindex/frontend/StateAndControls"}, + {"file": "learn/frontend/OffscreenCanvas.md", "route": "/learn/guides/devindex/frontend/OffscreenCanvas"}, + {"file": "learn/frontend/ContentEngine.md", "route": "/learn/guides/devindex/frontend/ContentEngine"}, + {"file": "learn/FAQ.md", "route": "/learn/guides/devindex/FAQ"} + ] +} diff --git a/package.json b/package.json index 8d071a9..60b0e65 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "devindex:spider" : "node ./services/cli.mjs spider", "devindex:update" : "node ./services/cli.mjs update --limit 200", "generate-docs-json" : "node ./node_modules/neo.mjs/buildScripts/docs/jsdocx.mjs", + "lint-public-docs" : "node ./buildScripts/lintPublicDocs.mjs", "server-start" : "webpack serve -c ./node_modules/neo.mjs/buildScripts/webpack/webpack.server.config.mjs --open", "test-e2e" : "playwright test -c test/playwright/playwright.config.e2e.mjs", "test-unit" : "playwright test -c test/playwright/playwright.config.unit.mjs", diff --git a/test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs b/test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs new file mode 100644 index 0000000..a017512 --- /dev/null +++ b/test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs @@ -0,0 +1,129 @@ +import {test, expect} from '@playwright/test'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import {lintPublicDocs, stripFencedCodeBlocks} from '../../../../buildScripts/lintPublicDocs.mjs'; + +/** + * @summary Builds a minimal valid spoke fixture under `os.tmpdir()` for targeted negative/positive lint tests. + * @param {Object} [overrides={}] + * @returns {String} Temporary fixture root path + */ +function createFixture({ + guideBody = '# Spider Engine\n\nSee [Intro](./Intro.md) and `apps/devindex/view/home/MainContainer.mjs` by `@neo-opus-grace`.\n', + extraTreeLeaf = false, + extraManifestEntry = false, + routeOverride = null +} = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'devindex-public-docs-')); + + fs.mkdirSync(path.join(root, 'docs'), {recursive: true}); + fs.mkdirSync(path.join(root, 'learn/data-factory'), {recursive: true}); + + fs.writeFileSync(path.join(root, 'learn/data-factory/Intro.md'), '# Intro\n'); + fs.writeFileSync(path.join(root, 'learn/data-factory/Engine.md'), guideBody); + + const treeData = [ + {name: 'The Data Factory', parentId: null, isLeaf: false, id: 'DataFactory'}, + {name: 'Introduction', parentId: 'DataFactory', id: 'data-factory/Intro'}, + {name: 'Spider Engine', parentId: 'DataFactory', id: 'data-factory/Engine'} + ]; + + if (extraTreeLeaf) { + fs.writeFileSync(path.join(root, 'learn/data-factory/Extra.md'), '# Extra\n'); + treeData.push({name: 'Extra', parentId: 'DataFactory', id: 'data-factory/Extra'}) + } + + fs.writeFileSync(path.join(root, 'learn/tree.json'), JSON.stringify({data: treeData}, null, 4)); + + const entries = [ + {file: 'learn/data-factory/Intro.md', route: '/learn/guides/devindex/data-factory/Intro'}, + {file: 'learn/data-factory/Engine.md', route: routeOverride || '/learn/guides/devindex/data-factory/Engine'} + ]; + + if (extraManifestEntry) { + fs.writeFileSync(path.join(root, 'learn/data-factory/Unlisted.md'), '# Unlisted\n'); + entries.push({file: 'learn/data-factory/Unlisted.md', route: '/learn/guides/devindex/data-factory/Unlisted'}) + } + + fs.writeFileSync(path.join(root, 'docs/public-manifest.json'), JSON.stringify({ + schema : 'public-manifest.v1', + spoke : 'neomjs/devindex', + treeFile : 'learn/tree.json', + treeMount: {parentId: 'Guides', id: 'guides/devindex', name: 'DevIndex'}, + entries + }, null, 4)); + + return root +} + +test.describe('buildScripts/lintPublicDocs', () => { + test('the live neomjs/devindex manifest admits all 26 guides at their pre-split /learn/guides/devindex/* routes', () => { + const result = lintPublicDocs(); + + expect(result.entryCount).toBe(26); + expect(result.routes).toContain('/learn/guides/devindex/data-factory/Engine'); + expect(result.routes).toContain('/learn/guides/devindex/Introduction') + }); + + test('passes clean prose containing repo-relative view/home/* paths, /Users// placeholders, and public @neo-* handles', () => { + const root = createFixture({ + guideBody: [ + '# Spider Engine', + '', + 'Clone into `/Users//workspace/devindex` and inspect `apps/devindex/view/home/MainContainer.mjs`.', + 'Reviewed by `@neo-gemini-julia` and `@neo-opus-grace`. See [Intro](./Intro.md).' + ].join('\n') + }); + + expect(lintPublicDocs({root}).entryCount).toBe(2) + }); + + test('rejects real Unix/macOS and Windows host filesystem paths', () => { + const unixRoot = createFixture({ + guideBody: '# Leak\n\nLoaded from `/Users/alice/.neo-ai/devindex/learn/Intro.md`.\n' + }); + expect(() => lintPublicDocs({root: unixRoot})).toThrow(/forbidden host filesystem path "\/Users\/alice\//); + + const winRoot = createFixture({ + guideBody: '# Leak\n\nLoaded from `C:\\Users\\alice\\devindex\\learn\\Intro.md`.\n' + }); + expect(() => lintPublicDocs({root: winRoot})).toThrow(/forbidden Windows host filesystem path/) + }); + + test('rejects private repository references across business, neo-stage, and neo-pro', () => { + for (const repo of ['neomjs/business', 'neomjs/neo-stage', 'neomjs/neo-pro']) { + const root = createFixture({ + guideBody: `# Leak\n\nSee ${repo}#42 for internal notes.\n` + }); + expect(() => lintPublicDocs({root})).toThrow(/forbidden private-repository reference/) + } + }); + + test('rejects manifest/tree drift and non-matching pre-split routes', () => { + expect(() => lintPublicDocs({root: createFixture({extraTreeLeaf: true})})) + .toThrow(/leaf "learn\/data-factory\/Extra\.md" is missing/); + + expect(() => lintPublicDocs({root: createFixture({extraManifestEntry: true})})) + .toThrow(/"learn\/data-factory\/Unlisted\.md" is not a leaf/); + + expect(() => lintPublicDocs({root: createFixture({routeOverride: '/learn/devindex/data-factory/Engine'})})) + .toThrow(/invalid route/); + + expect(() => lintPublicDocs({root: createFixture({routeOverride: '/learn/guides/devindex/WrongSlug'})})) + .toThrow(/does not match pre-split tree route/) + }); + + test('rejects broken relative Markdown links in prose while ignoring fenced code blocks', () => { + const fencedRoot = createFixture({ + guideBody: '# Spider Engine\n\n```markdown\n[Example](./DoesNotExist.md)\n```\n' + }); + expect(stripFencedCodeBlocks('a\n```js\nb\n```\nc')).toBe('a\n\n\n\nc'); + expect(lintPublicDocs({root: fencedRoot}).entryCount).toBe(2); + + const brokenRoot = createFixture({ + guideBody: '# Spider Engine\n\nSee [Missing](./DoesNotExist.md#top) for details.\n' + }); + expect(() => lintPublicDocs({root: brokenRoot})).toThrow(/broken relative Markdown link "\.\/DoesNotExist\.md"/) + }) +}); From 8ff3411b122953c60a63c5ca3f035504b7838124 Mon Sep 17 00:00:00 2001 From: Julia Date: Sun, 22 Nov 2026 01:46:28 +0100 Subject: [PATCH 2/4] docs(buildScripts): remove tracking refs from lintPublicDocs JSDoc (#161) --- buildScripts/lintPublicDocs.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildScripts/lintPublicDocs.mjs b/buildScripts/lintPublicDocs.mjs index 85c0696..d8d865a 100644 --- a/buildScripts/lintPublicDocs.mjs +++ b/buildScripts/lintPublicDocs.mjs @@ -6,7 +6,7 @@ import {fileURLToPath} from 'node:url'; /** * @module buildScripts.lintPublicDocs * @summary Validates `docs/public-manifest.json` against `learn/tree.json` and enforces public-publish hygiene across - * every manifest-admitted Markdown guide (`neomjs/neo` Discussion #20920 `A0 + A1`, `neomjs/devindex#161`). + * every manifest-admitted Markdown guide. * * Default-deny spoke federation requires two guarantees: * 1. **Structural parity:** Every `learn/tree.json` leaf is explicitly listed in `docs/public-manifest.json` at its From a2d61b485da93c2c083c9d1e1da3092bb3138ca3 Mon Sep 17 00:00:00 2001 From: Julia Date: Sun, 22 Nov 2026 01:57:30 +0100 Subject: [PATCH 3/4] feat(docs): align treeMount name to DevIndex App (#161) --- docs/public-manifest.json | 2 +- test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/public-manifest.json b/docs/public-manifest.json index 7d04164..99c11db 100644 --- a/docs/public-manifest.json +++ b/docs/public-manifest.json @@ -5,7 +5,7 @@ "treeMount": { "parentId": "Guides", "id" : "guides/devindex", - "name" : "DevIndex" + "name" : "DevIndex App" }, "entries" : [ {"file": "learn/Introduction.md", "route": "/learn/guides/devindex/Introduction"}, diff --git a/test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs b/test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs index a017512..0ddcc72 100644 --- a/test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs +++ b/test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs @@ -50,7 +50,7 @@ function createFixture({ schema : 'public-manifest.v1', spoke : 'neomjs/devindex', treeFile : 'learn/tree.json', - treeMount: {parentId: 'Guides', id: 'guides/devindex', name: 'DevIndex'}, + treeMount: {parentId: 'Guides', id: 'guides/devindex', name: 'DevIndex App'}, entries }, null, 4)); From 463267a493c1ca1e7d06a371582bb4a13f71ef99 Mon Sep 17 00:00:00 2001 From: Julia Date: Sun, 22 Nov 2026 02:42:22 +0100 Subject: [PATCH 4/4] fix(docs): enforce manifest-admitted link targets and file:// host-path rejection (#161) --- buildScripts/lintPublicDocs.mjs | 88 ++++++++++++------- .../unit/buildScripts/LintPublicDocs.spec.mjs | 53 ++++++++--- 2 files changed, 101 insertions(+), 40 deletions(-) diff --git a/buildScripts/lintPublicDocs.mjs b/buildScripts/lintPublicDocs.mjs index d8d865a..637260e 100644 --- a/buildScripts/lintPublicDocs.mjs +++ b/buildScripts/lintPublicDocs.mjs @@ -9,27 +9,27 @@ import {fileURLToPath} from 'node:url'; * every manifest-admitted Markdown guide. * * Default-deny spoke federation requires two guarantees: - * 1. **Structural parity:** Every `learn/tree.json` leaf is explicitly listed in `docs/public-manifest.json` at its - * canonical `/learn/guides/devindex/` route, and every manifest entry resolves to a real tracked `.md` file - * inside `learn/` with no path traversal or duplicate routes. + * 1. **Structural parity:** Every `learn/tree.json` leaf is explicitly listed in `docs/public-manifest.json` under the + * canonical `/learn//` route derived from `treeMount.id`, and every manifest entry resolves to a + * real tracked `.md` file inside `learn/` with no path traversal or duplicate routes. * 2. **Content hygiene:** Admitted product guides contain zero real host filesystem paths (`/Users//`, - * `/home//`, `C:\Users\\`), zero private-repository identifiers (`neomjs/business`, - * `neomjs/neo-stage`, `neomjs/neo-pro`), and zero broken relative `.md` links outside fenced code blocks, while - * permitting public `@neo-*` maintainer handles used in illustrative examples. + * `/home//`, `C:\Users\\`, or `file://` URLs targeting them), zero private-repository identifiers + * (`neomjs/business`, `neomjs/neo-stage`, `neomjs/neo-pro`), and zero broken or unpublished relative Markdown links + * outside fenced and inline code, while permitting public `@neo-*` maintainer handles used in illustrative examples. */ const __filename = fileURLToPath(import.meta.url), REPO_ROOT = path.resolve(path.dirname(__filename), '..'), - DEFAULT_PREFIX = '/learn/guides/devindex/', EXPECTED_SCHEMA = 'public-manifest.v1', EXPECTED_SPOKE = 'neomjs/devindex', - // Root-anchored Unix/macOS & Windows user home paths; ignores repo-relative segments like `view/home/MainContainer.mjs` - // and generic placeholders (``, ``, ``, `you`, `user`, `username`). - UNIX_HOST_PATH_RE = /(?:^|[\s`'"(=])(\/(?:Users|home)\/(?![<{]|(?:you|user|username)(?:\/|\b))[A-Za-z0-9._-]+\/[^\s`'")>\]]*)/gm, - WIN_HOST_PATH_RE = /(?:^|[\s`'"(=])([A-Za-z]:\\Users\\(?![<{]|(?:you|user|username)(?:\\|\b))[A-Za-z0-9._-]+\\[^\s`'")>\]]*)/gm, + // Root-anchored or file:// Unix/macOS & Windows user home paths; ignores repo-relative segments like + // `view/home/MainContainer.mjs` and generic placeholders (``, ``, ``, `you`, `user`, `username`). + UNIX_HOST_PATH_RE = /(?:^|[\s`'"(=:]|file:\/\/)(\/(?:Users|home)\/(?![<{]|(?:you|user|username)(?:\/|\b))[A-Za-z0-9._-]+\/[^\s`'")>\]]*)/gm, + WIN_HOST_PATH_RE = /(?:^|[\s`'"(=:]|file:\/\/\/?)([A-Za-z]:[\\/]Users[\\/](?![<{]|(?:you|user|username)(?:[\\/]|\b))[A-Za-z0-9._-]+[\\/][^\s`'")>\]]*)/gm, PRIVATE_REPO_RE = /\bneomjs\/(?:business|neo-stage|neo-pro)\b/gi, - MD_LINK_RE = /\[[^\]]*\]\(([^)#?\s]+\.md)(?:[?#][^)]*)?\)/g, + REL_LINK_RE = /\[(?:[^\[\]]|\[[^\]]*\])*\]\(([^)\s]+)\)/g, + INLINE_CODE_RE = /`[^`\n]+`/g, FENCE_RE = /^ {0,3}(`{3,}|~{3,})/; /** @@ -88,14 +88,14 @@ function lineAt(text, index) { * @param {Object} [options={}] * @param {String} [options.root=REPO_ROOT] Repository root directory. * @param {String} [options.manifestRelPath='docs/public-manifest.json'] Manifest path relative to `root`. - * @param {String} [options.routePrefix=DEFAULT_PREFIX] Required canonical URL prefix for routes in this spoke. + * @param {String|null} [options.routePrefix=null] Optional route prefix override; defaults to `/learn//`. * @param {String} [options.expectedSpoke=EXPECTED_SPOKE] Expected `spoke` identifier in the manifest. * @returns {{entryCount: Number, routes: String[]}} */ export function lintPublicDocs({ root = REPO_ROOT, manifestRelPath = 'docs/public-manifest.json', - routePrefix = DEFAULT_PREFIX, + routePrefix = null, expectedSpoke = EXPECTED_SPOKE } = {}) { const @@ -122,8 +122,16 @@ export function lintPublicDocs({ if (typeof manifest?.treeFile !== 'string' || !manifest.treeFile.trim()) { errors.push('treeFile must be a non-empty relative path') } - if (!manifest?.treeMount || typeof manifest.treeMount.id !== 'string' || typeof manifest.treeMount.name !== 'string') { - errors.push('treeMount must declare string id and name fields') + if ( + !manifest?.treeMount || + typeof manifest.treeMount.id !== 'string' || + !manifest.treeMount.id.trim() || + manifest.treeMount.id.startsWith('/') || + manifest.treeMount.id.endsWith('/') || + manifest.treeMount.id.includes('..') || + typeof manifest.treeMount.name !== 'string' + ) { + errors.push('treeMount must declare normalized string id and name fields') } if (!Array.isArray(manifest?.entries) || manifest.entries.length === 0) { errors.push('entries must be a non-empty array') @@ -133,6 +141,8 @@ export function lintPublicDocs({ throw new Error(`[public-docs] ${manifestRelPath} failed header validation:\n - ${errors.join('\n - ')}`) } + const effectivePrefix = routePrefix ?? `/learn/${manifest.treeMount.id}/`; + const treePath = path.resolve(root, manifest.treeFile); if (!fs.existsSync(treePath)) { throw new Error(`[public-docs] Missing treeFile declared by manifest: ${manifest.treeFile}`) @@ -155,8 +165,18 @@ export function lintPublicDocs({ } const - seenFiles = new Set(), - seenRoutes = new Set(); + seenFiles = new Set(), + seenRoutes = new Set(), + admittedAbsFiles = new Set(); + + for (const entry of manifest.entries) { + if (entry && typeof entry.file === 'string' && !entry.file.includes('..') && entry.file.startsWith('learn/') && entry.file.endsWith('.md')) { + const absCandidate = path.resolve(root, entry.file); + if (absCandidate.startsWith(learnRoot + path.sep) && fs.existsSync(absCandidate) && fs.statSync(absCandidate).isFile()) { + admittedAbsFiles.add(absCandidate) + } + } + } for (const [index, entry] of manifest.entries.entries()) { const label = `entries[${index}]`; @@ -177,13 +197,13 @@ export function lintPublicDocs({ } seenFiles.add(entry.file); - if (!entry.route.startsWith(routePrefix) || entry.route.endsWith('/') || entry.route.endsWith('.md') || entry.route.includes('..')) { - errors.push(`${label}: invalid route "${entry.route}" (must start with "${routePrefix}" without trailing "/" or ".md")`) + if (!entry.route.startsWith(effectivePrefix) || entry.route.endsWith('/') || entry.route.endsWith('.md') || entry.route.includes('..')) { + errors.push(`${label}: invalid route "${entry.route}" (must start with "${effectivePrefix}" without trailing "/" or ".md")`) } const expectedId = treeLeafByFile.get(entry.file); - if (expectedId && entry.route !== `${routePrefix}${expectedId}`) { - errors.push(`${label}: route "${entry.route}" does not match pre-split tree route "${routePrefix}${expectedId}"`) + if (expectedId && entry.route !== `${effectivePrefix}${expectedId}`) { + errors.push(`${label}: route "${entry.route}" does not match pre-split tree route "${effectivePrefix}${expectedId}"`) } if (seenRoutes.has(entry.route)) { @@ -220,15 +240,23 @@ export function lintPublicDocs({ errors.push(`${entry.file}:${line}: forbidden private-repository reference "${match[0]}"`) } - MD_LINK_RE.lastIndex = 0; - for (const match of proseOnly.matchAll(MD_LINK_RE)) { - const target = match[1]; - if (/^(?:https?:|mailto:)/i.test(target) || target.startsWith('/')) continue; + const linkScanText = proseOnly.replace(INLINE_CODE_RE, match => ' '.repeat(match.length)); + + REL_LINK_RE.lastIndex = 0; + for (const match of linkScanText.matchAll(REL_LINK_RE)) { + const + rawTarget = match[1], + target = rawTarget.split(/[?#]/)[0]; + + if (!target || /^(?:https?:|mailto:)/i.test(rawTarget) || rawTarget.startsWith('#') || rawTarget.startsWith('/')) continue; + + const + resolvedExact = path.resolve(path.dirname(absFile), target), + resolvedMd = target.endsWith('.md') ? resolvedExact : path.resolve(path.dirname(absFile), `${target}.md`); - const resolvedTarget = path.resolve(path.dirname(absFile), target); - if (!resolvedTarget.startsWith(learnRoot + path.sep) || !fs.existsSync(resolvedTarget)) { - const line = lineAt(proseOnly, match.index); - errors.push(`${entry.file}:${line}: broken relative Markdown link "${target}"`) + if (!admittedAbsFiles.has(resolvedExact) && !admittedAbsFiles.has(resolvedMd)) { + const line = lineAt(linkScanText, match.index); + errors.push(`${entry.file}:${line}: broken relative Markdown link "${target}" (must resolve to a manifest-admitted guide)`) } } } diff --git a/test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs b/test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs index 0ddcc72..dfddf61 100644 --- a/test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs +++ b/test/playwright/unit/buildScripts/LintPublicDocs.spec.mjs @@ -4,18 +4,23 @@ import os from 'node:os'; import path from 'node:path'; import {lintPublicDocs, stripFencedCodeBlocks} from '../../../../buildScripts/lintPublicDocs.mjs'; +const createdRoots = []; + /** * @summary Builds a minimal valid spoke fixture under `os.tmpdir()` for targeted negative/positive lint tests. * @param {Object} [overrides={}] * @returns {String} Temporary fixture root path */ function createFixture({ - guideBody = '# Spider Engine\n\nSee [Intro](./Intro.md) and `apps/devindex/view/home/MainContainer.mjs` by `@neo-opus-grace`.\n', - extraTreeLeaf = false, + guideBody = '# Spider Engine\n\nSee [Intro](./Intro.md) and `apps/devindex/view/home/MainContainer.mjs` by `@neo-opus-grace`.\n', + extraTreeLeaf = false, extraManifestEntry = false, - routeOverride = null + unlistedDiskFile = false, + mountIdOverride = null, + routeOverride = null } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'devindex-public-docs-')); + createdRoots.push(root); fs.mkdirSync(path.join(root, 'docs'), {recursive: true}); fs.mkdirSync(path.join(root, 'learn/data-factory'), {recursive: true}); @@ -23,6 +28,10 @@ function createFixture({ fs.writeFileSync(path.join(root, 'learn/data-factory/Intro.md'), '# Intro\n'); fs.writeFileSync(path.join(root, 'learn/data-factory/Engine.md'), guideBody); + if (unlistedDiskFile) { + fs.writeFileSync(path.join(root, 'learn/data-factory/Internal.md'), '# Internal\n') + } + const treeData = [ {name: 'The Data Factory', parentId: null, isLeaf: false, id: 'DataFactory'}, {name: 'Introduction', parentId: 'DataFactory', id: 'data-factory/Intro'}, @@ -50,7 +59,7 @@ function createFixture({ schema : 'public-manifest.v1', spoke : 'neomjs/devindex', treeFile : 'learn/tree.json', - treeMount: {parentId: 'Guides', id: 'guides/devindex', name: 'DevIndex App'}, + treeMount: {parentId: 'Guides', id: mountIdOverride || 'guides/devindex', name: 'DevIndex App'}, entries }, null, 4)); @@ -58,6 +67,11 @@ function createFixture({ } test.describe('buildScripts/lintPublicDocs', () => { + test.afterEach(() => { + while (createdRoots.length > 0) { + fs.rmSync(createdRoots.pop(), {recursive: true, force: true}) + } + }); test('the live neomjs/devindex manifest admits all 26 guides at their pre-split /learn/guides/devindex/* routes', () => { const result = lintPublicDocs(); @@ -79,16 +93,26 @@ test.describe('buildScripts/lintPublicDocs', () => { expect(lintPublicDocs({root}).entryCount).toBe(2) }); - test('rejects real Unix/macOS and Windows host filesystem paths', () => { + test('rejects real Unix/macOS and Windows host filesystem paths including file:// URLs', () => { const unixRoot = createFixture({ guideBody: '# Leak\n\nLoaded from `/Users/alice/.neo-ai/devindex/learn/Intro.md`.\n' }); expect(() => lintPublicDocs({root: unixRoot})).toThrow(/forbidden host filesystem path "\/Users\/alice\//); + const fileUrlRoot = createFixture({ + guideBody: '# Leak\n\nScreenshot at `file:///Users/alice/x/shot.png`.\n' + }); + expect(() => lintPublicDocs({root: fileUrlRoot})).toThrow(/forbidden host filesystem path "\/Users\/alice\/x\/shot\.png"/); + const winRoot = createFixture({ guideBody: '# Leak\n\nLoaded from `C:\\Users\\alice\\devindex\\learn\\Intro.md`.\n' }); - expect(() => lintPublicDocs({root: winRoot})).toThrow(/forbidden Windows host filesystem path/) + expect(() => lintPublicDocs({root: winRoot})).toThrow(/forbidden Windows host filesystem path/); + + const winFileUrlRoot = createFixture({ + guideBody: '# Leak\n\nScreenshot at `file:///C:/Users/alice/x/shot.png`.\n' + }); + expect(() => lintPublicDocs({root: winFileUrlRoot})).toThrow(/forbidden Windows host filesystem path/) }); test('rejects private repository references across business, neo-stage, and neo-pro', () => { @@ -100,13 +124,16 @@ test.describe('buildScripts/lintPublicDocs', () => { } }); - test('rejects manifest/tree drift and non-matching pre-split routes', () => { + test('rejects manifest/tree drift, treeMount.id drift, and non-matching pre-split routes', () => { expect(() => lintPublicDocs({root: createFixture({extraTreeLeaf: true})})) .toThrow(/leaf "learn\/data-factory\/Extra\.md" is missing/); expect(() => lintPublicDocs({root: createFixture({extraManifestEntry: true})})) .toThrow(/"learn\/data-factory\/Unlisted\.md" is not a leaf/); + expect(() => lintPublicDocs({root: createFixture({mountIdOverride: 'guides/other'})})) + .toThrow(/invalid route/); + expect(() => lintPublicDocs({root: createFixture({routeOverride: '/learn/devindex/data-factory/Engine'})})) .toThrow(/invalid route/); @@ -114,9 +141,9 @@ test.describe('buildScripts/lintPublicDocs', () => { .toThrow(/does not match pre-split tree route/) }); - test('rejects broken relative Markdown links in prose while ignoring fenced code blocks', () => { + test('rejects broken or unpublished relative Markdown links in prose while ignoring fenced and inline code', () => { const fencedRoot = createFixture({ - guideBody: '# Spider Engine\n\n```markdown\n[Example](./DoesNotExist.md)\n```\n' + guideBody: '# Spider Engine\n\n```markdown\n[Example](./DoesNotExist.md)\n```\nInline `[Example](./DoesNotExist.md)` stays inert.\n' }); expect(stripFencedCodeBlocks('a\n```js\nb\n```\nc')).toBe('a\n\n\n\nc'); expect(lintPublicDocs({root: fencedRoot}).entryCount).toBe(2); @@ -124,6 +151,12 @@ test.describe('buildScripts/lintPublicDocs', () => { const brokenRoot = createFixture({ guideBody: '# Spider Engine\n\nSee [Missing](./DoesNotExist.md#top) for details.\n' }); - expect(() => lintPublicDocs({root: brokenRoot})).toThrow(/broken relative Markdown link "\.\/DoesNotExist\.md"/) + expect(() => lintPublicDocs({root: brokenRoot})).toThrow(/broken relative Markdown link "\.\/DoesNotExist\.md"/); + + const unlistedTargetRoot = createFixture({ + unlistedDiskFile: true, + guideBody : '# Spider Engine\n\nSee [Internal](./Internal.md) notes.\n' + }); + expect(() => lintPublicDocs({root: unlistedTargetRoot})).toThrow(/broken relative Markdown link "\.\/Internal\.md"/) }) });