Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ jobs:
- name: Assert no derived data is tracked
run: npm run check-data-tracking

- name: Lint public documentation manifest
run: npm run lint-public-docs

# `npm ci` above ran the postinstall linker, so this asserts the projection actually happened
# rather than re-doing it: both skill surfaces point at the installed package, and no skill
# byte is tracked here. An `--ignore-scripts` install resolves the dependency without linking
Expand Down
303 changes: 303 additions & 0 deletions buildScripts/lintPublicDocs.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,303 @@
import fs from 'node:fs';
import path from 'node:path';
import process from 'node:process';
import {fileURLToPath} from 'node:url';

/**
* @module buildScripts.lintPublicDocs
* @summary Validates `docs/public-manifest.json` against `learn/tree.json` and enforces public-publish hygiene across
* every manifest-admitted Markdown guide.
*
* Default-deny spoke federation requires two guarantees:
* 1. **Structural parity:** Every `learn/tree.json` leaf is explicitly listed in `docs/public-manifest.json` under the
* canonical `/learn/<treeMount.id>/<id>` route derived from `treeMount.id`, and every manifest entry resolves to a
* real tracked `.md` file inside `learn/` with no path traversal or duplicate routes.
* 2. **Content hygiene:** Admitted product guides contain zero real host filesystem paths (`/Users/<name>/`,
* `/home/<name>/`, `C:\Users\<name>\`, or `file://` URLs targeting them), zero private-repository identifiers
* (`neomjs/business`, `neomjs/neo-stage`, `neomjs/neo-pro`), and zero broken or unpublished relative Markdown links
* outside fenced and inline code, while permitting public `@neo-*` maintainer handles used in illustrative examples.
*/

const
__filename = fileURLToPath(import.meta.url),
REPO_ROOT = path.resolve(path.dirname(__filename), '..'),
EXPECTED_SCHEMA = 'public-manifest.v1',
EXPECTED_SPOKE = 'neomjs/devindex',
// Root-anchored or file:// Unix/macOS & Windows user home paths; ignores repo-relative segments like
// `view/home/MainContainer.mjs` and generic placeholders (`<you>`, `<user>`, `<username>`, `you`, `user`, `username`).
UNIX_HOST_PATH_RE = /(?:^|[\s`'"(=:]|file:\/\/)(\/(?:Users|home)\/(?![<{]|(?:you|user|username)(?:\/|\b))[A-Za-z0-9._-]+\/[^\s`'")>\]]*)/gm,
WIN_HOST_PATH_RE = /(?:^|[\s`'"(=:]|file:\/\/\/?)([A-Za-z]:[\\/]Users[\\/](?![<{]|(?:you|user|username)(?:[\\/]|\b))[A-Za-z0-9._-]+[\\/][^\s`'")>\]]*)/gm,
PRIVATE_REPO_RE = /\bneomjs\/(?:business|neo-stage|neo-pro)\b/gi,
REL_LINK_RE = /\[(?:[^\[\]]|\[[^\]]*\])*\]\(([^)\s]+)\)/g,
INLINE_CODE_RE = /`[^`\n]+`/g,
FENCE_RE = /^ {0,3}(`{3,}|~{3,})/;

/**
* @summary Strips fenced Markdown code blocks while preserving line numbers for diagnostic reporting.
* @param {String} markdown
* @returns {String}
*/
export function stripFencedCodeBlocks(markdown) {
const
lines = String(markdown).split('\n'),
out = [];

let fence = null;

for (const rawLine of lines) {
const
line = rawLine.replace(/\r$/, ''),
match = FENCE_RE.exec(line);

if (fence) {
if (match && match[1][0] === fence[0] && match[1].length >= fence.length) {
fence = null
}
out.push('');
continue
}

if (match) {
fence = match[1];
out.push('');
continue
}

out.push(line)
}

return out.join('\n')
}

/**
* @summary Computes the 1-based line number for a character offset in a string.
* @param {String} text
* @param {Number} index
* @returns {Number}
*/
function lineAt(text, index) {
let line = 1;
for (let i = 0; i < index && i < text.length; i++) {
if (text[i] === '\n') line++
}
return line
}

/**
* @summary Validates `docs/public-manifest.json`, `learn/tree.json`, and all admitted Markdown files under `root`.
* @param {Object} [options={}]
* @param {String} [options.root=REPO_ROOT] Repository root directory.
* @param {String} [options.manifestRelPath='docs/public-manifest.json'] Manifest path relative to `root`.
* @param {String|null} [options.routePrefix=null] Optional route prefix override; defaults to `/learn/<treeMount.id>/`.
* @param {String} [options.expectedSpoke=EXPECTED_SPOKE] Expected `spoke` identifier in the manifest.
* @returns {{entryCount: Number, routes: String[]}}
*/
export function lintPublicDocs({
root = REPO_ROOT,
manifestRelPath = 'docs/public-manifest.json',
routePrefix = null,
expectedSpoke = EXPECTED_SPOKE
} = {}) {
const
errors = [],
manifestPath = path.resolve(root, manifestRelPath);

if (!fs.existsSync(manifestPath)) {
throw new Error(`[public-docs] Missing manifest: ${manifestRelPath}`)
}

let manifest;
try {
manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'))
} catch (err) {
throw new Error(`[public-docs] Invalid JSON in ${manifestRelPath}: ${err.message}`)
}

if (manifest?.schema !== EXPECTED_SCHEMA) {
errors.push(`schema must be "${EXPECTED_SCHEMA}" (got ${JSON.stringify(manifest?.schema)})`)
}
if (manifest?.spoke !== expectedSpoke) {
errors.push(`spoke must be "${expectedSpoke}" (got ${JSON.stringify(manifest?.spoke)})`)
}
if (typeof manifest?.treeFile !== 'string' || !manifest.treeFile.trim()) {
errors.push('treeFile must be a non-empty relative path')
}
if (
!manifest?.treeMount ||
typeof manifest.treeMount.id !== 'string' ||
!manifest.treeMount.id.trim() ||
manifest.treeMount.id.startsWith('/') ||
manifest.treeMount.id.endsWith('/') ||
manifest.treeMount.id.includes('..') ||
typeof manifest.treeMount.name !== 'string'
) {
errors.push('treeMount must declare normalized string id and name fields')
}
if (!Array.isArray(manifest?.entries) || manifest.entries.length === 0) {
errors.push('entries must be a non-empty array')
}

if (errors.length > 0) {
throw new Error(`[public-docs] ${manifestRelPath} failed header validation:\n - ${errors.join('\n - ')}`)
}

const effectivePrefix = routePrefix ?? `/learn/${manifest.treeMount.id}/`;

const treePath = path.resolve(root, manifest.treeFile);
if (!fs.existsSync(treePath)) {
throw new Error(`[public-docs] Missing treeFile declared by manifest: ${manifest.treeFile}`)
}

const tree = JSON.parse(fs.readFileSync(treePath, 'utf-8'));
if (!Array.isArray(tree?.data)) {
throw new Error(`[public-docs] ${manifest.treeFile} must contain a top-level "data" array`)
}

const
learnRoot = path.resolve(root, 'learn'),
treeLeafByFile = new Map();

for (const node of tree.data) {
if (node.isLeaf !== false) {
const relFile = `learn/${node.id}.md`;
treeLeafByFile.set(relFile, node.id)
}
}

const
seenFiles = new Set(),
seenRoutes = new Set(),
admittedAbsFiles = new Set();

for (const entry of manifest.entries) {
if (entry && typeof entry.file === 'string' && !entry.file.includes('..') && entry.file.startsWith('learn/') && entry.file.endsWith('.md')) {
const absCandidate = path.resolve(root, entry.file);
if (absCandidate.startsWith(learnRoot + path.sep) && fs.existsSync(absCandidate) && fs.statSync(absCandidate).isFile()) {
admittedAbsFiles.add(absCandidate)
}
}
}

for (const [index, entry] of manifest.entries.entries()) {
const label = `entries[${index}]`;

if (!entry || typeof entry.file !== 'string' || typeof entry.route !== 'string') {
errors.push(`${label}: each entry requires string "file" and "route"`);
continue
}

const normalizedFile = path.posix.normalize(entry.file);
if (normalizedFile !== entry.file || !entry.file.startsWith('learn/') || !entry.file.endsWith('.md') || entry.file.includes('..')) {
errors.push(`${label}: invalid file path "${entry.file}" (must be normalized "learn/<path>.md")`);
continue
}

if (seenFiles.has(entry.file)) {
errors.push(`${label}: duplicate file "${entry.file}"`)
}
seenFiles.add(entry.file);

if (!entry.route.startsWith(effectivePrefix) || entry.route.endsWith('/') || entry.route.endsWith('.md') || entry.route.includes('..')) {
errors.push(`${label}: invalid route "${entry.route}" (must start with "${effectivePrefix}" without trailing "/" or ".md")`)
}

const expectedId = treeLeafByFile.get(entry.file);
if (expectedId && entry.route !== `${effectivePrefix}${expectedId}`) {
errors.push(`${label}: route "${entry.route}" does not match pre-split tree route "${effectivePrefix}${expectedId}"`)
}

if (seenRoutes.has(entry.route)) {
errors.push(`${label}: duplicate route "${entry.route}"`)
}
seenRoutes.add(entry.route);

const absFile = path.resolve(root, entry.file);
if (!absFile.startsWith(learnRoot + path.sep) || !fs.existsSync(absFile) || !fs.statSync(absFile).isFile()) {
errors.push(`${label}: missing Markdown file "${entry.file}"`);
continue
}

const
rawMarkdown = fs.readFileSync(absFile, 'utf-8'),
proseOnly = stripFencedCodeBlocks(rawMarkdown);

for (const [regex, kind] of [
[UNIX_HOST_PATH_RE, 'host filesystem path'],
[WIN_HOST_PATH_RE, 'Windows host filesystem path']
]) {
regex.lastIndex = 0;
for (const match of rawMarkdown.matchAll(regex)) {
const
token = match[1],
line = lineAt(rawMarkdown, match.index + match[0].indexOf(token));
errors.push(`${entry.file}:${line}: forbidden ${kind} "${token}"`)
}
}

PRIVATE_REPO_RE.lastIndex = 0;
for (const match of rawMarkdown.matchAll(PRIVATE_REPO_RE)) {
const line = lineAt(rawMarkdown, match.index);
errors.push(`${entry.file}:${line}: forbidden private-repository reference "${match[0]}"`)
}

const linkScanText = proseOnly.replace(INLINE_CODE_RE, match => ' '.repeat(match.length));

REL_LINK_RE.lastIndex = 0;
for (const match of linkScanText.matchAll(REL_LINK_RE)) {
const
rawTarget = match[1],
target = rawTarget.split(/[?#]/)[0];

if (!target || /^(?:https?:|mailto:)/i.test(rawTarget) || rawTarget.startsWith('#') || rawTarget.startsWith('/')) continue;

const
resolvedExact = path.resolve(path.dirname(absFile), target),
resolvedMd = target.endsWith('.md') ? resolvedExact : path.resolve(path.dirname(absFile), `${target}.md`);

if (!admittedAbsFiles.has(resolvedExact) && !admittedAbsFiles.has(resolvedMd)) {
const line = lineAt(linkScanText, match.index);
errors.push(`${entry.file}:${line}: broken relative Markdown link "${target}" (must resolve to a manifest-admitted guide)`)
}
}
}

for (const relFile of treeLeafByFile.keys()) {
if (!seenFiles.has(relFile)) {
errors.push(`${manifest.treeFile}: leaf "${relFile}" is missing from ${manifestRelPath}`)
}
}

for (const relFile of seenFiles) {
if (!treeLeafByFile.has(relFile)) {
errors.push(`${manifestRelPath}: "${relFile}" is not a leaf in ${manifest.treeFile}`)
}
}

if (errors.length > 0) {
throw new Error(`[public-docs] Validation failed (${errors.length} issue(s)):\n - ${errors.join('\n - ')}`)
}

return {
entryCount: manifest.entries.length,
routes : [...seenRoutes]
}
}

const isMainModule = (() => {
if (!process.argv[1]) return false;
try {
return fs.realpathSync(process.argv[1]) === fs.realpathSync(__filename)
} catch {
return path.resolve(process.argv[1]) === __filename
}
})();

if (isMainModule) {
try {
const {entryCount} = lintPublicDocs();
console.log(`[public-docs] OK — verified ${entryCount} manifest-listed guides against learn/tree.json.`)
} catch (err) {
console.error(err.message);
process.exit(1)
}
}
38 changes: 38 additions & 0 deletions docs/public-manifest.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"schema" : "public-manifest.v1",
"spoke" : "neomjs/devindex",
"treeFile" : "learn/tree.json",
"treeMount": {
"parentId": "Guides",
"id" : "guides/devindex",
"name" : "DevIndex App"
},
"entries" : [
{"file": "learn/Introduction.md", "route": "/learn/guides/devindex/Introduction"},
{"file": "learn/EthicalManifesto.md", "route": "/learn/guides/devindex/EthicalManifesto"},
{"file": "learn/Methodology.md", "route": "/learn/guides/devindex/Methodology"},
{"file": "learn/OptIn.md", "route": "/learn/guides/devindex/OptIn"},
{"file": "learn/OptOut.md", "route": "/learn/guides/devindex/OptOut"},
{"file": "learn/UserGuide.md", "route": "/learn/guides/devindex/UserGuide"},
{"file": "learn/personas/TalentScouts.md", "route": "/learn/guides/devindex/personas/TalentScouts"},
{"file": "learn/personas/DataScientists.md", "route": "/learn/guides/devindex/personas/DataScientists"},
{"file": "learn/personas/PolicyMakers.md", "route": "/learn/guides/devindex/personas/PolicyMakers"},
{"file": "learn/data-factory/Intro.md", "route": "/learn/guides/devindex/data-factory/Intro"},
{"file": "learn/data-factory/Orchestrator.md", "route": "/learn/guides/devindex/data-factory/Orchestrator"},
{"file": "learn/data-factory/Storage.md", "route": "/learn/guides/devindex/data-factory/Storage"},
{"file": "learn/data-factory/GitHubAPI.md", "route": "/learn/guides/devindex/data-factory/GitHubAPI"},
{"file": "learn/data-factory/Engine.md", "route": "/learn/guides/devindex/data-factory/Engine"},
{"file": "learn/data-factory/Updater.md", "route": "/learn/guides/devindex/data-factory/Updater"},
{"file": "learn/data-factory/DataEnrichment.md", "route": "/learn/guides/devindex/data-factory/DataEnrichment"},
{"file": "learn/data-factory/DataHygiene.md", "route": "/learn/guides/devindex/data-factory/DataHygiene"},
{"file": "learn/data-factory/OptIn.md", "route": "/learn/guides/devindex/data-factory/OptIn"},
{"file": "learn/data-factory/OptOut.md", "route": "/learn/guides/devindex/data-factory/OptOut"},
{"file": "learn/Backend.md", "route": "/learn/guides/devindex/Backend"},
{"file": "learn/frontend/Architecture.md", "route": "/learn/guides/devindex/frontend/Architecture"},
{"file": "learn/frontend/TheGrid.md", "route": "/learn/guides/devindex/frontend/TheGrid"},
{"file": "learn/frontend/StateAndControls.md", "route": "/learn/guides/devindex/frontend/StateAndControls"},
{"file": "learn/frontend/OffscreenCanvas.md", "route": "/learn/guides/devindex/frontend/OffscreenCanvas"},
{"file": "learn/frontend/ContentEngine.md", "route": "/learn/guides/devindex/frontend/ContentEngine"},
{"file": "learn/FAQ.md", "route": "/learn/guides/devindex/FAQ"}
]
}
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
"devindex:spider" : "node ./services/cli.mjs spider",
"devindex:update" : "node ./services/cli.mjs update --limit 200",
"generate-docs-json" : "node ./node_modules/neo.mjs/buildScripts/docs/jsdocx.mjs",
"lint-public-docs" : "node ./buildScripts/lintPublicDocs.mjs",
"server-start" : "webpack serve -c ./node_modules/neo.mjs/buildScripts/webpack/webpack.server.config.mjs --open",
"test-e2e" : "playwright test -c test/playwright/playwright.config.e2e.mjs",
"test-unit" : "playwright test -c test/playwright/playwright.config.unit.mjs",
Expand Down
Loading