diff --git a/.changeset/strictness-ledger-gate.md b/.changeset/strictness-ledger-gate.md new file mode 100644 index 0000000000..782222b1db --- /dev/null +++ b/.changeset/strictness-ledger-gate.md @@ -0,0 +1,34 @@ +--- +"@objectstack/spec": patch +--- + +Add a gate that holds the #4001 strictness ledger to the code it describes. + +`docs/audits/2026-07-unknown-key-strictness-ledger.md` is the campaign's map: +which `z.object` sites are authorable (the `.strict()` ratchet target), which are +wire, which are deliberately open. Every step reads it to pick the next move, and +nothing kept it honest. It went stale twice in one week — a classification that +verification disproved (`hook.zod.ts`, #4207) and a "next step" that had shipped +months earlier (the warning layer, #4218). A map that drifts is worse than no +map, because it gets followed. + +`check:strictness-ledger` enforces the claims in it that are mechanically +checkable: + +- **Site counts** — the ledger states its own method (`z.object(` occurrences per + file), so every number is verifiable. A stale count means schemas were added or + removed under a `Class` verdict nobody re-examined. +- **Coverage** — every `*.zod.ts` with sites in a triaged directory must have a + row. Zero-site files (pure enum/token modules) are skipped, and become + reportable the day they grow their first `z.object(`. +- **Section totals**, and that a row claiming "strict as of" names a file that + really contains `.strict()`. + +The `Class` column is deliberately not checked — authorable vs wire vs open is a +human judgement, and the campaign's rule is verify-before-tightening. The gate +protects the arithmetic and coverage so that judgement is made against current +code. + +First run found 11 drifts, including six moved counts (`ui/app.zod.ts` had gone +11 → 18), two unbalanced section totals, and one genuinely unclassified file +(`automation/io-node-config.zod.ts`). All corrected here. diff --git a/.github/workflows/spec-liveness-check.yml b/.github/workflows/spec-liveness-check.yml index ace1d49945..997e7a4d32 100644 --- a/.github/workflows/spec-liveness-check.yml +++ b/.github/workflows/spec-liveness-check.yml @@ -21,6 +21,9 @@ on: # The variant/doc gate reads hand-written pages, so editing one can break it # without touching packages/spec at all. - 'content/docs/**' + # Same for the strictness ledger — it is a doc, and editing it can break + # the gate that now holds it to the code. + - 'docs/audits/**' permissions: contents: read @@ -61,3 +64,12 @@ jobs: # apps.mdx said the nav tree had "eight item types" while the schema had nine. - name: Check discriminated-union variants are documented run: pnpm --filter @objectstack/spec check:variant-docs + + # #4001 follow-up. The strictness ledger is the campaign's map — which + # z.object sites are authorable, wire or open. Every step reads it to pick + # the next move, and nothing kept it honest: it went stale twice in one + # week (a classification verification disproved, and a "next step" that had + # shipped months earlier). A map that drifts is worse than no map, because + # it gets followed. This holds its arithmetic and its coverage to the code. + - name: Check the strictness ledger matches the code + run: pnpm --filter @objectstack/spec check:strictness-ledger diff --git a/docs/audits/2026-07-unknown-key-strictness-ledger.md b/docs/audits/2026-07-unknown-key-strictness-ledger.md index a7ff9942e8..7d9e1cd122 100644 --- a/docs/audits/2026-07-unknown-key-strictness-ledger.md +++ b/docs/audits/2026-07-unknown-key-strictness-ledger.md @@ -129,24 +129,24 @@ Classification is per the rule above; **(p)** marks a provisional call made from the file's exports/JSDoc rather than a full read — verify before tightening (the #4001 "sharing-rule lesson": candidates, not verdicts). -### `ui/` — 192 sites +### `ui/` — 197 sites | File | Sites | Class | Note / next action | |---|---|---|---| -| `action.zod.ts` | 8 | authorable | param schema strict (#3746); remaining blocks ride later steps | -| `view.zod.ts` | 51 | authorable | partially strict (ADR-0089); long tail of sub-blocks | +| `action.zod.ts` | 9 | authorable | param schema strict (#3746); remaining blocks ride later steps | +| `view.zod.ts` | 50 | authorable | partially strict (ADR-0089); long tail of sub-blocks | | `component.zod.ts` | 29 | authorable | **next candidate** — SDUI component defs; check React-prop open slots first (p) | | `theme.zod.ts` | 14 | authorable (p) | authored themes | -| `app.zod.ts` | 11 | authorable | **strict as of #4001 PR B** — `AppSchema` + branding / area / context-selector / contribution, and the nav-item union converted to `z.discriminatedUnion('type', …)` (the union-error question, settled empirically: matched-branch-only errors, exact recursive paths, `toJSONSchema` clean). Per-target `params` stay open. PR A (#4142) tombstoned the seven audit-dead keys first | +| `app.zod.ts` | 18 | authorable | **strict as of #4001 PR B** — `AppSchema` + branding / area / context-selector / contribution, and the nav-item union converted to `z.discriminatedUnion('type', …)` (the union-error question, settled empirically: matched-branch-only errors, exact recursive paths, `toJSONSchema` clean). Per-target `params` stay open. PR A (#4142) tombstoned the seven audit-dead keys first | | `dashboard.zod.ts` | 11 | authorable | partially strict | | `widget.zod.ts` | 9 | authorable (p) | | | `page.zod.ts` | 7 | authorable | partially strict (ADR-0089) | -| `chart.zod.ts` / `i18n.zod.ts` / `responsive.zod.ts` | 6+6+6 | authorable (p) | i18n label shapes are wide-open records by design — verify | -| `dataset.zod.ts` / `animation.zod.ts` / `dnd.zod.ts` / `keyboard.zod.ts` / `touch.zod.ts` | 4 ea | authorable (p) | interaction configs | +| `chart.zod.ts` / `i18n.zod.ts` / `responsive.zod.ts` | 6+6+4 | authorable (p) | i18n label shapes are wide-open records by design — verify | +| `dataset.zod.ts` / `animation.zod.ts` / `dnd.zod.ts` / `keyboard.zod.ts` / `touch.zod.ts` | 4+4+4+4+7 | authorable (p) | interaction configs | | `notification.zod.ts` / `offline.zod.ts` / `report.zod.ts` | 3 ea | authorable (p) | | | `sharing.zod.ts` | 2 | authorable (p) | public-sharing config | -### `data/` — 163 sites +### `data/` — 154 sites | File | Sites | Class | Note | |---|---|---|---| @@ -165,11 +165,11 @@ tightening (the #4001 "sharing-rule lesson": candidates, not verdicts). | `external-catalog.zod.ts` | 4 | wire (p) | | | `field-value.zod.ts` / `seed.zod.ts` / `validation.zod.ts` | 1 ea | mixed (p) | | -### `automation/` — 80 sites +### `automation/` — 89 sites | File | Sites | Class | Note | |---|---|---|---| -| `flow.zod.ts` | 12 | authorable | **strict as of #4001** (4 schemas; `FlowVersionHistorySchema` is runtime — stays tolerant) | +| `flow.zod.ts` | 11 | authorable | **strict as of #4001** (4 schemas; `FlowVersionHistorySchema` is runtime — stays tolerant) | | `sync.zod.ts` / `etl.zod.ts` | 12+10 | authorable (p) | authored pipelines — **candidates** | | `trigger-registry.zod.ts` | 11 | mixed | descriptors are code-registered (wire-ish); bindings authored | | `execution.zod.ts` | 8 | wire | run-state envelopes — never strict | @@ -178,6 +178,8 @@ tightening (the #4001 "sharing-rule lesson": candidates, not verdicts). | `bpmn-interop.zod.ts` | 5 | wire (p) | interop import shapes | | `approval.zod.ts` | 4 | authorable | **strict as of #4001 step 3** — all four authoring schemas (node config / approver / escalation / decision-output). The published JSON schema carries `additionalProperties: false` into the Studio form AND `registerFlow()` config validation (#4027/#4040), so an unknown key in an approval node's `config` is rejected at registration too — verified: `z.toJSONSchema` on the strict lazySchema does not throw (#3746 hazard checked) | | `node-executor.zod.ts` | 4 | wire | executor contract | +| `io-node-config.zod.ts` | 2 | authorable | `NotifyConfigSchema` / `HttpConfigSchema` (#4045) — the sibling contracts that validate the **open** `config` slot on flow `notify` / `http` nodes. Authored per-node, so the open-slot exemption above does not extend to them; candidate once the executors' own drift is verified | +| `builtin-node-config.zod.ts` | 8 | authorable | Same family (#4045): the CRUD quartet, `screen`, `map`. Written from what the executors read rather than from the descriptors' `configSchema` literals, and reconciled bidirectionally by `builtin-node-form-zod-ledger.test.ts` — so unlike most rows here, this one already has a drift check of its own. Same candidacy note as `io-node-config` | | `webhook.zod.ts` | 1 | authorable (p) | spec-only (#3461) | ### `security/` — 20 sites @@ -213,10 +215,7 @@ tightening (the #4001 "sharing-rule lesson": candidates, not verdicts). ## Next steps (verify-then-enforce, one shape at a time) -1. Promote this ledger to a machine-checked gate (pattern of - `packages/spec/liveness/` + `check:liveness`) once enough of the surface is - classified that the table above is enforceable rather than descriptive. -2. Let the warning layer run in the wild for a release, then schedule the v18 +1. Let the warning layer run in the wild for a release, then schedule the v18 strict close-out on what it actually reports — which is the whole point of having built it. Nothing more to do here until there is field data. @@ -292,5 +291,49 @@ apps (23 + 6 objects, 20 + 1 pages, 4 + 1 datasets, 3 + 1 dashboards) report **zero** unknown keys. No finding this time — worth recording precisely because the app step's `ACCOUNT_APP.defaultOpen` came from exactly this class of check. +## This file is now machine-checked + +`pnpm --filter @objectstack/spec check:strictness-ledger` (wired into the Spec +Liveness Check workflow) holds the two claims here that are mechanically +checkable, so this map cannot go stale in silence again: + +- **Site counts.** The method is stated above — `z.object(` occurrences per file + — so every number in the triage tables is verifiable. A count that no longer + matches means schemas were added or removed under a `Class` verdict nobody + re-examined. Touching a file forces you back through this ledger. +- **Coverage.** Every `*.zod.ts` in a triaged directory that HAS sites must have + a row. A new one is undeclared surface. Zero-site files (pure enum/token + modules like `data/date-macros.zod.ts`) are skipped — there is nothing to + classify — and become reportable the day they grow their first `z.object(`. +- **Section totals**, and that any row claiming "strict as of" names a file that + really contains `.strict()`. + +Deliberately NOT checked: the `Class` column. Authorable vs wire vs open is a +judgement about who writes the input, and this campaign's rule is +verify-before-tightening. The gate protects the arithmetic and the coverage so +that judgement is always made against current code. + +**What it found on its first run — 11 drifts, in a file being actively edited +by the campaign that owns it.** Six counts had moved (`ui/app.zod.ts` 11 → 18, +`ui/touch.zod.ts` 4 → 7, `ui/action.zod.ts` 8 → 9, `ui/view.zod.ts` 51 → 50, +`ui/responsive.zod.ts` 6 → 4, `automation/flow.zod.ts` 12 → 11), two section +totals no longer summed, and six files had no row at all. The `app.zod.ts` gap +was **self-inflicted**: the app step (#4165) added seven schemas to that file +and updated the row's prose without touching its count. Five of the six +undeclared files turned out to have zero sites — which is what motivated the +skip rule above — leaving `automation/io-node-config.zod.ts` as the one genuine +omission, now classified. + +That is the argument for the gate in one paragraph: the people most familiar +with this ledger, editing it in the same week, still left eleven drifts in it. + +**And then it worked for real, before it had even merged.** While the gate sat +in review, `main` landed `automation/builtin-node-config.zod.ts` (#4045/#4228) — +eight new sites, sibling to `io-node-config.zod.ts`. Merging `main` turned the +gate red on a branch whose own diff had not touched a single schema, which is +precisely the intended behaviour: the file arrived, so someone had to classify +it. It is now a row. Every existing count survived that merge unchanged, so the +failure was exactly as narrow as it should have been. + Long tail stays gated on a verification pass per shape — never a one-shot "make all ~453 sites strict" (ADR-0054 ratchet; #4001's own recommendation). diff --git a/packages/spec/package.json b/packages/spec/package.json index 035222023b..fb6045179f 100644 --- a/packages/spec/package.json +++ b/packages/spec/package.json @@ -210,6 +210,7 @@ "check:liveness": "tsx scripts/liveness/check-liveness.mts", "check:empty-state": "tsx scripts/liveness/check-empty-state.mts", "check:variant-docs": "tsx scripts/check-variant-docs.mts", + "check:strictness-ledger": "tsx scripts/check-strictness-ledger.mts", "gen:react-blocks": "tsx scripts/build-react-blocks-contract.ts", "check:react-blocks": "tsx scripts/build-react-blocks-contract.ts --check", "check:react-conformance": "tsx scripts/check-react-blocks-conformance.ts", diff --git a/packages/spec/scripts/check-strictness-ledger.mts b/packages/spec/scripts/check-strictness-ledger.mts new file mode 100644 index 0000000000..ba9cf0f7f1 --- /dev/null +++ b/packages/spec/scripts/check-strictness-ledger.mts @@ -0,0 +1,208 @@ +#!/usr/bin/env tsx +// Strictness-ledger gate — makes docs/audits/2026-07-unknown-key-strictness-ledger.md +// enforceable instead of merely descriptive. +// +// WHY THIS EXISTS. The ledger is the #4001 campaign's map: which `z.object` sites +// are authorable (the `.strict()` ratchet target), which are wire, which are +// deliberately open. Every step reads it to decide what to do next. Nothing kept it +// honest, and it went stale twice in one week: +// +// - `hook.zod.ts` carried a blanket `authorable (p)` that verification disproved — +// HookContextSchema in the same file is a runtime shape (#4207). +// - It listed "build the unknown-key WARNING layer" as the next step months after +// that layer shipped, sending the next reader off to rebuild it (#4218). +// +// A map that drifts is worse than no map: it is followed. This gate holds the two +// claims in it that are mechanically checkable. +// +// WHAT IT CHECKS +// 1. Site counts. The ledger states its own method — "site counts are `z.object(` +// occurrences per file" — so every count is verifiable. A count that no longer +// matches means someone added or removed a schema without reclassifying it, and +// the row's `Class` verdict now covers sites nobody triaged. This is the ratchet: +// touching a file forces you back through the ledger. +// 2. Coverage. Every `*.zod.ts` under a triaged directory that HAS `z.object(` sites +// must appear in that directory's table. A new one is undeclared surface — +// exactly what the ledger exists to prevent. Files with zero sites (pure enum / +// token modules like `data/date-macros.zod.ts`) are skipped: the ledger classifies +// sites, and they have none to classify. This is not a hole — the day such a file +// grows its first `z.object(` it becomes undeclared and this gate says so. +// 3. Section totals. `### \`ui/\` — 192 sites` must equal the sum of its rows. +// Cheap, and it catches a row edited without updating the header. +// 4. Strictness claims. A row whose note says "strict as of" must name a file that +// actually contains `.strict()`. This is deliberately weak — it proves the claim +// is not fiction, not that every site in the file is strict. Rows say things like +// "partially strict"; encoding which sites those are would need a second ledger, +// and the per-schema truth already lives in the code the note points at. +// +// WHAT IT DOES NOT CHECK. The `Class` column itself — authorable vs wire vs open is a +// human judgement about who writes the input, and the campaign's own rule is +// verify-before-tightening. This gate protects the ledger's ARITHMETIC and its +// COVERAGE so that judgement is made against current code. +// +// Usage: +// tsx check-strictness-ledger.mts # fail on drift +// tsx check-strictness-ledger.mts --list # print the parsed ledger + +import fs from 'node:fs'; +import path from 'node:path'; +import url from 'node:url'; + +const HERE = path.dirname(url.fileURLToPath(import.meta.url)); +const SPEC = path.resolve(HERE, '..'); +const REPO = path.resolve(SPEC, '../..'); +const LEDGER = path.join(REPO, 'docs/audits/2026-07-unknown-key-strictness-ledger.md'); +const SRC = path.join(SPEC, 'src'); +const LIST = process.argv.includes('--list'); + +interface Row { + dir: string; + files: string[]; + /** Declared `z.object(` count per file, index-aligned with `files`. */ + counts: number[]; + note: string; + line: number; +} + +/** + * Parse `8`, `6+2`, `10+9+2` or `4 ea` against a file list. + * `N ea` means every file in the row declares N — the ledger's shorthand for a + * run of same-sized files. + */ +function parseCounts(cell: string, fileCount: number): number[] | null { + const ea = cell.match(/^(\d+)\s*ea$/); + if (ea) return Array(fileCount).fill(Number(ea[1])); + const parts = cell.split('+').map((p) => p.trim()); + if (!parts.every((p) => /^\d+$/.test(p))) return null; + const nums = parts.map(Number); + if (nums.length === 1 && fileCount > 1) return null; + return nums.length === fileCount ? nums : null; +} + +/** `z.object(` occurrences — the ledger's own stated counting method. */ +function countSites(file: string): number { + return (fs.readFileSync(file, 'utf-8').match(/z\.object\(/g) ?? []).length; +} + +const md = fs.readFileSync(LEDGER, 'utf-8').split('\n'); +const rows: Row[] = []; +const sectionTotals = new Map(); +let dir: string | null = null; + +for (let i = 0; i < md.length; i++) { + const line = md[i]; + + const header = line.match(/^### `([a-z-]+)\/` — (\d+) sites/); + if (header) { + dir = header[1]; + sectionTotals.set(dir, { declared: Number(header[2]), line: i + 1 }); + continue; + } + // Any other h2/h3 closes the current section, so prose tables below the + // triage cannot be mistaken for rows. + if (/^##/.test(line) && !header) dir = null; + if (!dir || !line.startsWith('|')) continue; + + const cells = line.split('|').slice(1, -1).map((c) => c.trim()); + if (cells.length < 3) continue; + const files = [...cells[0].matchAll(/`([^`]+\.zod\.ts)`/g)].map((m) => m[1]); + if (files.length === 0) continue; + + const counts = parseCounts(cells[1], files.length); + if (counts === null) { + console.error(`✗ ${LEDGER}:${i + 1} — cannot parse site count "${cells[1]}" for ${files.length} file(s)`); + process.exit(1); + } + rows.push({ dir, files, counts, note: cells.slice(2).join(' '), line: i + 1 }); +} + +if (LIST) { + for (const r of rows) { + console.log(`${r.dir}/ ${r.files.map((f, i) => `${f}=${r.counts[i]}`).join(' ')}`); + } + process.exit(0); +} + +const errors: string[] = []; +const declaredByDir = new Map>(); + +for (const row of rows) { + const declared = declaredByDir.get(row.dir) ?? new Set(); + declaredByDir.set(row.dir, declared); + + row.files.forEach((file, idx) => { + declared.add(file); + const abs = path.join(SRC, row.dir, file); + if (!fs.existsSync(abs)) { + errors.push( + `ledger:${row.line} — lists \`${row.dir}/${file}\`, which does not exist.\n` + + ` → the file moved or was deleted; update or drop the row.`, + ); + return; + } + const actual = countSites(abs); + if (actual !== row.counts[idx]) { + errors.push( + `ledger:${row.line} — \`${row.dir}/${file}\` declares ${row.counts[idx]} site(s), found ${actual}.\n` + + ` → ${actual > row.counts[idx] ? 'new' : 'removed'} \`z.object(\` site(s) since the row was written. ` + + `Re-read the file, confirm the Class verdict still covers it, and update the count.`, + ); + } + }); + + if (/strict as of/i.test(row.note)) { + const anyStrict = row.files.some((f) => { + const abs = path.join(SRC, row.dir, f); + return fs.existsSync(abs) && /\.strict\(\)/.test(fs.readFileSync(abs, 'utf-8')); + }); + if (!anyStrict) { + errors.push( + `ledger:${row.line} — row claims "strict as of" but no listed file contains \`.strict()\`:\n` + + ` ${row.files.map((f) => `${row.dir}/${f}`).join(', ')}\n` + + ` → the claim is stale; the ratchet was reverted or the schema moved.`, + ); + } + } +} + +// Coverage ratchet — a new .zod.ts in a triaged directory is undeclared surface. +for (const [d, declared] of declaredByDir) { + const dirPath = path.join(SRC, d); + if (!fs.existsSync(dirPath)) continue; + const onDisk = fs.readdirSync(dirPath).filter((f) => f.endsWith('.zod.ts')); + // Zero-site files carry nothing to classify (see the header note). They become + // reportable the moment they grow a `z.object(`. + const missing = onDisk.filter((f) => !declared.has(f) && countSites(path.join(dirPath, f)) > 0); + if (missing.length) { + errors.push( + `\`${d}/\` has ${missing.length} undeclared schema file(s) with sites: ` + + `${missing.map((f) => `${f} (${countSites(path.join(dirPath, f))})`).join(', ')}\n` + + ` → add a row to the ${d}/ table with its site count and a Class verdict ` + + `(authorable / wire / open — see the classification rule at the top of the ledger).`, + ); + } +} + +// Section arithmetic. +for (const [d, { declared, line }] of sectionTotals) { + const sum = rows.filter((r) => r.dir === d).reduce((a, r) => a + r.counts.reduce((x, y) => x + y, 0), 0); + if (sum !== declared) { + errors.push( + `ledger:${line} — \`${d}/\` header says ${declared} sites, rows sum to ${sum}.\n` + + ` → update the header to match the rows.`, + ); + } +} + +if (errors.length) { + console.error(`\n✗ strictness ledger: ${errors.length} drift(s)\n`); + for (const e of errors) console.error(` ${e}\n`); + console.error(` The ledger is ${path.relative(REPO, LEDGER)}.\n`); + process.exit(1); +} + +const fileCount = rows.reduce((a, r) => a + r.files.length, 0); +console.log( + `✓ strictness ledger: ${fileCount} file(s) across ${sectionTotals.size} triaged director(ies) — ` + + `site counts match, no undeclared schema files, section totals balance.`, +);