diff --git a/scripts/verify-self-audit-proof.test.ts b/scripts/verify-self-audit-proof.test.ts index 080fea1..01f03a4 100644 --- a/scripts/verify-self-audit-proof.test.ts +++ b/scripts/verify-self-audit-proof.test.ts @@ -865,3 +865,62 @@ test("main() creates diff.txt in retain bundle when reports diverge", () => { rmSync(tmp, { recursive: true, force: true }); } }); + +// ─── PR-mined theme table is live data, not a determinism signal ───────────── + +const MINED_TABLE = (rows: string) => `${CLEAN_REPORT} +## Observed Failure Modes (PR Review Mining) + +*Why this matters:* This run analyzed 25 PRs and surfaced 4 recurring rule candidates. + +Repo: \`lambda-curry/anvil\` +PRs analyzed: 25 · Comments reviewed: 29 · Substantive comments: 29 · Candidates: 4 + +| Theme | Frequency | PR Spread | Severity | Rule Signal Match | Comment Alignment | +|-------|-----------|-----------|----------|-------------------|------------------| +${rows} +`; + +test("a new PR-mined theme row is not a determinism failure", () => { + // The break that took main red after #41: a code review pushed a fifth theme + // over its threshold between a branch build and the merge build of the same + // commit, the rows shifted, and "Documentation" was compared against a newly + // inserted "Error Handling". + const checkedIn = MINED_TABLE( + `| Naming | 9 comments | 3 PRs (medium) | low | 🟡 signal match | 100% strong | +| Documentation | 5 comments | 2 PRs (medium) | low | 🟡 signal match | 100% strong |`, + ); + const fresh = MINED_TABLE( + `| Naming | 11 comments | 5 PRs (high) | medium | 🟡 signal match | 100% strong | +| Error Handling | 3 comments | 3 PRs (medium) | medium | 🟡 signal match | 100% strong | +| Documentation | 5 comments | 2 PRs (medium) | low | 🟡 signal match | 100% strong |`, + ); + + expect(compareSelfAuditReports(checkedIn, fresh).failures).toEqual([]); +}); + +test("the mined table disappearing entirely is still a failure", () => { + // The line held deliberately: rows changing is churn and normalizes away, but + // the table vanishing means mining itself broke, and that should stay loud. + const checkedIn = MINED_TABLE( + `| Naming | 9 comments | 3 PRs (medium) | low | 🟡 signal match | 100% strong |`, + ); + const fresh = MINED_TABLE(""); + + expect( + compareSelfAuditReports(checkedIn, fresh).failures.length, + ).toBeGreaterThan(0); +}); + +test("a real scoring change is still caught through the mined table", () => { + // The guard: normalizing live PR data must not blind the proof to the + // deterministic surface it exists to protect. + const checkedIn = MINED_TABLE( + `| Naming | 9 comments | 3 PRs (medium) | low | 🟡 signal match | 100% strong |`, + ); + const fresh = checkedIn.replace("Issues found | none", "Issues found | 3"); + + expect( + compareSelfAuditReports(checkedIn, fresh).failures.length, + ).toBeGreaterThan(0); +}); diff --git a/scripts/verify-self-audit-proof.ts b/scripts/verify-self-audit-proof.ts index e5894be..f10de01 100644 --- a/scripts/verify-self-audit-proof.ts +++ b/scripts/verify-self-audit-proof.ts @@ -53,6 +53,18 @@ type CliOptions = { retainDir: string | null; }; +/** One row of the PR-mined theme table: `| Naming | 9 comments | 3 PRs (medium) | low | … |`. */ +const MINED_THEME_ROW_PATTERN = + /^\| .+? \| \d+ comments \| \d+ PRs \([a-z]+\) \| [a-z]+ \|.*$/gm; + +const MINED_THEME_ROW = "| |"; + +/** Consecutive placeholder rows, so the NUMBER of themes stops mattering too. */ +const COLLAPSE_MINED_THEME_ROWS = new RegExp( + `(?:${MINED_THEME_ROW.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`)}\n?)+`, + "g", +); + function normalizeVolatileReportFields(reportText: string): string { return ( reportText @@ -73,10 +85,16 @@ function normalizeVolatileReportFields(reportText: string): string { // already normalized. Anvil mines its own PR history, so these move whenever anyone opens // a PR here and the packet needs a hand refresh: 20->22, then 22->24 days later. // Normalizing them ends the chore. - .replaceAll( - /^\| (.+?) \| \d+ comments \| \d+ PRs \([a-z]+\) \| [a-z]+ \|/gm, - "| $1 | comments | PRs () | |", - ) + // + // The theme SET is volatile too, not only the counts inside each row. Normalizing just the + // numbers still broke the moment a code review pushed a fifth theme over its threshold: + // the rows shifted, and "Documentation" was compared against a freshly inserted "Error + // Handling". Which themes appear is derived wholly from live GitHub comments and can differ + // between two runs minutes apart — a branch build and the merge build of the same commit + // disagreed exactly that way. So the block collapses to one placeholder. What this proof + // exists to protect — scores, stages, checks, every deterministic line — is untouched. + .replaceAll(MINED_THEME_ROW_PATTERN, MINED_THEME_ROW) + .replace(COLLAPSE_MINED_THEME_ROWS, `${MINED_THEME_ROW}\n`) ); }