Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions scripts/verify-self-audit-proof.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -865,3 +865,62 @@ test("main() creates diff.txt in retain bundle when reports diverge", () => {
rmSync(tmp, { recursive: true, force: true });
}
});

// ─── PR-mined theme table is live data, not a determinism signal ─────────────

const MINED_TABLE = (rows: string) => `${CLEAN_REPORT}
## Observed Failure Modes (PR Review Mining)

*Why this matters:* This run analyzed 25 PRs and surfaced 4 recurring rule candidates.

Repo: \`lambda-curry/anvil\`
PRs analyzed: 25 · Comments reviewed: 29 · Substantive comments: 29 · Candidates: 4

| Theme | Frequency | PR Spread | Severity | Rule Signal Match | Comment Alignment |
|-------|-----------|-----------|----------|-------------------|------------------|
${rows}
`;

test("a new PR-mined theme row is not a determinism failure", () => {
// The break that took main red after #41: a code review pushed a fifth theme
// over its threshold between a branch build and the merge build of the same
// commit, the rows shifted, and "Documentation" was compared against a newly
// inserted "Error Handling".
const checkedIn = MINED_TABLE(
`| Naming | 9 comments | 3 PRs (medium) | low | 🟡 signal match | 100% strong |
| Documentation | 5 comments | 2 PRs (medium) | low | 🟡 signal match | 100% strong |`,
);
const fresh = MINED_TABLE(
`| Naming | 11 comments | 5 PRs (high) | medium | 🟡 signal match | 100% strong |
| Error Handling | 3 comments | 3 PRs (medium) | medium | 🟡 signal match | 100% strong |
| Documentation | 5 comments | 2 PRs (medium) | low | 🟡 signal match | 100% strong |`,
);

expect(compareSelfAuditReports(checkedIn, fresh).failures).toEqual([]);
});

test("the mined table disappearing entirely is still a failure", () => {
// The line held deliberately: rows changing is churn and normalizes away, but
// the table vanishing means mining itself broke, and that should stay loud.
const checkedIn = MINED_TABLE(
`| Naming | 9 comments | 3 PRs (medium) | low | 🟡 signal match | 100% strong |`,
);
const fresh = MINED_TABLE("");

expect(
compareSelfAuditReports(checkedIn, fresh).failures.length,
).toBeGreaterThan(0);
});

test("a real scoring change is still caught through the mined table", () => {
// The guard: normalizing live PR data must not blind the proof to the
// deterministic surface it exists to protect.
const checkedIn = MINED_TABLE(
`| Naming | 9 comments | 3 PRs (medium) | low | 🟡 signal match | 100% strong |`,
);
const fresh = checkedIn.replace("Issues found | none", "Issues found | 3");

expect(
compareSelfAuditReports(checkedIn, fresh).failures.length,
).toBeGreaterThan(0);
});
26 changes: 22 additions & 4 deletions scripts/verify-self-audit-proof.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,18 @@ type CliOptions = {
retainDir: string | null;
};

/** One row of the PR-mined theme table: `| Naming | 9 comments | 3 PRs (medium) | low | … |`. */
const MINED_THEME_ROW_PATTERN =
/^\| .+? \| \d+ comments \| \d+ PRs \([a-z]+\) \| [a-z]+ \|.*$/gm;

const MINED_THEME_ROW = "| <normalized PR-mined theme row> |";

/** Consecutive placeholder rows, so the NUMBER of themes stops mattering too. */
const COLLAPSE_MINED_THEME_ROWS = new RegExp(
`(?:${MINED_THEME_ROW.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`)}\n?)+`,
"g",
);

function normalizeVolatileReportFields(reportText: string): string {
return (
reportText
Expand All @@ -73,10 +85,16 @@ function normalizeVolatileReportFields(reportText: string): string {
// already normalized. Anvil mines its own PR history, so these move whenever anyone opens
// a PR here and the packet needs a hand refresh: 20->22, then 22->24 days later.
// Normalizing them ends the chore.
.replaceAll(
/^\| (.+?) \| \d+ comments \| \d+ PRs \([a-z]+\) \| [a-z]+ \|/gm,
"| $1 | <normalized> comments | <normalized> PRs (<normalized>) | <normalized> |",
)
//
// The theme SET is volatile too, not only the counts inside each row. Normalizing just the
// numbers still broke the moment a code review pushed a fifth theme over its threshold:
// the rows shifted, and "Documentation" was compared against a freshly inserted "Error
// Handling". Which themes appear is derived wholly from live GitHub comments and can differ
// between two runs minutes apart — a branch build and the merge build of the same commit
// disagreed exactly that way. So the block collapses to one placeholder. What this proof
// exists to protect — scores, stages, checks, every deterministic line — is untouched.
.replaceAll(MINED_THEME_ROW_PATTERN, MINED_THEME_ROW)
.replace(COLLAPSE_MINED_THEME_ROWS, `${MINED_THEME_ROW}\n`)
);
}

Expand Down