From d2e4effa1511ee92e409804b1343ebd1360200bb Mon Sep 17 00:00:00 2001 From: Jake Ruesink Date: Sun, 9 Aug 2026 08:26:41 -0500 Subject: [PATCH 1/3] fix(audit): a pointer is not a rule document, so stop scoring it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit robinhood-trader reported Low-Yield 1/2. The failing file is its 10-line CLAUDE.md, an `@AGENTS.md` shim — a redirect to the canonical document, which carries no rules of its own. Demanding rationale and examples of a redirect asks the maintainer to pad it. Pointers now leave the Low-Yield DENOMINATOR rather than being counted and forgiven, because they were never rule documents to score. robinhood-trader goes 1/2 -> 0/1. Scoped to Low-Yield deliberately, and this is the interesting constraint: the same file still counts toward Context Load Pressure, which held at 343 lines across the change. A shim is additive there — Claude loads the shim AND the file it imports — so a blanket removal from the scoring surface would have silently undercounted load by 11 lines. Filtering the whole surface was my first instinct and it was wrong. The signal is the import, not the length. A regression test pairs an `@AGENTS.md` shim with a non-pointer file of the same size that has neither Why nor examples; the first leaves the denominator (0/1), the second stays and fails (1/2). On the documentRole consolidation: no, not now, and the code is why. See the report for the full argument, but in short — the four predicates are consulted at four different stages and the same role earns OPPOSITE treatment across checks, so a shared enum would unify the classification while leaving four distinct policies behind. isSymlinkAlias is also consumed by exactly one caller (drift), while audit reaches the same outcome by fingerprint dedup, so folding them together would change audit behaviour nobody asked to change. What the code DID agree with is smaller and real: importsItsMirrorSource (audit.ts) and importsRootMirror (rule-loading.ts) were two implementations of the same regex, one re-reading the file from disk on every call. I introduced the second in #39 without collapsing the first. Both now delegate to isPointerDocument in scripts/lib/document-role.ts, which names the role without committing to the full refactor — the seam a future documentRole would grow from, if the fifth case ever justifies it. Corpus census across the same 138 files used for alpha.11: exactly ONE pointer document exists fleet-wide, robinhood-trader/CLAUDE.md. The per-file hasWhy pass rate is unchanged at 61/138 (44%), exactly as it should be — this change alters set membership, not per-file classification, so the alpha.11 instrument is expected to be flat here and its flatness is the evidence that nothing was loosened. Co-Authored-By: Claude Opus 5 (1M context) --- docs/rubric.md | 2 + scripts/audit.ts | 25 ++++--- scripts/lib/document-role.ts | 26 ++++++++ scripts/lib/rule-loading.ts | 13 ++-- scripts/shim-denominator.test.ts | 108 +++++++++++++++++++++++++++++++ 5 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 scripts/lib/document-role.ts create mode 100644 scripts/shim-denominator.test.ts diff --git a/docs/rubric.md b/docs/rubric.md index c714de5..e44c289 100644 --- a/docs/rubric.md +++ b/docs/rubric.md @@ -170,6 +170,8 @@ Globs: [if glob-matched] / alwaysApply: [true/false] / on-demand: [how to pull] The two paths cover different genres, and short files depend on the first. A 35-line Cursor `.mdc` rule states its rationale in one dense paragraph, which rarely trips two separate prose families — the label is what carries it. Long-form instruction files usually pass on prose alone. **You need one of the two, not both.** +*Pointers are not scored.* A `CLAUDE.md` whose body is an `@AGENTS.md` import is a redirect to the canonical document, not a rule document, so it leaves the Low-Yield denominator entirely rather than being counted and forgiven — requiring rationale and examples of a redirect would only invite padding it. The signal is the import, not the length: a genuinely thin rule document of the same size is still scored, and still fails. Such a file does still count toward Context Load Pressure, because Claude loads the shim in addition to the file it imports. + **Examples are load-bearing.** A rule without examples is a hypothesis. Examples demonstrate the failure mode in a form the model can pattern-match against. **Imperative voice.** Rules give instructions. Use "Use X" not "X should be used." Use "Never modify Y" not "Y shouldn't be modified." diff --git a/scripts/audit.ts b/scripts/audit.ts index f4a20f6..6a479c0 100644 --- a/scripts/audit.ts +++ b/scripts/audit.ts @@ -1313,11 +1313,9 @@ function classifyMirrorStatus( * the maintainer to "repair" a working cross-tool contract. */ function importsItsMirrorSource(file: RuleFile): boolean { - try { - return /^\s*@AGENTS\.md\s*$/m.test(readFileSync(file.path, "utf8")); - } catch { - return false; - } + // Classified once at parse time; this used to re-read the file from disk with + // a second copy of the same regex. + return file.importsRootMirror; } function comparableMirrorFingerprint(file: RuleFile): string { @@ -2143,10 +2141,19 @@ export function assessStageD( ): { stage: StageResult; metrics: OverkillMetrics } { const load = summarizeLoad(countableLoadFiles(scoringRuleFiles)); const alwaysOnLines = load.alwaysOnLines; - const lowYieldRules = scoringRuleFiles.filter( + // A pointer redirects to the canonical document; it carries no rules, so + // asking it for rationale and examples asks the maintainer to pad a redirect. + // It leaves the denominator, not just the numerator — it was never a rule + // document to score. Note this is scoped to Low-Yield deliberately: the same + // file still counts toward Context Load Pressure, because Claude really does + // load the shim in addition to what it imports. + const lowYieldCandidates = scoringRuleFiles.filter( + (ruleFile) => !ruleFile.importsRootMirror, + ); + const lowYieldRules = lowYieldCandidates.filter( (ruleFile) => !ruleFile.hasWhySection || !ruleFile.hasExamplesSection, ).length; - const lowYieldRatio = ratio(lowYieldRules, scoringRuleFiles.length); + const lowYieldRatio = ratio(lowYieldRules, lowYieldCandidates.length); const keywordConflicts = detectKeywordConflicts(scoringRuleFiles); const redundancyPressure = clamp01(inventory.accidentalDuplicationRate * 3); @@ -2220,8 +2227,8 @@ export function assessStageD( label: "Low-Yield Rule Ratio", status: lowYieldStatus, detail: toolNativeAdvisoryLowYield - ? `${lowYieldRules}/${scoringRuleFiles.length || 0} scoring files miss Why or Examples; tool-native-first surface keeps this advisory while duplication/conflict/load stay healthy` - : `${lowYieldRules}/${scoringRuleFiles.length || 0} scoring files miss Why or Examples`, + ? `${lowYieldRules}/${lowYieldCandidates.length || 0} scoring files miss Why or Examples; tool-native-first surface keeps this advisory while duplication/conflict/load stay healthy` + : `${lowYieldRules}/${lowYieldCandidates.length || 0} scoring files miss Why or Examples`, }, ]; diff --git a/scripts/lib/document-role.ts b/scripts/lib/document-role.ts new file mode 100644 index 0000000..073795f --- /dev/null +++ b/scripts/lib/document-role.ts @@ -0,0 +1,26 @@ +/** + * What *kind* of document a rule file is, as distinct from what any check does + * with it. + * + * Anvil has accumulated several "this file is not really a rule document" + * predicates — symlink aliases, agent-workspace personas, generated mirror + * twins, and pointers. They are not interchangeable, and deliberately do not + * live behind one shared policy: the same role earns opposite treatment in + * different checks. A pointer is *excluded* from Low-Yield because it is not a + * rule document, and simultaneously *included* in Context Load Pressure because + * Claude genuinely loads it in addition to the file it imports. + * + * So this module names roles. It does not decide what to do about them. + */ + +/** + * A file whose body redirects to the canonical document rather than carrying + * rules itself — the sanctioned alternative to a symlink, used where Claude + * needs extras that Codex must not receive. + * + * Pointer-ness is about the import, not about length: a genuinely thin 11-line + * rule document is still a rule document and still has to earn its keep. + */ +export function isPointerDocument(content: string): boolean { + return /^\s*@AGENTS\.md\s*$/m.test(content); +} diff --git a/scripts/lib/rule-loading.ts b/scripts/lib/rule-loading.ts index 184dea6..0d0a269 100644 --- a/scripts/lib/rule-loading.ts +++ b/scripts/lib/rule-loading.ts @@ -25,6 +25,8 @@ * what gets loaded — not a smaller number. */ +import { isPointerDocument } from "./document-role.ts"; + /** Frontmatter keys that scope a rule to a subset of files. */ const SCOPE_KEYS = [ "paths", @@ -150,10 +152,13 @@ export type TieredFile = { importsRootMirror?: boolean; }; -/** Whether a file's body imports its mirror source rather than copying it. */ -export function importsRootMirror(content: string): boolean { - return /^\s*@AGENTS\.md\s*$/m.test(content); -} +/** + * Whether a file's body imports its mirror source rather than copying it. + * + * Same question as {@link isPointerDocument}; kept as a named re-export so the + * loading model reads in its own vocabulary. + */ +export const importsRootMirror = isPointerDocument; export function summarizeLoad(files: TieredFile[]): LoadBreakdown { let alwaysOn = 0; diff --git a/scripts/shim-denominator.test.ts b/scripts/shim-denominator.test.ts new file mode 100644 index 0000000..58b7e42 --- /dev/null +++ b/scripts/shim-denominator.test.ts @@ -0,0 +1,108 @@ +import { afterAll, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +import { isPointerDocument } from "./lib/document-role.ts"; + +const REPO_ROOT = resolve(import.meta.dir, ".."); +const CLI = resolve(REPO_ROOT, "bin/anvil.ts"); +const created: string[] = []; + +afterAll(() => { + for (const dir of created) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +/** A complete rule document that passes Low-Yield on its own. */ +const CANONICAL = `# Working in this repo + +Last validated: 2026-08-09 + +**Why:** we pin the toolchain because a floating version broke the build twice, +and the breakage only surfaced in the deploy step. + +**DO** +\`\`\`bash +bun install --frozen-lockfile +\`\`\` + +**DON'T** +\`\`\`bash +npm install +\`\`\` +`; + +/** robinhood-trader's shape: an import line plus a short note. */ +const POINTER = `@AGENTS.md +@data-model.md + +Read \`AGENTS.md\` first — it is canonical for this repo and everything else is +linked from its map. +`; + +/** Same length as the pointer, but it is a rule document, not a redirect. */ +const THIN_RULE_DOC = `# Extra Claude rules + +Prefer the repo formatter over your own. +Keep generated files out of review. +Run the suite before claiming done. +`; + +function makeRepo(claudeBody: string, prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)); + created.push(dir); + mkdirSync(join(dir, "src"), { recursive: true }); + writeFileSync(join(dir, "AGENTS.md"), CANONICAL, "utf8"); + writeFileSync(join(dir, "CLAUDE.md"), claudeBody, "utf8"); + writeFileSync( + join(dir, "package.json"), + JSON.stringify({ name: "fixture", version: "1.0.0" }, null, 2), + "utf8", + ); + writeFileSync(join(dir, "src", "index.ts"), "export const x = 1;\n", "utf8"); + return dir; +} + +async function lowYieldDetail(target: string): Promise { + const proc = Bun.spawn( + ["bun", CLI, "audit", "--target", target, "--ci", "--json"], + { cwd: REPO_ROOT, stdout: "pipe", stderr: "pipe" }, + ); + const stdout = await new Response(proc.stdout).text(); + await proc.exited; + const report = JSON.parse(stdout) as { + stageD: { checks: Array<{ id: string; detail: string }> }; + }; + const check = report.stageD.checks.find((c) => c.id === "low-yield-rules"); + return check?.detail ?? ""; +} + +test("isPointerDocument keys on the import, not on being short", () => { + expect(isPointerDocument(POINTER)).toBe(true); + expect(isPointerDocument(THIN_RULE_DOC)).toBe(false); + expect(isPointerDocument(CANONICAL)).toBe(false); + // A mention of the file is not an import of it. + expect(isPointerDocument("See AGENTS.md for the rules.\n")).toBe(false); +}); + +test("an @AGENTS.md shim is not counted as a scoring file", async () => { + const dir = makeRepo(POINTER, "anvil-shim-pointer-"); + + const detail = await lowYieldDetail(dir); + + // Denominator is 1, not 2: the pointer left the set entirely rather than + // being counted and forgiven. + expect(detail).toContain("0/1 scoring files"); +}, 180_000); + +test("an equally short non-pointer file is still scored", async () => { + // The guard against 'exclude short files'. This one is the same size as the + // shim and has neither Why nor examples, so it must still count and fail. + const dir = makeRepo(THIN_RULE_DOC, "anvil-shim-thin-"); + + const detail = await lowYieldDetail(dir); + + expect(detail).toContain("1/2 scoring files"); +}, 180_000); From 5924c3f0522aa4522b49188cb1815638662344bf Mon Sep 17 00:00:00 2001 From: Jake Ruesink Date: Sun, 9 Aug 2026 08:27:31 -0500 Subject: [PATCH 2/3] chore: bump 0.1.0-alpha.12 Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 8 ++++---- docs-site/public/llms-full.txt | 2 +- docs-site/src/content/docs/reference/cli.md | 2 +- docs/byok-trust-model.md | 2 +- docs/first-user-proof-packet.md | 6 +++--- docs/first-user-proof.md | 4 ++-- docs/getting-started.md | 6 +++--- docs/proofs/current-outside-tester-send-packet.md | 14 +++++++------- package.json | 2 +- 9 files changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 2406e1c..c6e2602 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Relative `--target` paths resolve from your current shell cwd. If you are alread Choose the lane that matches your setup before your first run. -If you are collecting the outside-user proof, stay on the exact pinned version and launcher from [First User Proof](docs/first-user-proof.md) instead of switching to the unpinned examples in this README. The current pinned `0.1.0-alpha.11` proof packet uses one repo-root `bunx` command with `--ci --output ./anvil-audit.md` so the saved report comes back from the first run. +If you are collecting the outside-user proof, stay on the exact pinned version and launcher from [First User Proof](docs/first-user-proof.md) instead of switching to the unpinned examples in this README. The current pinned `0.1.0-alpha.12` proof packet uses one repo-root `bunx` command with `--ci --output ./anvil-audit.md` so the saved report comes back from the first run. ### Local-only first pass (no provider required) @@ -151,7 +151,7 @@ bun run ./bin/anvil.ts --version Verified on the current alpha packet: - `--help` prints the four shipped entry commands: `audit`, `drift`, `bootstrap`, `mine-pr` -- `--version` prints `0.1.0-alpha.11` +- `--version` prints `0.1.0-alpha.12` Why you might choose this lane: @@ -173,10 +173,10 @@ For first-run setup and CI/lint guidance, see: Lambda Curry maintains this project with internal automation behind it, but that machinery is secondary to the public product path above. -- **Status:** Report as Decision Tool shipped; current charter follow-through is to collect outside-Lambda-Curry first-run proof on pinned `0.1.0-alpha.11` +- **Status:** Report as Decision Tool shipped; current charter follow-through is to collect outside-Lambda-Curry first-run proof on pinned `0.1.0-alpha.12` - **Verification posture:** CI artifact (audit report) + downstream observed impact in rule quality - **Current checked-in self-audit:** `docs/audits/anvil-audit-2026-08-08.md` reports `98/100` Structural Lint, `35/35` Guardrail Readiness, `0` issues, and `0` remediation tasks on current `main` -- **Current proof packet:** `docs/proofs/current-outside-tester-send-packet.md` keeps the external proof lane on one canonical repo-root command that saves `./anvil-audit.md`; the pinned packet stays on `@lambdacurry/anvil@0.1.0-alpha.11` +- **Current proof packet:** `docs/proofs/current-outside-tester-send-packet.md` keeps the external proof lane on one canonical repo-root command that saves `./anvil-audit.md`; the pinned packet stays on `@lambdacurry/anvil@0.1.0-alpha.12` Anvil is not primarily a UI project. Its real proof surface is whether downstream outputs and consumers reflect the intended rule behavior correctly. diff --git a/docs-site/public/llms-full.txt b/docs-site/public/llms-full.txt index bb764ac..002f336 100644 --- a/docs-site/public/llms-full.txt +++ b/docs-site/public/llms-full.txt @@ -745,7 +745,7 @@ anvil audit --target ./my-repo [options] Relative `--target` paths resolve from your current shell cwd. -If you arrived here from the external first-user proof docs, use the exact pinned command from that packet. The current `0.1.0-alpha.11` packet uses the public `--ci` spelling; `--no-ai` remains only as a deprecated compatibility alias. +If you arrived here from the external first-user proof docs, use the exact pinned command from that packet. The current `0.1.0-alpha.12` packet uses the public `--ci` spelling; `--no-ai` remains only as a deprecated compatibility alias. ## `anvil drift` diff --git a/docs-site/src/content/docs/reference/cli.md b/docs-site/src/content/docs/reference/cli.md index ff4ca77..7a2f4be 100644 --- a/docs-site/src/content/docs/reference/cli.md +++ b/docs-site/src/content/docs/reference/cli.md @@ -32,7 +32,7 @@ anvil audit --target ./my-repo [options] Relative `--target` paths resolve from your current shell cwd. -If you arrived here from the external first-user proof docs, use the exact pinned command from that packet. The current `0.1.0-alpha.11` packet uses the public `--ci` spelling; `--no-ai` remains only as a deprecated compatibility alias. +If you arrived here from the external first-user proof docs, use the exact pinned command from that packet. The current `0.1.0-alpha.12` packet uses the public `--ci` spelling; `--no-ai` remains only as a deprecated compatibility alias. ## `anvil drift` diff --git a/docs/byok-trust-model.md b/docs/byok-trust-model.md index 2f9b2e0..0f04e37 100644 --- a/docs/byok-trust-model.md +++ b/docs/byok-trust-model.md @@ -18,7 +18,7 @@ By default, Anvil scans your repo locally, then expects a working AI provider fo If you want the privacy-first path, run: -> **Current alpha note:** The published `0.1.0-alpha.11` proof packet uses one canonical repo-root `bunx` command with `--ci --output ./anvil-audit.md`. Packaged relative `--target` and `--output` paths still resolve from your shell cwd, so normal repo-relative first-run commands are honest when you use the unpinned command (it tracks the latest published build). +> **Current alpha note:** The published `0.1.0-alpha.12` proof packet uses one canonical repo-root `bunx` command with `--ci --output ./anvil-audit.md`. Packaged relative `--target` and `--output` paths still resolve from your shell cwd, so normal repo-relative first-run commands are honest when you use the unpinned command (it tracks the latest published build). ```bash # zero-install diff --git a/docs/first-user-proof-packet.md b/docs/first-user-proof-packet.md index fb7364b..18f876f 100644 --- a/docs/first-user-proof-packet.md +++ b/docs/first-user-proof-packet.md @@ -20,7 +20,7 @@ Send back whether it worked first try, the first useful fix the report pointed t bunx @lambdacurry/anvil@ audit --target . --ci --output ./anvil-audit.md ``` -Replace `` with the specific published build you want validated. The current `0.1.0-alpha.11` proof packet sends only the repo-root saved-report command above so the artifact comes back from the same first run without asking the tester to choose between layouts. +Replace `` with the specific published build you want validated. The current `0.1.0-alpha.12` proof packet sends only the repo-root saved-report command above so the artifact comes back from the same first run without asking the tester to choose between layouts. Helpful docs: - Getting started: https://lambda-curry.github.io/anvil/getting-started/first-audit @@ -54,7 +54,7 @@ Before sending the note above, make sure: ## 3. Exact command blocks to send -Pick one install path and one shell layout, then send only that exact command so the tester is not choosing between multiple moving parts. For the current `0.1.0-alpha.11` packet, the canonical layout is Bun zero-install from the target repo root. +Pick one install path and one shell layout, then send only that exact command so the tester is not choosing between multiple moving parts. For the current `0.1.0-alpha.12` packet, the canonical layout is Bun zero-install from the target repo root. Replace `` before you send anything. Do not use the floating `@alpha` tag in the external proof packet. @@ -128,7 +128,7 @@ bun run verify:first-user-proof -- docs/proofs/YYYY-MM-DD--first-user-pr ``` The validator returns a deterministic `counts` / `does-not-count` result and names the missing proof fields or contract mismatches directly. -For the current pinned `0.1.0-alpha.11` proof lane, that includes checking that the retained audit command keeps the packet's `--ci` spelling. +For the current pinned `0.1.0-alpha.12` proof lane, that includes checking that the retained audit command keeps the packet's `--ci` spelling. When the packet keeps a local report artifact, it also requires `Saved report path or screenshot link` to match the retained audit command's `--output` path. Save one small packet with these fields: diff --git a/docs/first-user-proof.md b/docs/first-user-proof.md index ba0e324..4ad362a 100644 --- a/docs/first-user-proof.md +++ b/docs/first-user-proof.md @@ -15,7 +15,7 @@ Capture one real outside-Lambda-Curry run that proves: Do this only after the exact published version you want to validate is live, and before Milestone 3 is called complete. -Do not send this packet with the floating `@alpha` tag. Replace `` in the command below with the specific published build you are validating, for example `0.1.0-alpha.11`. +Do not send this packet with the floating `@alpha` tag. Replace `` in the command below with the specific published build you are validating, for example `0.1.0-alpha.12`. ## Suggested tester profile @@ -115,7 +115,7 @@ bun run verify:first-user-proof -- docs/proofs/YYYY-MM-DD--first-user-pr ``` That validator checks the outside-tester status, pinned CLI version, first-try success, returned artifact, and other minimum packet fields, then returns `counts` or `does-not-count` with explicit reasons. -For the current pinned `0.1.0-alpha.11` proof lane, it requires the retained audit command to keep the exact `--ci` spelling from the packet. +For the current pinned `0.1.0-alpha.12` proof lane, it requires the retained audit command to keep the exact `--ci` spelling from the packet. ## Done signal for Milestone 3 gate diff --git a/docs/getting-started.md b/docs/getting-started.md index 8b51d16..c838a9a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -64,7 +64,7 @@ anvil --version What you should see in the current alpha: - `--help` lists the four shipped commands: `audit`, `drift`, `bootstrap`, `mine-pr` -- `--version` prints `0.1.0-alpha.11` +- `--version` prints `0.1.0-alpha.12` If you are validating Anvil from a cloned checkout instead of a global install, run: @@ -125,7 +125,7 @@ Top 5 improvements: ## Save the report to a file -> **Current alpha note:** The published `0.1.0-alpha.11` proof packet uses one canonical repo-root `bunx` command with `--ci --output ./anvil-audit.md`, while the packaged CLI still resolves relative `--target` and `--output` paths from your shell cwd on `bunx`, `npx`, and Bun global install. Normal relative-path examples are honest when you use the unpinned command (it tracks the latest published build). +> **Current alpha note:** The published `0.1.0-alpha.12` proof packet uses one canonical repo-root `bunx` command with `--ci --output ./anvil-audit.md`, while the packaged CLI still resolves relative `--target` and `--output` paths from your shell cwd on `bunx`, `npx`, and Bun global install. Normal relative-path examples are honest when you use the unpinned command (it tracks the latest published build). ```bash # zero-install with bunx @@ -235,7 +235,7 @@ anvil audit \ `--ci` keeps discovery, drift detection, coverage scoring, and markdown output local. The report headline becomes `Structural Lint Score`, and the improvement section is generated from repo-local heuristics instead of a provider. -`--no-ai` still works as a deprecated compatibility alias for the same mode. The current external first-user proof packet stays pinned to `0.1.0-alpha.11` and uses `--ci` for the local-only lane. +`--no-ai` still works as a deprecated compatibility alias for the same mode. The current external first-user proof packet stays pinned to `0.1.0-alpha.12` and uses `--ci` for the local-only lane. Privacy-first example artifact from the same example target: diff --git a/docs/proofs/current-outside-tester-send-packet.md b/docs/proofs/current-outside-tester-send-packet.md index 51b4ce4..2b2b71c 100644 --- a/docs/proofs/current-outside-tester-send-packet.md +++ b/docs/proofs/current-outside-tester-send-packet.md @@ -2,7 +2,7 @@ Use this packet to route one outside-Lambda-Curry tester through Anvil's remaining Milestone 3 proof lane. -This packet stays pinned to `@lambdacurry/anvil@0.1.0-alpha.11`. Do not swap the tester onto the floating `@alpha` tag. +This packet stays pinned to `@lambdacurry/anvil@0.1.0-alpha.12`. Do not swap the tester onto the floating `@alpha` tag. ## Three-line opener @@ -15,7 +15,7 @@ Send back whether it worked first try, the first useful fix the report pointed t ## Exact command to send ```bash -bunx @lambdacurry/anvil@0.1.0-alpha.11 audit --target . --ci --output ./anvil-audit.md +bunx @lambdacurry/anvil@0.1.0-alpha.12 audit --target . --ci --output ./anvil-audit.md ``` Send this as the only command. It assumes the tester is already in the target repo root, guarantees the saved report path, and keeps the local-only flag aligned with current public docs. @@ -27,7 +27,7 @@ Could you try one first-run Anvil audit on a real repo of yours? Paste the single command below from that repo's root; it saves `./anvil-audit.md`, stays local, and does not require an AI provider. ```bash -bunx @lambdacurry/anvil@0.1.0-alpha.11 audit --target . --ci --output ./anvil-audit.md +bunx @lambdacurry/anvil@0.1.0-alpha.12 audit --target . --ci --output ./anvil-audit.md ``` Helpful docs: @@ -39,7 +39,7 @@ What I'd love back: 1. Whether the exact command worked on the first try 2. If it did not, what failed first 3. If you changed the launcher or command, what you used instead - - If you switched to global `anvil`, keep both the pinned `bun add -g @lambdacurry/anvil@0.1.0-alpha.11` line and the `anvil audit ...` line together in `Exact command`. + - If you switched to global `anvil`, keep both the pinned `bun add -g @lambdacurry/anvil@0.1.0-alpha.12` line and the `anvil audit ...` line together in `Exact command`. 4. Whether you ran it from the repo root or somewhere else 5. The first useful fix the report pointed to, if any 6. Anything that felt confusing, too internal, or too hand-wavy @@ -47,7 +47,7 @@ What I'd love back: - If you send back the saved report path itself, keep `./anvil-audit.md`, the exact path the retained command wrote with `--output`. If you want one extra cross-check, this should print the same pinned version: -`bunx @lambdacurry/anvil@0.1.0-alpha.11 --version` +`bunx @lambdacurry/anvil@0.1.0-alpha.12 --version` If you changed launchers before the successful run, use the matching `--version` command from that same install path instead of mixing launchers in the saved packet. Do not append `anvil --version` to a `bunx` or `npx` proof packet. @@ -57,7 +57,7 @@ Count this as Milestone 3 proof only if all of these are true: - the tester is outside Lambda Curry - the tester completes a successful first run on a real repo -- the retained audit command keeps the pinned `0.1.0-alpha.11` local-only `--ci` spelling +- the retained audit command keeps the pinned `0.1.0-alpha.12` local-only `--ci` spelling - the exact command and returned artifact are retained in a saved proof packet - any rough edge found is captured as follow-up work @@ -79,7 +79,7 @@ bun run verify:first-user-proof -- docs/proofs/YYYY-MM-DD--first-user-pr ``` Run that verifier from an Anvil repo checkout or an unpacked published Anvil package root; the verifier now ships with the same proof-doc bundle. -It keys validation off the saved packet's `Pinned CLI version`, so this retained `0.1.0-alpha.11` packet can still be checked after current `main` advances to a later package version. +It keys validation off the saved packet's `Pinned CLI version`, so this retained `0.1.0-alpha.12` packet can still be checked after current `main` advances to a later package version. Historical note: the original dated retained packet for this same pinned proof lane remains at `docs/proofs/2026-05-23-alpha4-outside-tester-send-packet.md`. diff --git a/package.json b/package.json index 29c9370..1cc6cc4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lambdacurry/anvil", - "version": "0.1.0-alpha.11", + "version": "0.1.0-alpha.12", "description": "AI rules + engineering guardrails audit engine for AI-assisted codebases", "keywords": [ "agents", From 5757058a1b3473be474b8dd916e87cc268a63056 Mon Sep 17 00:00:00 2001 From: Jake Ruesink Date: Sun, 9 Aug 2026 08:38:45 -0500 Subject: [PATCH 3/3] test: assert the shim stays on the load budget, and drain stderr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two CodeRabbit findings on #41, both valid. The Low-Yield assertions alone would stay green if a later change also dropped pointers from summarizeLoad — which would violate the contract this fix rests on, since a shim is additive for context load. Now asserted directly: the same fixture with and without the shim differs by the shim's own lines (15 vs 10). Both fixture helpers also piped stderr without draining it. `--json` writes one progress line per discovered rule file there, so a full pipe buffer would deadlock the child mid-read. Latent at fixture size, real at repo size. Fixed in rule-load-fixture.test.ts too, which shipped with the same shape in alpha.10. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/rule-load-fixture.test.ts | 7 +++- scripts/shim-denominator.test.ts | 58 ++++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/scripts/rule-load-fixture.test.ts b/scripts/rule-load-fixture.test.ts index 929d807..f1f9f5d 100644 --- a/scripts/rule-load-fixture.test.ts +++ b/scripts/rule-load-fixture.test.ts @@ -65,7 +65,12 @@ async function auditJson(target: string): Promise<{ ["bun", CLI, "audit", "--target", target, "--ci", "--json"], { cwd: REPO_ROOT, stdout: "pipe", stderr: "pipe" }, ); - const stdout = await new Response(proc.stdout).text(); + // Drain both pipes concurrently: `--json` emits per-file progress on stderr, + // and a full stderr buffer would deadlock the child while we read stdout. + const [stdout] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); await proc.exited; return JSON.parse(stdout) as { overkill: { alwaysOnLines: number } }; } diff --git a/scripts/shim-denominator.test.ts b/scripts/shim-denominator.test.ts index 58b7e42..ea2b903 100644 --- a/scripts/shim-denominator.test.ts +++ b/scripts/shim-denominator.test.ts @@ -65,18 +65,27 @@ function makeRepo(claudeBody: string, prefix: string): string { return dir; } -async function lowYieldDetail(target: string): Promise { +type StageDChecks = Record; + +async function stageDChecks(target: string): Promise { const proc = Bun.spawn( ["bun", CLI, "audit", "--target", target, "--ci", "--json"], { cwd: REPO_ROOT, stdout: "pipe", stderr: "pipe" }, ); - const stdout = await new Response(proc.stdout).text(); + // `--json` writes one progress line per discovered rule file to stderr. Both + // pipes must be drained concurrently, or a full stderr buffer deadlocks the + // child while we are still reading stdout. + const [stdout] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); await proc.exited; const report = JSON.parse(stdout) as { stageD: { checks: Array<{ id: string; detail: string }> }; }; - const check = report.stageD.checks.find((c) => c.id === "low-yield-rules"); - return check?.detail ?? ""; + return Object.fromEntries( + report.stageD.checks.map((check) => [check.id, check.detail]), + ); } test("isPointerDocument keys on the import, not on being short", () => { @@ -90,11 +99,44 @@ test("isPointerDocument keys on the import, not on being short", () => { test("an @AGENTS.md shim is not counted as a scoring file", async () => { const dir = makeRepo(POINTER, "anvil-shim-pointer-"); - const detail = await lowYieldDetail(dir); + const checks = await stageDChecks(dir); // Denominator is 1, not 2: the pointer left the set entirely rather than // being counted and forgiven. - expect(detail).toContain("0/1 scoring files"); + expect(checks["low-yield-rules"]).toContain("0/1 scoring files"); +}, 180_000); + +test("the excluded shim still counts toward context load", async () => { + // The other half of the contract, and the half that would rot silently. + // A shim is additive for Claude — it loads the shim AND what it imports — so + // dropping pointers from the scoring surface wholesale would undercount load + // while leaving the Low-Yield assertion above perfectly green. + const withPointer = await stageDChecks(makeRepo(POINTER, "anvil-shim-load-")); + const withoutClaude = mkdtempSync(join(tmpdir(), "anvil-shim-solo-")); + created.push(withoutClaude); + mkdirSync(join(withoutClaude, "src"), { recursive: true }); + writeFileSync(join(withoutClaude, "AGENTS.md"), CANONICAL, "utf8"); + writeFileSync( + join(withoutClaude, "package.json"), + JSON.stringify({ name: "fixture", version: "1.0.0" }, null, 2), + "utf8", + ); + writeFileSync( + join(withoutClaude, "src", "index.ts"), + "export const x = 1;\n", + "utf8", + ); + const soloChecks = await stageDChecks(withoutClaude); + + const linesOf = (detail: string): number => + Number.parseInt(/^(\d+) always-on lines/.exec(detail)?.[1] ?? "-1", 10); + + const withShim = linesOf(withPointer["context-load-pressure"] ?? ""); + const solo = linesOf(soloChecks["context-load-pressure"] ?? ""); + + expect(solo).toBeGreaterThan(0); + // The shim's own lines are still on the session budget. + expect(withShim).toBeGreaterThan(solo); }, 180_000); test("an equally short non-pointer file is still scored", async () => { @@ -102,7 +144,7 @@ test("an equally short non-pointer file is still scored", async () => { // shim and has neither Why nor examples, so it must still count and fail. const dir = makeRepo(THIN_RULE_DOC, "anvil-shim-thin-"); - const detail = await lowYieldDetail(dir); + const checks = await stageDChecks(dir); - expect(detail).toContain("1/2 scoring files"); + expect(checks["low-yield-rules"]).toContain("1/2 scoring files"); }, 180_000);