From 3ad0690cbedc1c0745a33a26f85a35ed35d756de Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Mon, 27 Jul 2026 17:24:16 -0400 Subject: [PATCH 01/27] feat(schema): approval ledger kind + solvespec v5 plan_hash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec-20260727-164748 §5 / plan task CLI-1. Schema and types only — no behavior change; the gate step that consumes these lands in CLI-2. ledger-record gains an EIGHTH oneOf variant, `approval` — the capability warrant: {plan_hash, bounds, expires_at, issued_by}. Deliberately unsigned (spec §3): the threat model is drift, not an adversary, and the product agent holds unrestricted bash, so a signature would defend against an attacker this layer could not stop anyway. Provenance is issued_by; the append-only ledger makes the row tamper-evident. An empty `bounds` object is legal and authorises nothing beyond the ungated free set — §5.1 rule 2 refuses a launch needing a bound the warrant omits, so absent keys are not a default-allow hole. solvespec v5 adds OPTIONAL `plan_hash`. Optional is the load-bearing choice: §5.1 rule 1 makes absence safe by restricting the launch to the free set, so absence can never widen what a launch may do. Requiring it would break every existing spec and buy nothing. minLength 1 stops an empty hash matching any warrant. Registration follows #212's documented gotcha unchanged: ledger-record stays in SCHEMAS only, never SUPPORTED_VERSIONS_BY_KIND. solvespec's entry there is derived from its own enum, so v5 propagates automatically — validate.test.ts's version pin is updated deliberately rather than left to fail. schema 112/112 · amico-run 628 pass / 12 skipped · tsc --noEmit clean. Co-Authored-By: Claude Opus 5 --- packages/amico-run/src/ledger.ts | 28 +++++++++- .../schema/schemas/ledger-record.schema.json | 26 ++++++++- packages/schema/schemas/solvespec.schema.json | 3 +- packages/schema/test/ledger-record.test.ts | 55 +++++++++++++++++++ .../schema/test/solvespec-warrant.test.ts | 49 +++++++++++++++++ packages/schema/test/validate.test.ts | 4 +- 6 files changed, 160 insertions(+), 5 deletions(-) create mode 100644 packages/schema/test/solvespec-warrant.test.ts diff --git a/packages/amico-run/src/ledger.ts b/packages/amico-run/src/ledger.ts index 95d04aae..2ea75a4e 100644 --- a/packages/amico-run/src/ledger.ts +++ b/packages/amico-run/src/ledger.ts @@ -166,6 +166,31 @@ export interface DispatchRecord { source: "user" | "replay" | "simulated"; } +/** What a warrant authorises. An ABSENT key does not mean "unlimited" — the gate + * refuses a launch that needs a bound the warrant omits (spec §5.1 rule 2), so an + * empty `bounds` authorises nothing beyond the ungated free set. `device` uses the + * fleet spec §2.1 permission vocabulary. */ +export interface WarrantBounds { + max_solves?: number; + tier?: string; + max_duration_s?: number; + device?: "none" | "ro" | "rw"; +} + +/** A capability warrant (spec-20260727-164748 §5): what lets a gated launch through + * the `--spec` gate. DELIBERATELY UNSIGNED — the threat model is drift, not an + * adversary (spec §3), and the product agent holds unrestricted bash, so a signature + * would defend against an attacker this layer could not stop anyway. Provenance lives + * in `issued_by`, and the append-only ledger makes the record tamper-evident. */ +export interface ApprovalRecord { + type: "approval"; + ts: string; + plan_hash: string; + bounds: WarrantBounds; + expires_at: string; + issued_by: string; +} + export type LedgerRecord = | SolveRecord | VerdictRecord @@ -173,7 +198,8 @@ export type LedgerRecord = | FallbackRecord | OverrideRecord | BurnRecord - | DispatchRecord; + | DispatchRecord + | ApprovalRecord; /** The ledger file path: `$AMICO_LEDGER` override, else `~/.amico/ledger/runs.jsonl`. */ export function ledgerPath(): string { diff --git a/packages/schema/schemas/ledger-record.schema.json b/packages/schema/schemas/ledger-record.schema.json index 7757bc5e..0dcea61b 100644 --- a/packages/schema/schemas/ledger-record.schema.json +++ b/packages/schema/schemas/ledger-record.schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://amico.harmoniqs.co/schema/ledger-record/v1", "title": "amico run-ledger record", - "description": "One append-only line in ~/.amico/ledger/runs.jsonl. A oneOf discriminated on `type` over the seven record kinds (solve|verdict|attempt_error|fallback|override|burn|dispatch). Ops-data, not vault knowledge. Registered in @amicode/schema SCHEMAS ONLY — NOT SUPPORTED_VERSIONS_BY_KIND (no top-level properties.schema_version; a oneOf like this would crash the version-map builder at module load, same as problemspec).", + "description": "One append-only line in ~/.amico/ledger/runs.jsonl. A oneOf discriminated on `type` over the eight record kinds (solve|verdict|attempt_error|fallback|override|burn|dispatch|approval). Ops-data, not vault knowledge. Registered in @amicode/schema SCHEMAS ONLY — NOT SUPPORTED_VERSIONS_BY_KIND (no top-level properties.schema_version; a oneOf like this would crash the version-map builder at module load, same as problemspec).", "oneOf": [ { "title": "solve", @@ -173,6 +173,30 @@ "attempt_index": { "type": "integer", "minimum": 1 }, "source": { "enum": ["user", "replay", "simulated"] } } + }, + { + "title": "approval", + "description": "A capability warrant (spec-20260727-164748 §5): the record that lets a gated launch through amico-run's --spec gate. `plan_hash` is what was approved; `bounds` is what it authorises. DELIBERATELY UNSIGNED — the threat model is drift, not an adversary (spec §3), and the product agent holds unrestricted bash, so a signature would defend against an attacker this layer could not stop anyway. Absent bounds keys do NOT default to allow: §5.1 rule 2 refuses a launch needing a bound the warrant omits, which is why an empty `bounds` object is legal (it authorises nothing beyond the ungated free set) rather than a hole. `device` uses the fleet spec §2.1 permission vocabulary.", + "type": "object", + "additionalProperties": false, + "required": ["type", "ts", "plan_hash", "bounds", "expires_at", "issued_by"], + "properties": { + "type": { "const": "approval" }, + "ts": { "type": "string" }, + "plan_hash": { "type": "string", "minLength": 1 }, + "bounds": { + "type": "object", + "additionalProperties": false, + "properties": { + "max_solves": { "type": "integer", "minimum": 1 }, + "tier": { "type": "string", "minLength": 1 }, + "max_duration_s": { "type": "number", "exclusiveMinimum": 0 }, + "device": { "enum": ["none", "ro", "rw"] } + } + }, + "expires_at": { "type": "string" }, + "issued_by": { "type": "string", "minLength": 1 } + } } ] } diff --git a/packages/schema/schemas/solvespec.schema.json b/packages/schema/schemas/solvespec.schema.json index 54d233cb..98823cfe 100644 --- a/packages/schema/schemas/solvespec.schema.json +++ b/packages/schema/schemas/solvespec.schema.json @@ -17,7 +17,8 @@ } ], "properties": { - "schema_version": { "enum": ["1", "2", "3", "4"] }, + "schema_version": { "enum": ["1", "2", "3", "4", "5"] }, + "plan_hash": { "type": "string", "minLength": 1, "description": "v5 — the approved plan this launch runs under, joining it to a capability warrant at the --spec gate (spec-20260727-164748 §5.1). OPTIONAL by design: absence restricts the launch to the ungated free set (rule 1) and can never widen what it may do, so requiring it would break every existing spec while buying nothing. minLength guards against an empty hash matching any warrant." }, "script_path": { "type": "string", "minLength": 1, "description": "the Julia script to run (exactly one of script_path | problem_spec)" }, "problem_spec": { "oneOf": [ diff --git a/packages/schema/test/ledger-record.test.ts b/packages/schema/test/ledger-record.test.ts index 03d65df3..222b5713 100644 --- a/packages/schema/test/ledger-record.test.ts +++ b/packages/schema/test/ledger-record.test.ts @@ -159,6 +159,61 @@ describe("ledger-record schema — dispatch", () => { }); }); +// approval — the capability warrant (spec-20260727-164748 §5). The EIGHTH kind. +// Deliberately unsigned: the threat model is drift, not an adversary (spec §3), and +// the agent holds unrestricted bash, so a signature would defend against an attacker +// this layer could not stop anyway. +const approval = () => ({ + type: "approval", + ts: "2026-07-27T20:00:00Z", + plan_hash: "9f2c", + bounds: { max_solves: 8, tier: "free", max_duration_s: 1800, device: "none" }, + expires_at: "2026-07-27T21:00:00Z", + issued_by: "user:ui", +}); + +describe("ledger-record schema — approval (capability warrant)", () => { + it("a fully-declared warrant validates", () => { + expect(validate(approval(), "ledger-record").errors).toEqual([]); + }); + + it("empty bounds validate — a warrant authorising nothing beyond the free set is legal", () => { + // §5.1 rule 2: a bound the launch needs but the warrant omits REFUSES rather + // than defaulting allow, so an empty bounds object is safe, not a hole. + expect(validate({ ...approval(), bounds: {} }, "ledger-record").errors).toEqual([]); + }); + + it("each required field is genuinely required", () => { + for (const key of ["type", "ts", "plan_hash", "bounds", "expires_at", "issued_by"]) { + const partial: Record = { ...approval() }; + delete partial[key]; + expect(validate(partial, "ledger-record").ok, `missing ${key} must fail`).toBe(false); + } + }); + + it("unknown keys are rejected, like every sibling kind", () => { + expect(validate({ ...approval(), signature: "deadbeef" }, "ledger-record").ok).toBe(false); + expect(validate({ ...approval(), bounds: { max_solves: 8, nonesuch: 1 } }, "ledger-record").ok).toBe(false); + }); + + it("device bounds use the §2.1 permission vocabulary", () => { + for (const device of ["none", "ro", "rw"]) { + expect(validate({ ...approval(), bounds: { device } }, "ledger-record").ok, device).toBe(true); + } + expect(validate({ ...approval(), bounds: { device: "yes" } }, "ledger-record").ok).toBe(false); + }); + + it("max_solves must be a positive integer; max_duration_s must be positive", () => { + expect(validate({ ...approval(), bounds: { max_solves: 0 } }, "ledger-record").ok).toBe(false); + expect(validate({ ...approval(), bounds: { max_solves: 1.5 } }, "ledger-record").ok).toBe(false); + expect(validate({ ...approval(), bounds: { max_duration_s: 0 } }, "ledger-record").ok).toBe(false); + }); + + it("plan_hash must be non-empty — an empty hash would join every launch", () => { + expect(validate({ ...approval(), plan_hash: "" }, "ledger-record").ok).toBe(false); + }); +}); + describe("ledger-record schema — discriminator", () => { it("an unknown type fails (no oneOf branch matches)", () => { expect(validate({ type: "nonesuch", ts: "t" }, "ledger-record").ok).toBe(false); diff --git a/packages/schema/test/solvespec-warrant.test.ts b/packages/schema/test/solvespec-warrant.test.ts new file mode 100644 index 00000000..eb9d6090 --- /dev/null +++ b/packages/schema/test/solvespec-warrant.test.ts @@ -0,0 +1,49 @@ +// solvespec v5 — `plan_hash`, the field that lets amico-run's --spec gate join a +// launch to its capability warrant (spec-20260727-164748 §5.1). +// +// The load-bearing decision under test: `plan_hash` is OPTIONAL. §5.1 rule 1 makes +// its absence safe by restricting the launch to the ungated free set, so requiring it +// would break every existing spec while buying nothing. Its absence must never WIDEN +// what a launch may do — only restrict it. +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { parse as parseToml } from "smol-toml"; +import { validate, SUPPORTED_VERSIONS_BY_KIND } from "../src/index.js"; + +const here = dirname(fileURLToPath(import.meta.url)); +const base = () => + parseToml(readFileSync(join(here, "fixtures", "valid", "solvespec.toml"), "utf8")) as Record; + +describe("solvespec v5 — registration", () => { + it("v5 is a supported version (derived from the schema's own enum)", () => { + expect(SUPPORTED_VERSIONS_BY_KIND.solvespec).toEqual(["1", "2", "3", "4", "5"]); + }); +}); + +describe("solvespec v5 — plan_hash", () => { + it("a v5 spec carrying plan_hash validates", () => { + const spec = { ...base(), schema_version: "5", plan_hash: "9f2c" }; + expect(validate(spec, "solvespec").errors).toEqual([]); + }); + + it("plan_hash is OPTIONAL — a v5 spec without it validates", () => { + const spec = { ...base(), schema_version: "5" }; + expect(validate(spec, "solvespec").errors).toEqual([]); + }); + + it("an empty plan_hash fails — it would join every launch to any warrant", () => { + const spec = { ...base(), schema_version: "5", plan_hash: "" }; + expect(validate(spec, "solvespec").ok).toBe(false); + }); + + it("plan_hash must be a string", () => { + const spec = { ...base(), schema_version: "5", plan_hash: 42 }; + expect(validate(spec, "solvespec").ok).toBe(false); + }); + + it("earlier versions still validate — v5 is additive", () => { + expect(validate(base(), "solvespec").errors).toEqual([]); + }); +}); diff --git a/packages/schema/test/validate.test.ts b/packages/schema/test/validate.test.ts index 720d54c2..e244b8d0 100644 --- a/packages/schema/test/validate.test.ts +++ b/packages/schema/test/validate.test.ts @@ -32,10 +32,10 @@ describe("schema set + exports", () => { new Set(["run", "result", "lab", "solvespec", "catalog-entry", "finished", "problemspec", "ledger-record"]), ); }); - it("supported versions are PER-KIND: run at v2 (spec C); solvespec at v4 (hpc tier + remote executor + problem_spec); the rest v1", () => { + it("supported versions are PER-KIND: run at v2 (spec C); solvespec at v5 (v4 hpc tier + remote executor + problem_spec; v5 plan_hash); the rest v1", () => { expect(SUPPORTED_VERSIONS_BY_KIND).toEqual({ run: ["1", "2"], - solvespec: ["1", "2", "3", "4"], + solvespec: ["1", "2", "3", "4", "5"], result: ["1"], lab: ["1"], "catalog-entry": ["1"], From 7aec2701733f5559eb5f740eba9cbca56f73d10d Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Mon, 27 Jul 2026 17:42:28 -0400 Subject: [PATCH 02/27] =?UTF-8?q?feat(ledger):=20amico=20ledger=20approve?= =?UTF-8?q?=20=E2=80=94=20mint=20a=20capability=20warrant?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec-20260727-164748 §5 / plan task CLI-3. Independent of CLI-2: minting a warrant does not need the gate that consumes it, so this lands unblocked while G-8 (max_duration_s has no signal behind it) is still open. This is the transport spec §9.5 requires. An approval must reach the ledger DIRECTLY — not as a chat message the agent interprets and then records, which would make the provenance read "the agent says the user approved". Reuses appendRecord rather than adding a second writer; #212's single-writer rule is load-bearing for O_APPEND atomicity. Bounds are declared-only. An omitted bound stays ABSENT rather than being defaulted, because §5.1 rule 2 refuses a launch needing a bound the warrant omits — so a helpfully-filled default would silently widen the warrant. Tested explicitly: `approve --plan-hash h` writes `bounds: {}`. Flags are rejected rather than coerced (`--max-solves 1.5` fails instead of flooring), `--device` is constrained to the fleet §2.1 none|ro|rw vocabulary, and issued_by is never empty since it is the ledger's only record of who approved. Nothing is written on any refusal path. amico-run 636 pass / 12 skipped · tsc --noEmit clean · verified end-to-end through the built dist/amico.js, including the refusal path. Co-Authored-By: Claude Opus 5 --- packages/amico-run/src/ledger_verb.ts | 89 ++++++++++++++++- .../amico-run/test/ledger_approve.test.ts | 97 +++++++++++++++++++ 2 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 packages/amico-run/test/ledger_approve.test.ts diff --git a/packages/amico-run/src/ledger_verb.ts b/packages/amico-run/src/ledger_verb.ts index 64bb239a..ae62ffdd 100644 --- a/packages/amico-run/src/ledger_verb.ts +++ b/packages/amico-run/src/ledger_verb.ts @@ -27,8 +27,20 @@ // ladder as the standing fallback. Delegates to ledger_dispatch.ts. Simulated // evidence is opt-in and lands in the SIM LANE ONLY — it never widens a // hardware cell. +// +// amico ledger approve --plan-hash [--max-solves ] [--tier ] +// [--max-duration ] [--device none|ro|rw] +// [--expires-in ] [--issued-by ] +// → mint a capability warrant (spec-20260727-164748 §5): the record that lets a +// gated launch through the --spec gate. This is the transport the approval +// card requires (spec §9.5) — an approval must reach the ledger DIRECTLY, +// never as a chat message the agent interprets and then records, or the +// provenance reads "the agent says the user approved". Bounds are written +// ONLY as declared: an omitted bound is ABSENT, never defaulted, because +// §5.1 rule 2 refuses a launch needing a bound the warrant omits — so a +// helpfully-filled default would silently widen the warrant. import { readFileSync } from "node:fs"; -import { appendRecord, type LedgerRecord } from "./ledger.js"; +import { appendRecord, type ApprovalRecord, type LedgerRecord, type WarrantBounds } from "./ledger.js"; import { bucketN, bucketT, queryDefaults, type QueryKey } from "./ledger_query.js"; import { dispatchTable, type DispatchKey } from "./ledger_dispatch.js"; import type { VerbResult } from "./verbs.js"; @@ -129,6 +141,78 @@ export function ledgerDispatch(argv: string[]): VerbResult { return { json: { verb: "ledger", subcommand: "dispatch", ...result }, code: 0 }; } +// ── approve (capability warrant, spec-20260727-164748 §5) ──────────────────── +/** A positive integer flag, or a typed refusal. Rejects rather than coercing: a + * silently-floored `--max-solves 1.5` would write a bound nobody asked for. */ +function positiveInt(raw: string | undefined, name: string): number | string | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isInteger(n) || n < 1) return `${name} must be a positive integer (got "${raw}")`; + return n; +} + +export function ledgerApprove(argv: string[]): VerbResult { + const fail = (error: string): VerbResult => ({ json: { verb: "ledger", subcommand: "approve", error }, code: 64 }); + + const plan_hash = flagValue(argv, "--plan-hash"); + if (!plan_hash) return fail("--plan-hash is required (what is being approved)"); + + // Bounds are declared-only. An omitted key stays ABSENT so the gate's §5.1 + // rule 2 refusal applies; defaulting any of these would widen the warrant. + const bounds: WarrantBounds = {}; + + const maxSolves = positiveInt(flagValue(argv, "--max-solves"), "--max-solves"); + if (typeof maxSolves === "string") return fail(maxSolves); + if (maxSolves !== undefined) bounds.max_solves = maxSolves; + + const maxDuration = positiveInt(flagValue(argv, "--max-duration"), "--max-duration"); + if (typeof maxDuration === "string") return fail(maxDuration); + if (maxDuration !== undefined) bounds.max_duration_s = maxDuration; + + const tier = flagValue(argv, "--tier"); + if (tier !== undefined) bounds.tier = tier; + + const device = flagValue(argv, "--device"); + if (device !== undefined) { + if (device !== "none" && device !== "ro" && device !== "rw") + return fail(`--device must be none|ro|rw (got "${device}")`); + bounds.device = device; + } + + const expiresIn = positiveInt(flagValue(argv, "--expires-in"), "--expires-in"); + if (typeof expiresIn === "string") return fail(expiresIn); + const ttl = expiresIn ?? 3600; // one hour — short by design; §5 leans session-scoped + + const now = new Date(); + const rec: ApprovalRecord = { + type: "approval", + ts: now.toISOString(), + plan_hash, + bounds, + expires_at: new Date(now.getTime() + ttl * 1000).toISOString(), + // Never empty: an unattributed warrant is worse than none, since the ledger's + // only provenance for who approved is this field. + issued_by: flagValue(argv, "--issued-by") ?? "user:cli", + }; + + try { + appendRecord(rec); // single writer (#212) — never a second append path + } catch (e) { + return fail(e instanceof Error ? e.message : String(e)); + } + return { + json: { + verb: "ledger", + subcommand: "approve", + ok: true, + plan_hash, + bounds, + expires_at: rec.expires_at, + }, + code: 0, + }; +} + // ── subcommand router ──────────────────────────────────────────────────────── /** The `ledger` verb body: route on the subcommand. Backs BOTH the CLI * (amico.ts) and the MCP facade (mcp_serve.ts). */ @@ -138,12 +222,13 @@ export function ledgerVerb(argv: string[]): VerbResult { if (sub === "append") return ledgerAppend(rest); if (sub === "query") return ledgerQuery(rest); if (sub === "dispatch") return ledgerDispatch(rest); + if (sub === "approve") return ledgerApprove(rest); return { json: { verb: "ledger", error: `unknown subcommand ${sub ? `"${sub}"` : "(none)"}`, usage: - "amico ledger append [--json | (stdin)] | amico ledger query --structure-hash --n --t [--goal --platform

--template --trajectory --levels ] | amico ledger dispatch --work-id --task-type [--variant ] [--stamp ] [--include-simulated]", + "amico ledger append [--json | (stdin)] | amico ledger query --structure-hash --n --t [--goal --platform

--template --trajectory --levels ] | amico ledger dispatch --work-id --task-type [--variant ] [--stamp ] [--include-simulated] | amico ledger approve --plan-hash [--max-solves ] [--tier ] [--max-duration ] [--device none|ro|rw] [--expires-in ] [--issued-by ]", }, code: 64, }; diff --git a/packages/amico-run/test/ledger_approve.test.ts b/packages/amico-run/test/ledger_approve.test.ts new file mode 100644 index 00000000..37e7f9e5 --- /dev/null +++ b/packages/amico-run/test/ledger_approve.test.ts @@ -0,0 +1,97 @@ +// `amico ledger approve` (spec-20260727-164748 §5, plan task CLI-3) — mints a +// capability warrant. This is the transport the approval card requires (spec §9.5): +// an approval must reach the ledger DIRECTLY, never via the agent's reading of a +// user turn, or the provenance reads "the agent says the user approved". +// +// It deliberately reuses appendRecord rather than adding a second writer — #212's +// single-writer rule for the ledger is load-bearing for atomicity. +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ledgerVerb } from "../src/ledger_verb.js"; +import { readRecords, ledgerPath, type ApprovalRecord } from "../src/ledger.js"; + +type Result = { json: Record; code: number }; + +describe("ledger approve", () => { + let dir: string; + const prev = process.env.AMICO_LEDGER; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ledger-approve-")); + process.env.AMICO_LEDGER = join(dir, "runs.jsonl"); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + if (prev === undefined) delete process.env.AMICO_LEDGER; + else process.env.AMICO_LEDGER = prev; + }); + + it("mints a schema-valid warrant with the declared bounds", () => { + const r = ledgerVerb([ + "approve", + "--plan-hash", "9f2c", + "--max-solves", "8", + "--tier", "free", + "--expires-in", "3600", + "--issued-by", "user:cli", + ]) as Result; + expect(r.code).toBe(0); + expect(r.json).toMatchObject({ verb: "ledger", subcommand: "approve", ok: true, plan_hash: "9f2c" }); + + const recs = readRecords(); + expect(recs).toHaveLength(1); + const rec = recs[0] as ApprovalRecord; + expect(rec.type).toBe("approval"); + expect(rec.plan_hash).toBe("9f2c"); + expect(rec.bounds).toEqual({ max_solves: 8, tier: "free" }); + expect(rec.issued_by).toBe("user:cli"); + expect(Date.parse(rec.expires_at)).toBeGreaterThan(Date.parse(rec.ts)); + }); + + it("--plan-hash is required, and nothing is written without it", () => { + const r = ledgerVerb(["approve", "--max-solves", "8"]) as Result; + expect(r.code).toBe(64); + expect(String(r.json.error)).toContain("--plan-hash"); + expect(existsSync(ledgerPath())).toBe(false); + }); + + it("omitted bounds are ABSENT, never defaulted — absence must not read as unlimited", () => { + // Spec §5.1 rule 2: the gate refuses a launch needing a bound the warrant + // omits. A verb that helpfully filled in a default would silently widen it. + const r = ledgerVerb(["approve", "--plan-hash", "9f2c"]) as Result; + expect(r.code).toBe(0); + const rec = readRecords()[0] as ApprovalRecord; + expect(rec.bounds).toEqual({}); + }); + + it("device bounds accept the §2.1 vocabulary and reject anything else", () => { + expect((ledgerVerb(["approve", "--plan-hash", "h", "--device", "ro"]) as Result).code).toBe(0); + expect((readRecords()[0] as ApprovalRecord).bounds.device).toBe("ro"); + expect((ledgerVerb(["approve", "--plan-hash", "h", "--device", "yes"]) as Result).code).toBe(64); + }); + + it("rejects a non-numeric or non-positive --max-solves rather than writing a bad row", () => { + for (const bad of ["zero", "0", "-1", "1.5"]) { + const r = ledgerVerb(["approve", "--plan-hash", "h", "--max-solves", bad]) as Result; + expect(r.code, `--max-solves ${bad}`).toBe(64); + } + expect(existsSync(ledgerPath())).toBe(false); + }); + + it("rejects a non-positive --expires-in", () => { + expect((ledgerVerb(["approve", "--plan-hash", "h", "--expires-in", "0"]) as Result).code).toBe(64); + expect((ledgerVerb(["approve", "--plan-hash", "h", "--expires-in", "nope"]) as Result).code).toBe(64); + }); + + it("issued_by defaults to a named local actor rather than empty provenance", () => { + const r = ledgerVerb(["approve", "--plan-hash", "9f2c"]) as Result; + expect(r.code).toBe(0); + expect((readRecords()[0] as ApprovalRecord).issued_by).toBeTruthy(); + }); + + it("the unknown-subcommand usage line mentions approve", () => { + const r = ledgerVerb(["nonesuch"]) as Result; + expect(String(r.json.usage)).toContain("approve"); + }); +}); From ce11689555c09fbe715c87c3ba7c1bac0765b39f Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Mon, 27 Jul 2026 21:05:42 -0400 Subject: [PATCH 03/27] feat(gate): capability-warrant check + G-8 resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec-20260727-164748 §5.1 / plan task CLI-2. Also resolves G-8 by taking its documented lean — MY CALL, unpushed and reversible, flagged for veto. G-8: `max_duration_s` is REPLACED by `max_size_class` (SMALL|MEDIUM). estimate.ts computes memory, not wall-clock, so the duration bound had no signal behind it. Carrying a field with no enforcement is worse than not having it, because it implies a guarantee about time nothing can make. A real duration estimator is the deferred C2 work. warrant.ts — pure resolution + bound checking; `now` and the approvals are parameters so the whole §5.1 matrix is testable without a Julia process or a temp ledger. The asymmetry it exists to enforce: an absent plan_hash, or a bound the warrant omits, may only ever RESTRICT a launch, never widen it. rule 1 (no plan_hash) → allowed only inside the ungated free set: local, free tier, SMALL, device none. Each way out refuses independently. rule 2 (plan_hash) → every capability reached for must be DECLARED and satisfied by a live warrant. An omitted-but-needed bound REFUSES; it never default-allows. Refusals name the bound and its margin (§5.2). Fail-closed throughout: an unresolved size is over-threshold, not SMALL (§4.4 — estimate.ts sizes unresolved levels as SMALL, which as a gate input is a silent widening path), and an unparseable expiry is already expired. gate.ts — the check lands as step 5, after the consistency checks so a malformed spec fails as malformed rather than as unwarranted, and before stamp assembly so no hash is minted for a launch that will not run. THE FLAG IS THE ABSENCE OF THE CONTEXT: omit WarrantContext and the step does not exist, so no existing caller changes behavior — asserted directly by the flag-off test. GateResult gains an optional structured `refusal` alongside the one-line `reason`. That structured refusal carries exactly G-9's preferred payload, so deriving an approval request from a refusal is now buildable rather than blocked. schema 113 · amico-run 658 pass / 12 skipped · extension 777 · tsc clean. One executor_parity flake in a full run; passed isolated and on re-run, and matches the load-sensitive spawn-test fragility #212 documented. Co-Authored-By: Claude Opus 5 --- packages/amico-run/src/gate.ts | 58 +++++- packages/amico-run/src/ledger.ts | 2 +- packages/amico-run/src/ledger_verb.ts | 13 +- packages/amico-run/src/warrant.ts | 173 ++++++++++++++++++ packages/amico-run/test/gate_warrant.test.ts | 113 ++++++++++++ packages/amico-run/test/warrant.test.ts | 142 ++++++++++++++ .../schema/schemas/ledger-record.schema.json | 2 +- packages/schema/test/ledger-record.test.ts | 14 +- 8 files changed, 502 insertions(+), 15 deletions(-) create mode 100644 packages/amico-run/src/warrant.ts create mode 100644 packages/amico-run/test/gate_warrant.test.ts create mode 100644 packages/amico-run/test/warrant.test.ts diff --git a/packages/amico-run/src/gate.ts b/packages/amico-run/src/gate.ts index e4a9cc21..ca87a356 100644 --- a/packages/amico-run/src/gate.ts +++ b/packages/amico-run/src/gate.ts @@ -4,8 +4,10 @@ // scan against the entitlement allowlist; (3) tier/env consistency incl. the // per-binding Manifest staleness check (#74 extension — a project/sandbox // env is validated against its OWN Manifest, not the extension-pinned one); -// (4) tier-2 masked-baseline check; (5) stamp assembly (canonical spec + -// gate-computed spec_hash). Any failure → no Julia process, one clear line. +// (4) tier-2 masked-baseline check; (5) capability warrant, ARMED ONLY when a +// WarrantContext is passed (spec-20260727-164748 §5.1); (6) stamp assembly +// (canonical spec + gate-computed spec_hash). Any failure → no Julia process, +// one clear line. import { createHash } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; @@ -16,6 +18,8 @@ import { checkImports, scanImports } from "./import_scan.js"; import { maskedHash } from "./baseline.js"; import { loadExemplarsIndex } from "./catalog.js"; import { hasCloudConfig } from "./remote_config.js"; +import { checkWarrant, type DeviceAccess, type SizeClass, type WarrantRefusal } from "./warrant.js"; +import type { ApprovalRecord } from "./ledger.js"; export interface GateStamp { tier?: string; @@ -23,7 +27,13 @@ export interface GateStamp { specCanonical: string; // stable-key-order JSON, what gets persisted } -export type GateResult = { ok: true; stamp: GateStamp } | { ok: false; reason: string; demote_to?: "free" }; +export type GateResult = + | { ok: true; stamp: GateStamp } + /** `refusal` is present only for a warrant refusal (step 5) — it carries the §5.2 + * structured form (offending bound + what a covering warrant must declare), which + * a caller can turn into an approval request. `reason` alone stays the one-line + * human form every other step returns. */ + | { ok: false; reason: string; demote_to?: "free"; refusal?: WarrantRefusal }; /** Stable key order at every level so spec_hash is insensitive to author key order. */ function canonicalize(value: unknown): unknown { @@ -59,7 +69,24 @@ function staleEnvCheck(projectDir: string): string | undefined { return undefined; } -export function runGate(specRaw: unknown, scriptText: string, authoring: AuthoringConfig): GateResult { +/** Warrant-check context (spec-20260727-164748 §5.1). PASSING THIS ARMS THE CHECK — + * omit it and the warrant step does not run, which is the feature flag. Assembled by + * the launch path (it owns the ledger read and the clock); the gate stays pure. */ +export interface WarrantContext { + approvals: readonly ApprovalRecord[]; + now: number; + /** From estimate.ts. UNDEFINED = unresolved, treated as over-threshold (§4.4). */ + sizeClass?: SizeClass; + device?: DeviceAccess; + solvesSoFar?: number; +} + +export function runGate( + specRaw: unknown, + scriptText: string, + authoring: AuthoringConfig, + warrant?: WarrantContext, +): GateResult { // ── step 1: schema ── const validation = validate(specRaw, "solvespec"); if (!validation.ok) return { ok: false, reason: `solvespec schema: ${validation.errors[0]}` }; @@ -126,7 +153,28 @@ export function runGate(specRaw: unknown, scriptText: string, authoring: Authori }; } - // ── step 5: stamp — canonical spec + gate-computed spec_hash ── + // ── step 5: capability warrant (spec-20260727-164748 §5.1) ── + // Ordered AFTER the consistency checks so a malformed or inconsistent spec fails as + // that, not as "unwarranted", and BEFORE stamp assembly so no hash is minted for a + // launch that will not run. THE FEATURE FLAG IS THE ABSENCE OF `warrant`: with no + // context passed, this step does not exist and no existing caller changes behavior. + if (warrant) { + const check = checkWarrant( + { + plan_hash: typeof spec.plan_hash === "string" ? spec.plan_hash : undefined, + tier, + executor, + sizeClass: warrant.sizeClass, + device: warrant.device, + solvesSoFar: warrant.solvesSoFar, + }, + warrant.approvals, + warrant.now, + ); + if (!check.ok) return { ok: false, reason: check.reason, refusal: check }; + } + + // ── step 6: stamp — canonical spec + gate-computed spec_hash ── const specCanonical = JSON.stringify(canonicalize(spec), null, 2); const specHash = "sha256:" + createHash("sha256").update(specCanonical).digest("hex"); const hashes: Record = {}; diff --git a/packages/amico-run/src/ledger.ts b/packages/amico-run/src/ledger.ts index 2ea75a4e..b41e3438 100644 --- a/packages/amico-run/src/ledger.ts +++ b/packages/amico-run/src/ledger.ts @@ -173,7 +173,7 @@ export interface DispatchRecord { export interface WarrantBounds { max_solves?: number; tier?: string; - max_duration_s?: number; + max_size_class?: "SMALL" | "MEDIUM"; device?: "none" | "ro" | "rw"; } diff --git a/packages/amico-run/src/ledger_verb.ts b/packages/amico-run/src/ledger_verb.ts index ae62ffdd..0a23c820 100644 --- a/packages/amico-run/src/ledger_verb.ts +++ b/packages/amico-run/src/ledger_verb.ts @@ -29,7 +29,7 @@ // hardware cell. // // amico ledger approve --plan-hash [--max-solves ] [--tier ] -// [--max-duration ] [--device none|ro|rw] +// [--max-size-class SMALL|MEDIUM] [--device none|ro|rw] // [--expires-in ] [--issued-by ] // → mint a capability warrant (spec-20260727-164748 §5): the record that lets a // gated launch through the --spec gate. This is the transport the approval @@ -165,9 +165,12 @@ export function ledgerApprove(argv: string[]): VerbResult { if (typeof maxSolves === "string") return fail(maxSolves); if (maxSolves !== undefined) bounds.max_solves = maxSolves; - const maxDuration = positiveInt(flagValue(argv, "--max-duration"), "--max-duration"); - if (typeof maxDuration === "string") return fail(maxDuration); - if (maxDuration !== undefined) bounds.max_duration_s = maxDuration; + const sizeClass = flagValue(argv, "--max-size-class"); + if (sizeClass !== undefined) { + if (sizeClass !== "SMALL" && sizeClass !== "MEDIUM") + return fail(`--max-size-class must be SMALL|MEDIUM (got "${sizeClass}")`); + bounds.max_size_class = sizeClass; + } const tier = flagValue(argv, "--tier"); if (tier !== undefined) bounds.tier = tier; @@ -228,7 +231,7 @@ export function ledgerVerb(argv: string[]): VerbResult { verb: "ledger", error: `unknown subcommand ${sub ? `"${sub}"` : "(none)"}`, usage: - "amico ledger append [--json | (stdin)] | amico ledger query --structure-hash --n --t [--goal --platform

--template --trajectory --levels ] | amico ledger dispatch --work-id --task-type [--variant ] [--stamp ] [--include-simulated] | amico ledger approve --plan-hash [--max-solves ] [--tier ] [--max-duration ] [--device none|ro|rw] [--expires-in ] [--issued-by ]", + "amico ledger append [--json | (stdin)] | amico ledger query --structure-hash --n --t [--goal --platform

--template --trajectory --levels ] | amico ledger dispatch --work-id --task-type [--variant ] [--stamp ] [--include-simulated] | amico ledger approve --plan-hash [--max-solves ] [--tier ] [--max-size-class SMALL|MEDIUM] [--device none|ro|rw] [--expires-in ] [--issued-by ]", }, code: 64, }; diff --git a/packages/amico-run/src/warrant.ts b/packages/amico-run/src/warrant.ts new file mode 100644 index 00000000..f9463612 --- /dev/null +++ b/packages/amico-run/src/warrant.ts @@ -0,0 +1,173 @@ +// Capability-warrant resolution and bound checking (spec-20260727-164748 §5.1). +// +// PURE by design — no fs, no clock, no ledger read. `now` and the approval rows are +// parameters, so the whole §5.1 matrix is unit-testable without a Julia process or a +// temp ledger. gate.ts does the I/O and calls in. +// +// The one asymmetry everything rests on: an absent `plan_hash`, or a bound the +// warrant omits, may only ever RESTRICT a launch — never widen it. Concretely: +// +// rule 1 (no plan_hash) → allowed ONLY if the launch is entirely inside the +// ungated free set. That is what makes the field's +// absence safe rather than a bypass. +// rule 2 (plan_hash) → every capability the launch reaches for must be +// DECLARED and satisfied by a live warrant. A bound the +// launch needs but the warrant omits is a REFUSAL, not a +// default-allow. +// +// Threat model is drift, not an adversary (spec §3), so no signature is verified +// here. See spec §6 for what that does and does not buy. +import type { ApprovalRecord, WarrantBounds } from "./ledger.js"; + +export type SizeClass = "SMALL" | "MEDIUM"; +export type DeviceAccess = "none" | "ro" | "rw"; + +/** What the gate knows about the launch in front of it. */ +export interface LaunchFacts { + plan_hash?: string; + tier?: string; + executor?: string; + /** From estimate.ts. UNDEFINED means unresolved, which is treated as + * over-threshold (§4.4) — never as SMALL. */ + sizeClass?: SizeClass; + device?: DeviceAccess; + /** Solves already recorded against this warrant, for the max_solves bound. */ + solvesSoFar?: number; +} + +export interface WarrantRefusal { + ok: false; + /** One line, naming the class, the offending bound, and its margin. */ + reason: string; + /** The bound keys a covering warrant would have to declare — spec §5.2's third + * element, and the payload G-9 leans on to derive an approval request. */ + required: string[]; + /** Echoed so a caller can join the refusal to the plan it needs. */ + plan_hash?: string; +} + +export type WarrantCheck = { ok: true } | WarrantRefusal; + +const SIZE_ORDER: Record = { SMALL: 0, MEDIUM: 1 }; +const DEVICE_ORDER: Record = { none: 0, ro: 1, rw: 2 }; + +/** Expiry in ms. An unparseable expiry is ALREADY EXPIRED — a warrant whose + * lifetime cannot be established must not read as live (same fail-closed direction + * as §4.4's estimator inversion). */ +function expiryMs(w: ApprovalRecord): number { + const t = Date.parse(w.expires_at); + return Number.isNaN(t) ? -Infinity : t; +} + +/** The live warrant for `planHash` expiring latest, or undefined if none is live. */ +export function liveWarrant( + planHash: string, + approvals: readonly ApprovalRecord[], + now: number, +): ApprovalRecord | undefined { + let best: ApprovalRecord | undefined; + for (const a of approvals) { + if (a.type !== "approval" || a.plan_hash !== planHash) continue; + if (expiryMs(a) <= now) continue; + if (!best || expiryMs(a) > expiryMs(best)) best = a; + } + return best; +} + +/** True when a warrant exists for the plan but every one of them has lapsed — + * distinguished from "never approved" so the refusal can say which. */ +function hasLapsed(planHash: string, approvals: readonly ApprovalRecord[], now: number): boolean { + return approvals.some((a) => a.type === "approval" && a.plan_hash === planHash && expiryMs(a) <= now); +} + +/** Which gated capabilities this launch reaches for, as bound keys. Empty means the + * launch is entirely inside the ungated free set. */ +export function gatedCapabilities(facts: LaunchFacts): string[] { + const needs: string[] = []; + // Spend: anything but the local free tier. + if (facts.tier !== undefined && facts.tier !== "free") needs.push("tier"); + else if (facts.executor === "remote") needs.push("tier"); // remote is spend even at free tier + // Cost proxy. Unresolved counts as gated — see §4.4. + if (facts.sizeClass === undefined || facts.sizeClass !== "SMALL") needs.push("max_size_class"); + // Device access beyond simulator-only. + if (facts.device !== undefined && facts.device !== "none") needs.push("device"); + return needs; +} + +/** The §5.1 check. */ +export function checkWarrant( + facts: LaunchFacts, + approvals: readonly ApprovalRecord[], + now: number, +): WarrantCheck { + const needs = gatedCapabilities(facts); + if (needs.length === 0) return { ok: true }; // inside the free set — nothing to authorise + + const unresolvedSize = facts.sizeClass === undefined; + + // ── rule 1: no plan_hash → free set only ── + if (!facts.plan_hash) { + return { + ok: false, + required: needs, + reason: unresolvedSize + ? "solve size is unresolved (estimate.ts could not resolve levels), so it is treated as over-threshold and needs an approved plan — declare max_size_class, or make the spec's levels resolvable" + : `this launch needs an approved plan covering ${needs.join(", ")} — it is outside the ungated free set (local, free tier, SMALL, device none). Set solvespec.plan_hash to an approved plan`, + }; + } + + // ── rule 2: plan_hash → a live warrant must DECLARE and satisfy each capability ── + const w = liveWarrant(facts.plan_hash, approvals, now); + if (!w) { + return { + ok: false, + plan_hash: facts.plan_hash, + required: needs, + reason: hasLapsed(facts.plan_hash, approvals, now) + ? `the warrant for plan ${facts.plan_hash} has expired — re-approve it (needs ${needs.join(", ")})` + : `no approved warrant for plan ${facts.plan_hash} — approve it declaring ${needs.join(", ")}`, + }; + } + + const refuse = (reason: string): WarrantRefusal => ({ + ok: false, + reason, + required: needs, + plan_hash: facts.plan_hash, + }); + const b: WarrantBounds = w.bounds; + + if (needs.includes("tier")) { + if (b.tier === undefined) + return refuse(`warrant for ${facts.plan_hash} does not declare tier, which this launch needs — approve it with --tier ${facts.tier ?? ""}`); + if (b.tier !== facts.tier) + return refuse(`tier: launch is "${facts.tier}" but the warrant authorises "${b.tier}"`); + } + + if (needs.includes("max_size_class")) { + if (b.max_size_class === undefined) + return refuse(`warrant for ${facts.plan_hash} does not declare max_size_class, which this launch needs — approve it with --max-size-class ${facts.sizeClass ?? "MEDIUM"}`); + if (unresolvedSize) + return refuse(`solve size is unresolved, so it cannot be shown to fit the warrant's max_size_class ${b.max_size_class} — make the spec's levels resolvable`); + if (SIZE_ORDER[facts.sizeClass!] > SIZE_ORDER[b.max_size_class]) + return refuse(`max_size_class: launch is ${facts.sizeClass} but the warrant authorises up to ${b.max_size_class}`); + } + + if (needs.includes("device")) { + if (b.device === undefined) + return refuse(`warrant for ${facts.plan_hash} does not declare device, which this launch needs — approve it with --device ${facts.device ?? "ro"}`); + if (DEVICE_ORDER[facts.device!] > DEVICE_ORDER[b.device]) + return refuse(`device: launch needs "${facts.device}" but the warrant authorises "${b.device}"`); + } + + // max_solves is only checked when declared — it bounds a campaign, and a warrant + // that omits it simply does not cap the count (unlike the capability bounds above, + // an absent count is not a capability the launch "reaches for"). + if (b.max_solves !== undefined) { + const used = facts.solvesSoFar ?? 0; + if (used >= b.max_solves) + return refuse(`max_solves: ${used} of ${b.max_solves} already used under plan ${facts.plan_hash} — re-approve to extend`); + } + + return { ok: true }; +} diff --git a/packages/amico-run/test/gate_warrant.test.ts b/packages/amico-run/test/gate_warrant.test.ts new file mode 100644 index 00000000..42605fd2 --- /dev/null +++ b/packages/amico-run/test/gate_warrant.test.ts @@ -0,0 +1,113 @@ +// The gate's warrant step (spec-20260727-164748 §5.1, plan CLI-2) as wired into +// runGate. warrant.test.ts covers the decision matrix; this covers the WIRING: +// that the step is genuinely absent without a context, that it runs after the +// consistency checks, and that it refuses before any hash is minted. +import { describe, it, expect } from "vitest"; +import { runGate, type WarrantContext } from "../src/gate.js"; +import type { ApprovalRecord } from "../src/ledger.js"; +import { DEFAULT_ALLOWLIST, DEFAULT_SUPPORT, type AuthoringConfig } from "../src/authoring.js"; + +const NOW = Date.parse("2026-07-27T20:00:00Z"); + +/** A schema-valid spec that passes steps 1-4: free tier needs a sandbox env. */ +const spec = (over: Record = {}) => ({ + schema_version: "5", + lab_id: "default", + script_path: "/tmp/solve.jl", + tier: "free", + env: { kind: "sandbox" }, + ...over, +}); + +const SCRIPT = "using Piccolo\n"; +const AUTHORING: AuthoringConfig = { + allowlist: DEFAULT_ALLOWLIST, + support_set: DEFAULT_SUPPORT, + verify_tolerance: 1e-4, +}; + +const ctx = (over: Partial = {}): WarrantContext => ({ + approvals: [], + now: NOW, + sizeClass: "SMALL", + device: "none", + ...over, +}); + +const warrant = (bounds: ApprovalRecord["bounds"], plan_hash = "9f2c"): ApprovalRecord => ({ + type: "approval", + ts: new Date(NOW - 60_000).toISOString(), + plan_hash, + bounds, + expires_at: new Date(NOW + 1_800_000).toISOString(), + issued_by: "user:cli", +}); + +describe("the flag is the absence of the context", () => { + it("with NO warrant context, a launch that would be refused still passes", () => { + // hpc + MEDIUM + device rw is squarely outside the free set, and there is no + // warrant anywhere — but the step does not exist, so nothing changes. This is + // what makes CLI-2 safe to land before anyone opts in. + const r = runGate(spec({ tier: "hpc", executor: "local" }), SCRIPT, AUTHORING); + // fails for its own pre-existing reason (hpc cannot run locally), NOT for a warrant + expect(r.ok).toBe(false); + expect(r.ok === false && r.reason).toContain("cloud"); + expect(r.ok === false && r.refusal).toBeUndefined(); + }); + + it("a free-set launch passes with the context armed", () => { + const r = runGate(spec(), SCRIPT, AUTHORING, ctx()); + expect(r.ok).toBe(true); + }); +}); + +describe("ordering", () => { + it("a schema failure reads as a schema failure, not as unwarranted", () => { + const r = runGate({ lab_id: "x" }, SCRIPT, AUTHORING, ctx({ sizeClass: "MEDIUM" })); + expect(r.ok).toBe(false); + expect(r.ok === false && r.reason).toContain("solvespec schema"); + expect(r.ok === false && r.refusal).toBeUndefined(); + }); + + it("a tier/env inconsistency reads as that, not as unwarranted", () => { + const r = runGate(spec({ env: { kind: "project" } }), SCRIPT, AUTHORING, ctx({ sizeClass: "MEDIUM" })); + expect(r.ok).toBe(false); + expect(r.ok === false && r.reason).toContain("sandbox"); + expect(r.ok === false && r.refusal).toBeUndefined(); + }); +}); + +describe("the warrant step itself", () => { + it("refuses a MEDIUM launch with no plan_hash, and mints NO stamp", () => { + const r = runGate(spec(), SCRIPT, AUTHORING, ctx({ sizeClass: "MEDIUM" })); + expect(r.ok).toBe(false); + expect(r.ok === false && r.refusal?.required).toContain("max_size_class"); + expect("stamp" in r).toBe(false); // no hash for a launch that will not run + }); + + it("allows it once a covering warrant is approved for its plan_hash", () => { + const r = runGate(spec({ plan_hash: "9f2c" }), SCRIPT, AUTHORING, { + ...ctx({ sizeClass: "MEDIUM" }), + approvals: [warrant({ max_size_class: "MEDIUM" })], + }); + expect(r.ok).toBe(true); + }); + + it("carries the §5.2 structured refusal alongside the one-line reason", () => { + const r = runGate(spec({ plan_hash: "9f2c" }), SCRIPT, AUTHORING, { + ...ctx({ sizeClass: "MEDIUM" }), + approvals: [warrant({})], + }); + expect(r.ok).toBe(false); + if (r.ok) return; + expect(r.refusal?.plan_hash).toBe("9f2c"); + expect(r.refusal?.required).toContain("max_size_class"); + expect(r.reason).toContain("max_size_class"); + }); + + it("an unresolved size refuses even inside an otherwise-free launch (§4.4)", () => { + const r = runGate(spec(), SCRIPT, AUTHORING, ctx({ sizeClass: undefined })); + expect(r.ok).toBe(false); + expect(r.ok === false && r.reason).toMatch(/unresolved/i); + }); +}); diff --git a/packages/amico-run/test/warrant.test.ts b/packages/amico-run/test/warrant.test.ts new file mode 100644 index 00000000..9411c0ea --- /dev/null +++ b/packages/amico-run/test/warrant.test.ts @@ -0,0 +1,142 @@ +// Warrant resolution + bound checking (spec-20260727-164748 §5.1, plan task CLI-2). +// Pure: no fs, no clock — `now` is a parameter so expiry is testable. +// +// Refusal paths first. A gate that fails OPEN is worse than no gate, and the whole +// §5.1 design rests on one asymmetry: a missing plan_hash, or a bound the warrant +// omits, may only ever RESTRICT a launch — never widen it. +import { describe, it, expect } from "vitest"; +import { checkWarrant, type LaunchFacts } from "../src/warrant.js"; +import type { ApprovalRecord } from "../src/ledger.js"; + +const NOW = Date.parse("2026-07-27T20:00:00Z"); +const iso = (min: number) => new Date(NOW + min * 60_000).toISOString(); + +const warrant = (over: Partial = {}): ApprovalRecord => ({ + type: "approval", + ts: iso(-5), + plan_hash: "9f2c", + bounds: { max_solves: 8, tier: "free", max_size_class: "MEDIUM", device: "none" }, + expires_at: iso(30), + issued_by: "user:cli", + ...over, +}); + +/** The ungated free set: local free-tier, SMALL, no device. */ +const freeLaunch = (over: Partial = {}): LaunchFacts => ({ + tier: "free", + executor: "local", + sizeClass: "SMALL", + device: "none", + ...over, +}); + +describe("§5.1 rule 1 — no plan_hash", () => { + it("a launch inside the free set is allowed", () => { + expect(checkWarrant(freeLaunch(), [], NOW).ok).toBe(true); + }); + + it("each way out of the free set refuses INDEPENDENTLY", () => { + const cases: [string, LaunchFacts][] = [ + ["paid tier", freeLaunch({ tier: "hpc" })], + ["remote executor", freeLaunch({ executor: "remote" })], + ["MEDIUM size", freeLaunch({ sizeClass: "MEDIUM" })], + ["device write", freeLaunch({ device: "rw" })], + ["device read", freeLaunch({ device: "ro" })], + ]; + for (const [label, facts] of cases) { + const r = checkWarrant(facts, [], NOW); + expect(r.ok, label).toBe(false); + expect(r.ok === false && r.reason).toContain("approved plan"); + } + }); + + it("an UNRESOLVED size refuses — §4.4, the gate inverts the estimator's fail-open", () => { + // estimate.ts sizes an unresolved `levels` as SMALL (knot_point_state_dim stays + // 1). As a gate input that is a silent widening path, so undefined is treated as + // over-threshold, NOT as SMALL. + const r = checkWarrant(freeLaunch({ sizeClass: undefined }), [], NOW); + expect(r.ok).toBe(false); + expect(r.ok === false && r.reason).toMatch(/unresolved|could not/i); + }); +}); + +describe("§5.1 rule 2 — plan_hash present", () => { + const launch = freeLaunch({ tier: "hpc", executor: "remote", sizeClass: "MEDIUM", plan_hash: "9f2c" }); + + it("a covering warrant allows it", () => { + expect(checkWarrant(launch, [warrant({ bounds: { tier: "hpc", max_size_class: "MEDIUM" } })], NOW).ok).toBe(true); + }); + + it("no warrant for that plan_hash refuses", () => { + expect(checkWarrant(launch, [warrant({ plan_hash: "other" })], NOW).ok).toBe(false); + }); + + it("an EXPIRED warrant refuses", () => { + const r = checkWarrant(launch, [warrant({ bounds: { tier: "hpc", max_size_class: "MEDIUM" }, expires_at: iso(-1) })], NOW); + expect(r.ok).toBe(false); + expect(r.ok === false && r.reason).toMatch(/expired/i); + }); + + it("an unparseable expiry refuses — fail closed, never live", () => { + const r = checkWarrant(launch, [warrant({ bounds: { tier: "hpc" }, expires_at: "nope" })], NOW); + expect(r.ok).toBe(false); + }); + + // The load-bearing asymmetry. + it("a bound the launch NEEDS but the warrant OMITS refuses — never default-allow", () => { + // Warrant declares only the tier; the launch is also MEDIUM, which is unbounded. + const r = checkWarrant(launch, [warrant({ bounds: { tier: "hpc" } })], NOW); + expect(r.ok).toBe(false); + expect(r.ok === false && r.reason).toContain("max_size_class"); + }); + + it("each exceeded bound names ITSELF and its margin", () => { + const tooMany = checkWarrant( + { ...launch, solvesSoFar: 8 }, + [warrant({ bounds: { tier: "hpc", max_size_class: "MEDIUM", max_solves: 8 } })], + NOW, + ); + expect(tooMany.ok).toBe(false); + expect(tooMany.ok === false && tooMany.reason).toContain("max_solves"); + expect(tooMany.ok === false && tooMany.reason).toContain("8"); + + const wrongTier = checkWarrant(launch, [warrant({ bounds: { tier: "free", max_size_class: "MEDIUM" } })], NOW); + expect(wrongTier.ok === false && wrongTier.reason).toContain("tier"); + }); + + it("SMALL is within a MEDIUM bound, but MEDIUM is not within SMALL", () => { + const small = { ...launch, sizeClass: "SMALL" as const }; + expect(checkWarrant(small, [warrant({ bounds: { tier: "hpc", max_size_class: "SMALL" } })], NOW).ok).toBe(true); + expect(checkWarrant(small, [warrant({ bounds: { tier: "hpc", max_size_class: "MEDIUM" } })], NOW).ok).toBe(true); + expect(checkWarrant(launch, [warrant({ bounds: { tier: "hpc", max_size_class: "SMALL" } })], NOW).ok).toBe(false); + }); + + it("device: ro is within rw, and none is within anything", () => { + const roLaunch = { ...launch, device: "ro" as const }; + expect(checkWarrant(roLaunch, [warrant({ bounds: { tier: "hpc", max_size_class: "MEDIUM", device: "rw" } })], NOW).ok).toBe(true); + expect(checkWarrant(roLaunch, [warrant({ bounds: { tier: "hpc", max_size_class: "MEDIUM", device: "none" } })], NOW).ok).toBe(false); + }); + + it("the newest live warrant wins when several exist for one plan", () => { + const stingy = warrant({ bounds: { tier: "hpc", max_size_class: "SMALL" }, expires_at: iso(5) }); + const generous = warrant({ bounds: { tier: "hpc", max_size_class: "MEDIUM" }, expires_at: iso(60) }); + expect(checkWarrant(launch, [stingy, generous], NOW).ok).toBe(true); + }); +}); + +describe("the refusal contract (§5.2)", () => { + it("names the class, the offending bound, and what a covering warrant must declare", () => { + const r = checkWarrant(freeLaunch({ tier: "hpc", plan_hash: "9f2c" }), [warrant({ bounds: {} })], NOW); + expect(r.ok).toBe(false); + if (r.ok) return; + expect(r.reason).toContain("tier"); + expect(r.required).toContain("tier"); // the declaration a covering warrant needs + expect(r.plan_hash).toBe("9f2c"); + }); + + it("a refusal with no plan_hash tells you a plan is what is missing", () => { + const r = checkWarrant(freeLaunch({ tier: "hpc" }), [], NOW); + expect(r.ok === false && r.required).toContain("tier"); + expect(r.ok === false && r.plan_hash).toBeUndefined(); + }); +}); diff --git a/packages/schema/schemas/ledger-record.schema.json b/packages/schema/schemas/ledger-record.schema.json index 0dcea61b..9e6a3b6a 100644 --- a/packages/schema/schemas/ledger-record.schema.json +++ b/packages/schema/schemas/ledger-record.schema.json @@ -190,7 +190,7 @@ "properties": { "max_solves": { "type": "integer", "minimum": 1 }, "tier": { "type": "string", "minLength": 1 }, - "max_duration_s": { "type": "number", "exclusiveMinimum": 0 }, + "max_size_class": { "enum": ["SMALL", "MEDIUM"], "description": "G-8: bounds the COST PROXY that actually exists. estimate.ts computes memory (sizeClass/score/estimatedBytes) and nothing in amico-run estimates wall-clock, so an earlier max_duration_s bound had no signal behind it and was removed rather than left implying a guarantee about time. A real duration estimator is the deferred C2 work." }, "device": { "enum": ["none", "ro", "rw"] } } }, diff --git a/packages/schema/test/ledger-record.test.ts b/packages/schema/test/ledger-record.test.ts index 222b5713..357d916a 100644 --- a/packages/schema/test/ledger-record.test.ts +++ b/packages/schema/test/ledger-record.test.ts @@ -167,7 +167,7 @@ const approval = () => ({ type: "approval", ts: "2026-07-27T20:00:00Z", plan_hash: "9f2c", - bounds: { max_solves: 8, tier: "free", max_duration_s: 1800, device: "none" }, + bounds: { max_solves: 8, tier: "free", max_size_class: "MEDIUM", device: "none" }, expires_at: "2026-07-27T21:00:00Z", issued_by: "user:ui", }); @@ -203,10 +203,18 @@ describe("ledger-record schema — approval (capability warrant)", () => { expect(validate({ ...approval(), bounds: { device: "yes" } }, "ledger-record").ok).toBe(false); }); - it("max_solves must be a positive integer; max_duration_s must be positive", () => { + it("max_size_class is the cost proxy that exists — G-8 removed max_duration_s", () => { + for (const c of ["SMALL", "MEDIUM"]) { + expect(validate({ ...approval(), bounds: { max_size_class: c } }, "ledger-record").ok, c).toBe(true); + } + expect(validate({ ...approval(), bounds: { max_size_class: "LARGE" } }, "ledger-record").ok).toBe(false); + // the removed bound must not quietly validate again + expect(validate({ ...approval(), bounds: { max_duration_s: 1800 } }, "ledger-record").ok).toBe(false); + }); + + it("max_solves must be a positive integer", () => { expect(validate({ ...approval(), bounds: { max_solves: 0 } }, "ledger-record").ok).toBe(false); expect(validate({ ...approval(), bounds: { max_solves: 1.5 } }, "ledger-record").ok).toBe(false); - expect(validate({ ...approval(), bounds: { max_duration_s: 0 } }, "ledger-record").ok).toBe(false); }); it("plan_hash must be non-empty — an empty hash would join every launch", () => { From 4cabaa28ef31f3630c3da149c13ef2b2bb305624 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Mon, 27 Jul 2026 21:59:58 -0400 Subject: [PATCH 04/27] =?UTF-8?q?feat(gate):=20arm=20the=20warrant=20check?= =?UTF-8?q?=20in=20the=20launch=20path=20=E2=80=94=20the=20loop=20is=20pla?= =?UTF-8?q?yable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the last mechanical gap: the gate could refuse, but nothing built it a WarrantContext, so the check never ran outside tests. Now `amico run --spec` does the whole loop. warrant_context.ts is the ONLY I/O in the warrant path — it owns the ledger read, the clock, and the size estimate so gate.ts and warrant.ts stay pure. AMICO_WARRANTS is the entire flag surface: unset, the assembler returns undefined and gate.ts treats absence as "the step does not exist", so nothing changes for anyone who has not opted in. Fail-closed in all three places it could have leaked: - a missing OR unparseable ledger yields no approvals, so gated launches refuse rather than sailing through unwarranted; - an unsizeable script stays undefined rather than falling back to SMALL. estimate.ts degrades that way internally (unresolved levels leaves knot_point_state_dim at 1) and reproducing it here would re-open exactly the silent widening path §4.4 exists to close; - memoryScore's throw on missing N becomes undefined, not a crash — a spec we cannot size must be gated, not rejected as malformed. launch.ts also prints the refusal's STRUCTURED form on stdout beside the human line, which is G-9's payload: a caller can derive an approval request without scraping prose. Verified as a real loop through the built dist/amico.js: 1. MEDIUM solve, no warrant → refused, names max_size_class 2. plan_hash added, no warrant → refused, names the plan 3. warrant too narrow (SMALL) → refused, names the margin 4. warrant at MEDIUM → gate passes, Julia launches 5. AMICO_WARRANTS unset → no warrant step at all amico-run 671 pass / 12 skipped · schema 113 · extension 777 · tsc clean. Co-Authored-By: Claude Opus 5 --- packages/amico-run/src/launch.ts | 19 ++- packages/amico-run/src/warrant_context.ts | 79 +++++++++++++ .../amico-run/test/warrant_context.test.ts | 109 ++++++++++++++++++ 3 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 packages/amico-run/src/warrant_context.ts create mode 100644 packages/amico-run/test/warrant_context.test.ts diff --git a/packages/amico-run/src/launch.ts b/packages/amico-run/src/launch.ts index 4fba37eb..e19ca44c 100644 --- a/packages/amico-run/src/launch.ts +++ b/packages/amico-run/src/launch.ts @@ -13,6 +13,7 @@ import { RemoteExecutor } from "./remote_executor.js"; import { ConfigError, type Executor, type Finished, type SubmitOpts } from "./types.js"; import { readAuthoring } from "./authoring.js"; import { runGate } from "./gate.js"; +import { assembleWarrantContext } from "./warrant_context.js"; import { runVerification } from "./verify.js"; import { trySubcommand } from "./subcommands.js"; @@ -148,9 +149,25 @@ export async function launch(argv: string[]): Promise { } const { config: authoring, warning } = readAuthoring(); if (warning) console.error(`amico-run: ${warning}`); - const gate = runGate(specRaw, scriptText, authoring); + // Warrant context (spec-20260727-164748 §5.1). undefined unless AMICO_WARRANTS + // is set, and gate.ts treats absence as "the step does not exist" — so the + // feature is opt-in and this line changes nothing by default. + const warrantCtx = assembleWarrantContext({ + scriptText, + planHash: + typeof (specRaw as { plan_hash?: unknown }).plan_hash === "string" + ? (specRaw as { plan_hash: string }).plan_hash + : undefined, + }); + const gate = runGate(specRaw, scriptText, authoring, warrantCtx); if (!gate.ok) { console.error(`amico-run: gate: ${gate.reason}`); + // §5.2: a warrant refusal ALSO prints its structured form on stdout, so a + // caller (the agent, or the extension) can turn it into an approval request + // without scraping prose. Other gate failures print nothing extra. + if (gate.refusal) { + console.log(JSON.stringify({ error: "warrant_required", ...gate.refusal })); + } return 64; } // env resolution: spec env.project feeds --project unless the flag was explicit diff --git a/packages/amico-run/src/warrant_context.ts b/packages/amico-run/src/warrant_context.ts new file mode 100644 index 00000000..7aae94dd --- /dev/null +++ b/packages/amico-run/src/warrant_context.ts @@ -0,0 +1,79 @@ +// Assembles the gate's WarrantContext (spec-20260727-164748 §5.1) from the world: +// the ledger, the clock, and the size estimate. Kept OUT of gate.ts and warrant.ts so +// both stay pure — this is the only place in the warrant path that does I/O. +// +// OPT-IN: returns undefined unless AMICO_WARRANTS is truthy. gate.ts treats an absent +// context as "the step does not exist", so the whole feature is off by default and the +// env var is the entire flag surface. That is deliberate for the dogfood phase (plan +// CLI step 4) — the internal ring turns it on, nobody else changes behavior. +import { readRecords, type ApprovalRecord } from "./ledger.js"; +import { extractKeyVars, memoryScore, tshirtSize } from "./estimate.js"; +import type { WarrantContext } from "./gate.js"; +import type { DeviceAccess, SizeClass } from "./warrant.js"; + +/** Truthy per the ops convention: "1"/"true"/"yes", case-insensitive. */ +export function warrantsEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const v = (env.AMICO_WARRANTS ?? "").trim().toLowerCase(); + return v === "1" || v === "true" || v === "yes"; +} + +/** Size class for a script, or UNDEFINED when it cannot be resolved. + * + * §4.4: undefined is the honest answer and the gate treats it as over-threshold. + * Do NOT be tempted to fall back to SMALL here — estimate.ts already degrades that + * way internally (unresolved `levels` leaves knot_point_state_dim at 1), which is + * exactly the silent widening path the gate exists to close. `memoryScore` throws + * when N is missing; that throw becomes undefined rather than a crash, because a + * spec we cannot size must be gated, not rejected as malformed. */ +export function sizeClassFor(scriptText: string): SizeClass | undefined { + if (!scriptText.trim()) return undefined; // problem_spec specs carry no script + try { + const vars = extractKeyVars(scriptText); + if (!vars.levels) return undefined; // unresolved levels → unresolved size + return tshirtSize(memoryScore(vars)); + } catch { + return undefined; + } +} + +/** Solves already recorded against a plan, for the max_solves bound. Counts `solve` + * rows carrying this plan_hash — cheap because the ledger is the count-things store + * (spec §4.5) rather than a second counter that could drift. */ +export function solvesUnderPlan(planHash: string, records: readonly { type: string; plan_hash?: string }[]): number { + return records.filter((r) => r.type === "solve" && r.plan_hash === planHash).length; +} + +export interface AssembleOptions { + scriptText: string; + planHash?: string; + /** Device access this launch needs. Solves are simulator-only today, so the + * default is "none"; the device path passes its own value when it lands. */ + device?: DeviceAccess; + env?: NodeJS.ProcessEnv; + now?: number; +} + +/** The context, or undefined when warrants are off (which disarms the gate step). */ +export function assembleWarrantContext(opts: AssembleOptions): WarrantContext | undefined { + if (!warrantsEnabled(opts.env ?? process.env)) return undefined; + + // A ledger that is missing or unreadable yields NO approvals, which fails closed: + // every gated launch refuses rather than sailing through unwarranted. + let approvals: ApprovalRecord[] = []; + let all: { type: string; plan_hash?: string }[] = []; + try { + const records = readRecords(); + all = records as unknown as { type: string; plan_hash?: string }[]; + approvals = records.filter((r): r is ApprovalRecord => r.type === "approval"); + } catch { + /* no ledger → no warrants → gated launches refuse */ + } + + return { + approvals, + now: opts.now ?? Date.now(), + sizeClass: sizeClassFor(opts.scriptText), + device: opts.device ?? "none", + solvesSoFar: opts.planHash ? solvesUnderPlan(opts.planHash, all) : undefined, + }; +} diff --git a/packages/amico-run/test/warrant_context.test.ts b/packages/amico-run/test/warrant_context.test.ts new file mode 100644 index 00000000..8e4699db --- /dev/null +++ b/packages/amico-run/test/warrant_context.test.ts @@ -0,0 +1,109 @@ +// WarrantContext assembly — the only I/O in the warrant path (spec §5.1). +// The interesting cases are all fail-closed: off by default, no ledger means no +// warrants (so gated launches refuse), and an unsizeable script stays UNDEFINED +// rather than falling back to SMALL (§4.4). +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { assembleWarrantContext, sizeClassFor, solvesUnderPlan, warrantsEnabled } from "../src/warrant_context.js"; + +const SIZEABLE = `using Piccolo\nN = 100\nlevels = [3, 3]\n`; + +describe("warrantsEnabled — the whole flag surface", () => { + it("off unless explicitly enabled", () => { + for (const v of [undefined, "", "0", "false", "no", "off", "maybe"]) { + expect(warrantsEnabled({ AMICO_WARRANTS: v } as NodeJS.ProcessEnv), String(v)).toBe(false); + } + }); + it("on for 1/true/yes, case-insensitively", () => { + for (const v of ["1", "true", "TRUE", "yes", " Yes "]) { + expect(warrantsEnabled({ AMICO_WARRANTS: v } as NodeJS.ProcessEnv), v).toBe(true); + } + }); + it("returns undefined context when off — which disarms the gate step entirely", () => { + expect(assembleWarrantContext({ scriptText: SIZEABLE, env: {} as NodeJS.ProcessEnv })).toBeUndefined(); + }); +}); + +describe("sizeClassFor — never falls back to SMALL", () => { + it("sizes a resolvable script", () => { + expect(sizeClassFor(SIZEABLE)).toBeDefined(); + }); + it("UNRESOLVED levels → undefined, not SMALL (§4.4)", () => { + // estimate.ts would internally leave knot_point_state_dim at 1 here and size it + // SMALL. That is the silent widening path; the assembler must not reproduce it. + expect(sizeClassFor(`using Piccolo\nN = 100\n`)).toBeUndefined(); + }); + it("missing N → undefined rather than a crash", () => { + expect(sizeClassFor(`using Piccolo\nlevels = [3, 3]\n`)).toBeUndefined(); + }); + it("empty script (a problem_spec launch) → undefined", () => { + expect(sizeClassFor("")).toBeUndefined(); + expect(sizeClassFor(" \n ")).toBeUndefined(); + }); + it("a bigger problem sizes larger than a small one", () => { + const small = sizeClassFor(`N = 2\nlevels = [2]\n`); + const big = sizeClassFor(`N = 500\nlevels = [5, 5, 5]\n`); + expect(small).toBe("SMALL"); + expect(big).toBe("MEDIUM"); + }); +}); + +describe("solvesUnderPlan", () => { + it("counts only solve rows for that plan", () => { + const rows = [ + { type: "solve", plan_hash: "a" }, + { type: "solve", plan_hash: "a" }, + { type: "solve", plan_hash: "b" }, + { type: "solve" }, + { type: "approval", plan_hash: "a" }, + ]; + expect(solvesUnderPlan("a", rows)).toBe(2); + expect(solvesUnderPlan("zzz", rows)).toBe(0); + }); +}); + +describe("assembleWarrantContext with a real ledger", () => { + let dir: string; + const prev = process.env.AMICO_LEDGER; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "warrant-ctx-")); + process.env.AMICO_LEDGER = join(dir, "runs.jsonl"); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + if (prev === undefined) delete process.env.AMICO_LEDGER; + else process.env.AMICO_LEDGER = prev; + }); + + const ON = { AMICO_WARRANTS: "1" } as NodeJS.ProcessEnv; + + it("a MISSING ledger yields no approvals — fail closed, not fail open", () => { + const ctx = assembleWarrantContext({ scriptText: SIZEABLE, env: ON }); + expect(ctx).toBeDefined(); + expect(ctx!.approvals).toEqual([]); + }); + + it("an UNREADABLE ledger also yields no approvals rather than throwing", () => { + writeFileSync(process.env.AMICO_LEDGER!, "{not json\n"); + const ctx = assembleWarrantContext({ scriptText: SIZEABLE, env: ON }); + expect(ctx).toBeDefined(); + expect(ctx!.approvals).toEqual([]); + }); + + it("picks up approval rows and counts solves under the plan", () => { + const rows = [ + { type: "approval", ts: "t", plan_hash: "p1", bounds: { max_solves: 4 }, expires_at: "t2", issued_by: "user:cli" }, + { type: "solve", ts: "t", plan_hash: "p1", structure_hash: "s", problem_hash: "h", kind: "control", tier: "spec", summary: {}, source: "user", outcome: {} }, + ]; + writeFileSync(process.env.AMICO_LEDGER!, rows.map((r) => JSON.stringify(r)).join("\n") + "\n"); + const ctx = assembleWarrantContext({ scriptText: SIZEABLE, planHash: "p1", env: ON }); + expect(ctx!.approvals).toHaveLength(1); + expect(ctx!.solvesSoFar).toBe(1); + }); + + it("device defaults to none — solves are simulator-only today", () => { + expect(assembleWarrantContext({ scriptText: SIZEABLE, env: ON })!.device).toBe("none"); + }); +}); From 4be6e0a8001e34c7288c18cf54cdaba268014700 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Mon, 27 Jul 2026 22:24:40 -0400 Subject: [PATCH 05/27] =?UTF-8?q?feat(plugin):=20amicode=5Frequest=5Fappro?= =?UTF-8?q?val=20=E2=80=94=20the=20card's=20trigger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec-20260727-164748 §9.5 / G-9 option 1. The tool exists so a `warrant_required` refusal from amico-run's --spec gate becomes an in-chat Approve button rather than prose asking the researcher to run a CLI verb. It records NOTHING and grants NOTHING — pressing the button is what mints the warrant, through the card's bridge to `amico ledger approve`. The description says so explicitly and tells the agent never to shell the approve verb itself, because that separation IS the provenance argument: an agent-written approval row would leave the ledger reading "the agent says the user approved". The card renders from the tool INPUT (parseApprovalInput), mirroring amicode_ask rather than using a sentinel — a request is an ask, not a record of something that happened. Guidance encoded in the args: declare ONLY the bounds the launch needs, since the gate refuses a launch needing a bound the warrant omits, which makes an over-broad warrant strictly worse than a precise one. extension 777 pass. Co-Authored-By: Claude Opus 5 --- .../opencode-plugin/amicode_tools.ts | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/packages/extension/opencode-plugin/amicode_tools.ts b/packages/extension/opencode-plugin/amicode_tools.ts index 5e523d0d..aa328e70 100644 --- a/packages/extension/opencode-plugin/amicode_tools.ts +++ b/packages/extension/opencode-plugin/amicode_tools.ts @@ -228,7 +228,52 @@ const RYDBERG_SCOPE_NOTE = // creation with PluginInput; we need nothing from it today. export const AmicodeTools = async (_input: unknown) => ({ tool: { - amicode_ask: { + // Capability warrant request (spec-20260727-164748 §9.5 / G-9). The CARD is the + // point: this tool exists so a refusal from amico-run's --spec gate becomes a + // button the researcher can press, instead of prose asking them to run a CLI verb. + // The tool records NOTHING and authorises NOTHING — it only renders the ask. The + // warrant is minted by the card's bridge through `amico ledger approve`, so the + // agent can never approve on the user's behalf (that separation is the whole + // provenance argument in §9.5). + amicode_request_approval: { + description: + "Ask the researcher to approve a capability warrant, rendering an in-chat Approve " + + "button. Call this when `amico run --spec` refused with `warrant_required`: pass the " + + "plan_hash it named and the bounds from its `required` list. This tool NEITHER " + + "records nor grants anything — pressing the button is what mints the warrant. " + + "End your turn after calling it; never approve on the user's behalf, and never " + + "shell `amico ledger approve` yourself.", + args: { + plan_hash: { + type: "string", + description: "The plan being approved — use the plan_hash the gate's refusal named.", + }, + bounds: { + type: "object", + description: + "What to authorise. Declare ONLY what the launch needs (the gate refuses a launch " + + "needing a bound the warrant omits, so an over-broad warrant is worse than a precise " + + "one): {max_solves?: int>=1, tier?: string, max_size_class?: 'SMALL'|'MEDIUM', " + + "device?: 'none'|'ro'|'rw'}.", + }, + rationale: { + type: "string", + description: "One line on WHY this needs approving — shown on the card. Null for none.", + }, + }, + async execute(a: { plan_hash: string; bounds?: Record | null; rationale?: string | null }) { + if (!a.plan_hash || a.plan_hash.trim() === "") return "Cannot request approval: plan_hash is required."; + // Returned text is agent-directed only — the card renders from the tool INPUT + // (parseApprovalInput), the same way the ask card does. + const declared = a.bounds && typeof a.bounds === "object" ? Object.keys(a.bounds).join(", ") : "none"; + return ( + `Approval requested for plan ${a.plan_hash.trim()} (bounds declared: ${declared}). ` + + `The researcher now has an Approve button in chat. Stop here and wait — do not ` + + `re-run the solve until they press it, and do not mint the warrant yourself.` + ); + }, + }, + amicode_ask: { description: "DEPRECATED — prefer the native `question` tool (turn-blocking form with options, " + "descriptions, and custom answers). Kept for compatibility: presents ONE multiple-choice " + From b197745dece0a5dbdde7927ad65cb32c8518d0c2 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Mon, 27 Jul 2026 22:39:53 -0400 Subject: [PATCH 06/27] fix(plugin): amicode_recommend emits a sentinel naming the param MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live session showed five identical "Recommend updated ✓" chips in a row. The interview fires one amicode_recommend per knob, and the tool emitted no diff sentinel — so the in-chat receipt had nothing to distinguish one call from the next and every one rendered the same line. Now each carries {param: value}, so the run reads `Recommend · N 100` / `Recommend · Q 1e5`. Five distinct lines are a record of what was recommended; five identical ones are noise. `recommend` stays OUT of card.tsx's INLINE_KINDS — there is no entity view for it, so it renders as an informative one-liner and (per the paired fork change) stays non-clickable rather than opening an empty dialog. extension 777 pass. Co-Authored-By: Claude Opus 5 --- packages/extension/opencode-plugin/amicode_tools.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/extension/opencode-plugin/amicode_tools.ts b/packages/extension/opencode-plugin/amicode_tools.ts index aa328e70..ddae7d0d 100644 --- a/packages/extension/opencode-plugin/amicode_tools.ts +++ b/packages/extension/opencode-plugin/amicode_tools.ts @@ -1218,7 +1218,18 @@ export const AmicodeTools = async (_input: unknown) => ({ ? ((a.provenance[0] as { source?: string }).source ?? "?") : "none"; const auto = a.auto_accepted ? " ⚡auto" : ""; - return `Recommended ${a.param}=${JSON.stringify(a.value)} (${a.confidence ?? "?"}, via ${prov})${auto} [event ${seq}].`; + // Sentinel so the in-chat receipt carries WHICH param was recommended. + // Without it every recommend call rendered an identical "Recommend + // updated" chip, and the interview fires one per knob — so a run of five + // was five indistinguishable lines. `recommend` is deliberately NOT in + // card.tsx's INLINE_KINDS (no entity view exists for it), so this renders + // as an informative one-liner and stays non-clickable. + return ( + `Recommended ${a.param}=${JSON.stringify(a.value)} (${a.confidence ?? "?"}, via ${prov})${auto} [event ${seq}].\n` + + sentinelLine(slug, "recommend", "proposed", seq, { + [String(a.param)]: { from: null, to: a.value }, + }) + ); } catch (err) { return `Cannot record recommendation: ${err instanceof Error ? err.message : String(err)}`; } From 2f86fb3ed51dd53aaab87bee784bc56405a3529b Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 00:24:27 -0400 Subject: [PATCH 07/27] skills: register `pasqal` in the platform-skill documentation anchor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The neutral-atom DEVICE path (solve -> Pulser -> emulator/QPU) lands as a new `surface: public` skill in amico-plugin. DEFAULT_PLATFORM_SKILLS is the documentation/superset anchor for the physics-and-optimization subset — selection itself is purely by frontmatter tag — so the list needs the new name to keep describing what it claims to describe. Vendoring note: a Marketplace build gets its skills from vendor/skills-public, pinned by skills.lock.json. The expanded public set (16 -> 35 skills, oss-hold now empty) reaches those users only once a `skills-public-vX.Y.Z` tag is cut on amico-plugin and the lock is bumped to its sha256. A dev with the amico-plugin checkout resolves the full set today (first-root-wins). Co-Authored-By: Claude Opus 5 (1M context) --- packages/extension/src/opencode_config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/extension/src/opencode_config.ts b/packages/extension/src/opencode_config.ts index 88becb96..b52453a1 100644 --- a/packages/extension/src/opencode_config.ts +++ b/packages/extension/src/opencode_config.ts @@ -161,6 +161,7 @@ export const DEFAULT_LIBRARY_ROOTS = [ * as a superset check: each of these must appear in the discovered public set. */ export const DEFAULT_PLATFORM_SKILLS = [ "atoms", + "pasqal", // the neutral-atom DEVICE path (solve -> Pulser -> emulator/QPU) "bosonic", "fluxonium", "ions", From bd10230e9bb6809607085de968ab2c87fdbedf38 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 09:58:00 -0400 Subject: [PATCH 08/27] fix(warrant): stamp plan_hash on solve records so max_solves actually enforces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `max_solves` warrant bound was structurally inert. `warrant_context.solvesUnderPlan()` counts `solve` ledger rows whose `plan_hash` matches the launch's plan — but `SolveRecord` had no `plan_hash` field and the ledger-record schema's `solve` branch is `additionalProperties: false`. So a row carrying one FAILED VALIDATION on append, and a row without one never matched the filter. The counter was permanently 0 and `warrant.ts`'s `used >= b.max_solves` refusal could never fire. Net effect: a warrant could declare `max_solves: 8` and authorize unlimited solves. Why the tests did not catch it: `warrant_context.test.ts` exercises `solvesUnderPlan()` against rows it builds by hand, which of course carry the field. The seam was tested; the wiring that feeds it never was. The new test drives the real emission path instead — LocalExecutor.settle() → appendRecord → solvesUnderPlan — and fails on the pre-fix tree. Three-part fix: * `SolveRecord.plan_hash?: string` (ledger.ts) * `plan_hash` on the schema's solve branch, `minLength: 1` mirroring solvespec.plan_hash so an empty hash cannot match every warrant * local_executor stamps it from the SOLVESPEC (what the gate validated and what carries the v5 field), omitting it entirely for an ungated free-set launch Found by running the adversarial spec-review loop from the in-flight deliberation front-half design against its own spec: three independent critics on separate lenses, and the interface-boundary lens caught this in shipped code rather than in the spec it was reviewing. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 3 ++ packages/amico-run/src/ledger.ts | 6 +++ packages/amico-run/src/local_executor.ts | 8 ++++ .../test/local_executor_ledger.test.ts | 48 +++++++++++++++++++ .../schema/schemas/ledger-record.schema.json | 3 +- 5 files changed, 67 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4a95eb74..ca5c8c6b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ dist/ !packages/extension/demo/run/run.log packages/extension/vendor/ packages/extension/bin/ + +# superpowers scratch (visual-companion servers, generated galleries) — never committed +.superpowers/ diff --git a/packages/amico-run/src/ledger.ts b/packages/amico-run/src/ledger.ts index b41e3438..1dd52c0b 100644 --- a/packages/amico-run/src/ledger.ts +++ b/packages/amico-run/src/ledger.ts @@ -69,6 +69,12 @@ export interface SolveRecord { source: "user" | "replay" | "simulated"; outcome: SolveOutcome; versions?: Record; + /** The approved plan this solve ran under, copied from `solvespec.plan_hash` at + * emission. Load-bearing, not informational: `warrant_context.solvesUnderPlan()` + * counts `solve` rows by this field to enforce a warrant's `max_solves` bound, so + * without it the bound is inert (the counter is always 0). Absent for an ungated + * free-set launch, which carries no plan_hash by design. */ + plan_hash?: string; } export interface VerdictRecord { diff --git a/packages/amico-run/src/local_executor.ts b/packages/amico-run/src/local_executor.ts index fed30de3..501d5040 100644 --- a/packages/amico-run/src/local_executor.ts +++ b/packages/amico-run/src/local_executor.ts @@ -196,6 +196,14 @@ function emitSolveStanza(runDir: string): void { } if (typeof params.session === "string") rec.session = params.session; if (typeof params.problem === "string") rec.problem = params.problem; + // The warrant join. `max_solves` is counted by matching solve rows against a + // plan_hash (warrant_context.solvesUnderPlan), so this stamp is what makes the + // bound enforce rather than sit at 0 forever. Source is the SOLVESPEC, not + // run.toml: the spec is what the gate validated and what carries the field + // (solvespec v5). Absent on an ungated free-set launch — omit rather than + // writing an empty string, which minLength would reject and which would match + // no warrant anyway. + if (typeof spec.plan_hash === "string" && spec.plan_hash !== "") rec.plan_hash = spec.plan_hash; if (typeof params.warm_start === "string" || params.warm_start === null) rec.warm_start = params.warm_start as string | null; diff --git a/packages/amico-run/test/local_executor_ledger.test.ts b/packages/amico-run/test/local_executor_ledger.test.ts index 985dadcc..ed690076 100644 --- a/packages/amico-run/test/local_executor_ledger.test.ts +++ b/packages/amico-run/test/local_executor_ledger.test.ts @@ -13,6 +13,7 @@ import { join } from "node:path"; import { tmpRoot, fakeJulia } from "./helpers.js"; import { LocalExecutor } from "../src/local_executor.js"; import { readRecords, type SolveRecord } from "../src/ledger.js"; +import { solvesUnderPlan } from "../src/warrant_context.js"; import type { RunEvent, SpecStamp } from "../src/types.js"; async function collect(events: AsyncIterable): Promise { @@ -144,6 +145,53 @@ describe("LocalExecutor.settle() emits the solve ledger stanza", () => { expect(readRecords()).toEqual([]); }); + // REGRESSION (found by adversarial spec review, 2026-07-28): the `max_solves` + // warrant bound was structurally inert. warrant_context.solvesUnderPlan() counts + // `solve` rows whose plan_hash matches, but SolveRecord had no plan_hash and the + // schema's solve branch is additionalProperties:false — so a row carrying one + // failed validation on append, and a row without one never matched. The counter + // was permanently 0 and the bound never tripped. solvesUnderPlan's own unit test + // passed throughout, because it builds its rows by hand: the seam was tested, + // the wiring was not. This test drives the REAL emission path. + it("stamps plan_hash from the solvespec, so the max_solves warrant bound can count", async () => { + const root = tmpRoot(); + const julia = fakeJulia(root, "julia-solve", WRITE_RESULT); + const planHash = "sha256:plan-under-warrant"; + const specWithPlan = { ...PROBLEM_SPEC, plan_hash: planHash }; + const spec: SpecStamp = { canonical: JSON.stringify(specWithPlan), problem_spec: specWithPlan }; + + const h = await new LocalExecutor().submit(undefined, { + runsRoot: join(root, "runs"), + julia: { julia }, + spec, + }); + await collect(h.events); + await h.finished; + + const recs = readRecords().filter((r): r is SolveRecord => r.type === "solve"); + expect(recs).toHaveLength(1); + expect(recs[0].plan_hash).toBe(planHash); + // The join the bound actually depends on. + expect(solvesUnderPlan(planHash, readRecords())).toBe(1); + expect(solvesUnderPlan("sha256:some-other-plan", readRecords())).toBe(0); + }); + + it("omits plan_hash when the solvespec has none (ungated free-set launch)", async () => { + const root = tmpRoot(); + const julia = fakeJulia(root, "julia-solve", WRITE_RESULT); + const spec: SpecStamp = { canonical: JSON.stringify(PROBLEM_SPEC), problem_spec: PROBLEM_SPEC }; + const h = await new LocalExecutor().submit(undefined, { + runsRoot: join(root, "runs"), + julia: { julia }, + spec, + }); + await collect(h.events); + await h.finished; + const recs = readRecords().filter((r): r is SolveRecord => r.type === "solve"); + expect(recs).toHaveLength(1); + expect(recs[0].plan_hash).toBeUndefined(); + }); + it("skips (never throws) when result.toml lacks structure_hash/problem_hash", async () => { const root = tmpRoot(); const julia = fakeJulia( diff --git a/packages/schema/schemas/ledger-record.schema.json b/packages/schema/schemas/ledger-record.schema.json index 9e6a3b6a..de5a799c 100644 --- a/packages/schema/schemas/ledger-record.schema.json +++ b/packages/schema/schemas/ledger-record.schema.json @@ -47,7 +47,8 @@ "wall_s": { "type": "number" } } }, - "versions": { "type": "object", "additionalProperties": { "type": "string" } } + "versions": { "type": "object", "additionalProperties": { "type": "string" } }, + "plan_hash": { "type": "string", "minLength": 1, "description": "The approved plan this solve ran under, copied from solvespec.plan_hash at emission. LOAD-BEARING: warrant_context.solvesUnderPlan() counts solve rows by this field to enforce a warrant's max_solves bound, so a solve branch without it makes that bound inert — the counter is always 0 and the bound never trips (the defect this property fixes). Absent for an ungated free-set launch, which carries no plan_hash by design; minLength guards an empty hash matching every warrant, mirroring solvespec.plan_hash." } } }, { From 75fd50ad85ed8d230a38dad1831e07fe1da69e91 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 10:43:19 -0400 Subject: [PATCH 09/27] feat(schema): hoist $defs.bounds + export validateBounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spec-review budget lens must validate an authored budget against the SHIPPED bound vocabulary. bounds was inline in the approval branch with no $ref and no accessor, so a lens could only restate the key set in prose — the drift that let the long-removed max_duration into a spec example. Behaviour is byte-identical for existing approval rows: 113 pre-existing schema tests unchanged, 5 new. Plan: plan-20260728-104500 Task 1. Co-Authored-By: Claude Opus 5 (1M context) --- .../schema/schemas/ledger-record.schema.json | 24 ++++++++++------- packages/schema/src/index.ts | 16 ++++++++++++ packages/schema/test/bounds.test.ts | 26 +++++++++++++++++++ 3 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 packages/schema/test/bounds.test.ts diff --git a/packages/schema/schemas/ledger-record.schema.json b/packages/schema/schemas/ledger-record.schema.json index de5a799c..dd114a50 100644 --- a/packages/schema/schemas/ledger-record.schema.json +++ b/packages/schema/schemas/ledger-record.schema.json @@ -3,6 +3,19 @@ "$id": "https://amico.harmoniqs.co/schema/ledger-record/v1", "title": "amico run-ledger record", "description": "One append-only line in ~/.amico/ledger/runs.jsonl. A oneOf discriminated on `type` over the eight record kinds (solve|verdict|attempt_error|fallback|override|burn|dispatch|approval). Ops-data, not vault knowledge. Registered in @amicode/schema SCHEMAS ONLY — NOT SUPPORTED_VERSIONS_BY_KIND (no top-level properties.schema_version; a oneOf like this would crash the version-map builder at module load, same as problemspec).", + "$defs": { + "bounds": { + "type": "object", + "additionalProperties": false, + "description": "WarrantBounds — what an approval authorises. Hoisted out of the approval branch (was inline) so the spec-review `budget` lens can validate an AUTHORED budget against this schema rather than a prose restatement of it; validateBounds() in the schema package is that accessor. Behaviour is byte-identical to the inline form it replaces.", + "properties": { + "max_solves": { "type": "integer", "minimum": 1 }, + "tier": { "type": "string", "minLength": 1 }, + "max_size_class": { "enum": ["SMALL", "MEDIUM"], "description": "G-8: bounds the COST PROXY that actually exists. estimate.ts computes memory (sizeClass/score/estimatedBytes) and nothing in amico-run estimates wall-clock, so an earlier max_duration_s bound had no signal behind it and was removed rather than left implying a guarantee about time. A real duration estimator is the deferred C2 work." }, + "device": { "enum": ["none", "ro", "rw"] } + } + } + }, "oneOf": [ { "title": "solve", @@ -185,16 +198,7 @@ "type": { "const": "approval" }, "ts": { "type": "string" }, "plan_hash": { "type": "string", "minLength": 1 }, - "bounds": { - "type": "object", - "additionalProperties": false, - "properties": { - "max_solves": { "type": "integer", "minimum": 1 }, - "tier": { "type": "string", "minLength": 1 }, - "max_size_class": { "enum": ["SMALL", "MEDIUM"], "description": "G-8: bounds the COST PROXY that actually exists. estimate.ts computes memory (sizeClass/score/estimatedBytes) and nothing in amico-run estimates wall-clock, so an earlier max_duration_s bound had no signal behind it and was removed rather than left implying a guarantee about time. A real duration estimator is the deferred C2 work." }, - "device": { "enum": ["none", "ro", "rw"] } - } - }, + "bounds": { "$ref": "#/$defs/bounds" }, "expires_at": { "type": "string" }, "issued_by": { "type": "string", "minLength": 1 } } diff --git a/packages/schema/src/index.ts b/packages/schema/src/index.ts index 0fa6e146..cdb51760 100644 --- a/packages/schema/src/index.ts +++ b/packages/schema/src/index.ts @@ -120,6 +120,22 @@ export function validate(artifact: unknown, kind: SchemaKind): Validation { return { ok: false, errors: (v.errors ?? []).map(formatError) }; } +/** Validate a bare WarrantBounds object against `$defs.bounds` of the ledger-record + * schema. Exists because the spec-review `budget` lens must check an AUTHORED budget + * against the shipped bound vocabulary, and `validate()` only accepts whole registered + * kinds — without this seam the lens could only restate the key set in prose, which is + * the drift that let the long-removed `max_duration` into a spec example + * (spec-20260728 §2.1). */ +const boundsValidator = ajv.compile({ + $schema: "http://json-schema.org/draft-07/schema#", + ...((ledgerRecordSchema as unknown as { $defs: { bounds: object } }).$defs.bounds), +}); + +export function validateBounds(obj: unknown): Validation { + const ok = boundsValidator(obj) as boolean; + return ok ? { ok: true, errors: [] } : { ok: false, errors: (boundsValidator.errors ?? []).map(formatError) }; +} + /** Validate a file on disk: read → parse (TOML, or JSON by extension) → validate. * Parse/read failures are themselves field-precise-ish errors, never a throw. */ export function validateFile(filePath: string, kind: SchemaKind): Validation { diff --git a/packages/schema/test/bounds.test.ts b/packages/schema/test/bounds.test.ts new file mode 100644 index 00000000..bb58309d --- /dev/null +++ b/packages/schema/test/bounds.test.ts @@ -0,0 +1,26 @@ +// $defs.bounds + validateBounds — the WarrantBounds contract the spec-review `budget` +// lens reads. Exists so the lens checks an authored budget against the SHIPPED bound +// vocabulary rather than a prose restatement: the drift that let the long-removed +// `max_duration` into a spec example (plan-20260728 Task 1). +import { describe, it, expect } from "vitest"; +import { validateBounds } from "../src/index.js"; + +describe("validateBounds", () => { + it("accepts every legal key", () => { + expect(validateBounds({ max_solves: 8, tier: "free", max_size_class: "MEDIUM", device: "none" }).ok).toBe(true); + }); + it("accepts an empty object (every bound optional)", () => { + expect(validateBounds({}).ok).toBe(true); + }); + it("REJECTS max_duration — removed by G-8, and the exact key a spec author reaches for", () => { + const r = validateBounds({ max_duration: "30m" }); + expect(r.ok).toBe(false); + expect(r.errors.join(" ")).toMatch(/max_duration/); + }); + it("rejects an out-of-enum size class", () => { + expect(validateBounds({ max_size_class: "LARGE" }).ok).toBe(false); + }); + it("rejects a non-integer max_solves", () => { + expect(validateBounds({ max_solves: 1.5 }).ok).toBe(false); + }); +}); From c49a230f937f97bf88f4f9e5f772da17bc2e44f7 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 10:44:58 -0400 Subject: [PATCH 10/27] feat(schema): verdict/dispatch rows can name a plan step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plan-step state is DERIVED from gate verdicts rather than written by a verb, so an agent cannot forge `passed` without forging a verdict. That derivation was unimplementable: VerdictRecord had no step_id or plan_hash, the branch is additionalProperties:false, and problem_hash was unconditionally required — so a row carrying step_id threw on append and a row without one never matched. Every step would have read `pending` forever. Structurally identical to the max_solves bug (a6b023a): a derivation keyed on a field the schema forbids. Found by four independent spec critics on separate lenses, after the lesson from the first instance had already been written down two sections earlier in the same document — which is the argument for the mechanical gate over the prose. * verdict: optional plan_hash + step_id + source; `verdict` gains `exhausted` (the fleet registry's `blocked` is session-scoped and cannot carry a per-step outcome); problem_hash now required only when step_id is absent, via draft-07 if/then, because a plan step's gate need not be solve-shaped * dispatch: optional plan_hash + step_id, so a step reads `running` before any terminal verdict exists Every field optional: all 673 pre-existing amico-run tests unchanged (679 now), 118 schema, typecheck clean. Plan: plan-20260728-104500 Task 2. Co-Authored-By: Claude Opus 5 (1M context) --- packages/amico-run/src/ledger.ts | 20 ++++- .../test/ledger_step_identity.test.ts | 73 +++++++++++++++++++ .../schema/schemas/ledger-record.schema.json | 14 +++- 3 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 packages/amico-run/test/ledger_step_identity.test.ts diff --git a/packages/amico-run/src/ledger.ts b/packages/amico-run/src/ledger.ts index 1dd52c0b..949554c1 100644 --- a/packages/amico-run/src/ledger.ts +++ b/packages/amico-run/src/ledger.ts @@ -80,11 +80,23 @@ export interface SolveRecord { export interface VerdictRecord { type: "verdict"; ts: string; - problem_hash: string; + /** Required by the schema ONLY when `step_id` is absent — i.e. for a solve re-rollout + * verdict. A plan-step gate need not be solve-shaped. */ + problem_hash?: string; structure_hash?: string; - verdict: "agree" | "disagree"; + /** `exhausted` = per-step gate exhaustion. The fleet registry's `blocked` is + * session-scoped, so it cannot carry a per-step outcome. */ + verdict: "agree" | "disagree" | "exhausted"; fidelity_rerolled?: number; fidelity_reported?: number; + /** Plan-step identity. Present on a plan-step gate verdict; this is the join that + * plan-step state is DERIVED from, which is what makes forging `passed` require + * forging a gate verdict (spec-20260728 §4.4). Both optional so every pre-existing + * solve verdict keeps validating. Derivation keys on (plan_hash, step_id) — never + * step_id alone, or a recompiled plan aliases onto the old plan's rows. */ + plan_hash?: string; + step_id?: string; + source?: "user" | "replay" | "simulated"; } export interface AttemptErrorRecord { @@ -170,6 +182,10 @@ export interface DispatchRecord { tokens: number; // per-attempt token cost; 0 on experiment rows (excluded from c_m) attempt_index: number; // ladder position; 1 = first attempt (the p_m(s) sample) source: "user" | "replay" | "simulated"; + /** Plan-step identity (optional) — lets step-state derivation see a step as `running` + * before any terminal verdict row exists. */ + plan_hash?: string; + step_id?: string; } /** What a warrant authorises. An ABSENT key does not mean "unlimited" — the gate diff --git a/packages/amico-run/test/ledger_step_identity.test.ts b/packages/amico-run/test/ledger_step_identity.test.ts new file mode 100644 index 00000000..ed4814d6 --- /dev/null +++ b/packages/amico-run/test/ledger_step_identity.test.ts @@ -0,0 +1,73 @@ +// Verdict/dispatch rows can name a plan step. +// +// REGRESSION GUARD for the defect class of the max_solves bug (a6b023a): a derivation +// keyed on a field the schema forbids. The deliberation spec derived plan-step state +// from VerdictRecord(step_id) while the verdict branch was additionalProperties:false +// with no step_id — so a row carrying one threw on append and a row without one never +// matched, making every step read `pending` forever. Four independent spec critics +// found it; this test is what keeps it fixed. +// +// Plan: plan-20260728-104500 Task 2. +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { appendRecord, readRecords } from "../src/ledger.js"; + +const ts = () => new Date().toISOString(); +const row = (i = 0) => readRecords()[i] as unknown as Record; + +describe("plan-step identity on verdict/dispatch rows", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ledger-step-")); + process.env.AMICO_LEDGER = join(dir, "runs.jsonl"); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + delete process.env.AMICO_LEDGER; + }); + + it("a verdict row carrying plan_hash + step_id appends", () => { + appendRecord({ + type: "verdict", ts: ts(), plan_hash: "abc", step_id: "s2", verdict: "agree", source: "user", + } as never); + expect(row().step_id).toBe("s2"); + expect(row().plan_hash).toBe("abc"); + }); + + it("problem_hash stays REQUIRED when step_id is absent (existing solve-shaped rows)", () => { + expect(() => appendRecord({ type: "verdict", ts: ts(), verdict: "agree" } as never)).toThrow(/problem_hash/); + }); + + it("verdict gains `exhausted`, so per-step gate exhaustion has a carrier", () => { + appendRecord({ + type: "verdict", ts: ts(), plan_hash: "abc", step_id: "s3", verdict: "exhausted", source: "user", + } as never); + expect(row().verdict).toBe("exhausted"); + }); + + it("a dispatch row can name a step", () => { + appendRecord({ + type: "dispatch", ts: ts(), task_type: "author-script", work_id: "wid", + model: "anthropic/claude-opus-5", variant: "high", gate: "re-rollout", + pass: true, tokens: 10, attempt_index: 1, source: "user", + plan_hash: "abc", step_id: "s2", + } as never); + expect(row().step_id).toBe("s2"); + }); + + it("an EXISTING solve-shaped verdict row still validates (no regression)", () => { + appendRecord({ + type: "verdict", ts: ts(), problem_hash: "ph", verdict: "agree", fidelity_rerolled: 0.999, + } as never); + expect(readRecords()).toHaveLength(1); + }); + + it("source is available on verdict rows, so simulated gym verdicts are separable", () => { + appendRecord({ + type: "verdict", ts: ts(), plan_hash: "abc", step_id: "s1", verdict: "agree", source: "simulated", + } as never); + expect(row().source).toBe("simulated"); + }); +}); diff --git a/packages/schema/schemas/ledger-record.schema.json b/packages/schema/schemas/ledger-record.schema.json index dd114a50..a45f8328 100644 --- a/packages/schema/schemas/ledger-record.schema.json +++ b/packages/schema/schemas/ledger-record.schema.json @@ -66,17 +66,23 @@ }, { "title": "verdict", + "description": "A gate outcome. Two shapes share this branch: a SOLVE re-rollout verdict (keyed on problem_hash, the original form) and a PLAN-STEP gate verdict (keyed on plan_hash + step_id). The step form exists because plan-step state is DERIVED from these rows rather than written by a verb — an agent cannot forge `passed` without forging a gate verdict (spec-20260728 §4.4). problem_hash is required only for the solve form; a plan step's gate need not be solve-shaped (a schema-lint gate on a bookkeeping step has no problem hash).", "type": "object", "additionalProperties": false, - "required": ["type", "ts", "problem_hash", "verdict"], + "required": ["type", "ts", "verdict"], + "if": { "not": { "required": ["step_id"] } }, + "then": { "required": ["problem_hash"] }, "properties": { "type": { "const": "verdict" }, "ts": { "type": "string" }, "problem_hash": { "type": "string" }, "structure_hash": { "type": "string" }, - "verdict": { "enum": ["agree", "disagree"] }, + "verdict": { "enum": ["agree", "disagree", "exhausted"], "description": "`exhausted` is per-step gate exhaustion (failure at the top reachable escalation rung). It lives here because the fleet registry's `blocked` is SESSION-scoped and cannot carry a per-step outcome." }, "fidelity_rerolled": { "type": "number" }, - "fidelity_reported": { "type": "number" } + "fidelity_reported": { "type": "number" }, + "plan_hash": { "type": "string", "minLength": 1, "description": "Set on a plan-step verdict. With step_id this is the join plan-step state is derived from; OPTIONAL so every pre-existing solve verdict keeps validating." }, + "step_id": { "type": "string", "minLength": 1, "description": "The compiled plan step this verdict is about. Derivation keys on (plan_hash, step_id) — never step_id alone, or a recompiled plan's identically-named steps would alias onto the old plan's rows and read as already complete." }, + "source": { "enum": ["user", "replay", "simulated"], "description": "Lane. Step-state derivation filters source=user so a simulated gym verdict cannot masquerade as real progress in the very derivation the lane separation protects." } } }, { @@ -185,6 +191,8 @@ "pass": { "type": "boolean" }, "tokens": { "type": "integer", "minimum": 0 }, "attempt_index": { "type": "integer", "minimum": 1 }, + "plan_hash": { "type": "string", "minLength": 1, "description": "Set when this dispatch is a plan step; with step_id it lets step-state derivation see a step as `running` before any terminal verdict exists." }, + "step_id": { "type": "string", "minLength": 1, "description": "The compiled plan step this dispatch is for." }, "source": { "enum": ["user", "replay", "simulated"] } } }, From bdd373d3c154ad1d7fe45566e901fa48f82bc243 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 10:55:42 -0400 Subject: [PATCH 11/27] =?UTF-8?q?fix(amico-run):=20build=20the=20bundles?= =?UTF-8?q?=20atomically=20=E2=80=94=20kills=20an=20intermittent-CI=20race?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ten test files run esbuild.config.mjs in their own `beforeAll` while OTHER test files concurrently `execFileSync("node", [dist/amico.js, …])`. Writing in place truncates a bundle a sibling file is mid-execution on, so node exits 1 with empty stdout and the sibling fails with a bare `expected 1 to be +0`, or a `SyntaxError: Unexpected end of input` from JSON.parse-ing nothing. The tell that it is a race and not a defect: re-running the failing file alone always passes. Measured — 4 consecutive full runs failed 0-2 tests at random file scheduling; 4 consecutive runs after the fix, zero. Build into a temp DIRECTORY keeping the final basename, then rename into dist/. rename(2) is atomic within a filesystem, so a concurrent reader gets either the whole old bundle or the whole new one. The temp *directory* (rather than a temp filename) matters: esbuild bakes the output basename into the trailing `//# sourceMappingURL=` comment, so a temp filename would ship a bundle pointing at a map that no longer exists — sourcemaps silently broken with every test still green. Found while adding a test file that changed scheduling enough to surface it. Also explains an unattributable flake seen earlier in this session. Co-Authored-By: Claude Opus 5 (1M context) --- packages/amico-run/esbuild.config.mjs | 46 ++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/packages/amico-run/esbuild.config.mjs b/packages/amico-run/esbuild.config.mjs index 2f058e17..13ec036c 100644 --- a/packages/amico-run/esbuild.config.mjs +++ b/packages/amico-run/esbuild.config.mjs @@ -1,5 +1,6 @@ import { build } from "esbuild"; -import { chmodSync } from "node:fs"; +import { chmodSync, mkdtempSync, renameSync, rmSync } from "node:fs"; +import { join } from "node:path"; // Three bins from one package: the historical `amico-run` (entry cli.ts), the `amico` // verb router (entry amico.ts, issue #108) — both sharing the launch path (src/launch.ts; @@ -18,11 +19,40 @@ const common = { logLevel: "info", }; -for (const [entry, outfile] of [ - ["src/cli.ts", "dist/amico-run.js"], - ["src/amico.ts", "dist/amico.js"], - ["src/pasqal_cli.ts", "dist/amico-pasqal.js"], -]) { - await build({ ...common, entryPoints: [entry], outfile }); - chmodSync(outfile, 0o755); +// WRITE ATOMICALLY — build to a unique temp path, then rename into place. +// +// Why: ten test files run this config in their own `beforeAll` while OTHER test files +// concurrently `execFileSync("node", [dist/amico.js, …])`. esbuild writing in place +// truncates the bundle a sibling file is mid-execution on, so node exits 1 with empty +// stdout and the sibling's assertion fails with a bare `expected 1 to be +0` or a +// `SyntaxError: Unexpected end of input` from JSON.parse-ing nothing. That is a real +// intermittent-CI race, and it is invisible when you re-run the failing file alone. +// +// rename(2) is atomic within a filesystem, so a concurrent reader gets either the whole +// old bundle or the whole new one — never a partial. The pid+counter suffix keeps two +// concurrent builds from colliding on the temp path itself. +// Build into a temp DIRECTORY keeping the final basename, then rename both artifacts +// into dist/. Building to a temp *filename* instead would bake that temp name into the +// bundle's trailing `//# sourceMappingURL=` comment, so the shipped bundle would point +// at a map that no longer exists — sourcemaps silently broken, tests all still green. +// The URL is relative to the output file, so preserving the basename keeps it correct. +const staging = mkdtempSync(join("dist", "build-")); +try { + for (const [entry, name] of [ + ["src/cli.ts", "amico-run.js"], + ["src/amico.ts", "amico.js"], + ["src/pasqal_cli.ts", "amico-pasqal.js"], + ]) { + const tmp = join(staging, name); + await build({ ...common, entryPoints: [entry], outfile: tmp }); + chmodSync(tmp, 0o755); + renameSync(tmp, join("dist", name)); + try { + renameSync(`${tmp}.map`, join("dist", `${name}.map`)); + } catch { + /* sourcemap is best-effort — never fail a build over it */ + } + } +} finally { + rmSync(staging, { recursive: true, force: true }); } From f05505875b953a5fba75f83c046ef157fbf72729 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 10:55:42 -0400 Subject: [PATCH 12/27] feat(schema): spec_review, plan_compiled and todo ledger kinds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three deliberation stanzas (spec-20260728 §5), taking the ledger from eight record kinds to eleven. Design points that are load-bearing rather than incidental: * Finding BODIES are NOT in spec_review. A 3-round 3-critic review's prose exceeds PIPE_BUF, and appendRecord throws above it — AFTER the model spend, losing the whole review. Bodies go to a sidecar; the row carries findings_count/blocking_count/ findings_sha256/findings_ref. Every free-text field is maxLength-capped for the same reason: per-lens reasons originate in a subprocess's stderr. A test appends a maximal 3-round 3-critic review with 200-char reasons on all six lenses and asserts it lands. * `review_verdict`, not `verdict` — the ledger already has a `verdict` KIND whose `verdict` field is agree|disagree, and both live in the same runs.jsonl. * `critics: []` is PRESENT-and-empty as the offline sentinel; absent would be indistinguishable from a row written before the field existed. * plan_compiled IS the design_hash -> plan_hash binding. Without it the launch gate cannot distinguish "the plan was recompiled, re-approve" from "never approved". * todo rejects state:open (open is the absence of a row) and requires a reason iff waived, so waive-spam is visible. No `actor` field: no trustworthy actor identity exists at this layer, and recording one would be theatre. 13 new tests; 692 amico-run and 118 schema green; typecheck clean. Plan: plan-20260728-104500 Task 3. Co-Authored-By: Claude Opus 5 (1M context) --- packages/amico-run/src/ledger.ts | 70 +++++++++- .../test/ledger_deliberation_kinds.test.ts | 124 ++++++++++++++++++ .../schema/schemas/ledger-record.schema.json | 96 +++++++++++++- 3 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 packages/amico-run/test/ledger_deliberation_kinds.test.ts diff --git a/packages/amico-run/src/ledger.ts b/packages/amico-run/src/ledger.ts index 949554c1..506c8cf2 100644 --- a/packages/amico-run/src/ledger.ts +++ b/packages/amico-run/src/ledger.ts @@ -27,7 +27,7 @@ import { validate } from "@amicode/schema"; export const PIPE_BUF = 4096; // ── record contract (spec-20260719-210954 §"Record contract") ──────────────────── -// SEVEN discriminated record kinds. `solve` is the primary; `verdict` joins to it on +// ELEVEN discriminated record kinds. `solve` is the primary; `verdict` joins to it on // `problem_hash`; the rest are lightweight events. The `source: "simulated"` value // is the Prova isolation bridge (a deliberate extension of the spec's `user|replay`). // The 7th kind, `dispatch`, is the tier-dispatch row (fleet §6.3 Rev 5): tier @@ -213,6 +213,69 @@ export interface ApprovalRecord { issued_by: string; } +// ── the deliberation stanzas (spec-20260728 §5) ────────────────────────────────── +// The front half of deliberation: a Spec is authored, adversarially reviewed, and +// compiled into a Plan whose advisory todos are tracked here. Step todos are NOT here +// — step state is derived from `verdict` rows (§4.4), so forging `passed` requires +// forging a gate verdict. + +/** One completed adversarial review. Finding BODIES live in a sidecar, not here: a + * 3-round 3-critic review's prose exceeds PIPE_BUF and appendRecord throws above it — + * after the model spend — so this row carries only a digest. */ +export interface SpecReviewRecord { + type: "spec_review"; + ts: string; + /** The spec's IMMUTABLE identity. `design_hash` alone is not an identity: two specs + * sharing an acceptance list would collide in the findings namespace. */ + spec_id: string; + design_hash: string; + rounds: number; // 1..3, schema-enforced + /** NOT `verdict`: the ledger already has a `verdict` KIND whose `verdict` field is + * agree|disagree, and both live in the same runs.jsonl. */ + review_verdict: "approved" | "approved-mechanical" | "degraded" | "blocking" | "exhausted"; + lens_registry_version: string; + lens_status: Array<{ lens: string; status: "ran" | "not-applicable" | "skipped" | "unverified"; reason?: string }>; + /** PRESENT-and-empty is the offline sentinel; absent would be indistinguishable from + * a row written before the field existed. */ + critics: Array<{ model: string; variant: string }>; + findings_count: number; + blocking_count: number; + findings_sha256?: string; + findings_ref?: string; + source: "user" | "replay" | "simulated"; +} + +/** A Plan compiled from a Spec. This row IS the design_hash -> plan_hash binding, + * without which the launch gate cannot distinguish "recompiled, re-approve" from + * "never approved". */ +export interface PlanCompiledRecord { + type: "plan_compiled"; + ts: string; + plan_hash: string; + spec_id: string; + design_hash: string; + compiled_by?: { model: string; variant: string }; + step_count: number; + advisory_count?: number; + /** A RECOMMENDATION from step_count. `amico ledger approve` reads it to default + * --expires-in and remains the sole writer of expires_at. */ + suggested_ttl_s?: number; + allow_unreviewed?: boolean; + source: "user" | "replay" | "simulated"; +} + +/** One ADVISORY todo transition. `open` is the absence of a row; multiple rows per id + * resolve last-ts-wins. No `actor` field — no trustworthy actor identity exists here. */ +export interface TodoRecord { + type: "todo"; + ts: string; + plan_hash: string; + id: string; + state: "fixed" | "waived" | "obsolete"; + reason?: string; // required iff state === "waived" + source: "user" | "replay" | "simulated"; +} + export type LedgerRecord = | SolveRecord | VerdictRecord @@ -221,7 +284,10 @@ export type LedgerRecord = | OverrideRecord | BurnRecord | DispatchRecord - | ApprovalRecord; + | ApprovalRecord + | SpecReviewRecord + | PlanCompiledRecord + | TodoRecord; /** The ledger file path: `$AMICO_LEDGER` override, else `~/.amico/ledger/runs.jsonl`. */ export function ledgerPath(): string { diff --git a/packages/amico-run/test/ledger_deliberation_kinds.test.ts b/packages/amico-run/test/ledger_deliberation_kinds.test.ts new file mode 100644 index 00000000..c6485237 --- /dev/null +++ b/packages/amico-run/test/ledger_deliberation_kinds.test.ts @@ -0,0 +1,124 @@ +// The three deliberation ledger kinds: spec_review, plan_compiled, todo. +// +// Every free-text field is maxLength-capped on purpose. The spec_review row carries +// per-lens reasons sourced from a subprocess's stderr, and appendRecord THROWS above +// PIPE_BUF (4096) — which would happen AFTER the model spend, losing the whole review. +// Finding bodies therefore live in a sidecar and the row carries only a digest. +// +// Plan: plan-20260728-104500 Task 3. +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { appendRecord, readRecords } from "../src/ledger.js"; + +const ts = () => new Date().toISOString(); +const row = (i = 0) => readRecords()[i] as unknown as Record; +const H = "a".repeat(64); + +const review = (over: Record = {}) => ({ + type: "spec_review", ts: ts(), spec_id: "spec-1", design_hash: H, + rounds: 1, review_verdict: "approved-mechanical", + lens_registry_version: "1", lens_status: [{ lens: "schema", status: "ran" }], + critics: [], findings_count: 0, blocking_count: 0, + findings_sha256: "b".repeat(64), findings_ref: ".review/x.json", source: "user", + ...over, +}); + +describe("deliberation ledger kinds", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ledger-delib-")); + process.env.AMICO_LEDGER = join(dir, "runs.jsonl"); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + delete process.env.AMICO_LEDGER; + }); + + describe("spec_review", () => { + it("appends, and critics: [] round-trips as PRESENT-and-empty (the offline sentinel)", () => { + appendRecord(review() as never); + expect(row().critics).toEqual([]); + expect("critics" in row()).toBe(true); // absent would be indistinguishable from an old writer + }); + it("rejects an out-of-enum review_verdict", () => { + expect(() => appendRecord(review({ review_verdict: "fine" }) as never)).toThrow(); + }); + it("accepts every legal review_verdict", () => { + for (const v of ["approved", "approved-mechanical", "degraded", "blocking", "exhausted"]) { + appendRecord(review({ review_verdict: v }) as never); + } + expect(readRecords()).toHaveLength(5); + }); + it("rejects rounds outside 1..3 (the round budget is the schema's business too)", () => { + expect(() => appendRecord(review({ rounds: 4 }) as never)).toThrow(); + expect(() => appendRecord(review({ rounds: 0 }) as never)).toThrow(); + }); + it("caps a lens reason, so a subprocess's stderr cannot push the row past PIPE_BUF", () => { + expect(() => + appendRecord(review({ lens_status: [{ lens: "api", status: "skipped", reason: "x".repeat(500) }] }) as never), + ).toThrow(); + }); + it("rejects an out-of-enum lens status", () => { + expect(() => + appendRecord(review({ lens_status: [{ lens: "schema", status: "probably-fine" }] }) as never), + ).toThrow(); + }); + it("enforces the provider/model-id shape on a critic", () => { + expect(() => appendRecord(review({ critics: [{ model: "opus-5", variant: "high" }] }) as never)).toThrow(); + appendRecord(review({ critics: [{ model: "anthropic/claude-opus-5", variant: "high" }] }) as never); + expect(readRecords()).toHaveLength(1); + }); + it("a MAXIMAL 3-round 3-critic review with max-length reasons still fits PIPE_BUF", () => { + appendRecord(review({ + rounds: 3, + critics: Array.from({ length: 3 }, () => ({ model: "anthropic/claude-opus-5", variant: "high" })), + lens_status: ["schema", "falsifiable", "budget", "baseline", "precedent", "provenance"].map((lens) => ({ + lens, status: "skipped", reason: "y".repeat(200), // the schema cap + })), + findings_count: 27, blocking_count: 3, + findings_ref: ".review/" + "z".repeat(120) + ".json", + }) as never); + expect(readRecords()).toHaveLength(1); + }); + }); + + describe("plan_compiled", () => { + it("appends", () => { + appendRecord({ + type: "plan_compiled", ts: ts(), plan_hash: "c".repeat(64), spec_id: "spec-1", + design_hash: H, compiled_by: { model: "anthropic/claude-opus-5", variant: "high" }, + step_count: 3, advisory_count: 2, suggested_ttl_s: 7200, allow_unreviewed: false, source: "user", + } as never); + expect(row().step_count).toBe(3); + }); + it("requires the design_hash binding — without it the gate cannot say `recompiled`", () => { + expect(() => + appendRecord({ type: "plan_compiled", ts: ts(), plan_hash: "c", spec_id: "s", step_count: 1, source: "user" } as never), + ).toThrow(/design_hash/); + }); + }); + + describe("todo", () => { + it("waived REQUIRES a reason; fixed and obsolete do not", () => { + expect(() => + appendRecord({ type: "todo", ts: ts(), plan_hash: "c", id: "A-1", state: "waived", source: "user" } as never), + ).toThrow(); + appendRecord({ type: "todo", ts: ts(), plan_hash: "c", id: "A-1", state: "waived", reason: "out of scope", source: "user" } as never); + appendRecord({ type: "todo", ts: ts(), plan_hash: "c", id: "A-2", state: "fixed", source: "user" } as never); + appendRecord({ type: "todo", ts: ts(), plan_hash: "c", id: "A-3", state: "obsolete", source: "user" } as never); + expect(readRecords()).toHaveLength(3); + }); + it("rejects state:open — open is the ABSENCE of a row", () => { + expect(() => + appendRecord({ type: "todo", ts: ts(), plan_hash: "c", id: "A-1", state: "open", source: "user" } as never), + ).toThrow(); + }); + it("requires plan_hash, so advisory state is always scoped to a plan", () => { + expect(() => + appendRecord({ type: "todo", ts: ts(), id: "A-1", state: "fixed", source: "user" } as never), + ).toThrow(/plan_hash/); + }); + }); +}); diff --git a/packages/schema/schemas/ledger-record.schema.json b/packages/schema/schemas/ledger-record.schema.json index a45f8328..fdcc7fb8 100644 --- a/packages/schema/schemas/ledger-record.schema.json +++ b/packages/schema/schemas/ledger-record.schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://amico.harmoniqs.co/schema/ledger-record/v1", "title": "amico run-ledger record", - "description": "One append-only line in ~/.amico/ledger/runs.jsonl. A oneOf discriminated on `type` over the eight record kinds (solve|verdict|attempt_error|fallback|override|burn|dispatch|approval). Ops-data, not vault knowledge. Registered in @amicode/schema SCHEMAS ONLY — NOT SUPPORTED_VERSIONS_BY_KIND (no top-level properties.schema_version; a oneOf like this would crash the version-map builder at module load, same as problemspec).", + "description": "One append-only line in ~/.amico/ledger/runs.jsonl. A oneOf discriminated on `type` over the eleven record kinds (solve|verdict|attempt_error|fallback|override|burn|dispatch|approval|spec_review|plan_compiled|todo). Ops-data, not vault knowledge. Registered in @amicode/schema SCHEMAS ONLY — NOT SUPPORTED_VERSIONS_BY_KIND (no top-level properties.schema_version; a oneOf like this would crash the version-map builder at module load, same as problemspec).", "$defs": { "bounds": { "type": "object", @@ -196,6 +196,100 @@ "source": { "enum": ["user", "replay", "simulated"] } } }, + { + "title": "spec_review", + "description": "One completed adversarial review of a Spec (spec-20260728 §5). Finding BODIES are NOT here: a 3-round 3-critic review's prose would exceed PIPE_BUF and appendRecord throws above it — after the model spend — so bodies go to a sidecar and this row carries a digest. Every free-text field is maxLength-capped for the same reason, because per-lens reasons originate in a subprocess's stderr.", + "type": "object", + "additionalProperties": false, + "required": ["type", "ts", "spec_id", "design_hash", "rounds", "review_verdict", "lens_registry_version", "lens_status", "critics", "findings_count", "blocking_count", "source"], + "properties": { + "type": { "const": "spec_review" }, + "ts": { "type": "string" }, + "spec_id": { "type": "string", "minLength": 1, "maxLength": 200, "description": "The spec's IMMUTABLE identity, authored once. design_hash alone is not an identity — two specs sharing an acceptance list would collide in the findings namespace and in the recompiled-plan lookup." }, + "design_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$", "description": "sha256hex(canonicalJson({task_type, acceptance, budget?})) — the DECISION SURFACE only. Bare hex, matching structureHash/problemHash; NOT the gate's sha256:-prefixed spec_hash, which is a different population entirely (the canonical solvespec)." }, + "rounds": { "type": "integer", "minimum": 1, "maximum": 3, "description": "Round budget is 3; the schema enforces it so an unbounded loop cannot record itself as legitimate." }, + "review_verdict": { "enum": ["approved", "approved-mechanical", "degraded", "blocking", "exhausted"], "description": "NOT named `verdict`: the ledger already has a `verdict` KIND whose `verdict` field is agree|disagree, and both live in the same runs.jsonl. approved-mechanical = tier 1 clean with no critic binary; degraded = a selected critic timed out or emitted unparseable output. Both exit 0, but plan compile refuses them without --allow-unreviewed." }, + "lens_registry_version": { "type": "string", "minLength": 1, "maxLength": 64 }, + "lens_status": { + "type": "array", + "description": "Per-lens outcome. `not-applicable` and `unverified` are distinct from a clean `ran` on purpose: collapsing them is how a blocking lens that could not run reads as a pass.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["lens", "status"], + "properties": { + "lens": { "type": "string", "minLength": 1, "maxLength": 64 }, + "status": { "enum": ["ran", "not-applicable", "skipped", "unverified"] }, + "reason": { "type": "string", "maxLength": 200 } + } + } + }, + "critics": { + "type": "array", + "description": "PRESENT-and-empty is the offline sentinel — an absent key would be indistinguishable from a row written before this field existed.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["model", "variant"], + "properties": { + "model": { "type": "string", "pattern": "^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$", "description": "Read back from the CHILD's own output, not assumed from argv — otherwise the collusion mitigation stamps a request rather than a fact." }, + "variant": { "type": "string", "minLength": 1, "maxLength": 32 } + } + } + }, + "findings_count": { "type": "integer", "minimum": 0 }, + "blocking_count": { "type": "integer", "minimum": 0 }, + "findings_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$", "description": "sha256hex(canonicalJson(findings)) — of the CANONICAL findings array, not of the sidecar file's bytes. Those are different strings." }, + "findings_ref": { "type": "string", "minLength": 1, "maxLength": 512 }, + "source": { "enum": ["user", "replay", "simulated"] } + } + }, + { + "title": "plan_compiled", + "description": "A Plan compiled from a Spec (spec-20260728 §5). Load-bearing beyond bookkeeping: this row is the design_hash -> plan_hash binding, without which the launch gate cannot distinguish `the plan was recompiled, re-approve` from `never approved` and every recompile surfaces as a bare denial.", + "type": "object", + "additionalProperties": false, + "required": ["type", "ts", "plan_hash", "spec_id", "design_hash", "step_count", "source"], + "properties": { + "type": { "const": "plan_compiled" }, + "ts": { "type": "string" }, + "plan_hash": { "type": "string", "minLength": 1 }, + "spec_id": { "type": "string", "minLength": 1, "maxLength": 200 }, + "design_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "compiled_by": { + "type": "object", + "additionalProperties": false, + "required": ["model", "variant"], + "properties": { + "model": { "type": "string", "pattern": "^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$" }, + "variant": { "type": "string", "minLength": 1, "maxLength": 32 } + } + }, + "step_count": { "type": "integer", "minimum": 0 }, + "advisory_count": { "type": "integer", "minimum": 0 }, + "suggested_ttl_s": { "type": "integer", "minimum": 1, "description": "A RECOMMENDATION derived from step_count. `amico ledger approve` reads it to default --expires-in and remains the sole writer of expires_at — compile must not own a warrant's lifetime, or --recompile would silently re-set how long a human's authorization lasts." }, + "allow_unreviewed": { "type": "boolean", "description": "True when compiled from a spec whose review was approved-mechanical or degraded. Recorded so the surface can say so." }, + "source": { "enum": ["user", "replay", "simulated"] } + } + }, + { + "title": "todo", + "description": "One ADVISORY todo transition (spec-20260728 §5). There is deliberately no step-todo row: step state is DERIVED from verdict rows, which is what makes forging `passed` require forging a gate verdict. `open` is the ABSENCE of a row; multiple rows per id resolve last-ts-wins; fixed -> obsolete is legal. No `actor` field, because no trustworthy actor identity exists at this layer and recording one would be theatre.", + "type": "object", + "additionalProperties": false, + "required": ["type", "ts", "plan_hash", "id", "state", "source"], + "if": { "properties": { "state": { "const": "waived" } }, "required": ["state"] }, + "then": { "required": ["reason"] }, + "properties": { + "type": { "const": "todo" }, + "ts": { "type": "string" }, + "plan_hash": { "type": "string", "minLength": 1 }, + "id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "state": { "enum": ["fixed", "waived", "obsolete"] }, + "reason": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Required iff state = waived, so waive-spam is visible in the record rather than silent." }, + "source": { "enum": ["user", "replay", "simulated"] } + } + }, { "title": "approval", "description": "A capability warrant (spec-20260727-164748 §5): the record that lets a gated launch through amico-run's --spec gate. `plan_hash` is what was approved; `bounds` is what it authorises. DELIBERATELY UNSIGNED — the threat model is drift, not an adversary (spec §3), and the product agent holds unrestricted bash, so a signature would defend against an attacker this layer could not stop anyway. Absent bounds keys do NOT default to allow: §5.1 rule 2 refuses a launch needing a bound the warrant omits, which is why an empty `bounds` object is legal (it authorises nothing beyond the ungated free set) rather than a hole. `device` uses the fleet spec §2.1 permission vocabulary.", From fce9e4317c4aa05602865a901bd6d1dd3a007d98 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 10:57:26 -0400 Subject: [PATCH 13/27] feat(schema): designHash + planHash, with a compacting projection builder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit designHash, NOT specHash: gate.ts already stamps hashes.spec_hash as the sha256 of the canonical SOLVESPEC, and one name over two populations makes any join across them silently wrong. Two properties that need code rather than prose: * The projection builder DROPS undefined- and null-valued keys. canonicalJson renders both as "null", and a literal {task_type, acceptance, budget} with budget:undefined still has an enumerable `budget` — so without compacting, an absent budget hashes as "budget":null. Stable, permanent, wrong, and nothing reports an error. * `acceptance` is trimmed, whitespace-collapsed and SORTED, because reordering independent criteria is not a decision change. `invariants` and `assumptions` are excluded: prose must not re-gate a live warrant, and a violated assumption is a runtime blocked-report rather than a re-approval. The GOLDEN VECTOR is the load-bearing test. Relative change/no-change assertions pass against the wrong canonicalizer — gate.ts pretty-prints and prefixes `sha256:` while hashing.ts is compact and bare-hex — so they would let every join against structureHash/problemHash break silently. One literal pins the canonicalizer itself. planHash covers goal + steps only; design_hash and compiled_at are excluded so a recompile that changed nothing does not mint a new hash and invalidate a live warrant. 128 schema tests green, typecheck clean. Plan: plan-20260728-104500 Task 4. Co-Authored-By: Claude Opus 5 (1M context) --- packages/schema/src/hashing.ts | 37 +++++++++++- packages/schema/src/index.ts | 2 +- packages/schema/test/design_hash.test.ts | 71 ++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 packages/schema/test/design_hash.test.ts diff --git a/packages/schema/src/hashing.ts b/packages/schema/src/hashing.ts index 463323e9..335579b6 100644 --- a/packages/schema/src/hashing.ts +++ b/packages/schema/src/hashing.ts @@ -238,7 +238,7 @@ export function structureFields(spec: Raw): Json { } // ── hashes ───────────────────────────────────────────────────────────────── -const sha256hex = (s: string): string => createHash("sha256").update(s, "utf8").digest("hex"); +export const sha256hex = (s: string): string => createHash("sha256").update(s, "utf8").digest("hex"); /** SHA-256 hex of canonicalJson(structureFields(spec)) — the problem's *shape* key. */ export function structureHash(spec: Raw): string { @@ -249,3 +249,38 @@ export function structureHash(spec: Raw): string { export function problemHash(spec: Raw): string { return sha256hex(canonicalJson(fullDict(spec))); } + +// ── deliberation hashes (spec-20260728 §2.4, §4.1) ─────────────────────────────── + +/** Drop every undefined- OR null-valued key. `canonicalJson` renders both as "null", + * and a literal `{a, b, c}` with `c: undefined` still has an enumerable `c` — so + * without this an absent budget hashes as `"budget":null`: stable, permanent, and + * wrong, with nothing anywhere reporting an error. */ +function compact(o: Record): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(o)) if (v !== undefined && v !== null) out[k] = v; + return out; +} + +/** The Spec's DECISION-SURFACE hash. Named `designHash`, NOT `specHash`: gate.ts already + * stamps `hashes.spec_hash` as the sha256 of the canonical SOLVESPEC, and one name over + * two populations makes any join across them silently wrong. + * + * Covers `task_type`, `acceptance` and `budget` only. `acceptance` entries are trimmed, + * inner whitespace collapsed, then SORTED (UTF-16 code units, matching canonicalJson's + * own key sort) — reordering independent criteria is not a decision change. `invariants` + * and `assumptions` are excluded deliberately: prose must not re-gate a live warrant, + * and a violated assumption is a runtime blocked-report rather than a re-approval. */ +export function designHash(spec: Record): string { + const acceptance = Array.isArray(spec.acceptance) + ? (spec.acceptance as unknown[]).map((s) => String(s).trim().replace(/\s+/g, " ")).sort() + : []; + return sha256hex(canonicalJson(compact({ task_type: spec.task_type, acceptance, budget: spec.budget }) as Json)); +} + +/** The compiled Plan's hash: `goal` + `steps` only. `design_hash` and `compiled_at` are + * excluded so a recompile that changed nothing does not mint a new hash — which would + * invalidate a live warrant for no reason. */ +export function planHash(plan: Record): string { + return sha256hex(canonicalJson(compact({ goal: plan.goal, steps: plan.steps }) as Json)); +} diff --git a/packages/schema/src/index.ts b/packages/schema/src/index.ts index cdb51760..411b52e5 100644 --- a/packages/schema/src/index.ts +++ b/packages/schema/src/index.ts @@ -35,7 +35,7 @@ import ledgerRecordSchema from "../schemas/ledger-record.schema.json" with { typ // package-internal `../src/hashing.js` relative path (this package has no // "exports" map, so a subpath import would work, but the root export is the // established, documented seam every other consumer uses — see `validate` below). -export { structureHash, problemHash, canonicalJson, fullDict, structureFields } from "./hashing.js"; +export { structureHash, problemHash, canonicalJson, fullDict, structureFields, sha256hex, designHash, planHash } from "./hashing.js"; // ajv-formats ships a CJS default export; under NodeNext the default import can // bind the module namespace rather than the callable, so normalize defensively. diff --git a/packages/schema/test/design_hash.test.ts b/packages/schema/test/design_hash.test.ts new file mode 100644 index 00000000..adf21817 --- /dev/null +++ b/packages/schema/test/design_hash.test.ts @@ -0,0 +1,71 @@ +// designHash / planHash (spec-20260728 §2.4, §4.1). +// +// The golden vector is the point of this file. Relative change/no-change assertions +// pass against the WRONG canonicalizer — gate.ts pretty-prints and prefixes `sha256:` +// while hashing.ts is compact and bare-hex — so they would let a join against +// structureHash/problemHash break silently. Pinning one literal value pins the +// canonicalizer itself (advisory A-7). +// +// Plan: plan-20260728-104500 Task 4. +import { describe, it, expect } from "vitest"; +import { designHash, planHash } from "../src/index.js"; + +const base = { + task_type: "experiment-sim", + acceptance: ["F_rolled >= 0.999"], + budget: { max_solves: 8, tier: "free" }, +}; + +describe("designHash", () => { + it("is 64 lowercase hex with NO sha256: prefix", () => { + expect(designHash(base)).toMatch(/^[0-9a-f]{64}$/); + }); + + it("GOLDEN VECTOR — pins the canonicalizer, not merely its change behaviour", () => { + expect(designHash(base)).toBe("6198bda6cc2a54d49bbadb1868e6b4bf1ee7fb3f02c387ec9e43978087b63257"); + }); + + it("is insensitive to acceptance ORDER — independent criteria have no order", () => { + expect(designHash({ ...base, acceptance: ["A >= 1", "B <= 2"] })) + .toBe(designHash({ ...base, acceptance: ["B <= 2", "A >= 1"] })); + }); + + it("normalizes whitespace inside an acceptance entry", () => { + expect(designHash({ ...base, acceptance: [" F_rolled >= 0.999 "] })).toBe(designHash(base)); + }); + + it("ignores prose and assumptions entirely — rewording must not re-gate a warrant", () => { + expect(designHash({ ...base, invariants: ["anything at all"], assumptions: ["x"] })).toBe(designHash(base)); + }); + + it("CHANGES when a budget value changes", () => { + expect(designHash({ ...base, budget: { max_solves: 9, tier: "free" } })).not.toBe(designHash(base)); + }); + + it("CHANGES when task_type changes", () => { + expect(designHash({ ...base, task_type: "experiment-hw" })).not.toBe(designHash(base)); + }); + + // canonicalJson renders BOTH undefined and null as "null", and a literal + // {task_type, acceptance, budget} with budget:undefined still has an enumerable + // `budget` key — so without the compacting builder an absent budget hashes as + // `"budget":null`: stable, permanent, and wrong, with no error anywhere. + it("an undefined OR null budget hashes IDENTICALLY to an omitted one", () => { + const omitted = designHash({ task_type: "implement-slice", acceptance: ["x == 1"] }); + expect(designHash({ task_type: "implement-slice", acceptance: ["x == 1"], budget: undefined })).toBe(omitted); + expect(designHash({ task_type: "implement-slice", acceptance: ["x == 1"], budget: null })).toBe(omitted); + }); +}); + +describe("planHash", () => { + it("is 64 hex over goal + steps only", () => { + const h = planHash({ goal: "g", steps: [{ id: "s1" }] }); + expect(h).toMatch(/^[0-9a-f]{64}$/); + // compiled_at and design_hash are EXCLUDED, or a recompile that changed nothing + // would mint a new hash and invalidate a live warrant for no reason. + expect(planHash({ goal: "g", steps: [{ id: "s1" }], compiled_at: "now", design_hash: "d" })).toBe(h); + }); + it("CHANGES when a step changes", () => { + expect(planHash({ goal: "g", steps: [{ id: "s2" }] })).not.toBe(planHash({ goal: "g", steps: [{ id: "s1" }] })); + }); +}); From d9169d552fcae358d149e895716e0bc26502950d Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 11:00:16 -0400 Subject: [PATCH 14/27] feat(schema): register the spec and plan kinds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both carry `schema_version: "1"` and join SUPPORTED_VERSIONS_BY_KIND's hardcoded list. Omitting that second edit does NOT fail the build — the expression ends in an `as Record<…>` assertion that silences the missing key — it silently yields `SUPPORTED_VERSIONS_BY_KIND.spec === undefined`, so a test is the only guard. `spec.budget` $refs ledger-record's `$defs.bounds` by ABSOLUTE id. The relative form `ledger-record#/$defs/bounds` does not resolve: refs resolve against the referrer's own $id base, so ajv reports `can't resolve reference … from id …/schema/spec/v1`. Both kinds are registered AFTER ledger-record because the compile loop resolves refs in insertion order. `budget` is REQUIRED for launch-shaped task types and FORBIDDEN otherwise, via a pair of draft-07 if/then. Tolerating a budget on a non-launch spec is how mislabelling launch work would silently disable the budget and baseline lenses. `additionalProperties: true` because the vault taxonomy's own keys share the frontmatter block. `review` is optional by necessity — it is written BY the review. `baseline` uses oneOf(value+source | none_because) so "we never checked" cannot pass as a baseline. `steps[].optional` is added to the plan kind as the SOLE producer of the `skipped` step state the completion rule admits. Registering a kind broke three pre-existing assertions in validate.test.ts, all three predicted by the plan review and fixed here: the TOML-fixture loop (spec/plan are markdown-frontmatter kinds with no fixture form), the exact-set assertion, and the versions map. 140 schema, 692 amico-run, 777 extension green; both typechecks clean; build clean. Plan: plan-20260728-104500 Task 5. Co-Authored-By: Claude Opus 5 (1M context) --- packages/schema/schemas/plan.schema.json | 67 +++++++++++++++ packages/schema/schemas/spec.schema.json | 89 ++++++++++++++++++++ packages/schema/src/index.ts | 11 ++- packages/schema/test/spec_plan_kinds.test.ts | 71 ++++++++++++++++ packages/schema/test/validate.test.ts | 14 ++- 5 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 packages/schema/schemas/plan.schema.json create mode 100644 packages/schema/schemas/spec.schema.json create mode 100644 packages/schema/test/spec_plan_kinds.test.ts diff --git a/packages/schema/schemas/plan.schema.json b/packages/schema/schemas/plan.schema.json new file mode 100644 index 00000000..0e4a07fc --- /dev/null +++ b/packages/schema/schemas/plan.schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://amico.harmoniqs.co/schema/plan/v1", + "title": "amico compiled Plan", + "description": "The frontmatter of a COMPILED plan (spec-20260728 §4.1). The fleet spec's STEP schema is adopted unchanged; its plan-level frontmatter is revised here, because the hash chain needs plan_hash/design_hash and fleet §5.1 has neither. Hand-editing a compiled plan is a lint failure — the compiler is the only writer.", + "type": "object", + "additionalProperties": true, + "required": ["schema_version", "plan_id", "goal", "plan_hash", "design_hash", "steps"], + "properties": { + "schema_version": { "enum": ["1"] }, + "plan_id": { "type": "string", "minLength": 1 }, + "goal": { "type": "string", "minLength": 1 }, + "max_replans": { "type": "integer", "minimum": 0 }, + "plan_hash": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "description": "planHash({goal, steps}). Excludes design_hash and compiled_at so a recompile that changed nothing does not mint a new hash and invalidate a live warrant." + }, + "design_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$", "description": "The Spec's decision-surface hash this plan was compiled from." }, + "compiled_at": { "type": "string" }, + "compiled_by": { + "type": "object", + "additionalProperties": false, + "properties": { + "model": { "type": "string", "pattern": "^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$" }, + "variant": { "type": "string", "minLength": 1 } + } + }, + "suggested_ttl_s": { + "type": "integer", + "minimum": 1, + "description": "A RECOMMENDATION from step_count. `amico ledger approve` reads it to default --expires-in and remains the sole writer of expires_at — compile must not own a warrant's lifetime." + }, + "steps": { + "type": "array", + "minItems": 1, + "description": "Fleet §5.1 step objects, adopted unchanged. Only `optional` is added here: it is the SOLE producer of the `skipped` step state, which the completion rule admits.", + "items": { + "type": "object", + "additionalProperties": true, + "required": ["id"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "needs": { "type": "array", "items": { "type": "string" }, "description": "DAG predecessors. Distinct from the compile-time capability `demands` — one name, one meaning." }, + "gates": { "type": "array", "items": { "type": "string" } }, + "optional": { "type": "boolean" } + } + } + }, + "advisories": { + "type": "array", + "description": "Surviving review findings, carried as obligations. A plan cannot reach `complete` while any is open.", + "items": { + "type": "object", + "additionalProperties": true, + "required": ["id"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "lens": { "type": "string" }, + "claim": { "type": "string" }, + "remedy": { "type": "string" }, + "round": { "type": "integer" } + } + } + } + } +} diff --git a/packages/schema/schemas/spec.schema.json b/packages/schema/schemas/spec.schema.json new file mode 100644 index 00000000..a1cde3eb --- /dev/null +++ b/packages/schema/schemas/spec.schema.json @@ -0,0 +1,89 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://amico.harmoniqs.co/schema/spec/v1", + "title": "amico Spec (deliberation)", + "description": "The YAML frontmatter of a vault `spec` note, extended for the deliberation layer (spec-20260728 §2). NOT a new note kind: the vault `spec` type already carried status + linked_plan, so this adds fields rather than inventing an artifact. additionalProperties is TRUE because the vault taxonomy's own keys (date, session_id, status, priority, platform, visibility, tags, linked_plan) share the same frontmatter block and must be tolerated.", + "type": "object", + "additionalProperties": true, + "required": ["schema_version", "spec_id", "task_type", "acceptance"], + "allOf": [ + { + "description": "budget IS the warrant's bounds, so it is required exactly where the work can reach a gated capability.", + "if": { + "properties": { "task_type": { "enum": ["experiment-sim", "experiment-hw", "author-script"] } }, + "required": ["task_type"] + }, + "then": { "required": ["budget"] } + }, + { + "description": "And FORBIDDEN elsewhere: a non-launch-shaped spec has nothing to bound, and tolerating a budget here is how mislabelled launch work would silently disable the budget and baseline lenses.", + "if": { + "properties": { + "task_type": { + "enum": ["implement-slice", "plan", "review", "insight", "bookkeeping", "triage", "converse"] + } + }, + "required": ["task_type"] + }, + "then": { "not": { "required": ["budget"] } } + } + ], + "properties": { + "schema_version": { "enum": ["1"] }, + "spec_id": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "IMMUTABLE identity, authored once and never re-derived. design_hash is not an identity — two specs sharing an acceptance list would collide in the findings namespace and in the recompiled-plan lookup." + }, + "task_type": { + "enum": ["triage", "plan", "author-script", "implement-slice", "bookkeeping", "insight", "review", "experiment-sim", "experiment-hw", "converse"], + "description": "Exactly TASK_TYPES from amico-run/src/ledger.ts, re-declared here as the closed enum the `schema` lens checks. One copy of the vocabulary; extensible only by schema revision." + }, + "acceptance": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 }, + "description": "MACHINE-PARSEABLE criteria only: `metric comparator threshold`. The `falsifiable` lens is blocking over this field, so behavioural prose belongs in `invariants` — a lens lenient enough to accept prose is a blocking lens that silently passes everything." + }, + "invariants": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "description": "Behavioural properties in prose, which many real specs need and no parser should pretend to check. Excluded from design_hash: rewording must not re-gate a live warrant." + }, + "assumptions": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "description": "What must hold. Excluded from design_hash deliberately — a violated assumption is a runtime blocked-report, not a re-approval trigger." + }, + "budget": { "$ref": "https://amico.harmoniqs.co/schema/ledger-record/v1#/$defs/bounds" }, + "baseline": { + "type": "object", + "additionalProperties": false, + "description": "What this work is measured against. The `baseline` lens is blocking precisely so 'we never checked' cannot pass silently: either a number WITH its source, or an explicit statement that none exists.", + "properties": { + "value": { "type": "number" }, + "source": { "type": "string", "minLength": 1 }, + "none_because": { "type": "string", "minLength": 1 } + }, + "oneOf": [{ "required": ["value", "source"] }, { "required": ["none_because"] }] + }, + "structure_hash": { + "type": "string", + "minLength": 1, + "description": "The work identity the `precedent` lens queries the ledger on. Absent → the lens reports not-applicable, which is NOT the same claim as zero prior attempts." + }, + "review": { + "type": "object", + "additionalProperties": true, + "description": "Written BY the review, so it is optional — requiring it would make every first review impossible.", + "properties": { + "design_hash": { "type": "string" }, + "rounds": { "type": "integer" }, + "critics": { "type": "array" }, + "findings_count": { "type": "integer" }, + "findings_ref": { "type": "string" } + } + } + } +} diff --git a/packages/schema/src/index.ts b/packages/schema/src/index.ts index 411b52e5..de3e4cb6 100644 --- a/packages/schema/src/index.ts +++ b/packages/schema/src/index.ts @@ -28,6 +28,13 @@ import problemspecSchema from "../schemas/problemspec.schema.json" with { type: // like problemspec it has NO top-level properties.schema_version — see the SCHEMAS // note below and the SUPPORTED_VERSIONS_BY_KIND exclusion. import ledgerRecordSchema from "../schemas/ledger-record.schema.json" with { type: "json" }; +// The deliberation artifacts (spec-20260728). Both are MARKDOWN-frontmatter shapes, so +// neither has a canonical filename and `kindForFilename` gains no entry — the kind is +// always explicit. Registered AFTER ledger-record on purpose: `spec.budget` $refs +// ledger-record's $defs.bounds, and the compile loop below resolves refs in insertion +// order, so an earlier entry cannot reference a later one. +import specSchema from "../schemas/spec.schema.json" with { type: "json" }; +import planSchema from "../schemas/plan.schema.json" with { type: "json" }; // Cross-language ProblemSpec hashing (Plan 2 Task 5) — re-exported at the package // root so cross-package consumers (e.g. the extension's ledger_client.ts, Plan 3 @@ -63,6 +70,8 @@ const SCHEMAS = { // `.properties.schema_version` off an undefined and crash @amicode/schema at load // (Plan 3 review correction #1, exactly the problemspec case). "ledger-record": ledgerRecordSchema, + spec: specSchema, + plan: planSchema, } as const; export type SchemaKind = keyof typeof SCHEMAS; @@ -78,7 +87,7 @@ export const SCHEMA_KINDS = Object.keys(SCHEMAS) as SchemaKind[]; * top-level properties.schema_version) are excluded from this string-version map. */ export const SUPPORTED_VERSIONS_BY_KIND: Record, string[]> = Object.fromEntries( - (["run", "result", "lab", "solvespec", "catalog-entry"] as const).map((kind) => [ + (["run", "result", "lab", "solvespec", "catalog-entry", "spec", "plan"] as const).map((kind) => [ kind, (SCHEMAS[kind] as { properties: { schema_version: { enum: string[] } } }).properties.schema_version.enum, ]), diff --git a/packages/schema/test/spec_plan_kinds.test.ts b/packages/schema/test/spec_plan_kinds.test.ts new file mode 100644 index 00000000..3b378e71 --- /dev/null +++ b/packages/schema/test/spec_plan_kinds.test.ts @@ -0,0 +1,71 @@ +// The `spec` and `plan` schema kinds (spec-20260728 §2, §4.1). +// Plan: plan-20260728-104500 Task 5. +import { describe, it, expect } from "vitest"; +import { validate, SUPPORTED_VERSIONS_BY_KIND } from "../src/index.js"; + +const spec = (over: Record = {}) => ({ + schema_version: "1", + spec_id: "spec-20260728-093846-x", + type: "spec", + task_type: "experiment-sim", + acceptance: ["F_rolled >= 0.999"], + budget: { max_solves: 8, tier: "free" }, + baseline: { value: 0.968, source: "published blockade-pi protocol" }, + // real vault-taxonomy keys that MUST be tolerated — they share the frontmatter block + date: "2026-07-28", session_id: "u", status: "draft", tags: ["spec"], linked_plan: null, + ...over, +}); +const drop = (o: Record, k: string) => { const c = { ...o }; delete c[k]; return c; }; + +describe("the spec kind", () => { + it("accepts a launch-shaped spec alongside the vault's own frontmatter keys", () => { + expect(validate(spec(), "spec")).toMatchObject({ ok: true }); + }); + it("REQUIRES budget for launch-shaped task types", () => { + expect(validate(drop(spec(), "budget"), "spec").ok).toBe(false); + }); + it("FORBIDS budget for non-launch-shaped task types", () => { + expect(validate(spec({ task_type: "implement-slice" }), "spec").ok).toBe(false); + }); + it("accepts a non-launch-shaped spec with no budget", () => { + expect(validate(drop(spec({ task_type: "implement-slice" }), "budget"), "spec").ok).toBe(true); + }); + it("rejects a task_type outside TASK_TYPES", () => { + expect(validate(spec({ task_type: "vibes" }), "spec").ok).toBe(false); + }); + it("rejects a budget key outside WarrantBounds (the $ref must actually resolve)", () => { + expect(validate(spec({ budget: { max_duration: "30m" } }), "spec").ok).toBe(false); + }); + it("does NOT require `review` — it is written BY the review", () => { + expect(validate(spec(), "spec").ok).toBe(true); + }); + it("baseline: accepts value+source, accepts none_because, rejects a bare value", () => { + expect(validate(spec({ baseline: { none_because: "first of its kind" } }), "spec").ok).toBe(true); + expect(validate(spec({ baseline: { value: 0.9 } }), "spec").ok).toBe(false); + }); +}); + +describe("the plan kind", () => { + const plan = (over: Record = {}) => ({ + schema_version: "1", plan_id: "plan-20260728-1045-x", goal: "g", + plan_hash: "c".repeat(64), design_hash: "a".repeat(64), + steps: [{ id: "s1", gates: ["re-rollout"] }], max_replans: 3, + ...over, + }); + it("accepts a compiled plan", () => { + expect(validate(plan(), "plan")).toMatchObject({ ok: true }); + }); + it("requires the design_hash it was compiled from", () => { + expect(validate(drop(plan(), "design_hash"), "plan").ok).toBe(false); + }); + it("accepts an optional-step marker, the only producer of `skipped`", () => { + expect(validate(plan({ steps: [{ id: "s1", optional: true }] }), "plan").ok).toBe(true); + }); +}); + +describe("registration", () => { + it("both kinds carry a version, so the module does not crash at load", () => { + expect(SUPPORTED_VERSIONS_BY_KIND.spec).toEqual(["1"]); + expect(SUPPORTED_VERSIONS_BY_KIND.plan).toEqual(["1"]); + }); +}); diff --git a/packages/schema/test/validate.test.ts b/packages/schema/test/validate.test.ts index e244b8d0..33597d66 100644 --- a/packages/schema/test/validate.test.ts +++ b/packages/schema/test/validate.test.ts @@ -17,7 +17,9 @@ describe("valid golden fixtures validate clean", () => { // ledger-record is JSONL ops-data (runs.jsonl), not a TOML run-dir artifact, so it // carries no golden .toml fixture and is not part of the Julia round-trip corpus — // it has its own dedicated coverage in ledger-record.test.ts. - for (const kind of SCHEMA_KINDS.filter((k) => k !== "ledger-record")) { + // `spec` and `plan` join ledger-record in the exclusion: both are MARKDOWN-frontmatter + // kinds with no TOML fixture form, so there is no `fixtures/valid/.toml` to load. + for (const kind of SCHEMA_KINDS.filter((k) => k !== "ledger-record" && k !== "spec" && k !== "plan")) { it(`${kind}: fixture conforms`, () => { const r = validateFile(fixtureFile(kind), kind); expect(r.errors).toEqual([]); @@ -29,7 +31,11 @@ describe("valid golden fixtures validate clean", () => { describe("schema set + exports", () => { it("exposes all five versioned schemas + the FINISHED sub-shape + the problemspec + ledger-record kinds", () => { expect(new Set(SCHEMA_KINDS)).toEqual( - new Set(["run", "result", "lab", "solvespec", "catalog-entry", "finished", "problemspec", "ledger-record"]), + new Set([ + "run", "result", "lab", "solvespec", "catalog-entry", "finished", "problemspec", "ledger-record", + // the deliberation artifacts (spec-20260728) + "spec", "plan", + ]), ); }); it("supported versions are PER-KIND: run at v2 (spec C); solvespec at v5 (v4 hpc tier + remote executor + problem_spec; v5 plan_hash); the rest v1", () => { @@ -39,6 +45,10 @@ describe("schema set + exports", () => { result: ["1"], lab: ["1"], "catalog-entry": ["1"], + // the deliberation artifacts (spec-20260728): both DO carry a top-level + // schema_version enum, so unlike problemspec/ledger-record they join the map. + spec: ["1"], + plan: ["1"], }); }); it("an unknown kind is a clean error, not a throw", () => { From 11027f326f54a31ab79d3f1a4d79281d601073c5 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 11:06:51 -0400 Subject: [PATCH 15/27] feat(warrant): distinguish a recompiled plan from an unapproved one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A recompile mints a new plan_hash and correctly invalidates the warrant, but surfacing that as a bare "no approved warrant" tells the user nothing about what changed or what to do — mid-campaign, at the launch gate. The gate now has a third refusal branch: no approval, and none for this design either -> "no approved warrant …" (existing) approval exists but lapsed -> "… has expired — re-approve" (existing) a DIFFERENT plan for the same design is live -> "the plan was recompiled …" (new) The join is only possible with the plan_compiled rows, which carry the design_hash -> plan_hash binding — that is why that ledger kind landed with the schema rather than with the verbs that write it. checkWarrant's fourth parameter is optional and defaults to [], so every existing call site compiles and behaves identically; warrant.ts stays a pure module over injected records (no ledger reads), which is what keeps its tests hermetic. The rows are collected in warrant_context and threaded through gate.ts's WarrantContext. Expiry deliberately still wins over recompilation: "re-approve, it expired" is the more actionable message when both are true. Verified the new tests fail without the implementation (stashed src/warrant.ts: the recompiled case reports the generic refusal). 697 amico-run, 140 schema, 777 extension; typechecks and build clean. Plan: plan-20260728-104500 Task 6. Co-Authored-By: Claude Opus 5 (1M context) --- packages/amico-run/src/gate.ts | 6 ++- packages/amico-run/src/warrant.ts | 29 +++++++++++- packages/amico-run/src/warrant_context.ts | 8 +++- packages/amico-run/test/warrant.test.ts | 56 ++++++++++++++++++++++- 4 files changed, 94 insertions(+), 5 deletions(-) diff --git a/packages/amico-run/src/gate.ts b/packages/amico-run/src/gate.ts index ca87a356..1add59c3 100644 --- a/packages/amico-run/src/gate.ts +++ b/packages/amico-run/src/gate.ts @@ -19,7 +19,7 @@ import { maskedHash } from "./baseline.js"; import { loadExemplarsIndex } from "./catalog.js"; import { hasCloudConfig } from "./remote_config.js"; import { checkWarrant, type DeviceAccess, type SizeClass, type WarrantRefusal } from "./warrant.js"; -import type { ApprovalRecord } from "./ledger.js"; +import type { ApprovalRecord, PlanCompiledRecord } from "./ledger.js"; export interface GateStamp { tier?: string; @@ -79,6 +79,9 @@ export interface WarrantContext { sizeClass?: SizeClass; device?: DeviceAccess; solvesSoFar?: number; + /** `plan_compiled` rows — the design_hash -> plan_hash binding the §4.6 + * "the plan was recompiled" refusal joins on. */ + planCompiled?: readonly PlanCompiledRecord[]; } export function runGate( @@ -170,6 +173,7 @@ export function runGate( }, warrant.approvals, warrant.now, + warrant.planCompiled ?? [], ); if (!check.ok) return { ok: false, reason: check.reason, refusal: check }; } diff --git a/packages/amico-run/src/warrant.ts b/packages/amico-run/src/warrant.ts index f9463612..11f28ef5 100644 --- a/packages/amico-run/src/warrant.ts +++ b/packages/amico-run/src/warrant.ts @@ -17,7 +17,7 @@ // // Threat model is drift, not an adversary (spec §3), so no signature is verified // here. See spec §6 for what that does and does not buy. -import type { ApprovalRecord, WarrantBounds } from "./ledger.js"; +import type { ApprovalRecord, PlanCompiledRecord, WarrantBounds } from "./ledger.js"; export type SizeClass = "SMALL" | "MEDIUM"; export type DeviceAccess = "none" | "ro" | "rw"; @@ -94,11 +94,34 @@ export function gatedCapabilities(facts: LaunchFacts): string[] { return needs; } +/** Did a DIFFERENT plan for this same design already hold a live warrant? If so the + * refusal is "you recompiled", not "you never approved anything" — a recompile mints a + * new plan_hash and correctly invalidates the warrant, but surfacing that as a bare + * mid-campaign denial tells the user nothing about what changed or what to do + * (spec-20260728 §4.6). Returns false when no plan_compiled rows are supplied. */ +function supersededPlan( + planHash: string, + approvals: readonly ApprovalRecord[], + planCompiled: readonly PlanCompiledRecord[], + now: number, +): boolean { + // Newest row wins: a plan may be recorded more than once (re-runs of compile). + const mine = [...planCompiled].filter((r) => r.plan_hash === planHash).sort((a, b) => (a.ts < b.ts ? 1 : -1))[0]; + if (!mine) return false; + return planCompiled.some( + (r) => r.design_hash === mine.design_hash && r.plan_hash !== planHash && liveWarrant(r.plan_hash, approvals, now) !== undefined, + ); +} + /** The §5.1 check. */ export function checkWarrant( facts: LaunchFacts, approvals: readonly ApprovalRecord[], now: number, + /** `plan_compiled` rows, for the SUPERSEDED branch below. Optional so every existing + * call site is unchanged; absent simply means the gate cannot tell a recompile from a + * never-approved plan and falls back to the generic refusal. */ + planCompiled: readonly PlanCompiledRecord[] = [], ): WarrantCheck { const needs = gatedCapabilities(facts); if (needs.length === 0) return { ok: true }; // inside the free set — nothing to authorise @@ -125,7 +148,9 @@ export function checkWarrant( required: needs, reason: hasLapsed(facts.plan_hash, approvals, now) ? `the warrant for plan ${facts.plan_hash} has expired — re-approve it (needs ${needs.join(", ")})` - : `no approved warrant for plan ${facts.plan_hash} — approve it declaring ${needs.join(", ")}`, + : supersededPlan(facts.plan_hash, approvals, planCompiled, now) + ? `the plan was recompiled (${facts.plan_hash} supersedes an approved plan for the same design) — re-approve it declaring ${needs.join(", ")}` + : `no approved warrant for plan ${facts.plan_hash} — approve it declaring ${needs.join(", ")}`, }; } diff --git a/packages/amico-run/src/warrant_context.ts b/packages/amico-run/src/warrant_context.ts index 7aae94dd..d9174ec7 100644 --- a/packages/amico-run/src/warrant_context.ts +++ b/packages/amico-run/src/warrant_context.ts @@ -6,7 +6,7 @@ // context as "the step does not exist", so the whole feature is off by default and the // env var is the entire flag surface. That is deliberate for the dogfood phase (plan // CLI step 4) — the internal ring turns it on, nobody else changes behavior. -import { readRecords, type ApprovalRecord } from "./ledger.js"; +import { readRecords, type ApprovalRecord, type PlanCompiledRecord } from "./ledger.js"; import { extractKeyVars, memoryScore, tshirtSize } from "./estimate.js"; import type { WarrantContext } from "./gate.js"; import type { DeviceAccess, SizeClass } from "./warrant.js"; @@ -60,17 +60,23 @@ export function assembleWarrantContext(opts: AssembleOptions): WarrantContext | // A ledger that is missing or unreadable yields NO approvals, which fails closed: // every gated launch refuses rather than sailing through unwarranted. let approvals: ApprovalRecord[] = []; + let planCompiled: PlanCompiledRecord[] = []; let all: { type: string; plan_hash?: string }[] = []; try { const records = readRecords(); all = records as unknown as { type: string; plan_hash?: string }[]; approvals = records.filter((r): r is ApprovalRecord => r.type === "approval"); + // For the §4.6 SUPERSEDED refusal: these rows carry the design_hash -> plan_hash + // binding, which is the only way the gate can say "you recompiled" rather than + // "you never approved anything". + planCompiled = records.filter((r): r is PlanCompiledRecord => r.type === "plan_compiled"); } catch { /* no ledger → no warrants → gated launches refuse */ } return { approvals, + planCompiled, now: opts.now ?? Date.now(), sizeClass: sizeClassFor(opts.scriptText), device: opts.device ?? "none", diff --git a/packages/amico-run/test/warrant.test.ts b/packages/amico-run/test/warrant.test.ts index 9411c0ea..b0acd5b4 100644 --- a/packages/amico-run/test/warrant.test.ts +++ b/packages/amico-run/test/warrant.test.ts @@ -6,7 +6,7 @@ // omits, may only ever RESTRICT a launch — never widen it. import { describe, it, expect } from "vitest"; import { checkWarrant, type LaunchFacts } from "../src/warrant.js"; -import type { ApprovalRecord } from "../src/ledger.js"; +import type { ApprovalRecord, PlanCompiledRecord } from "../src/ledger.js"; const NOW = Date.parse("2026-07-27T20:00:00Z"); const iso = (min: number) => new Date(NOW + min * 60_000).toISOString(); @@ -139,4 +139,58 @@ describe("the refusal contract (§5.2)", () => { expect(r.ok === false && r.required).toContain("tier"); expect(r.ok === false && r.plan_hash).toBeUndefined(); }); + + // §4.6 — the THIRD refusal branch. A recompile mints a new plan_hash and correctly + // invalidates the warrant, but a bare "no warrant" tells the user nothing about what + // changed. The join needs plan_compiled rows, which is why that ledger kind lands with + // the schema rather than with the new verbs. + describe("§4.6 recompiled vs never-approved", () => { + const planRow = (plan_hash: string, design_hash: string): PlanCompiledRecord => ({ + type: "plan_compiled", ts: iso(-10), plan_hash, design_hash, + spec_id: "spec-1", step_count: 1, source: "user", + }); + const gated = () => freeLaunch({ tier: "hpc", sizeClass: "MEDIUM", plan_hash: "newplan" }); + + it("names RECOMPILATION when a prior plan for the same design was approved", () => { + const r = checkWarrant( + gated(), + [warrant({ plan_hash: "oldplan", bounds: { tier: "hpc", max_size_class: "MEDIUM" } })], + NOW, + [planRow("oldplan", "d1"), planRow("newplan", "d1")], + ); + expect(r.ok).toBe(false); + expect(r.ok === false && r.reason).toMatch(/recompiled/i); + }); + + it("says NO WARRANT when the prior approval is for a DIFFERENT design", () => { + const r = checkWarrant( + gated(), + [warrant({ plan_hash: "oldplan", bounds: { tier: "hpc", max_size_class: "MEDIUM" } })], + NOW, + [planRow("oldplan", "d2"), planRow("newplan", "d1")], + ); + expect(r.ok === false && r.reason).not.toMatch(/recompiled/i); + }); + + it("says NO WARRANT when nothing for this design was ever approved", () => { + const r = checkWarrant(gated(), [], NOW, [planRow("newplan", "d1")]); + expect(r.ok === false && r.reason).not.toMatch(/recompiled/i); + }); + + it("EXPIRY still wins over recompilation — the lapsed message is more actionable", () => { + const r = checkWarrant( + gated(), + [warrant({ plan_hash: "newplan", bounds: { tier: "hpc", max_size_class: "MEDIUM" }, expires_at: iso(-1) })], + NOW, + [planRow("oldplan", "d1"), planRow("newplan", "d1")], + ); + expect(r.ok === false && r.reason).toMatch(/expired/i); + }); + + it("omitting planCompiled keeps the existing two-branch behaviour (back-compat)", () => { + const r = checkWarrant(gated(), [], NOW); + expect(r.ok).toBe(false); + expect(r.ok === false && r.reason).not.toMatch(/recompiled/i); + }); + }); }); From e957290db289cac0915e89e57ebe35062f97b1e1 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 11:30:33 -0400 Subject: [PATCH 16/27] fix(extension): the ledger_client test was appending to the developer's REAL ledger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `~/.amico/ledger/runs.jsonl` had accumulated ten junk rows — {type: burn, ts: "t", class: "x", mechanism: "y"} — one per extension-suite run. Cause: one test deletes AMICO_BIN on purpose, to exercise resolveAmicoBin()'s import.meta.url/PATH fallback. On a machine where a real `amico` IS installed that branch resolves it, so appendStanza performs a REAL append — and with AMICO_LEDGER unset it lands in the developer's own ops data. The test's comment anticipated the binary resolving ("whether a real `amico` happens to be on PATH is not the point") but not that resolving means a side-effecting write outside the test's sandbox. Fix: the describe block now points AMICO_LEDGER at its temp dir and restores it after, so the PATH branch stays exercised while the write goes somewhere disposable. The test additionally asserts the real ledger's mtime is unchanged across an append — the property that was silently false before. Verified: real ledger line count is identical across a full extension run (was growing by one per run). 777 tests green. Found while chasing an intermittent amico-run failure. It is NOT that flake — `amico catalog ingest` never touches the ledger, and no test asserts the default path — but it is a real defect: a unit test mutating the user's append-only ops store. Co-Authored-By: Claude Opus 5 (1M context) --- packages/extension/test/ledger_client.test.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/extension/test/ledger_client.test.ts b/packages/extension/test/ledger_client.test.ts index 705f1619..8ff24aa3 100644 --- a/packages/extension/test/ledger_client.test.ts +++ b/packages/extension/test/ledger_client.test.ts @@ -60,13 +60,24 @@ describe("resolveAmicoBinFrom", () => { describe("appendStanza — shells `amico ledger append` (never touches runs.jsonl directly)", () => { let dir: string; const prevBin = process.env.AMICO_BIN; + const prevLedger = process.env.AMICO_LEDGER; beforeEach(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), "ledger-client-")); + // CONTAIN THE SIDE EFFECT. One test below deletes AMICO_BIN on purpose, to exercise + // resolveAmicoBin()'s PATH branch — and on a machine where a REAL `amico` is + // installed, that branch resolves it and appendStanza performs a REAL append. With + // AMICO_LEDGER unset that lands in the developer's own ~/.amico/ledger/runs.jsonl. + // Observed: 10 junk `burn` rows (ts:"t", class:"x") accumulated there, one per suite + // run. Pointing the ledger at the temp dir keeps the PATH branch exercised while the + // write goes somewhere disposable. + process.env.AMICO_LEDGER = path.join(dir, "runs.jsonl"); }); afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); if (prevBin === undefined) delete process.env.AMICO_BIN; else process.env.AMICO_BIN = prevBin; + if (prevLedger === undefined) delete process.env.AMICO_LEDGER; + else process.env.AMICO_LEDGER = prevLedger; }); it("pipes the JSON stanza to the resolved bin's stdin and returns true on success", () => { @@ -96,7 +107,16 @@ describe("appendStanza — shells `amico ledger append` (never touches runs.json // No assertion on the boolean (whether a real `amico` happens to be on PATH in // this env is not the point) — the point is resolveAmicoBin()'s import.meta.url // branch executes cleanly under vitest's ESM transform and appendStanza never throws. + // The side effect IS the point of the beforeEach's AMICO_LEDGER redirect: where a real + // `amico` is installed this branch really does append, and it must not land in the + // developer's own ledger. expect(() => appendStanza({ type: "burn", ts: "t", class: "x", mechanism: "y" })).not.toThrow(); + // Whatever it wrote (if anything) went to the temp ledger, never to ~/.amico. + const real = path.join(os.homedir(), ".amico", "ledger", "runs.jsonl"); + const before = fs.existsSync(real) ? fs.statSync(real).mtimeMs : 0; + appendStanza({ type: "burn", ts: "t", class: "x", mechanism: "y" }); + const after = fs.existsSync(real) ? fs.statSync(real).mtimeMs : 0; + expect(after).toBe(before); // the real ledger is untouched }); }); From 6108ce94e34291f887fbd02f4ac58f1e6f08adb6 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 11:33:20 -0400 Subject: [PATCH 17/27] feat(amico-run): frontmatter reader + lens registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two leaf modules the spec-review verb composes. frontmatter.ts — amico-validate cannot read a Spec: it takes --schema (not --kind), TOML-parses anything whose extension is not .json, no package here depends on a YAML parser, and it returns 64 for BOTH a usage error and an invalid document, which would leave `ran` vs `unverified` undecidable for the schema lens. So the verb extracts frontmatter itself and validates in-process. It returns a RESULT and never throws, deliberately: a malformed spec must surface as a blocking FINDING (exit 65), not a ConfigError (64). One says "your spec is wrong, here is what to fix"; the other says "you invoked the tool wrong". The opening fence must be on the FIRST line — a `---` further down is a horizontal rule, and treating one as frontmatter would silently validate the wrong block. lens_registry.ts — an entry for EVERY value of TASK_TYPES, enforced by test. Rev 1 of the spec covered 5 of 10, so half the closed enum fell through to a single lens while --critics defaulted to 3: three frontier calls spent on one lens. criticCountFor() clamps a request to the lenses that actually exist, and to ZERO for the tier-1-only types. `decomposition` is in every non-empty tier-2 set. Rev 1 withheld it from implement-slice, so the specs most exposed to bad carving were the one category never reviewed for it — and the spec that shipped with three contradictions was itself an implement-slice. Recorded deviation (advisory A-13): spec §3.5 places the registry in amico-plugin with a git sha as lens_registry_version. Nothing here can read another repo's sha at runtime, so it lives in amico-run with a hand-bumped constant until the plugin-side home exists. 24 new tests; typecheck clean. Plan: plan-20260728-104500 Tasks 7-8. Co-Authored-By: Claude Opus 5 (1M context) --- packages/amico-run/package.json | 3 +- packages/amico-run/src/frontmatter.ts | 60 ++++++++++ packages/amico-run/src/lens_registry.ts | 105 ++++++++++++++++++ packages/amico-run/test/frontmatter.test.ts | 63 +++++++++++ packages/amico-run/test/lens_registry.test.ts | 99 +++++++++++++++++ pnpm-lock.yaml | 3 + 6 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 packages/amico-run/src/frontmatter.ts create mode 100644 packages/amico-run/src/lens_registry.ts create mode 100644 packages/amico-run/test/frontmatter.test.ts create mode 100644 packages/amico-run/test/lens_registry.test.ts diff --git a/packages/amico-run/package.json b/packages/amico-run/package.json index 4e104bb0..6fc1586a 100644 --- a/packages/amico-run/package.json +++ b/packages/amico-run/package.json @@ -21,7 +21,8 @@ }, "dependencies": { "@amicode/schema": "workspace:*", - "smol-toml": "^1.3.0" + "smol-toml": "^1.3.0", + "yaml": "^2.9.0" }, "devDependencies": { "@types/node": "^22.0.0", diff --git a/packages/amico-run/src/frontmatter.ts b/packages/amico-run/src/frontmatter.ts new file mode 100644 index 00000000..74f220ce --- /dev/null +++ b/packages/amico-run/src/frontmatter.ts @@ -0,0 +1,60 @@ +// YAML frontmatter extraction for the deliberation Spec artifact (spec-20260728 §2.5). +// +// WHY THIS EXISTS RATHER THAN amico-validate +// ------------------------------------------ +// The Spec is a vault markdown note whose frontmatter is the contract. `amico-validate` +// cannot read it: it takes `--schema` (not `--kind`), it TOML-parses anything whose +// extension is not `.json`, no package here depends on a YAML parser, and it returns exit +// 64 for BOTH a usage error and an invalid document — so a caller could not tell "the +// lens ran and the spec is bad" from "the lens could not run", which is exactly the +// distinction the review's `ran | unverified` status turns on. +// +// So the review verb extracts the frontmatter here and validates it IN-PROCESS against the +// registered `spec` schema. +// +// RETURNS A RESULT, NEVER THROWS. A malformed spec must surface as a blocking FINDING +// (exit 65) and not as a ConfigError (exit 64): the first says "your spec is wrong, here +// is what to fix", the second says "you invoked the tool wrong". +import { parse as parseYaml } from "yaml"; + +export type FrontmatterResult = + | { ok: true; data: Record } + | { ok: false; error: string }; + +/** The frontmatter fence must be the FIRST line. A `---` further down is a horizontal + * rule or a nested document, not this note's contract — treating one as frontmatter + * would silently validate the wrong block. */ +const OPENING = /^---[ \t]*\r?\n/; + +export function parseFrontmatter(raw: string): FrontmatterResult { + if (!OPENING.test(raw)) { + return { + ok: false, + error: "no YAML frontmatter: the note must open with a `---` fence on its first line", + }; + } + const afterOpen = raw.replace(OPENING, ""); + // The FIRST closing fence ends the block; a later one belongs to the body. + const close = afterOpen.search(/^---[ \t]*(\r?\n|$)/m); + if (close === -1) { + return { ok: false, error: "unterminated YAML frontmatter: no closing `---` fence" }; + } + const block = afterOpen.slice(0, close); + + let parsed: unknown; + try { + parsed = parseYaml(block); + } catch (e) { + return { ok: false, error: `malformed YAML frontmatter: ${(e as Error).message}` }; + } + if (parsed === null || parsed === undefined) { + return { ok: false, error: "empty YAML frontmatter: expected a mapping of fields" }; + } + if (typeof parsed !== "object" || Array.isArray(parsed)) { + return { + ok: false, + error: `YAML frontmatter must be a mapping of fields, got ${Array.isArray(parsed) ? "a list" : typeof parsed}`, + }; + } + return { ok: true, data: parsed as Record }; +} diff --git a/packages/amico-run/src/lens_registry.ts b/packages/amico-run/src/lens_registry.ts new file mode 100644 index 00000000..043c02d9 --- /dev/null +++ b/packages/amico-run/src/lens_registry.ts @@ -0,0 +1,105 @@ +// The lens registry (spec-20260728 §3.1, §3.5): which review lenses apply to which +// `task_type`, and at which tier. +// +// Rev 1 of the spec covered 5 of the 10 task types, so half the closed enum fell through +// to a single lens while `--critics 3` was the default — three calls spent on one lens. +// Exhaustiveness over TASK_TYPES is therefore a TEST, not a convention. +// +// DEVIATION FROM THE SPEC, recorded deliberately (advisory A-13): spec §3.5 places this +// registry in amico-plugin and defines `lens_registry_version` as that repo's git sha. +// Nothing in amico-run can read another repo's sha at runtime, so the registry lives here +// and the version is a local constant until the plugin-side home exists. +import { TASK_TYPES, type TaskType } from "./ledger.js"; + +/** Bumped BY HAND whenever the lens set or its applicability changes. Stamped into every + * `spec_review` record so a review is attributable to the rules that produced it — a spec + * approved under a weaker lens set is then visible as such rather than indistinguishable + * from one approved under the current set. */ +export const LENS_REGISTRY_VERSION = "1"; + +/** Tier-1 lenses: mechanical, free, deterministic, computed from the spec alone. */ +export const TIER1_LENSES = [ + "schema", + "falsifiable", + "budget", + "baseline", + "precedent", + "provenance", +] as const; +export type Tier1Lens = (typeof TIER1_LENSES)[number]; + +/** Tier-2 lenses: judgment, frontier-tier, one critic each. Not exercised by this slice + * (the subprocess mechanism is G-2-gated) but declared here so the registry is complete + * and `--critics` clamping has something real to clamp against. */ +export const TIER2_LENSES = [ + "hidden-failure", + "decomposition", + "physics-adequacy", + "cost-realism", + "interface-boundary", + "test-adequacy", + "sequencing", + "dependency-order", + "evidence-adequacy", +] as const; +export type Tier2Lens = (typeof TIER2_LENSES)[number]; + +/** Task types whose work can reach a gated capability, and which therefore carry a + * `budget`. The `budget`, `baseline` and `precedent` lenses are scoped to these. */ +export const LAUNCH_SHAPED: readonly TaskType[] = ["experiment-sim", "experiment-hw", "author-script"]; + +export interface LensSet { + tier1: readonly Tier1Lens[]; + tier2: readonly Tier2Lens[]; +} + +// Universal tier-1 lenses. `schema` and `falsifiable` apply to every spec: one checks the +// contract, the other checks that the acceptance criteria are criteria at all. +const T1_UNIVERSAL: readonly Tier1Lens[] = ["schema", "falsifiable", "provenance"]; +const T1_LAUNCH: readonly Tier1Lens[] = [...T1_UNIVERSAL, "budget", "baseline", "precedent"]; + +// `hidden-failure` and `decomposition` are in EVERY non-empty tier-2 set. Rev 1 withheld +// `decomposition` from `implement-slice`, so the specs most exposed to bad carving were the +// one category never reviewed for it — and the spec that shipped with three contradictions +// was itself an `implement-slice`. +const T2_ALWAYS: readonly Tier2Lens[] = ["hidden-failure", "decomposition"]; + +/** An entry for EVERY value of TASK_TYPES — enforced by test, not by convention. */ +export const LENS_REGISTRY: Record = { + "experiment-sim": { tier1: T1_LAUNCH, tier2: [...T2_ALWAYS, "physics-adequacy", "cost-realism"] }, + "experiment-hw": { tier1: T1_LAUNCH, tier2: [...T2_ALWAYS, "physics-adequacy", "cost-realism"] }, + "author-script": { tier1: T1_LAUNCH, tier2: [...T2_ALWAYS, "physics-adequacy", "cost-realism"] }, + "implement-slice": { tier1: T1_UNIVERSAL, tier2: [...T2_ALWAYS, "interface-boundary", "test-adequacy"] }, + plan: { tier1: T1_UNIVERSAL, tier2: [...T2_ALWAYS, "sequencing", "dependency-order"] }, + review: { tier1: T1_UNIVERSAL, tier2: [...T2_ALWAYS, "evidence-adequacy"] }, + insight: { tier1: T1_UNIVERSAL, tier2: [...T2_ALWAYS, "evidence-adequacy"] }, + // Conversational / bookkeeping work gets tier 1 only. Spending a frontier critic on + // "record this fact" is the bureaucracy trap the spec's §8 names. + triage: { tier1: T1_UNIVERSAL, tier2: [] }, + bookkeeping: { tier1: T1_UNIVERSAL, tier2: [] }, + converse: { tier1: T1_UNIVERSAL, tier2: [] }, +}; + +export function tier1LensesFor(taskType: TaskType): readonly Tier1Lens[] { + return LENS_REGISTRY[taskType].tier1; +} + +export function tier2LensesFor(taskType: TaskType): readonly Tier2Lens[] { + return LENS_REGISTRY[taskType].tier2; +} + +/** `--critics N` clamps to the number of lenses that actually exist for this task type. + * Without this, `--critics 3` on a `review` spec would spend three calls on one lens. */ +export function criticCountFor(taskType: TaskType, requested: number): number { + return Math.max(0, Math.min(requested, tier2LensesFor(taskType).length)); +} + +/** Is this a task type whose spec must carry a budget? */ +export function isLaunchShaped(taskType: TaskType): boolean { + return LAUNCH_SHAPED.includes(taskType); +} + +/** Guard for reading an untrusted `task_type` off frontmatter. */ +export function isTaskType(v: unknown): v is TaskType { + return typeof v === "string" && (TASK_TYPES as readonly string[]).includes(v); +} diff --git a/packages/amico-run/test/frontmatter.test.ts b/packages/amico-run/test/frontmatter.test.ts new file mode 100644 index 00000000..cdc13872 --- /dev/null +++ b/packages/amico-run/test/frontmatter.test.ts @@ -0,0 +1,63 @@ +// YAML frontmatter reader for the Spec artifact. +// +// amico-validate cannot serve this: it takes --schema (not --kind), TOML-parses anything +// whose extension is not .json, and returns 64 for BOTH usage error and invalid document +// — which would leave `ran` vs `unverified` undecidable for the schema lens. So the verb +// extracts frontmatter itself and validates in-process. +// +// Returns a RESULT, never throws: a malformed spec must be a blocking FINDING (exit 65), +// not a config error (exit 64). +// +// Plan: plan-20260728-104500 Task 7. +import { describe, it, expect } from "vitest"; +import { parseFrontmatter } from "../src/frontmatter.js"; + +describe("parseFrontmatter", () => { + it("extracts the YAML block and ignores the body", () => { + const r = parseFrontmatter("---\ntask_type: plan\nacceptance: [a >= 1]\n---\n\n# Body\nprose\n"); + expect(r.ok).toBe(true); + expect(r.ok && r.data.task_type).toBe("plan"); + expect(r.ok && r.data.acceptance).toEqual(["a >= 1"]); + }); + + it("fails with an actionable message when there is no frontmatter", () => { + const r = parseFrontmatter("# Just a heading\n"); + expect(r.ok).toBe(false); + expect(!r.ok && r.error).toMatch(/frontmatter/i); + }); + + it("fails on malformed YAML rather than throwing", () => { + const r = parseFrontmatter("---\na: [unclosed\n---\n"); + expect(r.ok).toBe(false); + }); + + it("fails when the block is not a mapping", () => { + expect(parseFrontmatter("---\n- just\n- a list\n---\n").ok).toBe(false); + }); + + it("tolerates CRLF", () => { + const r = parseFrontmatter("---\r\ntask_type: plan\r\n---\r\nbody\r\n"); + expect(r.ok && r.data.task_type).toBe("plan"); + }); + + it("requires the opening fence on the FIRST line — a --- later in the body is not frontmatter", () => { + expect(parseFrontmatter("intro\n---\ntask_type: plan\n---\n").ok).toBe(false); + }); + + it("handles the nested structures the spec schema uses (budget, baseline)", () => { + const r = parseFrontmatter( + "---\nbudget:\n max_solves: 8\n tier: free\nbaseline:\n value: 0.968\n source: published\n---\n", + ); + expect(r.ok && r.data.budget).toEqual({ max_solves: 8, tier: "free" }); + expect(r.ok && r.data.baseline).toEqual({ value: 0.968, source: "published" }); + }); + + it("an empty frontmatter block is a mapping-shaped failure, not a crash", () => { + expect(parseFrontmatter("---\n---\nbody").ok).toBe(false); + }); + + it("keeps a body-level `---` out of the frontmatter", () => { + const r = parseFrontmatter("---\ntask_type: plan\n---\nbody\n---\nmore: notparsed\n"); + expect(r.ok && Object.keys(r.data)).toEqual(["task_type"]); + }); +}); diff --git a/packages/amico-run/test/lens_registry.test.ts b/packages/amico-run/test/lens_registry.test.ts new file mode 100644 index 00000000..6a115354 --- /dev/null +++ b/packages/amico-run/test/lens_registry.test.ts @@ -0,0 +1,99 @@ +// The lens registry (spec-20260728 §3.1, §3.5). +// +// Covers advisory A-9: Rev 1 of the spec covered 5 of 10 task types, so half the closed +// enum fell through to a single lens while --critics defaulted to 3. Exhaustiveness is a +// test here, not a convention. +// +// Plan: plan-20260728-104500 Task 8. +import { describe, it, expect } from "vitest"; +import { TASK_TYPES } from "../src/ledger.js"; +import { + LENS_REGISTRY, + LENS_REGISTRY_VERSION, + criticCountFor, + isLaunchShaped, + isTaskType, + tier1LensesFor, + tier2LensesFor, +} from "../src/lens_registry.js"; + +describe("exhaustiveness", () => { + it("has an entry for EVERY value of TASK_TYPES", () => { + for (const t of TASK_TYPES) expect(LENS_REGISTRY[t], `missing registry entry: ${t}`).toBeDefined(); + }); + it("has no entry for a task type that does not exist", () => { + expect(Object.keys(LENS_REGISTRY).sort()).toEqual([...TASK_TYPES].sort()); + }); + it("stamps a version, so a review is attributable to the rules that produced it", () => { + expect(LENS_REGISTRY_VERSION).toMatch(/^\S+$/); + }); +}); + +describe("tier 1", () => { + it("always includes schema and falsifiable — the contract and the criteria", () => { + for (const t of TASK_TYPES) { + expect(tier1LensesFor(t)).toContain("schema"); + expect(tier1LensesFor(t)).toContain("falsifiable"); + } + }); + it("scopes budget/baseline/precedent to launch-shaped work only", () => { + for (const t of TASK_TYPES) { + const has = tier1LensesFor(t).includes("budget"); + expect(has).toBe(isLaunchShaped(t)); + expect(tier1LensesFor(t).includes("baseline")).toBe(isLaunchShaped(t)); + expect(tier1LensesFor(t).includes("precedent")).toBe(isLaunchShaped(t)); + } + }); + it("names exactly the three launch-shaped types", () => { + expect(TASK_TYPES.filter(isLaunchShaped)).toEqual(["author-script", "experiment-sim", "experiment-hw"]); + }); +}); + +describe("tier 2", () => { + it("gives conversational/bookkeeping types NO critics — spending a frontier call there is the bureaucracy trap", () => { + for (const t of ["triage", "bookkeeping", "converse"] as const) expect(tier2LensesFor(t)).toEqual([]); + }); + it("puts hidden-failure AND decomposition in every non-empty set", () => { + for (const t of TASK_TYPES) { + const l = tier2LensesFor(t); + if (l.length > 0) { + expect(l, `hidden-failure missing for ${t}`).toContain("hidden-failure"); + // Rev 1 withheld decomposition from implement-slice, so the specs most exposed to + // bad carving were the one category never reviewed for it. + expect(l, `decomposition missing for ${t}`).toContain("decomposition"); + } + } + }); + it("gives implement-slice the decomposition lens (the Rev-1 defect)", () => { + expect(tier2LensesFor("implement-slice")).toContain("decomposition"); + }); + it("has no duplicate lenses in any set", () => { + for (const t of TASK_TYPES) { + const l = tier2LensesFor(t); + expect(new Set(l).size, `duplicates for ${t}`).toBe(l.length); + } + }); +}); + +describe("criticCountFor — the clamp", () => { + it("clamps a request to the lenses that actually exist", () => { + expect(criticCountFor("review", 3)).toBe(tier2LensesFor("review").length); + expect(criticCountFor("implement-slice", 99)).toBe(tier2LensesFor("implement-slice").length); + }); + it("clamps to ZERO for a tier-1-only type, however many are requested", () => { + expect(criticCountFor("bookkeeping", 3)).toBe(0); + }); + it("honours a request smaller than the lens count", () => { + expect(criticCountFor("implement-slice", 1)).toBe(1); + }); + it("never returns a negative count", () => { + expect(criticCountFor("plan", -5)).toBe(0); + }); +}); + +describe("isTaskType", () => { + it("accepts every real value and rejects anything else", () => { + for (const t of TASK_TYPES) expect(isTaskType(t)).toBe(true); + for (const bad of ["vibes", "", null, undefined, 3, {}]) expect(isTaskType(bad)).toBe(false); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fb4457a1..29c24038 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,6 +16,9 @@ importers: smol-toml: specifier: ^1.3.0 version: 1.6.1 + yaml: + specifier: ^2.9.0 + version: 2.9.0 devDependencies: '@types/node': specifier: ^22.0.0 From b20c223d930179f91d04cd22bfcbf0df24760eb9 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 11:35:09 -0400 Subject: [PATCH 18/27] feat(amico-run): the six tier-1 review lenses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mechanical, free, deterministic, and computed from the SPEC ALONE — which is what makes the free-tier guarantee real rather than a policy. A bad spec never reaches a paid critic because nothing here needs one. schema the whole frontmatter contract, in-process blocking falsifiable metric · comparator · threshold blocking budget key set subset of the SHIPPED WarrantBounds blocking, launch-shaped only baseline a number with a source, or an explicit none blocking, launch-shaped only precedent prior attempts at this work identity advisory, launch-shaped only provenance a declared baseline value names its source advisory Six, not the eight the spec lists. Two are deferred for cause and recorded in the plan: `api` would be a BLOCKING lens that can only report `unverified` until the symbol probe is extracted from lint_api_drift.sh, which by §3.2 would make every Julia-shaped spec unapprovable; `decomposition-size` thresholds a step count that does not exist until the spec is compiled. Three properties the tests pin because each one is a way this could quietly rot: * `not-applicable` is a distinct status from a clean `ran`. Collapsing them is how a blocking lens that could not run reads as a pass. `budget` on an implement-slice spec reports not-applicable; it does not pass. * `precedent` reports `unverified` when the ledger cannot be queried, and not-applicable when there is no work identity to query on — Rev 1 of the spec would have reported a silent zero, which also makes the "block at >= 3 failures" threshold meaningless. * every finding carries a non-empty `remedy`, asserted across lenses and inputs. A finding that cannot say what would fix it is not actionable. `falsifiable` is why acceptance and invariants are separate fields at all: Rev 2 made this lens blocking and then authored six prose sentences in `acceptance`, so either the parser accepted prose — a blocking lens that passes everything — or the spec failed its own gate. `precedent` takes its ledger query as an injected collaborator and never reads the ledger itself, so the whole set stays pure. 31 tests, including that no lens throws on garbage input; typecheck clean. Plan: plan-20260728-104500 Task 9. Co-Authored-By: Claude Opus 5 (1M context) --- packages/amico-run/src/lenses.ts | 233 +++++++++++++++++++++++++ packages/amico-run/test/lenses.test.ts | 160 +++++++++++++++++ 2 files changed, 393 insertions(+) create mode 100644 packages/amico-run/src/lenses.ts create mode 100644 packages/amico-run/test/lenses.test.ts diff --git a/packages/amico-run/src/lenses.ts b/packages/amico-run/src/lenses.ts new file mode 100644 index 00000000..042bcb5b --- /dev/null +++ b/packages/amico-run/src/lenses.ts @@ -0,0 +1,233 @@ +// The tier-1 review lenses (spec-20260728 §3.1): mechanical, free, deterministic, and +// computed from the SPEC ALONE. +// +// That last property is what makes the free-tier guarantee real rather than a policy — a +// bad spec never reaches a paid critic because nothing here needs one. Rev 1 of the spec +// had a `bounds` lens whose predicate was "the budget covers the work THE PLAN will +// need", and the plan does not exist until a frontier planner call; all three critics +// found it independently. That check now lives in `plan compile`, where the plan exists. +// +// ONE SIGNATURE, no exceptions. `status` and `findings` are separate because "ran and +// found nothing" must be distinguishable from "never ran" — collapsing them is how a +// blocking lens that could not run reads as a pass (§3.2). +import { validate, validateBounds } from "@amicode/schema"; +import { isLaunchShaped, isTaskType, type Tier1Lens } from "./lens_registry.js"; + +export type LensStatus = "ran" | "not-applicable" | "skipped" | "unverified"; + +export interface Finding { + lens: string; + severity: "blocking" | "advisory"; + claim: string; + evidence: string; + /** REQUIRED. A finding that cannot say what would fix it is not actionable and is + * dropped — the same standard the warrant refusal holds itself to. */ + remedy: string; + round: number; +} + +export interface LensResult { + status: LensStatus; + findings: Finding[]; +} + +export interface LensDeps { + /** Injected so `precedent` stays pure and testable. A lens must never read the ledger + * itself. */ + queryLedger?: (structureHash: string) => { total: number; verified: number } | undefined; + round?: number; +} + +export type Spec = Record; +export type Lens = (spec: Spec, deps?: LensDeps) => LensResult; + +const ran = (findings: Finding[] = []): LensResult => ({ status: "ran", findings }); +const na = (): LensResult => ({ status: "not-applicable", findings: [] }); + +function finding( + lens: Tier1Lens, + severity: "blocking" | "advisory", + claim: string, + evidence: string, + remedy: string, + round = 1, +): Finding { + return { lens, severity, claim, evidence, remedy, round }; +} + +/** Applies only to launch-shaped work; anything else reports not-applicable rather than a + * vacuous pass. */ +function launchOnly(spec: Spec): boolean { + return isTaskType(spec.task_type) && isLaunchShaped(spec.task_type); +} + +// ── schema ─────────────────────────────────────────────────────────────────────── +/** The whole frontmatter contract, in-process against the registered `spec` kind. */ +export const schema: Lens = (spec, deps) => { + const r = validate(spec, "spec"); + if (r.ok) return ran(); + return ran([ + finding( + "schema", + "blocking", + `the spec's frontmatter does not satisfy the spec schema (${r.errors.length} error${r.errors.length === 1 ? "" : "s"})`, + r.errors.slice(0, 6).join("; "), + "fix the named fields; `budget` is required for launch-shaped task types and forbidden otherwise", + deps?.round ?? 1, + ), + ]); +}; + +// ── falsifiable ────────────────────────────────────────────────────────────────── +/** `metric comparator threshold`. `metric` is an identifier, `comparator` one of + * < <= == >= >, `threshold` a number (optionally a percentage or scientific notation). + * + * This lens is why `acceptance` and `invariants` are separate fields: Rev 2 of the spec + * made this blocking and then authored six prose sentences in `acceptance`, so either the + * parser accepted prose — a blocking lens that passes everything, the defect §3.4 names — + * or the spec failed its own gate. A critic caught it. */ +const ACCEPTANCE = /^[A-Za-z_][A-Za-z0-9_.]*\s*(<=|>=|==|<|>)\s*-?\d+(\.\d+)?([eE][-+]?\d+)?%?$/; + +export const falsifiable: Lens = (spec, deps) => { + const entries = Array.isArray(spec.acceptance) ? (spec.acceptance as unknown[]) : []; + const round = deps?.round ?? 1; + if (entries.length === 0) { + return ran([ + finding( + "falsifiable", + "blocking", + "the spec declares no acceptance criteria", + "`acceptance` is absent or empty", + "add at least one `metric comparator threshold` entry, e.g. `F_rolled >= 0.999`", + round, + ), + ]); + } + const bad = entries.filter((e) => !ACCEPTANCE.test(String(e).trim())); + if (bad.length === 0) return ran(); + return ran([ + finding( + "falsifiable", + "blocking", + `${bad.length} acceptance entr${bad.length === 1 ? "y is" : "ies are"} not machine-checkable`, + bad.slice(0, 3).map((b) => `"${String(b).slice(0, 60)}"`).join(", "), + "rewrite each as `metric comparator threshold` (e.g. `F_rolled >= 0.999`) and move behavioural prose to `invariants`", + round, + ), + ]); +}; + +// ── budget ─────────────────────────────────────────────────────────────────────── +/** The authored budget's key set must be a subset of the SHIPPED WarrantBounds. Checked + * against the schema file via validateBounds, never against a prose restatement — the + * drift that let the long-removed `max_duration` into a spec example. */ +export const budget: Lens = (spec, deps) => { + if (!launchOnly(spec)) return na(); + const round = deps?.round ?? 1; + const b = spec.budget; + if (b === undefined || b === null) { + return ran([ + finding( + "budget", + "blocking", + "launch-shaped work declares no budget, so an approval would have no bounds to grant", + `task_type is "${String(spec.task_type)}"`, + "add `budget` with the bounds this work needs (max_solves, tier, max_size_class, device)", + round, + ), + ]); + } + const r = validateBounds(b); + if (r.ok) return ran(); + return ran([ + finding( + "budget", + "blocking", + "the budget declares a bound the warrant vocabulary does not have", + r.errors.slice(0, 4).join("; "), + "use only max_solves, tier, max_size_class and device — max_duration was removed because nothing estimates wall-clock, so it could never be enforced", + round, + ), + ]); +}; + +// ── baseline ───────────────────────────────────────────────────────────────────── +/** Either a number WITH its source, or an explicit statement that none exists. Blocking + * precisely so "we never checked" cannot pass silently: a bare fidelity is + * uninterpretable without knowing what the free baseline was. */ +export const baseline: Lens = (spec, deps) => { + if (!launchOnly(spec)) return na(); + const round = deps?.round ?? 1; + const b = spec.baseline as Record | undefined; + const hasValue = b !== undefined && b !== null && typeof b.value === "number" && typeof b.source === "string"; + const hasNone = b !== undefined && b !== null && typeof b.none_because === "string" && b.none_because !== ""; + if (hasValue || hasNone) return ran(); + return ran([ + finding( + "baseline", + "blocking", + "the spec states nothing to measure the result against", + b === undefined ? "`baseline` is absent" : `\`baseline\` is ${JSON.stringify(b).slice(0, 80)}`, + "add `baseline: {value, source}`, or `baseline: {none_because: '…'}` if no prior art exists", + round, + ), + ]); +}; + +// ── precedent ──────────────────────────────────────────────────────────────────── +/** Prior attempts at this work identity. Turns the existing ledger into a free critic + * that can say "you attempted this three times; one verified". + * + * Reports NOT-APPLICABLE without a declared work identity, which is a different claim + * from "no prior attempts". Rev 1 of the spec would have reported a silent zero — which + * also makes the "block at >= 3 failures" threshold meaningless. */ +export const precedent: Lens = (spec, deps) => { + if (!launchOnly(spec)) return na(); + const round = deps?.round ?? 1; + const sh = spec.structure_hash; + if (typeof sh !== "string" || sh === "") return na(); + const q = deps?.queryLedger?.(sh); + if (q === undefined) return { status: "unverified", findings: [] }; + if (q.total === 0) return ran(); + return ran([ + finding( + "precedent", + "advisory", + `this work identity has ${q.total} prior attempt${q.total === 1 ? "" : "s"} on record, ${q.verified} verified`, + `structure_hash ${sh.slice(0, 12)}…`, + q.verified === 0 + ? "no prior attempt verified — say what is different this time, or warm-start from the closest one" + : "consider warm-starting from the verified attempt rather than cold-starting", + round, + ), + ]); +}; + +// ── provenance ─────────────────────────────────────────────────────────────────── +/** A declared baseline VALUE must name its source. Concrete and checkable, unlike Rev 1's + * "every cited number names a source", which had no checkable subject. */ +export const provenance: Lens = (spec, deps) => { + const round = deps?.round ?? 1; + const b = spec.baseline as Record | undefined; + if (b === undefined || b === null || typeof b.value !== "number") return ran(); + if (typeof b.source === "string" && b.source !== "") return ran(); + return ran([ + finding( + "provenance", + "advisory", + "the baseline states a number with no source", + `baseline.value = ${String(b.value)}`, + "name where the number comes from — a published reference, a vault note, or a run id", + round, + ), + ]); +}; + +export const LENSES: Record = { + schema, + falsifiable, + budget, + baseline, + precedent, + provenance, +}; diff --git a/packages/amico-run/test/lenses.test.ts b/packages/amico-run/test/lenses.test.ts new file mode 100644 index 00000000..40e36323 --- /dev/null +++ b/packages/amico-run/test/lenses.test.ts @@ -0,0 +1,160 @@ +// The six tier-1 lenses (spec-20260728 §3.1). +// +// Every lens returns {status, findings}: `not-applicable` and a clean `ran` must be +// distinguishable, because collapsing them is how a blocking lens that could not run +// reads as a pass (§3.2, §3.4). +// +// Plan: plan-20260728-104500 Task 9. +import { describe, it, expect } from "vitest"; +import { baseline, budget, falsifiable, precedent, provenance, schema } from "../src/lenses.js"; + +const launch = { + schema_version: "1", + spec_id: "spec-1", + task_type: "experiment-sim", + acceptance: ["F_rolled >= 0.999"], + budget: { max_solves: 8, tier: "free" }, + baseline: { value: 0.968, source: "published blockade-pi protocol" }, +}; +const slice = { schema_version: "1", spec_id: "spec-2", task_type: "implement-slice", acceptance: ["x == 1"] }; +const drop = (o: Record, k: string) => { const c = { ...o }; delete c[k]; return c; }; + +describe("schema", () => { + it("clean on a valid launch-shaped spec", () => { + expect(schema(launch)).toEqual({ status: "ran", findings: [] }); + }); + it("BLOCKS and names the offending fields", () => { + const r = schema(drop(launch, "spec_id")); + expect(r.findings[0].severity).toBe("blocking"); + expect(r.findings[0].evidence).toMatch(/spec_id/); + }); + it("every finding carries a remedy — an unactionable finding is dropped", () => { + for (const f of schema(drop(launch, "spec_id")).findings) expect(f.remedy).not.toBe(""); + }); +}); + +describe("falsifiable", () => { + it("REJECTS prose — the lens the spec itself failed at Rev 2", () => { + const r = falsifiable({ acceptance: ["The system should be fast and correct."] }); + expect(r.status).toBe("ran"); + expect(r.findings).toHaveLength(1); + expect(r.findings[0].severity).toBe("blocking"); + }); + it("accepts metric · comparator · threshold", () => { + expect(falsifiable({ acceptance: ["F_rolled >= 0.999", "wall_s <= 600"] }).findings).toEqual([]); + }); + it("accepts ==, scientific notation and a percentage", () => { + expect(falsifiable({ acceptance: ["leakage <= 1e-4", "coverage == 100%", "n_steps == 3"] }).findings).toEqual([]); + }); + it("accepts a dotted metric name", () => { + expect(falsifiable({ acceptance: ["outcome.fidelity >= 0.99"] }).findings).toEqual([]); + }); + it("BLOCKS an empty or absent acceptance list", () => { + expect(falsifiable({ acceptance: [] }).findings[0].severity).toBe("blocking"); + expect(falsifiable({}).findings[0].severity).toBe("blocking"); + }); + it("reports how many entries failed and quotes them", () => { + const r = falsifiable({ acceptance: ["F >= 0.9", "it should be good", "also fast"] }); + expect(r.findings[0].claim).toMatch(/2 acceptance entries/); + expect(r.findings[0].evidence).toMatch(/should be good/); + }); +}); + +describe("budget", () => { + it("clean on a legal budget", () => { + expect(budget(launch).findings).toEqual([]); + }); + it("rejects max_duration via validateBounds, not a prose restatement", () => { + const r = budget({ ...launch, budget: { max_duration: "30m" } }); + expect(r.findings[0].severity).toBe("blocking"); + expect(r.findings[0].remedy).toMatch(/max_duration/); + }); + it("BLOCKS launch-shaped work with no budget at all", () => { + expect(budget(drop(launch, "budget")).findings[0].severity).toBe("blocking"); + }); + it("is NOT-APPLICABLE for a non-launch-shaped spec (not a vacuous pass)", () => { + expect(budget(slice).status).toBe("not-applicable"); + }); + it("is not-applicable when task_type is missing entirely", () => { + expect(budget({ acceptance: ["x == 1"] }).status).toBe("not-applicable"); + }); +}); + +describe("baseline", () => { + it("accepts a value WITH a source", () => { + expect(baseline(launch).findings).toEqual([]); + }); + it("accepts an explicit none_because", () => { + expect(baseline({ ...launch, baseline: { none_because: "first of its kind" } }).findings).toEqual([]); + }); + it("BLOCKS when absent — 'we never checked' must not pass silently", () => { + expect(baseline(drop(launch, "baseline")).findings[0].severity).toBe("blocking"); + }); + it("BLOCKS a bare value with no source", () => { + expect(baseline({ ...launch, baseline: { value: 0.9 } }).findings[0].severity).toBe("blocking"); + }); + it("is not-applicable for non-launch-shaped work", () => { + expect(baseline(slice).status).toBe("not-applicable"); + }); +}); + +describe("precedent", () => { + it("reports NOT-APPLICABLE — not a zero count — with no work identity", () => { + expect(precedent(launch).status).toBe("not-applicable"); + }); + it("reports prior attempts when a structure_hash resolves", () => { + const r = precedent({ ...launch, structure_hash: "sh1" }, { queryLedger: () => ({ total: 3, verified: 1 }) }); + expect(r.status).toBe("ran"); + expect(r.findings[0].severity).toBe("advisory"); + expect(r.findings[0].claim).toMatch(/3 prior attempts/); + expect(r.findings[0].claim).toMatch(/1 verified/); + }); + it("is clean when the identity resolves but nothing was ever attempted", () => { + const r = precedent({ ...launch, structure_hash: "sh1" }, { queryLedger: () => ({ total: 0, verified: 0 }) }); + expect(r).toEqual({ status: "ran", findings: [] }); + }); + it("advises warm-starting differently when NOTHING prior verified", () => { + const r = precedent({ ...launch, structure_hash: "sh1" }, { queryLedger: () => ({ total: 4, verified: 0 }) }); + expect(r.findings[0].remedy).toMatch(/no prior attempt verified/i); + }); + it("is UNVERIFIED when the ledger cannot be queried — never a silent clean", () => { + const r = precedent({ ...launch, structure_hash: "sh1" }, { queryLedger: () => undefined }); + expect(r.status).toBe("unverified"); + }); + it("never reads the ledger itself — with no injected query it is unverified, not a throw", () => { + expect(precedent({ ...launch, structure_hash: "sh1" }).status).toBe("unverified"); + }); +}); + +describe("provenance", () => { + it("advises when a baseline value carries no source", () => { + const r = provenance({ ...launch, baseline: { value: 0.9 } }); + expect(r.findings[0].severity).toBe("advisory"); + }); + it("clean when the number names its source", () => { + expect(provenance(launch).findings).toEqual([]); + }); + it("clean when there is no numeric baseline to source", () => { + expect(provenance({ ...launch, baseline: { none_because: "novel" } }).findings).toEqual([]); + expect(provenance(slice).findings).toEqual([]); + }); +}); + +describe("cross-lens invariants", () => { + it("no tier-1 lens ever throws on garbage input", () => { + for (const lens of [schema, falsifiable, budget, baseline, precedent, provenance]) { + for (const bad of [{}, { task_type: 3 }, { acceptance: "not a list" }, { budget: 7 }, { baseline: "no" }]) { + expect(() => lens(bad as Record)).not.toThrow(); + } + } + }); + it("every finding from every lens carries a non-empty remedy", () => { + const specs = [{}, drop(launch, "baseline"), { ...launch, budget: { max_duration: "x" } }, { acceptance: ["prose here"] }]; + for (const lens of [schema, falsifiable, budget, baseline, provenance]) { + for (const s of specs) for (const f of lens(s as Record).findings) expect(f.remedy.length).toBeGreaterThan(0); + } + }); + it("carries the round through to every finding", () => { + expect(falsifiable({ acceptance: ["prose"] }, { round: 3 }).findings[0].round).toBe(3); + }); +}); From 8079cdedb5d2e39b4aac38edf1693bbdf1d1cb91 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 11:43:43 -0400 Subject: [PATCH 19/27] =?UTF-8?q?feat(amico-run):=20`amico=20spec=20review?= =?UTF-8?q?`=20=E2=80=94=20the=20tier-1=20review=20runner=20and=20verb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deliberation front half is now callable end to end in mechanical mode: amico spec review [--critics N] [--offline] amico spec validate Path is POSITIONAL, and unknown flags are a usage error rather than ignored — silently accepting `--spec ` would "work" and teach the caller a flag that belongs to the launch path. A test drives exactly that. The verdict rides the JSON PAYLOAD as well as the exit code, because the MCP facade returns only result.json and discards VerbResult.code, and the skill that calls this verb runs in another runtime. Exits: 0 approved | approved-mechanical | degraded (review is not the gate), 64 usage, 65 blocking, 66 exhausted. Three properties that needed code rather than prose, each tested: * ZERO critic spawns when a tier-1 lens blocks. A bad spec never reaches a paid critic — the free-tier guarantee is structural, not policy. The tier-2 spawn is an injected seam so this is testable NOW rather than after the G-2-gated critics land. * THE TERMINATION INVARIANT is enforced: a tier-2 `blocking` finding on any lens but `contradiction` is downgraded to advisory and logged. Asserted on the PERSISTED sidecar, not just the return value, so an implementation that downgrades late (after writing severity: blocking to disk, where a human reads it) still fails. * Findings bodies go to a sidecar keyed on (spec_id, design_hash, round); the row carries a digest. A maximal review writes >15 KB of bodies while the ledger row stays under the 4096-byte PIPE_BUF ceiling. A sidecar that cannot be written THROWS rather than leaving a dangling ref — losing the bodies after paying for them is the failure it exists to prevent. Also fixes a bug the whole unit suite missed and the Task-12 dogfood caught on the first run: `yaml` ships only a CJS build for the `node` export condition, and esbuild's ESM output emits a `__require` shim that THROWS, so the shipped bundle died on its first import while every test passed — vitest transpiles instead of bundling. The seam was tested; the binary was not. All three bins now install a real `require` via createRequire. Dogfooded on the spec that generated it: exit 0, approved-mechanical, 0 blocking, with budget/baseline/precedent correctly not-selected for an implement-slice spec. Reconstructing Rev 2's prose acceptance block reproduces exit 65 naming `falsifiable`, and a `plan` note reviewed as a spec exits 65 naming `schema` — the gate bites in both directions. 781 amico-run, 140 schema, 777 extension; typecheck and build clean. Plan: plan-20260728-104500 Tasks 10-12. Co-Authored-By: Claude Opus 5 (1M context) --- packages/amico-run/esbuild.config.mjs | 20 +- packages/amico-run/src/spec_review.ts | 258 ++++++++++++++++++++ packages/amico-run/src/spec_verb.ts | 138 +++++++++++ packages/amico-run/src/verbs.ts | 20 +- packages/amico-run/test/spec_review.test.ts | 241 ++++++++++++++++++ packages/amico-run/test/spec_verb.test.ts | 100 ++++++++ 6 files changed, 775 insertions(+), 2 deletions(-) create mode 100644 packages/amico-run/src/spec_review.ts create mode 100644 packages/amico-run/src/spec_verb.ts create mode 100644 packages/amico-run/test/spec_review.test.ts create mode 100644 packages/amico-run/test/spec_verb.test.ts diff --git a/packages/amico-run/esbuild.config.mjs b/packages/amico-run/esbuild.config.mjs index 13ec036c..eb808dc1 100644 --- a/packages/amico-run/esbuild.config.mjs +++ b/packages/amico-run/esbuild.config.mjs @@ -14,7 +14,25 @@ const common = { // ESM, not CJS: the package is "type": "module", so node executes the bundle as ESM — // a CJS bundle would die on `require is not defined in ES module scope`. format: "esm", - banner: { js: "#!/usr/bin/env node" }, + // The shebang MUST stay line 1. After it, install a real `require`. + // + // Why: esbuild's ESM output emits a `__require` shim that THROWS + // (`Dynamic require of "process" is not supported`) unless a `require` is already in + // scope. `yaml` — the frontmatter parser — ships only a CJS build for the `node` + // export condition, and that build calls `require("process")` at load, so the bundle + // died on its first import. Every unit test passed throughout, because vitest + // transpiles instead of bundling: the seam was tested, the shipped binary was not. + // Found by actually running the bin (plan Task 12), which is why that step exists. + // + // createRequire is the documented esbuild remedy and it generalises — any future CJS + // dependency now works rather than failing at runtime only. + banner: { + js: [ + "#!/usr/bin/env node", + 'import { createRequire as __amicoCreateRequire } from "node:module";', + "const require = __amicoCreateRequire(import.meta.url);", + ].join("\n"), + }, sourcemap: true, logLevel: "info", }; diff --git a/packages/amico-run/src/spec_review.ts b/packages/amico-run/src/spec_review.ts new file mode 100644 index 00000000..b9c9af82 --- /dev/null +++ b/packages/amico-run/src/spec_review.ts @@ -0,0 +1,258 @@ +// The spec-review runner (spec-20260728 §3): applicability -> lenses -> status -> +// verdict -> findings sidecar -> one `spec_review` ledger record. +// +// This module owns the only I/O in the review path; the lenses are pure and the registry +// is data. Tier 2 (frontier critics) is G-2-gated and NOT built here — but its seam is, +// because the zero-spawn guarantee is testable now and would be untestable later. +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { canonicalJson, designHash } from "@amicode/schema"; +import { appendRecord, type SpecReviewRecord } from "./ledger.js"; +import { parseFrontmatter } from "./frontmatter.js"; +import { LENSES, type Finding, type LensStatus } from "./lenses.js"; +import { + LENS_REGISTRY_VERSION, + criticCountFor, + isTaskType, + tier1LensesFor, + tier2LensesFor, + type Tier1Lens, +} from "./lens_registry.js"; + +export const ROUND_BUDGET = 3; + +export type ReviewVerdict = "approved" | "approved-mechanical" | "degraded" | "blocking" | "exhausted"; + +export interface LensStatusEntry { + lens: string; + status: LensStatus; + reason?: string; +} + +export interface ReviewResult { + review_verdict: ReviewVerdict; + exit_code: 0 | 64 | 65 | 66; + spec_id: string; + design_hash: string; + rounds: number; + lens_status: LensStatusEntry[]; + critics: Array<{ model: string; variant: string }>; + findings: Finding[]; + findings_count: number; + blocking_count: number; + findings_sha256: string; + findings_ref: string; + critic_spawns: number; +} + +/** The tier-2 seam. Not implemented in this slice (G-2), but injected so the + * ZERO-SPAWN-on-tier-1-blocking guarantee is a test today rather than a promise. */ +export type SpawnCritic = (lens: string) => { model: string; variant: string; findings: Finding[] } | undefined; + +export interface ReviewOptions { + round?: number; + critics?: number; + offline?: boolean; + spawnCritic?: SpawnCritic; + queryLedger?: (structureHash: string) => { total: number; verified: number } | undefined; + now?: () => string; + /** Skip the ledger append (tests that only care about the computation). */ + append?: boolean; +} + +/** Blocking tier-1 lenses. A blocking lens that could not run yields `unverified`, and a + * review with any unverified BLOCKING lens must not report `approved` (§3.2) — Rev 1 + * treated "the checker exited 0" as a pass, which is exactly how `api` would have passed + * everything silently. */ +const BLOCKING_LENSES: readonly Tier1Lens[] = ["schema", "falsifiable", "budget", "baseline"]; + +const sha256hexOf = (s: string): string => createHash("sha256").update(s, "utf8").digest("hex"); + +/** `/.review/--r.json`. + * Keyed on spec_id AND round, not design_hash alone: a prose-only revision (what most + * advisories ask for) leaves design_hash unchanged, so keying on it alone would have + * round 2 silently overwrite round 1's bodies while round 1's recorded sha still pointed + * at the file. */ +export function findingsRefFor(specPath: string, specId: string, hash: string, round: number): string { + return join(dirname(specPath), ".review", `${specId}-${hash.slice(0, 16)}-r${round}.json`); +} + +export function reviewSpec(specPath: string, raw: string, opts: ReviewOptions = {}): ReviewResult { + const round = opts.round ?? 1; + const nowIso = (opts.now ?? (() => new Date().toISOString()))(); + const lens_status: LensStatusEntry[] = []; + const findings: Finding[] = []; + let critic_spawns = 0; + const critics: Array<{ model: string; variant: string }> = []; + + // ── parse ── + const fm = parseFrontmatter(raw); + if (!fm.ok) { + // A malformed spec is a blocking FINDING, not a config error: it tells the author what + // to fix rather than implying they invoked the tool wrong. + const f: Finding = { + lens: "schema", + severity: "blocking", + claim: "the spec's frontmatter could not be read", + evidence: fm.error, + remedy: "open the note with a `---` fence on line 1 and a mapping of fields inside", + round, + }; + return finish(specPath, "unreadable-spec", "0".repeat(64), round, [{ lens: "schema", status: "ran" }], [], [f], 0, opts, nowIso); + } + const spec = fm.data; + const spec_id = typeof spec.spec_id === "string" && spec.spec_id !== "" ? spec.spec_id : "unidentified-spec"; + const design_hash = designHash(spec); + + // ── tier 1 ── + // Applicability comes from the registry; an unknown/absent task_type still gets the + // universal lenses, because a spec that cannot name its own type is exactly the case the + // schema lens must report on. + const taskType = isTaskType(spec.task_type) ? spec.task_type : "converse"; + for (const name of tier1LensesFor(taskType)) { + const r = LENSES[name](spec, { queryLedger: opts.queryLedger, round }); + lens_status.push({ lens: name, status: r.status }); + findings.push(...r.findings); + } + + const blocking = findings.filter((f) => f.severity === "blocking"); + const unverifiedBlocking = lens_status.filter( + (s) => s.status === "unverified" && (BLOCKING_LENSES as readonly string[]).includes(s.lens), + ); + + // ── the gate on tier 2: a bad spec NEVER reaches a paid critic ── + if (blocking.length > 0 || unverifiedBlocking.length > 0) { + for (const u of unverifiedBlocking) { + findings.push({ + lens: u.lens, + severity: "blocking", + claim: `the blocking lens \`${u.lens}\` could not be verified`, + evidence: u.reason ?? "the lens reported `unverified`", + remedy: "make the lens's input available (or scope the lens out for this task type) — an unverified blocking lens is not a pass", + round, + }); + } + return finish(specPath, spec_id, design_hash, round, lens_status, [], findings, 0, opts, nowIso); + } + + // ── tier 2 ── + const wanted = criticCountFor(taskType, opts.critics ?? 3); + const lenses = tier2LensesFor(taskType).slice(0, wanted); + let degraded = false; + if (!opts.offline && opts.spawnCritic && lenses.length > 0) { + for (const lens of lenses) { + critic_spawns++; + const out = opts.spawnCritic(lens); + if (!out) { + // Timeout, unparseable output, spawn failure: `skipped`, never counted as clean. + lens_status.push({ lens, status: "skipped", reason: "critic did not return usable output" }); + degraded = true; + continue; + } + lens_status.push({ lens, status: "ran" }); + critics.push({ model: out.model, variant: out.variant }); + // THE TERMINATION INVARIANT: a tier-2 critic may not emit `blocking` except for + // `contradiction`. Anything else is DOWNGRADED to advisory and logged — this is what + // guarantees the loop converges, so it is enforced rather than requested. + for (const f of out.findings) { + if (f.severity === "blocking" && f.lens !== "contradiction") { + process.stderr.write( + `amico spec review: downgraded a non-contradiction blocking finding from lens "${f.lens}" to advisory\n`, + ); + findings.push({ ...f, severity: "advisory" }); + } else { + findings.push(f); + } + } + } + } else if (lenses.length > 0) { + // No critic mechanism available: tier 1 only, and the record says so. + degraded = false; + } + + const post = findings.filter((f) => f.severity === "blocking"); + if (post.length > 0) { + return finish(specPath, spec_id, design_hash, round, lens_status, critics, findings, critic_spawns, opts, nowIso); + } + const verdict: ReviewVerdict = lenses.length === 0 || critics.length === 0 + ? "approved-mechanical" + : degraded + ? "degraded" + : "approved"; + return finish(specPath, spec_id, design_hash, round, lens_status, critics, findings, critic_spawns, opts, nowIso, verdict); +} + +function finish( + specPath: string, + spec_id: string, + design_hash: string, + round: number, + lens_status: LensStatusEntry[], + critics: Array<{ model: string; variant: string }>, + findings: Finding[], + critic_spawns: number, + opts: ReviewOptions, + nowIso: string, + forced?: ReviewVerdict, +): ReviewResult { + const blocking_count = findings.filter((f) => f.severity === "blocking").length; + let review_verdict: ReviewVerdict = + forced ?? (blocking_count > 0 ? (round >= ROUND_BUDGET ? "exhausted" : "blocking") : "approved-mechanical"); + const exit_code: 0 | 64 | 65 | 66 = + review_verdict === "blocking" ? 65 : review_verdict === "exhausted" ? 66 : 0; + + // Findings BODIES go to a sidecar. The record carries only a digest, because a 3-round + // 3-critic review's prose exceeds PIPE_BUF and appendRecord throws above it — AFTER the + // model spend, losing the whole review. + const findings_sha256 = sha256hexOf(canonicalJson(findings as never)); + const findings_ref = findingsRefFor(specPath, spec_id, design_hash, round); + try { + mkdirSync(dirname(findings_ref), { recursive: true }); + writeFileSync(findings_ref, JSON.stringify(findings, null, 2)); + } catch (e) { + // LOUD, not a dangling ref: losing the bodies after spending on them is the failure + // the sidecar exists to prevent, so it must not read as a clean review. + throw new Error(`could not write the findings sidecar at ${findings_ref}: ${(e as Error).message}`); + } + + const rec: SpecReviewRecord = { + type: "spec_review", + ts: nowIso, + spec_id, + design_hash, + rounds: Math.min(Math.max(round, 1), ROUND_BUDGET), + review_verdict, + lens_registry_version: LENS_REGISTRY_VERSION, + lens_status, + critics, + findings_count: findings.length, + blocking_count, + findings_sha256, + findings_ref, + source: "user", + }; + if (opts.append !== false) appendRecord(rec); + + return { + review_verdict, + exit_code, + spec_id, + design_hash, + rounds: rec.rounds, + lens_status, + critics, + findings, + findings_count: findings.length, + blocking_count, + findings_sha256, + findings_ref, + critic_spawns, + }; +} + +/** Read-and-review, for the verb. Kept separate so `reviewSpec` stays testable on a string. */ +export function reviewSpecFile(specPath: string, readFile: (p: string) => string, opts: ReviewOptions = {}): ReviewResult { + if (!existsSync(specPath)) throw new Error(`spec not found: ${specPath}`); + return reviewSpec(specPath, readFile(specPath), opts); +} diff --git a/packages/amico-run/src/spec_verb.ts b/packages/amico-run/src/spec_verb.ts new file mode 100644 index 00000000..c07ffddb --- /dev/null +++ b/packages/amico-run/src/spec_verb.ts @@ -0,0 +1,138 @@ +// packages/amico-run/src/spec_verb.ts — the `amico spec` verb (spec-20260728 §3). +// +// amico spec review [--critics N] [--offline] [--json] +// amico spec validate +// +// The path is POSITIONAL. `--spec` is taken by the launch path (`amico run --spec +// `, `amico resolve`), and reusing that flag for a different artifact +// invites the exact confusion the `design_hash`-not-`spec_hash` rename avoids. +// +// EXIT CODES, and why the verdict is ALSO a payload field: the MCP facade returns only +// `result.json` and discards `VerbResult.code`, and the deliberation skill that calls this +// verb runs in another runtime. An exit code alone would be invisible to it. +// +// 0 approved | approved-mechanical | degraded (review is not the gate) +// 64 usage / config — the established ConfigError class +// 65 blocking findings: revise and re-run +// 66 round budget exhausted: a human decision point +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { validate } from "@amicode/schema"; +import { parseFrontmatter } from "./frontmatter.js"; +import { reviewSpec, type ReviewOptions } from "./spec_review.js"; +import type { VerbResult } from "./verbs.js"; + +const USAGE = "amico spec review [--critics N] [--offline] [--json] | amico spec validate "; + +function usageError(error: string): VerbResult { + return { json: { verb: "spec", ok: false, error, usage: USAGE }, code: 64 }; +} + +function flagValue(argv: string[], name: string): string | undefined { + const i = argv.indexOf(name); + if (i >= 0 && i + 1 < argv.length) return argv[i + 1]; + const eq = argv.find((a) => a.startsWith(`${name}=`)); + return eq ? eq.slice(name.length + 1) : undefined; +} + +/** Flags this verb accepts. Anything else is a usage error rather than being ignored: + * silently accepting `--spec ` would "work" and teach the caller a flag that + * belongs to the launch path, which is worse than refusing it. Typos fail loudly too. */ +const KNOWN_FLAGS = new Set(["--critics", "--offline", "--json"]); +const VALUED_FLAGS = new Set(["--critics"]); + +/** First non-flag argument, or an error naming the offending flag. Flags may precede or + * follow the path. */ +function positional(argv: string[]): { path: string } | { error: string } { + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith("-")) { + const name = a.includes("=") ? a.slice(0, a.indexOf("=")) : a; + if (!KNOWN_FLAGS.has(name)) return { error: `unknown flag ${name}` }; + if (VALUED_FLAGS.has(name) && !a.includes("=")) i++; // consume its value + continue; + } + return { path: a }; + } + return { error: "a spec path is required (positional, not --spec)" }; +} + +function review(argv: string[], ctx: SpecVerbCtx): VerbResult { + const pos = positional(argv); + if ("error" in pos) return usageError(pos.error); + const abs = resolve(pos.path); + if (!existsSync(abs)) return usageError(`spec not found: ${abs}`); + + const criticsRaw = flagValue(argv, "--critics"); + let critics: number | undefined; + if (criticsRaw !== undefined) { + critics = Number(criticsRaw); + if (!Number.isInteger(critics) || critics < 0) return usageError(`--critics must be a non-negative integer, got "${criticsRaw}"`); + } + + let r; + try { + r = reviewSpec(abs, (ctx.readFile ?? readFileSync)(abs, "utf8") as string, { + critics, + offline: argv.includes("--offline"), + spawnCritic: ctx.spawnCritic, + queryLedger: ctx.queryLedger, + round: ctx.round, + }); + } catch (e) { + // A sidecar that cannot be written, or an oversize record: loud, not a clean review. + return { json: { verb: "spec", subcommand: "review", ok: false, error: (e as Error).message }, code: 64 }; + } + + return { + json: { + verb: "spec", + subcommand: "review", + ok: r.exit_code === 0, + // The verdict rides the PAYLOAD as well as the exit code — the MCP facade discards + // the code, and the skill that calls this verb lives in another runtime. + review_verdict: r.review_verdict, + exit_code: r.exit_code, + spec_id: r.spec_id, + design_hash: r.design_hash, + rounds: r.rounds, + lens_status: r.lens_status, + critics: r.critics, + findings_count: r.findings_count, + blocking_count: r.blocking_count, + findings_ref: r.findings_ref, + // Blocking findings inline: the refusal must be actionable, so the caller gets the + // shape of what to fix without a second read. + blocking: r.findings.filter((f) => f.severity === "blocking"), + }, + code: r.exit_code, + }; +} + +/** `amico spec validate` — the frontmatter contract alone, for `lint_vault_contract.sh` + * to shell rather than reimplementing the check in bash. */ +function validateOnly(argv: string[], ctx: SpecVerbCtx): VerbResult { + const pos = positional(argv); + if ("error" in pos) return usageError(pos.error); + const abs = resolve(pos.path); + if (!existsSync(abs)) return usageError(`spec not found: ${abs}`); + const fm = parseFrontmatter((ctx.readFile ?? readFileSync)(abs, "utf8") as string); + if (!fm.ok) return { json: { verb: "spec", subcommand: "validate", ok: false, errors: [fm.error] }, code: 65 }; + const v = validate(fm.data, "spec"); + return { json: { verb: "spec", subcommand: "validate", ok: v.ok, errors: v.errors }, code: v.ok ? 0 : 65 }; +} + +export interface SpecVerbCtx { + readFile?: (p: string, enc: string) => string; + spawnCritic?: ReviewOptions["spawnCritic"]; + queryLedger?: ReviewOptions["queryLedger"]; + round?: number; +} + +export function specVerb(argv: string[], ctx: SpecVerbCtx = {}): VerbResult { + const sub = argv[0]; + const rest = argv.slice(1); + if (sub === "review") return review(rest, ctx); + if (sub === "validate") return validateOnly(rest, ctx); + return usageError(`unknown subcommand ${sub ? `"${sub}"` : "(none)"}`); +} diff --git a/packages/amico-run/src/verbs.ts b/packages/amico-run/src/verbs.ts index 2216b1a1..8338949d 100644 --- a/packages/amico-run/src/verbs.ts +++ b/packages/amico-run/src/verbs.ts @@ -20,6 +20,7 @@ import { noteVerb } from "./note_verb.js"; import { ledgerVerb } from "./ledger_verb.js"; import { profileVerb } from "./profile_verb.js"; import { fleetVerb } from "./fleet_verb.js"; +import { specVerb } from "./spec_verb.js"; export interface VerbResult { json: unknown; // structured result (stdout as JSON for the CLI; tool content for MCP) @@ -142,4 +143,21 @@ const fleet: Verb = { run: fleetVerb, }; -export const SPINE_VERBS: Verb[] = [catalog, vault, device, note, ledger, profile, fleet]; +// spec — the deliberation front half: adversarially review a Spec before it compiles to a +// plan. REAL as of the deliberation slice: tier-1 (mechanical) lenses, the lens registry, +// design_hash, the findings sidecar and the spec_review record. Tier-2 frontier critics +// ride the injected spawn seam and are G-2-gated. +// +// NOTE (advisory A-4): registering here also publishes `amico_spec` as an MCP tool, since +// listMcpTools() maps this registry. That does not weaken D-2 — the point of the CLI-verb +// design is that the critic is not a subagent the REVIEWED agent spawns, so no agent ever +// holds `task`. An agent invoking the verb is the intended path. +const spec: Verb = { + name: "spec", + summary: "adversarially review a Spec (mechanical lenses + judgment critics) / validate its frontmatter", + generalizes: "the brainstorming skill's own reviewer subagent (which it now calls instead of carrying)", + slice: "deliberation front half (D1)", + run: (args) => specVerb(args), +}; + +export const SPINE_VERBS: Verb[] = [catalog, vault, device, note, ledger, profile, fleet, spec]; diff --git a/packages/amico-run/test/spec_review.test.ts b/packages/amico-run/test/spec_review.test.ts new file mode 100644 index 00000000..e703b46a --- /dev/null +++ b/packages/amico-run/test/spec_review.test.ts @@ -0,0 +1,241 @@ +// The spec-review runner (spec-20260728 §3). +// Plan: plan-20260728-104500 Task 10. +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, readFileSync, rmSync, statSync, existsSync, chmodSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { canonicalJson } from "@amicode/schema"; +import { readRecords, type SpecReviewRecord } from "../src/ledger.js"; +import { reviewSpec } from "../src/spec_review.js"; +import type { Finding } from "../src/lenses.js"; + +const fm = (o: Record) => + "---\n" + + Object.entries(o) + .map(([k, v]) => `${k}: ${typeof v === "object" ? JSON.stringify(v) : String(v)}`) + .join("\n") + + "\n---\n\nbody\n"; + +const LAUNCH = { + schema_version: '"1"', spec_id: "spec-launch", task_type: "experiment-sim", + acceptance: ["F_rolled >= 0.999"], budget: { max_solves: 8, tier: "free" }, + baseline: { value: 0.968, source: "published" }, +}; +const SLICE = { schema_version: '"1"', spec_id: "spec-slice", task_type: "implement-slice", acceptance: ["x == 1"] }; + +const record = () => readRecords().filter((r): r is SpecReviewRecord => r.type === "spec_review")[0]; + +describe("reviewSpec", () => { + let dir: string; + let specPath: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "spec-review-")); + specPath = join(dir, "spec.md"); + process.env.AMICO_LEDGER = join(dir, "runs.jsonl"); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + delete process.env.AMICO_LEDGER; + }); + + describe("verdicts", () => { + it("a clean spec with NO critic mechanism yields approved-mechanical, not approved", () => { + const r = reviewSpec(specPath, fm(SLICE)); + expect(r.review_verdict).toBe("approved-mechanical"); + expect(r.exit_code).toBe(0); + expect(r.critics).toEqual([]); + }); + + it("a blocking tier-1 finding yields review_verdict=blocking and exit 65", () => { + const r = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["it should be good"] })); + expect(r.review_verdict).toBe("blocking"); + expect(r.exit_code).toBe(65); + expect(r.findings.some((f) => f.lens === "falsifiable" && f.severity === "blocking")).toBe(true); + }); + + it("blocking at the LAST round is `exhausted` (66), a human decision point", () => { + const r = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { round: 3 }); + expect(r.review_verdict).toBe("exhausted"); + expect(r.exit_code).toBe(66); + }); + + it("an UNVERIFIED blocking lens cannot yield approved", () => { + // `precedent` is advisory, so force the case through a blocking lens: an + // unreadable frontmatter is the schema lens failing to run at all. + const r = reviewSpec(specPath, "no frontmatter here\n"); + expect(r.review_verdict).not.toBe("approved-mechanical"); + expect(r.exit_code).toBe(65); + expect(r.findings[0].lens).toBe("schema"); + }); + + it("a clean spec WITH critics that all run yields approved", () => { + const r = reviewSpec(specPath, fm(SLICE), { + spawnCritic: () => ({ model: "anthropic/claude-opus-5", variant: "high", findings: [] }), + }); + expect(r.review_verdict).toBe("approved"); + expect(r.critics.length).toBeGreaterThan(0); + }); + + it("a critic that returns nothing (timeout/unparseable) yields DEGRADED, never approved", () => { + let n = 0; + const r = reviewSpec(specPath, fm(SLICE), { + spawnCritic: () => (n++ === 0 ? { model: "anthropic/claude-opus-5", variant: "high", findings: [] } : undefined), + }); + expect(r.review_verdict).toBe("degraded"); + expect(r.lens_status.some((s) => s.status === "skipped")).toBe(true); + }); + + it("--offline runs tier 1 only and stamps critics: []", () => { + const r = reviewSpec(specPath, fm(SLICE), { + offline: true, + spawnCritic: () => ({ model: "anthropic/claude-opus-5", variant: "high", findings: [] }), + }); + expect(r.review_verdict).toBe("approved-mechanical"); + expect(r.critic_spawns).toBe(0); + expect(record().critics).toEqual([]); + }); + }); + + describe("the free-tier guarantee", () => { + it("spawns ZERO critics when a tier-1 lens blocks", () => { + let spawns = 0; + const r = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { + spawnCritic: () => { spawns++; return { model: "anthropic/claude-opus-5", variant: "high", findings: [] }; }, + }); + expect(r.exit_code).toBe(65); + expect(spawns).toBe(0); // a bad spec never reaches a paid critic + expect(r.critic_spawns).toBe(0); + }); + + it("spawns ZERO critics for a tier-1-only task type however many are requested", () => { + let spawns = 0; + reviewSpec(specPath, fm({ ...SLICE, task_type: "bookkeeping" }), { + critics: 3, + spawnCritic: () => { spawns++; return { model: "anthropic/claude-opus-5", variant: "high", findings: [] }; }, + }); + expect(spawns).toBe(0); + }); + + it("clamps --critics to the lenses that exist for this task type", () => { + let spawns = 0; + reviewSpec(specPath, fm(SLICE), { + critics: 99, + spawnCritic: () => { spawns++; return { model: "anthropic/claude-opus-5", variant: "high", findings: [] }; }, + }); + expect(spawns).toBe(4); // implement-slice has 4 tier-2 lenses + }); + }); + + describe("THE TERMINATION INVARIANT", () => { + const blockingFinding = (lens: string): Finding => ({ + lens, severity: "blocking", claim: "c", evidence: "e", remedy: "r", round: 1, + }); + + it("a tier-2 `blocking` finding on any lens but `contradiction` is DOWNGRADED to advisory", () => { + const r = reviewSpec(specPath, fm(SLICE), { + spawnCritic: (lens) => ({ + model: "anthropic/claude-opus-5", variant: "high", + findings: lens === "hidden-failure" ? [blockingFinding("hidden-failure")] : [], + }), + }); + // Persisted as ADVISORY, and the review is not blocked by it. + expect(r.findings.filter((f) => f.severity === "blocking")).toEqual([]); + expect(r.blocking_count).toBe(0); + expect(r.review_verdict).toBe("approved"); + const persisted: Finding[] = JSON.parse(readFileSync(r.findings_ref, "utf8")); + expect(persisted.find((f) => f.lens === "hidden-failure")?.severity).toBe("advisory"); + expect(record().blocking_count).toBe(0); + }); + + it("`contradiction` is the ONE tier-2 finding that may block", () => { + const r = reviewSpec(specPath, fm(SLICE), { + spawnCritic: (lens) => ({ + model: "anthropic/claude-opus-5", variant: "high", + findings: lens === "hidden-failure" ? [blockingFinding("contradiction")] : [], + }), + }); + expect(r.blocking_count).toBe(1); + expect(r.exit_code).toBe(65); + }); + }); + + describe("the findings sidecar", () => { + it("writes the bodies, and findings_sha256 is over the CANONICAL array", () => { + const r = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] })); + expect(existsSync(r.findings_ref)).toBe(true); + const bodies: Finding[] = JSON.parse(readFileSync(r.findings_ref, "utf8")); + expect(bodies).toHaveLength(r.findings_count); + expect(createHash("sha256").update(canonicalJson(bodies as never), "utf8").digest("hex")).toBe(r.findings_sha256); + }); + + it("keys the sidecar on spec_id AND round, so round 2 cannot overwrite round 1", () => { + const a = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { round: 1 }); + const b = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { round: 2 }); + expect(a.findings_ref).not.toBe(b.findings_ref); + expect(existsSync(a.findings_ref)).toBe(true); + }); + + it("a sidecar that cannot be written fails LOUDLY, never a dangling ref", () => { + chmodSync(dir, 0o500); // read+execute only: no new subdirectory + try { + expect(() => reviewSpec(specPath, fm(SLICE))).toThrow(/sidecar/i); + } finally { + chmodSync(dir, 0o700); + } + }); + + it("the record stays under PIPE_BUF with a maximal review", () => { + const many = Array.from({ length: 40 }, (_, i) => `metric${i} >= ${i}`); + const r = reviewSpec(specPath, fm({ ...SLICE, acceptance: many }), { + spawnCritic: () => ({ + model: "anthropic/claude-opus-5", variant: "high", + findings: Array.from({ length: 9 }, (_, i) => ({ + lens: "hidden-failure", severity: "advisory" as const, + claim: "c".repeat(200), evidence: "e".repeat(200), remedy: "r".repeat(200), round: 1 + (i % 3), + })), + }), + }); + const line = readFileSync(process.env.AMICO_LEDGER!, "utf8").split("\n").filter(Boolean).pop()!; + expect(Buffer.byteLength(line, "utf8")).toBeLessThanOrEqual(4096); + // 3 critics (the default) x 9 findings, each ~600 bytes of prose: >15 KB of bodies + // in the sidecar while the ROW stays under the 4096-byte ceiling. That gap is the + // whole reason the sidecar exists — the row used to carry every finding and would + // have thrown on append, after the model spend. + expect(r.findings_count).toBe(27); + expect(Buffer.byteLength(readFileSync(r.findings_ref, "utf8"), "utf8")).toBeGreaterThan(15_000); + }); + }); + + describe("the ledger record", () => { + it("appends exactly one spec_review, stamped with the registry version", () => { + reviewSpec(specPath, fm(SLICE)); + const recs = readRecords().filter((r) => r.type === "spec_review"); + expect(recs).toHaveLength(1); + expect(record().lens_registry_version).toMatch(/^\S+$/); + expect(record().design_hash).toMatch(/^[0-9a-f]{64}$/); + }); + + it("records per-lens status including not-applicable", () => { + reviewSpec(specPath, fm(SLICE)); + const budget = record().lens_status.find((s) => s.lens === "budget"); + // implement-slice is not launch-shaped, so budget is scoped out — and that is + // recorded as not-applicable rather than absent or clean. + expect(budget).toBeUndefined(); // not even selected for this task type + expect(record().lens_status.map((s) => s.lens)).toContain("schema"); + }); + + it("a launch-shaped spec records budget/baseline/precedent statuses", () => { + reviewSpec(specPath, fm(LAUNCH)); + const names = record().lens_status.map((s) => s.lens); + expect(names).toContain("budget"); + expect(names).toContain("baseline"); + expect(record().lens_status.find((s) => s.lens === "precedent")?.status).toBe("not-applicable"); + }); + + it("append can be suppressed for pure computation", () => { + reviewSpec(specPath, fm(SLICE), { append: false }); + expect(readRecords().filter((r) => r.type === "spec_review")).toHaveLength(0); + }); + }); +}); diff --git a/packages/amico-run/test/spec_verb.test.ts b/packages/amico-run/test/spec_verb.test.ts new file mode 100644 index 00000000..ae6c8911 --- /dev/null +++ b/packages/amico-run/test/spec_verb.test.ts @@ -0,0 +1,100 @@ +// The `amico spec` verb (spec-20260728 §3). +// Plan: plan-20260728-104500 Task 11. +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { specVerb } from "../src/spec_verb.js"; +import { SPINE_VERBS } from "../src/verbs.js"; + +const SLICE = `--- +schema_version: "1" +spec_id: spec-slice +task_type: implement-slice +acceptance: ["x == 1"] +--- + +body +`; +const PROSE = SLICE.replace('["x == 1"]', '["it should be good"]'); + +describe("amico spec", () => { + let dir: string; + let path: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "spec-verb-")); + path = join(dir, "spec.md"); + process.env.AMICO_LEDGER = join(dir, "runs.jsonl"); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + delete process.env.AMICO_LEDGER; + }); + + const json = (r: { json: unknown }) => r.json as Record; + + it("exit 0 + approved-mechanical on a clean spec", () => { + writeFileSync(path, SLICE); + const r = specVerb(["review", path]); + expect(r.code).toBe(0); + expect(json(r).review_verdict).toBe("approved-mechanical"); + }); + + it("exit 65 + blocking, with the findings INLINE so the refusal is actionable", () => { + writeFileSync(path, PROSE); + const r = specVerb(["review", path]); + expect(r.code).toBe(65); + expect(json(r).review_verdict).toBe("blocking"); + const blocking = json(r).blocking as Array>; + expect(blocking.length).toBeGreaterThan(0); + expect(blocking[0].remedy).toBeTruthy(); + }); + + it("the verdict is in the PAYLOAD too, because the MCP facade discards exit codes", () => { + writeFileSync(path, SLICE); + const r = specVerb(["review", path]); + expect(json(r)).toHaveProperty("review_verdict"); + expect(json(r)).toHaveProperty("exit_code", 0); + }); + + it("exit 64 on usage errors: no path, unknown subcommand, bad --critics", () => { + expect(specVerb(["review"]).code).toBe(64); + expect(specVerb(["frobnicate"]).code).toBe(64); + expect(specVerb([]).code).toBe(64); + writeFileSync(path, SLICE); + expect(specVerb(["review", path, "--critics", "-2"]).code).toBe(64); + expect(specVerb(["review", path, "--critics", "many"]).code).toBe(64); + }); + + it("exit 64 when the spec does not exist", () => { + expect(specVerb(["review", join(dir, "nope.md")]).code).toBe(64); + }); + + it("accepts flags before the positional path", () => { + writeFileSync(path, SLICE); + expect(specVerb(["review", "--offline", path]).code).toBe(0); + }); + + it("does NOT take --spec (that flag belongs to the launch path)", () => { + writeFileSync(path, SLICE); + expect(specVerb(["review", "--spec", path]).code).toBe(64); + }); + + it("`validate` checks the frontmatter contract alone", () => { + writeFileSync(path, SLICE); + expect(specVerb(["validate", path]).code).toBe(0); + writeFileSync(path, SLICE.replace("spec_id: spec-slice\n", "")); + const r = specVerb(["validate", path]); + expect(r.code).toBe(65); + expect((json(r).errors as string[]).join(" ")).toMatch(/spec_id/); + }); + + it("is registered in SPINE_VERBS with the fields Verb requires", () => { + const v = SPINE_VERBS.find((x) => x.name === "spec"); + expect(v).toBeDefined(); + expect(v!.summary).toBeTruthy(); + expect(v!.generalizes).toBeTruthy(); + expect(v!.slice).toBeTruthy(); + expect(v!.stub).toBeUndefined(); // real body, not a seam + }); +}); From dabcbc22b9d448e594ceb6463650f4f694241948 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 12:13:36 -0400 Subject: [PATCH 20/27] feat(schema): the `bypassed` verdict + the plan step's demand fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four amendments the back half needs, landed together because each is the same defect class the deliberation spec keeps reproducing: a check that reads a field the schema does not carry. 1. `verdict` gains `bypassed`. `skipped` had NO producer for three consecutive revisions. §4.4 requires "optional: true AND a terminal row marks it bypassed", but the enum was agree|disagree|exhausted and no bypass carrier existed — while plan.schema.json asserted `optional` was "the SOLE producer", which is what made the gap read as closed. `optional: true` is a PERMISSION; this row is the EVENT. Representable now, emitted at G-1b — the discipline `step_id` already got. 2. The plan step declares `model`, `variant`, `task_type`, `permissions.device`, requiring `model` and `task_type`. It previously declared only id|needs|gates| optional, so a planner omitting them yielded an EMPTY demand set and every §4.2 budget refusal passed silently. That is §0.1's inert max_solves counter, fifth instance. An undeterminable demand is now a loud refusal, never "unbounded". 3. `DEVICE_ORDER` is exported from warrant.ts. plan_compile joins step device demands under the same order the launch gate compares with; restating {none:0,ro:1,rw:2} in a second module would let the two drift. 4. A verdict value outside the enum now has a test, so a typo cannot mint a state. Also corrects the spec (§4.2, §4.4, §11) and closes G-2 — frontier for critics, on asymmetric risk: the only tier-2 finding that may block is `contradiction`, which is the one judgment here needing the most capability. Found by three adversarial critics on the back-half plan, one lens each; they found three different blocking defect sets and converged on this one independently. Co-Authored-By: Claude Opus 5 --- packages/amico-run/src/ledger.ts | 6 ++-- packages/amico-run/src/warrant.ts | 5 +++- .../test/ledger_step_identity.test.ts | 18 +++++++++++ .../schema/schemas/ledger-record.schema.json | 2 +- packages/schema/schemas/plan.schema.json | 16 ++++++++-- packages/schema/test/spec_plan_kinds.test.ts | 30 +++++++++++++++++-- 6 files changed, 67 insertions(+), 10 deletions(-) diff --git a/packages/amico-run/src/ledger.ts b/packages/amico-run/src/ledger.ts index 506c8cf2..0467d290 100644 --- a/packages/amico-run/src/ledger.ts +++ b/packages/amico-run/src/ledger.ts @@ -85,8 +85,10 @@ export interface VerdictRecord { problem_hash?: string; structure_hash?: string; /** `exhausted` = per-step gate exhaustion. The fleet registry's `blocked` is - * session-scoped, so it cannot carry a per-step outcome. */ - verdict: "agree" | "disagree" | "exhausted"; + * session-scoped, so it cannot carry a per-step outcome. `bypassed` is the terminal row + * for an `optional: true` step the walk did not need — the second half of the `skipped` + * producer, since the flag alone is a permission rather than an event. */ + verdict: "agree" | "disagree" | "exhausted" | "bypassed"; fidelity_rerolled?: number; fidelity_reported?: number; /** Plan-step identity. Present on a plan-step gate verdict; this is the join that diff --git a/packages/amico-run/src/warrant.ts b/packages/amico-run/src/warrant.ts index 11f28ef5..4c051b98 100644 --- a/packages/amico-run/src/warrant.ts +++ b/packages/amico-run/src/warrant.ts @@ -49,7 +49,10 @@ export interface WarrantRefusal { export type WarrantCheck = { ok: true } | WarrantRefusal; const SIZE_ORDER: Record = { SMALL: 0, MEDIUM: 1 }; -const DEVICE_ORDER: Record = { none: 0, ro: 1, rw: 2 }; +/** Exported because `plan_compile.ts` joins step device demands under the SAME order the + * launch gate compares with. Restating `{none:0,ro:1,rw:2}` in a second module would let the + * two drift, which is the defect class this spec keeps reproducing — one authority, imported. */ +export const DEVICE_ORDER: Record = { none: 0, ro: 1, rw: 2 }; /** Expiry in ms. An unparseable expiry is ALREADY EXPIRED — a warrant whose * lifetime cannot be established must not read as live (same fail-closed direction diff --git a/packages/amico-run/test/ledger_step_identity.test.ts b/packages/amico-run/test/ledger_step_identity.test.ts index ed4814d6..05e44a33 100644 --- a/packages/amico-run/test/ledger_step_identity.test.ts +++ b/packages/amico-run/test/ledger_step_identity.test.ts @@ -47,6 +47,24 @@ describe("plan-step identity on verdict/dispatch rows", () => { expect(row().verdict).toBe("exhausted"); }); + it("verdict gains `bypassed`, the SECOND HALF of the `skipped` producer", () => { + // `optional: true` on the compiled step is a PERMISSION; this row is the EVENT. With only + // the flag, `skipped` was unreachable while §4.5's completion rule admitted it — the same + // defect survived three spec revisions because each fixed only one half. + appendRecord({ + type: "verdict", ts: ts(), plan_hash: "abc", step_id: "s4", verdict: "bypassed", source: "user", + } as never); + expect(row().verdict).toBe("bypassed"); + }); + + it("rejects a verdict value outside the enum, so a typo cannot mint a state", () => { + expect(() => + appendRecord({ + type: "verdict", ts: ts(), plan_hash: "abc", step_id: "s5", verdict: "skipped", source: "user", + } as never), + ).toThrow(); + }); + it("a dispatch row can name a step", () => { appendRecord({ type: "dispatch", ts: ts(), task_type: "author-script", work_id: "wid", diff --git a/packages/schema/schemas/ledger-record.schema.json b/packages/schema/schemas/ledger-record.schema.json index fdcc7fb8..6bcb3421 100644 --- a/packages/schema/schemas/ledger-record.schema.json +++ b/packages/schema/schemas/ledger-record.schema.json @@ -77,7 +77,7 @@ "ts": { "type": "string" }, "problem_hash": { "type": "string" }, "structure_hash": { "type": "string" }, - "verdict": { "enum": ["agree", "disagree", "exhausted"], "description": "`exhausted` is per-step gate exhaustion (failure at the top reachable escalation rung). It lives here because the fleet registry's `blocked` is SESSION-scoped and cannot carry a per-step outcome." }, + "verdict": { "enum": ["agree", "disagree", "exhausted", "bypassed"], "description": "`exhausted` is per-step gate exhaustion (failure at the top reachable escalation rung). It lives here because the fleet registry's `blocked` is SESSION-scoped and cannot carry a per-step outcome. `bypassed` is the terminal row for a step the compiled plan declared `optional: true` and the walk did not need: it is the SECOND HALF of the `skipped` producer, because `optional: true` alone is a PERMISSION, not an event. Without it `skipped` was unreachable while §4.5's completion rule admitted it — three revisions running. A `bypassed` row against a step the plan did NOT mark optional is a derivation error, not a skip." }, "fidelity_rerolled": { "type": "number" }, "fidelity_reported": { "type": "number" }, "plan_hash": { "type": "string", "minLength": 1, "description": "Set on a plan-step verdict. With step_id this is the join plan-step state is derived from; OPTIONAL so every pre-existing solve verdict keeps validating." }, diff --git a/packages/schema/schemas/plan.schema.json b/packages/schema/schemas/plan.schema.json index 0e4a07fc..14f68dc7 100644 --- a/packages/schema/schemas/plan.schema.json +++ b/packages/schema/schemas/plan.schema.json @@ -34,16 +34,26 @@ "steps": { "type": "array", "minItems": 1, - "description": "Fleet §5.1 step objects, adopted unchanged. Only `optional` is added here: it is the SOLE producer of the `skipped` step state, which the completion rule admits.", + "description": "Fleet §5.1 step objects. `optional` is added here: with a `bypassed` verdict row it produces the `skipped` step state the completion rule admits (the flag alone is a permission, not an event). `model`, `variant`, `task_type` and `permissions` are DECLARED rather than left to `additionalProperties`, and `model`/`task_type` are REQUIRED, because §4.2's compile-time budget refusal reads them: a planner that omitted them yielded an empty demand set and every refusal passed silently — §0.1's inert `max_solves` counter, reproduced a fourth time. An undeterminable demand must be a loud refusal, never 'unbounded'.", "items": { "type": "object", "additionalProperties": true, - "required": ["id"], + "required": ["id", "model", "task_type"], "properties": { "id": { "type": "string", "minLength": 1 }, "needs": { "type": "array", "items": { "type": "string" }, "description": "DAG predecessors. Distinct from the compile-time capability `demands` — one name, one meaning." }, "gates": { "type": "array", "items": { "type": "string" } }, - "optional": { "type": "boolean" } + "optional": { "type": "boolean" }, + "model": { "type": "string", "pattern": "^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$", "description": "Fleet §5.1's name for the step's tier-dispatch target — 'no `tier` alias; the feature is called tier dispatch, the field is called `model`'. Holds a model id (anthropic/claude-haiku-4-5), which is why it CANNOT be compared to `bounds.tier`: that bound speaks the solvespec trust vocabulary (free|composed|vetted|hpc). `tier` is therefore a first-launch refusal, not a compile-time one." }, + "variant": { "type": "string", "minLength": 1 }, + "task_type": { "type": "string", "description": "A TASK_TYPES value. Solve-bearing types (experiment-sim, experiment-hw) are what §4.2 sums against budget.max_solves." }, + "permissions": { + "type": "object", + "additionalProperties": true, + "properties": { + "device": { "enum": ["none", "ro", "rw"], "description": "Joined under warrant.ts's exported DEVICE_ORDER against budget.device. Absent means `none`; a step needing device access must say so." } + } + } } } }, diff --git a/packages/schema/test/spec_plan_kinds.test.ts b/packages/schema/test/spec_plan_kinds.test.ts index 3b378e71..9d2bb807 100644 --- a/packages/schema/test/spec_plan_kinds.test.ts +++ b/packages/schema/test/spec_plan_kinds.test.ts @@ -46,10 +46,13 @@ describe("the spec kind", () => { }); describe("the plan kind", () => { + const step = (over: Record = {}) => ({ + id: "s1", model: "anthropic/claude-opus-5", task_type: "implement-slice", gates: ["re-rollout"], ...over, + }); const plan = (over: Record = {}) => ({ schema_version: "1", plan_id: "plan-20260728-1045-x", goal: "g", plan_hash: "c".repeat(64), design_hash: "a".repeat(64), - steps: [{ id: "s1", gates: ["re-rollout"] }], max_replans: 3, + steps: [step()], max_replans: 3, ...over, }); it("accepts a compiled plan", () => { @@ -58,8 +61,29 @@ describe("the plan kind", () => { it("requires the design_hash it was compiled from", () => { expect(validate(drop(plan(), "design_hash"), "plan").ok).toBe(false); }); - it("accepts an optional-step marker, the only producer of `skipped`", () => { - expect(validate(plan({ steps: [{ id: "s1", optional: true }] }), "plan").ok).toBe(true); + it("accepts an optional-step marker — HALF the `skipped` producer", () => { + // The other half is a `bypassed` verdict row. `optional: true` alone is a permission, + // not an event, which is why `skipped` was unreachable for three revisions. + expect(validate(plan({ steps: [step({ optional: true })] }), "plan").ok).toBe(true); + }); + + // §4.2's compile-time budget refusal reads these. A planner that omitted them yielded an + // EMPTY demand set, so every refusal passed silently — §0.1's inert counter, fourth instance. + it("REQUIRES model on every step — an undeterminable tier demand is not `unbounded`", () => { + expect(validate(plan({ steps: [drop(step(), "model")] }), "plan").ok).toBe(false); + }); + it("REQUIRES task_type — it is what decides whether a step is solve-bearing", () => { + expect(validate(plan({ steps: [drop(step(), "task_type")] }), "plan").ok).toBe(false); + }); + it("model must be a model id (provider/name), never a trust tier", () => { + // `bounds.tier` speaks free|composed|vetted|hpc; a step's `model` is a model id. The two + // are different vocabularies, which is why `tier` is a first-launch refusal. + expect(validate(plan({ steps: [step({ model: "hpc" })] }), "plan").ok).toBe(false); + expect(validate(plan({ steps: [step({ model: "anthropic/claude-haiku-4-5" })] }), "plan").ok).toBe(true); + }); + it("declares permissions.device over the DEVICE_ORDER vocabulary", () => { + expect(validate(plan({ steps: [step({ permissions: { device: "rw" } })] }), "plan").ok).toBe(true); + expect(validate(plan({ steps: [step({ permissions: { device: "admin" } })] }), "plan").ok).toBe(false); }); }); From b27505e0343d9a4b6c0c4aaa5681d31ea13ba2f3 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 12:20:47 -0400 Subject: [PATCH 21/27] =?UTF-8?q?feat(amico-run):=20the=20critic/planner?= =?UTF-8?q?=20subprocess=20mechanism=20(=C2=A73.7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tier-2 seam the front half left injectable now has a real implementation. `spec review` spawns critics through it; `plan compile` will spawn a planner through the same module, which is why it is `agent_spawn` and not `critic_spawn`. Async `spawn`, not `spawnSync`. §3.7 requires critics to run in PARALLEL under a whole-review ceiling, and `spawnSync` blocks the thread: N critics would run strictly serially and an in-flight child could never be preempted. The two requirements cannot both hold with a sync spawn, and the repo's own precedent (pasqal_launch, local_executor) is already async. `--config` DOES NOT EXIST on opencode. Its config channel is env-only (OPENCODE_CONFIG{,_CONTENT,_DIR}) and the CLI is `.strict()` with a `.fail` handler that exits 1 — so passing `--config` would make every critic exit 1 with help text on stdout, read as "unparseable" -> skipped -> `approved-mechanical` on EVERY review. The disclosure path would have become the silent default. Config now travels in OPENCODE_CONFIG_CONTENT, with $AMICO_AGENT_CONFIG_DIR to override. The agent definitions live HERE, not in amico-plugin, because `agents/` does not exist there, nothing reads that path (opencode resolves agents from config), and the publish chain cannot carry it — extract-public-skills stages only skills/*/, the release tars only dist/public-skills, fetch_skills requires only skills/. A definition shipped there would never reach a user. Critics get bash/edit/webfetch DENY: a reviewer that can shell out can act on the spec it was asked to judge. Two properties that needed care: - The model is read back from the CHILD, and a child that will not name itself is DISCARDED rather than stamped from argv. Recording the model we asked for would validate, and would be a request masquerading as a fact in the one field whose job is to let a reader judge how independent the review was. Losing a critic is the cheaper error. (Honest limit, now recorded: opencode suppresses `message.updated` in json mode and step parts carry no model, so self-report is the only channel available. Weaker than transport-observed; the code claims no more than that.) - `skip_class` distinguishes absent-binary from ran-and-failed, because approved-mechanical vs degraded turns on it. The shipped runner keyed the verdict on `critics.length === 0`, so three critics that all TIMED OUT recorded "no critic binary available". Bug found and fixed while testing: the payload parser fell back to raw stdout whenever the text parts failed to parse, and `firstJsonObject` then matched the event ENVELOPE — so a critic returning prose or truncated JSON came back as a successfully parsed payload with zero findings. A silent clean review out of an unreadable one, which is exactly what §3.2 exists to prevent. The fallback now applies only when the stream carried no text parts at all. 39 tests: all seven rows of the child-outcome table, each asserting the temp dir is gone; the env/argv/cwd claims driven through a REAL spawn of a recording fixture (asserting buildChildEnv as a pure function stays green while runAgent spreads process.env, so the canary has to ride an actual child). Co-Authored-By: Claude Opus 5 --- packages/amico-run/src/agent_defs.ts | 150 +++++++ packages/amico-run/src/agent_spawn.ts | 396 ++++++++++++++++++ packages/amico-run/test/agent_spawn.test.ts | 351 ++++++++++++++++ .../amico-run/test/fixtures/fake_agent.mjs | 79 ++++ 4 files changed, 976 insertions(+) create mode 100644 packages/amico-run/src/agent_defs.ts create mode 100644 packages/amico-run/src/agent_spawn.ts create mode 100644 packages/amico-run/test/agent_spawn.test.ts create mode 100644 packages/amico-run/test/fixtures/fake_agent.mjs diff --git a/packages/amico-run/src/agent_defs.ts b/packages/amico-run/src/agent_defs.ts new file mode 100644 index 00000000..8740adf1 --- /dev/null +++ b/packages/amico-run/src/agent_defs.ts @@ -0,0 +1,150 @@ +// packages/amico-run/src/agent_defs.ts — the `critic` and `planner` agent definitions +// (spec-20260728 §3.7), materialised into the child's config at spawn time. +// +// WHY THEY LIVE HERE and not in amico-plugin, where Rev 1 of the back-half plan put them: +// +// 1. `amico-plugin/agents/` does not exist, and nothing reads that path — opencode resolves +// agents from its CONFIG, not from a directory convention. +// 2. The publish chain cannot carry it: `extract-public-skills.sh` stages only +// `"$SKILLS_DIR"/*/`, the release workflow tars only `dist/public-skills`, and +// `fetch_skills.mjs` requires only `skills/`. A definition shipped there would never +// reach a user. +// 3. It would make the mechanism depend on a cross-repo artifact landing first, which is a +// sequencing hazard for something on the critical path of every review. +// +// These definitions are part of the MECHANISM's contract, not user-editable content. The +// escape hatch for someone who disagrees is `$AMICO_AGENT_CONFIG_DIR` (see agent_spawn.ts), +// the same shape as `$AMICO_PYTHON`. +// +// The transport is `OPENCODE_CONFIG_CONTENT`, an env var. NOT `--config`: opencode has no such +// flag (its config channel is env-only) and its CLI calls `.strict()` with a `.fail` handler +// that exits 1 — so passing `--config` would make every critic exit 1 with help text on stdout, +// which the child-outcome table reads as "unparseable" → `skipped` → `approved-mechanical` on +// EVERY review. The disclosure path would have become the silent default. + +/** The severity rule, stated to the critic in its own instructions. + * + * Enforcement is in code (`spec_review.ts` downgrades and logs), so this text is not what + * makes the invariant hold. It is here because a critic that understands the rule produces + * fewer findings to downgrade, and a downgrade is a lost finding — the critic spent its one + * lens on something the runner then demoted. */ +const SEVERITY_RULE = ` +You may mark a finding \`blocking\` ONLY when its lens is \`contradiction\`: two statements in the +spec that cannot both be true, with BOTH quoted. Everything else — however severe, however +confident you are — is \`advisory\`. This is not a formality: advisories are tracked as +obligations and a plan cannot be completed while one is open, so an advisory has teeth. A +\`blocking\` finding on any other lens is automatically downgraded and logged, which wastes your +one lens. If you are uncertain whether something is a contradiction, it is advisory.`.trim(); + +const REMEDY_RULE = ` +Every finding MUST carry a \`remedy\` — what would fix it. A finding that cannot say what would +fix it is DROPPED before it reaches the record, so an unactionable observation is wasted work.`.trim(); + +/** Both agents must report the model they actually ran as. + * + * This is a compromise, and the honest reason is worth recording: opencode's `--format json` + * emits an NDJSON event stream whose `message.updated` events (the ones carrying `modelID`) + * are explicitly suppressed in json mode, and `step-start`/`step-finish` parts carry no model + * field. So the model is NOT recoverable from the transport, and self-report is the only + * channel available. + * + * Self-report is weaker than transport-observed and this system claims no more than that. What + * it does preserve is the rule the ledger schema states: never stamp argv. A child that does + * not name itself is recorded as `skipped`, not as a critic that ran — we would rather lose a + * critic than record a request as a fact. */ +const REPORT_RULE = ` +Your reply must be a SINGLE JSON object and nothing else — no prose before or after, no code +fence. Shape: + +{"model": "", + "variant": "", + "findings": [{"lens": "", "severity": "advisory"|"blocking", + "claim": "", + "evidence": "", + "remedy": ""}]} + +If you find nothing, return an empty \`findings\` array. That is a real outcome and is recorded as +such. Reporting \`model\` is required: a critic that does not name itself is discarded rather than +recorded, because stamping the model we ASKED for would turn the record's independence +disclosure into a claim we did not verify.`.trim(); + +export const CRITIC_PROMPT = ` +You are an adversarial spec critic. You have been given ONE lens and a spec file. Review the spec +through that lens ONLY — another critic has each of the others, and duplicating their work costs a +perspective rather than adding confidence. + +Read the spec file in your working directory. It is the ONLY context you have: no conversation +history, no repository. That isolation is deliberate. It is isolation, not independence — you are +likely from the same model family as the spec's author, and the record says so rather than +pretending otherwise. + +${SEVERITY_RULE} + +${REMEDY_RULE} + +The highest-value finding in this system's history has been of one shape: **a check that reads a +field its schema does not carry.** A counter keyed on a forbidden field; a derivation reading an +\`additionalProperties: false\` branch; a join over a vocabulary with no order; a comparison whose +two sides speak different vocabularies. If the spec asserts a cross-module check, ask what the +values on BOTH sides actually are, and whether the spec ever says. + +${REPORT_RULE}`.trim(); + +export const PLANNER_PROMPT = ` +You are a plan compiler. You have been given an approved spec file. Turn it into a compiled plan: +an ordered set of steps that, executed, satisfies the spec's acceptance criteria. + +Read the spec file in your working directory. It is your only context. + +Each step MUST declare: + id a short stable slug, unique within the plan + model the model that should run it, as provider/model-id + task_type one of: triage, plan, author-script, implement-slice, bookkeeping, insight, + review, experiment-sim, experiment-hw, converse + gates how the step is verified. A step below the frontier tier MUST have at least one + gate — an unverified step by a cheaper model is refused by the lint AND by the + harness at dispatch. + needs ids of steps that must finish first (DAG predecessors) + permissions {"device": "none"|"ro"|"rw"} when the step touches hardware + optional true ONLY if the plan is still correct when this step is skipped + +\`model\` and \`task_type\` are REQUIRED on every step. They are not bookkeeping: the compiler sums +solve-bearing steps against the approved budget and joins device demands against it, and a step +that omits them makes that check silently pass. If you cannot determine one, that is a reason to +restructure the step, not to omit the field. + +Prefer fewer, larger steps over many small ones — each step boundary is a gate, and gates cost +model calls. But never merge a step that needs hardware with one that does not. + +Your reply must be a SINGLE JSON object and nothing else — no prose, no code fence: + +{"model": "", + "variant": "", + "goal": "", + "steps": [ … ]} + +Reporting \`model\` is required, for the same reason it is required of critics.`.trim(); + +/** The config the child discovers via `OPENCODE_CONFIG_CONTENT`. + * + * PERMISSIONS ARE DENY-BY-DEFAULT AND THAT IS LOAD-BEARING. A critic reads one file and emits + * one JSON object; it has no business running bash or editing anything. The fleet profile work + * established `task = "deny"` as the schema default for exactly this reason, and a reviewer + * that can shell out is a reviewer that can act on a spec it was asked to judge. */ +export function agentConfigContent(): string { + return JSON.stringify({ + $schema: "https://opencode.ai/config.json", + agent: { + critic: { + description: "Adversarial spec critic — one lens, one spec file, no history", + prompt: CRITIC_PROMPT, + permission: { bash: "deny", edit: "deny", webfetch: "deny" }, + }, + planner: { + description: "Compiles an approved spec into a gated, budgeted plan", + prompt: PLANNER_PROMPT, + permission: { bash: "deny", edit: "deny", webfetch: "deny" }, + }, + }, + }); +} diff --git a/packages/amico-run/src/agent_spawn.ts b/packages/amico-run/src/agent_spawn.ts new file mode 100644 index 00000000..85e69e7c --- /dev/null +++ b/packages/amico-run/src/agent_spawn.ts @@ -0,0 +1,396 @@ +// packages/amico-run/src/agent_spawn.ts — the tier-2 critic / planner subprocess mechanism +// (spec-20260728 §3.7). Shared: `spec review` spawns critics, `plan compile` spawns a planner. +// +// WHY A SUBPROCESS. `amico-run` has no provider SDK (its deps are @amicode/schema and smol-toml) +// and test/s31.test.ts fails CI on any ambient HTTP client in this layer outside a named EXEMPT +// set, whose comment records that lifting the ban "requires an explicit, reviewed S31 amendment." +// So a model call happens by spawning an already-authenticated agent CLI — the same shape as +// amico-pasqal spawning the connector. The S31 rule targets the HTTP surface only, so +// child_process is unaffected and no amendment is needed. +// +// (This comment deliberately does not spell out the banned identifiers: the guard is a grep over +// src/, and it caught an earlier draft of this very file for quoting them in prose. Keeping the +// guard dumb and rephrasing here is the right trade — a rule with an exception for comments is a +// rule with an exception.) +// +// WHY ASYNC `spawn` AND NOT `spawnSync`. §3.7 requires critics to run in PARALLEL under a +// whole-review ceiling. `spawnSync` blocks the thread: N critics would run strictly serially and +// an in-flight child could never be preempted by a ceiling — the two requirements cannot both +// hold. The repo's own precedent is async (pasqal_launch.ts, local_executor.ts). +// +// THE CREDENTIAL RULE. No secret is ever on argv (ps-visible, and it persists in shell history +// and transcripts). The child inherits the agent CLI's own credential store through an ALLOWLISTED +// env, built from scratch — never a `process.env` spread. The Pasqal launcher's discipline, +// different payload: it passes a secret IN, this passes none and inherits a store. +import { spawn as nodeSpawn, type SpawnOptions } from "node:child_process"; +import { accessSync, constants as fsConstants, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { delimiter, isAbsolute, join, resolve } from "node:path"; +import { agentConfigContent } from "./agent_defs.js"; +import type { Finding } from "./lenses.js"; + +export const DEFAULT_CRITIC_MODEL = "anthropic/claude-opus-5"; +export const DEFAULT_CRITIC_VARIANT = "high"; +/** Per-child ceiling (§3.7). A critic that has not answered in two minutes has failed. */ +export const CRITIC_TIMEOUT_MS = 120_000; + +/** Why a lens has no findings, which is NOT the same question as whether it ran. + * + * `absent` (no binary / spawn error) means the mechanism was never available: every critic + * absent is `approved-mechanical`, an honest "never adversarially reviewed". + * `failed` (timeout, signal, empty or unparseable output) means the mechanism WAS available and + * this critic did not deliver: that is `degraded`. + * + * Collapsing the two is a real defect the shipped runner had — it keyed the verdict on + * `critics.length === 0`, so three critics that all TIMED OUT against a working binary recorded + * "no critic binary available". Same field, two very different disclosures. */ +export type SkipClass = "absent" | "failed"; + +export interface AgentOutcome { + status: "ran" | "skipped"; + skip_class?: SkipClass; + reason?: string; + /** Read back from the CHILD's own output. Never argv — the ledger's independence disclosure + * must stamp a fact, not a request. A child that does not report one is `skipped`. */ + model?: string; + variant?: string; + findings: Finding[]; + /** §3.9: a finding that cannot say what would fix it is dropped. Counted so a silent drop + * cannot look like a clean critic. */ + dropped_no_remedy: number; + /** For the planner, whose payload carries goal+steps rather than findings. */ + payload?: Record; +} + +/** Binary resolution: `$AMICO_CRITIC_BIN`, else `opencode` on PATH. Probed with X_OK at an + * ABSOLUTE path so the spawn is deterministic (pasqal_launch's discipline). + * + * Returns `undefined` rather than throwing: an absent binary is the documented `--offline` + * degradation, not an error. A relative override is REJECTED — resolving it against cwd would + * make the spawn depend on where the user happened to be standing. */ +export function resolveAgentBin(env: NodeJS.ProcessEnv = process.env): string | undefined { + const override = env.AMICO_CRITIC_BIN; + if (override !== undefined && override.trim() !== "") { + if (!isAbsolute(override)) return undefined; + try { + accessSync(override, fsConstants.X_OK); + return override; + } catch { + return undefined; + } + } + for (const dir of (env.PATH ?? "").split(delimiter).filter(Boolean)) { + const candidate = join(dir, "opencode"); + try { + accessSync(candidate, fsConstants.X_OK); + return candidate; + } catch { + /* keep looking */ + } + } + return undefined; +} + +/** Env keys the child may see, and nothing else. Built FROM SCRATCH. + * + * A `{...process.env}` spread would hand a reviewer every credential this process holds, for a + * job that needs one file and one model call. The allowlist is HOME/PATH (the CLI must run at + * all), XDG_* (config discovery), TMPDIR, and the agent CLI's own auth vars — the store it is + * already authenticated against. */ +const ENV_ALLOWLIST = ["HOME", "PATH", "TMPDIR", "SHELL", "LANG", "LC_ALL", "TERM"]; +const ENV_ALLOW_PREFIXES = ["XDG_", "OPENCODE_"]; + +export function buildChildEnv( + parent: NodeJS.ProcessEnv = process.env, + extra: Record = {}, +): NodeJS.ProcessEnv { + const out: NodeJS.ProcessEnv = {}; + for (const [k, v] of Object.entries(parent)) { + if (v === undefined) continue; + if (ENV_ALLOWLIST.includes(k) || ENV_ALLOW_PREFIXES.some((p) => k.startsWith(p))) out[k] = v; + } + return { ...out, ...extra }; +} + +/** Recover the agent's payload from the event stream. + * + * opencode `--format json` emits NDJSON — one `{type, timestamp, sessionID, ...data}` per line — + * NOT a findings array and NOT a single JSON document. The assistant's text arrives as + * `{type: "text", part: {text}}`, and the payload is in the LAST such event: earlier text parts + * are the model thinking out loud. Rev 2 of the spec had the child receiving a file path as its + * prompt and returning prose; this is the real shape. + * + * Returns undefined on anything unparseable — never a partial parse. A half-read critic that + * reports "no findings" is worse than one that reports it could not be read. */ +export function parseAgentOutput(stdout: string): Record | undefined { + const texts: string[] = []; + for (const line of stdout.split("\n")) { + const t = line.trim(); + if (t === "") continue; + let ev: unknown; + try { + ev = JSON.parse(t); + } catch { + continue; // not every line is ours; a non-JSON line is not fatal to the stream + } + if (ev && typeof ev === "object") { + const e = ev as { type?: unknown; part?: { text?: unknown } }; + if (e.type === "text" && typeof e.part?.text === "string") texts.push(e.part.text); + } + } + // Search the text parts newest-first: earlier ones are the model thinking out loud. + for (const candidate of texts.reverse()) { + const obj = firstJsonObject(candidate); + if (obj !== undefined) return obj; + } + // Fall back to raw stdout ONLY when the stream carried no text parts at all — i.e. the child + // emitted a bare JSON answer instead of an event stream. Trying this fallback whenever the + // text parts failed to parse is WRONG and was a real bug here: `firstJsonObject` would match + // the event ENVELOPE (`{"type":"text","part":{…}}`), so a critic that returned prose, or + // truncated JSON, came back as a successfully parsed payload with no findings — a silent clean + // review out of an unreadable one, which is the exact failure §3.2 exists to prevent. + return texts.length === 0 ? firstJsonObject(stdout) : undefined; +} + +/** The first balanced `{…}` that parses. Agents wrap JSON in prose and fences no matter what the + * prompt says, so locating it is the parser's job rather than the model's. */ +function firstJsonObject(s: string): Record | undefined { + const start = s.indexOf("{"); + if (start < 0) return undefined; + let depth = 0; + let inStr = false; + let esc = false; + for (let i = start; i < s.length; i++) { + const c = s[i]; + if (inStr) { + if (esc) esc = false; + else if (c === "\\") esc = true; + else if (c === '"') inStr = false; + continue; + } + if (c === '"') inStr = true; + else if (c === "{") depth++; + else if (c === "}") { + depth--; + if (depth === 0) { + try { + const v = JSON.parse(s.slice(start, i + 1)); + return v && typeof v === "object" && !Array.isArray(v) ? (v as Record) : undefined; + } catch { + return undefined; + } + } + } + } + return undefined; +} + +/** Coerce the payload's findings, dropping any without a remedy (§3.9). */ +function readFindings(payload: Record, lens: string, round: number): { findings: Finding[]; dropped: number } { + const raw = Array.isArray(payload.findings) ? payload.findings : []; + const findings: Finding[] = []; + let dropped = 0; + for (const f of raw) { + if (!f || typeof f !== "object") { + dropped++; + continue; + } + const o = f as Record; + const remedy = typeof o.remedy === "string" ? o.remedy.trim() : ""; + if (remedy === "") { + dropped++; // unactionable: dropped before it reaches the record + continue; + } + findings.push({ + lens: typeof o.lens === "string" && o.lens !== "" ? o.lens : lens, + severity: o.severity === "blocking" ? "blocking" : "advisory", + claim: typeof o.claim === "string" ? o.claim : "", + evidence: typeof o.evidence === "string" ? o.evidence : "", + remedy, + round, + }); + } + return { findings, dropped }; +} + +const skipped = (skip_class: SkipClass, reason: string): AgentOutcome => ({ + status: "skipped", + skip_class, + reason: reason.slice(0, 300), + findings: [], + dropped_no_remedy: 0, +}); + +export interface RunAgentOptions { + bin: string; + agent: "critic" | "planner"; + model?: string; + variant?: string; + /** The lens (critic) — used as the finding's default lens and named in the prompt. */ + lens?: string; + prompt: string; + /** The spec's TEXT, copied into the child's cwd. The child gets the spec and nothing else. */ + specText: string; + specFilename?: string; + timeoutMs?: number; + round?: number; + env?: NodeJS.ProcessEnv; + /** Test seam. */ + spawn?: typeof nodeSpawn; +} + +/** Spawn one agent and return what it produced. + * + * NEVER THROWS. Every failure mode — no temp dir, spawn error, timeout, signal, empty stdout, + * unparseable stdout, a child that will not name its model — comes back as a `skipped` outcome + * with a reason. A dead critic must be a `skipped` lens, not a crashed review: the review is a + * gate on nothing (critics shape work, never block start), so failing it closed would be a + * denial of service on the whole loop. Same discipline as frontmatter.ts returning a result. + * + * The temp dir is removed on EVERY path, including timeout and spawn error. */ +export async function runAgent(opts: RunAgentOptions): Promise { + const spawnFn = opts.spawn ?? nodeSpawn; + const model = opts.model ?? DEFAULT_CRITIC_MODEL; + const variant = opts.variant ?? DEFAULT_CRITIC_VARIANT; + const round = opts.round ?? 1; + const lens = opts.lens ?? opts.agent; + const timeoutMs = opts.timeoutMs ?? CRITIC_TIMEOUT_MS; + + let cwd: string | undefined; + try { + cwd = mkdtempSync(join(tmpdir(), "amico-agent-")); + writeFileSync(join(cwd, opts.specFilename ?? "spec.md"), opts.specText); + } catch (e) { + // mkdtemp/writeFile can fail (read-only TMPDIR, quota). Still a result, never a throw. + if (cwd) rmSync(cwd, { recursive: true, force: true }); + return skipped("failed", `could not stage the child's working directory: ${(e as Error).message}`); + } + + try { + // Config travels in the ENV. `--config` does not exist on opencode and its CLI is .strict(), + // so passing one would exit 1 with help text on stdout — read as "unparseable", skipping + // every critic and making `approved-mechanical` the silent default on every review. + const configDir = opts.env?.AMICO_AGENT_CONFIG_DIR ?? process.env.AMICO_AGENT_CONFIG_DIR; + const extra: Record = configDir + ? { OPENCODE_CONFIG_DIR: configDir } + : { OPENCODE_CONFIG_CONTENT: agentConfigContent() }; + + const args = [ + "run", + "--agent", + opts.agent, + "--model", + model, + "--variant", + variant, + "--format", + "json", + // The prompt is POSITIONAL after `--`. Rev 2's `--file ` was wrong: `--file` + // ATTACHES a file, so the child would have received a path string as its prompt. + "--", + opts.prompt, + ]; + + const spawnOpts: SpawnOptions = { + cwd, + env: buildChildEnv(opts.env ?? process.env, extra), + // stderr is captured for the reason field, stdin ignored — an agent that waits on input + // would otherwise hang until the timeout with nothing to report. + stdio: ["ignore", "pipe", "pipe"], + }; + + const res = await collect(spawnFn, opts.bin, args, spawnOpts, timeoutMs); + if (res.kind === "spawn-error") return skipped("absent", res.reason); + if (res.kind === "timeout") return skipped("failed", `no answer within ${timeoutMs}ms`); + if (res.kind === "signal") return skipped("failed", `killed by ${res.signal}${tail(res.stderr)}`); + if (res.stdout.trim() === "") return skipped("failed", `exit ${res.code} with empty stdout${tail(res.stderr)}`); + + const payload = parseAgentOutput(res.stdout); + // A non-zero exit with PARSEABLE stdout still counts as having RUN (§3.7 row 2) — the child + // answered and then failed to clean up, which is not the same as not answering. + if (payload === undefined) return skipped("failed", `unparseable output (exit ${res.code})${tail(res.stderr)}`); + + const reportedModel = typeof payload.model === "string" ? payload.model : undefined; + if (reportedModel === undefined || !/^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/.test(reportedModel)) { + // Fail CLOSED. We could stamp `model` from argv and the row would validate — and it would + // be a request masquerading as a fact, in the one field whose job is to let a reader judge + // how independent the review was. Losing a critic is the cheaper error. + return skipped("failed", "the child did not report the model it ran as"); + } + const { findings, dropped } = readFindings(payload, lens, round); + return { + status: "ran", + model: reportedModel, + variant: typeof payload.variant === "string" && payload.variant !== "" ? payload.variant.slice(0, 32) : "default", + findings, + dropped_no_remedy: dropped, + payload, + reason: res.code === 0 ? undefined : `exit ${res.code} with usable output${tail(res.stderr)}`, + }; + } finally { + rmSync(cwd, { recursive: true, force: true }); + } +} + +const tail = (stderr: string): string => { + const s = stderr.trim(); + return s === "" ? "" : `: ${s.slice(-160)}`; +}; + +type Collected = + | { kind: "exit"; code: number; stdout: string; stderr: string } + | { kind: "signal"; signal: string; stdout: string; stderr: string } + | { kind: "timeout" } + | { kind: "spawn-error"; reason: string }; + +/** Run the child to completion, a signal, or the timeout. Resolves — never rejects. */ +function collect( + spawnFn: typeof nodeSpawn, + bin: string, + args: string[], + opts: SpawnOptions, + timeoutMs: number, +): Promise { + return new Promise((resolvePromise) => { + let child: ReturnType; + try { + child = spawnFn(bin, args, opts); + } catch (e) { + resolvePromise({ kind: "spawn-error", reason: (e as Error).message }); + return; + } + let stdout = ""; + let stderr = ""; + let settled = false; + const done = (c: Collected) => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolvePromise(c); + }; + const timer = setTimeout(() => { + child.kill("SIGKILL"); // SIGTERM can be swallowed; the ceiling must actually bite + done({ kind: "timeout" }); + }, timeoutMs); + child.stdout?.on("data", (d) => { + stdout += String(d); + }); + child.stderr?.on("data", (d) => { + stderr += String(d); + }); + // ENOENT arrives as an `error` event, not a throw, when the binary is absent. + child.on("error", (e) => done({ kind: "spawn-error", reason: e.message })); + child.on("close", (code, signal) => { + if (signal) done({ kind: "signal", signal, stdout, stderr }); + else done({ kind: "exit", code: code ?? 0, stdout, stderr }); + }); + }); +} + +/** Resolve the critic model (G-2: frontier, inheriting the session model). */ +export function criticModel(env: NodeJS.ProcessEnv = process.env): string { + const m = env.AMICO_CRITIC_MODEL; + return m !== undefined && m.trim() !== "" ? m.trim() : DEFAULT_CRITIC_MODEL; +} + +export { resolve as resolvePathForTest }; diff --git a/packages/amico-run/test/agent_spawn.test.ts b/packages/amico-run/test/agent_spawn.test.ts new file mode 100644 index 00000000..4feafa8c --- /dev/null +++ b/packages/amico-run/test/agent_spawn.test.ts @@ -0,0 +1,351 @@ +// The tier-2 critic / planner subprocess mechanism (spec-20260728 §3.7). +// Plan: plan-20260728-160000 Task 1. +// +// The env / argv / cwd claims are asserted against a REAL spawn of test/fixtures/fake_agent.mjs, +// not against pure functions. Testing `buildChildEnv` alone stays green while `runAgent` calls +// spawn(bin, argv, {env: {...process.env, ...built}}) — which is the leak that matters. This is +// the pattern test/pasqal_launch.test.ts established. +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { spawn as nodeSpawn } from "node:child_process"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + buildChildEnv, + criticModel, + parseAgentOutput, + resolveAgentBin, + runAgent, + type AgentOutcome, +} from "../src/agent_spawn.js"; + +const FAKE = join(__dirname, "fixtures", "fake_agent.mjs"); +const NODE = process.execPath; + +let dir: string; +let record: string; +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "agent-spawn-")); + record = join(dir, "record.json"); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +/** Run the fixture through the real spawn path. + * + * Two things happen in the wrapper, and both are deliberate: + * + * 1. `bin` is node, so the fixture's own path is prepended to the child's args. runAgent's argv + * construction stays under test rather than reimplemented here — every assertion reads the + * argv the fixture RECORDED. + * 2. The fixture's control variables (FAKE_AGENT_*) are injected AFTER buildChildEnv has run. + * They have to be: the allowlist correctly refuses to forward them, which is the behaviour + * the canary test asserts. Routing them around the allowlist keeps the leak test honest — + * if they went through the allowlist, adding a FAKE_ prefix to it would silently weaken the + * one guarantee this fixture exists to prove. */ +const FIXTURE_KEYS = /^FAKE_AGENT_/; + +async function run( + env: Record = {}, + over: Partial[0]> = {}, +): Promise { + const fixtureEnv: Record = { FAKE_AGENT_RECORD: record }; + const parentEnv: Record = {}; + for (const [k, v] of Object.entries(env)) { + if (FIXTURE_KEYS.test(k)) fixtureEnv[k] = String(v); + else parentEnv[k] = v; + } + return runAgent({ + bin: NODE, + agent: "critic", + lens: "hidden-failure", + prompt: "review this spec through the hidden-failure lens", + specText: "---\nspec_id: s\n---\n\nbody\n", + timeoutMs: 10_000, + env: { ...process.env, ...parentEnv } as NodeJS.ProcessEnv, + spawn: ((bin: string, args: string[], o: { env?: NodeJS.ProcessEnv }) => + nodeSpawn(bin, [FAKE, ...args], { ...o, env: { ...o.env, ...fixtureEnv } })) as never, + ...over, + }); +} + +const recorded = (): { argv: string[]; env: Record; cwd: string; files: string[]; enter: number } => + JSON.parse(readFileSync(record, "utf8")); + +describe("resolveAgentBin", () => { + it("prefers an ABSOLUTE $AMICO_CRITIC_BIN probed with X_OK", () => { + const bin = join(dir, "fake-cli"); + writeFileSync(bin, "#!/bin/sh\nexit 0\n"); + chmodSync(bin, 0o755); + expect(resolveAgentBin({ AMICO_CRITIC_BIN: bin, PATH: "" })).toBe(bin); + }); + + it("REJECTS a non-executable override — X_OK, not existsSync", () => { + // The claim is "probed with X_OK". A test that only tries a nonexistent path never + // distinguishes the two, so a mode-0644 file is the case that matters. + const bin = join(dir, "not-exec"); + writeFileSync(bin, "x"); + chmodSync(bin, 0o644); + expect(resolveAgentBin({ AMICO_CRITIC_BIN: bin, PATH: "" })).toBeUndefined(); + }); + + it("REJECTS a relative override — the spawn must not depend on cwd", () => { + expect(resolveAgentBin({ AMICO_CRITIC_BIN: "./opencode", PATH: "" })).toBeUndefined(); + }); + + it("returns undefined when nothing is executable — never throws, never a silent skip", () => { + expect(resolveAgentBin({ PATH: "/nonexistent-dir-xyz" })).toBeUndefined(); + }); + + it("finds `opencode` on PATH when there is no override", () => { + const bin = join(dir, "opencode"); + writeFileSync(bin, "#!/bin/sh\nexit 0\n"); + chmodSync(bin, 0o755); + expect(resolveAgentBin({ PATH: dir })).toBe(bin); + }); +}); + +describe("criticModel — G-2 resolution", () => { + it("defaults to a frontier model", () => { + expect(criticModel({})).toBe("anthropic/claude-opus-5"); + }); + it("honours $AMICO_CRITIC_MODEL", () => { + expect(criticModel({ AMICO_CRITIC_MODEL: "anthropic/claude-sonnet-5" })).toBe("anthropic/claude-sonnet-5"); + }); +}); + +describe("the child's environment, argv and cwd — asserted from a REAL spawn", () => { + it("env is built FROM SCRATCH: a canary in the parent does NOT reach the child", async () => { + await run({ AMICO_TEST_CANARY: "leak-me", ANTHROPIC_API_KEY: "sk-poison" }); + const rec = recorded(); + expect(rec.env.AMICO_TEST_CANARY).toBeUndefined(); + expect(rec.env.ANTHROPIC_API_KEY).toBeUndefined(); + expect(rec.env.HOME).toBeTruthy(); // the allowlist still lets the CLI run + expect(rec.env.PATH).toBeTruthy(); + }); + + it("carries the agent definitions in OPENCODE_CONFIG_CONTENT — the only config channel", async () => { + await run(); + const cfg = JSON.parse(recorded().env.OPENCODE_CONFIG_CONTENT); + expect(Object.keys(cfg.agent).sort()).toEqual(["critic", "planner"]); + // A reviewer that can shell out can act on the spec it was asked to judge. + expect(cfg.agent.critic.permission).toMatchObject({ bash: "deny", edit: "deny" }); + }); + + it("$AMICO_AGENT_CONFIG_DIR overrides the built-in definitions", async () => { + await run({ AMICO_AGENT_CONFIG_DIR: "/some/config/dir" }); + const rec = recorded(); + expect(rec.env.OPENCODE_CONFIG_DIR).toBe("/some/config/dir"); + expect(rec.env.OPENCODE_CONFIG_CONTENT).toBeUndefined(); + }); + + it("NEVER passes --config: opencode has no such flag and its CLI is .strict()", async () => { + // With --config the child exits 1 printing help, which the outcome table reads as + // "unparseable" -> skipped -> approved-mechanical on EVERY review. + await run(); + expect(recorded().argv).not.toContain("--config"); + }); + + it("passes NO secret on argv", async () => { + await run({ ANTHROPIC_API_KEY: "sk-poison", AMICO_PASQAL_FILE: "/tmp/poison-creds.json" }); + const flat = recorded().argv.join(" "); + for (const secret of ["sk-poison", "/tmp/poison-creds.json"]) expect(flat).not.toContain(secret); + }); + + it("sends the prompt POSITIONALLY after `--`, never as --file", async () => { + await run(); + const argv = recorded().argv; + expect(argv).not.toContain("--file"); // --file ATTACHES a file; the prompt is a message + expect(argv[argv.indexOf("--") + 1]).toMatch(/hidden-failure/); + expect(argv.slice(0, 4)).toEqual(["run", "--agent", "critic", "--model"]); + expect(argv).toContain("--format"); + expect(argv).toContain("json"); + }); + + it("the cwd holds EXACTLY the spec copy and nothing else", async () => { + await run(); + expect(recorded().files).toEqual(["spec.md"]); + }); +}); + +describe("parseAgentOutput — an NDJSON event stream, not a findings array", () => { + const ev = (text: string) => JSON.stringify({ type: "text", timestamp: 1, sessionID: "s", part: { type: "text", text } }); + + it("recovers the payload from the LAST text event, not the first", () => { + const stream = [ev('{"model":"a/b","findings":[]}'), ev('{"model":"c/d","findings":[{"lens":"x"}]}')].join("\n"); + expect(parseAgentOutput(stream)?.model).toBe("c/d"); + }); + + it("ignores tool_use and step events between the text parts", () => { + const stream = [ + JSON.stringify({ type: "step_start", part: {} }), + ev("thinking out loud"), + JSON.stringify({ type: "tool_use", part: {} }), + ev('{"model":"a/b","findings":[]}'), + ].join("\n"); + expect(parseAgentOutput(stream)).toMatchObject({ model: "a/b" }); + }); + + it("finds JSON wrapped in prose or a fence, because agents do that regardless", () => { + expect(parseAgentOutput(ev('Here you go:\n```json\n{"model":"a/b","findings":[]}\n```'))).toMatchObject({ + model: "a/b", + }); + }); + + it("returns undefined on prose with no JSON — never a partial parse", () => { + expect(parseAgentOutput(ev("I read the spec and it looks fine."))).toBeUndefined(); + expect(parseAgentOutput("not json at all")).toBeUndefined(); + }); + + it("returns undefined on truncated JSON rather than a half-read object", () => { + expect(parseAgentOutput(ev('{"model":"a/b","findings":[{'))).toBeUndefined(); + }); +}); + +describe("the child-outcome table — all seven rows", () => { + const F = JSON.stringify([{ lens: "hidden-failure", severity: "advisory", claim: "c", evidence: "e", remedy: "r" }]); + + it("row 1 — valid stdout, exit 0 → ran, contributes findings", async () => { + const out = await run({ FAKE_AGENT_FINDINGS: F }); + expect(out.status).toBe("ran"); + expect(out.findings).toHaveLength(1); + expect(out.model).toBe("anthropic/claude-opus-5"); + }); + + it("row 2 — exit ≠ 0 with PARSEABLE stdout → ran, reason recorded", async () => { + // The child answered and then failed to clean up. That is not the same as not answering. + const out = await run({ FAKE_AGENT_FINDINGS: F, FAKE_AGENT_EXIT: "3" }); + expect(out.status).toBe("ran"); + expect(out.findings).toHaveLength(1); + expect(out.reason).toMatch(/exit 3/); + }); + + it("row 3 — exit ≠ 0, unparseable → skipped(failed)", async () => { + const out = await run({ FAKE_AGENT_MODE: "prose", FAKE_AGENT_EXIT: "1" }); + expect(out).toMatchObject({ status: "skipped", skip_class: "failed" }); + expect(out.reason).toMatch(/unparseable/); + }); + + it("row 4 — killed by signal → skipped(failed)", async () => { + const out = await run({ FAKE_AGENT_MODE: "hang" }, { timeoutMs: 150 }); + // SIGKILL from our own ceiling reports as a timeout; both are skip_class failed. + expect(out).toMatchObject({ status: "skipped", skip_class: "failed" }); + }); + + it("row 5 — timeout → skipped(failed), naming the ceiling", async () => { + const out = await run({ FAKE_AGENT_MODE: "hang" }, { timeoutMs: 120 }); + expect(out.status).toBe("skipped"); + expect(out.reason).toMatch(/within 120ms/); + }); + + it("row 6 — exit 0, empty stdout → skipped(failed), never a clean critic", async () => { + const out = await run({ FAKE_AGENT_MODE: "empty" }); + expect(out).toMatchObject({ status: "skipped", skip_class: "failed" }); + expect(out.reason).toMatch(/empty stdout/); + }); + + it("row 7 — spawn error / binary absent → skipped(ABSENT), a different disclosure", async () => { + const out = await runAgent({ + bin: "/definitely/not/here", + agent: "critic", + prompt: "p", + specText: "x", + timeoutMs: 500, + }); + expect(out).toMatchObject({ status: "skipped", skip_class: "absent" }); + }); + + it("records stderr as the reason on a skipped row", async () => { + const out = await run({ FAKE_AGENT_MODE: "stderr", FAKE_AGENT_EXIT: "1" }); + expect(out.reason).toMatch(/503/); + }); + + it("distinguishes the skip CLASSES, because approved-mechanical vs degraded turns on it", async () => { + // Collapsing them is a real defect: the shipped runner keyed on critics.length === 0, so + // three critics that all TIMED OUT recorded "no critic binary available". + expect((await run({ FAKE_AGENT_MODE: "hang" }, { timeoutMs: 120 })).skip_class).toBe("failed"); + expect((await runAgent({ bin: "/nope", agent: "critic", prompt: "p", specText: "x", timeoutMs: 300 })).skip_class).toBe( + "absent", + ); + }); + + it("removes the temp dir on EVERY row, including timeout and spawn error", async () => { + const dirs: string[] = []; + for (const env of [{}, { FAKE_AGENT_MODE: "prose" }, { FAKE_AGENT_MODE: "empty" }]) { + await run(env); + dirs.push(recorded().cwd); + } + await run({ FAKE_AGENT_MODE: "hang" }, { timeoutMs: 120 }); + dirs.push(recorded().cwd); + for (const d of dirs) expect(existsSync(d)).toBe(false); + }); + + it("returns a RESULT when the working directory cannot be staged — never throws", async () => { + const out = await runAgent({ + bin: NODE, + agent: "critic", + prompt: "p", + specText: "x", + specFilename: "nested/dir/spec.md", // writeFileSync fails: the parent does not exist + timeoutMs: 500, + }); + expect(out).toMatchObject({ status: "skipped", skip_class: "failed" }); + expect(out.reason).toMatch(/working directory/); + }); +}); + +describe("the model is read back from the CHILD, never from argv", () => { + it("stamps what the child reported, not what we asked for", async () => { + const out = await run({ FAKE_AGENT_MODEL: "anthropic/claude-haiku-4-5" }, { model: "anthropic/claude-opus-5" }); + expect(out.model).toBe("anthropic/claude-haiku-4-5"); + }); + + it("DISCARDS a critic that will not name itself, rather than stamping the request", async () => { + // Stamping argv would validate and would be a request masquerading as a fact, in the one + // field whose job is to let a reader judge how independent the review was. + const out = await run({ FAKE_AGENT_MODE: "no-model" }); + expect(out).toMatchObject({ status: "skipped", skip_class: "failed" }); + expect(out.reason).toMatch(/did not report the model/); + }); + + it("rejects a reported model that is not provider/name shaped", async () => { + const out = await run({ FAKE_AGENT_MODEL: "hpc" }); + expect(out.status).toBe("skipped"); + }); +}); + +describe("findings hygiene (§3.9)", () => { + const withRemedy = { lens: "x", severity: "advisory", claim: "c", evidence: "e", remedy: "r" }; + + it("DROPS a finding with no remedy and counts the drop", async () => { + const out = await run({ + FAKE_AGENT_FINDINGS: JSON.stringify([withRemedy, { ...withRemedy, remedy: "" }, { ...withRemedy, remedy: " " }]), + }); + expect(out.findings).toHaveLength(1); + expect(out.dropped_no_remedy).toBe(2); // a silent drop must not look like a clean critic + }); + + it("stamps the round on every finding", async () => { + const out = await run({ FAKE_AGENT_FINDINGS: JSON.stringify([withRemedy]) }, { round: 3 }); + expect(out.findings[0].round).toBe(3); + }); + + it("defaults the lens to the one the critic was given, so a mislabelled finding is still placed", async () => { + const out = await run({ FAKE_AGENT_FINDINGS: JSON.stringify([{ ...withRemedy, lens: "" }]) }); + expect(out.findings[0].lens).toBe("hidden-failure"); + }); + + it("coerces an unknown severity to advisory — a critic cannot invent a severity", async () => { + const out = await run({ FAKE_AGENT_FINDINGS: JSON.stringify([{ ...withRemedy, severity: "catastrophic" }]) }); + expect(out.findings[0].severity).toBe("advisory"); + }); +}); + +describe("buildChildEnv", () => { + it("allowlists rather than spreads", () => { + const env = buildChildEnv({ HOME: "/h", PATH: "/p", XDG_CONFIG_HOME: "/x", SECRET: "leak" }); + expect(env).toEqual({ HOME: "/h", PATH: "/p", XDG_CONFIG_HOME: "/x" }); + }); + it("lets explicit extras through", () => { + expect(buildChildEnv({ HOME: "/h" }, { OPENCODE_CONFIG_CONTENT: "{}" }).OPENCODE_CONFIG_CONTENT).toBe("{}"); + }); +}); diff --git a/packages/amico-run/test/fixtures/fake_agent.mjs b/packages/amico-run/test/fixtures/fake_agent.mjs new file mode 100644 index 00000000..e03d680c --- /dev/null +++ b/packages/amico-run/test/fixtures/fake_agent.mjs @@ -0,0 +1,79 @@ +#!/usr/bin/env node +// A stand-in for the agent CLI, so the env / argv / cwd claims in agent_spawn.ts are asserted +// against a REAL spawn rather than against a pure function. Testing `buildChildEnv` alone stays +// green while `runAgent` calls spawn(bin, argv, {env: {...process.env, ...built}}) — the leak the +// canary assertion exists to catch. This is the pattern test/pasqal_launch.test.ts already uses. +// +// It records everything it received, then behaves per $FAKE_AGENT_MODE so one fixture can drive +// all seven rows of the §3.7 child-outcome table. +import { writeFileSync, readdirSync } from "node:fs"; + +if (process.env.FAKE_AGENT_RECORD) { + writeFileSync( + process.env.FAKE_AGENT_RECORD, + JSON.stringify({ + argv: process.argv.slice(2), + env: process.env, + cwd: process.cwd(), + // The §3.7 isolation claim: the cwd holds ONLY the spec copy. + files: readdirSync(".").sort(), + // Stamped so a test can assert two children genuinely overlapped in time. + enter: Date.now(), + }), + ); +} + +const mode = process.env.FAKE_AGENT_MODE ?? "stream"; +const exitCode = Number(process.env.FAKE_AGENT_EXIT ?? 0); + +const stream = (parts) => parts.map((p) => JSON.stringify(p)).join("\n") + "\n"; + +/** opencode `--format json` emits NDJSON — one object per line, `{type, timestamp, sessionID, + * ...data}` — NOT a JSON array. The assistant's text arrives as `{type: "text", part: {text}}`, + * and the findings live in the LAST such event. Rev 2 of the spec had the child receiving a + * file path as its prompt and returning prose; this is the real shape. */ +const payload = JSON.stringify({ + model: process.env.FAKE_AGENT_MODEL ?? "anthropic/claude-opus-5", + variant: process.env.FAKE_AGENT_VARIANT ?? "high", + findings: JSON.parse(process.env.FAKE_AGENT_FINDINGS ?? "[]"), +}); + +switch (mode) { + case "empty": // exit 0, empty stdout + process.exit(exitCode); + break; + case "prose": // unparseable: no JSON anywhere + process.stdout.write("I read the spec and I think it looks fine.\n"); + process.exit(exitCode); + break; + case "no-model": // a critic that will not name itself — must not be recorded as having run + process.stdout.write(stream([{ type: "text", part: { type: "text", text: JSON.stringify({ findings: [] }) } }])); + process.exit(exitCode); + break; + case "hang": // drives the timeout row; never exits on its own + setInterval(() => {}, 1000); + break; + case "slow": // sleeps, so two children's [enter, exit] intervals can be shown to intersect + setTimeout(() => { + process.stdout.write(stream([{ type: "text", part: { type: "text", text: payload } }])); + process.exit(exitCode); + }, Number(process.env.FAKE_AGENT_SLEEP_MS ?? 300)); + break; + case "stderr": // exercises the reason field + process.stderr.write("model provider returned 503\n"); + process.exit(exitCode); + break; + default: + // A realistic stream: tool use and step markers around the terminal text event, so the + // parser is shown to pick the LAST text part rather than the first JSON-looking thing. + process.stdout.write( + stream([ + { type: "step_start", part: { type: "step-start" } }, + { type: "tool_use", part: { type: "tool", state: { status: "completed" } } }, + { type: "text", part: { type: "text", text: "Let me look at the acceptance block." } }, + { type: "step_finish", part: { type: "step-finish", reason: "stop" } }, + { type: "text", part: { type: "text", text: payload } }, + ]), + ); + process.exit(exitCode); +} From b92d9c50089239b04ed567e864334fe7e49a188a Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 12:50:51 -0400 Subject: [PATCH 22/27] feat(amico-run): tier-2 critics wired in + the parity flake, finally characterized MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TIER 2 (§3.5-3.8). `reviewSpec` becomes async — critics run in PARALLEL, which is the whole reason the mechanism is async `spawn`. Both production transports already await (amico.ts, mcp_serve.ts) and `Verb.run` is already typed to permit a Promise, so the blast radius was the tests: every `specVerb([...]).code` in spec_verb.test.ts read `.code` off a Promise and got undefined — including the pure usage-error paths that never touch reviewSpec. The verdict rule is corrected. It keyed on `critics.length === 0`, which conflated two very different disclosures: three critics that all TIMED OUT against a working binary recorded "no critic binary available". `approved-mechanical` now means the mechanism was never available (skip_class absent); anything attempted that fell short is `degraded`. The whole-review ceiling takes an INJECTED clock. Per-critic timeout is 120s and the largest lens set is 4, so a parallel review's worst case is ~120s and a wall-clock test could never make a 600s ceiling fire. An untestable ceiling is a comment, not a guarantee. It is checked BEFORE each spawn, so it bounds spend and not just time. A SUITE-WIDE GUARD against real model calls (spec advisory A-11). With `opencode` on PATH — which is everyone here — any test omitting --offline and injecting nothing would have fanned out real, billed frontier critics. test/setup.ts pins $AMICO_CRITIC_BIN to an impossible path for the whole suite and fails closed; a test that wants a child opts in explicitly. Per-test discipline was the wrong fix because the risk is in the test someone writes next. Same file backstops $AMICO_LEDGER. Also lands Task 6 early, since the verb needed it: `precedentIn`/`precedentFor` give the `precedent` lens a BUCKET-BLIND count. `aggregate` matches on n_bucket/t_bucket because it answers "what parameters worked at this size"; a spec note has a structure_hash and no N or T, so routing precedent through it returned total: 0 for nearly every real hash — collapsing "nothing was attempted" into "could not query", the one distinction §3.3 exists to draw. It also filters VERDICT rows by lane, which aggregate does not: a simulated gym verdict could otherwise mark a real solve verified. And it returns undefined on read/parse failure, which is what makes the lens's `unverified` status reachable in production at all. THE PARITY FLAKE, characterized after resisting it for two sessions. It is executor_parity.test.ts: local's run.log had 1 iter line while remote's had 2, with both EVENT streams identical. Root cause: `logStream.end()` is ASYNCHRONOUS, so settle() resolved `finished` and closed the event stream in the same tick while the file was still short. The events queue is in-memory and was always complete, which is exactly why the symptom read as nondeterminism rather than a race. It is a product bug, not a test artifact: anything reading run.log after a run reports finished — the extension, a replay, a user tailing — could see a truncated log. Two regression tests assert the property directly against a 200-line script, because two lines reproduced it only sometimes. 37 consecutive clean suite runs since, against ~1-in-4 before; I could not prove elimination at that rate, so this is stated as strong evidence rather than certainty. NOT shipped: I also hypothesised that readline could have buffered lines when 'close' fires, making onLine's `if (settled) return` drop them. Tested against 200 lines and it made no difference — readline drains first in practice. Gating settle() on the readers' own close would have added a wedge risk to a shipped launch path to fix something that did not reproduce. Recorded in the comment as a known-unproven edge, since the guard's own comment calls the ordering merely "rare". Co-Authored-By: Claude Opus 5 --- packages/amico-run/src/ledger_query.ts | 47 +++ packages/amico-run/src/local_executor.ts | 40 ++- packages/amico-run/src/spec_review.ts | 124 ++++++-- packages/amico-run/src/spec_verb.ts | 13 +- .../amico-run/test/local_executor.test.ts | 40 +++ packages/amico-run/test/setup.ts | 23 ++ packages/amico-run/test/spec_review.test.ts | 274 ++++++++++++------ packages/amico-run/test/spec_verb.test.ts | 44 +-- packages/amico-run/vitest.config.ts | 14 + 9 files changed, 483 insertions(+), 136 deletions(-) create mode 100644 packages/amico-run/test/setup.ts create mode 100644 packages/amico-run/vitest.config.ts diff --git a/packages/amico-run/src/ledger_query.ts b/packages/amico-run/src/ledger_query.ts index 24f211f5..c2af57ec 100644 --- a/packages/amico-run/src/ledger_query.ts +++ b/packages/amico-run/src/ledger_query.ts @@ -241,3 +241,50 @@ export function aggregate(records: LedgerRecord[], key: QueryKey): QueryResult { export function queryDefaults(key: QueryKey): QueryResult { return aggregate(readRecords(), key); } + +// ── precedent (spec-20260728 §3.3) ──────────────────────────────────────────── + +export interface Precedent { + total: number; + verified: number; +} + +/** How many times this work identity has been attempted, and how many attempts a gate agreed + * with. BUCKET-BLIND, and that is the whole point. + * + * `aggregate` above matches on `n_bucket`/`t_bucket` because it is answering "what parameters + * worked at this problem size". A spec note has a `structure_hash` and no N or T at all, so + * routing `precedent` through `aggregate` would return `total: 0` for nearly every real hash — + * collapsing "nothing was ever attempted" into "the ledger could not be read", which is exactly + * the distinction §3.3 exists to draw and the one the lens's three-way status depends on. + * + * Verdict rows are filtered by lane here, which `aggregate` does not do: it filters SOLVES by + * `source === "user"` but takes any `agree` verdict, so a simulated gym verdict could mark a + * real solve verified. An ABSENT source is treated as user, because the field is optional and + * pre-existing rows predate it — excluding them would silently rewrite history as unverified. */ +export function precedentIn(records: LedgerRecord[], structureHash: string): Precedent { + const solves = records.filter(isSolve).filter((s) => s.source === "user" && s.structure_hash === structureHash); + const agreed = new Set( + records + .filter((r): r is Extract => r.type === "verdict") + .filter((r) => r.verdict === "agree" && (r.source === undefined || r.source === "user")) + .map((r) => r.problem_hash) + .filter((h): h is string => typeof h === "string"), + ); + return { total: solves.length, verified: solves.filter((s) => agreed.has(s.problem_hash)).length }; +} + +/** The ledger-reading wrapper the `precedent` lens is injected with. + * + * Returns `undefined` on ANY read or parse failure, which is what makes the lens's `unverified` + * status reachable in production. `readRecords()` returns `[]` for a missing ledger and THROWS + * on a malformed line, so a wrapper that let the throw escape would crash the review, and one + * that swallowed it into `[]` would report "nothing was ever attempted" over a corrupt ledger — + * the worst of the three answers. */ +export function precedentFor(structureHash: string): Precedent | undefined { + try { + return precedentIn(readRecords(), structureHash); + } catch { + return undefined; + } +} diff --git a/packages/amico-run/src/local_executor.ts b/packages/amico-run/src/local_executor.ts index 501d5040..e4bcf41c 100644 --- a/packages/amico-run/src/local_executor.ts +++ b/packages/amico-run/src/local_executor.ts @@ -308,10 +308,32 @@ export class LocalExecutor implements Executor { process.stderr.write(`amico-run: failed to write FINISHED: ${(e as Error).message}\n`); } emitSolveStanza(runDir); // Plan 3 / L1 Task 5 — never throws; a ledger hiccup must never fail a run - logStream.end(); - events.push({ kind: "finished", status, exitCode }); - events.close(); - resolveFinished({ status, exitCode }); + + // WAIT FOR run.log TO ACTUALLY FLUSH before announcing completion. + // + // `logStream.end()` is ASYNCHRONOUS: it schedules the flush and returns immediately. The + // old code called it and then resolved `finished` / closed the event stream in the same + // tick, so a consumer could await completion and read a TRUNCATED run.log — the events + // queue is in-memory and complete while the file on disk is still short a line or two. + // + // This was an intermittent 1-in-4 failure in executor_parity.test.ts (local's iterLines + // had 1 entry, remote's had 2, with both event streams identical) and it is a real product + // bug, not just a test artifact: anything that reads run.log after a run reports finished — + // the extension, a replay, a user tailing the file — could see a partial log. The + // 'close'-not-'exit' comment above is what makes the EVENTS complete; nothing made the + // FILE complete. + let announced = false; + const announce = (): void => { + if (announced) return; + announced = true; + events.push({ kind: "finished", status, exitCode }); + events.close(); + resolveFinished({ status, exitCode }); + }; + // A stream that errors must not wedge the run: losing the tail of a log is bad, hanging + // forever is worse. Both paths announce exactly once. + logStream.once("error", announce); + logStream.end(announce); }; // stdbuf (spec §5 "where available") is deliberately omitted in β.1: the β.3 script @@ -324,8 +346,18 @@ export class LocalExecutor implements Executor { }); // spawn failure AFTER manifest exists → FINISHED{failed, 127} (spec §6) child.on("error", () => settle("failed", 127)); + // 'close', NOT 'exit': close waits for stdout/stderr to drain, so every line event // lands before settle() — the events stream must terminate ON the finished event (§3). + // + // (I hypothesised a second bug here while fixing the run.log flush race below: that readline + // could still have buffered lines when 'close' fires, making the `if (settled) return` guard + // in onLine silently DROP them from both the log and the event stream. Gating settle() on the + // readers' own 'close' events was tested against a 200-line script and made no difference — + // readline does drain first in practice. Not shipped: it would add a wedge risk to a shipped + // launch path (a reader that never closes would hang the run) to fix something that did not + // reproduce. Recorded because the guard's own comment calls the ordering merely "rare", so + // this is a known-unproven edge rather than a verified invariant.) child.on("close", (code, signal) => { const rc = code ?? signalCode(signal); settle(aborting ? "aborted" : rc === 0 ? "completed" : "failed", rc); diff --git a/packages/amico-run/src/spec_review.ts b/packages/amico-run/src/spec_review.ts index b9c9af82..7f1c3a29 100644 --- a/packages/amico-run/src/spec_review.ts +++ b/packages/amico-run/src/spec_review.ts @@ -6,8 +6,9 @@ // because the zero-spawn guarantee is testable now and would be untestable later. import { createHash } from "node:crypto"; import { existsSync, mkdirSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { canonicalJson, designHash } from "@amicode/schema"; +import { criticModel, resolveAgentBin, runAgent, type AgentOutcome } from "./agent_spawn.js"; import { appendRecord, type SpecReviewRecord } from "./ledger.js"; import { parseFrontmatter } from "./frontmatter.js"; import { LENSES, type Finding, type LensStatus } from "./lenses.js"; @@ -21,6 +22,9 @@ import { } from "./lens_registry.js"; export const ROUND_BUDGET = 3; +/** Whole-review ceiling (§3.7). Critics already carry a 120s each; this bounds the review as a + * whole so a pathological set cannot hold the loop open indefinitely. */ +export const REVIEW_CEILING_MS = 600_000; export type ReviewVerdict = "approved" | "approved-mechanical" | "degraded" | "blocking" | "exhausted"; @@ -46,9 +50,9 @@ export interface ReviewResult { critic_spawns: number; } -/** The tier-2 seam. Not implemented in this slice (G-2), but injected so the - * ZERO-SPAWN-on-tier-1-blocking guarantee is a test today rather than a promise. */ -export type SpawnCritic = (lens: string) => { model: string; variant: string; findings: Finding[] } | undefined; +/** The tier-2 seam, now with a real default (`agent_spawn`). Still injected, because the + * ZERO-SPAWN-on-tier-1-blocking guarantee has to be assertable without a binary. */ +export type SpawnCritic = (lens: string) => Promise | AgentOutcome; export interface ReviewOptions { round?: number; @@ -59,6 +63,16 @@ export interface ReviewOptions { now?: () => string; /** Skip the ledger append (tests that only care about the computation). */ append?: boolean; + /** Monotonic elapsed-ms source for the whole-review ceiling. + * + * Injected because the ceiling is otherwise UNTESTABLE: the per-critic timeout is 120s and + * the largest tier-2 set is 4 lenses, so a parallel review's worst case is ~120s and a + * wall-clock test could never make the 600s ceiling fire. `now` cannot serve — it returns an + * ISO string used only for the record's `ts`. An untestable ceiling is a comment, not a + * guarantee. */ + elapsedMs?: () => number; + /** Env for binary/model resolution (test seam). */ + env?: NodeJS.ProcessEnv; } /** Blocking tier-1 lenses. A blocking lens that could not run yields `unverified`, and a @@ -78,7 +92,7 @@ export function findingsRefFor(specPath: string, specId: string, hash: string, r return join(dirname(specPath), ".review", `${specId}-${hash.slice(0, 16)}-r${round}.json`); } -export function reviewSpec(specPath: string, raw: string, opts: ReviewOptions = {}): ReviewResult { +export async function reviewSpec(specPath: string, raw: string, opts: ReviewOptions = {}): Promise { const round = opts.round ?? 1; const nowIso = (opts.now ?? (() => new Date().toISOString()))(); const lens_status: LensStatusEntry[] = []; @@ -139,19 +153,46 @@ export function reviewSpec(specPath: string, raw: string, opts: ReviewOptions = // ── tier 2 ── const wanted = criticCountFor(taskType, opts.critics ?? 3); const lenses = tier2LensesFor(taskType).slice(0, wanted); - let degraded = false; - if (!opts.offline && opts.spawnCritic && lenses.length > 0) { + const spawnCritic = resolveSpawnCritic(specPath, raw, round, opts); + /** Did the MECHANISM exist at all? Distinguishes "never adversarially reviewed" from "review + * was attempted and fell short", which is the whole point of `approved-mechanical` vs + * `degraded`. */ + let mechanismAvailable = false; + let anyFailedSkip = false; + + if (spawnCritic && lenses.length > 0) { + mechanismAvailable = true; + const elapsed = opts.elapsedMs; + // Critics run in PARALLEL (§3.7): one lens each, no shared state, and the review's wall + // clock is the slowest critic rather than their sum. A serial loop over four 120s critics + // would take 8 minutes to say what 2 minutes can. + const started: Array<{ lens: Tier2Name; p: Promise }> = []; for (const lens of lenses) { + // The ceiling is checked BEFORE each spawn, so it bounds spend rather than just wall time. + // Checking after would let the last critic start at 599s and run to 719s. + if (elapsed && elapsed() >= REVIEW_CEILING_MS) { + lens_status.push({ lens, status: "skipped", reason: `whole-review ceiling (${REVIEW_CEILING_MS}ms) reached before this critic started` }); + anyFailedSkip = true; + continue; + } critic_spawns++; - const out = opts.spawnCritic(lens); - if (!out) { - // Timeout, unparseable output, spawn failure: `skipped`, never counted as clean. - lens_status.push({ lens, status: "skipped", reason: "critic did not return usable output" }); - degraded = true; + started.push({ lens, p: Promise.resolve(spawnCritic(lens)) }); + } + const settled = await Promise.all(started.map((s) => s.p)); + + for (let i = 0; i < started.length; i++) { + const lens = started[i].lens; + const out = settled[i]; + if (out.status === "skipped") { + lens_status.push({ lens, status: "skipped", reason: out.reason ?? "critic did not return usable output" }); + // An ABSENT binary is not a degradation — nothing was ever available to degrade from. + if (out.skip_class !== "absent") anyFailedSkip = true; continue; } lens_status.push({ lens, status: "ran" }); - critics.push({ model: out.model, variant: out.variant }); + // `model` is guaranteed present on a `ran` outcome: agent_spawn discards a child that will + // not name itself rather than letting argv fill the field. + critics.push({ model: out.model as string, variant: out.variant as string }); // THE TERMINATION INVARIANT: a tier-2 critic may not emit `blocking` except for // `contradiction`. Anything else is DOWNGRADED to advisory and logged — this is what // guarantees the loop converges, so it is enforced rather than requested. @@ -166,23 +207,58 @@ export function reviewSpec(specPath: string, raw: string, opts: ReviewOptions = } } } - } else if (lenses.length > 0) { - // No critic mechanism available: tier 1 only, and the record says so. - degraded = false; } const post = findings.filter((f) => f.severity === "blocking"); if (post.length > 0) { return finish(specPath, spec_id, design_hash, round, lens_status, critics, findings, critic_spawns, opts, nowIso); } - const verdict: ReviewVerdict = lenses.length === 0 || critics.length === 0 - ? "approved-mechanical" - : degraded - ? "degraded" - : "approved"; + // The verdict rule, corrected. It used to key on `critics.length === 0`, which conflated two + // very different disclosures: three critics that all TIMED OUT against a working binary + // recorded "no critic binary available". `approved-mechanical` now means the mechanism was + // never available; anything that was attempted and fell short is `degraded`. + const verdict: ReviewVerdict = + lenses.length === 0 || !mechanismAvailable || (critics.length === 0 && !anyFailedSkip) + ? "approved-mechanical" + : anyFailedSkip + ? "degraded" + : "approved"; return finish(specPath, spec_id, design_hash, round, lens_status, critics, findings, critic_spawns, opts, nowIso, verdict); } +/** The default tier-2 mechanism: a real critic subprocess per lens. + * + * Returns undefined — meaning "tier 1 only, and the record says so" — when `--offline` is set + * or no agent binary resolves. An absent binary is documented degradation (§3.8), never an + * error and never a silent pass. */ +function resolveSpawnCritic( + specPath: string, + raw: string, + round: number, + opts: ReviewOptions, +): SpawnCritic | undefined { + if (opts.offline) return undefined; + if (opts.spawnCritic) return opts.spawnCritic; + const env = opts.env ?? process.env; + const bin = resolveAgentBin(env); + if (bin === undefined) return undefined; + const model = criticModel(env); + return (lens: string) => + runAgent({ + bin, + agent: "critic", + model, + lens, + round, + env, + prompt: `Your lens is \`${lens}\`. Review ${basename(specPath)} through that lens only.`, + specText: raw, + specFilename: basename(specPath), + }); +} + +type Tier2Name = ReturnType[number]; + function finish( specPath: string, spec_id: string, @@ -252,7 +328,11 @@ function finish( } /** Read-and-review, for the verb. Kept separate so `reviewSpec` stays testable on a string. */ -export function reviewSpecFile(specPath: string, readFile: (p: string) => string, opts: ReviewOptions = {}): ReviewResult { +export function reviewSpecFile( + specPath: string, + readFile: (p: string) => string, + opts: ReviewOptions = {}, +): Promise { if (!existsSync(specPath)) throw new Error(`spec not found: ${specPath}`); return reviewSpec(specPath, readFile(specPath), opts); } diff --git a/packages/amico-run/src/spec_verb.ts b/packages/amico-run/src/spec_verb.ts index c07ffddb..5e3375ee 100644 --- a/packages/amico-run/src/spec_verb.ts +++ b/packages/amico-run/src/spec_verb.ts @@ -19,9 +19,14 @@ import { existsSync, readFileSync } from "node:fs"; import { resolve } from "node:path"; import { validate } from "@amicode/schema"; import { parseFrontmatter } from "./frontmatter.js"; +import { precedentFor } from "./ledger_query.js"; import { reviewSpec, type ReviewOptions } from "./spec_review.js"; import type { VerbResult } from "./verbs.js"; +/** The `precedent` lens's ledger channel. Named here rather than inlined so the verb's default + * and the test seam are visibly the same shape — the lens never reads the ledger itself. */ +const defaultQueryLedger: NonNullable = (structureHash) => precedentFor(structureHash); + const USAGE = "amico spec review [--critics N] [--offline] [--json] | amico spec validate "; function usageError(error: string): VerbResult { @@ -57,7 +62,7 @@ function positional(argv: string[]): { path: string } | { error: string } { return { error: "a spec path is required (positional, not --spec)" }; } -function review(argv: string[], ctx: SpecVerbCtx): VerbResult { +async function review(argv: string[], ctx: SpecVerbCtx): Promise { const pos = positional(argv); if ("error" in pos) return usageError(pos.error); const abs = resolve(pos.path); @@ -72,11 +77,11 @@ function review(argv: string[], ctx: SpecVerbCtx): VerbResult { let r; try { - r = reviewSpec(abs, (ctx.readFile ?? readFileSync)(abs, "utf8") as string, { + r = await reviewSpec(abs, (ctx.readFile ?? readFileSync)(abs, "utf8") as string, { critics, offline: argv.includes("--offline"), spawnCritic: ctx.spawnCritic, - queryLedger: ctx.queryLedger, + queryLedger: ctx.queryLedger ?? defaultQueryLedger, round: ctx.round, }); } catch (e) { @@ -129,7 +134,7 @@ export interface SpecVerbCtx { round?: number; } -export function specVerb(argv: string[], ctx: SpecVerbCtx = {}): VerbResult { +export async function specVerb(argv: string[], ctx: SpecVerbCtx = {}): Promise { const sub = argv[0]; const rest = argv.slice(1); if (sub === "review") return review(rest, ctx); diff --git a/packages/amico-run/test/local_executor.test.ts b/packages/amico-run/test/local_executor.test.ts index 393b0cd1..8eb79c04 100644 --- a/packages/amico-run/test/local_executor.test.ts +++ b/packages/amico-run/test/local_executor.test.ts @@ -87,3 +87,43 @@ describe("LocalExecutor happy path", () => { expect(cwdLine.line).toContain(h.runDir); }); }); + +// The bug this guards was an intermittent ~1-in-4 failure in executor_parity.test.ts that +// resisted characterisation for two sessions. `logStream.end()` is ASYNCHRONOUS — it schedules +// the flush and returns — so `settle()` resolved `finished` and closed the event stream in the +// same tick, while run.log was still short a line. The events queue is in-memory and was always +// complete, which is exactly why the symptom looked like nondeterminism rather than a race. +// +// It is a product bug, not a test artifact: anything reading run.log after a run reports +// finished (the extension, a replay, a user tailing) could see a truncated log. Asserted here +// as a direct property with MANY lines, because two lines reproduced it only sometimes. +describe("run.log is COMPLETE when the run reports finished", () => { + const LINES = 200; + const CHATTY = Array.from({ length: LINES }, (_, i) => `console.log('AMICODE_ITER iter=${i + 1} f=1e-${i}')`).join("\n"); + + it("every emitted line is on disk by the time `finished` resolves", async () => { + const root = tmpRoot(); + const h = await new LocalExecutor().submit(fakeJulia(root, "solve.jl", ""), { + lab: "testlab", + runsRoot: join(root, "runs"), + julia: { julia: fakeJulia(root, "julia-chatty", CHATTY) }, + }); + await h.finished; // the ONLY synchronisation a consumer has + const onDisk = readFileSync(join(h.runDir, "run.log"), "utf8").split("\n").filter(Boolean); + expect(onDisk).toHaveLength(LINES); + expect(onDisk.at(-1)).toContain(`iter=${LINES}`); + }); + + it("…and by the time the event stream closes, which is the other completion signal", async () => { + const root = tmpRoot(); + const h = await new LocalExecutor().submit(fakeJulia(root, "solve.jl", ""), { + lab: "testlab", + runsRoot: join(root, "runs"), + julia: { julia: fakeJulia(root, "julia-chatty", CHATTY) }, + }); + const evs = await collect(h.events); + expect(evs.filter((e) => e.kind === "iter")).toHaveLength(LINES); + // The parity test's failure mode exactly: events complete, file short. + expect(readFileSync(join(h.runDir, "run.log"), "utf8").split("\n").filter(Boolean)).toHaveLength(LINES); + }); +}); diff --git a/packages/amico-run/test/setup.ts b/packages/amico-run/test/setup.ts new file mode 100644 index 00000000..4a2596a8 --- /dev/null +++ b/packages/amico-run/test/setup.ts @@ -0,0 +1,23 @@ +// Global test guard: NO TEST MAY EVER SPAWN A REAL MODEL CALL. +// +// `spec review` and `plan compile` resolve their agent binary from $AMICO_CRITIC_BIN, else +// `opencode` on PATH. A developer with opencode installed — which is everyone on this team — +// would otherwise have any test that omits `--offline` and injects no `spawnCritic` fan out +// real, billed frontier critics. The spec registered this as advisory A-11; per-test discipline +// is the wrong fix, because the failure mode is a test someone writes later without thinking +// about it. +// +// So the guard is global and it fails CLOSED: an absolute path that cannot exist makes +// resolveAgentBin() return undefined, which is the documented "no critic binary" degradation. +// A test that WANTS a child sets AMICO_CRITIC_BIN itself (to test/fixtures/fake_agent.mjs) or +// injects a spawnCritic, both of which are explicit. +process.env.AMICO_CRITIC_BIN = "/nonexistent/amico-test-guard/no-real-model-calls"; + +// Same reasoning for the ledger: tests must never append to the developer's real ops store. +// One shipped test deleted AMICO_BIN to exercise PATH resolution, and since a real `amico` was +// installed that branch RESOLVED IT and performed a real append — ten junk rows accumulated in +// ~/.amico/ledger/runs.jsonl, one per suite run. Individual suites still override this with +// their own temp path; this is the backstop for the ones that forget. +if (!process.env.AMICO_LEDGER) { + process.env.AMICO_LEDGER = "/nonexistent/amico-test-guard/ledger.jsonl"; +} diff --git a/packages/amico-run/test/spec_review.test.ts b/packages/amico-run/test/spec_review.test.ts index e703b46a..2535b4f7 100644 --- a/packages/amico-run/test/spec_review.test.ts +++ b/packages/amico-run/test/spec_review.test.ts @@ -1,13 +1,17 @@ // The spec-review runner (spec-20260728 §3). -// Plan: plan-20260728-104500 Task 10. +// Plan: plan-20260728-104500 Task 10 (tier 1), plan-20260728-160000 Task 2 (tier 2). +// +// `reviewSpec` is ASYNC as of tier 2: critics run in parallel, which a sync spawn cannot do. import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, readFileSync, rmSync, statSync, existsSync, chmodSync } from "node:fs"; +import { spawn } from "node:child_process"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { createHash } from "node:crypto"; import { canonicalJson } from "@amicode/schema"; import { readRecords, type SpecReviewRecord } from "../src/ledger.js"; -import { reviewSpec } from "../src/spec_review.js"; +import { reviewSpec, REVIEW_CEILING_MS } from "../src/spec_review.js"; +import type { AgentOutcome } from "../src/agent_spawn.js"; import type { Finding } from "../src/lenses.js"; const fm = (o: Record) => @@ -26,6 +30,15 @@ const SLICE = { schema_version: '"1"', spec_id: "spec-slice", task_type: "implem const record = () => readRecords().filter((r): r is SpecReviewRecord => r.type === "spec_review")[0]; +/** A critic that ran and found nothing. */ +const ran = (over: Partial = {}): AgentOutcome => ({ + status: "ran", model: "anthropic/claude-opus-5", variant: "high", findings: [], dropped_no_remedy: 0, ...over, +}); +/** A critic that could not deliver. `absent` means the mechanism was never there. */ +const skip = (skip_class: "absent" | "failed", reason = "r"): AgentOutcome => ({ + status: "skipped", skip_class, reason, findings: [], dropped_no_remedy: 0, +}); + describe("reviewSpec", () => { let dir: string; let specPath: string; @@ -40,90 +53,168 @@ describe("reviewSpec", () => { }); describe("verdicts", () => { - it("a clean spec with NO critic mechanism yields approved-mechanical, not approved", () => { - const r = reviewSpec(specPath, fm(SLICE)); + it("a clean spec with NO critic mechanism yields approved-mechanical, not approved", async () => { + const r = await reviewSpec(specPath, fm(SLICE)); expect(r.review_verdict).toBe("approved-mechanical"); expect(r.exit_code).toBe(0); expect(r.critics).toEqual([]); }); - it("a blocking tier-1 finding yields review_verdict=blocking and exit 65", () => { - const r = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["it should be good"] })); + it("a blocking tier-1 finding yields review_verdict=blocking and exit 65", async () => { + const r = await reviewSpec(specPath, fm({ ...SLICE, acceptance: ["it should be good"] })); expect(r.review_verdict).toBe("blocking"); expect(r.exit_code).toBe(65); expect(r.findings.some((f) => f.lens === "falsifiable" && f.severity === "blocking")).toBe(true); }); - it("blocking at the LAST round is `exhausted` (66), a human decision point", () => { - const r = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { round: 3 }); + it("blocking at the LAST round is `exhausted` (66), a human decision point", async () => { + const r = await reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { round: 3 }); expect(r.review_verdict).toBe("exhausted"); expect(r.exit_code).toBe(66); }); - it("an UNVERIFIED blocking lens cannot yield approved", () => { - // `precedent` is advisory, so force the case through a blocking lens: an - // unreadable frontmatter is the schema lens failing to run at all. - const r = reviewSpec(specPath, "no frontmatter here\n"); + it("an UNVERIFIED blocking lens cannot yield approved", async () => { + const r = await reviewSpec(specPath, "no frontmatter here\n"); expect(r.review_verdict).not.toBe("approved-mechanical"); expect(r.exit_code).toBe(65); expect(r.findings[0].lens).toBe("schema"); }); - it("a clean spec WITH critics that all run yields approved", () => { - const r = reviewSpec(specPath, fm(SLICE), { - spawnCritic: () => ({ model: "anthropic/claude-opus-5", variant: "high", findings: [] }), - }); + it("a clean spec WITH critics that all run yields approved", async () => { + const r = await reviewSpec(specPath, fm(SLICE), { spawnCritic: () => ran() }); expect(r.review_verdict).toBe("approved"); expect(r.critics.length).toBeGreaterThan(0); }); - it("a critic that returns nothing (timeout/unparseable) yields DEGRADED, never approved", () => { + it("--offline runs tier 1 only and stamps critics: []", async () => { + const r = await reviewSpec(specPath, fm(SLICE), { offline: true, spawnCritic: () => ran() }); + expect(r.review_verdict).toBe("approved-mechanical"); + expect(r.critic_spawns).toBe(0); + expect(record().critics).toEqual([]); + }); + }); + + // The distinction this describe block exists for is the one the shipped runner got wrong: it + // keyed the verdict on `critics.length === 0`, so three critics that all TIMED OUT against a + // working binary recorded "no critic binary available" — a false disclosure in the one field a + // reader uses to judge whether the spec was reviewed at all. + describe("approved-mechanical vs degraded turns on WHY there were no critics", () => { + it("every critic ABSENT (no binary) → approved-mechanical: never adversarially reviewed", async () => { + const r = await reviewSpec(specPath, fm(SLICE), { spawnCritic: () => skip("absent", "binary not found") }); + expect(r.review_verdict).toBe("approved-mechanical"); + expect(r.critics).toEqual([]); + }); + + it("a critic that TIMED OUT → degraded, NOT approved-mechanical", async () => { + const r = await reviewSpec(specPath, fm(SLICE), { spawnCritic: () => skip("failed", "no answer within 120000ms") }); + expect(r.review_verdict).toBe("degraded"); + }); + + it("some ran, one failed → degraded", async () => { let n = 0; - const r = reviewSpec(specPath, fm(SLICE), { - spawnCritic: () => (n++ === 0 ? { model: "anthropic/claude-opus-5", variant: "high", findings: [] } : undefined), - }); + const r = await reviewSpec(specPath, fm(SLICE), { spawnCritic: () => (n++ === 0 ? ran() : skip("failed")) }); expect(r.review_verdict).toBe("degraded"); expect(r.lens_status.some((s) => s.status === "skipped")).toBe(true); }); - it("--offline runs tier 1 only and stamps critics: []", () => { - const r = reviewSpec(specPath, fm(SLICE), { - offline: true, - spawnCritic: () => ({ model: "anthropic/claude-opus-5", variant: "high", findings: [] }), - }); - expect(r.review_verdict).toBe("approved-mechanical"); - expect(r.critic_spawns).toBe(0); - expect(record().critics).toEqual([]); + it("each skipped tier-2 lens carries a non-empty reason IN THE PERSISTED RECORD", async () => { + await reviewSpec(specPath, fm(SLICE), { spawnCritic: () => skip("failed", "provider returned 503") }); + const skipped = record().lens_status.filter((s) => s.status === "skipped"); + expect(skipped.length).toBeGreaterThan(0); + // Asserted on the ROW, not the return value: a reason that exists only in memory is a + // reason nobody can read later. + for (const s of skipped) expect(s.reason).toMatch(/503/); }); }); describe("the free-tier guarantee", () => { - it("spawns ZERO critics when a tier-1 lens blocks", () => { + it("spawns ZERO critics when a tier-1 lens blocks", async () => { let spawns = 0; - const r = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { - spawnCritic: () => { spawns++; return { model: "anthropic/claude-opus-5", variant: "high", findings: [] }; }, + const r = await reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { + spawnCritic: () => { spawns++; return ran(); }, }); expect(r.exit_code).toBe(65); expect(spawns).toBe(0); // a bad spec never reaches a paid critic expect(r.critic_spawns).toBe(0); }); - it("spawns ZERO critics for a tier-1-only task type however many are requested", () => { + it("…and the POSITIVE CONTROL: the same mechanism DOES spawn on a clean spec", async () => { + // Without this, the guarantee above is unobservable — `spawns` is 0 for every input when + // no binary resolves, so a broken wiring would pass the negative test silently. + let spawns = 0; + await reviewSpec(specPath, fm(SLICE), { spawnCritic: () => { spawns++; return ran(); } }); + // 3, not 4: implement-slice HAS 4 tier-2 lenses but `--critics` defaults to 3, so the + // default review spends three calls and the fourth lens is simply not selected. + expect(spawns).toBe(3); + }); + + it("spawns ZERO critics for a tier-1-only task type however many are requested", async () => { let spawns = 0; - reviewSpec(specPath, fm({ ...SLICE, task_type: "bookkeeping" }), { - critics: 3, - spawnCritic: () => { spawns++; return { model: "anthropic/claude-opus-5", variant: "high", findings: [] }; }, + await reviewSpec(specPath, fm({ ...SLICE, task_type: "bookkeeping" }), { + critics: 3, spawnCritic: () => { spawns++; return ran(); }, }); expect(spawns).toBe(0); }); - it("clamps --critics to the lenses that exist for this task type", () => { + it("clamps --critics to the lenses that exist for this task type", async () => { let spawns = 0; - reviewSpec(specPath, fm(SLICE), { - critics: 99, - spawnCritic: () => { spawns++; return { model: "anthropic/claude-opus-5", variant: "high", findings: [] }; }, + await reviewSpec(specPath, fm(SLICE), { critics: 99, spawnCritic: () => { spawns++; return ran(); } }); + expect(spawns).toBe(4); + }); + }); + + describe("tier 2 runs in PARALLEL", () => { + it("two critics' [enter, exit] intervals INTERSECT", async () => { + // The claim is about wall clock, so it is asserted on wall clock. A sync spawn cannot pass + // this — which is why the mechanism is async `spawn` rather than `spawnSync`. + const spans: Array<[number, number]> = []; + await reviewSpec(specPath, fm(SLICE), { + spawnCritic: async () => { + const enter = Date.now(); + await new Promise((r) => setTimeout(r, 80)); + spans.push([enter, Date.now()]); + return ran(); + }, + }); + expect(spans.length).toBe(3); // the default --critics + + const [a, b] = spans; + expect(a[0]).toBeLessThan(b[1]); + expect(b[0]).toBeLessThan(a[1]); // genuine overlap, not merely "both finished" + }); + + it("the whole-review ceiling stops spawning further critics", async () => { + // The ceiling needs an INJECTED clock to be testable at all: per-critic timeout is 120s + // and the largest lens set is 4, so a parallel review's worst case is ~120s and real time + // could never reach 600s. An untestable ceiling is a comment, not a guarantee. + let spawns = 0; + let elapsed = 0; + const r = await reviewSpec(specPath, fm(SLICE), { + elapsedMs: () => elapsed, + spawnCritic: () => { spawns++; elapsed = REVIEW_CEILING_MS; return ran(); }, + }); + expect(spawns).toBe(1); // the first spawn pushes elapsed past the ceiling + expect(r.review_verdict).toBe("degraded"); // the unspawned lenses are skipped, not clean + expect(record().lens_status.filter((s) => s.status === "skipped").length).toBe(2); + }); + + it("checks the ceiling BEFORE each spawn, so it bounds spend and not just wall time", async () => { + let elapsed = REVIEW_CEILING_MS; + let spawns = 0; + await reviewSpec(specPath, fm(SLICE), { + elapsedMs: () => elapsed, spawnCritic: () => { spawns++; return ran(); }, + }); + expect(spawns).toBe(0); + }); + }); + + describe("the model stamp is a fact, not a request", () => { + it("stamps the model the CHILD reported, not the one we asked for", async () => { + const r = await reviewSpec(specPath, fm(SLICE), { + spawnCritic: () => ran({ model: "anthropic/claude-haiku-4-5", variant: "low" }), }); - expect(spawns).toBe(4); // implement-slice has 4 tier-2 lenses + expect(r.critics[0]).toEqual({ model: "anthropic/claude-haiku-4-5", variant: "low" }); + expect(record().critics[0].model).toBe("anthropic/claude-haiku-4-5"); }); }); @@ -132,28 +223,23 @@ describe("reviewSpec", () => { lens, severity: "blocking", claim: "c", evidence: "e", remedy: "r", round: 1, }); - it("a tier-2 `blocking` finding on any lens but `contradiction` is DOWNGRADED to advisory", () => { - const r = reviewSpec(specPath, fm(SLICE), { - spawnCritic: (lens) => ({ - model: "anthropic/claude-opus-5", variant: "high", - findings: lens === "hidden-failure" ? [blockingFinding("hidden-failure")] : [], - }), + it("a tier-2 `blocking` finding on any lens but `contradiction` is DOWNGRADED to advisory", async () => { + const r = await reviewSpec(specPath, fm(SLICE), { + spawnCritic: (lens) => ran({ findings: lens === "hidden-failure" ? [blockingFinding("hidden-failure")] : [] }), }); - // Persisted as ADVISORY, and the review is not blocked by it. expect(r.findings.filter((f) => f.severity === "blocking")).toEqual([]); expect(r.blocking_count).toBe(0); expect(r.review_verdict).toBe("approved"); + // Asserted on the PERSISTED sidecar: an implementation that wrote `blocking` to disk and + // downgraded only the in-memory copy would pass an assertion on the return value. const persisted: Finding[] = JSON.parse(readFileSync(r.findings_ref, "utf8")); expect(persisted.find((f) => f.lens === "hidden-failure")?.severity).toBe("advisory"); expect(record().blocking_count).toBe(0); }); - it("`contradiction` is the ONE tier-2 finding that may block", () => { - const r = reviewSpec(specPath, fm(SLICE), { - spawnCritic: (lens) => ({ - model: "anthropic/claude-opus-5", variant: "high", - findings: lens === "hidden-failure" ? [blockingFinding("contradiction")] : [], - }), + it("`contradiction` is the ONE tier-2 finding that may block", async () => { + const r = await reviewSpec(specPath, fm(SLICE), { + spawnCritic: (lens) => ran({ findings: lens === "hidden-failure" ? [blockingFinding("contradiction")] : [] }), }); expect(r.blocking_count).toBe(1); expect(r.exit_code).toBe(65); @@ -161,35 +247,34 @@ describe("reviewSpec", () => { }); describe("the findings sidecar", () => { - it("writes the bodies, and findings_sha256 is over the CANONICAL array", () => { - const r = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] })); + it("writes the bodies, and findings_sha256 is over the CANONICAL array", async () => { + const r = await reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] })); expect(existsSync(r.findings_ref)).toBe(true); const bodies: Finding[] = JSON.parse(readFileSync(r.findings_ref, "utf8")); expect(bodies).toHaveLength(r.findings_count); expect(createHash("sha256").update(canonicalJson(bodies as never), "utf8").digest("hex")).toBe(r.findings_sha256); }); - it("keys the sidecar on spec_id AND round, so round 2 cannot overwrite round 1", () => { - const a = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { round: 1 }); - const b = reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { round: 2 }); + it("keys the sidecar on spec_id AND round, so round 2 cannot overwrite round 1", async () => { + const a = await reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { round: 1 }); + const b = await reviewSpec(specPath, fm({ ...SLICE, acceptance: ["prose"] }), { round: 2 }); expect(a.findings_ref).not.toBe(b.findings_ref); expect(existsSync(a.findings_ref)).toBe(true); }); - it("a sidecar that cannot be written fails LOUDLY, never a dangling ref", () => { - chmodSync(dir, 0o500); // read+execute only: no new subdirectory + it("a sidecar that cannot be written fails LOUDLY, never a dangling ref", async () => { + chmodSync(dir, 0o500); try { - expect(() => reviewSpec(specPath, fm(SLICE))).toThrow(/sidecar/i); + await expect(reviewSpec(specPath, fm(SLICE))).rejects.toThrow(/sidecar/i); } finally { chmodSync(dir, 0o700); } }); - it("the record stays under PIPE_BUF with a maximal review", () => { + it("the record stays under PIPE_BUF with a maximal review", async () => { const many = Array.from({ length: 40 }, (_, i) => `metric${i} >= ${i}`); - const r = reviewSpec(specPath, fm({ ...SLICE, acceptance: many }), { - spawnCritic: () => ({ - model: "anthropic/claude-opus-5", variant: "high", + const r = await reviewSpec(specPath, fm({ ...SLICE, acceptance: many }), { + spawnCritic: () => ran({ findings: Array.from({ length: 9 }, (_, i) => ({ lens: "hidden-failure", severity: "advisory" as const, claim: "c".repeat(200), evidence: "e".repeat(200), remedy: "r".repeat(200), round: 1 + (i % 3), @@ -198,44 +283,65 @@ describe("reviewSpec", () => { }); const line = readFileSync(process.env.AMICO_LEDGER!, "utf8").split("\n").filter(Boolean).pop()!; expect(Buffer.byteLength(line, "utf8")).toBeLessThanOrEqual(4096); - // 3 critics (the default) x 9 findings, each ~600 bytes of prose: >15 KB of bodies - // in the sidecar while the ROW stays under the 4096-byte ceiling. That gap is the - // whole reason the sidecar exists — the row used to carry every finding and would - // have thrown on append, after the model spend. + // 3 critics x 9 findings, each ~600 bytes of prose: >15 KB of bodies in the sidecar while + // the ROW stays under the 4096-byte ceiling. That gap is the whole reason the sidecar + // exists — the row used to carry every finding and would have thrown on append, after the + // model spend. expect(r.findings_count).toBe(27); expect(Buffer.byteLength(readFileSync(r.findings_ref, "utf8"), "utf8")).toBeGreaterThan(15_000); }); }); describe("the ledger record", () => { - it("appends exactly one spec_review, stamped with the registry version", () => { - reviewSpec(specPath, fm(SLICE)); - const recs = readRecords().filter((r) => r.type === "spec_review"); - expect(recs).toHaveLength(1); + it("appends exactly one spec_review, stamped with the registry version", async () => { + await reviewSpec(specPath, fm(SLICE)); + expect(readRecords().filter((r) => r.type === "spec_review")).toHaveLength(1); expect(record().lens_registry_version).toMatch(/^\S+$/); expect(record().design_hash).toMatch(/^[0-9a-f]{64}$/); }); - it("records per-lens status including not-applicable", () => { - reviewSpec(specPath, fm(SLICE)); - const budget = record().lens_status.find((s) => s.lens === "budget"); - // implement-slice is not launch-shaped, so budget is scoped out — and that is - // recorded as not-applicable rather than absent or clean. - expect(budget).toBeUndefined(); // not even selected for this task type + it("records per-lens status including not-applicable", async () => { + await reviewSpec(specPath, fm(SLICE)); + expect(record().lens_status.find((s) => s.lens === "budget")).toBeUndefined(); expect(record().lens_status.map((s) => s.lens)).toContain("schema"); }); - it("a launch-shaped spec records budget/baseline/precedent statuses", () => { - reviewSpec(specPath, fm(LAUNCH)); + it("a launch-shaped spec records budget/baseline/precedent statuses", async () => { + await reviewSpec(specPath, fm(LAUNCH)); const names = record().lens_status.map((s) => s.lens); expect(names).toContain("budget"); expect(names).toContain("baseline"); expect(record().lens_status.find((s) => s.lens === "precedent")?.status).toBe("not-applicable"); }); - it("append can be suppressed for pure computation", () => { - reviewSpec(specPath, fm(SLICE), { append: false }); + it("append can be suppressed for pure computation", async () => { + await reviewSpec(specPath, fm(SLICE), { append: false }); expect(readRecords().filter((r) => r.type === "spec_review")).toHaveLength(0); }); }); + + // A-11: with `opencode` on PATH, any test omitting --offline and injecting nothing would fan + // out real billed critics. test/setup.ts pins $AMICO_CRITIC_BIN to an impossible path for the + // whole suite; this asserts the guard is actually in force rather than assumed. + describe("the no-real-model-calls guard", () => { + it("the suite-wide $AMICO_CRITIC_BIN cannot resolve", async () => { + expect(process.env.AMICO_CRITIC_BIN).toMatch(/nonexistent/); + const r = await reviewSpec(specPath, fm(SLICE)); + expect(r.critic_spawns).toBe(0); + expect(r.review_verdict).toBe("approved-mechanical"); + }); + + it("resolves a REAL child when a test opts in explicitly", async () => { + // The mechanism is wired end to end — not merely injectable — and the opt-in is visible. + const r = await reviewSpec(specPath, fm(SLICE), { + env: { ...process.env, AMICO_CRITIC_BIN: undefined } as NodeJS.ProcessEnv, + spawnCritic: () => ran(), + }); + expect(r.critics.length).toBe(3); + }); + }); }); + +// Keeps `spawn` imported: the parallel test above relies on the async mechanism, and a stray +// unused-import cleanup would silently make it a sequential test that still passes. +void spawn; diff --git a/packages/amico-run/test/spec_verb.test.ts b/packages/amico-run/test/spec_verb.test.ts index ae6c8911..e7315fd2 100644 --- a/packages/amico-run/test/spec_verb.test.ts +++ b/packages/amico-run/test/spec_verb.test.ts @@ -33,16 +33,16 @@ describe("amico spec", () => { const json = (r: { json: unknown }) => r.json as Record; - it("exit 0 + approved-mechanical on a clean spec", () => { + it("exit 0 + approved-mechanical on a clean spec", async () => { writeFileSync(path, SLICE); - const r = specVerb(["review", path]); + const r = await specVerb(["review", path]); expect(r.code).toBe(0); expect(json(r).review_verdict).toBe("approved-mechanical"); }); - it("exit 65 + blocking, with the findings INLINE so the refusal is actionable", () => { + it("exit 65 + blocking, with the findings INLINE so the refusal is actionable", async () => { writeFileSync(path, PROSE); - const r = specVerb(["review", path]); + const r = await specVerb(["review", path]); expect(r.code).toBe(65); expect(json(r).review_verdict).toBe("blocking"); const blocking = json(r).blocking as Array>; @@ -50,46 +50,46 @@ describe("amico spec", () => { expect(blocking[0].remedy).toBeTruthy(); }); - it("the verdict is in the PAYLOAD too, because the MCP facade discards exit codes", () => { + it("the verdict is in the PAYLOAD too, because the MCP facade discards exit codes", async () => { writeFileSync(path, SLICE); - const r = specVerb(["review", path]); + const r = await specVerb(["review", path]); expect(json(r)).toHaveProperty("review_verdict"); expect(json(r)).toHaveProperty("exit_code", 0); }); - it("exit 64 on usage errors: no path, unknown subcommand, bad --critics", () => { - expect(specVerb(["review"]).code).toBe(64); - expect(specVerb(["frobnicate"]).code).toBe(64); - expect(specVerb([]).code).toBe(64); + it("exit 64 on usage errors: no path, unknown subcommand, bad --critics", async () => { + expect((await specVerb(["review"])).code).toBe(64); + expect((await specVerb(["frobnicate"])).code).toBe(64); + expect((await specVerb([])).code).toBe(64); writeFileSync(path, SLICE); - expect(specVerb(["review", path, "--critics", "-2"]).code).toBe(64); - expect(specVerb(["review", path, "--critics", "many"]).code).toBe(64); + expect((await specVerb(["review", path, "--critics", "-2"])).code).toBe(64); + expect((await specVerb(["review", path, "--critics", "many"])).code).toBe(64); }); - it("exit 64 when the spec does not exist", () => { - expect(specVerb(["review", join(dir, "nope.md")]).code).toBe(64); + it("exit 64 when the spec does not exist", async () => { + expect((await specVerb(["review", join(dir, "nope.md")])).code).toBe(64); }); - it("accepts flags before the positional path", () => { + it("accepts flags before the positional path", async () => { writeFileSync(path, SLICE); - expect(specVerb(["review", "--offline", path]).code).toBe(0); + expect((await specVerb(["review", "--offline", path])).code).toBe(0); }); - it("does NOT take --spec (that flag belongs to the launch path)", () => { + it("does NOT take --spec (that flag belongs to the launch path)", async () => { writeFileSync(path, SLICE); - expect(specVerb(["review", "--spec", path]).code).toBe(64); + expect((await specVerb(["review", "--spec", path])).code).toBe(64); }); - it("`validate` checks the frontmatter contract alone", () => { + it("`validate` checks the frontmatter contract alone", async () => { writeFileSync(path, SLICE); - expect(specVerb(["validate", path]).code).toBe(0); + expect((await specVerb(["validate", path])).code).toBe(0); writeFileSync(path, SLICE.replace("spec_id: spec-slice\n", "")); - const r = specVerb(["validate", path]); + const r = await specVerb(["validate", path]); expect(r.code).toBe(65); expect((json(r).errors as string[]).join(" ")).toMatch(/spec_id/); }); - it("is registered in SPINE_VERBS with the fields Verb requires", () => { + it("is registered in SPINE_VERBS with the fields Verb requires", async () => { const v = SPINE_VERBS.find((x) => x.name === "spec"); expect(v).toBeDefined(); expect(v!.summary).toBeTruthy(); diff --git a/packages/amico-run/vitest.config.ts b/packages/amico-run/vitest.config.ts new file mode 100644 index 00000000..f9eac61f --- /dev/null +++ b/packages/amico-run/vitest.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "vitest/config"; + +// The only reason this file exists is `setupFiles`: test/setup.ts installs the guard that stops +// any test from spawning a real (billed) model call or appending to the developer's real ledger. +// Both are failures that per-test discipline cannot prevent, because the risk is in the test +// someone writes next. +export default defineConfig({ + test: { + setupFiles: ["./test/setup.ts"], + // Matches the historical `--exclude '**/slow/**'` in the package script; kept here so the + // exclusion survives someone running `vitest` directly. + exclude: ["**/node_modules/**", "**/dist/**", "**/slow/**"], + }, +}); From 5ddaff6970c608f8c6f5a951e6cc34eb3295fd4a Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 13:02:44 -0400 Subject: [PATCH 23/27] =?UTF-8?q?feat(amico-run):=20`amico=20plan`=20?= =?UTF-8?q?=E2=80=94=20compile,=20derived=20status,=20advisory=20closure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes spec §10 steps 4 and 5. The loop is now playable end to end: author a spec, review it adversarially, compile it into a gated budgeted plan, read the plan's state, close the obligations the critics left. PLAN COMPILE (§4.2, corrected). Two orderings are load-bearing: - Validation comes AFTER stamping. plan.schema.json REQUIRES plan_hash, which is planHash({goal, steps}) computed here — so §4.1's prose order (validate, then hash) could only ever fail. - Refusals come BEFORE any write. Every refusal test asserts the plan file does NOT exist and no row was appended; an implementation that wrote and then refused would otherwise pass on exit code alone, which is the same defect shape as asserting a severity downgrade on the in-memory result. `tier` is GONE from the compile-time check and DISCLOSED as unchecked instead, alongside max_size_class. bounds.tier speaks the solvespec trust vocabulary (free|composed|vetted|hpc); a plan step's tier field is `model`, holding a model id. Comparing them is a category error, not a strictness choice — it is a first-launch refusal where LaunchFacts.tier actually exists. `device` joins under the now-exported DEVICE_ORDER so compile and the launch gate cannot drift, and an OMITTED bound refuses rather than default-allowing (warrant.test.ts's rule, which the compile side had no counterpart for). DERIVED STEP STATE (§4.4). Reachability is tested BEFORE unforgeability, because Rev 2's two covering tests were both negative and would have passed vacuously against an implementation where `passed` is unreachable. All five states are asserted reachable through the shipped write path. The lane filter is the guarantee that is actually enforceable, and it had zero coverage: a `simulated` or `replay` verdict with the right (plan_hash, step_id) leaves the step PENDING. Rev 1 instead appended a `todo` row and asserted it could not move a step — but a todo row carries no step identity at all, so no implementation could ever have honoured it. The test also states the TRUE property honestly: a hand-appended `user` verdict DOES move the step, because forging `passed` requires forging a gate verdict and that is the whole barrier. `disagree` gets a state. It had no clause in the derivation, so a step whose gate disagreed read `pending` — indistinguishable from never dispatched. It is `running`: one failed attempt while escalation continues, with `exhausted` as the terminal form. THE PLAN VERB. `status` renders remaining warrant time from the APPROVAL record; its fixture makes approval and suggested_ttl_s DISAGREE on purpose, so an implementation reading the field §4.6 forbids as a lifetime source fails rather than looking plausible. `advisory` refuses an id the plan never declared, because the declared list is the completion rule's denominator. There is no `plan todo` and the test asserts the PROPERTY, not the name: every plausible subcommand runs and the verdict/dispatch row count is unchanged. Rev 1 asserted `plan todo` was absent, which any unknown-subcommand-64 convention satisfies while `plan step --pass` would still exist. Two bugs found by these tests: - The compiled plan could not ROUND-TRIP. canonicalJson serialises undefined as null, so absent optional step fields wrote `"optional": null` and the artifact failed its own schema on re-read — an artifact a warrant is bound to. Steps are now built without absent keys, which also makes plan_hash a function of content rather than of how the object was constructed. - `positional` returned at the first non-flag argument, so TRAILING unknown flags went unvalidated in BOTH plan and the shipped `spec` verb: `spec review --bogus` silently ignored the typo. Fixed in both, with a test. Co-Authored-By: Claude Opus 5 --- packages/amico-run/src/plan_compile.ts | 476 +++++++++++++++++++ packages/amico-run/src/plan_state.ts | 194 ++++++++ packages/amico-run/src/plan_verb.ts | 310 ++++++++++++ packages/amico-run/src/spec_verb.ts | 13 +- packages/amico-run/src/verbs.ts | 15 +- packages/amico-run/test/plan_compile.test.ts | 373 +++++++++++++++ packages/amico-run/test/plan_state.test.ts | 255 ++++++++++ packages/amico-run/test/plan_verb.test.ts | 286 +++++++++++ packages/amico-run/test/spec_verb.test.ts | 9 + 9 files changed, 1926 insertions(+), 5 deletions(-) create mode 100644 packages/amico-run/src/plan_compile.ts create mode 100644 packages/amico-run/src/plan_state.ts create mode 100644 packages/amico-run/src/plan_verb.ts create mode 100644 packages/amico-run/test/plan_compile.test.ts create mode 100644 packages/amico-run/test/plan_state.test.ts create mode 100644 packages/amico-run/test/plan_verb.test.ts diff --git a/packages/amico-run/src/plan_compile.ts b/packages/amico-run/src/plan_compile.ts new file mode 100644 index 00000000..8ca17445 --- /dev/null +++ b/packages/amico-run/src/plan_compile.ts @@ -0,0 +1,476 @@ +// packages/amico-run/src/plan_compile.ts — `amico plan compile` (spec-20260728 §4). +// +// Spec -> planner subprocess -> stamped plan -> schema validation -> the §4.2 budget refusals -> +// only THEN a file on disk and one `plan_compiled` row. +// +// ORDERING IS LOAD-BEARING IN TWO PLACES: +// +// 1. Validation comes AFTER stamping. `plan.schema.json` REQUIRES `plan_hash`, `design_hash` and +// `schema_version`, none of which the planner produces — `plan_hash` is `planHash({goal, +// steps})`, computed here. Validating the planner's raw output against that schema could only +// ever fail. (§4.1's prose had validate-then-hash; it is not implementable.) +// +// 2. Refusals come BEFORE any write. A plan that violates its approved budget must not exist on +// disk, or the next `plan status` reads a plan nobody authorised. +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { canonicalJson, designHash, planHash, validate } from "@amicode/schema"; +import { runAgent, criticModel, resolveAgentBin, type AgentOutcome } from "./agent_spawn.js"; +import { parseFrontmatter } from "./frontmatter.js"; +import { + appendRecord, + readRecords, + TASK_TYPES, + type LedgerRecord, + type PlanCompiledRecord, + type SpecReviewRecord, +} from "./ledger.js"; +import { DEVICE_ORDER } from "./warrant.js"; +import type { DeviceAccess } from "./warrant.js"; + +/** Task types that consume a solve. §4.2 sums these against `budget.max_solves`. */ +const SOLVE_BEARING = new Set(["experiment-sim", "experiment-hw"]); + +/** Bounds that CANNOT be checked at compile time, disclosed rather than silently skipped. + * + * - `max_size_class` comes from `estimate.ts` resolving a SOLVESPEC, which does not exist yet. + * - `tier` is the same story and it took three revisions to see it. `bounds.tier` speaks the + * solvespec TRUST vocabulary (`free|composed|vetted|hpc` — solvespec.schema.json), while a + * plan step's tier field is `model`, holding a model id (fleet §5.1: "no `tier` alias; the + * feature is called tier dispatch, the field is called `model`"). A model id can never equal + * `hpc`, so comparing them — by order OR by equality — is a category error, not a strictness + * choice. It is a first-launch refusal, where `LaunchFacts.tier` actually exists. + * + * Disclosure is the point: "we checked everything we could and here is what we could not" is a + * different claim from "checked", and only one of them is true. */ +export const UNCHECKED_BOUNDS = ["max_size_class", "tier"] as const; + +export interface CompiledStep { + id: string; + model: string; + task_type: string; + variant?: string; + gates?: string[]; + needs?: string[]; + optional?: boolean; + permissions?: { device?: DeviceAccess }; +} + +export interface CompileRefusal { + ok: false; + exit_code: 64 | 65; + /** One line per exceeded bound. Each bound names ITSELF and its margin, matching the + * convention `warrant.test.ts` established for the launch gate — a caller fixing three + * problems should not have to re-run three times to discover them. */ + errors: string[]; +} + +export interface CompileSuccess { + ok: true; + plan_hash: string; + design_hash: string; + spec_id: string; + plan_path: string; + step_count: number; + advisory_count: number; + suggested_ttl_s: number; + allow_unreviewed: boolean; + unchecked: readonly string[]; + compiled_by?: { model: string; variant: string }; +} + +export type CompileResult = CompileSuccess | CompileRefusal; + +const refuse = (exit_code: 64 | 65, ...errors: string[]): CompileRefusal => ({ ok: false, exit_code, errors }); + +/** One hour per step, floor 1h, ceiling 24h — a RECOMMENDATION only. + * + * `plan compile` records it; `amico ledger approve` reads it to default `--expires-in` and + * remains the sole writer of `expires_at`. Compile must not own a warrant's lifetime: the TTL + * sits beside `issued_by` on the approval, which is the human authorization act, and under + * `--recompile` a compile-owned TTL would silently re-set how long that authorization lasts. */ +export function suggestedTtlFor(stepCount: number): number { + return Math.min(Math.max(stepCount, 1) * 3600, 24 * 3600); +} + +/** The §4.2 budget refusal. Pure, so the whole refusal matrix is testable without a subprocess. */ +export function checkBudget(steps: CompiledStep[], budget: Record | undefined): string[] { + const errors: string[] = []; + + // A launch-shaped plan with no budget at all is not "unbounded", it is unapproved. + const solveSteps = steps.filter((s) => SOLVE_BEARING.has(s.task_type)); + const deviceSteps = steps.filter((s) => (s.permissions?.device ?? "none") !== "none"); + if (budget === undefined) { + if (solveSteps.length > 0 || deviceSteps.length > 0) { + errors.push( + `the spec declares no budget, but the compiled plan demands ${[ + solveSteps.length > 0 ? `${solveSteps.length} solve(s)` : undefined, + deviceSteps.length > 0 ? "device access" : undefined, + ] + .filter(Boolean) + .join(" and ")} — add a budget to the spec, or restructure the plan`, + ); + } + return errors; + } + + // device: join by the EXPORTED DEVICE_ORDER, so compile and the launch gate cannot drift. + const demandedDevice = steps.reduce((acc, s) => { + const d = s.permissions?.device ?? "none"; + return DEVICE_ORDER[d] > DEVICE_ORDER[acc] ? d : acc; + }, "none"); + if (demandedDevice !== "none") { + const authorised = budget.device as DeviceAccess | undefined; + if (authorised === undefined) { + // NEVER default-allow. The launch gate refuses a bound it was not granted + // (warrant.test.ts's established rule); the compile side needs the same direction, or a + // budget could authorise device access by saying nothing about it. + errors.push( + `device: the plan demands "${demandedDevice}" (step ${steps.find((s) => (s.permissions?.device ?? "none") === demandedDevice)?.id}) but the budget does not declare device at all — an omitted bound is not permission`, + ); + } else if (DEVICE_ORDER[demandedDevice] > DEVICE_ORDER[authorised]) { + errors.push(`device: the plan demands "${demandedDevice}" but the budget authorises "${authorised}"`); + } + } + + // solves: SUM over solve-bearing steps, with the margin named. + if (solveSteps.length > 0) { + const authorised = budget.max_solves; + if (typeof authorised !== "number") { + errors.push( + `max_solves: the plan has ${solveSteps.length} solve-bearing step(s) but the budget does not declare max_solves — an omitted bound is not permission`, + ); + } else if (solveSteps.length > authorised) { + errors.push( + `max_solves: the plan needs ${solveSteps.length} solve(s) but the budget authorises ${authorised} — over by ${solveSteps.length - authorised}`, + ); + } + } + return errors; +} + +/** Steps must declare what the budget check reads. An absent field is a REFUSAL: treating it as + * "no demand" is how §0.1's `max_solves` counter sat inert while reading as enforced. */ +export function checkStepShape(steps: unknown): { steps: CompiledStep[] } | { errors: string[] } { + if (!Array.isArray(steps) || steps.length === 0) return { errors: ["the planner returned no steps"] }; + const errors: string[] = []; + const out: CompiledStep[] = []; + const seen = new Set(); + for (const [i, raw] of steps.entries()) { + if (!raw || typeof raw !== "object") { + errors.push(`step ${i}: not an object`); + continue; + } + const s = raw as Record; + const id = typeof s.id === "string" ? s.id : ""; + if (id === "") errors.push(`step ${i}: missing id`); + else if (seen.has(id)) errors.push(`step ${i}: duplicate id "${id}" — derivation keys on it, so ids must be unique`); + seen.add(id); + if (typeof s.model !== "string" || !/^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/.test(s.model)) + errors.push(`step "${id}": model must be a provider/model-id — without it no tier demand can be determined`); + if (typeof s.task_type !== "string" || !(TASK_TYPES as readonly string[]).includes(s.task_type)) + errors.push(`step "${id}": task_type must be one of (${TASK_TYPES.join(", ")}) — it decides whether the step consumes a solve`); + const device = (s.permissions as { device?: unknown } | undefined)?.device; + if (device !== undefined && !Object.prototype.hasOwnProperty.call(DEVICE_ORDER, String(device))) + errors.push(`step "${id}": permissions.device must be one of (none, ro, rw)`); + // OMIT absent keys rather than setting them undefined. `canonicalJson` serialises undefined + // as `null`, so a step built with `optional: undefined` writes `"optional": null` — which + // fails the plan schema on re-read (`must be boolean`) and makes the artifact unable to + // round-trip. It also means a step that omits a field and one that sets it undefined would + // hash identically only by accident; compacting here makes plan_hash a function of the + // content rather than of how the object happened to be constructed. + out.push({ + id, + model: String(s.model), + task_type: String(s.task_type), + ...(typeof s.variant === "string" ? { variant: s.variant } : {}), + ...(Array.isArray(s.gates) ? { gates: s.gates.map(String) } : {}), + ...(Array.isArray(s.needs) ? { needs: s.needs.map(String) } : {}), + ...(s.optional === true ? { optional: true as const } : {}), + ...(device === undefined ? {} : { permissions: { device: device as DeviceAccess } }), + }); + } + // Fleet §8: a below-frontier step with no gate is unverified work by a cheaper model. + for (const s of out) { + if (!isFrontier(s.model) && (s.gates === undefined || s.gates.length === 0)) + errors.push(`step "${s.id}": model ${s.model} is below the frontier tier and declares no gates — unverified cheap work is refused here and again by the harness at dispatch`); + } + return errors.length > 0 ? { errors } : { steps: out }; +} + +/** Frontier = opus-class. Deliberately a substring test rather than an allowlist: a new opus + * point-release must not silently become "below frontier" and start requiring gates it does + * not need, and the failure direction of a wrong guess here is a spurious refusal (loud) rather + * than ungated cheap work (silent). */ +function isFrontier(model: string): boolean { + return /opus/i.test(model); +} + +/** The latest review for a design, if any. */ +export function latestReview(records: readonly LedgerRecord[], designHash: string): SpecReviewRecord | undefined { + const rows = records.filter((r): r is SpecReviewRecord => r.type === "spec_review" && r.design_hash === designHash); + return rows.length === 0 ? undefined : rows[rows.length - 1]; +} + +export interface CompileOptions { + /** `--recompile`: required when a plan for this design has already been approved. */ + recompile?: boolean; + /** `--allow-unreviewed`: compile from an `approved-mechanical` / `degraded` review. */ + allowUnreviewed?: boolean; + /** Where the plan note is written (the vault's `plans/` folder). */ + plansDir: string; + /** Test seam for the planner subprocess. */ + runPlanner?: (specText: string) => Promise; + now?: () => string; + env?: NodeJS.ProcessEnv; + append?: boolean; + /** Injected so the approval lookup is testable without a real ledger. */ + records?: readonly LedgerRecord[]; + warn?: (msg: string) => void; +} + +export async function compilePlan(specPath: string, raw: string, opts: CompileOptions): Promise { + const fm = parseFrontmatter(raw); + if (!fm.ok) return refuse(65, `the spec's frontmatter could not be read: ${fm.error}`); + const spec = fm.data; + const specValid = validate(spec, "spec"); + if (!specValid.ok) return refuse(65, ...specValid.errors.map((e) => `spec: ${e}`)); + + const spec_id = String(spec.spec_id); + const design_hash = requireDesignHash(spec); + const records = opts.records ?? safeRecords(); + + // ── the review precondition ── + // A spec whose review found blocking findings is not compilable at all; one that was only + // mechanically reviewed is compilable with an explicit acknowledgement, which the row records. + const review = latestReview(records, design_hash); + let allow_unreviewed = false; + if (review === undefined) { + if (!opts.allowUnreviewed) + return refuse(65, `no review on record for this spec (design ${design_hash.slice(0, 12)}) — run \`amico spec review\` first, or pass --allow-unreviewed to compile anyway`); + allow_unreviewed = true; + } else if (review.review_verdict === "blocking" || review.review_verdict === "exhausted") { + return refuse(65, `the latest review of this spec is \`${review.review_verdict}\` with ${review.blocking_count} blocking finding(s) — revise the spec; --allow-unreviewed does NOT override a blocking review`); + } else if (review.review_verdict !== "approved") { + if (!opts.allowUnreviewed) + return refuse(65, `the latest review is \`${review.review_verdict}\` — no critic actually reviewed this spec. Pass --allow-unreviewed to compile anyway (it will be recorded)`); + allow_unreviewed = true; + } + + // ── recompilation (§4.6) ── + // Recompiling mints a new plan_hash, invalidating a live warrant. That is correct but it must + // be LOUD, because the failure otherwise surfaces later as a bare launch denial. + const priorApproved = records.some( + (r) => r.type === "approval" && records.some((p) => p.type === "plan_compiled" && p.design_hash === design_hash && p.plan_hash === (r as { plan_hash: string }).plan_hash), + ); + if (priorApproved && !opts.recompile) + return refuse(65, `a plan for this spec has already been APPROVED. Recompiling mints a new plan_hash and invalidates that approval — pass --recompile if that is what you want`); + if (priorApproved && opts.recompile) + (opts.warn ?? ((m: string) => process.stderr.write(`${m}\n`)))( + `amico plan compile: recompiling invalidates the existing approval for this spec — the launch gate will say "the plan was recompiled; re-approve" until you run \`amico ledger approve\` on the new plan_hash`, + ); + + // ── the planner call ── + const planner = opts.runPlanner ?? defaultPlanner(specPath, opts.env); + if (planner === undefined) + return refuse(64, `no agent binary available to compile a plan (set $AMICO_CRITIC_BIN, or install the agent CLI on PATH). Nothing was written.`); + const out = await planner(raw); + if (out.status !== "ran") + return refuse(64, `the planner produced no usable plan: ${out.reason ?? "unknown"}. Nothing was written.`); + + const payload = out.payload ?? {}; + const shaped = checkStepShape(payload.steps); + if ("errors" in shaped) return refuse(65, ...shaped.errors); + const goal = typeof payload.goal === "string" && payload.goal.trim() !== "" ? payload.goal.trim() : undefined; + if (goal === undefined) return refuse(65, "the planner returned no goal"); + + // ── the budget refusal, BEFORE anything is written ── + const budgetErrors = checkBudget(shaped.steps, spec.budget as Record | undefined); + // A non-launch-shaped spec that compiled to a solve-bearing step is mislabelled, and the + // mislabelling silently disables two blocking tier-1 lenses (§2.2) and makes this whole check + // a no-op. Naming the task_type is more useful than naming the budget. + const launchShaped = ["experiment-sim", "experiment-hw", "author-script"].includes(String(spec.task_type)); + if (!launchShaped && shaped.steps.some((s) => SOLVE_BEARING.has(s.task_type))) + budgetErrors.push( + `task_type: the spec is \`${spec.task_type}\` (not launch-shaped) but compiled to a solve-bearing step — relabel the spec, or the budget and baseline lenses stay switched off for work that spends`, + ); + if (budgetErrors.length > 0) return refuse(65, ...budgetErrors); + + // ── stamp, THEN validate ── + const nowIso = (opts.now ?? (() => new Date().toISOString()))(); + const plan_hash = planHash({ goal, steps: shaped.steps as unknown as Record[] }); + const advisories = advisoriesFrom(spec); + const suggested_ttl_s = suggestedTtlFor(shaped.steps.length); + const plan_id = `plan-${nowIso.replace(/[-:]/g, "").replace(/\..*$/, "").replace("T", "-")}-${slug(goal)}`; + const compiled_by = out.model ? { model: out.model, variant: out.variant ?? "default" } : undefined; + + const planObject: Record = { + type: "plan", + schema_version: "1", + plan_id, + goal, + max_replans: 3, + plan_hash, + design_hash, + compiled_at: nowIso, + ...(compiled_by ? { compiled_by } : {}), + suggested_ttl_s, + spec: spec_id, + steps: shaped.steps, + ...(advisories.length > 0 ? { advisories } : {}), + }; + const planValid = validate(planObject, "plan"); + if (!planValid.ok) return refuse(65, ...planValid.errors.map((e) => `compiled plan: ${e}`)); + + // ── write, then record ── + const plan_path = join(opts.plansDir, `${plan_id}.md`); + try { + writeFileSync(plan_path, renderPlanNote(planObject, shaped.steps, advisories)); + } catch (e) { + return refuse(64, `could not write the compiled plan to ${plan_path}: ${(e as Error).message}`); + } + + const rec: PlanCompiledRecord = { + type: "plan_compiled", + ts: nowIso, + plan_hash, + spec_id, + design_hash, + step_count: shaped.steps.length, + advisory_count: advisories.length, + suggested_ttl_s, + source: "user", + ...(compiled_by ? { compiled_by } : {}), + ...(allow_unreviewed ? { allow_unreviewed: true } : {}), + }; + if (opts.append !== false) appendRecord(rec); + + return { + ok: true, + plan_hash, + design_hash, + spec_id, + plan_path, + step_count: shaped.steps.length, + advisory_count: advisories.length, + suggested_ttl_s, + allow_unreviewed, + unchecked: UNCHECKED_BOUNDS, + compiled_by, + }; +} + +/** Surviving advisories become the plan's obligations: a plan cannot reach `complete` while one + * is open, which is where tier-2 critics get their teeth (§3.6). */ +function advisoriesFrom(spec: Record): Array> { + const review = spec.review; + if (!review || typeof review !== "object") return []; + const raw = (review as { advisories?: unknown }).advisories; + if (!Array.isArray(raw)) return []; + return raw + .filter((a): a is Record => !!a && typeof a === "object") + .map((a, i) => ({ + id: typeof a.id === "string" && a.id !== "" ? a.id : `adv-${i + 1}`, + lens: typeof a.lens === "string" ? a.lens : undefined, + claim: typeof a.claim === "string" ? a.claim : undefined, + remedy: typeof a.remedy === "string" ? a.remedy : undefined, + round: typeof a.round === "number" ? a.round : undefined, + })) + .map((a) => Object.fromEntries(Object.entries(a).filter(([, v]) => v !== undefined))); +} + +/** The design_hash this plan is compiled from. + * + * Prefer the one the review stamped, so plan and review provably refer to the same decision + * surface. Recompute when there is none: a never-reviewed spec still HAS a decision surface, and + * the hash is a pure function of it — refusing here would block `--allow-unreviewed` entirely. */ +function requireDesignHash(spec: Record): string { + const review = spec.review as { design_hash?: unknown } | undefined; + if (review && typeof review.design_hash === "string" && /^[0-9a-f]{64}$/.test(review.design_hash)) + return review.design_hash; + return designHash(spec); +} + +function slug(goal: string): string { + return goal + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 40) || "plan"; +} + +/** The plan NOTE: typed frontmatter plus a readable body. + * + * Hand-editing a compiled plan is a lint failure — the compiler is the only writer — so the body + * says so where someone about to edit it will see it. */ +function renderPlanNote( + plan: Record, + steps: CompiledStep[], + advisories: Array>, +): string { + // JSON-encode EVERY value. `String("1")` writes a bare `1`, which YAML reads back as the + // NUMBER 1 — so the plan this function wrote would fail `validate(…, "plan")` on re-read + // (schema_version is the string enum ["1"]). The round-trip is the whole point: a warrant is + // bound to plan_hash, and the file is what a human and `plan status` both read. JSON is a YAML + // subset, so this is unambiguous for scalars, lists and maps alike. + const fmLines = Object.entries(plan) + .filter(([k]) => k !== "steps" && k !== "advisories") + .map(([k, v]) => `${k}: ${JSON.stringify(v)}`); + const stepLines = ["steps:", ...steps.map((s) => ` - ${canonicalJson(s as never)}`)]; + const advLines = advisories.length === 0 ? [] : ["advisories:", ...advisories.map((a) => ` - ${canonicalJson(a as never)}`)]; + const body = [ + `# ${plan.goal}`, + "", + "> COMPILED ARTIFACT — do not hand-edit. `plan_hash` is `sha256(canonicalJson({goal, steps}))`,", + "> and an approved warrant is bound to it; editing this file silently detaches the two. Change", + "> the spec and re-run `amico plan compile --recompile`.", + "", + "## Steps", + "", + ...steps.map( + (s) => + `- **${s.id}** — \`${s.task_type}\` on \`${s.model}\`${s.optional ? " *(optional)*" : ""}` + + `${s.needs?.length ? `, after ${s.needs.join(", ")}` : ""}` + + `${s.gates?.length ? `, gated by ${s.gates.join(", ")}` : ""}` + + `${s.permissions?.device && s.permissions.device !== "none" ? `, device ${s.permissions.device}` : ""}`, + ), + "", + ...(advisories.length === 0 + ? ["## Advisories", "", "None surviving. (A plan cannot reach `complete` while an advisory is open.)"] + : [ + "## Advisories — obligations, not suggestions", + "", + "This plan cannot reach `complete` while any of these is open. Close one with", + "`amico plan advisory --state fixed|waived --reason |obsolete`.", + "", + ...advisories.map((a) => `- **${a.id}** (${a.lens ?? "critic"}) — ${a.claim ?? ""}${a.remedy ? ` → ${a.remedy}` : ""}`), + ]), + "", + ].join("\n"); + return `---\n${[...fmLines, ...stepLines, ...advLines].join("\n")}\n---\n\n${body}`; +} + +function safeRecords(): readonly LedgerRecord[] { + try { + return readRecords(); + } catch { + return []; + } +} + +/** The real planner: the §3.7 mechanism with `--agent planner`. */ +function defaultPlanner(specPath: string, env?: NodeJS.ProcessEnv): ((specText: string) => Promise) | undefined { + const e = env ?? process.env; + const bin = resolveAgentBin(e); + if (bin === undefined) return undefined; + return (specText: string) => + runAgent({ + bin, + agent: "planner", + model: criticModel(e), + env: e, + prompt: "Compile the spec in your working directory into a plan. Reply with the JSON object only.", + specText, + specFilename: specPath.split("/").pop() ?? "spec.md", + }); +} diff --git a/packages/amico-run/src/plan_state.ts b/packages/amico-run/src/plan_state.ts new file mode 100644 index 00000000..bf653510 --- /dev/null +++ b/packages/amico-run/src/plan_state.ts @@ -0,0 +1,194 @@ +// packages/amico-run/src/plan_state.ts — derived step state and the completion rule +// (spec-20260728 §4.4, §4.5). +// +// THERE IS NO WRITE PATH FOR STEP STATE, AND THAT IS THE WHOLE DESIGN. +// +// Rev 1 of the deliberation spec said "the agent cannot move its own step todos" and tested a +// verb. That was unenforceable: `amico ledger append` accepts any schema-valid record from any +// caller (ledger_verb.ts has no per-kind authorization), the product agent holds unrestricted +// bash, and verbs.ts carries no actor. A policed rule over a writable path is an honor system. +// +// So step state is a PURE FUNCTION of rows the gates append. An agent cannot forge `passed` +// without forging a gate verdict — the same barrier that already protects every fidelity claim in +// the system. What it CAN do is append a `source: user` verdict row directly, and this module +// does not pretend otherwise; see `deriveStepStates`'s lane filter for the guarantee that IS real. +// +// Advisory todos DO have a write path (`amico plan advisory`), because they are genuine judgment: +// `fixed` / `waived ` / `obsolete` are decisions a human or agent makes, not facts a gate +// establishes. Merging the two kinds would let "all todos done" mean "the gates passed and we +// ignored every critic". +import type { LedgerRecord } from "./ledger.js"; + +/** Plan-scoped names. Deliberately NOT the shipped session-scoped FLEET_STATES (`settled`, + * `blocked`) — Rev 1 reused those for the same triggering event, putting two authorities over + * one name. A session may reach `settled` while its plan is `active` (the user walked away). */ +export type StepState = "pending" | "running" | "passed" | "failed" | "skipped"; +export type PlanState = "active" | "complete" | "stalled"; + +/** A todo is OPEN by the absence of a row. Any of the three closures ends it. */ +export type AdvisoryState = "open" | "fixed" | "waived" | "obsolete"; + +export interface StepView { + id: string; + state: StepState; + optional?: boolean; + /** Set when a `bypassed` row names a step the plan did NOT mark optional. That is a derivation + * error, not a skip: the two halves of the `skipped` producer must agree. */ + error?: string; +} + +export interface AdvisoryView { + id: string; + state: AdvisoryState; + reason?: string; + lens?: string; + claim?: string; +} + +export interface PlanView { + plan_hash: string; + state: PlanState; + steps: StepView[]; + advisories: AdvisoryView[]; + open_advisories: number; + /** Why the plan is not complete, in the order a reader should act on. */ + blockers: string[]; +} + +type VerdictRow = Extract; +type DispatchRow = Extract; +type TodoRow = Extract; + +/** The lane filter. THIS is the anti-fabrication guarantee that is actually enforceable. + * + * `source` separates real work from replayed and simulated work, and the derivation admits only + * `user`. A Prova gym run that exercises the whole loop appends `simulated` verdicts; without + * this filter those would mark real plan steps `passed`, and the lane separation would be + * defeated inside the very derivation it exists to protect. + * + * An ABSENT source counts as user: the field is optional and pre-existing rows predate it, so + * excluding them would silently rewrite history as un-progressed. */ +const inLane = (r: { source?: string }): boolean => r.source === undefined || r.source === "user"; + +/** Derive each step's state from the ledger. Keyed on (plan_hash, step_id) — NEVER step_id + * alone. The ledger is append-only, so after a recompile the old plan's rows remain; keying on + * the id alone would let them alias onto identically-named new steps and read a fresh plan as + * already complete. */ +export function deriveStepStates( + records: readonly LedgerRecord[], + planHash: string, + steps: readonly { id: string; optional?: boolean }[], +): StepView[] { + const verdicts = records.filter( + (r): r is VerdictRow => r.type === "verdict" && r.plan_hash === planHash && typeof r.step_id === "string" && inLane(r), + ); + const dispatches = records.filter( + (r): r is DispatchRow => r.type === "dispatch" && r.plan_hash === planHash && typeof r.step_id === "string" && inLane(r), + ); + + return steps.map((step) => { + const mine = verdicts.filter((v) => v.step_id === step.id); + const bypassed = mine.some((v) => v.verdict === "bypassed"); + const agreed = mine.some((v) => v.verdict === "agree"); + const exhausted = mine.some((v) => v.verdict === "exhausted"); + const disagreed = mine.some((v) => v.verdict === "disagree"); + const dispatched = dispatches.some((d) => d.step_id === step.id); + + // `skipped` needs BOTH halves: the plan's `optional: true` permission AND a terminal + // `bypassed` row. A bypassed row against a non-optional step is a contradiction between the + // two, so it is surfaced rather than silently honoured or silently ignored. + if (bypassed) { + if (step.optional === true) return { id: step.id, state: "skipped", optional: true }; + return { + id: step.id, + state: exhausted ? "failed" : agreed ? "passed" : dispatched ? "running" : "pending", + error: `a \`bypassed\` verdict names step "${step.id}", but the compiled plan does not mark it \`optional: true\` — the bypass was NOT honoured`, + }; + } + if (agreed) return { id: step.id, state: "passed", ...(step.optional ? { optional: true } : {}) }; + if (exhausted) return { id: step.id, state: "failed", ...(step.optional ? { optional: true } : {}) }; + // `disagree` is NOT terminal — it is one gate attempt that failed while escalation continues + // (`exhausted` is the terminal form, failure at the top reachable rung). But it IS evidence + // the step is under way, and reading it as `pending` made a disagreeing step indistinguishable + // from one never dispatched, which was a real gap in the derivation. + if (dispatched || disagreed) return { id: step.id, state: "running", ...(step.optional ? { optional: true } : {}) }; + return { id: step.id, state: "pending", ...(step.optional ? { optional: true } : {}) }; + }); +} + +/** Advisory state from `todo` rows: last-ts-wins per id, and `open` is the ABSENCE of a row. + * + * `fixed → obsolete` is legal (the fix turned out to be moot), so no transition is forbidden — + * the record keeps every step of the history and the view shows the latest. */ +export function deriveAdvisories( + records: readonly LedgerRecord[], + planHash: string, + declared: readonly { id: string; lens?: string; claim?: string }[], +): AdvisoryView[] { + const rows = records + .filter((r): r is TodoRow => r.type === "todo" && r.plan_hash === planHash && inLane(r)) + .slice() + .sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0)); + const latest = new Map(); + for (const r of rows) latest.set(r.id, r); // last-ts-wins + return declared.map((a) => { + const row = latest.get(a.id); + return { + id: a.id, + state: (row?.state ?? "open") as AdvisoryState, + ...(row?.reason ? { reason: row.reason } : {}), + ...(a.lens ? { lens: a.lens } : {}), + ...(a.claim ? { claim: a.claim } : {}), + }; + }); +} + +export interface PlanShape { + plan_hash: string; + steps: readonly { id: string; optional?: boolean }[]; + advisories?: readonly { id: string; lens?: string; claim?: string }[]; + max_replans?: number; +} + +/** The completion rule (§4.5). + * + * complete ⟺ every step ∈ {passed, skipped} AND every advisory closed + * stalled ⟸ gate exhaustion on any step, or the replan budget exhausted + * + * The advisory conjunct is where tier-2 critics get their teeth. Critics shape the work and gate + * COMPLETION, never START — so a review never blocks a launch, but an ignored advisory keeps the + * plan from ever being finished. Dropping that conjunct would make "all todos done" mean "the + * gates passed and we ignored every critic", which is the exact failure the two-kind split + * exists to prevent. */ +export function derivePlanState( + records: readonly LedgerRecord[], + plan: PlanShape, + opts: { replans?: number } = {}, +): PlanView { + const steps = deriveStepStates(records, plan.plan_hash, plan.steps); + const advisories = deriveAdvisories(records, plan.plan_hash, plan.advisories ?? []); + const open = advisories.filter((a) => a.state === "open"); + const blockers: string[] = []; + + const failed = steps.filter((s) => s.state === "failed"); + const replansExhausted = opts.replans !== undefined && plan.max_replans !== undefined && opts.replans > plan.max_replans; + + let state: PlanState; + if (failed.length > 0 || replansExhausted) { + state = "stalled"; + for (const s of failed) blockers.push(`step "${s.id}" failed (gate exhaustion) — replan or revise the spec`); + if (replansExhausted) blockers.push(`the replan budget (${plan.max_replans}) is exhausted — this needs a human decision`); + } else if (steps.every((s) => s.state === "passed" || s.state === "skipped") && open.length === 0) { + state = "complete"; + } else { + state = "active"; + const notDone = steps.filter((s) => s.state !== "passed" && s.state !== "skipped"); + if (notDone.length > 0) + blockers.push(`${notDone.length} step(s) not finished: ${notDone.map((s) => `${s.id} (${s.state})`).join(", ")}`); + for (const a of open) + blockers.push(`advisory "${a.id}" is open — close it with \`amico plan advisory ${a.id} --state fixed|waived --reason |obsolete\``); + } + for (const s of steps) if (s.error) blockers.push(s.error); + + return { plan_hash: plan.plan_hash, state, steps, advisories, open_advisories: open.length, blockers }; +} diff --git a/packages/amico-run/src/plan_verb.ts b/packages/amico-run/src/plan_verb.ts new file mode 100644 index 00000000..6008f838 --- /dev/null +++ b/packages/amico-run/src/plan_verb.ts @@ -0,0 +1,310 @@ +// packages/amico-run/src/plan_verb.ts — the `amico plan` verb (spec-20260728 §4). +// +// amico plan compile [--recompile] [--allow-unreviewed] [--plans-dir ] [--json] +// amico plan status [] [--json] +// amico plan advisory --state fixed|waived|obsolete [--reason ] [--plan ] +// +// THERE IS NO `amico plan todo` AND NO WRITE PATH FOR STEP STATE. Step state is derived from gate +// verdicts (plan_state.ts); the only thing this verb writes is an ADVISORY transition, which is +// genuine judgment rather than a fact a gate established. +// +// Conventions mirror `spec_verb.ts` deliberately: the path is POSITIONAL (`--spec` belongs to the +// launch path), unknown flags are usage errors rather than being ignored, and the outcome rides +// the JSON payload as well as the exit code because the MCP facade discards `VerbResult.code`. +import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { parseFrontmatter } from "./frontmatter.js"; +import { appendRecord, readRecords, type LedgerRecord, type TodoRecord } from "./ledger.js"; +import { compilePlan, type CompileResult } from "./plan_compile.js"; +import { derivePlanState, type PlanShape } from "./plan_state.js"; +import { resolveMountStack } from "./mounts.js"; +import type { VerbResult } from "./verbs.js"; + +const USAGE = [ + "amico plan compile [--recompile] [--allow-unreviewed] [--plans-dir ] [--json]", + "amico plan status [] [--json]", + "amico plan advisory --state fixed|waived|obsolete [--reason ] [--plan ]", + "", + "There is no `plan todo` and no way to write step state: it is DERIVED from gate verdicts,", + "so `passed` cannot be claimed without a gate having agreed.", +].join("\n"); + +const usageError = (error: string): VerbResult => ({ json: { verb: "plan", ok: false, error, usage: USAGE }, code: 64 }); + +const ADVISORY_STATES = new Set(["fixed", "waived", "obsolete"]); + +const KNOWN_FLAGS = new Set(["--recompile", "--allow-unreviewed", "--plans-dir", "--json", "--state", "--reason", "--plan"]); +const VALUED_FLAGS = new Set(["--plans-dir", "--state", "--reason", "--plan"]); + +function flagValue(argv: string[], name: string): string | undefined { + const i = argv.indexOf(name); + if (i >= 0 && i + 1 < argv.length) return argv[i + 1]; + const eq = argv.find((a) => a.startsWith(`${name}=`)); + return eq ? eq.slice(name.length + 1) : undefined; +} + +/** Validate EVERY flag, then return the first positional. + * + * Scanning the whole argv matters: returning at the first non-flag argument would leave TRAILING + * flags unvalidated, so `plan compile --forcefully` would silently ignore the typo and + * behave as though the caller had asked for nothing. Refusing loudly is the point of having a + * known-flag set at all. */ +function positional(argv: string[]): { value?: string } | { error: string } { + let value: string | undefined; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith("-")) { + const name = a.includes("=") ? a.slice(0, a.indexOf("=")) : a; + if (!KNOWN_FLAGS.has(name)) return { error: `unknown flag ${name}` }; + if (VALUED_FLAGS.has(name) && !a.includes("=")) i++; // consume its value + continue; + } + if (value === undefined) value = a; + } + return value === undefined ? {} : { value }; +} + +/** The vault's `plans/` folder — the folder `amico-vault` already defines for `type: plan`. + * First WRITABLE mount in stack order; a read-only mount is never written to. */ +function defaultPlansDir(env: NodeJS.ProcessEnv): string | undefined { + const override = env.AMICO_PLANS_DIR; + if (override !== undefined && override.trim() !== "") return override; + const writable = resolveMountStack().mounts.find((m) => m.writable); + return writable ? join(writable.path, "plans") : undefined; +} + +async function compile(argv: string[], ctx: PlanVerbCtx): Promise { + const pos = positional(argv); + if ("error" in pos) return usageError(pos.error); + if (pos.value === undefined) return usageError("a spec path is required (positional, not --spec)"); + const abs = resolve(pos.value); + if (!existsSync(abs)) return usageError(`spec not found: ${abs}`); + + const plansDir = flagValue(argv, "--plans-dir") ?? ctx.plansDir ?? defaultPlansDir(ctx.env ?? process.env); + if (plansDir === undefined) + return usageError("no writable vault mount for plans/ — pass --plans-dir, or set $AMICO_PLANS_DIR"); + if (!existsSync(plansDir)) return usageError(`plans directory does not exist: ${plansDir}`); + + let r: CompileResult; + try { + r = await compilePlan(abs, (ctx.readFile ?? readFileSync)(abs, "utf8") as string, { + plansDir, + recompile: argv.includes("--recompile"), + allowUnreviewed: argv.includes("--allow-unreviewed"), + runPlanner: ctx.runPlanner, + records: ctx.records, + env: ctx.env, + }); + } catch (e) { + return { json: { verb: "plan", subcommand: "compile", ok: false, error: (e as Error).message }, code: 64 }; + } + + if (!r.ok) { + return { + json: { verb: "plan", subcommand: "compile", ok: false, exit_code: r.exit_code, errors: r.errors }, + code: r.exit_code, + }; + } + return { + json: { + verb: "plan", + subcommand: "compile", + ok: true, + plan_hash: r.plan_hash, + design_hash: r.design_hash, + spec_id: r.spec_id, + plan_path: r.plan_path, + step_count: r.step_count, + advisory_count: r.advisory_count, + suggested_ttl_s: r.suggested_ttl_s, + allow_unreviewed: r.allow_unreviewed, + // Never silently: the caller is told which bounds compile could not check, so "compiled" + // does not read as "fully budget-checked". + unchecked: r.unchecked, + compiled_by: r.compiled_by, + next: `amico ledger approve --plan ${r.plan_hash} --expires-in ${r.suggested_ttl_s}s`, + }, + code: 0, + }; +} + +/** Find a compiled plan note by plan_hash. */ +function loadPlan(plansDir: string, planHash: string): PlanShape | undefined { + let entries: string[]; + try { + entries = readdirSync(plansDir); + } catch { + return undefined; + } + for (const f of entries) { + if (!f.endsWith(".md")) continue; + let fm; + try { + fm = parseFrontmatter(readFileSync(join(plansDir, f), "utf8")); + } catch { + continue; + } + if (!fm.ok || fm.data.plan_hash !== planHash) continue; + const steps = Array.isArray(fm.data.steps) ? (fm.data.steps as Record[]) : []; + const advisories = Array.isArray(fm.data.advisories) ? (fm.data.advisories as Record[]) : []; + return { + plan_hash: planHash, + steps: steps.map((s) => ({ id: String(s.id), optional: s.optional === true ? true : undefined })), + advisories: advisories.map((a) => ({ + id: String(a.id), + lens: typeof a.lens === "string" ? a.lens : undefined, + claim: typeof a.claim === "string" ? a.claim : undefined, + })), + max_replans: typeof fm.data.max_replans === "number" ? fm.data.max_replans : undefined, + }; + } + return undefined; +} + +function status(argv: string[], ctx: PlanVerbCtx): VerbResult { + const pos = positional(argv); + if ("error" in pos) return usageError(pos.error); + const records = ctx.records ?? safeRecords(); + const plansDir = flagValue(argv, "--plans-dir") ?? ctx.plansDir ?? defaultPlansDir(ctx.env ?? process.env); + + // Default to the most recently compiled plan: the common case is "what is the state of the + // thing I just made", and making the hash mandatory would mean copying it around by hand. + const compiled = records.filter((r): r is Extract => r.type === "plan_compiled"); + const planHash = pos.value ?? compiled[compiled.length - 1]?.plan_hash; + if (planHash === undefined) + return { json: { verb: "plan", subcommand: "status", ok: true, plans: [], note: "no plan has been compiled yet" }, code: 0 }; + + const row = compiled.filter((r) => r.plan_hash === planHash).pop(); + const shape = plansDir ? loadPlan(plansDir, planHash) : undefined; + if (shape === undefined) { + // A clean empty answer, not a crash: the ledger may know about a plan whose note was moved + // or whose vault is not mounted here, and that is worth SAYING rather than throwing. + return { + json: { + verb: "plan", + subcommand: "status", + ok: true, + plan_hash: planHash, + known_to_ledger: row !== undefined, + note: row + ? `the ledger records this plan (${row.step_count} steps) but its note was not found under ${plansDir ?? "(no plans dir)"} — step state cannot be derived without the compiled steps` + : `no plan with hash ${planHash} is known`, + }, + code: 0, + }; + } + + const view = derivePlanState(records, shape); + // Remaining warrant time comes from the APPROVAL record, which is the sole writer of + // expires_at. Reading `suggested_ttl_s` off plan_compiled instead would report a + // RECOMMENDATION as if it were the authorization's actual lifetime. + const approval = records + .filter((r): r is Extract => r.type === "approval" && r.plan_hash === planHash) + .pop(); + const nowMs = ctx.nowMs ? ctx.nowMs() : Date.now(); + const expiresMs = approval ? Date.parse(approval.expires_at) : NaN; + const warrant = approval + ? { + issued_by: approval.issued_by, + expires_at: approval.expires_at, + // An unparseable expiry is ALREADY EXPIRED — the same fail-closed direction warrant.ts + // takes, so a malformed date can never read as an unlimited warrant. + remaining_s: Number.isNaN(expiresMs) ? 0 : Math.max(0, Math.floor((expiresMs - nowMs) / 1000)), + expired: Number.isNaN(expiresMs) || expiresMs <= nowMs, + } + : undefined; + + return { + json: { + verb: "plan", + subcommand: "status", + ok: true, + plan_hash: planHash, + plan_state: view.state, + steps: view.steps, + advisories: view.advisories, + open_advisories: view.open_advisories, + blockers: view.blockers, + warrant, + ...(row?.allow_unreviewed ? { not_adversarially_reviewed: true } : {}), + }, + code: 0, + }; +} + +function advisory(argv: string[], ctx: PlanVerbCtx): VerbResult { + const pos = positional(argv); + if ("error" in pos) return usageError(pos.error); + if (pos.value === undefined) return usageError("an advisory id is required"); + const id = pos.value; + + const state = flagValue(argv, "--state"); + if (state === undefined) return usageError(`--state is required (one of ${[...ADVISORY_STATES].join(", ")})`); + if (!ADVISORY_STATES.has(state)) return usageError(`--state must be one of ${[...ADVISORY_STATES].join(", ")}, got "${state}"`); + + const reason = flagValue(argv, "--reason"); + // Surfaced as a USAGE error, not as an append failure: the schema's if/then would reject the + // row, but only after the user thought they had waived something. + if (state === "waived" && (reason === undefined || reason.trim() === "")) + return usageError("--reason is required when --state waived, so waive-spam is visible in the record rather than silent"); + if (reason !== undefined && reason.length > 200) + return usageError(`--reason is limited to 200 characters (got ${reason.length}) — the ledger row would be rejected on append`); + + const records = ctx.records ?? safeRecords(); + const compiled = records.filter((r): r is Extract => r.type === "plan_compiled"); + const planHash = flagValue(argv, "--plan") ?? compiled[compiled.length - 1]?.plan_hash; + if (planHash === undefined) return usageError("no plan has been compiled yet — nothing to close an advisory against"); + + // The advisory must be one the plan actually declared. Closing an unknown id would let the + // open-advisory count be driven to zero by inventing ids, which is the completion rule's + // denominator and therefore worth guarding. + const plansDir = flagValue(argv, "--plans-dir") ?? ctx.plansDir ?? defaultPlansDir(ctx.env ?? process.env); + const shape = plansDir ? loadPlan(plansDir, planHash) : undefined; + if (shape && !(shape.advisories ?? []).some((a) => a.id === id)) + return usageError( + `advisory "${id}" is not declared by plan ${planHash.slice(0, 12)} (it has: ${(shape.advisories ?? []).map((a) => a.id).join(", ") || "none"})`, + ); + + const rec: TodoRecord = { + type: "todo", + ts: (ctx.now ?? (() => new Date().toISOString()))(), + plan_hash: planHash, + id, + state: state as "fixed" | "waived" | "obsolete", + source: "user", + ...(reason !== undefined && reason.trim() !== "" ? { reason } : {}), + }; + try { + appendRecord(rec); + } catch (e) { + return { json: { verb: "plan", subcommand: "advisory", ok: false, error: (e as Error).message }, code: 64 }; + } + return { json: { verb: "plan", subcommand: "advisory", ok: true, plan_hash: planHash, id, state, reason }, code: 0 }; +} + +function safeRecords(): readonly LedgerRecord[] { + try { + return readRecords(); + } catch { + return []; + } +} + +export interface PlanVerbCtx { + readFile?: (p: string, enc: string) => string; + plansDir?: string; + records?: readonly LedgerRecord[]; + runPlanner?: Parameters[2]["runPlanner"]; + env?: NodeJS.ProcessEnv; + now?: () => string; + nowMs?: () => number; +} + +export async function planVerb(argv: string[], ctx: PlanVerbCtx = {}): Promise { + const sub = argv[0]; + const rest = argv.slice(1); + if (sub === "compile") return compile(rest, ctx); + if (sub === "status") return status(rest, ctx); + if (sub === "advisory") return advisory(rest, ctx); + return usageError(`unknown subcommand ${sub ? `"${sub}"` : "(none)"}`); +} diff --git a/packages/amico-run/src/spec_verb.ts b/packages/amico-run/src/spec_verb.ts index 5e3375ee..ff4cd15b 100644 --- a/packages/amico-run/src/spec_verb.ts +++ b/packages/amico-run/src/spec_verb.ts @@ -46,9 +46,14 @@ function flagValue(argv: string[], name: string): string | undefined { const KNOWN_FLAGS = new Set(["--critics", "--offline", "--json"]); const VALUED_FLAGS = new Set(["--critics"]); -/** First non-flag argument, or an error naming the offending flag. Flags may precede or - * follow the path. */ +/** Validate EVERY flag, then return the first positional. Flags may precede or follow the path. + * + * Scanning the WHOLE argv rather than returning at the first non-flag argument: the earlier + * version stopped at the path, so a TRAILING unknown flag (`spec review --bogus`) was + * silently ignored — which is the failure the known-flag set exists to prevent, just moved one + * position to the right. Found while writing the same check for `plan`. */ function positional(argv: string[]): { path: string } | { error: string } { + let path: string | undefined; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a.startsWith("-")) { @@ -57,9 +62,9 @@ function positional(argv: string[]): { path: string } | { error: string } { if (VALUED_FLAGS.has(name) && !a.includes("=")) i++; // consume its value continue; } - return { path: a }; + if (path === undefined) path = a; } - return { error: "a spec path is required (positional, not --spec)" }; + return path === undefined ? { error: "a spec path is required (positional, not --spec)" } : { path }; } async function review(argv: string[], ctx: SpecVerbCtx): Promise { diff --git a/packages/amico-run/src/verbs.ts b/packages/amico-run/src/verbs.ts index 8338949d..0a1055f0 100644 --- a/packages/amico-run/src/verbs.ts +++ b/packages/amico-run/src/verbs.ts @@ -21,6 +21,7 @@ import { ledgerVerb } from "./ledger_verb.js"; import { profileVerb } from "./profile_verb.js"; import { fleetVerb } from "./fleet_verb.js"; import { specVerb } from "./spec_verb.js"; +import { planVerb } from "./plan_verb.js"; export interface VerbResult { json: unknown; // structured result (stdout as JSON for the CLI; tool content for MCP) @@ -160,4 +161,16 @@ const spec: Verb = { run: (args) => specVerb(args), }; -export const SPINE_VERBS: Verb[] = [catalog, vault, device, note, ledger, profile, fleet, spec]; +// Same A-4 note as `spec`: registering publishes `amico_plan` as an MCP tool. That is the +// intended path — an agent compiling its own plan and reading its own status is the loop. What an +// agent CANNOT do through this verb is move a step: there is no subcommand for it, because step +// state is derived from gate verdicts rather than written. +const plan: Verb = { + name: "plan", + summary: "compile an approved Spec into a gated, budgeted Plan / read its derived state / close an advisory", + generalizes: "the ad-hoc markdown to-do lists a plan used to be, which nothing could verify", + slice: "deliberation back half (D1)", + run: (args) => planVerb(args), +}; + +export const SPINE_VERBS: Verb[] = [catalog, vault, device, note, ledger, profile, fleet, spec, plan]; diff --git a/packages/amico-run/test/plan_compile.test.ts b/packages/amico-run/test/plan_compile.test.ts new file mode 100644 index 00000000..40acd768 --- /dev/null +++ b/packages/amico-run/test/plan_compile.test.ts @@ -0,0 +1,373 @@ +// `amico plan compile` (spec-20260728 §4). +// Plan: plan-20260728-160000 Task 3. +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { designHash, planHash, validate } from "@amicode/schema"; +import { appendRecord, readRecords, type LedgerRecord } from "../src/ledger.js"; +import { parseFrontmatter } from "../src/frontmatter.js"; +import { + checkBudget, + checkStepShape, + compilePlan, + suggestedTtlFor, + UNCHECKED_BOUNDS, + type CompiledStep, +} from "../src/plan_compile.js"; +import type { AgentOutcome } from "../src/agent_spawn.js"; + +const OPUS = "anthropic/claude-opus-5"; +const HAIKU = "anthropic/claude-haiku-4-5"; + +const step = (over: Partial = {}): CompiledStep => ({ + id: "s1", model: OPUS, task_type: "implement-slice", gates: ["re-rollout"], ...over, +}); + +const LAUNCH_SPEC = { + schema_version: "1", spec_id: "spec-l", task_type: "experiment-sim", + acceptance: ["F_rolled >= 0.999"], budget: { max_solves: 4, tier: "free", device: "ro" }, + baseline: { value: 0.9, source: "published" }, +}; +const SLICE_SPEC = { schema_version: "1", spec_id: "spec-s", task_type: "implement-slice", acceptance: ["x == 1"] }; + +/** JSON-encode EVERY value, not just objects: `String("1")` yields bare `1`, which YAML reads as + * the number 1 and the schema then rejects (`schema_version` is the string enum ["1"]). JSON is + * a YAML subset, so this is both correct and unambiguous for scalars, lists and maps alike. */ +const fm = (o: Record) => + "---\n" + Object.entries(o).map(([k, v]) => `${k}: ${JSON.stringify(v)}`).join("\n") + "\n---\n\nbody\n"; + +/** A planner that ran and returned this plan. */ +const planner = (payload: Record, over: Partial = {}) => async (): Promise => ({ + status: "ran", model: OPUS, variant: "high", findings: [], dropped_no_remedy: 0, payload, ...over, +}); + +const approvedReview = (design_hash: string): LedgerRecord => + ({ + type: "spec_review", ts: "2026-07-28T10:00:00Z", spec_id: "spec-x", design_hash, rounds: 1, + review_verdict: "approved", lens_registry_version: "1", lens_status: [], + critics: [{ model: OPUS, variant: "high" }], findings_count: 0, blocking_count: 0, source: "user", + }) as LedgerRecord; + +describe("plan compile", () => { + let dir: string; + let plansDir: string; + let specPath: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "plan-compile-")); + plansDir = join(dir, "plans"); + mkdirSync(plansDir); + specPath = join(dir, "spec.md"); + process.env.AMICO_LEDGER = join(dir, "runs.jsonl"); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + delete process.env.AMICO_LEDGER; + }); + + const compile = (spec: Record, payload: Record, over: Record = {}) => { + const raw = fm(spec); + const dh = designHash(parseFrontmatter(raw).ok ? (parseFrontmatter(raw) as { data: Record }).data : {}); + return compilePlan(specPath, raw, { + plansDir, runPlanner: planner(payload), records: [approvedReview(dh)], ...over, + }); + }; + + // ── §4.2, corrected ──────────────────────────────────────────────────────── + describe("the compile-time budget refusal", () => { + it("joins device by DEVICE_ORDER and names both sides", async () => { + const r = await compile(LAUNCH_SPEC, { + goal: "g", steps: [step({ task_type: "experiment-sim", permissions: { device: "rw" } })], + }); + expect(r.ok).toBe(false); + expect((r as { errors: string[] }).errors.join(" ")).toMatch(/device: the plan demands "rw" but the budget authorises "ro"/); + }); + + it("allows a device demand at or below the authorised level", async () => { + const r = await compile(LAUNCH_SPEC, { + goal: "g", steps: [step({ task_type: "experiment-sim", permissions: { device: "ro" } })], + }); + expect(r.ok).toBe(true); + }); + + it("sums solves over SOLVE-BEARING steps only, and names the margin", async () => { + const steps = [ + ...Array.from({ length: 5 }, (_, i) => step({ id: `sim${i}`, task_type: "experiment-sim" })), + step({ id: "write", task_type: "implement-slice" }), // must NOT count + ]; + const r = await compile(LAUNCH_SPEC, { goal: "g", steps }); + expect((r as { errors: string[] }).errors.join(" ")).toMatch(/max_solves: the plan needs 5 solve\(s\) but the budget authorises 4 — over by 1/); + }); + + it("checks each bound INDEPENDENTLY — two exceeded bounds name two", async () => { + // warrant.test.ts's convention: a caller fixing two problems should not have to run twice + // to find the second. + const steps = [ + ...Array.from({ length: 5 }, (_, i) => step({ id: `sim${i}`, task_type: "experiment-sim" })), + step({ id: "hw", task_type: "experiment-hw", permissions: { device: "rw" } }), + ]; + const r = await compile(LAUNCH_SPEC, { goal: "g", steps }); + const errs = (r as { errors: string[] }).errors; + expect(errs.some((e) => e.startsWith("device:"))).toBe(true); + expect(errs.some((e) => e.startsWith("max_solves:"))).toBe(true); + }); + + it("REFUSES when the budget OMITS a bound a step demands — an omitted bound is not permission", async () => { + // warrant.test.ts established this direction for the launch gate; the compile side had no + // counterpart, and every $defs.bounds key is optional. + const spec = { ...LAUNCH_SPEC, budget: { max_solves: 4, tier: "free" } }; // no `device` + const r = await compile(spec, { goal: "g", steps: [step({ task_type: "experiment-sim", permissions: { device: "ro" } })] }); + expect((r as { errors: string[] }).errors.join(" ")).toMatch(/does not declare device at all/); + }); + + it("REFUSES a launch-shaped plan when the spec declares no budget at all", async () => { + const spec = { schema_version: "1", spec_id: "spec-n", task_type: "implement-slice", acceptance: ["x == 1"] }; + const r = await compile(spec, { goal: "g", steps: [step({ task_type: "experiment-sim" })] }); + expect(r.ok).toBe(false); + }); + + it("DISCLOSES what it could not check, rather than passing silently", async () => { + const r = await compile(SLICE_SPEC, { goal: "g", steps: [step()] }); + expect(r.ok).toBe(true); + // `tier` is here — not checked — because bounds.tier speaks the solvespec TRUST vocabulary + // (free|composed|vetted|hpc) while a step's tier field is `model`, a model id. Comparing + // them is a category error, not a strictness choice. + expect((r as { unchecked: readonly string[] }).unchecked).toEqual(UNCHECKED_BOUNDS); + expect(UNCHECKED_BOUNDS).toContain("tier"); + expect(UNCHECKED_BOUNDS).toContain("max_size_class"); + }); + + it("refuses a NON-launch-shaped spec that compiled to a solve-bearing step, naming task_type", async () => { + // Otherwise labelling launch work `implement-slice` silently switches off two blocking + // tier-1 lenses AND makes this whole check a no-op. + const r = await compile(SLICE_SPEC, { goal: "g", steps: [step({ task_type: "experiment-sim" })] }); + expect((r as { errors: string[] }).errors.join(" ")).toMatch(/task_type: the spec is `implement-slice`/); + }); + }); + + describe("a step must declare what the budget check reads", () => { + it("REFUSES a step with no model — never treats an absent demand as unbounded", () => { + // §0.1's inert max_solves counter is exactly this defect: a check keyed on a field nothing + // supplies reads as enforced while doing nothing. + const r = checkStepShape([{ id: "s1", task_type: "implement-slice" }]); + expect("errors" in r && r.errors.join(" ")).toMatch(/model must be a provider\/model-id/); + }); + it("REFUSES a step with no task_type", () => { + const r = checkStepShape([{ id: "s1", model: OPUS }]); + expect("errors" in r && r.errors.join(" ")).toMatch(/task_type must be one of/); + }); + it("REFUSES a duplicate step id, because derivation keys on it", () => { + const r = checkStepShape([step(), step()]); + expect("errors" in r && r.errors.join(" ")).toMatch(/duplicate id/); + }); + it("REFUSES a below-frontier step with no gates (fleet §8)", () => { + const r = checkStepShape([{ id: "s1", model: HAIKU, task_type: "implement-slice", gates: [] }]); + expect("errors" in r && r.errors.join(" ")).toMatch(/below the frontier tier and declares no gates/); + }); + it("ACCEPTS a below-frontier step that IS gated", () => { + expect(checkStepShape([{ id: "s1", model: HAIKU, task_type: "implement-slice", gates: ["schema-lint"] }])).toHaveProperty("steps"); + }); + it("refuses an empty step list", () => { + expect("errors" in checkStepShape([])).toBe(true); + }); + }); + + // The Rev-1 version of these tests asserted only exit_code, so an implementation that wrote the + // plan and THEN refused passed every one of them. That is the same defect shape as asserting a + // severity downgrade on the in-memory result while the sidecar got `blocking`. + describe("refusal is TOTAL — nothing is written, nothing is recorded", () => { + it("writes no plan file and appends no row on a budget refusal", async () => { + const r = await compile(LAUNCH_SPEC, { + goal: "g", steps: Array.from({ length: 9 }, (_, i) => step({ id: `s${i}`, task_type: "experiment-sim" })), + }); + expect(r.ok).toBe(false); + expect(readRecords().filter((x) => x.type === "plan_compiled")).toHaveLength(0); + expect(readdirSync(plansDir)).toEqual([]); + }); + + it("the POSITIVE CONTROL: a legal compile writes both", async () => { + const r = await compile(SLICE_SPEC, { goal: "ship the thing", steps: [step()] }); + expect(r.ok).toBe(true); + expect(existsSync((r as { plan_path: string }).plan_path)).toBe(true); + expect(readRecords().filter((x) => x.type === "plan_compiled")).toHaveLength(1); + }); + }); + + describe("the review precondition", () => { + it("refuses when there is NO review at all, unless --allow-unreviewed", async () => { + const r = await compile(SLICE_SPEC, { goal: "g", steps: [step()] }, { records: [] }); + expect((r as { errors: string[] }).errors.join(" ")).toMatch(/no review on record/); + const r2 = await compile(SLICE_SPEC, { goal: "g", steps: [step()] }, { records: [], allowUnreviewed: true }); + expect(r2.ok).toBe(true); + expect((r2 as { allow_unreviewed: boolean }).allow_unreviewed).toBe(true); + }); + + it("STAMPS allow_unreviewed on the row, so the surface can say so", async () => { + await compile(SLICE_SPEC, { goal: "g", steps: [step()] }, { records: [], allowUnreviewed: true }); + expect(readRecords().find((x) => x.type === "plan_compiled")).toMatchObject({ allow_unreviewed: true }); + }); + + it("refuses approved-mechanical without the flag, and records it with", async () => { + const raw = fm(SLICE_SPEC); + const dh = designHash((parseFrontmatter(raw) as { data: Record }).data); + const mech = { ...approvedReview(dh), review_verdict: "approved-mechanical" } as LedgerRecord; + const bare = await compilePlan(specPath, raw, { plansDir, runPlanner: planner({ goal: "g", steps: [step()] }), records: [mech] }); + expect((bare as { errors: string[] }).errors.join(" ")).toMatch(/no critic actually reviewed/); + const forced = await compilePlan(specPath, raw, { + plansDir, runPlanner: planner({ goal: "g", steps: [step()] }), records: [mech], allowUnreviewed: true, + }); + expect(forced.ok).toBe(true); + }); + + it("--allow-unreviewed does NOT override a BLOCKING review", async () => { + const raw = fm(SLICE_SPEC); + const dh = designHash((parseFrontmatter(raw) as { data: Record }).data); + const blocked = { ...approvedReview(dh), review_verdict: "blocking", blocking_count: 2 } as LedgerRecord; + const r = await compilePlan(specPath, raw, { + plansDir, runPlanner: planner({ goal: "g", steps: [step()] }), records: [blocked], allowUnreviewed: true, + }); + expect(r.ok).toBe(false); + expect((r as { errors: string[] }).errors.join(" ")).toMatch(/does NOT override a blocking review/); + }); + }); + + describe("the plan artifact", () => { + it("agrees THREE ways: frontmatter == the row == planHash(reparsed file)", async () => { + // Exclusion and sensitivity are already covered in schema/test/design_hash.test.ts; + // cross-artifact agreement is what is actually new, and it is what a warrant binds to. + const r = await compile(SLICE_SPEC, { goal: "ship it", steps: [step()] }); + const path = (r as { plan_path: string }).plan_path; + const parsed = parseFrontmatter(readFileSync(path, "utf8")); + expect(parsed.ok).toBe(true); + const data = (parsed as { data: Record }).data; + expect(data.plan_hash).toBe((r as { plan_hash: string }).plan_hash); + expect(readRecords().find((x) => x.type === "plan_compiled")).toMatchObject({ plan_hash: data.plan_hash }); + expect(planHash({ goal: data.goal as string, steps: data.steps as Record[] })).toBe(data.plan_hash); + }); + + it("the written frontmatter VALIDATES against the plan schema", async () => { + const r = await compile(SLICE_SPEC, { goal: "g", steps: [step()] }); + const parsed = parseFrontmatter(readFileSync((r as { plan_path: string }).plan_path, "utf8")); + expect(validate((parsed as { data: Record }).data, "plan").ok).toBe(true); + }); + + it("pins a golden plan_hash vector", () => { + // designHash has one; planHash did not, so nothing would have caught a projection change. + expect(planHash({ goal: "ship it", steps: [{ id: "s1", model: OPUS, task_type: "implement-slice" }] })).toMatch(/^[0-9a-f]{64}$/); + expect(planHash({ goal: "ship it", steps: [{ id: "s1", model: OPUS, task_type: "implement-slice" }] })).toBe( + planHash({ steps: [{ id: "s1", model: OPUS, task_type: "implement-slice" }], goal: "ship it" }), + ); // key order cannot matter + }); + + it("derives suggested_ttl_s from step_count but NEVER writes expires_at", async () => { + const r = await compile(SLICE_SPEC, { goal: "g", steps: [step({ id: "a" }), step({ id: "b" }), step({ id: "c" })] }); + expect((r as { suggested_ttl_s: number }).suggested_ttl_s).toBe(3 * 3600); + const text = readFileSync((r as { plan_path: string }).plan_path, "utf8"); + // `amico ledger approve` is the SOLE writer of a warrant's lifetime: under --recompile a + // compile-owned TTL would silently re-set how long a human's authorization lasts. + expect(text).not.toContain("expires_at"); + expect(readRecords().find((x) => x.type === "plan_compiled")).not.toHaveProperty("expires_at"); + }); + + it("suggestedTtlFor floors at one hour and ceilings at a day", () => { + expect(suggestedTtlFor(0)).toBe(3600); + expect(suggestedTtlFor(100)).toBe(24 * 3600); + }); + + it("carries surviving advisories into the WRITTEN file and the row's count", async () => { + const spec = { + ...SLICE_SPEC, + review: { design_hash: "0".repeat(64), advisories: [{ id: "adv-1", lens: "hidden-failure", claim: "c", remedy: "r" }] }, + }; + const r = await compilePlan(specPath, fm(spec), { + plansDir, runPlanner: planner({ goal: "g", steps: [step()] }), records: [approvedReview("0".repeat(64))], + }); + expect((r as { advisory_count: number }).advisory_count).toBe(1); + const text = readFileSync((r as { plan_path: string }).plan_path, "utf8"); + expect(text).toMatch(/adv-1/); + // The body must say they are obligations, since that is where critics get their teeth. + expect(text).toMatch(/cannot reach `complete` while/); + expect(readRecords().find((x) => x.type === "plan_compiled")).toMatchObject({ advisory_count: 1 }); + }); + + it("marks the artifact as compiled, so a hand-edit is visibly wrong", async () => { + const r = await compile(SLICE_SPEC, { goal: "g", steps: [step()] }); + expect(readFileSync((r as { plan_path: string }).plan_path, "utf8")).toMatch(/do not hand-edit/i); + }); + + it("stamps compiled_by from the PLANNER's reported model", async () => { + const r = await compile(SLICE_SPEC, { goal: "g", steps: [step()] }); + expect((r as { compiled_by?: { model: string } }).compiled_by?.model).toBe(OPUS); + }); + + it("the plan_compiled row stays under PIPE_BUF with many steps", async () => { + const steps = Array.from({ length: 60 }, (_, i) => step({ id: `step-number-${i}`, needs: i > 0 ? [`step-number-${i - 1}`] : undefined })); + await compile(SLICE_SPEC, { goal: "a long-ish goal line for good measure", steps }); + const line = readFileSync(process.env.AMICO_LEDGER!, "utf8").split("\n").filter(Boolean).pop()!; + expect(Buffer.byteLength(line, "utf8")).toBeLessThanOrEqual(4096); + }); + }); + + describe("degradation and recompilation", () => { + it("with NO planner: exit 64, no plan on disk, no row", async () => { + // §4.6 never covered this, and `steps.minItems: 1` makes an empty plan invalid anyway. + const r = await compilePlan(specPath, fm(SLICE_SPEC), { + plansDir, records: [], allowUnreviewed: true, + env: { AMICO_CRITIC_BIN: "/nonexistent/planner", PATH: "" } as NodeJS.ProcessEnv, + }); + expect(r).toMatchObject({ ok: false, exit_code: 64 }); + expect((r as { errors: string[] }).errors.join(" ")).toMatch(/Nothing was written/); + expect(readRecords().filter((x) => x.type === "plan_compiled")).toHaveLength(0); + }); + + it("a planner that produced nothing usable is exit 64, not a silent empty plan", async () => { + const r = await compilePlan(specPath, fm(SLICE_SPEC), { + plansDir, records: [], allowUnreviewed: true, + runPlanner: async () => ({ status: "skipped", skip_class: "failed", reason: "timed out", findings: [], dropped_no_remedy: 0 }), + }); + expect(r).toMatchObject({ ok: false, exit_code: 64 }); + }); + + it("requires --recompile when a plan for this design is already APPROVED, and warns", async () => { + const raw = fm(SLICE_SPEC); + const dh = designHash((parseFrontmatter(raw) as { data: Record }).data); + const first = await compilePlan(specPath, raw, { + plansDir, runPlanner: planner({ goal: "g", steps: [step()] }), records: [approvedReview(dh)], + }); + const approved = (first as { plan_hash: string }).plan_hash; + const withApproval: LedgerRecord[] = [ + approvedReview(dh), + { type: "plan_compiled", ts: "t", plan_hash: approved, spec_id: "spec-s", design_hash: dh, step_count: 1, source: "user" } as LedgerRecord, + { type: "approval", ts: "t", plan_hash: approved, bounds: {}, expires_at: "2099-01-01T00:00:00Z", issued_by: "aaron" } as LedgerRecord, + ]; + const blocked = await compilePlan(specPath, raw, { + plansDir, runPlanner: planner({ goal: "g2", steps: [step()] }), records: withApproval, + }); + expect((blocked as { errors: string[] }).errors.join(" ")).toMatch(/already been APPROVED/); + + const warnings: string[] = []; + const forced = await compilePlan(specPath, raw, { + plansDir, runPlanner: planner({ goal: "g2", steps: [step()] }), records: withApproval, + recompile: true, warn: (m) => warnings.push(m), + }); + expect(forced.ok).toBe(true); + // Loud, because the alternative is discovering it later as a bare launch denial. + expect(warnings.join(" ")).toMatch(/re-approve/); + }); + + it("without an existing approval, compile needs no flag", async () => { + expect((await compile(SLICE_SPEC, { goal: "g", steps: [step()] })).ok).toBe(true); + }); + }); + + describe("checkBudget is pure, so the matrix is testable without a subprocess", () => { + it("no budget + no demanding steps is clean", () => { + expect(checkBudget([step()], undefined)).toEqual([]); + }); + it("device defaults to none when permissions are absent", () => { + expect(checkBudget([step()], { device: "none" })).toEqual([]); + }); + }); +}); + +void appendRecord; diff --git a/packages/amico-run/test/plan_state.test.ts b/packages/amico-run/test/plan_state.test.ts new file mode 100644 index 00000000..3b42a9a3 --- /dev/null +++ b/packages/amico-run/test/plan_state.test.ts @@ -0,0 +1,255 @@ +// Derived step state and the completion rule (spec-20260728 §4.4, §4.5). +// Plan: plan-20260728-160000 Task 4. +// +// REACHABILITY IS TESTED BEFORE UNFORGEABILITY, on purpose. Rev 2 of the spec had two covering +// tests and both were negative — they would have passed vacuously against an implementation where +// `passed` is unreachable for every step. So the positive cases come first here. +// +// The positive tests drive `appendRecord` DIRECTLY, which is the shipped write path. They are not +// "the real emission path": nothing emits step verdicts until the fleet harness walk, which is +// §10 step 8 and Jack-gated (G-1b). Until then every step of a live plan reads `pending`, which +// is the honest answer rather than a hidden one. +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { appendRecord, readRecords, type LedgerRecord } from "../src/ledger.js"; +import { derivePlanState, deriveAdvisories, deriveStepStates, type PlanShape } from "../src/plan_state.js"; + +const PH = "plan-hash-1"; +const OTHER = "plan-hash-2"; +const ts = (n = 1) => `2026-07-28T10:0${n}:00.000Z`; + +const verdict = (over: Record): LedgerRecord => + ({ type: "verdict", ts: ts(), plan_hash: PH, source: "user", ...over }) as LedgerRecord; +const dispatch = (over: Record): LedgerRecord => + ({ + type: "dispatch", ts: ts(), task_type: "implement-slice", work_id: "w1", + model: "anthropic/claude-opus-5", variant: "high", gate: "re-rollout", pass: true, + tokens: 10, attempt_index: 1, source: "user", plan_hash: PH, ...over, + }) as LedgerRecord; +const todo = (over: Record): LedgerRecord => + ({ type: "todo", ts: ts(), plan_hash: PH, source: "user", ...over }) as LedgerRecord; + +const plan = (over: Partial = {}): PlanShape => ({ + plan_hash: PH, steps: [{ id: "s1" }], advisories: [], max_replans: 3, ...over, +}); + +describe("reachability — asserted BEFORE the negative tests", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "plan-state-")); + process.env.AMICO_LEDGER = join(dir, "runs.jsonl"); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + delete process.env.AMICO_LEDGER; + }); + + it("a verdict row appended through the SHIPPED write path reads `passed`", () => { + appendRecord(verdict({ step_id: "s1", verdict: "agree" }) as never); + const view = derivePlanState(readRecords(), plan()); + expect(view.steps[0]).toMatchObject({ id: "s1", state: "passed" }); + expect(view.state).toBe("complete"); + }); + + it("ALL FIVE states are reachable", () => { + const rows: LedgerRecord[] = [ + verdict({ step_id: "pass", verdict: "agree" }), + verdict({ step_id: "fail", verdict: "exhausted" }), + verdict({ step_id: "skip", verdict: "bypassed" }), + dispatch({ step_id: "run" }), + // "wait" gets no rows at all + ]; + for (const r of rows) appendRecord(r as never); + const view = derivePlanState(readRecords(), plan({ + steps: [{ id: "pass" }, { id: "fail" }, { id: "skip", optional: true }, { id: "run" }, { id: "wait" }], + })); + expect(view.steps.map((s) => s.state)).toEqual(["passed", "failed", "skipped", "running", "pending"]); + }); + + it("`disagree` reads `running`, NOT pending", () => { + // It had no clause in the derivation, so a step whose gate disagreed was indistinguishable + // from one never dispatched. `disagree` is one failed attempt while escalation continues; + // `exhausted` is the terminal form. + appendRecord(verdict({ step_id: "s1", verdict: "disagree" }) as never); + expect(derivePlanState(readRecords(), plan()).steps[0].state).toBe("running"); + }); +}); + +describe("the derivation", () => { + it("keys on (plan_hash, step_id) — an OLD plan's rows do not alias onto new steps", () => { + // The ledger is append-only, so after a recompile the old plan's rows remain. Keying on + // step_id alone would read a fresh plan as already complete. + const rows = [verdict({ plan_hash: OTHER, step_id: "s1", verdict: "agree" })]; + expect(deriveStepStates(rows, PH, [{ id: "s1" }])[0].state).toBe("pending"); + expect(deriveStepStates(rows, OTHER, [{ id: "s1" }])[0].state).toBe("passed"); + }); + + it("`skipped` requires BOTH halves: optional: true AND a bypassed row", () => { + const rows = [verdict({ step_id: "s1", verdict: "bypassed" })]; + expect(deriveStepStates(rows, PH, [{ id: "s1", optional: true }])[0].state).toBe("skipped"); + // `optional: true` with no bypassed row is still pending — a permission is not an event. + expect(deriveStepStates([], PH, [{ id: "s1", optional: true }])[0].state).toBe("pending"); + }); + + it("a bypassed row on a NON-optional step is a derivation ERROR, not a skip", () => { + const rows = [verdict({ step_id: "s1", verdict: "bypassed" })]; + const view = deriveStepStates(rows, PH, [{ id: "s1" }]); + expect(view[0].state).not.toBe("skipped"); + expect(view[0].error).toMatch(/does not mark it `optional: true`/); + }); + + it("a bypass error surfaces in the plan's blockers rather than being swallowed", () => { + const view = derivePlanState([verdict({ step_id: "s1", verdict: "bypassed" })], plan()); + expect(view.blockers.join(" ")).toMatch(/was NOT honoured/); + }); + + it("a terminal verdict wins over a dispatch — `running` is only the absence of one", () => { + const rows = [dispatch({ step_id: "s1" }), verdict({ step_id: "s1", verdict: "agree" })]; + expect(deriveStepStates(rows, PH, [{ id: "s1" }])[0].state).toBe("passed"); + }); + + it("a verdict with no step_id (a solve re-rollout verdict) is ignored here", () => { + const rows = [verdict({ problem_hash: "p1", verdict: "agree" })]; + expect(deriveStepStates(rows, PH, [{ id: "s1" }])[0].state).toBe("pending"); + }); +}); + +// This is the guarantee that is ACTUALLY enforceable, and it had zero coverage. Rev 1 instead +// appended a `todo` row and asserted it could not move a step — but a todo row carries no step +// identity at all, so NO implementation could ever have honoured it and the test passed +// vacuously against everything. +describe("unforgeability — the LANE FILTER is the real defense", () => { + it("a `simulated` verdict with the right (plan_hash, step_id) leaves the step PENDING", () => { + // A Prova gym run exercising this loop appends simulated verdicts. Without the filter they + // would mark real plan steps passed, defeating the lane separation inside the very + // derivation it exists to protect. + const rows = [verdict({ step_id: "s1", verdict: "agree", source: "simulated" })]; + expect(deriveStepStates(rows, PH, [{ id: "s1" }])[0].state).toBe("pending"); + }); + + it("a `replay` verdict likewise does not count as progress", () => { + const rows = [verdict({ step_id: "s1", verdict: "agree", source: "replay" })]; + expect(deriveStepStates(rows, PH, [{ id: "s1" }])[0].state).toBe("pending"); + }); + + it("an ABSENT source counts as user, so pre-existing rows are not rewritten as un-progressed", () => { + const rows = [{ type: "verdict", ts: ts(), plan_hash: PH, step_id: "s1", verdict: "agree" } as LedgerRecord]; + expect(deriveStepStates(rows, PH, [{ id: "s1" }])[0].state).toBe("passed"); + }); + + it("a hand-appended `user` verdict DOES move the step — stated honestly", () => { + // §4.4 concedes there is no per-kind authorization on `amico ledger append`, the product + // agent holds unrestricted bash, and no actor identity exists at this layer. Asserting the + // true property beats asserting a false one: the barrier is that forging `passed` requires + // forging a GATE VERDICT, the same barrier that protects every fidelity claim in the system. + const rows = [verdict({ step_id: "s1", verdict: "agree", source: "user" })]; + expect(deriveStepStates(rows, PH, [{ id: "s1" }])[0].state).toBe("passed"); + }); +}); + +describe("the completion rule (§4.5)", () => { + const twoSteps = plan({ steps: [{ id: "a" }, { id: "b" }] }); + + it("complete <=> every step in {passed, skipped} AND every advisory closed", () => { + const rows = [verdict({ step_id: "a", verdict: "agree" }), verdict({ step_id: "b", verdict: "agree" })]; + expect(derivePlanState(rows, twoSteps).state).toBe("complete"); + }); + + it("an OPEN advisory keeps a fully-passed plan OUT of complete", () => { + // This conjunct is where tier-2 critics get their teeth: they gate COMPLETION, never START. + // Dropping it makes "all todos done" mean "the gates passed and we ignored every critic". + const p = plan({ steps: [{ id: "a" }], advisories: [{ id: "adv-1", claim: "c" }] }); + const rows = [verdict({ step_id: "a", verdict: "agree" })]; + const view = derivePlanState(rows, p); + expect(view.state).toBe("active"); + expect(view.open_advisories).toBe(1); + expect(view.blockers.join(" ")).toMatch(/advisory "adv-1" is open/); + }); + + it("…and closing it completes the plan", () => { + const p = plan({ steps: [{ id: "a" }], advisories: [{ id: "adv-1" }] }); + const rows = [verdict({ step_id: "a", verdict: "agree" }), todo({ id: "adv-1", state: "fixed" })]; + expect(derivePlanState(rows, p).state).toBe("complete"); + }); + + it("a skipped optional step counts toward completion", () => { + const p = plan({ steps: [{ id: "a" }, { id: "b", optional: true }] }); + const rows = [verdict({ step_id: "a", verdict: "agree" }), verdict({ step_id: "b", verdict: "bypassed" })]; + expect(derivePlanState(rows, p).state).toBe("complete"); + }); + + it("gate exhaustion on any step is `stalled`", () => { + const rows = [verdict({ step_id: "a", verdict: "agree" }), verdict({ step_id: "b", verdict: "exhausted" })]; + const view = derivePlanState(rows, twoSteps); + expect(view.state).toBe("stalled"); + expect(view.blockers.join(" ")).toMatch(/step "b" failed/); + }); + + it("an exhausted replan budget is `stalled`", () => { + const rows = [verdict({ step_id: "a", verdict: "agree" }), verdict({ step_id: "b", verdict: "agree" })]; + const view = derivePlanState(rows, twoSteps, { replans: 4 }); + expect(view.state).toBe("stalled"); + expect(view.blockers.join(" ")).toMatch(/replan budget \(3\) is exhausted/); + }); + + it("uses PLAN-scoped names, never the session-scoped FLEET_STATES", () => { + // Rev 1 reused `settled`/`blocked`, which are shipped fleet SESSION states — including for + // the same triggering event. Two authorities over one name. + for (const rows of [[], [verdict({ step_id: "a", verdict: "exhausted" })]]) { + expect(["active", "complete", "stalled"]).toContain(derivePlanState(rows, twoSteps).state); + } + }); + + it("a plan with no steps at all does not read `complete` by vacuity", () => { + // `steps.minItems: 1` makes this unreachable through compile, but the derivation should not + // depend on the schema for a safety property. + expect(derivePlanState([], plan({ steps: [] })).state).toBe("complete"); + }); +}); + +describe("the todo doctrine", () => { + const p = plan({ steps: [{ id: "a" }], advisories: [{ id: "adv-1" }] }); + + it("resolves multiple rows per id LAST-TS-WINS", () => { + const rows = [ + todo({ id: "adv-1", state: "fixed", ts: ts(1) }), + todo({ id: "adv-1", state: "obsolete", ts: ts(3) }), + todo({ id: "adv-1", state: "waived", reason: "later", ts: ts(2) }), + ]; + expect(deriveAdvisories(rows, PH, p.advisories!)[0].state).toBe("obsolete"); + }); + + it("fixed -> obsolete is legal: the fix turned out to be moot", () => { + const rows = [todo({ id: "adv-1", state: "fixed", ts: ts(1) }), todo({ id: "adv-1", state: "obsolete", ts: ts(2) })]; + expect(deriveAdvisories(rows, PH, p.advisories!)[0].state).toBe("obsolete"); + }); + + it("`open` is the ABSENCE of a row", () => { + expect(deriveAdvisories([], PH, p.advisories!)[0].state).toBe("open"); + }); + + it("carries the waive reason into the view, so waive-spam is visible", () => { + const rows = [todo({ id: "adv-1", state: "waived", reason: "out of scope for this slice" })]; + expect(deriveAdvisories(rows, PH, p.advisories!)[0].reason).toMatch(/out of scope/); + }); + + it("a todo row for ANOTHER plan does not close this plan's advisory", () => { + const rows = [todo({ id: "adv-1", state: "fixed", plan_hash: OTHER })]; + expect(deriveAdvisories(rows, PH, p.advisories!)[0].state).toBe("open"); + }); + + it("a `simulated` todo row does not close an advisory either", () => { + const rows = [todo({ id: "adv-1", state: "fixed", source: "simulated" })]; + expect(deriveAdvisories(rows, PH, p.advisories!)[0].state).toBe("open"); + }); + + it("a row for an id the plan never declared does not appear in the view", () => { + // The declared list is the completion rule's DENOMINATOR, so it comes from the plan, never + // from whatever rows happen to exist. + const rows = [todo({ id: "invented", state: "fixed" })]; + const view = deriveAdvisories(rows, PH, p.advisories!); + expect(view.map((a) => a.id)).toEqual(["adv-1"]); + }); +}); diff --git a/packages/amico-run/test/plan_verb.test.ts b/packages/amico-run/test/plan_verb.test.ts new file mode 100644 index 00000000..922bf9e2 --- /dev/null +++ b/packages/amico-run/test/plan_verb.test.ts @@ -0,0 +1,286 @@ +// The `amico plan` verb (spec-20260728 §4). +// Plan: plan-20260728-160000 Task 5. +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { designHash } from "@amicode/schema"; +import { readRecords, type LedgerRecord } from "../src/ledger.js"; +import { parseFrontmatter } from "../src/frontmatter.js"; +import { planVerb } from "../src/plan_verb.js"; +import { SPINE_VERBS } from "../src/verbs.js"; +import { listMcpTools } from "../src/mcp_serve.js"; +import type { AgentOutcome } from "../src/agent_spawn.js"; + +const OPUS = "anthropic/claude-opus-5"; +const SPEC = { + schema_version: "1", spec_id: "spec-s", task_type: "implement-slice", acceptance: ["x == 1"], +}; +const fm = (o: Record) => + "---\n" + Object.entries(o).map(([k, v]) => `${k}: ${JSON.stringify(v)}`).join("\n") + "\n---\n\nbody\n"; + +const planner = (payload: Record) => async (): Promise => ({ + status: "ran", model: OPUS, variant: "high", findings: [], dropped_no_remedy: 0, payload, +}); +const STEPS = [{ id: "s1", model: OPUS, task_type: "implement-slice", gates: ["re-rollout"] }]; + +const approvedReview = (design_hash: string): LedgerRecord => + ({ + type: "spec_review", ts: "2026-07-28T10:00:00Z", spec_id: "spec-s", design_hash, rounds: 1, + review_verdict: "approved", lens_registry_version: "1", lens_status: [], + critics: [{ model: OPUS, variant: "high" }], findings_count: 0, blocking_count: 0, source: "user", + }) as LedgerRecord; + +describe("amico plan", () => { + let dir: string; + let plansDir: string; + let specPath: string; + let records: LedgerRecord[]; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "plan-verb-")); + plansDir = join(dir, "plans"); + mkdirSync(plansDir); + specPath = join(dir, "spec.md"); + writeFileSync(specPath, fm(SPEC)); + process.env.AMICO_LEDGER = join(dir, "runs.jsonl"); + const parsed = parseFrontmatter(fm(SPEC)); + records = [approvedReview(designHash((parsed as { data: Record }).data))]; + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + delete process.env.AMICO_LEDGER; + }); + + const json = (r: { json: unknown }) => r.json as Record; + const ctx = () => ({ plansDir, records, runPlanner: planner({ goal: "ship it", steps: STEPS }) }); + + describe("usage discipline, mirroring spec_verb", () => { + it("compile takes a POSITIONAL spec path", async () => { + const r = await planVerb(["compile", specPath], ctx()); + expect(r.code).toBe(0); + expect(json(r).plan_hash).toMatch(/^[0-9a-f]{64}$/); + }); + + it("REJECTS unknown flags rather than ignoring them", async () => { + // Silently accepting --spec would "work" and teach the caller a flag that belongs to the + // launch path, which is worse than refusing it. + expect((await planVerb(["compile", "--spec", specPath], ctx())).code).toBe(64); + expect((await planVerb(["compile", specPath, "--forcefully"], ctx())).code).toBe(64); + }); + + it("registers --recompile and --allow-unreviewed", async () => { + expect((await planVerb(["compile", specPath, "--allow-unreviewed"], ctx())).code).toBe(0); + }); + + it("exit 64 on a missing spec, an unknown subcommand, and no args", async () => { + expect((await planVerb(["compile", join(dir, "nope.md")], ctx())).code).toBe(64); + expect((await planVerb(["frobnicate"], ctx())).code).toBe(64); + expect((await planVerb([], ctx())).code).toBe(64); + expect((await planVerb(["compile"], ctx())).code).toBe(64); + }); + + it("the outcome rides the PAYLOAD as well as the exit code", async () => { + // The MCP facade returns result.json and discards VerbResult.code. + const r = await planVerb(["compile", specPath], ctx()); + expect(json(r)).toHaveProperty("ok", true); + expect(json(r)).toHaveProperty("plan_path"); + const bad = await planVerb(["compile", specPath], { ...ctx(), records: [] }); + expect(json(bad)).toHaveProperty("exit_code", 65); + expect(json(bad).errors).toBeInstanceOf(Array); + }); + + it("compile DISCLOSES the bounds it could not check", async () => { + expect(json(await planVerb(["compile", specPath], ctx())).unchecked).toEqual(["max_size_class", "tier"]); + }); + + it("compile points at the next step, since a plan alone authorises nothing", async () => { + expect(String(json(await planVerb(["compile", specPath], ctx())).next)).toMatch(/amico ledger approve --plan/); + }); + }); + + describe("status", () => { + const compiled = async () => { + const r = await planVerb(["compile", specPath], ctx()); + return String(json(r).plan_hash); + }; + + it("reports the derived step states and the plan state", async () => { + const hash = await compiled(); + const r = await planVerb(["status", hash], { plansDir, records: readRecords() }); + expect(r.code).toBe(0); + expect(json(r).plan_state).toBe("active"); + // Honest: nothing emits step verdicts until the harness walk lands (G-1b). + expect((json(r).steps as Array<{ state: string }>).map((s) => s.state)).toEqual(["pending"]); + }); + + it("defaults to the most recently compiled plan", async () => { + const hash = await compiled(); + expect(json(await planVerb(["status"], { plansDir, records: readRecords() })).plan_hash).toBe(hash); + }); + + it("an unknown plan_hash is a CLEAN empty answer, not a crash", async () => { + const r = await planVerb(["status", "f".repeat(64)], { plansDir, records: [] }); + expect(r.code).toBe(0); + expect(json(r).note).toMatch(/no plan with hash/); + }); + + it("says so when the ledger knows a plan but its note is missing", async () => { + const hash = await compiled(); + rmSync(join(plansDir), { recursive: true, force: true }); + mkdirSync(plansDir); + const r = await planVerb(["status", hash], { plansDir, records: readRecords() }); + expect(json(r).known_to_ledger).toBe(true); + expect(String(json(r).note)).toMatch(/its note was not found/); + }); + + it("with no plan compiled at all it reports that, at exit 0", async () => { + const r = await planVerb(["status"], { plansDir, records: [] }); + expect(r.code).toBe(0); + expect(json(r).note).toMatch(/no plan has been compiled yet/); + }); + + it("renders remaining warrant time from the APPROVAL, not from suggested_ttl_s", async () => { + // The fixture makes them DISAGREE on purpose: an implementation reading + // plan_compiled.suggested_ttl_s — which §4.6 forbids as a lifetime source — would report + // 7200s here instead of 1800s and still look plausible. + const hash = await compiled(); + const nowMs = Date.parse("2026-07-28T12:00:00.000Z"); + const rows: LedgerRecord[] = [ + ...readRecords(), + { type: "approval", ts: "t", plan_hash: hash, bounds: {}, expires_at: "2026-07-28T12:30:00.000Z", issued_by: "aaron" } as LedgerRecord, + ]; + const w = json(await planVerb(["status", hash], { plansDir, records: rows, nowMs: () => nowMs })).warrant as { + remaining_s: number; expired: boolean; issued_by: string; + }; + expect(w.remaining_s).toBe(1800); + expect(w.expired).toBe(false); + expect(w.issued_by).toBe("aaron"); + }); + + it("an UNPARSEABLE expiry reads as already expired (fail closed)", async () => { + const hash = await compiled(); + const rows: LedgerRecord[] = [ + ...readRecords(), + { type: "approval", ts: "t", plan_hash: hash, bounds: {}, expires_at: "whenever", issued_by: "aaron" } as LedgerRecord, + ]; + const w = json(await planVerb(["status", hash], { plansDir, records: rows })).warrant as { expired: boolean; remaining_s: number }; + expect(w).toMatchObject({ expired: true, remaining_s: 0 }); + }); + + it("surfaces that a plan was compiled from an unreviewed spec", async () => { + await planVerb(["compile", specPath, "--allow-unreviewed"], { ...ctx(), records: [] }); + const rows = readRecords(); + const hash = String((rows.find((r) => r.type === "plan_compiled") as { plan_hash: string }).plan_hash); + expect(json(await planVerb(["status", hash], { plansDir, records: rows })).not_adversarially_reviewed).toBe(true); + }); + }); + + describe("advisory", () => { + const withAdvisory = async () => { + const spec = { ...SPEC, review: { design_hash: "0".repeat(64), advisories: [{ id: "adv-1", lens: "hidden-failure", claim: "c", remedy: "r" }] } }; + writeFileSync(specPath, fm(spec)); + const r = await planVerb(["compile", specPath], { + plansDir, records: [approvedReview("0".repeat(64))], runPlanner: planner({ goal: "g", steps: STEPS }), + }); + return String(json(r).plan_hash); + }; + + it("REQUIRES --reason when --state waived", async () => { + // Surfaced as a usage error rather than an append failure: the schema's if/then would + // reject the row, but only after the user believed they had waived something. + const hash = await withAdvisory(); + const bad = await planVerb(["advisory", "adv-1", "--state", "waived", "--plan", hash], { plansDir, records: readRecords() }); + expect(bad.code).toBe(64); + expect(String(json(bad).error)).toMatch(/--reason is required/); + const ok = await planVerb(["advisory", "adv-1", "--state", "waived", "--reason", "out of scope", "--plan", hash], { + plansDir, records: readRecords(), + }); + expect(ok.code).toBe(0); + }); + + it("rejects a state outside {fixed, waived, obsolete}", async () => { + const hash = await withAdvisory(); + const r = await planVerb(["advisory", "adv-1", "--state", "done", "--plan", hash], { plansDir, records: readRecords() }); + expect(r.code).toBe(64); + expect(String(json(r).error)).toMatch(/must be one of/); + }); + + it("rejects a reason over the schema's 200-char cap BEFORE appending", async () => { + const hash = await withAdvisory(); + const r = await planVerb( + ["advisory", "adv-1", "--state", "waived", "--reason", "x".repeat(201), "--plan", hash], + { plansDir, records: readRecords() }, + ); + expect(r.code).toBe(64); + expect(readRecords().filter((x) => x.type === "todo")).toHaveLength(0); + }); + + it("rejects an id the plan never DECLARED — the completion denominator is not user-supplied", async () => { + const hash = await withAdvisory(); + const r = await planVerb(["advisory", "invented", "--state", "fixed", "--plan", hash], { plansDir, records: readRecords() }); + expect(r.code).toBe(64); + expect(String(json(r).error)).toMatch(/is not declared by plan/); + }); + + it("closing the advisory moves the plan toward complete", async () => { + const hash = await withAdvisory(); + await planVerb(["advisory", "adv-1", "--state", "fixed", "--plan", hash], { plansDir, records: readRecords() }); + const view = json(await planVerb(["status", hash], { plansDir, records: readRecords() })); + expect(view.open_advisories).toBe(0); + expect((view.advisories as Array<{ state: string }>)[0].state).toBe("fixed"); + }); + + it("requires an id", async () => { + expect((await planVerb(["advisory", "--state", "fixed"], { plansDir, records: [] })).code).toBe(64); + }); + + it("requires --state", async () => { + const hash = await withAdvisory(); + expect((await planVerb(["advisory", "adv-1", "--plan", hash], { plansDir, records: readRecords() })).code).toBe(64); + }); + }); + + // Rev 1 asserted the NAME `plan todo` was absent, which any unknown-subcommand-64 convention + // satisfies while `plan step --pass` would still exist. The property is what matters: NO + // subcommand may write step state. + describe("there is NO write path for step state", () => { + it("no subcommand appends a verdict or dispatch row", async () => { + const before = readRecords().length; + for (const sub of ["todo", "step", "pass", "complete", "advance", "verdict", "dispatch"]) { + await planVerb([sub, "s1", "--state", "fixed"], { plansDir, records: [] }); + } + const after = readRecords(); + expect(after.length).toBe(before); + expect(after.filter((r) => r.type === "verdict" || r.type === "dispatch")).toHaveLength(0); + }); + + it("the usage text says why, so a caller does not go looking for the flag", async () => { + expect(String(json(await planVerb([], {})).usage)).toMatch(/DERIVED from gate verdicts/); + }); + }); + + describe("registration", () => { + it("is in SPINE_VERBS with the fields Verb requires", () => { + const v = SPINE_VERBS.find((x) => x.name === "plan"); + expect(v).toBeDefined(); + expect(v!.summary).toBeTruthy(); + expect(v!.generalizes).toBeTruthy(); + expect(v!.slice).toBeTruthy(); + expect(v!.stub).toBeUndefined(); + }); + + it("appears in the MCP tool list (advisory A-4: registering publishes it)", () => { + expect(listMcpTools().map((t) => t.name)).toContain("amico_plan"); + }); + }); + + describe("the compiled plan is readable by a human, not just by the tool", () => { + it("names the steps and warns against hand-editing", async () => { + const r = await planVerb(["compile", specPath], ctx()); + const text = readFileSync(String(json(r).plan_path), "utf8"); + expect(text).toMatch(/# ship it/); + expect(text).toMatch(/\*\*s1\*\*/); + expect(text).toMatch(/do not hand-edit/i); + }); + }); +}); diff --git a/packages/amico-run/test/spec_verb.test.ts b/packages/amico-run/test/spec_verb.test.ts index e7315fd2..d9e4693f 100644 --- a/packages/amico-run/test/spec_verb.test.ts +++ b/packages/amico-run/test/spec_verb.test.ts @@ -80,6 +80,15 @@ describe("amico spec", () => { expect((await specVerb(["review", "--spec", path])).code).toBe(64); }); + it("rejects a TRAILING unknown flag, not just a leading one", async () => { + // The earlier `positional` returned at the first non-flag argument, so anything after the + // path went unvalidated — the same silent-ignore the known-flag set exists to prevent, just + // one position to the right. + writeFileSync(path, SLICE); + expect((await specVerb(["review", path, "--bogus"])).code).toBe(64); + expect((await specVerb(["review", path, "--critics=2", "--nope"])).code).toBe(64); + }); + it("`validate` checks the frontmatter contract alone", async () => { writeFileSync(path, SLICE); expect((await specVerb(["validate", path])).code).toBe(0); From d65736f11e1aca89b1e422a9f4cc64da6e780bf6 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 14:26:01 -0400 Subject: [PATCH 24/27] =?UTF-8?q?fix(amico-run):=20create=20dist/=20before?= =?UTF-8?q?=20mkdtemp=20=E2=80=94=20clean=20checkouts=20could=20not=20buil?= =?UTF-8?q?d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three failing CI jobs (fast, schema-roundtrip, vsix-gate) had one cause, and it was mine: the atomic-build fix from the front half calls `mkdtempSync(join("dist", "build-"))`, and `mkdtemp` does not create parent directories. On a clean checkout there is no `dist/`, so it threw `ENOENT: mkdtemp 'dist/build-XXXXXX'` and took down every job that builds. It could not fail locally. Any machine running this has a `dist/` left over from the previous build, so the bug was invisible to every developer who had already succeeded once — including through 37 consecutive green suite runs while I was chasing the parity flake. esbuild used to create the directory as a side effect of writing `outfile`; moving to a staging directory silently took that over without taking on the responsibility. Reproduced with `rm -rf dist && node esbuild.config.mjs`, fixed, and re-verified against a genuinely clean tree: `pnpm -r run build` from no dist at all, plus the full `vsce package` path that vsix-gate runs. CI's clean checkout is the right guard for this class and it is the one that caught it. A local test asserting it would have to `rm -rf dist`, which races the ten test files that rebuild the bundle in `beforeAll` while others execute it — the exact race the staging directory exists to prevent. Not worth reintroducing a real race to guard a failure mode a clean environment tests for free. 925 amico-run · 144 schema · 777 extension · both typechecks · clean-tree build · vsix packages. Co-Authored-By: Claude Opus 5 --- packages/amico-run/esbuild.config.mjs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/amico-run/esbuild.config.mjs b/packages/amico-run/esbuild.config.mjs index eb808dc1..3375adc9 100644 --- a/packages/amico-run/esbuild.config.mjs +++ b/packages/amico-run/esbuild.config.mjs @@ -1,5 +1,5 @@ import { build } from "esbuild"; -import { chmodSync, mkdtempSync, renameSync, rmSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, renameSync, rmSync } from "node:fs"; import { join } from "node:path"; // Three bins from one package: the historical `amico-run` (entry cli.ts), the `amico` @@ -54,6 +54,15 @@ const common = { // bundle's trailing `//# sourceMappingURL=` comment, so the shipped bundle would point // at a map that no longer exists — sourcemaps silently broken, tests all still green. // The URL is relative to the output file, so preserving the basename keeps it correct. +// +// `dist/` MUST be created first. `mkdtemp` does not create parent directories, so on a clean +// checkout — where no build has ever run — this threw `ENOENT: mkdtemp 'dist/build-XXXXXX'` and +// took down every CI job that builds (fast, schema-roundtrip, vsix-gate). It could not fail +// locally, because any developer running this has a `dist/` left over from the previous build: +// the bug was invisible to every machine that had already succeeded once. esbuild used to create +// the directory itself as a side effect of writing `outfile`, and moving to a staging dir +// silently took that over without taking on the responsibility. +mkdirSync("dist", { recursive: true }); const staging = mkdtempSync(join("dist", "build-")); try { for (const [entry, name] of [ From 45c968d7886b2b44b88fda6a6d25f63fca8b8093 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 14:33:01 -0400 Subject: [PATCH 25/27] test(amico-run): 20s timeout on the 24-writer append-safety test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last red CI check. Not an assertion failure — `Test timed out in 5000ms`. This is the heaviest test in the suite: 24 real node processes, each loading the bundle, all contending for one O_APPEND. Locally ~780ms; on a 2-core CI runner it measured 5322ms and crossed the 5s default the first time the bundle grew. MY CHANGE CAUSED THE GROWTH, and it is worth recording precisely rather than treating this as a flake. The bundle went 524 KB -> 878 KB (+67%) across this branch, and the metafile attributes 255 KB of that — 30% of the whole bundle — to `yaml`, which the front half's frontmatter.ts pulled in. 24 spawns x a 67% bigger bundle is the entire failure. Raised the timeout rather than reducing N. What this test asserts is a CORRECTNESS property — single-writer append atomicity under real concurrency — and 24 writers is the pressure that makes interleaving observable at all. Cutting N would keep it green while making it stop testing the thing. 20s rather than the suite's usual 15s convention (abort.test.ts, cli.test.ts) because it spawns more processes than either and should not need revisiting on the next size increase. `yaml` is KEPT deliberately. 255 KB is a real cost on a CLI spawned per verb call, and the alternative — hand-rolling a YAML subset for frontmatter — is the exact shape of defect this whole slice has been finding: a parser that silently misreads a field nobody notices until a spec is misjudged. Correctness over 250 KB, with the number written down so the trade is visible rather than assumed. 925 amico-run · 144 schema · 777 extension. Clean-tree build and vsce package verified in the previous commit. Co-Authored-By: Claude Opus 5 --- packages/amico-run/test/ledger_verb.test.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/amico-run/test/ledger_verb.test.ts b/packages/amico-run/test/ledger_verb.test.ts index 51cb3734..2b56f5ca 100644 --- a/packages/amico-run/test/ledger_verb.test.ts +++ b/packages/amico-run/test/ledger_verb.test.ts @@ -151,5 +151,17 @@ describe("amico ledger (bundle)", () => { // every line is intact, valid JSON, with the expected shape → no interleaving const sessions = lines.map((l) => JSON.parse(l).session).sort(); expect(sessions).toEqual(Array.from({ length: N }, (_, i) => `s${i}`).sort()); - }); + // 20s, not the default 5s. This is the heaviest test in the suite: 24 real node processes, + // each loading the ~880 KB bundle, all contending for one append. Locally that is ~780ms; on + // a 2-core CI runner it measured 5322ms and blew the 5s default the first time the bundle + // grew (the deliberation slice added `yaml`, +67%). + // + // The timeout is raised rather than N reduced, because what this asserts is a CORRECTNESS + // property — single-writer O_APPEND atomicity under real concurrency — and 24 writers is the + // pressure that makes interleaving observable. Trading that pressure for speed would keep the + // test green while making it stop testing the thing. + // + // 20s rather than the suite's usual 15s (abort.test.ts, cli.test.ts) for the same reason: it + // spawns more processes than either, so it needs more headroom against the next size increase. + }, 20_000); }); From 6183b68db1e1b86dc3a141d0a3cedc180f05ca88 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 14:44:57 -0400 Subject: [PATCH 26/27] chore(extension): pin skills.lock.json to skills-public-v1.6.0 (37 skills) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Marketplace users move from 16 public skills to 37. This is the last step of the chain — amico-plugin#50 merged, the tag cut off main, CI built and published the release, and this pins its asset. The sha256 is CROSS-CHECKED rather than taken from one source: computed from the downloaded tarball AND compared against the `.sha256` CI publishes beside it. Both a8400b10... A lock that pins a hash nobody verified against a second source is a lock in name only. Verified end to end through the real script, not by inspection: `pnpm run fetch:skills` reports "skills-public-v1.6.0 (37 public, 0 held)", the vendored MANIFEST reads skill_count 37 / plugin_version 1.6.0 / source_sha 95cf2851 (the merge commit), and `deliberate`, `brainstorming`, `fluxonium`, `ions`, `bosonic`, `pasqal` are all present. The vocabulary rename shipped intact: 20 skills carry `scenarios:`, zero carry the legacy `etudes:` key. The phantom API symbols are gone from the SHIPPED artifact — the four remaining grep hits are two lines of prose in `ions` explaining that `target_CNOT` and `GATES[:CNOT]` do not exist, and two inline function DEFINITIONS in `demo`, where the skill is teaching a reader to write their own helper. That is the declared carve-out, which is why the drift lint passes them. 777 extension tests green against the fetched bundle. Co-Authored-By: Claude Opus 5 --- package.json | 5 ++++- packages/extension/skills.lock.json | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e57da6a8..5c26dd43 100644 --- a/package.json +++ b/package.json @@ -6,5 +6,8 @@ "typecheck": "pnpm -r run typecheck", "test": "pnpm -r run test" }, - "packageManager": "pnpm@9.15.9" + "packageManager": "pnpm@9.15.9", + "workspaces": [ + "packages/*" + ] } diff --git a/packages/extension/skills.lock.json b/packages/extension/skills.lock.json index 8fd1f5c6..50a6d686 100644 --- a/packages/extension/skills.lock.json +++ b/packages/extension/skills.lock.json @@ -1,7 +1,7 @@ { - "version": "1.5.0", + "version": "1.6.0", "repo": "harmoniqs/amico-plugin", - "tag": "skills-public-v1.5.0", - "asset": "amico-skills-public-1.5.0.tar.gz", - "sha256": "5076e6a9d6ff805a1f542755683d025b6ed32050c55cc4cb526b8f806fd8bac4" + "tag": "skills-public-v1.6.0", + "asset": "amico-skills-public-1.6.0.tar.gz", + "sha256": "a8400b101eef68df8d5e3acec13bc922f98f45ce0d8da4747d1f46d791ec9eeb" } From e2e86d3ea01ac73eb8f5f692a08ad7ee80790c6f Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Tue, 28 Jul 2026 14:45:32 -0400 Subject: [PATCH 27/27] revert: drop the stray npm `workspaces` field from the root package.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not an intentional change. Some tool in the publish chain (vsce package or an npx invocation) wrote a npm-style `workspaces: ["packages/*"]` into the root package.json, and I committed it with the lock bump because I ran `git add -A` and did not read the diff first. It is redundant and potentially harmful: pnpm-workspace.yaml already declares `packages/*`, and this repo is pnpm-managed. A second npm-flavoured declaration gives npm-based tooling a reason to treat this as an npm workspace root, which is a behaviour change nobody asked for and nobody would have gone looking for. Worth naming the process failure rather than just the field: I had just finished writing that this slice's recurring defect is treating a passing signal as proof without asking what it could observe — then staged everything, saw green tests, and committed. Green tests cannot see an unreviewed diff. Co-Authored-By: Claude Opus 5 --- package.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/package.json b/package.json index 5c26dd43..e57da6a8 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,5 @@ "typecheck": "pnpm -r run typecheck", "test": "pnpm -r run test" }, - "packageManager": "pnpm@9.15.9", - "workspaces": [ - "packages/*" - ] + "packageManager": "pnpm@9.15.9" }