diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e04c58d..69c33dda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## 0.174.0 + +### `keepGoing` and `score`: spend the whole shot budget, ship the best tree + +`agenticGenerator`'s `Verifier` answered one boolean, and `ok` carried two meanings at once — "this tree is shippable" and "stop now". The shot loop returned inside `if (result.ok)` and `AgenticGeneratorOptions` exposed no field to override it, so ordinary best-of-n could not be expressed: a caller who wanted to spend the shots it was given and keep the best tree had no way to say so. + +A consumer building best-of-n hit exactly that and encoded it by hand (agent-lab's playproof study, `projects/playproof/verify-budget.ts`): reject every shot but the last whatever it measured, score each tree as it is produced, and physically write the best program back into the worktree just before the last shot is accepted — because with only one boolean, "ship the best tree" has to mean "make the best tree BE the worktree". Their measured runs are why it matters. Under first-acceptance-wins, 1 shot of 3 fired and the program that shipped was never once run by its author. Under the workaround, 3 of 3 fired and every shot ran its own program. + +`VerifyResult` gains two optional fields, so three separate questions get three separate answers: + +| field | question | omitted | +|---|---|---| +| `ok` | is this tree shippable | unchanged | +| `keepGoing` | should the budget stop here | the first passing tree ends the candidate, exactly as before | +| `score` | how does this tree rank against the other passing trees | every passing tree ties, so the later one wins | + +- **The loop owns the restore, and that is the point.** A passing tree whose verifier asks for another shot is snapshotted as a Git tree object (staged into a private index, so the index the driver commits from is untouched). When the budget ends, the highest-scoring tree is put back into the worktree — content, added files, and the removal of files only a losing shot wrote — and the restore is proved by re-snapshotting and comparing tree ids before the candidate is returned. The caller ranks; the runtime moves the bytes. Without this half, every caller wanting best-of-n still hand-rolls the write-back, which is the thing being fixed. +- **Compatibility is an explicit gate, not a claim.** `tests/agentic-generator.test.ts` drives the real `agenticGenerator` with a verifier returning today's shape and asserts today's behaviour: 1 of 3 shots fires, the disposition stream is exactly `['accepted']` with `restoredFromShot: null`, and the accepted shot's own tree is what lands. A second gate holds the failure path — a verifier that never passes still feeds `verification FAILED` into the next shot and still ships nothing. +- **A last shot that breaks or reverts the change no longer costs the candidate.** A banked tree passed verification, so it ships even when the final shot ends on a broken or empty tree. The failing shot's own `rejected` disposition is still emitted first, so its evidence survives. The invariant is intact: a tree that failed verification is never what ships. +- **A tie keeps the LATER tree.** It is already on disk, so no restore is needed, and it is the author's own refinement of the tree it tied with. +- **A set of trees that cannot be ordered fails the run.** Scoring one passing tree and not another throws rather than guessing an order, and a non-finite score throws. A tree that FAILED verification is never ranked, whatever it scored. +- **`onShotDisposition` gains `kept`**, the shot that passed and was sent back: it carries the `score`, whether the tree `best`s the candidate so far, and the verifier's `feedback`. `accepted` gains `restoredFromShot` — non-null is the record that best-of-n moved bytes rather than only ranking them. +- **The next-shot note is new text for a new state.** A passing tree sent back is told `verification PASSED`, that shots remain, and that the best version it produces is the one that ships — not the `verification FAILED` note, which would be a lie. +- **`cli-worktree`, `cli-in-place` and `commandVerifier` are unchanged.** This is the verify/shot-loop contract only; `commandVerifier` still answers `{ok:true}` / `{ok:false, feedback}` and still stops at the first passing tree. + ## 0.173.0 ### `cli-in-place`: a local coding CLI on the worktree you hand it diff --git a/api-surface.json b/api-surface.json index 213848d5..f07476e6 100644 --- a/api-surface.json +++ b/api-surface.json @@ -78,7 +78,7 @@ "AgentTaskStatus": "type 4087243dc453", "AgenticGeneratorExecutorForWorktree": "type a1493d23fed8", "AgenticGeneratorOptions": "type 1d5e47e57c3b", - "AgenticGeneratorShotDisposition": "type 657d4fd128fa", + "AgenticGeneratorShotDisposition": "type dc9795eb57ec", "AgenticGeneratorShotExecution": "type c0739e352ea6", "AgenticGeneratorShotReceipt": "type c73c53ad3197", "AnalystRegistry": "type 4be7b4a3eda8", @@ -323,7 +323,7 @@ "VerifiedAgentCandidate": "type a8159a2b2fc2", "VerifiedAgentCandidateTaskOutcome": "type e78360db85cb", "Verifier": "type c596e7edac69", - "VerifyResult": "type fe21ce933c15", + "VerifyResult": "type 0864be50c38c", "VetoedFact": "type b762517293cc", "WorkerTraceEvidence": "type 34812cbffcbf", "WorkerTraceUnavailableReason": "type 0dcb14d1071b", diff --git a/docs/api/index.md b/docs/api/index.md index 69301408..a9964ac7 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -4792,8 +4792,15 @@ so trace stitching survives nested orchestration. ### VerifyResult -Outcome of verifying a candidate worktree. `feedback` (compiler errors, - failing test output) is fed into the next shot when `ok` is false. +Outcome of verifying a candidate worktree. + +`ok` answers "is this tree shippable". `keepGoing` answers "should the budget +stop here", and `score` ranks this tree against the other trees the same +candidate produced — three separate questions, so a verifier can pass a tree +and still spend the shots it was given. + +`feedback` (compiler errors, failing test output, or the reason a passing +tree is being sent back) is fed into the next shot. #### Properties @@ -4805,6 +4812,28 @@ Outcome of verifying a candidate worktree. `feedback` (compiler errors, > `optional` **feedback?**: `string` +##### keepGoing? + +> `optional` **keepGoing?**: `boolean` + +Spend the remaining shots instead of returning this tree now. + +Read only when `ok` is true: a failed verification already spends the next +shot. Omitted means the first passing tree ends the candidate. + +##### score? + +> `optional` **score?**: `number` + +How good this tree is, for ranking it against the other passing trees of +this candidate. Higher wins; a tie keeps the LATER tree, which is the one +already on disk and the one the author refined last. + +Only a passing tree is ranked — a tree that failed verification is never a +candidate, whatever it scored. Score every passing tree or none of them: a +scored tree cannot be ranked against an unscored one, and mixing the two +fails the run rather than guessing an order. + *** ### AgenticGeneratorShotReceipt @@ -5020,8 +5049,10 @@ readonly `ProposalFinding`[] Verify the worktree after each dirtying shot. When set, a candidate that fails verification is NOT returned — the failure feeds the next shot (verify-in-session), up to `maxShots`; a candidate that never verifies is - discarded (`applied:false`), never shipped. Omitted means the first dirty - shot is the candidate. See `commandVerifier`. + discarded (`applied:false`), never shipped. A verifier that returns + `keepGoing` passes a tree AND spends the remaining shots, and the + best-scoring tree is the one that ships. Omitted means the first dirty + shot is the candidate. See `commandVerifier` and `VerifyResult`. ##### isDirty? @@ -11297,12 +11328,90 @@ Runtime's exact terminal turn plus its complete normalized event stream. ### AgenticGeneratorShotDisposition -> **AgenticGeneratorShotDisposition** = \{ `kind`: `"clean"`; `worktreePath`: `string`; \} \| \{ `kind`: `"rejected"`; `worktreePath`: `string`; `stage`: `"raw-trace-evidence"` \| `"verification"`; `feedback`: `string` \| `null`; \} \| \{ `kind`: `"accepted"`; `worktreePath`: `string`; `verified`: `boolean`; \} \| \{ `kind`: `"setup-error"`; `worktreePath`: `string`; `stage`: `"worktree-inspection"` \| `"raw-trace-evidence"` \| `"verification"`; `error`: \{ `name`: `string`; `message`: `string`; \}; \} +> **AgenticGeneratorShotDisposition** = \{ `kind`: `"clean"`; `worktreePath`: `string`; \} \| \{ `kind`: `"rejected"`; `worktreePath`: `string`; `stage`: `"raw-trace-evidence"` \| `"verification"`; `feedback`: `string` \| `null`; \} \| \{ `kind`: `"kept"`; `worktreePath`: `string`; `score`: `number` \| `null`; `best`: `boolean`; `feedback`: `string` \| `null`; \} \| \{ `kind`: `"accepted"`; `worktreePath`: `string`; `verified`: `boolean`; `restoredFromShot`: `number` \| `null`; \} \| \{ `kind`: `"setup-error"`; `worktreePath`: `string`; `stage`: `"worktree-inspection"` \| `"raw-trace-evidence"` \| `"verification"`; `error`: \{ `name`: `string`; `message`: `string`; \}; \} Worktree decision emitted before a completed shot is retried, accepted, or discarded. The callback runs while `worktreePath` is still available, so callers can persist the exact diff. +#### Union Members + +##### Type Literal + +\{ `kind`: `"clean"`; `worktreePath`: `string`; \} + +*** + +##### Type Literal + +\{ `kind`: `"rejected"`; `worktreePath`: `string`; `stage`: `"raw-trace-evidence"` \| `"verification"`; `feedback`: `string` \| `null`; \} + +*** + +##### Type Literal + +\{ `kind`: `"kept"`; `worktreePath`: `string`; `score`: `number` \| `null`; `best`: `boolean`; `feedback`: `string` \| `null`; \} + +###### kind + +> `readonly` **kind**: `"kept"` + +The tree passed verification and the verifier asked for another shot, + so it was snapshotted and the budget continues. + +###### worktreePath + +> `readonly` **worktreePath**: `string` + +###### score + +> `readonly` **score**: `number` \| `null` + +The rank the verifier gave this tree, or null when it scored nothing. + +###### best + +> `readonly` **best**: `boolean` + +Whether this tree is now the best one this candidate has produced. + +###### feedback + +> `readonly` **feedback**: `string` \| `null` + +*** + +##### Type Literal + +\{ `kind`: `"accepted"`; `worktreePath`: `string`; `verified`: `boolean`; `restoredFromShot`: `number` \| `null`; \} + +###### kind + +> `readonly` **kind**: `"accepted"` + +###### worktreePath + +> `readonly` **worktreePath**: `string` + +###### verified + +> `readonly` **verified**: `boolean` + +###### restoredFromShot + +> `readonly` **restoredFromShot**: `number` \| `null` + +One-based shot whose tree was put back into the worktree because it + outranked the tree on disk; null when the tree on disk is the one that + ships. Non-null is the record that best-of-n moved bytes rather than + only ranking them. + +*** + +##### Type Literal + +\{ `kind`: `"setup-error"`; `worktreePath`: `string`; `stage`: `"worktree-inspection"` \| `"raw-trace-evidence"` \| `"verification"`; `error`: \{ `name`: `string`; `message`: `string`; \}; \} + *** ### AgenticGeneratorExecutorForWorktree diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index dfaad340..fa2a66e5 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.173.0` and `@tangle-network/agent-eval@0.170.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.174.0` and `@tangle-network/agent-eval@0.170.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -231,7 +231,7 @@ Import from `@tangle-network/agent-runtime` — 440 exports. | `SpendGap` | interface | One journaled node whose usage accounting is incomplete — the named gap behind a `false` | | `SqlAdapter` | interface | Minimal SQL driver shape. Implementations forward to whichever client the | | `Supervisor` | interface | Owns the conserved pool, the spawn log, the abort cascade, the OTP intensity breaker, | -| `VerifyResult` | interface | Outcome of verifying a candidate worktree. `feedback` (compiler errors, | +| `VerifyResult` | interface | Outcome of verifying a candidate worktree. | | `AgentCandidateBundleInput` | type | Exact candidate wire shape before the runtime computes its canonical digest. | | `AgentCandidateCodeSource` | type | Explicit control/no-op code or one finalized CodeSurface whose bytes must still verify. | | `AgentCandidateExecutionClaimResult` | type | Result of atomically claiming one execution attempt. | diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 297f1594..93cb666e 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.173.0.** +> **Version 0.174.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.163.2 <0.171.0`. > `sandbox` must satisfy `>=0.31.0 <0.32.0`. diff --git a/package.json b/package.json index 3fa6d3e4..577bf49a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.173.0", + "version": "0.174.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/improvement/agentic-generator.ts b/src/improvement/agentic-generator.ts index 6e516292..3951c49f 100644 --- a/src/improvement/agentic-generator.ts +++ b/src/improvement/agentic-generator.ts @@ -23,10 +23,20 @@ * (the worktree persists, so the harness RESUMES atop its own failing * edits with the error in hand — no session-specific retry path needed) * - dirty + `verify` ok (or no verifier configured) → return the candidate + * - dirty + `verify` ok + `keepGoing` → bank the tree and spend the next shot * A candidate that never verifies within `maxShots` is discarded (`applied: * false`), never shipped — if you configured a verifier, a non-passing tree is * not a candidate. With no verifier, the first dirty shot is the candidate. * + * BEST-OF-N is the `keepGoing` path, and the loop owns it end to end. A + * verifier that passes a tree and asks for another shot has its tree + * snapshotted as a Git tree object; when the budget ends, the highest-`score` + * tree is RESTORED into the worktree and returned as the candidate. So the + * caller ranks and the loop moves the bytes — a caller never has to write a + * passing tree back itself. The budget ends on the last shot, or earlier on a + * `keepGoing`-less pass, and a last shot that broke or reverted the change does + * not cost the candidate the verified tree an earlier shot produced. + * * @stable */ @@ -62,16 +72,44 @@ import { createExecutor, type ExecutorConfig } from '../runtime/supervise/runtim import { detachedSnapshot } from '../runtime/supervise/snapshot' import type { CandidateGenerator } from './improvement-driver' import { optimizerMethod } from './optimizer-prompt' +import { restoreWorktreeTree, snapshotWorktreeTree } from './worktree-tree' const RAW_TRACE_ANALYST_ID = 'raw-trace-distiller' const RAW_TRACE_AREA = 'raw-trace-context' const RAW_TRACE_DIAGNOSIS_PATH = '.improve/raw-trace-diagnosis.md' -/** Outcome of verifying a candidate worktree. `feedback` (compiler errors, - * failing test output) is fed into the next shot when `ok` is false. */ +/** + * Outcome of verifying a candidate worktree. + * + * `ok` answers "is this tree shippable". `keepGoing` answers "should the budget + * stop here", and `score` ranks this tree against the other trees the same + * candidate produced — three separate questions, so a verifier can pass a tree + * and still spend the shots it was given. + * + * `feedback` (compiler errors, failing test output, or the reason a passing + * tree is being sent back) is fed into the next shot. + */ export interface VerifyResult { ok: boolean feedback?: string + /** + * Spend the remaining shots instead of returning this tree now. + * + * Read only when `ok` is true: a failed verification already spends the next + * shot. Omitted means the first passing tree ends the candidate. + */ + keepGoing?: boolean + /** + * How good this tree is, for ranking it against the other passing trees of + * this candidate. Higher wins; a tie keeps the LATER tree, which is the one + * already on disk and the one the author refined last. + * + * Only a passing tree is ranked — a tree that failed verification is never a + * candidate, whatever it scored. Score every passing tree or none of them: a + * scored tree cannot be ranked against an unscored one, and mixing the two + * fails the run rather than guessing an order. + */ + score?: number } /** Verifies the edited worktree. Sync or async; throws only on a setup fault @@ -129,10 +167,26 @@ export type AgenticGeneratorShotDisposition = readonly stage: 'raw-trace-evidence' | 'verification' readonly feedback: string | null } + | { + /** The tree passed verification and the verifier asked for another shot, + * so it was snapshotted and the budget continues. */ + readonly kind: 'kept' + readonly worktreePath: string + /** The rank the verifier gave this tree, or null when it scored nothing. */ + readonly score: number | null + /** Whether this tree is now the best one this candidate has produced. */ + readonly best: boolean + readonly feedback: string | null + } | { readonly kind: 'accepted' readonly worktreePath: string readonly verified: boolean + /** One-based shot whose tree was put back into the worktree because it + * outranked the tree on disk; null when the tree on disk is the one that + * ships. Non-null is the record that best-of-n moved bytes rather than + * only ranking them. */ + readonly restoredFromShot: number | null } | { readonly kind: 'setup-error' @@ -173,8 +227,10 @@ export interface AgenticGeneratorOptions { /** Verify the worktree after each dirtying shot. When set, a candidate that * fails verification is NOT returned — the failure feeds the next shot * (verify-in-session), up to `maxShots`; a candidate that never verifies is - * discarded (`applied:false`), never shipped. Omitted means the first dirty - * shot is the candidate. See `commandVerifier`. */ + * discarded (`applied:false`), never shipped. A verifier that returns + * `keepGoing` passes a tree AND spends the remaining shots, and the + * best-scoring tree is the one that ships. Omitted means the first dirty + * shot is the candidate. See `commandVerifier` and `VerifyResult`. */ verify?: Verifier /** Test seam — inject the worktree-dirty check (defaults to `git status`). */ isDirty?: (worktreePath: string) => boolean @@ -241,9 +297,46 @@ export function agenticGenerator(opts: AgenticGeneratorOptions): CandidateGenera const shots = Math.max(1, maxShots) // Feedback appended to the base prompt for the NEXT shot — empty on shot 0. let attemptNote = '' + // The best tree this candidate has produced, and where it lives. Only a + // verifier that returns `keepGoing` can put more than one tree here: a + // passing tree that ends the budget is banked and shipped in one step. + let best: BankedTree | null = null + // Whether the verifier scores its trees. Set by the first passing tree; + // a later tree that disagrees cannot be ranked and fails the run. + let scored: boolean | null = null + let lastReceipt: AgenticGeneratorShotReceipt | null = null + + /** Ship the best tree this candidate produced, restoring it when a later + * shot wrote over it. */ + const shipBankedTree = async ( + receipt: AgenticGeneratorShotReceipt, + banked: BankedTree, + ): Promise> => { + const restoredFromShot = banked.onDisk ? null : banked.shot + if (!banked.onDisk) { + if (banked.tree === null) { + throw new Error( + `agenticGenerator: shot ${banked.shot} produced the best tree but it was never snapshotted`, + ) + } + restoreWorktreeTree(worktreePath, banked.tree) + banked.onDisk = true + } + signal.throwIfAborted() + await emitShotDisposition(opts.onShotDisposition, receipt, { + kind: 'accepted', + worktreePath, + verified: true, + restoredFromShot, + }) + signal.throwIfAborted() + return acceptedCandidate(findings) + } for (let shot = 0; shot < shots; shot++) { signal.throwIfAborted() + // This shot may write over whatever the worktree held. + if (best) best.onDisk = false const taskPrompt = attemptNote ? `${basePrompt}\n\n${attemptNote}` : basePrompt const startedAt = new Date() let turn: CollectedAgentTurn | null = null @@ -317,6 +410,7 @@ export function agenticGenerator(opts: AgenticGeneratorOptions): CandidateGenera costReceipt, error: shotError, }) + lastReceipt = receipt await emitShotReceipt(opts.onShotCompleted, receipt, execution, shotError) signal.throwIfAborted() @@ -386,6 +480,7 @@ export function agenticGenerator(opts: AgenticGeneratorOptions): CandidateGenera kind: 'accepted', worktreePath, verified: false, + restoredFromShot: null, }) signal.throwIfAborted() return acceptedCandidate(findings) @@ -406,14 +501,39 @@ export function agenticGenerator(opts: AgenticGeneratorOptions): CandidateGenera ) } if (result.ok) { + const score = admittedScore(result, scored, shot) + scored = score !== null + const previousBest = best + let banked: BankedTree + // A tie keeps the LATER tree: it is already on disk, and it is the + // author's own refinement of the tree it tied with. `admittedScore` + // refuses a mix of scored and unscored trees, so an unscored set is a + // flat 0 and this rule reduces to "the later tree wins". + if (previousBest === null || (score ?? 0) >= (previousBest.score ?? 0)) { + banked = { shot: shot + 1, score, tree: null, onDisk: true } + } else { + banked = previousBest + } + const becomesBest = banked !== previousBest + best = banked + if (result.keepGoing !== true) { + signal.throwIfAborted() + return await shipBankedTree(receipt, banked) + } + // Shots remain, so a later one can write over this tree. Bank the + // bytes now, while they are still on disk. + if (becomesBest && shot < shots - 1) banked.tree = snapshotWorktreeTree(worktreePath) signal.throwIfAborted() await emitShotDisposition(opts.onShotDisposition, receipt, { - kind: 'accepted', + kind: 'kept', worktreePath, - verified: true, + score, + best: becomesBest, + feedback: result.feedback ?? null, }) signal.throwIfAborted() - return acceptedCandidate(findings) + attemptNote = keptNote(result.feedback) + continue } signal.throwIfAborted() await emitShotDisposition(opts.onShotDisposition, receipt, { @@ -427,12 +547,52 @@ export function agenticGenerator(opts: AgenticGeneratorOptions): CandidateGenera attemptNote = failureNote(result.feedback) } - // Shots exhausted: no verified candidate (or, sans verifier, no edits). + // Shots exhausted. A banked tree passed verification, so it ships even + // though the last shot did not end the budget on it — discarding paid, + // verified work because a later shot broke or reverted it is the loss + // best-of-n exists to prevent. + if (best !== null) { + if (!lastReceipt) { + throw new Error('agenticGenerator: a tree was banked without a shot receipt') + } + return await shipBankedTree(lastReceipt, best) + } + // No verified candidate (or, sans verifier, no edits). return { applied: false, summary: '' } }, } } +/** A tree that passed verification, and whether the worktree still holds it. */ +interface BankedTree { + /** One-based shot that produced it. */ + readonly shot: number + /** Its rank, or null when the verifier scores nothing. */ + readonly score: number | null + /** Its Git tree id, written only while a later shot can still overwrite it. */ + tree: string | null + onDisk: boolean +} + +/** The rank a passing tree carries, refusing a set of trees that cannot be ordered. */ +function admittedScore(result: VerifyResult, scored: boolean | null, shot: number): number | null { + const has = result.score !== undefined + if (scored !== null && has !== scored) { + throw new Error( + `agenticGenerator: verify ${has ? 'scored' : 'did not score'} the tree from shot ${shot + 1} and ${ + scored ? 'scored' : 'did not score' + } an earlier passing tree; a scored tree cannot be ranked against an unscored one`, + ) + } + if (!has) return null + if (typeof result.score !== 'number' || !Number.isFinite(result.score)) { + throw new Error( + `agenticGenerator: verify returned a non-finite score (${String(result.score)}) for shot ${shot + 1}`, + ) + } + return result.score +} + async function emitShotReceipt( callback: AgenticGeneratorOptions['onShotCompleted'], receipt: AgenticGeneratorShotReceipt, @@ -649,6 +809,19 @@ function failureNote(feedback?: string): string { ].join('\n') } +/** Next-shot feedback when the worktree PASSED and the verifier asked for + * another shot. The passing tree is banked, so the author is told to improve + * it rather than protect it — a worse tree cannot cost it the candidate. */ +function keptNote(feedback?: string): string { + const detail = feedback?.trim() + return [ + 'NOTE: your edits are in the working tree and verification PASSED.', + 'Shots remain in this budget — keep improving the change in place, do not revert it.', + 'The best version you produce is the one that ships.', + detail ? `Verifier output:\n${truncate(detail, 4000)}` : 'No verifier detail was captured.', + ].join('\n') +} + function rawTraceEvidenceProblem( worktreePath: string, findings: ReadonlyArray, diff --git a/src/improvement/worktree-tree.ts b/src/improvement/worktree-tree.ts new file mode 100644 index 00000000..db144c1f --- /dev/null +++ b/src/improvement/worktree-tree.ts @@ -0,0 +1,73 @@ +/** + * The exact content of a candidate worktree, as a Git tree object. + * + * A multi-shot candidate edits ONE directory in place, so the tree a shot + * produced is gone as soon as the next shot writes over it. Writing that + * content into the object store is what makes an earlier tree recoverable: + * `agenticGenerator` snapshots a tree that verified, and puts it back when a + * later shot ends the budget on a worse one. + * + * The snapshot stages into a PRIVATE index file, so the index the driver later + * commits from is untouched. + */ + +import { spawnSync } from 'node:child_process' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +/** Write the worktree's current content into the object store and return its tree id. */ +export function snapshotWorktreeTree(worktreePath: string): string { + const scratch = mkdtempSync(join(tmpdir(), 'agentic-generator-tree-')) + const indexFile = join(scratch, 'index') + try { + // HEAD first: a tracked file that `.gitignore` also matches stays in the + // tree. Against an EMPTY index `git add -A` reads that file as untracked + // and drops it, and restoring such a tree would delete a tracked file. + git(worktreePath, ['read-tree', 'HEAD'], indexFile) + git(worktreePath, ['add', '--all'], indexFile) + return git(worktreePath, ['write-tree'], indexFile) + } finally { + rmSync(scratch, { recursive: true, force: true }) + } +} + +/** + * Put a snapshotted tree back into the worktree, then prove the directory + * holds exactly that tree. + * + * The proof is not ceremony: a restore that lands the wrong bytes ships the + * wrong candidate, and every artifact downstream still reads as though the + * best tree shipped. + */ +export function restoreWorktreeTree(worktreePath: string, tree: string): void { + git(worktreePath, ['read-tree', '-u', '--reset', tree]) + // `read-tree` removes what the index tracked. A file a later shot added is + // untracked, so it survives that and would ship beside the restored tree. + // Ignored files are left alone, exactly as a commit leaves them. + git(worktreePath, ['clean', '--force', '-d', '--quiet']) + const restored = snapshotWorktreeTree(worktreePath) + if (restored !== tree) { + throw new Error( + `agenticGenerator: restoring tree ${tree} into ${worktreePath} produced ${restored}`, + ) + } +} + +function git(cwd: string, args: string[], indexFile?: string): string { + const env = { ...process.env } + if (indexFile) env.GIT_INDEX_FILE = indexFile + else delete env.GIT_INDEX_FILE + const result = spawnSync('git', args, { cwd, encoding: 'utf-8', env }) + if (result.error) { + throw new Error( + `agenticGenerator: git ${args[0]} failed to spawn in ${cwd}: ${result.error.message}`, + ) + } + if (result.status !== 0) { + throw new Error( + `agenticGenerator: git ${args[0]} exited ${result.status} in ${cwd}: ${result.stderr.trim()}`, + ) + } + return result.stdout.trim() +} diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 28666d4d..b80d1d62 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:3c686da480353cc11747a45c7250f7cc49f081cdb98ac9b1108540ac19513468", + "digest": "sha256:fb623d9857d9aa2e1655ec5b1b773f4ba1722af5d41381a6877a6707b8f8fefa", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.173.0" + "runtimeVersion": "0.174.0" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:cf79cfe76194bd2d120d247945b3f4af2ca24f05f7300d1dd829e92ae058babd", - "runId": "agent-runtime-0.173.0-proposal-fixture", + "recordDigest": "sha256:230b0be2647416eda41912e403fb94ba81bb2e2acb3026985b3e9a085f2c615d", + "runId": "agent-runtime-0.174.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.173.0-proposal-fixture" + "runId": "agent-runtime-0.174.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 485b824e..c02c0e21 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:24ef0f56f54f5f1cf2a8eae14b5d7375494566ae00bfeaf38e60fb0fede31387", + "digest": "sha256:d190ec44331b7a6223e18a5b6dd0bfb8281dc998365c09dde811b4c94acf6100", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.173.0" + "runtimeVersion": "0.174.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:9c90c1b08e40421738f375cffb0aa5f3755ead1ce49e6f5c4b540a3efd8c51f3", + "recordDigest": "sha256:d88298a446b40eea656d343f6f80169c04df2725c21bdceb22ce21b38c2958e2", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/agentic-generator.test.ts b/tests/agentic-generator.test.ts index ce94736d..ebcad0c4 100644 --- a/tests/agentic-generator.test.ts +++ b/tests/agentic-generator.test.ts @@ -1,5 +1,13 @@ import { execFileSync } from 'node:child_process' -import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs' +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { CostLedger, makeProposalFinding, type ProposalFinding } from '@tangle-network/agent-eval' @@ -667,6 +675,305 @@ describe('agenticGenerator on a cli-in-place placement', () => { }) }) +/** + * Spending the whole shot budget and shipping the best tree. + * + * `ok` used to answer two questions at once — "this tree is shippable" and + * "stop now" — so a caller who wanted best-of-n had to reject every shot but + * the last and write the winning tree back into the worktree itself. + * `keepGoing` and `score` separate the two, and the loop owns the restore. + */ +describe('agenticGenerator best-of-n over the shot budget', () => { + /** Write a distinct tree per shot, and record the prompt each shot was given. */ + function authorPerShot( + prompts: string[], + write: (worktreePath: string, shot: number) => void, + ): AgenticGeneratorExecutorForWorktree { + return routedExecutor(({ worktreePath, body, call }) => { + prompts.push( + String(messages(body).findLast((message) => message.role === 'user')?.content ?? ''), + ) + write(worktreePath, call) + }) + } + + it('COMPATIBILITY: a verifier returning todays shape still stops at the first passing tree', async () => { + const prompts: string[] = [] + const dispositions: AgenticGeneratorShotDisposition[] = [] + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: authorPerShot(prompts, (worktreePath, shot) => { + writeFileSync(join(worktreePath, 'app.ts'), `export const x = ${shot + 1}\n`) + }), + buildPrompt, + // Today's shape byte for byte: one boolean, and an optional string on failure. + verify: () => ({ ok: true }), + onShotDisposition: (_receipt, disposition) => dispositions.push(disposition), + }) + const worktreePath = await candidateWorktree('compatibility-first-acceptance') + + const result = await generator.generate(generateArgs(worktreePath, FINDINGS, 3)) + + expect(result.applied).toBe(true) + // One of three shots fired: the first passing tree still ends the candidate. + expect(prompts).toHaveLength(1) + expect(dispositions).toEqual([ + { kind: 'accepted', worktreePath, verified: true, restoredFromShot: null }, + ]) + // The tree the accepted shot left is the tree that ships. Nothing was restored. + expect(readFileSync(join(worktreePath, 'app.ts'), 'utf8')).toBe('export const x = 2\n') + }) + + it('COMPATIBILITY: a failing verifier still feeds the next shot and never ships', async () => { + const prompts: string[] = [] + const dispositions: AgenticGeneratorShotDisposition[] = [] + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: authorPerShot(prompts, (worktreePath, shot) => { + writeFileSync(join(worktreePath, 'app.ts'), `export const x = broken${shot}\n`) + }), + buildPrompt, + verify: () => ({ ok: false, feedback: 'cannot find name broken' }), + onShotDisposition: (_receipt, disposition) => dispositions.push(disposition), + }) + const worktreePath = await candidateWorktree('compatibility-never-verifies') + + const result = await generator.generate(generateArgs(worktreePath, FINDINGS, 3)) + + expect(result.applied).toBe(false) + expect(prompts).toHaveLength(3) + expect(prompts[1]).toContain('verification FAILED') + expect(prompts[1]).toContain('cannot find name broken') + expect(dispositions.map((disposition) => disposition.kind)).toEqual([ + 'rejected', + 'rejected', + 'rejected', + ]) + }) + + it('spends every shot when a passing verifier asks to keep going', async () => { + const prompts: string[] = [] + const dispositions: AgenticGeneratorShotDisposition[] = [] + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: authorPerShot(prompts, (worktreePath, shot) => { + writeFileSync(join(worktreePath, 'app.ts'), `export const x = ${shot + 1}\n`) + }), + buildPrompt, + verify: () => ({ ok: true, keepGoing: true, score: 1, feedback: 'it played 300 turns' }), + onShotDisposition: (_receipt, disposition) => dispositions.push(disposition), + }) + const worktreePath = await candidateWorktree('spends-every-shot') + + const result = await generator.generate(generateArgs(worktreePath, FINDINGS, 3)) + + expect(result.applied).toBe(true) + // Three of three shots fired, each one reading the passing note. + expect(prompts).toHaveLength(3) + expect(prompts[0]).not.toContain('verification PASSED') + expect(prompts[1]).toContain('verification PASSED') + expect(prompts[1]).toContain('it played 300 turns') + expect(prompts[2]).toContain('verification PASSED') + expect(dispositions.map((disposition) => disposition.kind)).toEqual([ + 'kept', + 'kept', + 'kept', + 'accepted', + ]) + expect(dispositions[0]).toMatchObject({ kind: 'kept', score: 1, best: true }) + // Every tree tied, so the LAST one wins and nothing had to be put back. + expect(dispositions.at(-1)).toEqual({ + kind: 'accepted', + worktreePath, + verified: true, + restoredFromShot: null, + }) + expect(readFileSync(join(worktreePath, 'app.ts'), 'utf8')).toBe('export const x = 4\n') + }) + + it('restores the best tree when the best shot is not the last', async () => { + const prompts: string[] = [] + const dispositions: AgenticGeneratorShotDisposition[] = [] + const scores = [1, 5, 2] + let shot = 0 + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: authorPerShot(prompts, (worktreePath, call) => { + writeFileSync(join(worktreePath, 'app.ts'), `export const x = ${call + 1}\n`) + // Shot 2's tree also adds a file; shot 3's adds a different one. Only + // the winner's extra file may survive the restore. + if (call === 2) writeFileSync(join(worktreePath, 'best.ts'), 'export const best = true\n') + if (call === 3) writeFileSync(join(worktreePath, 'worse.ts'), 'export const worse = true\n') + }), + buildPrompt, + verify: () => { + const score = scores[shot] + shot += 1 + return { ok: true, keepGoing: true, score } + }, + onShotDisposition: (_receipt, disposition) => dispositions.push(disposition), + }) + const worktreePath = await candidateWorktree('restores-the-best-tree') + + const result = await generator.generate(generateArgs(worktreePath, FINDINGS, 3)) + + expect(result.applied).toBe(true) + expect(prompts).toHaveLength(3) + expect(dispositions.map((disposition) => disposition.kind)).toEqual([ + 'kept', + 'kept', + 'kept', + 'accepted', + ]) + expect(dispositions[1]).toMatchObject({ kind: 'kept', score: 5, best: true }) + expect(dispositions[2]).toMatchObject({ kind: 'kept', score: 2, best: false }) + expect(dispositions.at(-1)).toEqual({ + kind: 'accepted', + worktreePath, + verified: true, + restoredFromShot: 2, + }) + // Shot 2's exact tree ships: its content, its file, and none of shot 3's. + expect(readFileSync(join(worktreePath, 'app.ts'), 'utf8')).toBe('export const x = 3\n') + expect(readFileSync(join(worktreePath, 'best.ts'), 'utf8')).toBe('export const best = true\n') + expect(existsSync(join(worktreePath, 'worse.ts'))).toBe(false) + }) + + it('ships the banked tree when the last shot breaks the change', async () => { + const dispositions: AgenticGeneratorShotDisposition[] = [] + const prompts: string[] = [] + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: authorPerShot(prompts, (worktreePath, call) => { + writeFileSync( + join(worktreePath, 'app.ts'), + call === 1 ? 'export const x = 2\n' : 'export const x = broken\n', + ) + }), + buildPrompt, + verify: (candidatePath) => + readFileSync(join(candidatePath, 'app.ts'), 'utf8').includes('broken') + ? { ok: false, feedback: 'cannot find name broken' } + : { ok: true, keepGoing: true, score: 3 }, + onShotDisposition: (_receipt, disposition) => dispositions.push(disposition), + }) + const worktreePath = await candidateWorktree('last-shot-regresses') + + const result = await generator.generate(generateArgs(worktreePath, FINDINGS, 2)) + + expect(result.applied).toBe(true) + expect(dispositions.map((disposition) => disposition.kind)).toEqual([ + 'kept', + 'rejected', + 'accepted', + ]) + // The rejection's own evidence survives; the verified tree still ships. + expect(dispositions[1]).toMatchObject({ + kind: 'rejected', + stage: 'verification', + feedback: 'cannot find name broken', + }) + expect(dispositions.at(-1)).toMatchObject({ kind: 'accepted', restoredFromShot: 1 }) + expect(readFileSync(join(worktreePath, 'app.ts'), 'utf8')).toBe('export const x = 2\n') + }) + + it('ships nothing when no shot of the budget ever verifies', async () => { + const dispositions: AgenticGeneratorShotDisposition[] = [] + const prompts: string[] = [] + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: authorPerShot(prompts, (worktreePath, call) => { + writeFileSync(join(worktreePath, 'app.ts'), `export const x = broken${call}\n`) + }), + buildPrompt, + // The floor: a tree that never played is refused at every shot, last included. + verify: () => ({ ok: false, feedback: 'the program never played' }), + onShotDisposition: (_receipt, disposition) => dispositions.push(disposition), + }) + const worktreePath = await candidateWorktree('nothing-ever-verifies') + + const result = await generator.generate(generateArgs(worktreePath, FINDINGS, 3)) + + expect(result.applied).toBe(false) + expect(result.summary).toBe('') + expect(prompts).toHaveLength(3) + expect(dispositions.map((disposition) => disposition.kind)).toEqual([ + 'rejected', + 'rejected', + 'rejected', + ]) + }) + + it('refuses a budget whose passing trees cannot be ordered', async () => { + const prompts: string[] = [] + let shot = 0 + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: authorPerShot(prompts, (worktreePath, call) => { + writeFileSync(join(worktreePath, 'app.ts'), `export const x = ${call + 1}\n`) + }), + buildPrompt, + verify: () => { + shot += 1 + return shot === 1 ? { ok: true, keepGoing: true, score: 4 } : { ok: true, keepGoing: true } + }, + }) + const worktreePath = await candidateWorktree('unorderable-scores') + + await expect(generator.generate(generateArgs(worktreePath, FINDINGS, 2))).rejects.toThrow( + /cannot be ranked against an unscored one/, + ) + }) + + it('refuses a non-finite score', async () => { + const prompts: string[] = [] + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: authorPerShot(prompts, (worktreePath) => { + writeFileSync(join(worktreePath, 'app.ts'), 'export const x = 2\n') + }), + buildPrompt, + verify: () => ({ ok: true, keepGoing: true, score: Number.NaN }), + }) + const worktreePath = await candidateWorktree('non-finite-score') + + await expect(generator.generate(generateArgs(worktreePath, FINDINGS, 2))).rejects.toThrow( + /non-finite score/, + ) + }) + + it('ships the banked tree when the last shot reverts every edit', async () => { + const dispositions: AgenticGeneratorShotDisposition[] = [] + const prompts: string[] = [] + const generator = agenticGenerator({ + profile: PROFILE, + executorForWorktree: authorPerShot(prompts, (worktreePath, call) => { + // Shot 2 puts the worktree back to its base state, so the dirty check + // reads it as an empty tree. + writeFileSync( + join(worktreePath, 'app.ts'), + call === 1 ? 'export const x = 2\n' : 'export const x = 1\n', + ) + }), + buildPrompt, + verify: () => ({ ok: true, keepGoing: true, score: 7 }), + onShotDisposition: (_receipt, disposition) => dispositions.push(disposition), + }) + const worktreePath = await candidateWorktree('last-shot-reverts') + + const result = await generator.generate(generateArgs(worktreePath, FINDINGS, 2)) + + expect(result.applied).toBe(true) + expect(dispositions.map((disposition) => disposition.kind)).toEqual([ + 'kept', + 'clean', + 'accepted', + ]) + expect(dispositions.at(-1)).toMatchObject({ kind: 'accepted', restoredFromShot: 1 }) + expect(readFileSync(join(worktreePath, 'app.ts'), 'utf8')).toBe('export const x = 2\n') + }) +}) + describe('commandVerifier', () => { it('reports command failure output and accepts exit zero', async () => { const pass = commandVerifier(process.execPath, ['-e', 'process.exit(0)'])