diff --git a/bench/HARNESS.md b/bench/HARNESS.md index 5f59d38..681ce99 100644 --- a/bench/HARNESS.md +++ b/bench/HARNESS.md @@ -102,6 +102,13 @@ self-describing (models + config). | `src/examples/strategy-demo.mts` | the 3-layer API demo (gym-free) | `WORKER_MODEL=gpt-4o-mini tsx src/examples/strategy-demo.mts` | | `src/examples/math-demo.mts` | any-domain proof: math via `createVerifierEnvironment` (the tax/legal/gtm answer-shape) | `BUDGET=3 tsx src/examples/math-demo.mts` | +`run-benchmarks-cli.mts` can run a bounded per-task refinement loop with +`LOOP_ATTEMPTS=N`: attempt 1 answers the original task; later attempts receive previous artifacts +plus redacted checker feedback; the loop stops early on pass. This is for testing whether agents can +use the benchmark's own feedback to solve the task, not for leaking gold answers into prompts. +For local subscription-backed workers, set `BACKEND=bridge`, `BRIDGE_URL`, `BRIDGE_BEARER`, and pass +the full bridge model id in `CELLS`, e.g. `CELLS=opencode/deepseek/deepseek-v4-pro`. + EOPS standup (one container): `docker run -d --rm --name eops -p 8006:8005 shivakrishnareddyma225/enterpriseops-gym-mcp-itsm:latest` + `EOPS_GYM_DBS_DIR=`; restart it FRESH per big run diff --git a/bench/src/run-benchmarks-cli.mts b/bench/src/run-benchmarks-cli.mts index d9b90d0..549bd22 100644 --- a/bench/src/run-benchmarks-cli.mts +++ b/bench/src/run-benchmarks-cli.mts @@ -20,9 +20,13 @@ const must = (k: string): string => { return v } -/** `harness/model` → in-box cell; bare `model` → off-box router cell. */ +/** `harness/model` → in-box cell; bare `model` → off-box router cell. For cli-bridge, the + * full model id is already `harness/model`, so do not split it. */ function parseCell(spec: string): BenchCell { const s = spec.trim() + if (process.env.BACKEND === 'bridge') { + return { label: s, model: s, backend: 'bridge', ...(process.env.SEARCH_PROVIDER ? { searchProvider: process.env.SEARCH_PROVIDER } : {}) } + } const slash = s.indexOf('/') if (slash > 0) { const harness = s.slice(0, slash) @@ -43,11 +47,14 @@ async function main(): Promise { cells, routerBaseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1', routerKey: must('TANGLE_API_KEY'), + ...(process.env.BRIDGE_URL ? { bridgeUrl: process.env.BRIDGE_URL } : {}), + ...(process.env.BRIDGE_BEARER ?? process.env.CLI_BRIDGE_BEARER ? { bridgeBearer: process.env.BRIDGE_BEARER ?? process.env.CLI_BRIDGE_BEARER } : {}), ...(process.env.SANDBOX_BASE ? { sandboxBaseUrl: process.env.SANDBOX_BASE } : {}), n: Number(process.env.N ?? 10), ...(process.env.IDS ? { ids: process.env.IDS.split(',') } : {}), ...(process.env.SPLIT ? { split: process.env.SPLIT } : {}), ...(process.env.REPS ? { reps: Number(process.env.REPS) } : {}), + ...(process.env.LOOP_ATTEMPTS ? { loopAttempts: Number(process.env.LOOP_ATTEMPTS) } : {}), concurrency: Number(process.env.CONCURRENCY ?? 4), ...(process.env.TIMEOUT_MS ? { timeoutMs: Number(process.env.TIMEOUT_MS) } : {}), ...(process.env.VERIFY_JUDGE === '0' ? { verifyJudge: false } : {}), diff --git a/bench/src/run-benchmarks.test.mts b/bench/src/run-benchmarks.test.mts index ce4b275..dbc631c 100644 --- a/bench/src/run-benchmarks.test.mts +++ b/bench/src/run-benchmarks.test.mts @@ -90,6 +90,53 @@ async function main(): Promise { }) assert.equal(repped.perTask.length, 6, 'reps × tasks') + // Looped cells feed safe failure feedback into the next attempt and stop once the judge passes. + let prompts: string[] = [] + const retryShot: BenchShot = async ({ task, prompt }) => { + prompts.push(prompt ?? task.prompt) + return { artifact: prompts.length === 1 ? 'WRONG' : String(task.metadata?.gold), ok: true } + } + const oneShot = await runBenchmarks({ + benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }], + routerBaseUrl: 'x', routerKey: 'x', runShot: retryShot, resolveAdapter: resolveStub, n: 1, + }) + assert.equal(oneShot.rows[0]!.resolveRate, 0, 'without the loop, the first bad attempt fails') + prompts = [] + const looped = await runBenchmarks({ + benchmarks: ['alpha'], cells: [{ label: 'retrying', model: 'm' }], + routerBaseUrl: 'x', routerKey: 'x', runShot: retryShot, resolveAdapter: resolveStub, n: 1, loopAttempts: 2, + }) + assert.equal(looped.rows[0]!.resolveRate, 1, 'with loopAttempts=2, the retry can pass') + assert.equal(prompts.length, 2, 'loop stops after the passing second attempt') + assert.match(prompts[1]!, /Previous attempts and safe checker feedback/) + assert.match(looped.perTask[0]!.detail ?? '', /"mode":"refine-loop"/) + + // A benchmark's detail may include hidden answer fields; those must never be fed back as hints. + const leakyGold = 'SECRET-GOLD' + const leaky: BenchmarkAdapter = { + name: 'leaky', + preflight: async () => {}, + loadTasks: async () => [{ id: 'leaky-0', prompt: 'Answer the hidden task.', metadata: { gold: leakyGold } }], + judge: async (_task, artifact) => ({ + resolved: artifact === leakyGold, + score: artifact === leakyGold ? 1 : 0, + detail: JSON.stringify({ bestGold: leakyGold, expectedAnswer: leakyGold, publicHint: 'retry' }), + }), + goldArtifact: async () => leakyGold, + } + const leakyPrompts: string[] = [] + const leakyShot: BenchShot = async ({ prompt }) => { + leakyPrompts.push(prompt ?? '') + return { artifact: leakyPrompts.length === 1 ? 'WRONG' : leakyGold, ok: true } + } + await runBenchmarks({ + benchmarks: ['leaky'], cells: [{ label: 'retrying', model: 'm' }], + routerBaseUrl: 'x', routerKey: 'x', runShot: leakyShot, resolveAdapter: () => leaky, loopAttempts: 2, + }) + assert.equal(leakyPrompts.length, 2) + assert.equal(leakyPrompts[1]!.includes(leakyGold), false, 'retry prompt redacts hidden gold fields') + assert.match(leakyPrompts[1]!, /publicHint/) + // An unavailable benchmark (preflight throws) is skipped, not fatal; the sweep still runs the rest. const flaky: Record = { ok: stubAdapter('ok', 2), diff --git a/bench/src/run-benchmarks.ts b/bench/src/run-benchmarks.ts index 148cd84..a0a8be4 100644 --- a/bench/src/run-benchmarks.ts +++ b/bench/src/run-benchmarks.ts @@ -33,6 +33,7 @@ import { openSandboxRun } from '@tangle-network/agent-runtime/loops' import type { SandboxEvent } from '@tangle-network/sandbox' import { resolveAdapter } from './adapters' import type { BenchmarkAdapter, BenchScore, BenchTask } from './benchmarks/types' +import { runRefineLoop } from './refine-loop' import { resolveBenchClient } from './resolve-client' import { runPool } from './run-pool' @@ -59,8 +60,14 @@ export type BenchShot = (input: { readonly adapter: BenchmarkAdapter readonly task: BenchTask readonly cell: BenchCell + /** Prompt to hand to the worker. Defaults to `task.prompt`; looped runs pass revised prompts. */ + readonly prompt?: string + /** 1-based attempt index for looped runs. */ + readonly attempt?: number readonly routerBaseUrl: string readonly routerKey: string + readonly bridgeUrl?: string + readonly bridgeBearer?: string readonly sandboxBaseUrl?: string readonly timeoutMs?: number }) => Promise<{ artifact: string; ok: boolean; detail?: string }> @@ -72,6 +79,8 @@ export interface RunBenchmarksOptions { readonly cells: readonly BenchCell[] readonly routerBaseUrl: string readonly routerKey: string + readonly bridgeUrl?: string + readonly bridgeBearer?: string readonly sandboxBaseUrl?: string /** Tasks per benchmark (the n). */ readonly n?: number @@ -83,6 +92,9 @@ export interface RunBenchmarksOptions { readonly concurrency?: number /** Per-shot wall-clock (ms). */ readonly timeoutMs?: number + /** Max attempts per (benchmark × cell × task). Default 1. Attempts after the first receive + * non-answer checker feedback and the previous artifacts; the loop stops early on pass. */ + readonly loopAttempts?: number /** Self-verify each benchmark's judge against its gold artifact on the first task before spending * model tokens; a benchmark whose judge rejects its own gold is recorded unavailable. Default true. */ readonly verifyJudge?: boolean @@ -152,12 +164,14 @@ function finalText(events: readonly SandboxEvent[]): string { /** The default real-agent shot: one `openSandboxRun` over the cell's harness+model, deliverable * extracted by the adapter's parser (or final text), abortable on `timeoutMs`. */ -const openSandboxShot: BenchShot = async ({ adapter, task, cell, routerBaseUrl, routerKey, sandboxBaseUrl, timeoutMs }) => { +const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerBaseUrl, routerKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs }) => { const client = resolveBenchClient({ backend: cell.backend ?? 'router', routerBaseUrl, routerKey, model: cell.model, + ...(bridgeUrl ? { bridgeUrl } : {}), + ...(bridgeBearer ? { bridgeBearer } : {}), ...(sandboxBaseUrl ? { sandboxBaseUrl } : {}), ...(cell.searchProvider ? { searchProvider: cell.searchProvider } : {}), ...(timeoutMs ? { timeoutMs } : {}), @@ -189,7 +203,7 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, routerBaseUrl, deliverable, ) try { - const turn = await run.start(task.prompt) + const turn = await run.start(prompt ?? task.prompt) const artifact = (turn.out ?? '').trim() return { artifact, @@ -202,6 +216,121 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, routerBaseUrl, } } +function parseMaybeJson(value: string): unknown { + try { + return JSON.parse(value) as unknown + } catch { + return value + } +} + +function redactJudgeLeak(value: unknown): unknown { + if (Array.isArray(value)) return value.map(redactJudgeLeak) + if (!value || typeof value !== 'object') return value + const out: Record = {} + for (const [key, child] of Object.entries(value as Record)) { + const normalizedKey = key.replace(/([a-z])([A-Z])/g, '$1_$2').toLowerCase() + if (/(^|_)(gold|expected|reference|solution|answer)(_|$)/.test(normalizedKey)) continue + out[key] = redactJudgeLeak(child) + } + return out +} + +function safeFeedback(score: BenchScore): Record { + return { + resolved: score.resolved, + score: score.score, + ...(score.detail ? { detail: redactJudgeLeak(parseMaybeJson(score.detail)) } : {}), + } +} + +function truncate(value: string, max = 4_000): string { + return value.length <= max ? value : `${value.slice(0, max)}\n...[truncated ${value.length - max} chars]` +} + +function retryPrompt(task: BenchTask, history: ReadonlyArray<{ round: number; artifact: string }>, scores: ReadonlyMap): string { + const attempts = history + .map((h) => { + const score = scores.get(h.round) + return [ + `Attempt ${h.round}:`, + 'Artifact:', + truncate(h.artifact), + score ? `Checker feedback: ${JSON.stringify(safeFeedback(score))}` : undefined, + ] + .filter(Boolean) + .join('\n') + }) + .join('\n\n') + return [ + 'Retry the benchmark task. The previous artifact did not pass the checker.', + 'Use only the original task statement and supplied context. Do not invent facts.', + 'Return the corrected artifact in exactly the format requested by the original task.', + '', + 'Original task:', + task.prompt, + '', + 'Previous attempts and safe checker feedback:', + attempts, + ].join('\n') +} + +async function loopedShot( + input: Parameters[0], + shot: BenchShot, + attempts: number, +): Promise<{ artifact: string; ok: boolean; detail?: string }> { + const scores = new Map() + const result = await runRefineLoop({ + rounds: attempts, + prompt: (round, history) => (round === 1 ? input.task.prompt : retryPrompt(input.task, history, scores)), + runShot: async (prompt, round) => { + const out = await shot({ ...input, prompt, attempt: round }) + return { artifact: out.artifact, note: out.detail } + }, + judge: async (artifact, round) => { + const score = await input.adapter.judge(input.task, artifact) + scores.set(round, score) + return { valid: score.resolved, score: score.score } + }, + }) + + const best = result.rounds.reduce((winner, candidate) => { + const a = scores.get(winner.round) + const b = scores.get(candidate.round) + if (!a) return candidate + if (!b) return winner + if (b.resolved && !a.resolved) return candidate + if (b.resolved === a.resolved && b.score > a.score) return candidate + return winner + }, result.rounds[0]!) + const bestScore = scores.get(best.round) + return { + artifact: best.artifact, + ok: best.artifact.trim().length > 0, + detail: JSON.stringify({ + mode: 'refine-loop', + attempts: result.rounds.length, + selectedAttempt: best.round, + resolvedDuringLoop: result.resolved, + selectedScore: bestScore?.score ?? null, + rounds: result.rounds.map((round) => ({ + attempt: round.round, + score: scores.get(round.round)?.score ?? null, + resolved: scores.get(round.round)?.resolved ?? null, + note: round.note ?? null, + })), + }), + } +} + +function combineDetails(runDetail: string | undefined, scoreDetail: string | undefined): string | undefined { + if (runDetail && scoreDetail) { + return JSON.stringify({ run: parseMaybeJson(runDetail), score: parseMaybeJson(scoreDetail) }) + } + return runDetail ?? scoreDetail +} + interface Job { readonly benchmark: string readonly adapter: BenchmarkAdapter @@ -254,6 +383,7 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise 1 ? await loopedShot(shotInput, shot, loopAttempts) : await shot(shotInput) const score: BenchScore = await job.adapter.judge(job.task, out.artifact) result = { benchmark: job.benchmark, @@ -285,7 +418,7 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise