From d95246c1d1f60bcb6ca0982fc1ff3b5d70008fb0 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 7 Jul 2026 17:41:34 -0600 Subject: [PATCH 1/2] feat(bench): add terminal-bench container executor --- .../src/coordination-mcp-container-reach.mts | 181 ++++++++++++++ bench/src/tb-container-executor.mts | 234 ++++++++++++++++++ bench/src/tb-container-executor.test.mts | 99 ++++++++ 3 files changed, 514 insertions(+) create mode 100644 bench/src/coordination-mcp-container-reach.mts create mode 100644 bench/src/tb-container-executor.mts create mode 100644 bench/src/tb-container-executor.test.mts diff --git a/bench/src/coordination-mcp-container-reach.mts b/bench/src/coordination-mcp-container-reach.mts new file mode 100644 index 00000000..ff68fb23 --- /dev/null +++ b/bench/src/coordination-mcp-container-reach.mts @@ -0,0 +1,181 @@ +/** + * Manual reachability probe for the coordination MCP from inside a Docker container. + * + * Run: `npx tsx bench/src/coordination-mcp-container-reach.mts`. + */ + +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) +import { + type Agent, + type AgentProfile, + type AgentSpec, + createExecutorRegistry, + createSupervisor, + type Executor, + type ExecutorResult, + InMemoryResultBlobStore, + InMemorySpawnJournal, + type Scope, + type UsageEvent, +} from '../../src/runtime/index' +import { serveCoordinationMcp } from '../../src/runtime/supervise/coordination-mcp' + +const HOST_BIND = '0.0.0.0' +const DOCKER_BRIDGE_GATEWAY = process.env.DOCKER_BRIDGE_GATEWAY ?? '172.17.0.1' + +function trivialWorker(name: string): Agent { + const ex: Executor = { + runtime: 'router', + execute() { + return (async function* () { + yield { kind: 'iteration' } as UsageEvent + })() + }, + teardown: () => Promise.resolve({ destroyed: true }), + resultArtifact: (): ExecutorResult => ({ + outRef: `w:${name}`, + out: { ok: true }, + verdict: { valid: true, score: 1 }, + spent: { iterations: 1, tokens: { input: 0, output: 0 }, usd: 0, ms: 0 }, + }), + } + const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor: ex } + return { name, act: async () => ({ ok: true }), executorSpec: spec } as Agent & { + executorSpec: AgentSpec + } +} + +interface ToolsListResponse { + result?: { tools?: Array<{ name: string; description?: string; inputSchema?: unknown }> } + error?: { code: number; message: string } +} + +async function dockerCurlToolsList( + containerUrl: string, +): Promise<{ raw: string; parsed: ToolsListResponse; cmd: string }> { + const rpc = JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list' }) + const args = [ + 'run', + '--rm', + '--add-host', + 'host.docker.internal:host-gateway', + 'curlimages/curl', + '-s', + '-X', + 'POST', + containerUrl, + '-H', + 'content-type: application/json', + '-d', + rpc, + ] + const cmd = `docker ${args.join(' ')}` + const { stdout } = await execFileAsync('docker', args, { encoding: 'utf8', timeout: 120_000 }) + return { raw: stdout, parsed: JSON.parse(stdout) as ToolsListResponse, cmd } +} + +function opencodeConfigSnippet(containerUrl: string): string { + return JSON.stringify( + { + $schema: 'https://opencode.ai/config.json', + permission: { + external_directory: 'allow', + bash: 'allow', + edit: 'allow', + read: 'allow', + write: 'allow', + webfetch: 'allow', + task: 'allow', + plan_enter: 'allow', + plan_exit: 'allow', + question: 'allow', + }, + mcp: { + coordination: { + type: 'remote', + url: containerUrl, + enabled: true, + }, + }, + }, + null, + 2, + ) +} + +async function main(): Promise { + const blobs = new InMemoryResultBlobStore() + let ok = false + + const root: Agent = { + name: 'coordination-mcp-container-reach', + async act(_t, scope: Scope) { + const mcp = await serveCoordinationMcp({ + scope, + blobs, + makeWorkerAgent: () => trivialWorker('w'), + perWorker: { maxIterations: 4, maxTokens: 2000 }, + host: HOST_BIND, + }) + // Docker containers reach the host through the bridge gateway, not the 0.0.0.0 bind URL. + const containerUrl = `http://${DOCKER_BRIDGE_GATEWAY}:${mcp.port}/mcp` + const hostDockerInternalUrl = `http://host.docker.internal:${mcp.port}/mcp` + + console.error(`[probe] host-binding used : ${HOST_BIND}`) + console.error(`[probe] server url (mcp.url) : ${mcp.url}`) + console.error(`[probe] container-reachable url : ${containerUrl}`) + console.error(`[probe] alt (host.docker.internal) : ${hostDockerInternalUrl}`) + console.error('') + + try { + console.error('[probe] POSTing tools/list from inside a docker container ...') + const { raw, parsed, cmd } = await dockerCurlToolsList(containerUrl) + console.error(`[probe] container ran: ${cmd}`) + console.error(`[probe] container got back (${raw.length} bytes):`) + console.error(raw) + console.error('') + + const toolNames = (parsed.result?.tools ?? []).map((t) => t.name) + const hasSpawn = toolNames.includes('spawn_agent') + const hasAwait = toolNames.includes('await_event') + console.error(`[probe] tools advertised: ${toolNames.join(', ')}`) + console.error(`[probe] spawn_agent present = ${hasSpawn}; await_event present = ${hasAwait}`) + ok = hasSpawn && hasAwait + + if (ok) { + console.error('') + console.error('[probe] opencode.json snippet (arm B writes this via OPENCODE_CONFIG):') + console.error(opencodeConfigSnippet(containerUrl)) + } + return ok ? { reachable: true } : undefined + } finally { + await mcp.close() + } + }, + } + + await createSupervisor().run(root, 'reach', { + budget: { maxIterations: 100, maxTokens: 400_000 }, + runId: 'coordination-mcp-container-reach', + journal: new InMemorySpawnJournal(), + blobs, + executors: createExecutorRegistry(), + maxDepth: 4, + now: () => Date.now(), + }) + + console.log( + ok + ? 'CONTAINER-REACHABLE: docker tools/list returned spawn_agent and await_event.' + : 'NOT reachable from container; see output above.', + ) + process.exit(ok ? 0 : 1) +} + +main().catch((e) => { + console.error(e instanceof Error ? (e.stack ?? e.message) : String(e)) + process.exit(1) +}) diff --git a/bench/src/tb-container-executor.mts b/bench/src/tb-container-executor.mts new file mode 100644 index 00000000..5d42043e --- /dev/null +++ b/bench/src/tb-container-executor.mts @@ -0,0 +1,234 @@ +/** + * Executor for workers that must mutate the same Terminal-Bench container the grader inspects. + * + * The container lifecycle stays owned by Terminal-Bench. This executor only runs a command in the + * provided container id and reports the captured process artifact back through the runtime. + */ +import { spawn } from 'node:child_process' +import type { + Agent, + AgentProfile, + AgentSpec, + Executor, + ExecutorContext, + ExecutorFactory, + ExecutorResult, + MakeWorkerAgent, + Runtime, + Spend, +} from '../../src/runtime/index' + +export interface TbExecOutput { + /** Primary artifact consumed by drivers that read `{ content }`. */ + readonly content: string + readonly stdout: string + readonly stderr: string + readonly exitCode: number | null + readonly containerId: string + readonly command: string +} + +export type ParseUsage = (io: { + stdout: string + stderr: string + exitCode: number | null +}) => { input: number; output: number; usd?: number } | undefined + +export interface TbContainerConfig { + readonly containerId?: string + readonly containerEnvVar?: string + readonly workdir?: string + readonly shell?: string + readonly env?: Readonly> + readonly wrapCommand?: (task: unknown) => string + readonly parseUsage?: ParseUsage + readonly budgetExempt?: boolean + readonly dockerBin?: string + readonly failOnNonZeroExit?: boolean + readonly runtime?: Runtime +} + +const DEFAULT_ENV_VAR = 'TB_TARGET_CONTAINER' + +function resolveContainerId(config: TbContainerConfig): string { + const envVar = config.containerEnvVar ?? DEFAULT_ENV_VAR + const id = config.containerId ?? process.env[envVar] + if (!id || id.trim().length === 0) { + throw new Error( + `tbContainerExecutor: no target container - set config.containerId or the ${envVar} env var ` + + `to the id of the terminal-bench container the verifier grades.`, + ) + } + return id.trim() +} + +function taskToCommand(task: unknown): string { + if (typeof task === 'string') return task + if (task && typeof task === 'object') { + const obj = task as Record + for (const k of ['command', 'prompt', 'content', 'task', 'message']) { + if (typeof obj[k] === 'string') return obj[k] as string + } + } + return JSON.stringify(task) +} + +/** + * Compose the `docker exec` argv that runs `command` inside `containerId` via ` -c`. + * Exported so tests can verify the container path without a live Docker daemon. + */ +export function buildTbDockerExecArgs( + containerId: string, + command: string, + opts: { shell?: string; workdir?: string; env?: Readonly> } = {}, +): string[] { + const out: string[] = ['exec', '-i'] + if (opts.workdir) out.push('--workdir', opts.workdir) + if (opts.env) { + for (const [k, v] of Object.entries(opts.env)) { + if (typeof v !== 'string' || v.length === 0) continue + out.push('-e', `${k}=${v}`) + } + } + out.push(containerId, opts.shell ?? '/bin/sh', '-c', command) + return out +} + +function contentRef(prefix: string, value: unknown): string { + let str: string + try { + str = JSON.stringify(value) ?? String(value) + } catch { + str = String(value) + } + let h = 0x811c9dc5 + for (let i = 0; i < str.length; i += 1) { + h ^= str.charCodeAt(i) + h = Math.imul(h, 0x01000193) + } + return `${prefix}:${(h >>> 0).toString(16).padStart(8, '0')}` +} + +export function createTbContainerExecutor( + config: TbContainerConfig = {}, +): ExecutorFactory { + return (_spec: AgentSpec, ctx: ExecutorContext): Executor => { + const containerId = resolveContainerId(config) + const dockerBin = config.dockerBin ?? 'docker' + const metered = config.parseUsage !== undefined + const budgetExempt = config.budgetExempt ?? !metered + const runtime: Runtime = config.runtime ?? 'tb-container' + + const controller = new AbortController() + const abortIfSignalled = () => { + if (ctx.signal.aborted) controller.abort() + } + abortIfSignalled() + if (!ctx.signal.aborted) ctx.signal.addEventListener('abort', abortIfSignalled, { once: true }) + + let proc: ReturnType | undefined + let artifact: ExecutorResult | undefined + + return { + runtime, + budgetExempt, + execute(task, signal): Promise> { + const command = (config.wrapCommand ?? taskToCommand)(task) + const args = buildTbDockerExecArgs(containerId, command, { + ...(config.shell ? { shell: config.shell } : {}), + ...(config.workdir ? { workdir: config.workdir } : {}), + ...(config.env ? { env: config.env } : {}), + }) + const started = Date.now() + + return new Promise>((resolve, reject) => { + const child = spawn(dockerBin, args, { stdio: ['ignore', 'pipe', 'pipe'] }) + proc = child + + const kill = () => { + if (child.exitCode === null && !child.killed) child.kill('SIGKILL') + } + if (signal.aborted || controller.signal.aborted) kill() + else { + signal.addEventListener('abort', kill, { once: true }) + controller.signal.addEventListener('abort', kill, { once: true }) + } + + const outChunks: string[] = [] + const errChunks: string[] = [] + child.stdout?.on('data', (d: Buffer) => outChunks.push(d.toString('utf8'))) + child.stderr?.on('data', (d: Buffer) => errChunks.push(d.toString('utf8'))) + + child.once('error', (err) => { + signal.removeEventListener('abort', kill) + controller.signal.removeEventListener('abort', kill) + reject( + new Error( + `tbContainerExecutor: failed to spawn '${dockerBin} exec' on ${containerId}: ${err.message}`, + { cause: err }, + ), + ) + }) + + child.once('close', (code) => { + signal.removeEventListener('abort', kill) + controller.signal.removeEventListener('abort', kill) + const stdout = outChunks.join('') + const stderr = errChunks.join('') + const exitCode = code + + if (config.failOnNonZeroExit && exitCode !== 0) { + reject( + new Error( + `tbContainerExecutor: command exited ${exitCode} in ${containerId}: ${stderr.slice(0, 200)}`, + ), + ) + return + } + + const out: TbExecOutput = { content: stdout, stdout, stderr, exitCode, containerId, command } + const usage = config.parseUsage?.({ stdout, stderr, exitCode }) + const spent: Spend = metered + ? { + iterations: 1, + tokens: { input: usage?.input ?? 0, output: usage?.output ?? 0 }, + usd: usage?.usd ?? 0, + ms: Date.now() - started, + } + : // Budget-exempt commands report no conserved token, dollar, or iteration spend. + { iterations: 0, tokens: { input: 0, output: 0 }, usd: 0, ms: Date.now() - started } + + artifact = { outRef: contentRef(String(runtime), out), out, spent } + resolve(artifact) + }) + }) + }, + teardown(_grace): Promise<{ destroyed: boolean }> { + controller.abort() + if (proc && proc.exitCode === null && !proc.killed) proc.kill('SIGKILL') + return Promise.resolve({ destroyed: true }) + }, + resultArtifact() { + if (!artifact) { + throw new Error('tbContainerExecutor: resultArtifact() read before execute() settled') + } + return artifact + }, + } + } +} + +export function makeTbContainerWorkerAgent(config: TbContainerConfig = {}): MakeWorkerAgent { + const factory = createTbContainerExecutor(config) + return (rawProfile) => { + const p = (rawProfile ?? {}) as { name?: unknown } + const name = typeof p.name === 'string' && p.name.length > 0 ? p.name : 'tb-worker' + const spec: AgentSpec = { profile: rawProfile as AgentProfile, harness: null } + const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } + const executor = factory(spec, ctx) + return { name, act: async () => '', executorSpec: { ...spec, executor } } as Agent< + unknown, + unknown + > & { executorSpec: AgentSpec } + } +} diff --git a/bench/src/tb-container-executor.test.mts b/bench/src/tb-container-executor.test.mts new file mode 100644 index 00000000..180f07d2 --- /dev/null +++ b/bench/src/tb-container-executor.test.mts @@ -0,0 +1,99 @@ +import assert from 'node:assert/strict' +import { chmod, mkdtemp, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import type { AgentSpec, ExecutorContext } from '../../src/runtime/index' +import { buildTbDockerExecArgs, createTbContainerExecutor } from './tb-container-executor.mts' + +const spec: AgentSpec = { profile: { name: 'tb-test-worker' }, harness: null } + +function context(): ExecutorContext { + return { signal: new AbortController().signal, seams: {} } +} + +async function executable(name: string, body: string): Promise { + const dir = await mkdtemp(path.join(tmpdir(), 'tb-container-executor-')) + const file = path.join(dir, name) + await writeFile(file, body) + await chmod(file, 0o755) + return file +} + +async function main(): Promise { + assert.deepEqual( + buildTbDockerExecArgs('cid', 'echo ok', { + workdir: '/app', + env: { A: 'B', EMPTY: '' }, + shell: '/bin/bash', + }), + ['exec', '-i', '--workdir', '/app', '-e', 'A=B', 'cid', '/bin/bash', '-c', 'echo ok'], + 'docker exec argv is deterministic and skips empty env values', + ) + + assert.throws( + () => createTbContainerExecutor({ containerEnvVar: 'TB_CONTAINER_EXECUTOR_TEST_MISSING' })(spec, context()), + /no target container/, + 'missing target container fails before spawning', + ) + + const fakeDocker = await executable( + 'fake-docker', + `#!/bin/sh +printf 'argv:%s\\n' "$*" +`, + ) + + const metered = createTbContainerExecutor({ + containerId: 'cid', + dockerBin: fakeDocker, + workdir: '/work', + env: { OPENAI_BASE_URL: 'http://router.test' }, + parseUsage: () => ({ input: 7, output: 11, usd: 0.004 }), + })(spec, context()) + + assert.equal(metered.budgetExempt, false, 'usage parser makes the executor metered') + const meteredResult = await metered.execute({ command: 'echo hello' }, new AbortController().signal) + assert.equal(meteredResult.out.containerId, 'cid') + assert.equal(meteredResult.out.command, 'echo hello') + assert.match( + meteredResult.out.stdout, + /argv:exec -i --workdir \/work -e OPENAI_BASE_URL=http:\/\/router\.test cid \/bin\/sh -c echo hello/, + ) + assert.deepEqual(meteredResult.spent.tokens, { input: 7, output: 11 }) + assert.equal(meteredResult.spent.usd, 0.004) + assert.equal(metered.resultArtifact().out, meteredResult.out) + + const free = createTbContainerExecutor({ containerId: 'cid', dockerBin: fakeDocker })(spec, context()) + assert.equal(free.budgetExempt, true, 'unmetered shell commands are explicit budget-exempt work') + const freeResult = await free.execute('printf ok', new AbortController().signal) + assert.equal(freeResult.spent.iterations, 0) + assert.deepEqual(freeResult.spent.tokens, { input: 0, output: 0 }) + assert.equal(freeResult.spent.usd, 0) + + const failingDocker = await executable( + 'failing-docker', + `#!/bin/sh +echo failed >&2 +exit 7 +`, + ) + const strict = createTbContainerExecutor({ + containerId: 'cid', + dockerBin: failingDocker, + failOnNonZeroExit: true, + })(spec, context()) + await assert.rejects( + strict.execute('do work', new AbortController().signal), + /command exited 7/, + 'strict mode treats non-zero command exit as infrastructure failure', + ) + + const lenient = createTbContainerExecutor({ containerId: 'cid', dockerBin: failingDocker })(spec, context()) + const lenientResult = await lenient.execute('do work', new AbortController().signal) + assert.equal(lenientResult.out.exitCode, 7, 'default mode returns non-zero exits as task artifacts') + assert.match(lenientResult.out.stderr, /failed/) + + console.log('tb-container-executor.test: OK') +} + +void main() From f9d3eb4b067286a74b16fe15cf750370fb02d4e6 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 7 Jul 2026 17:45:45 -0600 Subject: [PATCH 2/2] fix(bench): keep reachability probe output on stderr --- bench/src/coordination-mcp-container-reach.mts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/src/coordination-mcp-container-reach.mts b/bench/src/coordination-mcp-container-reach.mts index ff68fb23..2d8ccb20 100644 --- a/bench/src/coordination-mcp-container-reach.mts +++ b/bench/src/coordination-mcp-container-reach.mts @@ -167,7 +167,7 @@ async function main(): Promise { now: () => Date.now(), }) - console.log( + console.error( ok ? 'CONTAINER-REACHABLE: docker tools/list returned spawn_agent and await_event.' : 'NOT reachable from container; see output above.',