From 3cac8968031a74e8612dfb1448175509b33cf91c Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 7 Jul 2026 17:52:40 -0600 Subject: [PATCH] feat(bench): add terminal-bench supervisor arm --- .gitignore | 1 + bench/run-ab-arm.sh | 51 ++++ bench/smoke-sidecar.sh | 55 ++++ bench/src/tb-supervisor-sidecar.mts | 222 ++++++++++++++++ bench/tb_agents/opencode_supervisor_agent.py | 252 +++++++++++++++++++ 5 files changed, 581 insertions(+) create mode 100755 bench/run-ab-arm.sh create mode 100755 bench/smoke-sidecar.sh create mode 100644 bench/src/tb-supervisor-sidecar.mts create mode 100644 bench/tb_agents/opencode_supervisor_agent.py diff --git a/.gitignore b/.gitignore index e9135b9..e4f93b0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ node_modules/ bench/data/ bench/experiments/ bench/runs/ +bench/ab-runs/ **/__pycache__/ .claude/ bench/scripts/__pycache__/ diff --git a/bench/run-ab-arm.sh b/bench/run-ab-arm.sh new file mode 100755 index 0000000..8c90037 --- /dev/null +++ b/bench/run-ab-arm.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Run one raw/supervisor Terminal-Bench arm through the Tangle router. +# +# ./run-ab-arm.sh +# arm : raw | supervisor +# task-id : e.g. crack-7z-hash.easy +set -euo pipefail + +ARM="${1:?arm: raw|supervisor}" +TASK_ID="${2:?task-id}" +RUN_ID="${3:?run-id}" +OUT_DIR="${4:?out-dir}" + +BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SECRETS="${AGENT_STATE_ENV:-$HOME/company/devops/secrets/agent-state.env}" +TB_MODEL="${TB_MODEL:-zai-coding-plan/glm-5.2}" +ROUTER_KEY="$(dotenvx get TANGLE_API_KEY -f "$SECRETS" 2>/dev/null)" +if [ -z "$ROUTER_KEY" ]; then echo "FATAL: no TANGLE_API_KEY" >&2; exit 1; fi + +case "$ARM" in + raw) AGENT_IMPORT="tb_agents.opencode_router_agent:OpenCodeRouterAgent" ;; + supervisor) AGENT_IMPORT="tb_agents.opencode_supervisor_agent:OpenCodeSupervisorAgent" ;; + *) echo "FATAL: arm must be raw|supervisor" >&2; exit 1 ;; +esac + +mkdir -p "$OUT_DIR" +cd "$BENCH_DIR" + +# Container-id publish path (arm B's sidecar reads it; harmless for arm A). +export TB_CONTAINER_ID_FILE="$OUT_DIR/.tb-container-id" +rm -f "$TB_CONTAINER_ID_FILE" + +env \ + PYTHONPATH="$BENCH_DIR" \ + OPENAI_API_KEY="$ROUTER_KEY" \ + OPENAI_BASE_URL="https://router.tangle.tools/v1" \ + TB_CONTAINER_ID_FILE="$TB_CONTAINER_ID_FILE" \ + TB_AB_OUT_DIR="$OUT_DIR" \ + .venv-terminal-bench/bin/tb run \ + -d terminal-bench-core==0.1.1 \ + -t "$TASK_ID" \ + --agent-import-path "$AGENT_IMPORT" \ + -m "$TB_MODEL" \ + --output-path "$OUT_DIR/runs" \ + --run-id "$RUN_ID" \ + --n-concurrent 1 \ + --no-livestream \ + 2>&1 | tee "$OUT_DIR/tb-run.log" + +echo "=== results.json ===" +cat "$OUT_DIR/runs/$RUN_ID/results.json" 2>/dev/null || echo "(no results.json)" diff --git a/bench/smoke-sidecar.sh b/bench/smoke-sidecar.sh new file mode 100755 index 0000000..ab942cd --- /dev/null +++ b/bench/smoke-sidecar.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -uo pipefail + +BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "$BENCH_DIR/.." && pwd)" +cd "$BENCH_DIR" + +docker rm -f tb-sidecar-smoke >/dev/null 2>&1 || true +SMOKE_ID=$(docker run -d --name tb-sidecar-smoke alpine sleep 600) +echo "smoke container: $SMOKE_ID" + +TSX="${TSX:-$REPO_DIR/node_modules/.bin/tsx}" +rm -f /tmp/sc-port /tmp/sc-events.jsonl /tmp/sc-events.jsonl.final.json +env TB_TARGET_CONTAINER="$SMOKE_ID" \ + OPENAI_API_KEY=dummy OPENAI_BASE_URL=https://router.tangle.tools/v1 \ + WORKER_MODEL=zai-coding-plan/glm-5.2 \ + TB_SIDECAR_PORT_FILE=/tmp/sc-port TB_SIDECAR_LOG=/tmp/sc-events.jsonl \ + TB_WORKER_WORKDIR=/ \ + "$TSX" src/tb-supervisor-sidecar.mts > /tmp/sc-stdout.log 2>&1 & +SIDECAR_PID=$! +echo "sidecar pid $SIDECAR_PID" + +for i in $(seq 1 40); do + [ -f /tmp/sc-port ] && break + if ! kill -0 "$SIDECAR_PID" 2>/dev/null; then echo "SIDECAR DIED"; cat /tmp/sc-stdout.log; exit 1; fi + sleep 0.5 +done +PORT=$(cat /tmp/sc-port 2>/dev/null) +echo "sidecar port: $PORT" +URL="http://172.17.0.1:${PORT}/mcp" + +echo "=== tools/list ===" +curl -s -X POST "$URL" -H 'content-type: application/json' \ + -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' | head -c 600 +echo + +echo "=== spawn_agent (worker docker-execs into container) ===" +curl -s -X POST "$URL" -H 'content-type: application/json' \ + -d '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"spawn_agent","arguments":{"profile":{"name":"smoke-worker"},"task":"echo SMOKE_WORKER_RAN"}}}' +echo +sleep 6 +echo "=== await_event ===" +curl -s -X POST "$URL" -H 'content-type: application/json' \ + -d '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"await_event","arguments":{}}}' | head -c 800 +echo + +echo "=== stop sidecar (SIGTERM) -> dumps final ledger ===" +kill -TERM "$SIDECAR_PID" 2>/dev/null +wait "$SIDECAR_PID" 2>/dev/null +echo "=== events log ===" +cat /tmp/sc-events.jsonl +echo "=== final ledger ===" +cat /tmp/sc-events.jsonl.final.json 2>/dev/null | head -c 1500 +echo +docker rm -f tb-sidecar-smoke >/dev/null 2>&1 || true diff --git a/bench/src/tb-supervisor-sidecar.mts b/bench/src/tb-supervisor-sidecar.mts new file mode 100644 index 0000000..be84ce6 --- /dev/null +++ b/bench/src/tb-supervisor-sidecar.mts @@ -0,0 +1,222 @@ +/** + * Host process that lets an in-container Terminal-Bench agent delegate work to child agents. + * + * It exposes the coordination MCP on a container-reachable host address, then runs every spawned + * worker in the same Docker container the benchmark grader inspects. + */ + +import { appendFileSync, writeFileSync } from 'node:fs' +import { + type Agent, + createExecutorRegistry, + createSupervisor, + InMemoryResultBlobStore, + InMemorySpawnJournal, + type Scope, +} from '../../src/runtime/index' +import { serveCoordinationMcp } from '../../src/runtime/supervise/coordination-mcp' +import { makeTbContainerWorkerAgent, type ParseUsage } from './tb-container-executor.mts' + +const CONTAINER_ID = (process.env.TB_TARGET_CONTAINER ?? '').trim() +const ROUTER_KEY = process.env.OPENAI_API_KEY ?? '' +const ROUTER_BASE = process.env.OPENAI_BASE_URL ?? 'https://router.tangle.tools/v1' +const WORKER_MODEL = process.env.WORKER_MODEL ?? 'zai-coding-plan/glm-5.2' +const DOCKER_BRIDGE_GATEWAY = process.env.DOCKER_BRIDGE_GATEWAY ?? '172.17.0.1' +const PORT_FILE = process.env.TB_SIDECAR_PORT_FILE ?? '.tb-sidecar-port' +const LOG_FILE = process.env.TB_SIDECAR_LOG ?? '.tb-sidecar-events.jsonl' +const WORKER_WORKDIR = process.env.TB_WORKER_WORKDIR ?? '/app' +const WORKER_CONFIG_PATH = '/installed-agent/opencode-worker.json' + +if (!CONTAINER_ID) { + console.error('[sidecar] FATAL: TB_TARGET_CONTAINER not set') + process.exit(2) +} + +function logEvent(kind: string, payload: unknown): void { + const line = JSON.stringify({ ts: new Date().toISOString(), kind, payload }) + '\n' + try { + appendFileSync(LOG_FILE, line) + } catch { + /* logging is never fatal */ + } + console.error(`[sidecar] ${kind} ${JSON.stringify(payload).slice(0, 300)}`) +} + +function workerWrapCommand(task: unknown): string { + const instr = + typeof task === 'string' + ? task + : task && typeof task === 'object' + ? ((): string => { + const o = task as Record + for (const k of ['task', 'prompt', 'content', 'command', 'message', 'instruction']) { + if (typeof o[k] === 'string') return o[k] as string + } + return JSON.stringify(task) + })() + : String(task) + const q = `'${instr.replace(/'/g, `'\\''`)}'` + return [ + 'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"', + '{ [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; } 2>/dev/null || true', + '{ [ -s /root/.nvm/nvm.sh ] && . /root/.nvm/nvm.sh; } 2>/dev/null || true', + `export OPENCODE_CONFIG=${WORKER_CONFIG_PATH}`, + `cd ${WORKER_WORKDIR} 2>/dev/null || true`, + `opencode --model ${WORKER_MODEL} --format json run ${q}`, + ].join('; ') +} + +const parseWorkerUsage: ParseUsage = ({ stdout }) => { + let input = 0 + let output = 0 + let saw = false + for (const raw of stdout.split('\n')) { + const line = raw.trim() + if (!line || !(line.startsWith('{') || line.startsWith('['))) continue + let ev: unknown + try { + ev = JSON.parse(line) + } catch { + continue + } + const events = Array.isArray(ev) ? ev : [ev] + for (const e of events) { + if (!e || typeof e !== 'object') continue + const dicts: Array> = [e as Record] + for (const k of ['usage', 'tokens', 'part', 'info', 'message']) { + const sub = (e as Record)[k] + if (sub && typeof sub === 'object') { + dicts.push(sub as Record) + for (const k2 of ['usage', 'tokens']) { + const sub2 = (sub as Record)[k2] + if (sub2 && typeof sub2 === 'object') dicts.push(sub2 as Record) + } + } + } + for (const d of dicts) { + const i = (d.input_tokens ?? d.prompt_tokens ?? d.input) as unknown + const o = (d.output_tokens ?? d.completion_tokens ?? d.output) as unknown + if (typeof i === 'number' || typeof o === 'number') { + input += typeof i === 'number' ? i : 0 + output += typeof o === 'number' ? o : 0 + saw = true + break + } + } + } + } + return saw ? { input, output } : undefined +} + +async function main(): Promise { + writeFileSync(LOG_FILE, '') + const blobs = new InMemoryResultBlobStore() + + const makeWorkerAgent = makeTbContainerWorkerAgent({ + containerId: CONTAINER_ID, + workdir: WORKER_WORKDIR, + wrapCommand: workerWrapCommand, + parseUsage: parseWorkerUsage, + env: { OPENAI_API_KEY: ROUTER_KEY, OPENAI_BASE_URL: ROUTER_BASE, HOME: '/root' }, + runtime: 'tb-container', + }) + + const root: Agent = { + name: 'tb-supervisor-sidecar', + async act(_t, scope: Scope) { + const mcp = await serveCoordinationMcp({ + scope, + blobs, + makeWorkerAgent, + perWorker: { maxIterations: 40, maxTokens: 200_000 }, + host: '0.0.0.0', + onEvent: (event) => logEvent('bus', event), + }) + const containerUrl = `http://${DOCKER_BRIDGE_GATEWAY}:${mcp.port}/mcp` + writeFileSync(PORT_FILE, String(mcp.port)) + logEvent('ready', { port: mcp.port, url: mcp.url, containerUrl, containerId: CONTAINER_ID }) + + await new Promise((resolve) => { + const stop = (sig: string) => { + logEvent('stop', { signal: sig }) + resolve() + } + process.on('SIGTERM', () => stop('SIGTERM')) + process.on('SIGINT', () => stop('SIGINT')) + }) + + const drained = mcp.drainResolved() + const settled = mcp.settled() + const history = mcp.history() + const workerOutputs: Array> = [] + let workerInput = 0 + let workerOutput = 0 + for (const w of settled) { + if (!w.outRef) continue + try { + const out = (await blobs.get(w.outRef)) as + | { command?: string; stdout?: string; stderr?: string; exitCode?: number | null; containerId?: string } + | undefined + if (out) { + const usage = out.stdout ? parseWorkerUsage({ stdout: out.stdout, stderr: '', exitCode: null }) : undefined + if (usage) { + workerInput += usage.input + workerOutput += usage.output + } + workerOutputs.push({ + id: w.id, + status: w.status, + command: out.command, + exitCode: out.exitCode ?? null, + containerId: out.containerId, + stdoutBytes: out.stdout?.length ?? 0, + stderrTail: (out.stderr ?? '').slice(-500), + stdoutTail: (out.stdout ?? '').slice(-500), + usage: usage ?? null, + }) + } + } catch { + /* blob fetch is best-effort */ + } + } + logEvent('final', { + drained, + settled, + settledCount: settled.length, + historyLen: history.length, + workerTokens: { input: workerInput, output: workerOutput }, + stats: mcp.stats(), + }) + try { + writeFileSync( + `${LOG_FILE}.final.json`, + JSON.stringify( + { settled, drained, history, workerOutputs, workerTokens: { input: workerInput, output: workerOutput }, stats: mcp.stats() }, + null, + 2, + ), + ) + } catch { + /* never fatal */ + } + await mcp.close() + return { settledCount: settled.length } + }, + } + + await createSupervisor().run(root, 'tb-supervisor', { + budget: { maxIterations: 500, maxTokens: 2_000_000 }, + runId: 'tb-supervisor-sidecar', + journal: new InMemorySpawnJournal(), + blobs, + executors: createExecutorRegistry(), + maxDepth: 4, + now: () => Date.now(), + }) + process.exit(0) +} + +main().catch((e) => { + console.error('[sidecar] FATAL', e instanceof Error ? (e.stack ?? e.message) : String(e)) + process.exit(1) +}) diff --git a/bench/tb_agents/opencode_supervisor_agent.py b/bench/tb_agents/opencode_supervisor_agent.py new file mode 100644 index 0000000..7d9b42b --- /dev/null +++ b/bench/tb_agents/opencode_supervisor_agent.py @@ -0,0 +1,252 @@ +"""Terminal-Bench opencode agent with a coordination MCP mounted into the task container.""" + +import json +import os +import shlex +import subprocess +import time +from pathlib import Path + +from terminal_bench.agents.base_agent import AgentResult +from terminal_bench.agents.failure_mode import FailureMode +from terminal_bench.terminal.tmux_session import TmuxSession + +from tb_agents.opencode_router_agent import ( + _CONTAINER_DIR, + OpenCodeRouterAgent, +) + +DOCKER_BRIDGE_GATEWAY = os.environ.get("DOCKER_BRIDGE_GATEWAY", "172.17.0.1") +_WORKER_CONFIG_PATH = f"{_CONTAINER_DIR}/opencode-worker.json" +# The sidecar entrypoint, resolved relative to the bench dir (this file is bench/tb_agents/...). +_BENCH_DIR = Path(__file__).resolve().parent.parent +_REPO_DIR = _BENCH_DIR.parent +_SIDECAR_TS = _BENCH_DIR / "src" / "tb-supervisor-sidecar.mts" + + +def _resolve_tsx() -> list[str]: + """Prefer the pinned local tsx binary (no npx download / no shell hook rewrite).""" + for cand in ( + _BENCH_DIR / "node_modules" / ".bin" / "tsx", + _REPO_DIR / "node_modules" / ".bin" / "tsx", + ): + if cand.exists(): + return [str(cand)] + return ["npx", "tsx"] + + +class OpenCodeSupervisorAgent(OpenCodeRouterAgent): + """`OpenCodeRouterAgent` + a coordination MCP mounted into the in-container opencode. + + Adds a host orchestration sidecar and the `mcp.coordination` config block; otherwise + identical to Arm A. Worker model defaults to the same model as the supervisor. + """ + + def __init__(self, model_name: str, *args, **kwargs): + super().__init__(model_name, *args, **kwargs) + self._mcp_port: int | None = None + self._sidecar: subprocess.Popen | None = None + self._sidecar_stdout = None + self._sidecar_port_file: Path | None = None + self._sidecar_log_file: Path | None = None + + @staticmethod + def name() -> str: + return "agent-runtime-opencode-supervisor" + + def _mcp_url(self) -> str: + return f"http://{DOCKER_BRIDGE_GATEWAY}:{self._mcp_port}/mcp" + + def _build_router_config(self) -> str: + config = json.loads(super()._build_router_config()) + perm = config.setdefault("permission", {}) + perm.update( + { + "edit": "allow", + "bash": "allow", + "webfetch": "allow", + "read": "allow", + "write": "allow", + "task": "allow", + "plan_enter": "allow", + "plan_exit": "allow", + "question": "allow", + } + ) + if self._mcp_port is not None: + config["mcp"] = { + "coordination": { + "type": "remote", + "url": self._mcp_url(), + "enabled": True, + } + } + return json.dumps(config) + + def _write_config_to_container(self, session: TmuxSession) -> None: + super()._write_config_to_container(session) + worker_config = OpenCodeRouterAgent._build_router_config(self) + session.container.exec_run( + [ + "sh", + "-c", + f"printf '%s' {shlex.quote(worker_config)} > {_WORKER_CONFIG_PATH}", + ] + ) + + def _resolve_container_id(self, session: TmuxSession) -> str | None: + return getattr(session.container, "id", None) or getattr( + session.container, "short_id", None + ) + + def _start_sidecar(self, container_id: str, logging_dir: Path | None) -> None: + base = logging_dir if logging_dir is not None else Path.cwd() + base.mkdir(parents=True, exist_ok=True) + self._sidecar_port_file = base / ".tb-sidecar-port" + self._sidecar_log_file = base / "tb-sidecar-events.jsonl" + self._sidecar_port_file.unlink(missing_ok=True) + + env = dict(os.environ) + env.update( + { + "TB_TARGET_CONTAINER": container_id, + "OPENAI_API_KEY": self._router_api_key, + "OPENAI_BASE_URL": self._router_base_url, + "WORKER_MODEL": self._model_name, + "TB_SIDECAR_PORT_FILE": str(self._sidecar_port_file), + "TB_SIDECAR_LOG": str(self._sidecar_log_file), + "DOCKER_BRIDGE_GATEWAY": DOCKER_BRIDGE_GATEWAY, + } + ) + self._sidecar_stdout = open(base / "tb-sidecar.stdout.log", "w") + self._logger.info("Starting orchestration sidecar for container %s", container_id) + self._sidecar = subprocess.Popen( + [*_resolve_tsx(), str(_SIDECAR_TS)], + cwd=str(_BENCH_DIR), + env=env, + stdout=self._sidecar_stdout, + stderr=subprocess.STDOUT, + ) + deadline = time.time() + 60.0 + while time.time() < deadline: + if self._sidecar.poll() is not None: + raise RuntimeError( + f"orchestration sidecar exited early with code {self._sidecar.returncode} " + f"(see {base / 'tb-sidecar.stdout.log'})" + ) + if self._sidecar_port_file.exists(): + txt = self._sidecar_port_file.read_text().strip() + if txt.isdigit(): + self._mcp_port = int(txt) + self._logger.info( + "Sidecar ready: mcp port=%d url=%s", self._mcp_port, self._mcp_url() + ) + return + time.sleep(0.5) + raise RuntimeError("orchestration sidecar did not publish a port within 60s") + + def _stop_sidecar(self) -> None: + if self._sidecar is None: + return + try: + self._sidecar.terminate() + try: + self._sidecar.wait(timeout=30) + except subprocess.TimeoutExpired: + self._sidecar.kill() + except Exception as e: # noqa: BLE001 - teardown is never fatal + self._logger.warning("Could not stop sidecar cleanly: %s", e) + finally: + if self._sidecar_stdout is not None: + try: + self._sidecar_stdout.close() + except Exception: # noqa: BLE001 + pass + + def _orchestration_summary(self) -> dict: + summary: dict = { + "settled": [], + "worker_outputs": [], + "history_len": 0, + "worker_ran": False, + "worker_tokens": {"input": 0, "output": 0}, + } + if self._sidecar_log_file is None: + return summary + final = Path(str(self._sidecar_log_file) + ".final.json") + try: + if final.exists(): + data = json.loads(final.read_text()) + settled = data.get("settled", []) or [] + summary["settled"] = settled + summary["worker_outputs"] = data.get("workerOutputs", []) or [] + summary["history_len"] = len(data.get("history", []) or []) + summary["worker_tokens"] = data.get( + "workerTokens", {"input": 0, "output": 0} + ) + summary["worker_ran"] = any( + isinstance(w, dict) and w.get("containerId") + for w in summary["worker_outputs"] + ) or len(settled) > 0 + except Exception as e: # noqa: BLE001 + self._logger.warning("Could not read orchestration summary: %s", e) + return summary + + def perform_task( + self, + instruction: str, + session: TmuxSession, + logging_dir: Path | None = None, + ) -> AgentResult: + container_id = self._resolve_container_id(session) + if not container_id: + self._logger.error("Could not resolve container id for supervisor sidecar.") + return AgentResult(failure_mode=FailureMode.UNKNOWN_AGENT_ERROR) + + try: + self._start_sidecar(container_id, logging_dir) + except Exception as e: # noqa: BLE001 + self._logger.error("Sidecar failed to start: %s", e) + self._mcp_port = None + return AgentResult(failure_mode=FailureMode.UNKNOWN_AGENT_ERROR) + + try: + supervised_instruction = ( + "You are the supervisor. You have an MCP server named \"coordination\" with " + "tools including spawn_agent(profile, task), observe_agent(workerId), " + "await_event(), and stop(). A spawned worker runs a full coding agent INSIDE " + "THIS SAME container and its file changes persist here, so you may delegate " + "concrete sub-tasks (e.g. \"install perl and 7z\", \"crack the hash with john\") " + "to workers via spawn_agent, then observe_agent/await_event for their results. " + "Delegate independent or heavy sub-tasks to workers when useful; do the rest " + "yourself. Complete the task fully.\n\n" + f"TASK:\n{instruction}" + ) + result = super().perform_task(supervised_instruction, session, logging_dir) + finally: + self._stop_sidecar() + + summary = self._orchestration_summary() + if logging_dir is not None: + try: + (logging_dir / "orchestration-summary.json").write_text( + json.dumps(summary, indent=2) + ) + except Exception: # noqa: BLE001 + pass + self._logger.info( + "Arm B orchestration: workers_settled=%d history_len=%d worker_ran=%s", + len(summary.get("settled", [])), + summary.get("history_len", 0), + summary.get("worker_ran"), + ) + wt = summary.get("worker_tokens") or {} + worker_in = int(wt.get("input") or 0) + worker_out = int(wt.get("output") or 0) + if result.failure_mode == FailureMode.AGENT_INSTALLATION_FAILED: + return result + return AgentResult( + total_input_tokens=result.total_input_tokens + worker_in, + total_output_tokens=result.total_output_tokens + worker_out, + failure_mode=result.failure_mode, + )