feat(bench): add terminal-bench supervisor arm

.gitignore

@@ -7,6 +7,7 @@ node_modules/
 bench/data/
 bench/experiments/
 bench/runs/
+bench/ab-runs/
 **/__pycache__/
 .claude/
 bench/scripts/__pycache__/

bench/run-ab-arm.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Run one raw/supervisor Terminal-Bench arm through the Tangle router.
+#
+#   ./run-ab-arm.sh <arm> <task-id> <run-id> <out-dir>
+#     arm     : raw | supervisor
+#     task-id : e.g. crack-7z-hash.easy
+set -euo pipefail
+
+ARM="${1:?arm: raw|supervisor}"
+TASK_ID="${2:?task-id}"
+RUN_ID="${3:?run-id}"
+OUT_DIR="${4:?out-dir}"
+
+BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SECRETS="${AGENT_STATE_ENV:-$HOME/company/devops/secrets/agent-state.env}"
+TB_MODEL="${TB_MODEL:-zai-coding-plan/glm-5.2}"
+ROUTER_KEY="$(dotenvx get TANGLE_API_KEY -f "$SECRETS" 2>/dev/null)"
+if [ -z "$ROUTER_KEY" ]; then echo "FATAL: no TANGLE_API_KEY" >&2; exit 1; fi
+
+case "$ARM" in
+  raw)        AGENT_IMPORT="tb_agents.opencode_router_agent:OpenCodeRouterAgent" ;;
+  supervisor) AGENT_IMPORT="tb_agents.opencode_supervisor_agent:OpenCodeSupervisorAgent" ;;
+  *) echo "FATAL: arm must be raw|supervisor" >&2; exit 1 ;;
+esac
+
+mkdir -p "$OUT_DIR"
+cd "$BENCH_DIR"
+
+# Container-id publish path (arm B's sidecar reads it; harmless for arm A).
+export TB_CONTAINER_ID_FILE="$OUT_DIR/.tb-container-id"
+rm -f "$TB_CONTAINER_ID_FILE"
+
+env \
+  PYTHONPATH="$BENCH_DIR" \
+  OPENAI_API_KEY="$ROUTER_KEY" \
+  OPENAI_BASE_URL="https://router.tangle.tools/v1" \
+  TB_CONTAINER_ID_FILE="$TB_CONTAINER_ID_FILE" \
+  TB_AB_OUT_DIR="$OUT_DIR" \
+  .venv-terminal-bench/bin/tb run \
+    -d terminal-bench-core==0.1.1 \
+    -t "$TASK_ID" \
+    --agent-import-path "$AGENT_IMPORT" \
+    -m "$TB_MODEL" \
+    --output-path "$OUT_DIR/runs" \
+    --run-id "$RUN_ID" \
+    --n-concurrent 1 \
+    --no-livestream \
+    2>&1 | tee "$OUT_DIR/tb-run.log"
+
+echo "=== results.json ==="
+cat "$OUT_DIR/runs/$RUN_ID/results.json" 2>/dev/null || echo "(no results.json)"

bench/smoke-sidecar.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_DIR="$(cd "$BENCH_DIR/.." && pwd)"
+cd "$BENCH_DIR"
+
+docker rm -f tb-sidecar-smoke >/dev/null 2>&1 || true
+SMOKE_ID=$(docker run -d --name tb-sidecar-smoke alpine sleep 600)
+echo "smoke container: $SMOKE_ID"
+
+TSX="${TSX:-$REPO_DIR/node_modules/.bin/tsx}"
+rm -f /tmp/sc-port /tmp/sc-events.jsonl /tmp/sc-events.jsonl.final.json
+env TB_TARGET_CONTAINER="$SMOKE_ID" \
+    OPENAI_API_KEY=dummy OPENAI_BASE_URL=https://router.tangle.tools/v1 \
+    WORKER_MODEL=zai-coding-plan/glm-5.2 \
+    TB_SIDECAR_PORT_FILE=/tmp/sc-port TB_SIDECAR_LOG=/tmp/sc-events.jsonl \
+    TB_WORKER_WORKDIR=/ \
+    "$TSX" src/tb-supervisor-sidecar.mts > /tmp/sc-stdout.log 2>&1 &
+SIDECAR_PID=$!
+echo "sidecar pid $SIDECAR_PID"
+
+for i in $(seq 1 40); do
+  [ -f /tmp/sc-port ] && break
+  if ! kill -0 "$SIDECAR_PID" 2>/dev/null; then echo "SIDECAR DIED"; cat /tmp/sc-stdout.log; exit 1; fi
+  sleep 0.5
+done
+PORT=$(cat /tmp/sc-port 2>/dev/null)
+echo "sidecar port: $PORT"
+URL="http://172.17.0.1:${PORT}/mcp"
+
+echo "=== tools/list ==="
+curl -s -X POST "$URL" -H 'content-type: application/json' \
+  -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' | head -c 600
+echo
+
+echo "=== spawn_agent (worker docker-execs into container) ==="
+curl -s -X POST "$URL" -H 'content-type: application/json' \
+  -d '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"spawn_agent","arguments":{"profile":{"name":"smoke-worker"},"task":"echo SMOKE_WORKER_RAN"}}}'
+echo
+sleep 6
+echo "=== await_event ==="
+curl -s -X POST "$URL" -H 'content-type: application/json' \
+  -d '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"await_event","arguments":{}}}' | head -c 800
+echo
+
+echo "=== stop sidecar (SIGTERM) -> dumps final ledger ==="
+kill -TERM "$SIDECAR_PID" 2>/dev/null
+wait "$SIDECAR_PID" 2>/dev/null
+echo "=== events log ==="
+cat /tmp/sc-events.jsonl
+echo "=== final ledger ==="
+cat /tmp/sc-events.jsonl.final.json 2>/dev/null | head -c 1500
+echo
+docker rm -f tb-sidecar-smoke >/dev/null 2>&1 || true

bench/src/tb-supervisor-sidecar.mts

@@ -0,0 +1,222 @@
+/**
+ * Host process that lets an in-container Terminal-Bench agent delegate work to child agents.
+ *
+ * It exposes the coordination MCP on a container-reachable host address, then runs every spawned
+ * worker in the same Docker container the benchmark grader inspects.
+ */
+
+import { appendFileSync, writeFileSync } from 'node:fs'
+import {
+  type Agent,
+  createExecutorRegistry,
+  createSupervisor,
+  InMemoryResultBlobStore,
+  InMemorySpawnJournal,
+  type Scope,
+} from '../../src/runtime/index'
+import { serveCoordinationMcp } from '../../src/runtime/supervise/coordination-mcp'
+import { makeTbContainerWorkerAgent, type ParseUsage } from './tb-container-executor.mts'
+
+const CONTAINER_ID = (process.env.TB_TARGET_CONTAINER ?? '').trim()
+const ROUTER_KEY = process.env.OPENAI_API_KEY ?? ''
+const ROUTER_BASE = process.env.OPENAI_BASE_URL ?? 'https://router.tangle.tools/v1'
+const WORKER_MODEL = process.env.WORKER_MODEL ?? 'zai-coding-plan/glm-5.2'
+const DOCKER_BRIDGE_GATEWAY = process.env.DOCKER_BRIDGE_GATEWAY ?? '172.17.0.1'
+const PORT_FILE = process.env.TB_SIDECAR_PORT_FILE ?? '.tb-sidecar-port'
+const LOG_FILE = process.env.TB_SIDECAR_LOG ?? '.tb-sidecar-events.jsonl'
+const WORKER_WORKDIR = process.env.TB_WORKER_WORKDIR ?? '/app'
+const WORKER_CONFIG_PATH = '/installed-agent/opencode-worker.json'
+
+if (!CONTAINER_ID) {
+  console.error('[sidecar] FATAL: TB_TARGET_CONTAINER not set')
+  process.exit(2)
+}
+
+function logEvent(kind: string, payload: unknown): void {
+  const line = JSON.stringify({ ts: new Date().toISOString(), kind, payload }) + '\n'
+  try {
+    appendFileSync(LOG_FILE, line)
+  } catch {
+    /* logging is never fatal */
+  }
+  console.error(`[sidecar] ${kind} ${JSON.stringify(payload).slice(0, 300)}`)
+}
+
+function workerWrapCommand(task: unknown): string {
+  const instr =
+    typeof task === 'string'
+      ? task
+      : task && typeof task === 'object'
+        ? ((): string => {
+            const o = task as Record<string, unknown>
+            for (const k of ['task', 'prompt', 'content', 'command', 'message', 'instruction']) {
+              if (typeof o[k] === 'string') return o[k] as string
+            }
+            return JSON.stringify(task)
+          })()
+        : String(task)
+  const q = `'${instr.replace(/'/g, `'\\''`)}'`
+  return [
+    'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"',
+    '{ [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; } 2>/dev/null || true',
+    '{ [ -s /root/.nvm/nvm.sh ] && . /root/.nvm/nvm.sh; } 2>/dev/null || true',
+    `export OPENCODE_CONFIG=${WORKER_CONFIG_PATH}`,
+    `cd ${WORKER_WORKDIR} 2>/dev/null || true`,
+    `opencode --model ${WORKER_MODEL} --format json run ${q}`,
+  ].join('; ')
+}
+
+const parseWorkerUsage: ParseUsage = ({ stdout }) => {
+  let input = 0
+  let output = 0
+  let saw = false
+  for (const raw of stdout.split('\n')) {
+    const line = raw.trim()
+    if (!line || !(line.startsWith('{') || line.startsWith('['))) continue
+    let ev: unknown
+    try {
+      ev = JSON.parse(line)
+    } catch {
+      continue
+    }
+    const events = Array.isArray(ev) ? ev : [ev]
+    for (const e of events) {
+      if (!e || typeof e !== 'object') continue
+      const dicts: Array<Record<string, unknown>> = [e as Record<string, unknown>]
+      for (const k of ['usage', 'tokens', 'part', 'info', 'message']) {
+        const sub = (e as Record<string, unknown>)[k]
+        if (sub && typeof sub === 'object') {
+          dicts.push(sub as Record<string, unknown>)
+          for (const k2 of ['usage', 'tokens']) {
+            const sub2 = (sub as Record<string, unknown>)[k2]
+            if (sub2 && typeof sub2 === 'object') dicts.push(sub2 as Record<string, unknown>)
+          }
+        }
+      }
+      for (const d of dicts) {
+        const i = (d.input_tokens ?? d.prompt_tokens ?? d.input) as unknown
+        const o = (d.output_tokens ?? d.completion_tokens ?? d.output) as unknown
+        if (typeof i === 'number' || typeof o === 'number') {
+          input += typeof i === 'number' ? i : 0
+          output += typeof o === 'number' ? o : 0
+          saw = true
+          break
+        }
+      }
+    }
+  }
+  return saw ? { input, output } : undefined
+}
+
+async function main(): Promise<void> {
+  writeFileSync(LOG_FILE, '')
+  const blobs = new InMemoryResultBlobStore()
+
+  const makeWorkerAgent = makeTbContainerWorkerAgent({
+    containerId: CONTAINER_ID,
+    workdir: WORKER_WORKDIR,
+    wrapCommand: workerWrapCommand,
+    parseUsage: parseWorkerUsage,
+    env: { OPENAI_API_KEY: ROUTER_KEY, OPENAI_BASE_URL: ROUTER_BASE, HOME: '/root' },
+    runtime: 'tb-container',
+  })
+
+  const root: Agent<unknown, unknown> = {
+    name: 'tb-supervisor-sidecar',
+    async act(_t, scope: Scope<unknown>) {
+      const mcp = await serveCoordinationMcp({
+        scope,
+        blobs,
+        makeWorkerAgent,
+        perWorker: { maxIterations: 40, maxTokens: 200_000 },
+        host: '0.0.0.0',
+        onEvent: (event) => logEvent('bus', event),
+      })
+      const containerUrl = `http://${DOCKER_BRIDGE_GATEWAY}:${mcp.port}/mcp`
+      writeFileSync(PORT_FILE, String(mcp.port))
+      logEvent('ready', { port: mcp.port, url: mcp.url, containerUrl, containerId: CONTAINER_ID })
+
+      await new Promise<void>((resolve) => {
+        const stop = (sig: string) => {
+          logEvent('stop', { signal: sig })
+          resolve()
+        }
+        process.on('SIGTERM', () => stop('SIGTERM'))
+        process.on('SIGINT', () => stop('SIGINT'))
+      })
+
+      const drained = mcp.drainResolved()
+      const settled = mcp.settled()
+      const history = mcp.history()
+      const workerOutputs: Array<Record<string, unknown>> = []
+      let workerInput = 0
+      let workerOutput = 0
+      for (const w of settled) {
+        if (!w.outRef) continue
+        try {
+          const out = (await blobs.get(w.outRef)) as
+            | { command?: string; stdout?: string; stderr?: string; exitCode?: number | null; containerId?: string }
+            | undefined
+          if (out) {
+            const usage = out.stdout ? parseWorkerUsage({ stdout: out.stdout, stderr: '', exitCode: null }) : undefined
+            if (usage) {
+              workerInput += usage.input
+              workerOutput += usage.output
+            }
+            workerOutputs.push({
+              id: w.id,
+              status: w.status,
+              command: out.command,
+              exitCode: out.exitCode ?? null,
+              containerId: out.containerId,
+              stdoutBytes: out.stdout?.length ?? 0,
+              stderrTail: (out.stderr ?? '').slice(-500),
+              stdoutTail: (out.stdout ?? '').slice(-500),
+              usage: usage ?? null,
+            })
+          }
+        } catch {
+          /* blob fetch is best-effort */
+        }
+      }
+      logEvent('final', {
+        drained,
+        settled,
+        settledCount: settled.length,
+        historyLen: history.length,
+        workerTokens: { input: workerInput, output: workerOutput },
+        stats: mcp.stats(),
+      })
+      try {
+        writeFileSync(
+          `${LOG_FILE}.final.json`,
+          JSON.stringify(
+            { settled, drained, history, workerOutputs, workerTokens: { input: workerInput, output: workerOutput }, stats: mcp.stats() },
+            null,
+            2,
+          ),
+        )
+      } catch {
+        /* never fatal */
+      }
+      await mcp.close()
+      return { settledCount: settled.length }
+    },
+  }
+
+  await createSupervisor<unknown, unknown>().run(root, 'tb-supervisor', {
+    budget: { maxIterations: 500, maxTokens: 2_000_000 },
+    runId: 'tb-supervisor-sidecar',
+    journal: new InMemorySpawnJournal(),
+    blobs,
+    executors: createExecutorRegistry(),
+    maxDepth: 4,
+    now: () => Date.now(),
+  })
+  process.exit(0)
+}
+
+main().catch((e) => {
+  console.error('[sidecar] FATAL', e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})