From eb0e7d7cb4ed33af9b44758b4b4c7f6d1b7ecf89 Mon Sep 17 00:00:00 2001 From: yash-scaleai Date: Fri, 31 Jul 2026 23:31:24 +0000 Subject: [PATCH 1/6] tau3: add a gpt-5.4-mini variant, and stop the seed crashing on empty turns Follows the variant convention swe-atlas-qna set: build.yaml stays on deepseek-v4-flash and build.gpt54mini.yaml retargets the benchmark, so the change is reversible and the two configs are comparable side by side. The variant carries no baseline_reward, because 0.7321 was measured on deepseek with no reasoning_effort at all and a delta against it would be a model comparison. It reports an absolute held-out score instead. Reasoning effort stays at the seed's medium. At xhigh the seed lost 16 of 150 held-out cases to context-window overflow; at medium it lost none. The seed raised RuntimeError when the model returned neither text nor a tool call, which killed 17 of 150 cases on this target and put the run above the 0.1 error_rate_threshold that aborts an evaluation. gaia took the same fix in 4e90dace and tau3 never got it; it stayed invisible while the target was deepseek, which does not emit reason-only turns. Producer allow-list gains the bare and aux slots, after 8 of 157 producer calls in a gaia opencode run came back 403 model_denied. rescore_candidate.py gains --config so a retargeted variant can be scored by the same path as the pinned baselines. Without it the script only ever reads build.yaml, and swe-atlas-qna's new variant cannot be pinned at all. Co-Authored-By: Claude Opus 5 --- .../scripts/rescore_candidate.py | 22 +- .../tau3/baseline/build.gpt54mini.yaml | 200 ++++++++++++++++++ .../baseline/target/src/tau3_agent/agent.py | 23 +- 3 files changed, 239 insertions(+), 6 deletions(-) create mode 100644 harness-engineering-bench/tau3/baseline/build.gpt54mini.yaml diff --git a/harness-engineering-bench/scripts/rescore_candidate.py b/harness-engineering-bench/scripts/rescore_candidate.py index 7a4c9357..5c39639c 100644 --- a/harness-engineering-bench/scripts/rescore_candidate.py +++ b/harness-engineering-bench/scripts/rescore_candidate.py @@ -48,12 +48,21 @@ def log(message: str) -> None: print(f"[rescore] {message}", flush=True) -def load_build(benchmark: str) -> tuple[dict, Path]: +def load_build(benchmark: str, config: str = "build.yaml") -> tuple[dict, Path]: + """Load a benchmark's build config. + + `config` names the file inside `/baseline/`, defaulting to the + canonical `build.yaml`. A benchmark retargeted at a second model keeps that + canonical file untouched and adds a variant beside it (see + swe-atlas-qna/baseline/build.gpt54mini.yaml), and the variant needs scoring by + the same path as the pinned baselines or its number is not comparable to + anything. + """ import yaml # provided by the vero environment - path = BENCH_ROOT / benchmark / "baseline" / "build.yaml" + path = BENCH_ROOT / benchmark / "baseline" / config if not path.is_file(): - sys.exit(f"no build.yaml for benchmark {benchmark!r} at {path}") + sys.exit(f"no {config} for benchmark {benchmark!r} at {path}") return yaml.safe_load(path.read_text()), path @@ -208,6 +217,11 @@ def main() -> int: ), ) parser.add_argument("--benchmark", required=True) + parser.add_argument( + "--config", default="build.yaml", + help=("build config inside /baseline/ (default build.yaml). " + "Use this to score a retargeted variant, e.g. build.gpt54mini.yaml."), + ) parser.add_argument("--version", help="candidate sha (default: the shipped one)") parser.add_argument("--partition", default="test") parser.add_argument( @@ -227,7 +241,7 @@ def main() -> int: parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() - build, build_path = load_build(args.benchmark) + build, build_path = load_build(args.benchmark, args.config) outdir = Path(args.output).resolve() if args.output else Path( tempfile.mkdtemp(prefix=f"rescore-{args.benchmark}-")) outdir.mkdir(parents=True, exist_ok=True) diff --git a/harness-engineering-bench/tau3/baseline/build.gpt54mini.yaml b/harness-engineering-bench/tau3/baseline/build.gpt54mini.yaml new file mode 100644 index 00000000..3e1c522a --- /dev/null +++ b/harness-engineering-bench/tau3/baseline/build.gpt54mini.yaml @@ -0,0 +1,200 @@ +name: vero/optimize-tau3-baseline +description: >- + Improve a customer-service agent on canonical tau3-bench tasks while + preserving the task-owned MCP protocol and evaluation. +agent_repo: target +task_source: sierra-research/tau3-bench@sha256:a57304f682894ac061090769af771a3617664f3ff6e5417d4eadf8e30433e4d9 +task_manifest: ../partitions/manifest.json +agent_import_path: tau3_agent.agent:Tau3Agent +harbor_requirement: harbor[modal]==0.20.0 + +partition_files: + development: ../partitions/development.json + validation: ../partitions/validation.json + test: ../partitions/test.json + +agent_access: + - partition: development + disclosure: full + expose_case_resources: true + total_runs: 100 + total_cases: 300 + - partition: validation + disclosure: aggregate + expose_case_resources: false + min_aggregate_cases: 5 + total_runs: 100 + total_cases: 600 + +selection_partition: validation +targets: + - partition: test + reward_key: reward + # NO baseline_reward. The pinned 0.7321 was measured on + # fireworks_ai/deepseek-v4-flash with no reasoning_effort at all (the seed's + # _is_reasoning_model gate matches gpt-5/o1/o3/o4 only), so a delta against it + # would be a model comparison dressed up as an optimization result. + # With `score_baseline: false` below this run reports an ABSOLUTE held-out + # score and no delta. Pair it with the seed floor measured out of band on the + # same config; do not compare it to 0.7321. + failure_value: 0.0 + max_attempts: 1 + # The held-out eval is noisy: score the selected candidate 3x per case and + # average, so the final reward is comparable to the pinned baseline, which was + # itself pooled over 3 rounds. Without this the candidate carries ~sqrt(3) more + # standard error than the floor it is judged against. + # Per-target override - search/validation keep the global n_attempts (1). + n_attempts: 3 + aggregate_attempts: mean + +evaluation_set_name: tau3 +objective: + selector: + metric: score + direction: maximize +reward_mode: submit # agent picks; falls back to auto_best, then current version +baseline_floor: false # gates on validation while reward is on test; opt-in only +score_baseline: false +rescore_top_k: 3 +rescore_attempts: 1 + +# gpt-5.4-mini, not deepseek-v4-flash. Takes the target agent off the shared +# Fireworks per-minute bucket that is the suite's standing blocker, and tau3's own +# LLM services (TAU2_USER_MODEL / TAU2_NL_ASSERTIONS_MODEL below) already run on +# this model, so the benchmark now touches Fireworks nowhere. Unprefixed on +# purpose: the agent sends model_name.removeprefix("openai/"), so an +# openai/-prefixed name here would be allow-listed in one form and requested in +# another and the gateway would deny it. +# +# Reasoning effort stays at the seed's `medium`. Measured 2026-07-31 on the +# 150-case held-out set: at xhigh the seed lost 16 cases to "input exceeds the +# context window" (history is resent every turn and MAX_TURNS is 80), at medium it +# lost none. `high` is untested. +model: gpt-5.4-mini +environment_name: ${inner_env:-modal} +# inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default +extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"] +# NOTE: tau3's optimizer trial reliably dies at ~38-40 min with grpclib +# StreamTerminatedError ("Connection lost"). Four live experiments (default +# DinD, --ek modal_vm_runtime=true, --ek modal_sandbox_v2=true, and +# --max-retries 1) all failed identically, so no `optimizer_harbor_args` value +# is set here: none of them is a fix, and a config value that claims to be one +# is worse than nothing. Tracked upstream; see the PR that added this field. +harbor_python_version: "3.12" +n_attempts: 1 +max_retries: 1 +infrastructure_max_attempts: 3 +infrastructure_retry_delay_seconds: 5 +aggregate_attempts: best +feedback_transcripts: true +feedback_max_bytes: 16000 +expose_attempt_detail: false +# Unreachable: worst case is ceil(450/24) x 3600 = 68400s, every +# finalize trial (150 held-out x n_attempts=3) hitting its own cap. Assumes +# max_concurrency=24; recompute if that drops. +timeout_seconds: 79200 +# Exactly the dataset's declared [agent] timeout_sec, so vero's derived +# --agent-timeout-multiplier is 1.0 and the target agent gets precisely the +# clock the benchmark intends. Harbor times agent setup, environment build and +# verification on separate clocks with separate multipliers, so none of them +# eat into this budget and no buffer is warranted. +case_timeout_seconds: 3600 +task_agent_timeout_seconds: 3600 +max_concurrency: 24 # 8 -> 24; see officeqa for the measured headroom argument +error_rate_threshold: 0.1 +# Unreachable: worst-case finalize (68400) + worst-case rescore_top_k=3 +# validation rescore (68400). A verifier timeout loses the score outright. +verifier_timeout_seconds: 158400 +secrets: + - MODAL_TOKEN_ID + - MODAL_TOKEN_SECRET + - WANDB_API_KEY + - WANDB_BASE_URL # self-hosted or cloud W&B + +wandb: + # Pinned rather than left to each launcher's W&B default entity, which is their + # personal namespace: unset, the suite's runs split across as many namespaces as + # there are people running cells and the grid tooling reports "no runs matching". + entity: ${wandb_entity:-egp} + project: harness-engineering-bench # one project for the whole suite + group: tau3 # keeps the benchmark distinguishable in the shared project + name: ${wandb_run:-tau3-gpt54mini} # per-launch label, e.g. --param wandb_run=tau3__claude-sonnet-5 + tags: [tau3] + log_traces: true + +inference_gateway: + upstream_api_key_env: OPENAI_API_KEY + upstream_base_url_env: OPENAI_BASE_URL + # Stamp request-log records with a thread_id so per_trial_tokens.py can attribute + # gateway token usage to individual trials (trusted, vs. content-matching). + # Without it the fallback recovers only each conversation's root turn: measured + # 0-13% coverage unstamped against 90-98% stamped, even with --tasks-dir. + request_log_attribution: true + producer: + # Three slots, matching swe-atlas-qna/baseline/build.gpt54mini.yaml. The + # gateway compares the requested model as an exact string, and the harnesses + # strip the provider prefix, so the prefixed form alone 403s every openai- and + # anthropic-routed optimizer on its first call. The third slot is opencode's + # auxiliary summarisation model, which goes on the wire as a DATED id. + # Measured in the gaia claude-opus-5 x opencode r1 gateway log: 8 of 157 + # producer calls came back 403 model_denied, including + # claude-haiku-4-5-20251001 and a probe sweep of gpt-5-mini / gpt-4o-mini / + # o4-mini. Non-fatal there, but tau3 sessions are the longest in the suite and + # the officeqa measurement put aux traffic at 58% of producer calls, so the + # magnitude is workload-dependent and worth covering. + allowed_models: + - "${optimizer_model:-openai/gpt-5.4}" + - "${optimizer_model_bare:-gpt-5.4}" + - "${optimizer_aux_model:-gpt-5.4-nano}" + max_concurrency: 8 + # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is + # the spend control, so a token cap only needs to stop a runaway. tau3 is the + # largest set here, hence the bigger ceiling. + evaluation: + allowed_models: [gpt-5.4-mini] + max_requests: 200000 + max_tokens: 4000000000 # 900 agent case-runs (300 dev + 600 validation) + max_concurrency: 64 + # Reserved so a search-phase overspend can never starve held-out scoring. + finalization: + allowed_models: [gpt-5.4-mini] + max_requests: 200000 + max_tokens: 4000000000 # 150 test cases x3 attempts + rescore headroom + max_concurrency: 64 +instruct_multifidelity: true +instruct_exhaust_budget: true + +# tau3's environment runs its own LLM services — the user-simulator (inside the +# task container) and the NL-assertions grader (verifier) — which cannot reach the +# compose-internal inference gateway. Give them the real upstream directly while +# the candidate agent keeps the metered/allow-listed gateway (VERO_AGENT_INFERENCE_*). +task_services_use_upstream: true +# Harness isolation is off for tau3: its task-owned services need the real +# upstream credential, which uid isolation cannot hide from the harness env. +# Re-enable once task-service credentials are delivered off the harness env. +harness_user: null +# Optimizer-agent env (forwarded to the harbor claude-code agent as --ae KEY=VALUE). +# Claude Code's Bash tool caps a single call at BASH_MAX_TIMEOUT_MS (default +# 600000=10min), well under one inner eval, which pushed the officeqa optimizer +# into --detach + background-poll + end-turn -- and a headless --print run is +# never re-woken, so the search died there. Raise the cap so a whole eval fits in +# one blocking call. The background-task vars are defence in depth only: they gate +# *automatic* backgrounding and do NOT remove the Bash tool's run_in_background +# parameter, which the model can still choose. The instruction forbids that. +agent_env: + # Above this benchmark's widest single eval: a full validation pass is + # ceil(150/24) x 3600 = 25200s worst case. + BASH_MAX_TIMEOUT_MS: "32400000" + BASH_DEFAULT_TIMEOUT_MS: "32400000" # same as max: an un-timed eval must still block + ENABLE_BACKGROUND_TASKS: "0" + FORCE_AUTO_BACKGROUND_TASKS: "0" + # Harnesses installed with `uv tool install` (mini-swe-agent, swe-agent) + # default to symlinking their entry point into /usr/local/bin, which the + # unprivileged optimizer user cannot write: "Failed to install executable + # ... Permission denied". npm/nvm-based harnesses (claude-code, opencode) + # are unaffected, so this only bites when the harness changes. + UV_TOOL_BIN_DIR: "/home/agent/.local/bin" + +task_environment: + TAU2_USER_MODEL: openai/gpt-5.4-mini-2026-03-17 + TAU2_NL_ASSERTIONS_MODEL: openai/gpt-5.4-mini-2026-03-17 diff --git a/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py b/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py index 1cac3594..c501902e 100644 --- a/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py +++ b/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py @@ -428,9 +428,28 @@ async def run( text = (message.content or "").strip() if not text: - raise RuntimeError( - "model returned neither a customer message nor a tool call" + # No tool call and no message: the model only reasoned this turn, or + # was truncated at the token limit. Nudge and carry on rather than + # crashing -- MAX_TURNS plus the end_conversation fallback below + # already bound the loop. gaia's agent took the same fix in + # 4e90dace ("don't crash on reason/search-only turns"); tau3 never + # got it, and it stayed invisible while the target was + # deepseek-v4-flash, which does not emit reason-only turns. + # Measured 2026-07-31 on the 150-case held-out set with + # gpt-5.4-mini at medium effort: this raise killed 17 of 150 cases, + # putting the run above the 0.1 error_rate_threshold that aborts an + # evaluation outright. + self._trace({"turn": turn, "empty_turn": True}) + messages.append( + { + "role": "user", + "content": ( + "Continue. Either call a domain tool or use " + "send_message_to_user to reply to the customer." + ), + } ) + continue messages.append({"role": "assistant", "content": text}) fallback_tool = ( "end_conversation" if text == "###STOP###" else "send_message_to_user" From 6183984c46b9cd1ada2153e39392ee5734097074 Mon Sep 17 00:00:00 2001 From: yash-scaleai Date: Sat, 1 Aug 2026 00:38:05 +0000 Subject: [PATCH 2/6] launch_cell: forward extra --param pairs for multi-slot allow-lists The producer allow-list grew from one slot to three (swe-atlas-qna, and tau3's gpt-5.4-mini variant), and the launcher could only fill the first. An anthropic-routed optimizer therefore had no way to declare its bare and aux model names and took a 403 model_denied on each. Trailing KEY=VALUE args now become --param pairs, validated at launch so a typo fails there rather than at the gateway. Note the slots must resolve to distinct strings: allowed_models is uniqueness-checked, so passing the wire form for both the prefixed and bare slot fails config validation before any container starts. Co-Authored-By: Claude Opus 5 --- .../scripts/launch_cell.sh | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/harness-engineering-bench/scripts/launch_cell.sh b/harness-engineering-bench/scripts/launch_cell.sh index ca2205e0..36c0aa89 100755 --- a/harness-engineering-bench/scripts/launch_cell.sh +++ b/harness-engineering-bench/scripts/launch_cell.sh @@ -18,6 +18,13 @@ # via --param optimizer_model. Usually the launch-model minus the # harness-specific prefix: claude-opus-5, fireworks_ai/kimi-k3. # wandb-run ______r +# [extra...] optional trailing KEY=VALUE pairs, each forwarded as --param. +# Needed by configs whose producer allow-list has more than one +# slot: swe-atlas-qna and tau3's gpt-5.4-mini variant take +# optimizer_model_bare and optimizer_aux_model as well, and an +# anthropic-routed optimizer must pass them explicitly because the +# defaults resolve to the OpenAI family. The gateway compares exact +# strings, so a missing slot is a 403 model_denied on that model. # # Why the daemonizer: macOS has no setsid(1), and the harness SIGTERMs background # tasks belonging to an idle session's process group. A plain `nohup ... &` loses @@ -28,13 +35,24 @@ # worked, and that has already cost us a full grid once. set -euo pipefail -if [ "$#" -ne 8 ]; then - sed -n '2,28p' "$0" >&2 +if [ "$#" -lt 8 ]; then + sed -n '2,36p' "$0" >&2 exit 2 fi outdir=$1 config=$2 envfile=$3 environment=$4 agent=$5 launch_model=$6 wire_model=$7 wandb_run=$8 +shift 8 +# Each remaining KEY=VALUE becomes its own --param. Validated here rather than +# passed through blind, so a typo fails at launch instead of at the gateway. +extra_params="" +for pair in "$@"; do + case "$pair" in + *=*) extra_params="$extra_params \\ + --param \"$pair\"" ;; + *) echo "extra args must be KEY=VALUE, got: $pair" >&2; exit 2 ;; + esac +done here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) repo=$(cd "$here/../.." && pwd) @@ -64,7 +82,7 @@ exec uv run vero harbor run \\ --agent "$agent" \\ --model "$launch_model" \\ --param "optimizer_model=$wire_model" \\ - --param "wandb_run=$wandb_run" \\ + --param "wandb_run=$wandb_run"$extra_params \\ --yes \\ -o "$rundir/jobs" EOF From 3bb42228f1b03d6d40d030a8f2c560d46203a8fd Mon Sep 17 00:00:00 2001 From: yash-scaleai Date: Sat, 1 Aug 2026 17:06:25 +0000 Subject: [PATCH 3/6] tau3: add an Azure variant so the gpt-5.6 contestants can run codex strips the provider prefix, so --model azure_ai/gpt-5.6-sol reaches the gateway as bare gpt-5.6-sol and is forwarded unchanged; the proxy then load-balances it across two Azure deployments and every turn after the first fails with invalid_encrypted_content, because the reasoning payload cannot be decrypted by the deployment that did not create it. That killed all four gpt-5.6 cells earlier today at 2-4 minutes. Uses the gateway model_aliases added in PR #81 to pin both bare names to their Azure deployment, which is the only place the prefix can be reattached since the harness will not carry it. Varun measured the split against the live proxy: bare failed 5 of 5 encrypted-content replays, azure_ai/-prefixed passed 5 of 5. My own 3-chain probe passed both ways and was too small to see it. Also why this cannot wait for the shared config: the OpenAI contract expired 2026-07-31, so every GPT model now has to route through Azure. Co-Authored-By: Claude Opus 5 --- .../tau3/baseline/build.azure.yaml | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 harness-engineering-bench/tau3/baseline/build.azure.yaml diff --git a/harness-engineering-bench/tau3/baseline/build.azure.yaml b/harness-engineering-bench/tau3/baseline/build.azure.yaml new file mode 100644 index 00000000..2ab8d337 --- /dev/null +++ b/harness-engineering-bench/tau3/baseline/build.azure.yaml @@ -0,0 +1,216 @@ +name: vero/optimize-tau3-baseline +description: >- + Improve a customer-service agent on canonical tau3-bench tasks while + preserving the task-owned MCP protocol and evaluation. +agent_repo: target +task_source: sierra-research/tau3-bench@sha256:a57304f682894ac061090769af771a3617664f3ff6e5417d4eadf8e30433e4d9 +task_manifest: ../partitions/manifest.json +agent_import_path: tau3_agent.agent:Tau3Agent +harbor_requirement: harbor[modal]==0.20.0 + +partition_files: + development: ../partitions/development.json + validation: ../partitions/validation.json + test: ../partitions/test.json + +agent_access: + - partition: development + disclosure: full + expose_case_resources: true + total_runs: 100 + total_cases: 300 + - partition: validation + disclosure: aggregate + expose_case_resources: false + min_aggregate_cases: 5 + total_runs: 100 + total_cases: 600 + +selection_partition: validation +targets: + - partition: test + reward_key: reward + # NO baseline_reward. The pinned 0.7321 was measured on + # fireworks_ai/deepseek-v4-flash with no reasoning_effort at all (the seed's + # _is_reasoning_model gate matches gpt-5/o1/o3/o4 only), so a delta against it + # would be a model comparison dressed up as an optimization result. + # With `score_baseline: false` below this run reports an ABSOLUTE held-out + # score and no delta. Pair it with the seed floor measured out of band on the + # same config; do not compare it to 0.7321. + failure_value: 0.0 + max_attempts: 1 + # The held-out eval is noisy: score the selected candidate 3x per case and + # average, so the final reward is comparable to the pinned baseline, which was + # itself pooled over 3 rounds. Without this the candidate carries ~sqrt(3) more + # standard error than the floor it is judged against. + # Per-target override - search/validation keep the global n_attempts (1). + n_attempts: 3 + aggregate_attempts: mean + +evaluation_set_name: tau3 +objective: + selector: + metric: score + direction: maximize +reward_mode: submit # agent picks; falls back to auto_best, then current version +baseline_floor: false # gates on validation while reward is on test; opt-in only +score_baseline: false +rescore_top_k: 3 +rescore_attempts: 1 + +# gpt-5.4-mini, not deepseek-v4-flash. Takes the target agent off the shared +# Fireworks per-minute bucket that is the suite's standing blocker, and tau3's own +# LLM services (TAU2_USER_MODEL / TAU2_NL_ASSERTIONS_MODEL below) already run on +# this model, so the benchmark now touches Fireworks nowhere. Unprefixed on +# purpose: the agent sends model_name.removeprefix("openai/"), so an +# openai/-prefixed name here would be allow-listed in one form and requested in +# another and the gateway would deny it. +# +# Reasoning effort stays at the seed's `medium`. Measured 2026-07-31 on the +# 150-case held-out set: at xhigh the seed lost 16 cases to "input exceeds the +# context window" (history is resent every turn and MAX_TURNS is 80), at medium it +# lost none. `high` is untested. +model: gpt-5.4-mini +environment_name: ${inner_env:-modal} +# inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default +extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"] +# NOTE: tau3's optimizer trial reliably dies at ~38-40 min with grpclib +# StreamTerminatedError ("Connection lost"). Four live experiments (default +# DinD, --ek modal_vm_runtime=true, --ek modal_sandbox_v2=true, and +# --max-retries 1) all failed identically, so no `optimizer_harbor_args` value +# is set here: none of them is a fix, and a config value that claims to be one +# is worse than nothing. Tracked upstream; see the PR that added this field. +harbor_python_version: "3.12" +n_attempts: 1 +max_retries: 1 +infrastructure_max_attempts: 3 +infrastructure_retry_delay_seconds: 5 +aggregate_attempts: best +feedback_transcripts: true +feedback_max_bytes: 16000 +expose_attempt_detail: false +# Unreachable: worst case is ceil(450/24) x 3600 = 68400s, every +# finalize trial (150 held-out x n_attempts=3) hitting its own cap. Assumes +# max_concurrency=24; recompute if that drops. +timeout_seconds: 79200 +# Exactly the dataset's declared [agent] timeout_sec, so vero's derived +# --agent-timeout-multiplier is 1.0 and the target agent gets precisely the +# clock the benchmark intends. Harbor times agent setup, environment build and +# verification on separate clocks with separate multipliers, so none of them +# eat into this budget and no buffer is warranted. +case_timeout_seconds: 3600 +task_agent_timeout_seconds: 3600 +max_concurrency: 24 # 8 -> 24; see officeqa for the measured headroom argument +error_rate_threshold: 0.1 +# Unreachable: worst-case finalize (68400) + worst-case rescore_top_k=3 +# validation rescore (68400). A verifier timeout loses the score outright. +verifier_timeout_seconds: 158400 +secrets: + - MODAL_TOKEN_ID + - MODAL_TOKEN_SECRET + - WANDB_API_KEY + - WANDB_BASE_URL # self-hosted or cloud W&B + +wandb: + # Pinned rather than left to each launcher's W&B default entity, which is their + # personal namespace: unset, the suite's runs split across as many namespaces as + # there are people running cells and the grid tooling reports "no runs matching". + entity: ${wandb_entity:-egp} + project: harness-engineering-bench # one project for the whole suite + group: tau3 # keeps the benchmark distinguishable in the shared project + name: ${wandb_run:-tau3-azure} # per-launch label, e.g. --param wandb_run=tau3__claude-sonnet-5 + tags: [tau3] + log_traces: true + +inference_gateway: + upstream_api_key_env: OPENAI_API_KEY + upstream_base_url_env: OPENAI_BASE_URL + # Stamp request-log records with a thread_id so per_trial_tokens.py can attribute + # gateway token usage to individual trials (trusted, vs. content-matching). + # Without it the fallback recovers only each conversation's root turn: measured + # 0-13% coverage unstamped against 90-98% stamped, even with --tasks-dir. + request_log_attribution: true + producer: + # Three slots, matching swe-atlas-qna/baseline/build.gpt54mini.yaml. The + # gateway compares the requested model as an exact string, and the harnesses + # strip the provider prefix, so the prefixed form alone 403s every openai- and + # anthropic-routed optimizer on its first call. The third slot is opencode's + # auxiliary summarisation model, which goes on the wire as a DATED id. + # Measured in the gaia claude-opus-5 x opencode r1 gateway log: 8 of 157 + # producer calls came back 403 model_denied, including + # claude-haiku-4-5-20251001 and a probe sweep of gpt-5-mini / gpt-4o-mini / + # o4-mini. Non-fatal there, but tau3 sessions are the longest in the suite and + # the officeqa measurement put aux traffic at 58% of producer calls, so the + # magnitude is workload-dependent and worth covering. + allowed_models: + - "${optimizer_model:-openai/gpt-5.4}" + - "${optimizer_model_bare:-gpt-5.4}" + - "${optimizer_aux_model:-gpt-5.4-nano}" + # codex strips the provider prefix, so `--model azure_ai/gpt-5.6-sol` arrives at + # the gateway as bare `gpt-5.6-sol` and gets forwarded unchanged. The proxy then + # load-balances it across two Azure deployments, and every turn after the first + # fails with invalid_encrypted_content because the reasoning payload cannot be + # decrypted by the deployment that did not create it. Measured by Varun against + # the live proxy (PR #81): bare failed 5 of 5 encrypted-content replays, + # azure_ai/-prefixed passed 5 of 5. Aliasing pins the bare name to one + # deployment, which is the only place the prefix can be reattached, since the + # harness will not carry it. Both gpt-5.6 models are listed because this file + # serves the sol and terra contestants. + # + # Also relevant: the OpenAI contract expired 2026-07-31, so every GPT model now + # has to route through Azure regardless of harness. + model_aliases: + gpt-5.6-sol: azure_ai/gpt-5.6-sol + gpt-5.6-terra: azure_ai/gpt-5.6-terra + max_concurrency: 8 + # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is + # the spend control, so a token cap only needs to stop a runaway. tau3 is the + # largest set here, hence the bigger ceiling. + evaluation: + allowed_models: [gpt-5.4-mini] + max_requests: 200000 + max_tokens: 4000000000 # 900 agent case-runs (300 dev + 600 validation) + max_concurrency: 64 + # Reserved so a search-phase overspend can never starve held-out scoring. + finalization: + allowed_models: [gpt-5.4-mini] + max_requests: 200000 + max_tokens: 4000000000 # 150 test cases x3 attempts + rescore headroom + max_concurrency: 64 +instruct_multifidelity: true +instruct_exhaust_budget: true + +# tau3's environment runs its own LLM services — the user-simulator (inside the +# task container) and the NL-assertions grader (verifier) — which cannot reach the +# compose-internal inference gateway. Give them the real upstream directly while +# the candidate agent keeps the metered/allow-listed gateway (VERO_AGENT_INFERENCE_*). +task_services_use_upstream: true +# Harness isolation is off for tau3: its task-owned services need the real +# upstream credential, which uid isolation cannot hide from the harness env. +# Re-enable once task-service credentials are delivered off the harness env. +harness_user: null +# Optimizer-agent env (forwarded to the harbor claude-code agent as --ae KEY=VALUE). +# Claude Code's Bash tool caps a single call at BASH_MAX_TIMEOUT_MS (default +# 600000=10min), well under one inner eval, which pushed the officeqa optimizer +# into --detach + background-poll + end-turn -- and a headless --print run is +# never re-woken, so the search died there. Raise the cap so a whole eval fits in +# one blocking call. The background-task vars are defence in depth only: they gate +# *automatic* backgrounding and do NOT remove the Bash tool's run_in_background +# parameter, which the model can still choose. The instruction forbids that. +agent_env: + # Above this benchmark's widest single eval: a full validation pass is + # ceil(150/24) x 3600 = 25200s worst case. + BASH_MAX_TIMEOUT_MS: "32400000" + BASH_DEFAULT_TIMEOUT_MS: "32400000" # same as max: an un-timed eval must still block + ENABLE_BACKGROUND_TASKS: "0" + FORCE_AUTO_BACKGROUND_TASKS: "0" + # Harnesses installed with `uv tool install` (mini-swe-agent, swe-agent) + # default to symlinking their entry point into /usr/local/bin, which the + # unprivileged optimizer user cannot write: "Failed to install executable + # ... Permission denied". npm/nvm-based harnesses (claude-code, opencode) + # are unaffected, so this only bites when the harness changes. + UV_TOOL_BIN_DIR: "/home/agent/.local/bin" + +task_environment: + TAU2_USER_MODEL: openai/gpt-5.4-mini-2026-03-17 + TAU2_NL_ASSERTIONS_MODEL: openai/gpt-5.4-mini-2026-03-17 From 6211defbf3a49d239967fefae25a64d2d2590fa1 Mon Sep 17 00:00:00 2001 From: yash-scaleai Date: Sat, 1 Aug 2026 20:50:02 +0000 Subject: [PATCH 4/6] tau3: pin the measured floor at 0.5618 and backfill it into the finished cells Measured on the config the cells actually ran: 3 rounds x 3 attempts of the unmodified seed through rescore_candidate.py, n=1335, rounds 0.568/0.553/0.564, sd 0.0063. The old 0.7321 is not a valid comparator and the comment now says so in the imperative -- it was deepseek-v4-flash with no reasoning_effort, so a delta against it measures the model swap, not the optimizer. The 14 cells that finished before the pin existed wrote an empty baseline_rewards, which leaves tau3 the one benchmark whose gain cannot be derived from its own artifacts. backfill_baseline.py fills the field and pushes, so the analysis script reads it the same way it reads every other benchmark rather than carrying a tau3 special case. Provenance goes in a sibling file because VerificationResult forbids extra keys. Co-Authored-By: Claude Opus 5 --- .../scripts/backfill_baseline.py | 135 ++++++++++++++++++ .../tau3/baseline/build.azure.yaml | 14 +- .../tau3/baseline/build.gpt54mini.yaml | 12 +- 3 files changed, 153 insertions(+), 8 deletions(-) create mode 100644 harness-engineering-bench/scripts/backfill_baseline.py diff --git a/harness-engineering-bench/scripts/backfill_baseline.py b/harness-engineering-bench/scripts/backfill_baseline.py new file mode 100644 index 00000000..7085236d --- /dev/null +++ b/harness-engineering-bench/scripts/backfill_baseline.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Fill `baseline_rewards` into finalization.json for cells that ran without a pin. + + python3 harness-engineering-bench/scripts/backfill_baseline.py \ + --benchmark tau3 --reward-key reward --value 0.5679 \ + --provenance "K=3 rescore_candidate.py --seed on build.gpt54mini.yaml, ..." \ + [--push] [--dry-run] + +Why this is legitimate rather than editing results after the fact: nothing ever +measures `baseline_rewards` during a run. Every other benchmark carries a +`baseline_reward` that was measured out of band by rescore_candidate.py days +earlier and pasted into build.yaml, and the verifier simply copies it into the +finalization payload. Filling the same field, with a value from the same script on +the same seed and target, gives tau3's cells identical provenance -- it does not +invent a number, it supplies the one the config should have carried. + +Why it matters: the results pipeline computes gain from `baseline_rewards`. Left +empty, tau3 is the one benchmark whose gain cannot be derived, and the likely +failure mode is that nobody notices until the table is built. + +Two hard constraints this respects: + +- `VerificationResult` is a StrictModel with extra="forbid", and vero's report.py + validates the finalization payload through it. So NO new keys go in that file -- + provenance goes in a sibling baseline_reward_provenance.json that no schema reads. +- The candidate's own `rewards` are never touched. Only the comparator is added. + +Every patch is validated by re-parsing the file through VerificationResult before +it is written, so a schema mistake fails here rather than in someone's report. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent.parent +S3 = "s3://scale-ml/harness-engineering-bench" +PROFILE = "ml-worker" + + +def validates(payload: dict) -> tuple[bool, str]: + """Re-parse through vero's own model, so a bad edit cannot reach S3.""" + try: + sys.path.insert(0, str(REPO / "vero" / "src")) + from vero.sidecar.verifier import VerificationResult # type: ignore + except Exception as exc: # vero not importable here; skip rather than guess + return True, f"(schema check skipped: {exc})" + try: + VerificationResult.model_validate_json(json.dumps(payload)) + return True, "ok" + except Exception as exc: + return False, str(exc)[:200] + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--benchmark", required=True) + ap.add_argument("--reward-key", default="reward") + ap.add_argument("--value", type=float, required=True) + ap.add_argument("--provenance", required=True, + help="one line recording how the value was measured") + ap.add_argument("--push", action="store_true", help="also upload to S3") + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + pattern = f"runs/{args.benchmark}/*/jobs/*/task__*/verifier/finalization.json" + files = sorted(REPO.glob(pattern)) + if not files: + sys.exit(f"no finalization.json under runs/{args.benchmark}/") + + stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + touched = skipped = failed = 0 + for path in files: + cell = path.relative_to(REPO).parts[2] + payload = json.loads(path.read_text()) + existing = payload.get("baseline_rewards") or {} + if existing.get(args.reward_key) is not None: + print(f" skip {cell:34} already has {existing}") + skipped += 1 + continue + if not payload.get("shipped"): + print(f" skip {cell:34} shipped=false, not a reportable cell") + skipped += 1 + continue + + payload["baseline_rewards"] = dict(existing) | {args.reward_key: args.value} + ok, why = validates(payload) + if not ok: + print(f" FAIL {cell:34} schema rejected: {why}") + failed += 1 + continue + + prov = path.parent / "baseline_reward_provenance.json" + prov_doc = { + "backfilled_at": stamp, + "reward_key": args.reward_key, + "baseline_reward": args.value, + "measured_by": args.provenance, + "note": ("Added after the run. The build config carried no " + "baseline_reward, so the verifier wrote an empty " + "baseline_rewards. This value comes from the same script and " + "seed that every other benchmark's pinned baseline comes from; " + "the candidate's own rewards are untouched."), + } + if args.dry_run: + print(f" would {cell:34} set {args.reward_key}={args.value}") + touched += 1 + continue + + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + prov.write_text(json.dumps(prov_doc, indent=2) + "\n", encoding="utf-8") + print(f" set {cell:34} {args.reward_key}={args.value}") + touched += 1 + + if args.push: + rel = path.relative_to(REPO / "runs") + for local, key in ((path, rel), (prov, prov.relative_to(REPO / "runs"))): + r = subprocess.run( + ["aws", "--profile", PROFILE, "s3", "cp", str(local), + f"{S3}/{key}", "--only-show-errors"], + capture_output=True, text=True) + if r.returncode != 0: + print(f" UPLOAD FAILED {key}: {r.stderr.strip()[:120]}") + + print(f"\n{touched} patched, {skipped} skipped, {failed} failed") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness-engineering-bench/tau3/baseline/build.azure.yaml b/harness-engineering-bench/tau3/baseline/build.azure.yaml index 2ab8d337..a89a280c 100644 --- a/harness-engineering-bench/tau3/baseline/build.azure.yaml +++ b/harness-engineering-bench/tau3/baseline/build.azure.yaml @@ -30,13 +30,19 @@ selection_partition: validation targets: - partition: test reward_key: reward - # NO baseline_reward. The pinned 0.7321 was measured on + # Measured 2026-08-01 on THIS config: 3 rounds x 3 attempts of the unmodified + # seed through rescore_candidate.py, n=1335 held-out trials, rounds + # 0.568 / 0.553 / 0.564, sd 0.0063. Same target model and seed as + # build.gpt54mini.yaml -- this config differs only in gateway model_aliases, + # which route the optimizer, not the target, so the floor is shared. + baseline_reward: 0.5618 + # Do NOT restore the old 0.7321. That was measured on # fireworks_ai/deepseek-v4-flash with no reasoning_effort at all (the seed's # _is_reasoning_model gate matches gpt-5/o1/o3/o4 only), so a delta against it # would be a model comparison dressed up as an optimization result. - # With `score_baseline: false` below this run reports an ABSOLUTE held-out - # score and no delta. Pair it with the seed floor measured out of band on the - # same config; do not compare it to 0.7321. + # `score_baseline: false` below means the run does not re-measure this; it + # copies the number into the finalization payload, which is what makes the + # gain derivable downstream. Recompute it if the seed or target model changes. failure_value: 0.0 max_attempts: 1 # The held-out eval is noisy: score the selected candidate 3x per case and diff --git a/harness-engineering-bench/tau3/baseline/build.gpt54mini.yaml b/harness-engineering-bench/tau3/baseline/build.gpt54mini.yaml index 3e1c522a..22b12ee9 100644 --- a/harness-engineering-bench/tau3/baseline/build.gpt54mini.yaml +++ b/harness-engineering-bench/tau3/baseline/build.gpt54mini.yaml @@ -30,13 +30,17 @@ selection_partition: validation targets: - partition: test reward_key: reward - # NO baseline_reward. The pinned 0.7321 was measured on + # Measured 2026-08-01 on THIS config: 3 rounds x 3 attempts of the unmodified + # seed through rescore_candidate.py, n=1335 held-out trials, rounds + # 0.568 / 0.553 / 0.564, sd 0.0063. + baseline_reward: 0.5618 + # Do NOT restore the old 0.7321. That was measured on # fireworks_ai/deepseek-v4-flash with no reasoning_effort at all (the seed's # _is_reasoning_model gate matches gpt-5/o1/o3/o4 only), so a delta against it # would be a model comparison dressed up as an optimization result. - # With `score_baseline: false` below this run reports an ABSOLUTE held-out - # score and no delta. Pair it with the seed floor measured out of band on the - # same config; do not compare it to 0.7321. + # `score_baseline: false` below means the run does not re-measure this; it + # copies the number into the finalization payload, which is what makes the + # gain derivable downstream. Recompute it if the seed or target model changes. failure_value: 0.0 max_attempts: 1 # The held-out eval is noisy: score the selected candidate 3x per case and From 22b87f6460224129a088f054a178bb46e276e6a8 Mon Sep 17 00:00:00 2001 From: yash-scaleai Date: Sun, 2 Aug 2026 20:36:58 +0000 Subject: [PATCH 5/6] Add a benchmark-agnostic skill for reading what the optimizer actually did Squashed from four commits written while building and then running it once on tau3: the initial version, a fix for a fabricated citation the first real run caught in itself, a cut of two stages (blind lenses, completeness critic) that turned out to reproduce what a 20-line script already gave for free, and a restructure of the output into attributed tables instead of prose. Benchmark-agnostic by construction, not just in name: both scripts take --benchmark as an argument and walk runs// generically, with no hardcoded target-repo path or model name anywhere in either. The tau3 mentions in SKILL.md and lenses.md are calibration examples for the kind of finding expected, explicitly labeled as such -- run it on officeqa or swe-atlas and it reads their cells the same way. Co-Authored-By: Claude Opus 5 --- .../skills/analyze-optimizer-commits/SKILL.md | 254 ++++++++++++++++++ .../references/lenses.md | 136 ++++++++++ .../references/output-template.md | 118 ++++++++ .../scripts/cross_cell_stats.py | 76 ++++++ .../scripts/extract_candidates.py | 200 ++++++++++++++ .../tau3/OPTIMIZER-BEHAVIOR.md | 143 ++++++++++ 6 files changed, 927 insertions(+) create mode 100644 harness-engineering-bench/skills/analyze-optimizer-commits/SKILL.md create mode 100644 harness-engineering-bench/skills/analyze-optimizer-commits/references/lenses.md create mode 100644 harness-engineering-bench/skills/analyze-optimizer-commits/references/output-template.md create mode 100644 harness-engineering-bench/skills/analyze-optimizer-commits/scripts/cross_cell_stats.py create mode 100644 harness-engineering-bench/skills/analyze-optimizer-commits/scripts/extract_candidates.py create mode 100644 harness-engineering-bench/tau3/OPTIMIZER-BEHAVIOR.md diff --git a/harness-engineering-bench/skills/analyze-optimizer-commits/SKILL.md b/harness-engineering-bench/skills/analyze-optimizer-commits/SKILL.md new file mode 100644 index 00000000..824f1218 --- /dev/null +++ b/harness-engineering-bench/skills/analyze-optimizer-commits/SKILL.md @@ -0,0 +1,254 @@ +--- +name: analyze-optimizer-commits +description: >- + Analyze what the optimizer agent actually did inside a benchmark's grid cells — + read every candidate commit, classify the edits, find cross-cell trends, and + produce a bulleted, citation-bearing observations .md for the paper's discussion + section. Runs a fan-out of subagents (per-cell extraction → adversarial + verification → synthesis), with cross-cell stats computed by script rather than + by agent, so findings are comprehensive and every claim is checkable. Use + whenever someone + asks what the optimizer commits are doing, what the optimizer LLM changed or + learned, whether it reverted or iterated, what trends appear across cells or + models or harnesses, or asks for optimizer-behaviour observations for the + write-up — including vaguer forms like "any interesting patterns in these runs?" + or "why did this cell score higher than that one?" +--- + +# Analyzing what the optimizer did + +An optimizer agent is handed a target harness and a scoring CLI, and left to +improve the harness however it likes. The reward tells us *whether* it improved +things. This analysis answers *what it did* — the part the paper's discussion +section is made of, and the part no metric captures. + +The output is a bulleted `.md` of observations, each carrying a citation to a +specific cell and commit. Apaar's requirement, verbatim: *"bulleted, so not very +cloudy — an exhaustive compilation of interesting observations of what the LLM +did... keep it light because I will take that and put it in the discussion +section."* So: exhaustive in coverage, terse in prose. Every claim traceable. + +Different people run this across different benchmarks and the results get pooled, +so consistency matters as much as depth. Same procedure, same output shape, same +evidence standard, whether it's tau3 or GAIA or swe-atlas. + +## Why this is a fan-out and not one pass + +One agent reading twelve candidate repositories does three things badly, and all +three have already happened on this project: + +- **It runs out of attention before it runs out of cells.** Cells read late get + a paragraph; cells read early get a page. The write-up then describes the first + three cells and calls it a trend. +- **It sees the pattern it went looking for.** A single reader who notices reverts + early starts reading everything as evidence about reverts, and never notices + that two cells shipped nothing at all. +- **It reports plausible numbers it never counted.** "Roughly a third of commits + were reverts" is the kind of claim that reads fine, gets into a paper, and is + wrong. Nobody recomputes it. + +So: extraction is separated from interpretation, every cross-cell number comes +from a script rather than an agent's count, and every candidate observation is +attacked by a verifier that re-reads the artifact before it ships. + +An earlier version of this skill also ran the cross-cell interpretation step as +six independent "blind" agents (one per lens, unable to see each other's +findings) plus a completeness critic reading everything at the end. Cut after the +first real run: the blind-lens fan-out reproduced almost nothing that a 20-line +python script computing correlations and keyword hits didn't already surface +faster and for free, and with 16/16 cells covered by step 1 the completeness +critic had nothing left to find. If a future benchmark's corpus is large enough +that step 2 below misses real patterns, bring a lens back — but earn it, don't +default to it. + +## Step 0 — Extract the facts deterministically + +Run the bundled extractor first. It unpacks each cell's `session.tar.gz`, walks +the candidate git repo, and emits one record per candidate with its message, +diffstat, files touched, and per-partition scores. + +```bash +cd harness-engineering-bench +python3 skills/analyze-optimizer-commits/scripts/extract_candidates.py \ + --benchmark --json /tmp/-candidates.json +``` + +Read the stdout table yourself before dispatching anything. It tells you how many +cells there are, which are marked `[NOT REPORTABLE]`, and where the shipped +candidate sits in each chain. That shapes the fan-out — and it is also your first +sanity check: if the extractor finds two cells where you expected twelve, the +problem is the path or the archive, not the optimizer. + +**Only analyze reportable cells, and say how many you dropped.** A cell that +crashed still writes a `finalization.json` with `shipped: false` or reward 0.0, +and its candidate chain is real but truncated — the optimizer was interrupted, not +finished. Mixing those in makes the optimizer look like it abandons work. The +extractor's `reportable` flag encodes the check (shipped, zero error rate, +non-zero token spend); trust it, and report the excluded cells as their own +observation, because *why* cells died is itself a finding. + +## Step 0.5 — Find the cells that shipped the seed, before anything else + +For every cell, compare the seed's tree hash with the shipped candidate's: + +```bash +git --git-dir= rev-parse ^{tree} ^{tree} +git --git-dir= diff --name-only # ignore __pycache__, .gitignore +``` + +A cell where these agree shipped the **unmodified seed**, which makes its reward a +measurement of the baseline through the *finalization* path. That is the single most +valuable artifact in the corpus, and it is why this check comes before +interpretation rather than during it. + +On tau3's first run, two cells had this property and scored 0.4267 and 0.4511 — +while the seed measured **0.5618** on the same test partition through +`rescore_candidate.py`. Same code, same cases, two scripts, 0.12 apart, against a +floor whose own round-to-round sd was 0.0063. Every per-cell gain computed against +that floor was wrong by roughly the same amount, and the error inverted the +headline: 4 of 14 cells looked like improvements against the bare-path floor, 12 of +14 against the in-path seed. + +So: **if the benchmark's pinned baseline was measured by a different script than the +one that scored the candidates, no gain number means anything until that gap is +quantified.** Verify the tree hashes rather than believing a commit message — three +tau3 cells have messages saying "revert" while still carrying behavioural changes, +and one says only "Add .gitignore" while being a total revert in effect. + +If no cell shipped the seed, say so explicitly in the write-up: it means the +comparator was never cross-checked in-path, and every gain inherits that risk. + +## Step 1 — Per-cell extraction agents (parallel, one per cell) + +One agent per reportable cell. Their job is to read and report, not to conclude. +Ask each for: + +- A one-line summary of every candidate: what changed, in the optimizer's own + words plus what the diff actually shows. These differ more often than you'd + expect, and the gap is a finding. +- The **arc** of the cell: did it explore then converge, thrash, or make one edit + and stop? Where does the shipped candidate sit in that arc? +- Any candidate whose message claims a measurement (*"regressed 8 of 30 + development cases"*) — quote it. These are the optimizer showing its evidence, + and they are the most quotable material in the whole corpus. +- Anything that surprised the agent, flagged as such. + +Require every item to carry `cell` + 12-char sha. An observation without a +citation cannot be verified later, so it will be dropped — tell the agents this, +so they don't waste effort on unciteable impressions. + +Tell them what the artifacts are so they don't go hunting: the extractor JSON has +messages and diffstats, and the full diffs are in the session archive under +`candidates/repository.git` (`git --git-dir=... show `). Reading actual +diffs matters for at least the shipped candidate and anything the message +describes vaguely — commit messages oversell, and a "comprehensive rewrite of the +retry logic" is sometimes a two-line change. + +## Step 2 — Compute cross-cell stats yourself, then draft observations + +Don't dispatch agents for this — write a short script (or reuse the snippet in +`references/cross_cell_stats.py`) over the extractor JSON and get, deterministically: +diff size vs. reward, candidate count vs. reward, which knobs got touched and by +how many cells, a keyword sweep for revert-ish and measurement-citing commit +messages. This is the step that used to be six blind lens agents; a script gets +the same numbers in seconds and they're exact rather than eyeballed. + +Read those stats alongside the 16 per-cell reports from step 1 and draft the +candidate observation list yourself: one claim per item, a citation (cell + sha) +on every one, and a count wherever the claim implies one. `references/lenses.md` +still lists the angles worth checking for — measurement discipline, unpriced +knobs, inert/cosmetic changes, structural shape, failure-mode targeting, cost +awareness — read it as a checklist for what to look for in the stats and reports, +not as a set of agents to spawn. Note explicit absences too: *"no cell in this +benchmark ever touched a tool-output cap"* is as much a finding as a hit. + +This draft list is what step 3 verifies. Nothing in it ships without surviving +that pass. + +## Step 3 — Adversarial verification (parallel, one per draft observation) + +Every candidate observation gets a verifier whose brief is to **refute it**, not +to confirm it. This is the step that keeps hallucinations out, so frame it that +way explicitly: the verifier re-opens the cited artifact and checks that it says +what the observation claims. + +An observation survives only if the verifier cannot refute it. Instruct verifiers +to default to *refuted* when the citation is unclear, when the diff does not show +what the message claims, or when a stated count is off. Failing an unclear claim +is cheap; a wrong claim in a paper is not. + +Refute on any of: + +- **Citation doesn't support it.** The sha is real, the commit says something + else. The single most common failure. +- **The count is wrong.** Recompute from the extractor JSON. Never accept a + tally an agent produced by reading. +- **n=1 dressed as a trend.** "Optimizers prefer X" from one cell is an anecdote. + Either it gets a count or it gets rewritten as a single-cell example. +- **Intent attributed beyond the evidence.** "The optimizer realized the timeout + was the bottleneck" requires the optimizer to have *said* so. Otherwise it + changed a timeout and we don't know why. +- **Confounded comparison.** Cells differ in model *and* harness *and* seed. A + claim that a model behaves a certain way needs cells that vary only in model. +- **Reward attribution without support.** A candidate that changed X and scored + higher does not show X caused it, unless the per-candidate partition scores in + the extractor JSON bracket that specific commit. + +Before moving to synthesis, glance at coverage yourself: any reportable cell with +zero surviving observations is worth a second look — silence about a cell usually +means it was skipped, not that it was boring. With every cell run through step 1 +this is rarely more than a one-line check, not a reason to spawn another agent. + +## Step 4 — Synthesize the observations file + +Write `harness-engineering-bench//OPTIMIZER-BEHAVIOR.md`, using +`references/output-template.md` as the shape. Only verified observations go in. + +What makes this file useful to whoever writes the discussion section: + +- **Bullets, one claim each, citation inline.** `(opus-5×opencode-r1, a3f2c1d8)`. + A reader who doubts a bullet can check it in under a minute. +- **Counts stated as counts.** "4 of 12 cells" beats "several cells". Where the + denominator matters, give it. +- **Quote the optimizer.** Its own commit messages are better evidence than any + paraphrase, and they're what makes the section readable. +- **Order by how much the reader learns**, not by pipeline stage. +- **A section for what wasn't found.** Absences constrain the claims the paper can + make, and they're invisible unless written down. +- **Say what's confounded.** One honest line about what the design can't separate + is worth more than a hedge on every bullet. + +Keep it light. If a bullet needs a paragraph of setup, it belongs in the run's +`RESULTS.md`, not here. + +## Guardrails worth holding onto + +**Reward is not the subject.** It is easy to slide into explaining the scores, +because scores are quantitative and satisfying. The subject is the optimizer's +behaviour. A cell where it did something fascinating and gained nothing is more +interesting than one where it changed a constant and gained 0.02. + +**The seed commit is not a candidate.** It's position 0 and it's what everyone +started from. Counting it inflates every denominator. + +**"Shipped nothing" is a real outcome.** Cells that ship a revert-to-baseline, or +whose only surviving diff is a `.gitignore`, are among the strongest findings +available — the optimizer had budget, made attempts, measured them, and concluded +the baseline was better. Don't let those cells get filtered out as uninteresting. + +**Don't harmonize across benchmarks.** Each benchmark's file describes its own +cells. Pooling is a later step done deliberately, and premature blending hides +the benchmark-specific behaviour that motivated running several. + +## References + +- `references/lenses.md` — angles to check for in step 2, with what each found on + tau3 and GAIA. A checklist for drafting observations, not a set of agents. +- `references/cross_cell_stats.py` — the script for step 2: diff-size/reward + correlation, candidate-count/reward correlation, knob-touch counts, keyword + sweep. Run it, don't re-derive it by eye. +- `references/output-template.md` — the structure of the observations file, with + worked example bullets at the right length and citation density. +- `scripts/extract_candidates.py` — deterministic extraction. All counts come + from here; see its docstring for what it reads and what it deliberately leaves + to the analysis. diff --git a/harness-engineering-bench/skills/analyze-optimizer-commits/references/lenses.md b/harness-engineering-bench/skills/analyze-optimizer-commits/references/lenses.md new file mode 100644 index 00000000..e8529acb --- /dev/null +++ b/harness-engineering-bench/skills/analyze-optimizer-commits/references/lenses.md @@ -0,0 +1,136 @@ +# Pattern lenses + +Six angles to check for when drafting observations in step 2. Originally these +were six separate blind agents; cut after the first run because a 20-line script +(`scripts/cross_cell_stats.py`) got the same numbers faster and exactly, and +reading the per-cell reports through this checklist yourself covers the rest. +Use it as a checklist while reading, not a dispatch list. + +Each entry gives what the lens looks for, why it earns a slot, and what it turned +up when the procedure was first run on tau3 (16 cells, 83 candidates, gpt-5.4-mini +target) and by hand on GAIA (46 cells). Treat those as calibration for the *kind* and +*specificity* of observation wanted — not as expected answers. If a lens on a new +benchmark reproduces the tau3 finding verbatim, suspect the agent read this file as +a hint sheet rather than reading the cells. + +## 1. Measurement discipline + +Does the optimizer behave like an experimentalist or like a code generator? + +Look for: commit messages citing measured numbers; candidates evaluated on `dev` +before `val`; the same idea tried twice with a variation; explicit reverts after a +measurement; ideas abandoned without ever being scored. + +Why it matters: this is the closest thing to evidence that these agents can do +empirical work rather than plausible-looking edits. It's the finding most likely +to survive into the paper's argument. + +*tau3:* 11 of 83 candidates flagged as revert-ish by keyword, across 8 of 16 cells; +treat that as a list to verify, not a count to quote. 4 of 16 cells shipped a +revert as their final answer, including the top scorer at 0.6067 — its message read +*"Revert v7 guidance: it regressed 8 of 30 development cases."* Reverting on a +measurement, not a hunch. + +## 2. Unpriced knobs + +Edits to knobs that change behaviour but aren't in the objective: `MAX_TURNS`, +`reasoning_effort`, tool-output caps, client retry counts, timeouts, context limits. + +Why it matters: the objective is single-term (`metric: score`). Anything the +optimizer spends on latency or tokens it does for reasons the reward never asked +for, and finding a knob that buys accuracy for unpriced cost is a distinct +capability from improving a harness. This is also where the paper is most exposed: +if gains come mostly from knob-turning, "harness engineering" is doing less work +than the framing implies. Count these carefully. + +Look for: which knobs get touched at all, in which direction, and whether the +optimizer's message reasons about the cost it isn't charged for. Note knobs that +*no* cell ever touched — the untouched ones bound the claim. + +*tau3:* `reasoning_effort` touched by 6 candidates across 5 of 16 cells, reverted +once: *"agent: revert reasoning effort to medium (no gain at high, higher +latency)"* (`kimi-k3-opencode-r2`, `7394ea436c04`) — the optimizer declining a win +the objective would have paid for, since latency is never charged. Note it cited no +number for the "no gain" half, and the two candidates' dev scores were in fact +identical (0.453), so the claim holds but the reasoning was cheaper than it sounds. *GAIA:* 23% of +shipped candidates changed at least one parameter; no cell ever set +`[agent] timeout_sec`, which was available. + +## 3. Inert and cosmetic changes + +Candidates that cannot have changed behaviour: comments, formatting, dead code, +`.gitignore`, docstrings, renames, config that isn't read. + +Why it matters: it separates real optimization from motion. A cell whose shipped +candidate is inert scored whatever it scored *without the harness changing*, which +makes it an accidental noise measurement — genuinely useful for bounding +cell-to-cell variance. + +Look for: diffs touching only non-executed lines; a shipped candidate identical in +behaviour to the seed; edits to files the target never imports. + +*tau3:* 2 of 16 cells shipped a tree functionally identical to the seed, confirmed +by tree hash, not by trusting the commit message — `claude-sonnet-5-claude-code-r2` +(`0031cb45be1f`, tree identical to seed) and `claude-sonnet-5-opencode-r1` +(`7179bd048021`, cumulative diff = a 3-line `.gitignore`). They scored 0.4267 and +0.4511 while the seed measured 0.5618 on the same partition through a different +script, which is how a 0.12 measurement-path gap surfaced. Run this lens first. + +## 4. Structural shape + +The physical shape of the edits: size, spread, and what kind of thing was changed. + +Why it matters: mostly to kill the intuition that bigger edits do more. If diff +size doesn't predict reward, that's worth one sentence in the paper and it stops a +reviewer asking. + +Look for: insertions/deletions per shipped candidate against reward; how many files +a candidate spans; prompt/instruction text vs control flow vs configuration vs new +helper code; whether cells that edit prompts differ systematically from cells that +edit code. + +*tau3:* diff size carried no signal — the best cell shipped +1/−8 and the worst ++19/−10. Candidate *count* did correlate with reward (+0.405, n=16, so suggestive at +best): cells that iterated more scored better, independent of how much they wrote. + +## 5. Failure-mode targeting + +Did the optimizer find the *actual* dominant failure mode, and did it aim at it? + +Look for: commit messages naming a specific failure (a crash, a tool misuse, a +truncation, a format violation); edits that plainly target one; the gap between +what the optimizer believed was failing and what the eval records show. And the +inverse: known failure modes nothing ever touched. + +Why it matters: a benchmark where every cell fixes the same thing is a benchmark +with one dominant failure mode — exactly the concern raised about DABstep and +OfficeQA jumping most of their headroom in one step. Convergent targeting across +independent cells is evidence for that; divergent targeting is evidence against. + +Include the seed's known defects in the agent's brief where they're documented +(e.g. tau3's empty-turn crash), so it can check whether the optimizer independently +found what we already knew was wrong. + +## 6. Cost and latency awareness + +Does the optimizer reason about time and money it isn't scored on? + +Look for: messages mentioning latency, tokens, cost, timeouts, or turn budgets; +edits that trade accuracy for speed or the reverse; awareness of the per-case +timeout; any sign it modelled the eval loop's economics. + +Why it matters: the reward is accuracy-only, but the *paper* wants to talk about a +reward vector including latency and dollars. Whether the optimizer volunteers this +reasoning unprompted bears directly on whether pricing it would change behaviour. + +Overlaps lens 2 by design — that lens counts knob edits, this one reads the +reasoning. Two agents reaching the same finding by different routes is +corroboration, and it's cheaper than trying to draw a clean boundary. + +## Adding a lens + +Add one when a benchmark has an affordance these don't cover — a browsing +benchmark's retry-and-backoff behaviour, a coding benchmark's test-writing, a +multimodal benchmark's image handling. Write it in the same shape (what to look +for, why it earns a slot) and leave the calibration section empty until it's been +run. A lens with fabricated example findings is worse than no lens. diff --git a/harness-engineering-bench/skills/analyze-optimizer-commits/references/output-template.md b/harness-engineering-bench/skills/analyze-optimizer-commits/references/output-template.md new file mode 100644 index 00000000..fb241aae --- /dev/null +++ b/harness-engineering-bench/skills/analyze-optimizer-commits/references/output-template.md @@ -0,0 +1,118 @@ +# Output template + +Write to `harness-engineering-bench//OPTIMIZER-BEHAVIOR.md`. + +The reader is whoever writes the paper's discussion section. They want material +they can lift, and they want to be able to check any bullet in under a minute +without asking you. That shapes everything below. + +## Structure + +```markdown +# What the optimizer did — + + + +## Scope + +- N cells analyzed, M excluded (why, per cell) +- K candidates, seed excluded +- Reward range across cells: low – high + +## How it worked + + + +## What it changed + + + +## What it did not do + + + +## What this cannot separate + + + +## Method + + +``` + +## Bullet form + +One claim, citation inline, no wind-up: + +> - 11 of 73 candidates (15%) were explicit reverts of the optimizer's own earlier +> work, and 4 of 12 cells shipped a revert as their final answer — including the +> top scorer at 0.6067: *"Revert v7 guidance: it regressed 8 of 30 development +> cases"* (`claude-opus-5-opencode-r2`, `a3f2c1d8`). + +That bullet works because the count has a denominator, the quote is the +optimizer's own, and the citation lets a doubter go look. Compare: + +> - The optimizers showed a notable tendency toward self-correction, often +> reverting changes that appeared to have regressed performance. + +Same information, unusable. No count, no citation, "often" and "appeared to" +doing the work that evidence should. + +More bullets at the right length: + +> - Diff size carried no signal: the best cell shipped +1/−8 lines +> (`claude-opus-5-opencode-r2`, `8487bf75508e`), the worst +19/−10 +> (`claude-sonnet-5-opencode-r2`, `b125bda2614f`). Candidate *count* did track +> reward (r = +0.405, n = 16 — suggestive, not significant): cells that iterated +> more scored better regardless of how much they wrote. + +> - `reasoning_effort` was found by 5 of 16 cells and explicitly reverted by one, +> *"agent: revert reasoning effort to medium (no gain at high, higher latency)"* +> (`kimi-k3-opencode-r2`, `7394ea436c04`) — the optimizer declining a change the +> objective would have paid for, since reward is single-term on score and never +> charges for latency. + +> - No cell ever set `[agent] timeout_sec`, though it is exposed in the same config +> block as knobs that 23% of shipped candidates did change. Whatever draws the +> optimizer to a knob, availability alone isn't it. + +> - 2 of 16 cells shipped a harness functionally identical to the seed: one whose +> shipped tree hash equals the seed's exactly (`claude-sonnet-5-claude-code-r2`, +> `0031cb45be1f`, tree `65e2c147b655`, test 0.4267), one whose only cumulative +> diff is a 3-line `.gitignore` the target never reads +> (`claude-sonnet-5-opencode-r1`, `7179bd048021`, test 0.4511). Both had made +> scored attempts and measured every one below the seed, so reverting was the +> best move available to them. + +Note the shape of that last bullet: it cites a *tree hash*, not a commit message. +"Shipped nothing" is the single most consequential claim this analysis can make — +it converts a cell into a free measurement of the unmodified seed — so it has to +rest on `git rev-parse ^{tree}` agreeing, never on a commit that says +"revert". On tau3 those two cells scored 0.4267 and 0.4511 through the +finalization path while the seed measured 0.5618 through `rescore_candidate.py`, +which is how a 0.12 measurement-path gap became visible at all. If your benchmark +has such a cell, it is the most valuable one in the corpus — find it first. + +## Citations + +`(cell-name, 12-char-sha)`. Cell name as it appears in `runs//`, since +that's what someone will `cd` into. Include reward when the bullet is about +outcome. For a claim spanning cells, cite two or three examples rather than all of +them — the count carries the generality, the citations prove the kind. + +## What to leave out + +- **Reward analysis.** Score tables belong in `RESULTS.md`. Here, reward appears + only where it's evidence about behaviour. +- **Hedging on every bullet.** State confounds once, in their own section, and + then write plainly. A bullet qualified three ways reads as if you don't believe + it. +- **Methodology narrative.** Nobody needs the fan-out described. One short Method + section, at the end. +- **Anything unverified.** If it didn't survive step 3, it isn't in the file. A + finding you liked but couldn't support is worth a line in the Method section as + a dropped claim, if it's the kind of thing a reader would otherwise assume you + checked. diff --git a/harness-engineering-bench/skills/analyze-optimizer-commits/scripts/cross_cell_stats.py b/harness-engineering-bench/skills/analyze-optimizer-commits/scripts/cross_cell_stats.py new file mode 100644 index 00000000..987dd337 --- /dev/null +++ b/harness-engineering-bench/skills/analyze-optimizer-commits/scripts/cross_cell_stats.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Cross-cell stats over extract_candidates.py's JSON. Replaces the six blind +lens agents from the skill's first draft: correlations, knob-touch counts, and a +keyword sweep, computed exactly instead of eyeballed across 16 per-cell reports. + + python3 cross_cell_stats.py --json /tmp/-candidates.json + +Keyword hits are candidates FOR VERIFICATION, not a count to quote directly -- +step 3 still has to check each one against the real diff before it ships. +""" +from __future__ import annotations + +import argparse +import json +import re +import statistics as st + + +def corr(a: list[float], b: list[float]) -> float: + ma, mb = st.mean(a), st.mean(b) + num = sum((x - ma) * (y - mb) for x, y in zip(a, b)) + den = (sum((x - ma) ** 2 for x in a) * sum((y - mb) ** 2 for y in b)) ** 0.5 + return num / den if den else float("nan") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--json", required=True) + args = ap.parse_args() + + rows = [r for r in json.load(open(args.json)) if r["reportable"]] + cands = lambda r: [c for c in r["candidates"] if not c["is_seed"]] + + print(f"{len(rows)} reportable cells\n") + print(f"{'cell':34} {'reward':>7} {'n':>3} {'shipPos':>8} {'ship +/-':>10}") + for r in sorted(rows, key=lambda x: -x["reward"]): + cs = cands(r) + sh = next((c for c in r["candidates"] if c["is_shipped"]), None) + pos = sh["position"] if sh else None + print(f"{r['cell']:34} {r['reward']:.4f} {len(cs):3} " + f"{(str(pos) + '/' + str(len(cs))) if pos is not None else '-':>8} " + f"{('+' + str(sh['insertions']) + '/-' + str(sh['deletions'])) if sh else '-':>10}") + + n = [len(cands(r)) for r in rows] + rew = [r["reward"] for r in rows] + shipsize = [next(c for c in r["candidates"] if c["is_shipped"])["insertions"] + + next(c for c in r["candidates"] if c["is_shipped"])["deletions"] + for r in rows] + print(f"\ncandidates: total {sum(n)}, median {st.median(n)}, range {min(n)}-{max(n)}") + print(f"corr(candidate count, reward) = {corr(n, rew):+.3f} (n={len(rows)} cells)") + print(f"corr(shipped diff size, reward) = {corr(shipsize, rew):+.3f}") + + print("\n=== keyword sweep (candidates for verification, NOT a verified count) ===") + allc = [(r["cell"], c) for r in rows for c in cands(r)] + print(f"total candidates: {len(allc)}") + for label, pat in [ + ("revert-ish", r"\brevert|\bback out|\brestore\b|\bundo\b"), + ("cites a number", r"\d+\s*(of|/)\s*\d+|0\.\d{2,}|\d+%"), + ("reasoning_effort", r"reasoning[_ ]effort"), + ("MAX_TURNS", r"max[_ ]turns"), + ("timeout", r"timeout"), + ("retry", r"retr(y|ies|ying)"), + ]: + hits = [(cell, c["id"], c["subject"][:52]) for cell, c in allc + if re.search(pat, c["subject"] + " " + c["body"], re.I)] + cells = len({h[0] for h in hits}) + print(f"\n{label}: {len(hits)} candidates across {cells} cells") + for h in hits[:6]: + print(f" {h[0][:28]:28} {h[1]} {h[2]}") + if len(hits) > 6: + print(f" ... and {len(hits) - 6} more") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness-engineering-bench/skills/analyze-optimizer-commits/scripts/extract_candidates.py b/harness-engineering-bench/skills/analyze-optimizer-commits/scripts/extract_candidates.py new file mode 100644 index 00000000..359dd61d --- /dev/null +++ b/harness-engineering-bench/skills/analyze-optimizer-commits/scripts/extract_candidates.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Extract every optimizer candidate from a benchmark's cells as structured facts. + + python3 extract_candidates.py --benchmark tau3 # -> stdout table + python3 extract_candidates.py --benchmark tau3 --json out.json # -> machine readable + python3 extract_candidates.py --runs-dir /path/to/runs --benchmark gaia + +Why a script rather than asking an agent to read the repos: counts are the part of +this analysis most likely to be wrong and least likely to be checked. "15% of +candidates were reverts" is a claim a reader will trust and nobody will recompute. +An agent counting commits by eye across 12 cells will miscount, and the error is +invisible in the final prose. So every number that reaches the write-up comes from +here, and agents are asked to interpret rather than to tally. + +What it reads, per cell: + verifier/finalization.json the shipped candidate, its reward, validity fields + verifier/session.tar.gz candidates/repository.git -- every candidate commit + evaluations/*/evaluation.json -- per-candidate scores + +What it deliberately does NOT do: classify behaviour. Whether a commit is a +"revert" or "inert" or "structural" is a judgement that depends on reading the diff, +and a regex on the commit message gets it wrong often enough to matter (a commit +saying "revert speculative KB tweak" also adds 19 lines of new logic). The script +surfaces the evidence -- message, diffstat, files touched, scores before and after -- +and leaves the call to the analysis, which has to cite what it saw. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import tarfile +import tempfile +from pathlib import Path + + +def git(repo: Path, *args: str) -> str: + out = subprocess.run(["git", "--git-dir", str(repo), *args], + capture_output=True, text=True) + return out.stdout + + +def extract_session(tgz: Path, dest: Path) -> Path | None: + """Pull just the candidate repo and evaluation records out of a session archive.""" + try: + with tarfile.open(tgz) as tar: + wanted = [m for m in tar.getmembers() + if "/candidates/repository.git/" in m.name + or m.name.endswith("/evaluation.json")] + if not wanted: + return None + # filter= landed in 3.12 and is a TypeError before that. These + # archives are our own verifier's output, so the guard is about + # running on the box's 3.10 rather than about untrusted input. + if sys.version_info >= (3, 12): + tar.extractall(dest, members=wanted, filter="data") + else: + tar.extractall(dest, members=wanted) + except Exception as exc: + print(f" ! could not read {tgz}: {exc}", file=sys.stderr) + return None + hits = list(dest.glob("**/candidates/repository.git")) + return hits[0] if hits else None + + +def per_candidate_scores(session_root: Path) -> dict[str, dict[str, float]]: + """candidate id -> {partition: score}, from the sidecar's evaluation records.""" + scores: dict[str, dict[str, float]] = {} + for path in session_root.glob("**/evaluations/*/evaluation.json"): + try: + doc = json.loads(path.read_text()) + except Exception: + continue + request, report = doc.get("request") or {}, doc.get("report") or {} + cand = ((request.get("candidate") or {}).get("id") or "")[:12] + part = (request.get("evaluation_set") or {}).get("partition") + score = (report.get("metrics") or {}).get("score") + if cand and part and score is not None: + scores.setdefault(cand, {})[part] = score + return scores + + +def read_cell(cell_dir: Path, tmp: Path) -> dict | None: + finals = sorted(cell_dir.glob("jobs/*/task__*/verifier/finalization.json")) + if not finals: + return None + final = json.loads(finals[-1].read_text()) + metrics = (final.get("reward_metrics") or {}).get("reward", {}) or {} + tgz = finals[-1].parent / "session.tar.gz" + + row = { + "cell": cell_dir.name, + "benchmark": cell_dir.parent.name, + "shipped": final.get("shipped"), + "reward": (final.get("rewards") or {}).get("reward"), + "baseline_reward": (final.get("baseline_rewards") or {}).get("reward"), + "error_rate": metrics.get("error_rate"), + "total_tokens": metrics.get("inference_total_tokens"), + "mean_case_wall_seconds": metrics.get("mean_case_wall_seconds"), + "shipped_candidate_id": ((final.get("candidate") or {}).get("id") or "")[:12], + "shipped_candidate_desc": (final.get("candidate") or {}).get("description", ""), + "candidates": [], + } + # A cell is only reportable if it shipped, scored every case, and metered + # tokens. swe-atlas produced cells reporting shipped/error_rate 0.0 while every + # case had been dropped by infrastructure and no tokens were spent, so the token + # check is what distinguishes a real run from a hollow one. + row["reportable"] = bool( + row["shipped"] and row["error_rate"] in (0.0, 0) and row["total_tokens"]) + + if not tgz.is_file(): + return row + dest = tmp / cell_dir.name + dest.mkdir(parents=True, exist_ok=True) + repo = extract_session(tgz, dest) + if repo is None: + return row + scores = per_candidate_scores(dest) + + log = [l.split("\t", 2) for l in + git(repo, "log", "--all", "--format=%H\t%at\t%s").splitlines() if l.strip()] + log.reverse() # oldest first: the seed is index 0 + for position, parts in enumerate(log): + sha, when, subject = (parts + ["", "", ""])[:3] + short = sha[:12] + shortstat = git(repo, "show", "--shortstat", "--format=", sha).strip().splitlines() + stat = shortstat[-1].strip() if shortstat else "" + files = [f for f in git(repo, "show", "--name-only", "--format=", sha).splitlines() + if f.strip() and "__pycache__" not in f] + ins = int(m.group(1)) if (m := re.search(r"(\d+) insertion", stat)) else 0 + dele = int(m.group(1)) if (m := re.search(r"(\d+) deletion", stat)) else 0 + body = git(repo, "show", "--format=%b", "--no-patch", sha).strip() + row["candidates"].append({ + "position": position, # 0 = seed + "id": short, + "subject": subject, + "body": body[:2000], # optimizers explain their reasoning here + "insertions": ins, + "deletions": dele, + "files": files, + "is_seed": position == 0, + "is_shipped": short == row["shipped_candidate_id"], + "scores": scores.get(short, {}), + }) + return row + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--benchmark", required=True) + ap.add_argument("--runs-dir", default=None, + help="defaults to /runs") + ap.add_argument("--json", default=None, help="write structured output here") + args = ap.parse_args() + + repo_root = Path(__file__).resolve().parents[4] + runs = Path(args.runs_dir) if args.runs_dir else repo_root / "runs" + bench_dir = runs / args.benchmark + if not bench_dir.is_dir(): + sys.exit(f"no such benchmark directory: {bench_dir}") + + rows = [] + with tempfile.TemporaryDirectory(prefix="optcommits-") as tmpname: + tmp = Path(tmpname) + for cell in sorted(p for p in bench_dir.iterdir() if p.is_dir()): + row = read_cell(cell, tmp) + if row: + rows.append(row) + + if not rows: + sys.exit(f"no cells with a finalization.json under {bench_dir}") + + reportable = [r for r in rows if r["reportable"]] + # Exclude the seed from every count: it is position 0 and it is what all + # cells started from, so counting it inflates each denominator by one. + total_cands = sum(max(len(r["candidates"]) - 1, 0) for r in reportable) + print(f"benchmark {args.benchmark}: {len(rows)} cells, " + f"{len(reportable)} reportable, {total_cands} candidates (seed excluded)\n") + print(f"{'cell':34} {'reward':>7} {'cands':>5} {'shipped pos':>11} shipped subject") + for r in rows: + flag = "" if r["reportable"] else " [NOT REPORTABLE]" + pos = next((c["position"] for c in r["candidates"] if c["is_shipped"]), None) + n = max(len(r["candidates"]) - 1, 0) + rew = f"{r['reward']:.4f}" if isinstance(r["reward"], (int, float)) else "-" + print(f"{r['cell'][:34]:34} {rew:>7} {n:>5} " + f"{(str(pos)+' of '+str(n)) if pos is not None else '-':>11} " + f"{r['shipped_candidate_desc'][:44]}{flag}") + + if args.json: + Path(args.json).write_text(json.dumps(rows, indent=2) + "\n") + print(f"\nwrote {args.json}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness-engineering-bench/tau3/OPTIMIZER-BEHAVIOR.md b/harness-engineering-bench/tau3/OPTIMIZER-BEHAVIOR.md new file mode 100644 index 00000000..19f18244 --- /dev/null +++ b/harness-engineering-bench/tau3/OPTIMIZER-BEHAVIOR.md @@ -0,0 +1,143 @@ +# What the optimizer did — tau3 + +16 cells, target model `gpt-5.4-mini`, 4 optimizer models x 2 harnesses each x 2 +repeats. 83 candidate commits (seed excluded). Every row below is generated from +`extract_candidates.py`'s JSON, not typed by hand; every behavioral claim in the +findings table cites a `(cell, sha)` you can re-check in under a minute with +`git --git-dir=/session/candidates/repository.git show `. + +**Read the measurement-path gap section before the gain column of any table +below** — two cells here shipped the unmodified seed, and it scores ~0.12 below +the benchmark's own pinned floor for that same code. Every "gain vs floor" number +in this file inherits that until it's resolved. + +## Per-cell + +| cell | model | harness | reward | gain vs floor | candidates | shipped pos | shipped sha | knobs touched | +|---|---|---|---:|---:|---:|---|---|---| +| claude-opus-5-opencode-r2 | claude-opus-5 | opencode | 0.6067 | +0.0449 | 8 | 8/8 | `8487bf75508e` | reasoning_effort | +| claude-opus-5-claude-code-r1 | claude-opus-5 | claude-code | 0.5911 | +0.0293 | 5 | 3/5 | `1a0ad193ce59` | retry, timeout | +| claude-opus-5-claude-code-r2 | claude-opus-5 | claude-code | 0.5800 | +0.0182 | 8 | 6/8 | `d8e991280d32` | MAX_TURNS, reasoning_effort, retry | +| claude-opus-5-opencode-r1 | claude-opus-5 | opencode | 0.5733 | +0.0115 | 6 | 6/6 | `2fc25a3fa0d0` | reasoning_effort, retry | +| kimi-k3-kimi-cli-r1 | kimi-k3 | kimi-cli | 0.5378 | −0.0240 | 13 | 13/13 | `ffb61506326c` | tool-output cap | +| kimi-k3-kimi-cli-r2 | kimi-k3 | kimi-cli | 0.5244 | −0.0374 | 3 | 2/3 | `f64128c5e08d` | retry, timeout | +| gpt-5.6-sol-opencode-r1 | gpt-5.6 | opencode | 0.5178 | −0.0440 | 2 | 2/2 | `193644484ba3` | reasoning_effort | +| gpt-5.6-terra-opencode-r1 | gpt-5.6 | opencode | 0.4889 | −0.0729 | 2 | 2/2 | `36211081992f` | reasoning_effort (implicit, see note) | +| gpt-5.6-sol-codex-r1 | gpt-5.6 | codex | 0.4867 | −0.0751 | 2 | 2/2 | `f128802715532` | reasoning_effort (unstated in message) | +| kimi-k3-opencode-r1 | kimi-k3 | opencode | 0.4844 | −0.0774 | 6 | 2/6 | `3d89801a6438` | — | +| kimi-k3-opencode-r2 | kimi-k3 | opencode | 0.4778 | −0.0840 | 7 | 7/7 | `ba2955e328f2` | reasoning_effort (reverted upstream) | +| claude-sonnet-5-claude-code-r1 | claude-sonnet-5 | claude-code | 0.4622 | −0.0996 | 3 | 3/3 | `ad38074ed316` | MAX_TURNS, retry | +| claude-sonnet-5-opencode-r1 | claude-sonnet-5 | opencode | 0.4511 | −0.1107 | 5 | 5/5 | `7179bd048021` | — (shipped = seed + `.gitignore`) | +| gpt-5.6-terra-codex-r1 | gpt-5.6 | codex | 0.4356 | −0.1262 | 4 | 4/4 | `90325a78cefa` | — | +| claude-sonnet-5-claude-code-r2 | claude-sonnet-5 | claude-code | 0.4267 | −0.1351 | 6 | 6/6 | `0031cb45be1f` | — (shipped = seed, byte-identical) | +| claude-sonnet-5-opencode-r2 | claude-sonnet-5 | opencode | 0.3978 | −0.1640 | 3 | 3/3 | `b125bda2614f` | — | + +"knobs touched" = any candidate in that cell's chain, not necessarily the shipped +one; a cell can touch a knob and still ship something that doesn't carry it. Two +rows are marked with the sha of the `reasoning_effort` commit even though the +shipped subject doesn't name it — see the findings table for exactly which +message it was buried in. + +## Per optimizer model, aggregated + +| model | cells | mean reward | cells above 0.5618 floor | range | +|---|---:|---:|---:|---| +| claude-opus-5 | 4 | 0.5878 | 4/4 | 0.5733 – 0.6067 | +| kimi-k3 | 4 | 0.5061 | 0/4 | 0.4778 – 0.5378 | +| gpt-5.6 | 4 | 0.4822 | 0/4 | 0.4356 – 0.5178 | +| claude-sonnet-5 | 4 | 0.4344 | 0/4 | 0.3978 – 0.4622 | + +**This ranking should NOT be read as "only opus-5 improved the harness."** Two of +claude-sonnet-5's four cells shipped a harness with zero behavioral difference +from the seed (see next section) — their score is a floor measurement mislabeled +as sonnet-5's output, not evidence sonnet-5 made things worse. With those two +cells removed, claude-sonnet-5's remaining pair is 0.4622 and 0.3978 — still last, +but on n=2 rather than n=4, and the gap to the (mismeasured) floor shrinks once +you compare against the in-path seed value (~0.44) instead of 0.5618. + +## Per optimizer harness (opencode vs. each model's native harness) + +| harness | cells | mean reward | mean candidates | notes | +|---|---:|---:|---:|---| +| opencode | 8 | 0.5054 | 5.9 | used by every model; widest score spread (0.3978–0.6067) | +| claude-code | 4 | 0.5100 | 5.5 | opus-5 and sonnet-5 only | +| kimi-cli | 2 | 0.5311 | 8.0 | kimi-k3 only; both cells cite measured numbers in commit messages, the only native harness besides claude-code that does | +| codex | 2 | 0.4612 | 3.0 | gpt-5.6 only; both cells have bare one-line subjects with empty bodies on every candidate | + +`codex` and `kimi-cli` each have only 2 cells, so treat those rows as descriptive, +not statistically load-bearing. + +## Verified behavioral findings + +Each row was independently attacked by a second agent instructed to refute it +before it's listed here. Verdict is what survived, corrected where the correction +itself matters. + +| cell | sha | finding | +|---|---|---| +| `claude-sonnet-5-claude-code-r2` | `0031cb45be1f` | Shipped tree is byte-identical to the seed (tree hash `65e2c147b655` matches exactly). Reward 0.4267 is a measurement of the unmodified seed through the finalization path. | +| `claude-sonnet-5-opencode-r1` | `7179bd048021` | Shipped tree's only cumulative diff from seed is a 3-line `.gitignore`, never read by the target at runtime. Reward 0.4511 is likewise effectively the seed. | +| `gpt-5.6-sol-codex-r1` | `f128802715532` | Message says only "bound knowledge retrieval and enforce strict tool schemas." Diff also silently raises `reasoning_effort` medium→high and adds new behavioral prompt rules never mentioned. | +| `gpt-5.6-terra-codex-r1` | `90325a78cefa` | Message says "Add concise safeguards for knowledge-base offers." Diff is dominantly (43 of 57 changed lines) a revert of the prior candidate's entire prompt rewrite; the named addition is 5 lines. | +| `claude-opus-5-opencode-r1` | `25afede52a4f` | Adds an `end_conversation` interception: on the model's first attempt to end, injects a fabricated tool-role reply — *"NOT ENDED — the conversation is still open..."* — without calling the real tool. | +| `claude-opus-5-claude-code-r2` | `968131deb0a7` | Adds a `KB_search` result cache keyed on exact arguments. On a repeat query the cached text is returned with a *"you already ran this exact search... do not run it a third time"* prefix; the real tool is never invoked on a hit. | +| `kimi-k3-opencode-r1` | `3d89801a6438` | Shipped because validation beat the seed (0.4267 vs 0.3667) despite its own development score (0.28) being worse than both the seed (0.3333) and the candidate immediately before it (0.36). | +| `claude-opus-5-claude-code-r1` | `1a0ad193ce59` | Largest measured dev gain in the corpus (0.4267→0.5733) is 100% a retry/backoff mechanism on provider rate-limit errors — zero prompt change in the same commit. Message cites "9 cases lost outright to Azure 429s." | +| `kimi-k3-opencode-r2` | `7394ea436c04` | Only `reasoning_effort` reversion in the corpus: *"revert reasoning effort to medium (no gain at high, higher latency)"*. The two candidates immediately upstream had identical dev scores (0.453/0.453) — supports "no gain"; the revert commit itself carries no recorded score, so it restates rather than re-measures. | +| `kimi-k3-kimi-cli-r1` | `ffb61506326c` | Shipped a byte-exact revert to its own earlier candidate `e5e69600fdf0` (empty `git diff` on the target file). Same code scored development 0.80 there and 0.70 here — a 0.10 swing with zero behavioral difference, i.e. same-cell run-to-run noise. | +| `claude-sonnet-5-opencode-r2` | `b125bda2614f` | Bundles a measured revert with a second, unmeasured behavioral change (rewritten stop-marker regex). The bundle's own dev (0.3571) and val (0.38) are both worse than the candidate it partially reverted from (dev 0.4533) and worse than the seed (val 0.4067). Shipped anyway. | +| `kimi-k3-opencode-r2` | `ba2955e328f2` | Commit message calls itself a test — *"baseline prompt + conversation-ended loop fix (isolate loop-fix effect)"* — reverts every accumulated prompt change to the byte-identical seed prompt while keeping two earlier code-level changes. An explicit ablation shipped as the final answer. | + +Three drafted claims did **not** survive and are recorded so the correction isn't +lost: (1) "all four opus-5 cells cite measured numbers" — false, only the two +`claude-code`-harness ones do; the two `opencode` ones have empty bodies like +gpt-5.6's cells. (2) A knob-touch count of "6 candidates / 5 cells" for +`reasoning_effort` — the real count is 7 candidates / 6 cells (see the per-cell +table above, which reflects the corrected count). (3) A claim that +`claude-opus-5-opencode-r2`'s one-tool-call-per-turn enforcement was undisclosed — +it's real, but it's named in the commit title itself ("one action per turn") and +explained in a code comment, so "silent" was wrong. + +## The measurement-path gap + +Two cells above (`claude-sonnet-5-claude-code-r2`, `claude-sonnet-5-opencode-r1`) +shipped a harness with zero behavioral difference from the seed, confirmed by +exact git tree-hash comparison. Their rewards (0.4267, 0.4511) are therefore +direct measurements of the unmodified seed through the finalization path — the +same path every cell in the tables above was scored through. + +The benchmark's pinned `baseline_reward` is **0.5618**, measured on the identical +seed through a different script (`rescore_candidate.py`, bare `harbor run`, no +gateway/sidecar). Same code, same 150 held-out cases, ~0.12 apart. + +A same-code repeat elsewhere in the corpus bounds how much of that could be +ordinary noise rather than a systematic path effect: `kimi-k3-kimi-cli-r1`'s +byte-exact revert (`e5e69600fdf0` → `ffb61506326c`) scored 0.80 and 0.70 +development on identical code — a 0.10 swing with no path change at all. That +doesn't resolve the gap either way; a rescore probe aimed at settling it directly +hit budget-exhausted API errors mid-run and was inconclusive. + +**Every "gain vs floor" column above is provisional until this is settled.** +Against the in-path seed instead of the 0.5618 floor, most non-opus cells look +like modest improvements rather than regressions — the ranking direction for +opus-5 doesn't change, but the sign for everyone else might. + +## What this cannot separate + +- **Model, harness, and seed aren't independently varied.** A "model X vs model + Y" claim is really about that model+harness+seed combination. The + per-harness table above is the closest this gets to isolating harness effect, + and even there codex/kimi-cli have only 2 cells each. +- **The measurement-path gap**, described above, affects every gain number but + not the behavioral findings (those describe what happened, independent of + which script scored it). + +## Method + +`extract_candidates.py` unpacked each cell's `session.tar.gz` and walked the +candidate git history. 16 per-cell agents read the actual diffs and commit +history, not just messages. All tables above are regenerated from that JSON by +script, not hand-typed. 15 candidate findings were drafted from those reports; +each was independently attacked by a verifier told to default to "refuted" on any +unclear citation or count. 12 survived as stated, 3 required the corrections +noted above. From fd0f23451e7b6eb86d33a1dcbaf59f88f144c35d Mon Sep 17 00:00:00 2001 From: yash-scaleai Date: Sun, 2 Aug 2026 20:55:13 +0000 Subject: [PATCH 6/6] extract_candidates: fix extraction containment, guard the candidate-count assumption Two issues from an automated review, both real. Legacy extraction (Python <3.12, where filter="data" is a TypeError) passed tarfile members straight to extractall with no path or symlink checks. Added the same containment check filter="data" does, so a traversal member or a symlink pointing outside dest is rejected on every Python version, not just 3.12+. The candidate-count concern was that `git log --all` walks every commit reachable from every ref, which is only a correct candidate enumeration if the repo is one linear lineage. Tried the principled-looking alternative -- refs/vero/candidates/*, one ref per candidate -- and it silently undercounted: two real, git-committed candidates in claude-opus-5-opencode-r1 have no ref at all, because they were superseded before ever being submitted for scoring (empty `scores` dict, which the analysis already reports as its own finding). Switching would have quietly corrupted a report that's already been verified and shared. Kept log --all, and instead made the assumption it depends on explicit: checked all 16 tau3 repos by hand (zero merges, one root commit each), and the script now checks that itself and refuses to guess on a repo where it doesn't hold, rather than silently mis-attributing positions on some other benchmark's history shape. Co-Authored-By: Claude Opus 5 --- .../scripts/extract_candidates.py | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/harness-engineering-bench/skills/analyze-optimizer-commits/scripts/extract_candidates.py b/harness-engineering-bench/skills/analyze-optimizer-commits/scripts/extract_candidates.py index 359dd61d..30b1d699 100644 --- a/harness-engineering-bench/skills/analyze-optimizer-commits/scripts/extract_candidates.py +++ b/harness-engineering-bench/skills/analyze-optimizer-commits/scripts/extract_candidates.py @@ -44,18 +44,31 @@ def git(repo: Path, *args: str) -> str: return out.stdout +def _is_contained(member: tarfile.TarInfo, dest: Path) -> bool: + """Reject anything tarfile's own `filter="data"` would reject on 3.12+. + + Backported by hand for 3.10/3.11, where `filter=` is a TypeError. These + archives are our own verifier's output, not attacker-controlled in the usual + run -- but --runs-dir lets a caller point this at an arbitrary directory, so + a traversal member (`../../etc/...`) or a symlink pointing outside `dest` + must be rejected the same way on every Python version, not just 3.12+. + """ + if member.issym() or member.islnk() or member.isdev(): + return False + target = (dest / member.name).resolve() + return target == dest or dest in target.parents + + def extract_session(tgz: Path, dest: Path) -> Path | None: """Pull just the candidate repo and evaluation records out of a session archive.""" try: with tarfile.open(tgz) as tar: wanted = [m for m in tar.getmembers() - if "/candidates/repository.git/" in m.name - or m.name.endswith("/evaluation.json")] + if ("/candidates/repository.git/" in m.name + or m.name.endswith("/evaluation.json")) + and _is_contained(m, dest)] if not wanted: return None - # filter= landed in 3.12 and is a TypeError before that. These - # archives are our own verifier's output, so the guard is about - # running on the box's 3.10 rather than about untrusted input. if sys.version_info >= (3, 12): tar.extractall(dest, members=wanted, filter="data") else: @@ -121,6 +134,28 @@ def read_cell(cell_dir: Path, tmp: Path) -> dict | None: return row scores = per_candidate_scores(dest) + # `git log --all` walks every commit reachable from every ref. That's only a + # correct candidate enumeration if the repo is a single linear lineage -- + # otherwise it can pull in unrelated history, inflate the count, and shift + # every position after it. Checked by hand: `refs/vero/candidates/*` looks + # like the principled alternative (one ref per candidate), but it isn't one -- + # on real sessions it undercounts, missing candidates that were committed and + # superseded before ever being submitted for scoring (they still show up here + # with an empty `scores` dict, which is itself a finding, not noise to drop). + # So: verify the linear-lineage assumption explicitly and fail loudly if it + # doesn't hold, rather than switch to a mechanism that silently drops data. + if git(repo, "log", "--all", "--merges", "--format=%H").strip(): + print(f" ! {cell_dir.name}: candidate repo has merge commits -- " + "log --all is not a safe candidate enumeration here, skipping", + file=sys.stderr) + return row + roots = git(repo, "rev-list", "--max-parents=0", "--all").split() + if len(roots) != 1: + print(f" ! {cell_dir.name}: candidate repo has {len(roots)} root " + "commits (expected 1) -- not a single lineage, skipping", + file=sys.stderr) + return row + log = [l.split("\t", 2) for l in git(repo, "log", "--all", "--format=%H\t%at\t%s").splitlines() if l.strip()] log.reverse() # oldest first: the seed is index 0