tau3: gpt-5.4-mini baseline + a benchmark-agnostic optimizer-commit analysis skill

harness-engineering-bench/scripts/backfill_baseline.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Fill `baseline_rewards` into finalization.json for cells that ran without a pin.
+
+    python3 harness-engineering-bench/scripts/backfill_baseline.py \
+        --benchmark tau3 --reward-key reward --value 0.5679 \
+        --provenance "K=3 rescore_candidate.py --seed on build.gpt54mini.yaml, ..." \
+        [--push] [--dry-run]
+
+Why this is legitimate rather than editing results after the fact: nothing ever
+measures `baseline_rewards` during a run. Every other benchmark carries a
+`baseline_reward` that was measured out of band by rescore_candidate.py days
+earlier and pasted into build.yaml, and the verifier simply copies it into the
+finalization payload. Filling the same field, with a value from the same script on
+the same seed and target, gives tau3's cells identical provenance -- it does not
+invent a number, it supplies the one the config should have carried.
+
+Why it matters: the results pipeline computes gain from `baseline_rewards`. Left
+empty, tau3 is the one benchmark whose gain cannot be derived, and the likely
+failure mode is that nobody notices until the table is built.
+
+Two hard constraints this respects:
+
+- `VerificationResult` is a StrictModel with extra="forbid", and vero's report.py
+  validates the finalization payload through it. So NO new keys go in that file --
+  provenance goes in a sibling baseline_reward_provenance.json that no schema reads.
+- The candidate's own `rewards` are never touched. Only the comparator is added.
+
+Every patch is validated by re-parsing the file through VerificationResult before
+it is written, so a schema mistake fails here rather than in someone's report.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parent.parent.parent
+S3 = "s3://scale-ml/harness-engineering-bench"
+PROFILE = "ml-worker"
+
+
+def validates(payload: dict) -> tuple[bool, str]:
+    """Re-parse through vero's own model, so a bad edit cannot reach S3."""
+    try:
+        sys.path.insert(0, str(REPO / "vero" / "src"))
+        from vero.sidecar.verifier import VerificationResult  # type: ignore
+    except Exception as exc:  # vero not importable here; skip rather than guess
+        return True, f"(schema check skipped: {exc})"
+    try:
+        VerificationResult.model_validate_json(json.dumps(payload))
+        return True, "ok"
+    except Exception as exc:
+        return False, str(exc)[:200]
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    ap.add_argument("--benchmark", required=True)
+    ap.add_argument("--reward-key", default="reward")
+    ap.add_argument("--value", type=float, required=True)
+    ap.add_argument("--provenance", required=True,
+                    help="one line recording how the value was measured")
+    ap.add_argument("--push", action="store_true", help="also upload to S3")
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    pattern = f"runs/{args.benchmark}/*/jobs/*/task__*/verifier/finalization.json"
+    files = sorted(REPO.glob(pattern))
+    if not files:
+        sys.exit(f"no finalization.json under runs/{args.benchmark}/")
+
+    stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    touched = skipped = failed = 0
+    for path in files:
+        cell = path.relative_to(REPO).parts[2]
+        payload = json.loads(path.read_text())
+        existing = payload.get("baseline_rewards") or {}
+        if existing.get(args.reward_key) is not None:
+            print(f"  skip  {cell:34} already has {existing}")
+            skipped += 1
+            continue
+        if not payload.get("shipped"):
+            print(f"  skip  {cell:34} shipped=false, not a reportable cell")
+            skipped += 1
+            continue
+
+        payload["baseline_rewards"] = dict(existing) | {args.reward_key: args.value}
+        ok, why = validates(payload)
+        if not ok:
+            print(f"  FAIL  {cell:34} schema rejected: {why}")
+            failed += 1
+            continue
+
+        prov = path.parent / "baseline_reward_provenance.json"
+        prov_doc = {
+            "backfilled_at": stamp,
+            "reward_key": args.reward_key,
+            "baseline_reward": args.value,
+            "measured_by": args.provenance,
+            "note": ("Added after the run. The build config carried no "
+                     "baseline_reward, so the verifier wrote an empty "
+                     "baseline_rewards. This value comes from the same script and "
+                     "seed that every other benchmark's pinned baseline comes from; "
+                     "the candidate's own rewards are untouched."),
+        }
+        if args.dry_run:
+            print(f"  would {cell:34} set {args.reward_key}={args.value}")
+            touched += 1
+            continue
+
+        path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+        prov.write_text(json.dumps(prov_doc, indent=2) + "\n", encoding="utf-8")
+        print(f"  set   {cell:34} {args.reward_key}={args.value}")
+        touched += 1
+
+        if args.push:
+            rel = path.relative_to(REPO / "runs")
+            for local, key in ((path, rel), (prov, prov.relative_to(REPO / "runs"))):
+                r = subprocess.run(
+                    ["aws", "--profile", PROFILE, "s3", "cp", str(local),
+                     f"{S3}/{key}", "--only-show-errors"],
+                    capture_output=True, text=True)
+                if r.returncode != 0:
+                    print(f"        UPLOAD FAILED {key}: {r.stderr.strip()[:120]}")
+
+    print(f"\n{touched} patched, {skipped} skipped, {failed} failed")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

harness-engineering-bench/scripts/launch_cell.sh

@@ -18,6 +18,13 @@
 #                 via --param optimizer_model. Usually the launch-model minus the
 #                 harness-specific prefix: claude-opus-5, fireworks_ai/kimi-k3.
 #   wandb-run     <benchmark>__<model>__<harness>__r<n>
+#   [extra...]    optional trailing KEY=VALUE pairs, each forwarded as --param.
+#                 Needed by configs whose producer allow-list has more than one
+#                 slot: swe-atlas-qna and tau3's gpt-5.4-mini variant take
+#                 optimizer_model_bare and optimizer_aux_model as well, and an
+#                 anthropic-routed optimizer must pass them explicitly because the
+#                 defaults resolve to the OpenAI family. The gateway compares exact
+#                 strings, so a missing slot is a 403 model_denied on that model.
 #
 # Why the daemonizer: macOS has no setsid(1), and the harness SIGTERMs background
 # tasks belonging to an idle session's process group. A plain `nohup ... &` loses
@@ -28,13 +35,24 @@
 # worked, and that has already cost us a full grid once.
 set -euo pipefail
 
-if [ "$#" -ne 8 ]; then
-  sed -n '2,28p' "$0" >&2
+if [ "$#" -lt 8 ]; then
+  sed -n '2,36p' "$0" >&2
   exit 2
 fi
 
 outdir=$1 config=$2 envfile=$3 environment=$4
 agent=$5 launch_model=$6 wire_model=$7 wandb_run=$8
+shift 8
+# Each remaining KEY=VALUE becomes its own --param. Validated here rather than
+# passed through blind, so a typo fails at launch instead of at the gateway.
+extra_params=""
+for pair in "$@"; do
+  case "$pair" in
+    *=*) extra_params="$extra_params \\
+  --param \"$pair\"" ;;
+    *) echo "extra args must be KEY=VALUE, got: $pair" >&2; exit 2 ;;
+  esac
+done
 
 here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 repo=$(cd "$here/../.." && pwd)
@@ -64,7 +82,7 @@ exec uv run vero harbor run \\
   --agent "$agent" \\
   --model "$launch_model" \\
   --param "optimizer_model=$wire_model" \\
-  --param "wandb_run=$wandb_run" \\
+  --param "wandb_run=$wandb_run"$extra_params \\
   --yes \\
   -o "$rundir/jobs"
 EOF

harness-engineering-bench/scripts/rescore_candidate.py

@@ -48,12 +48,21 @@ def log(message: str) -> None:
     print(f"[rescore] {message}", flush=True)
 
 
-def load_build(benchmark: str) -> tuple[dict, Path]:
+def load_build(benchmark: str, config: str = "build.yaml") -> tuple[dict, Path]:
+    """Load a benchmark's build config.
+
+    `config` names the file inside `<benchmark>/baseline/`, defaulting to the
+    canonical `build.yaml`. A benchmark retargeted at a second model keeps that
+    canonical file untouched and adds a variant beside it (see
+    swe-atlas-qna/baseline/build.gpt54mini.yaml), and the variant needs scoring by
+    the same path as the pinned baselines or its number is not comparable to
+    anything.
+    """
     import yaml  # provided by the vero environment
 
-    path = BENCH_ROOT / benchmark / "baseline" / "build.yaml"
+    path = BENCH_ROOT / benchmark / "baseline" / config
     if not path.is_file():
-        sys.exit(f"no build.yaml for benchmark {benchmark!r} at {path}")
+        sys.exit(f"no {config} for benchmark {benchmark!r} at {path}")
     return yaml.safe_load(path.read_text()), path
 
 
@@ -208,6 +217,11 @@ def main() -> int:
         ),
     )
     parser.add_argument("--benchmark", required=True)
+    parser.add_argument(
+        "--config", default="build.yaml",
+        help=("build config inside <benchmark>/baseline/ (default build.yaml). "
+              "Use this to score a retargeted variant, e.g. build.gpt54mini.yaml."),
+    )
     parser.add_argument("--version", help="candidate sha (default: the shipped one)")
     parser.add_argument("--partition", default="test")
     parser.add_argument(
@@ -227,7 +241,7 @@ def main() -> int:
     parser.add_argument("--dry-run", action="store_true")
     args = parser.parse_args()
 
-    build, build_path = load_build(args.benchmark)
+    build, build_path = load_build(args.benchmark, args.config)
     outdir = Path(args.output).resolve() if args.output else Path(
         tempfile.mkdtemp(prefix=f"rescore-{args.benchmark}-"))
     outdir.mkdir(parents=True, exist_ok=True)