Reference-harness grid: scoring convention, installed-harness support, producer fixes

harness-engineering-bench/gaia/baseline/build.shell.yaml

@@ -160,6 +160,16 @@ inference_gateway:
   request_log_attribution: true
   producer:
     allowed_models: ["${optimizer_model:-openai/gpt-5.4}"]
+    # Pin the gpt-5.6 optimizers to one upstream deployment. An unqualified model group
+    # load-balances across deployments, and Responses-API encrypted reasoning is
+    # decryptable only by the deployment that produced it, so every turn after the
+    # first fails `invalid_encrypted_content`. Measured here 2026-08-02 10:12Z: the
+    # gpt-5.6-sol x codex canary died on it. Same fault and same fix as
+    # swe-atlas-qna/baseline/build.yaml. Applied AFTER the allow-list check, so
+    # allowed_models still governs what the optimizer may ask for.
+    model_aliases:
+      gpt-5.6-sol: azure_ai/gpt-5.6-sol
+      gpt-5.6-terra: azure_ai/gpt-5.6-terra
     max_concurrency: 8
   # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is
   # the spend control, so a token cap only needs to stop a runaway.

harness-engineering-bench/officeqa/baseline/build.yaml

@@ -154,6 +154,19 @@ inference_gateway:
   request_log_attribution: true
   producer:
     allowed_models: ["${optimizer_model:-openai/gpt-5.4}"]
+    # Pin every gpt-5.x optimizer to one upstream deployment. An unqualified model group
+    # load-balances across deployments, and Responses-API encrypted reasoning is
+    # decryptable only by the deployment that produced it, so every turn after the first
+    # fails invalid_encrypted_content. Measured here 2026-08-03 09:55Z: the generational
+    # sweep's gpt-5.5 x codex cell died on exactly that ("Received Model Group=gpt-5.5,
+    # Available Model Group Fallbacks=None"). Same fault and same fix as
+    # swe-atlas-qna/baseline/build.yaml and gaia/baseline/build.shell.yaml. Applied
+    # AFTER the allow-list check, so allowed_models still governs what may be asked for.
+    model_aliases:
+      gpt-5.1: azure_ai/gpt-5.1
+      gpt-5.2: azure_ai/gpt-5.2
+      gpt-5.4: azure_ai/gpt-5.4
+      gpt-5.5: azure_ai/gpt-5.5
     max_concurrency: 8
   # Token caps are a runaway backstop, not the spend control: the work is already
   # bounded by the agent case budget above and by the fixed held-out set, so a cap

harness-engineering-bench/scripts/rescore_candidate.py

@@ -139,6 +139,10 @@ def harbor_command(
     attempts: int,
     concurrency: int,
     model: str,
+    agent: str | None = None,
+    setup_timeout_multiplier: float | None = None,
+    agent_env: list[str] | None = None,
+    extra_requirements: list[str] | None = None,
 ) -> list[str]:
     """Mirror vero/src/vero/harbor/backend.py::_command and the baseline runs.
 
@@ -159,23 +163,83 @@ def harbor_command(
         "--no-config", "--no-env-file",
         "--project", str(workspace),
         "--with", build.get("harbor_requirement", "harbor[modal]==0.20.0"),
+        # Harbor-native harnesses import their framework in the ORCHESTRATOR, not the
+        # task container, and harbor does not declare those imports: dspy-rlm dies on
+        # ModuleNotFoundError: No module named 'dspy' before the agent ever starts.
+        *[arg for req in (extra_requirements or []) for arg in ("--with", req)],
         "harbor", "run", "--yes",
         *source_args,
-        "--agent-import-path", build["agent_import_path"],
+        # A reference run swaps the seed program for an installed harness
+        # (claude-code, opencode, ...) and changes NOTHING else -- same dataset
+        # ref, same partition, same rounds, same aggregation -- so the number
+        # stays comparable to baseline_reward and to every contestant. The target
+        # model stays pinned, so the harness is the only variable.
+        *(["--agent", agent] if agent
+          else ["--agent-import-path", build["agent_import_path"]]),
         "-e", resolve_param(build.get("environment_name", "modal")),
         "-m", str(model),
         "-n", str(concurrency),
         "--n-attempts", str(attempts),
         "--jobs-dir", str(jobs_dir),
     ]
+    # opencode's install (nvm -> node 22 -> npm -g) is far heavier than the seed's
+    # zero setup and can exceed harbor's default. swe-atlas already gives its
+    # OPTIMIZER agent a 4x multiplier for the same reason.
+    if setup_timeout_multiplier:
+        command += ["--agent-setup-timeout-multiplier", str(setup_timeout_multiplier)]
+    for pair in agent_env or []:
+        command += ["--ae", pair]
     for task in tasks:
         command.extend(["-i", task])
     command.extend(str(a) for a in build.get("extra_harbor_args", []))
     return command
 
 
+# Convention (2026-07-31, adopted on Greptile's PR #75 catch) -- IDENTICAL to
+# runs/recompute.py, which produced the pinned baselines. A trial the HARNESS killed
+# scores 0: the harness owns its own install, its context management and its step
+# budget, so a reference harness that cannot install or cannot finish must pay for it
+# exactly as a candidate does at finalization, which zero-fills dead attempts. A trial
+# the PLATFORM killed is dropped: a retry cannot score a trial that never ran, and
+# zero-filling infra bakes outage luck into the number.
+#
+# Dropping them instead -- which this script did until 2026-08-02 -- silently inflates
+# every reference score by scoring only the trials that survived. It cost 73 zeroes
+# across the reference grid, worst on swe-atlas x mini-swe-agent (n=122 of 150).
+HARNESS_EXCEPTIONS = {
+    "RuntimeError",                # swe-atlas seed: empty-completion fail-fast
+    "UnicodeDecodeError",          # terminal-bench seed: undecodable command output
+    "AgentTimeoutError",           # wall-clock exhaustion: the step budget is harness-owned
+    "BadRequestError",             # gpt-oss 128k overflow: context management is harness-owned
+    "NonZeroAgentExitCodeError",   # the harness crashed or never installed its binary
+    "AgentSetupTimeoutError",      # the harness owns how long its own install takes
+    "AgentAuthenticationError",    # subclasses NonZeroAgentExitCodeError: the CLI reports
+                                   # no login, i.e. the harness did not read a credential
+                                   # surface we set (goose reads OPENAI_HOST, not _BASE_URL)
+    "AdapterParseError",           # dspy's own output parser gave up: harness-owned
+}
+INFRA_EXCEPTIONS = {
+    "RateLimitError",
+    "ApiRateLimitError",
+    "NetworkConnectionError",
+    "VerifierTimeoutError",
+    "EnvironmentStartTimeoutError",
+    "SandboxFilesystemNotFoundError",
+    "AddTestsDirError",            # harbor could not stage the tests: platform, not agent
+    "ConnectionError",
+    "UnknownApiError",             # harbor's ApiError subclass for a provider error it
+                                   # could not classify: upstream, like the two RateLimits
+    "RewardFileNotFoundError",     # the verifier ran and produced no reward file
+    "CancelledError",              # harbor cancelled the trial (job.py CANCELLED_ERROR_TYPE)
+}
+# ValueError is deliberately in NEITHER set. Both instances on disk are harbor's
+# "ContextVar ... was created in a different Context" orchestration bug, which is
+# platform-side, but the name is generic enough that an agent-side ValueError would
+# land here too. A rescore that meets one should stop and be looked at, not guess.
+
+
 def trial_rewards(round_dir: Path) -> list[float]:
-    """Same extraction as runs/recompute.py, so numbers are comparable."""
+    """Same extraction AND the same failure convention as runs/recompute.py."""
     rewards = []
     for path in glob.glob(f"{round_dir}/**/result.json", recursive=True):
         if "/verifier/" in path:
@@ -189,8 +253,18 @@ def trial_rewards(round_dir: Path) -> list[float]:
         verifier = data.get("verifier_result") or {}
         block = verifier.get("rewards")
         reward = block.get("reward") if isinstance(block, dict) else verifier.get("reward")
+        exception = (data.get("exception_info") or {}).get("exception_type")
         if reward is not None:
             rewards.append(float(reward))
+        elif exception in HARNESS_EXCEPTIONS:
+            rewards.append(0.0)  # the harness killed it: price the defect
+        elif exception in INFRA_EXCEPTIONS:
+            continue             # the platform killed it: drop, do not bake in outage luck
+        elif exception:
+            message = (data.get("exception_info") or {}).get("exception_message") or ""
+            sys.exit(f"unclassified exception {exception!r} in {round_dir}: add it to "
+                     "HARNESS_EXCEPTIONS or INFRA_EXCEPTIONS before quoting a number"
+                     f"\n  {message.splitlines()[0][:200] if message else '(no message)'}")
     return rewards
 
 
@@ -222,12 +296,36 @@ def main() -> int:
                         help="independent rounds, pooled (default 3, as the baselines)")
     parser.add_argument("--attempts", type=int, default=1,
                         help="attempts per case within a round (default 1)")
+    parser.add_argument("--agent",
+                        help="run an INSTALLED harbor harness (claude-code, opencode, "
+                             "...) instead of the seed program, at the benchmark's "
+                             "pinned model. Measures a SOTA reference, not a bound.")
+    parser.add_argument("--harbor-requirement",
+                        help="override build.yaml's harbor_requirement for this run "
+                             "only. Also relaxes the copied workspace's own harbor pin, "
+                             "which build.yaml and target/pyproject.toml must otherwise "
+                             "keep in lockstep (see a70c572) or uv cannot resolve.")
+    parser.add_argument("--setup-timeout-multiplier", type=float,
+                        help="scale harbor's agent-setup timeout (installed harnesses "
+                             "with heavy toolchains need this)")
+    parser.add_argument("--agent-env", action="append", metavar="KEY=VALUE",
+                        default=[],
+                        help="extra env var for the agent (repeatable). goose reads "
+                             "OPENAI_HOST/OPENAI_BASE_PATH rather than OPENAI_BASE_URL, "
+                             "so it needs the proxy pointed at explicitly.")
+    parser.add_argument("--with-requirement", action="append", metavar="SPEC",
+                        default=[], dest="extra_requirements",
+                        help="extra package for the orchestrator uv env (repeatable). "
+                             "Harbor-native harnesses import their framework here and "
+                             "harbor does not declare it -- dspy-rlm needs 'dspy'.")
     parser.add_argument("--concurrency", type=int, default=24)
     parser.add_argument("--output", help="output dir (default: a temp dir)")
     parser.add_argument("--dry-run", action="store_true")
     args = parser.parse_args()
 
     build, build_path = load_build(args.benchmark)
+    if args.harbor_requirement:
+        build["harbor_requirement"] = args.harbor_requirement
     outdir = Path(args.output).resolve() if args.output else Path(
         tempfile.mkdtemp(prefix=f"rescore-{args.benchmark}-"))
     outdir.mkdir(parents=True, exist_ok=True)
@@ -245,6 +343,15 @@ def main() -> int:
             "__pycache__", "*.pyc", ".venv", ".git"))
         version = "seed"
         log(f"seed harness from {origin}")
+        if args.harbor_requirement:
+            pyproject = workspace / "pyproject.toml"
+            if pyproject.is_file():
+                import re as _re
+                text = pyproject.read_text()
+                relaxed = _re.sub(r'"harbor==[^"]+"', '"harbor"', text)
+                if relaxed != text:
+                    pyproject.write_text(relaxed)
+                    log("relaxed the copied workspace's harbor pin so the override resolves")
     else:
         session_dir = open_session(args.session, outdir)
         version = shipped_version(
@@ -284,7 +391,10 @@ def main() -> int:
         command = harbor_command(
             build=build, build_path=build_path, workspace=workspace, tasks=tasks,
             jobs_dir=jobs_dir, attempts=args.attempts, concurrency=args.concurrency,
-            model=args.model or build["model"],
+            model=args.model or build["model"], agent=args.agent,
+            setup_timeout_multiplier=args.setup_timeout_multiplier,
+            agent_env=args.agent_env,
+            extra_requirements=args.extra_requirements,
         )
         if args.dry_run:
             print(" ".join(command))

harness-engineering-bench/swe-atlas-qna/baseline/build.yaml

@@ -165,6 +165,22 @@ inference_gateway:
       - "${optimizer_model:-openai/gpt-5.4}"
       - "${optimizer_model_bare:-gpt-5.4}"
       - "${optimizer_aux_model:-gpt-5.4-nano}"
+    # Pin the gpt-5.6 optimizers to one upstream deployment. An unqualified model
+    # group is load-balanced across deployments, and Responses-API encrypted
+    # reasoning content is decryptable only by the deployment that produced it, so
+    # replaying it against a sibling fails `invalid_encrypted_content` on every turn
+    # after the first. Measured 2026-08-01 on the bare group: 8 of 8 replays failed,
+    # against 0 of 5 on azure_ai. It killed both gpt-5.6-sol cells here at 2m58s --
+    # opencode AND codex, so this is not a codex-only fault as terminal-bench's
+    # instance suggested.
+    #
+    # Applied AFTER the allow-list check, so allowed_models still governs what the
+    # optimizer may ask for and a cell's label still means what it says. Keys outside
+    # the allow-list are permitted and inert (specs.py validate_aliases), which is why
+    # both models can sit here in a config shared by every cell of the grid.
+    model_aliases:
+      gpt-5.6-sol: azure_ai/gpt-5.6-sol
+      gpt-5.6-terra: azure_ai/gpt-5.6-terra
     max_concurrency: 8
   # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is
   # the spend control, so a token cap only needs to stop a runaway.

vero/src/vero/harbor/cli.py

@@ -528,6 +528,20 @@ def _compiled_run_environment(
     environment["ANTHROPIC_BASE_URL"] = producer_base_url[: -len("/v1")] if (
         producer_base_url.endswith("/v1")
     ) else producer_base_url
+    # openhands-sdk reads its own LLM_* surface and ignores OPENAI_*/ANTHROPIC_*
+    # entirely (harbor/agents/installed/openhands_sdk.py). The producer token is
+    # minted per run, so it cannot be supplied through build.yaml agent_env --
+    # it has to be injected here alongside the other two surfaces. Same
+    # unconditional rule: a harness that does not read these ignores them.
+    environment["LLM_API_KEY"] = producer_api_key
+    environment["LLM_BASE_URL"] = producer_base_url
+    # goose reads OPENAI_HOST + OPENAI_BASE_PATH and never OPENAI_BASE_URL, and it
+    # JOINS them, so the host must not already carry /v1 or the request goes to
+    # .../v1/v1/chat/completions and the proxy 403s.
+    environment["OPENAI_HOST"] = producer_base_url[: -len("/v1")] if (
+        producer_base_url.endswith("/v1")
+    ) else producer_base_url
+    environment["OPENAI_BASE_PATH"] = "v1/chat/completions"
     return environment