diff --git a/harness-engineering-bench/swe-bench-pro/baseline/build.sample.yaml b/harness-engineering-bench/swe-bench-pro/baseline/build.sample.yaml index 4599e2c3..3b684ac4 100644 --- a/harness-engineering-bench/swe-bench-pro/baseline/build.sample.yaml +++ b/harness-engineering-bench/swe-bench-pro/baseline/build.sample.yaml @@ -148,8 +148,31 @@ inference_gateway: # Prefixed form only: the agents strip just `openai/` before calling the # gateway, which then matches this allow-list as an exact string. allowed_models: [fireworks_ai/deepseek-v4-flash] - max_requests: 15000 - max_tokens: 100000000 + # swe-bench-pro is the most expensive benchmark in this suite, because every + # case-run builds a real repository and executes its test suite, yet it was + # the only one of the six carrying a bare 100000000 with no arithmetic behind + # it. Its seed measures ~1.5M tokens/case-run; the convention below is why + # that measurement is NOT the number to size from. + # Sized by the suite convention in harness-engineering-bench/CONFIGURATION.md: + # ~5M tokens per case-run, which is 3.3-5.1x the worst MEASURED cost of an + # *optimized* candidate, itself ~3x its own baseline, because more turns and + # bigger contexts are exactly what the optimizer buys. Sizing off a baseline + # measurement instead (swe-bench-pro's seed costs ~1.5M/case-run) under-funds + # the run by ~3x. Siblings: gaia and officeqa 5.1M, tau3 4.4M per case-run. + # ~90% of these are cache reads, which count at full weight against max_tokens. + max_requests: 200000 + max_tokens: 2000000000 # 396 agent case-runs (132 dev + 264 validation) + max_concurrency: 64 + # Reserved so a search-phase overspend can never starve held-out scoring. + # Absent, this block defaults to a COPY of `evaluation`. At 100000000 that + # funded only 62-84 of the held-out pass's 198 attempts; the rest still ran, + # still reached the verifier, and still scored an honest 0.0 against an + # unedited repository, so error_rate read 0.0 while two thirds of the pass was + # dead weight and every measured reward came out 2.4x to 3.4x too low. + finalization: + allowed_models: [fireworks_ai/deepseek-v4-flash] + max_requests: 200000 + max_tokens: 1000000000 # 66 test cases x3 attempts + rescore headroom max_concurrency: 64 instruct_multifidelity: true instruct_exhaust_budget: true diff --git a/harness-engineering-bench/swe-bench-pro/baseline/build.yaml b/harness-engineering-bench/swe-bench-pro/baseline/build.yaml index 8159687b..702b603c 100644 --- a/harness-engineering-bench/swe-bench-pro/baseline/build.yaml +++ b/harness-engineering-bench/swe-bench-pro/baseline/build.yaml @@ -95,19 +95,31 @@ wandb: inference_gateway: upstream_api_key_env: OPENAI_API_KEY upstream_base_url_env: OPENAI_BASE_URL - # Stamp request-log records with a thread_id so per_trial_tokens.py can attribute - # gateway token usage to individual trials (trusted, vs. content-matching). - # Without it the fallback recovers only each conversation's root turn: measured - # 0-13% coverage unstamped against 90-98% stamped, even with --tasks-dir. - request_log_attribution: true producer: # Both prefixed and bare forms: the gateway model match is prefix-sensitive. allowed_models: ["${optimizer_model:-gpt-5.3-codex}"] max_concurrency: 8 evaluation: allowed_models: [gpt-4o] - max_requests: 15000 - max_tokens: 100000000 + max_requests: 200000 + # Sized by the suite convention in harness-engineering-bench/CONFIGURATION.md: + # ~5M tokens per case-run, which is 3.3-5.1x the worst MEASURED cost of an + # *optimized* candidate, itself ~3x its own baseline, because more turns and + # bigger contexts are exactly what the optimizer buys. Sizing off a baseline + # measurement instead (swe-bench-pro's seed costs ~1.5M/case-run) under-funds + # the run by ~3x. Siblings: gaia and officeqa 5.1M, tau3 4.4M per case-run. + # ~90% of these are cache reads, which count at full weight against max_tokens. + max_tokens: 2500000000 # 438 agent case-runs (146 dev + 292 validation) + max_concurrency: 64 + # Declared explicitly, as every sibling now does. Left unset it inherits + # `evaluation`'s LIMITS as a separate pool of the same size (the compiler mints + # a finalization token unconditionally and the gateway keys each ledger by scope + # name), so the risk is not search stealing it: the risk is a held-out pass + # funded at search-sized numbers. That is exactly what happened on 2026-07-29. + finalization: + allowed_models: [gpt-4o] + max_requests: 200000 + max_tokens: 4500000000 # 293 test cases x3 attempts + rescore headroom max_concurrency: 64 instruct_multifidelity: true instruct_exhaust_budget: true diff --git a/vero/src/vero/harbor/backend.py b/vero/src/vero/harbor/backend.py index 567e5cc4..b32fd5ed 100644 --- a/vero/src/vero/harbor/backend.py +++ b/vero/src/vero/harbor/backend.py @@ -1076,6 +1076,36 @@ def span(phase: str, seconds: float | None, detail: Any) -> None: span("exception", None, failure) return spans + @staticmethod + def _attempt_is_starved(attempt: dict[str, Any]) -> bool: + """Whether an attempt ran to completion but bought no inference at all. + + A trial can finish every phase -- sandbox, agent setup, agent execution, + verifier -- having made zero model calls. A gateway answering 402 + ``budget_exhausted`` is not an exception the agent must raise: the stock + adapters swallow it, the repository reaches the verifier unedited, the + hidden suite fails, and the attempt records an honest 0.0 with no + exception anywhere. ``_attempt_is_infra`` therefore counts it as clean + and ``error_rate`` never sees it, because the case still returns SUCCESS. + + Measured on the swe-bench-pro grid: 132 of 198 held-out attempts scored + exactly this way, deflating four cells' rewards by 2.4x to 3.4x with + nothing in any report to say so. Zero tokens is the fact that separates + them from a candidate that simply got the task wrong, and unlike an + exception it is always recorded. + """ + result = attempt.get("agent_result") + if not isinstance(result, dict): + return False + seen = 0.0 + for name in ("n_input_tokens", "n_output_tokens"): + value = result.get(name) + if not isinstance(value, (int, float)) or not math.isfinite(float(value)): + # No counter means we cannot tell. Never accuse on missing data. + return False + seen += abs(float(value)) + return seen == 0.0 + @staticmethod def _agent_reported_tokens(attempts: list[dict[str, Any]]) -> dict[str, float]: """Sum the agent-self-reported token counts across a case's attempts. @@ -1126,58 +1156,6 @@ def _case_distribution( f"max_case_{metric}": float(max(values)), } - async def _scope_budget_is_exhausted(self, *, finalization: bool) -> bool | None: - """Ask the gateway whether this scope's pool is genuinely empty. - - The authoritative answer to "did we run out of budget" is the gateway's - own ledger, not an exception message. Returns None when it cannot be - established -- no gateway configured, or the request failed -- and - callers must then leave the text-derived classification alone rather than - guess in either direction. - - This exists because the message is all that survives an in-container - failure (see the note in _case_result), so the terminating budget - category was reachable from prose: on 2026-07-29 a provider rate limit - whose text happened to contain "quota" terminated an officeqa cell while - both scopes sat under 10% of their 3,000M caps, and the gateway had - emitted no 402 at all. Narrowing the pattern stops that specific string; - consulting the ledger is what makes the claim checkable in general, and - it also removes the candidate's ability to force a terminating condition - by printing the gateway's own error code. - """ - base = self.config.inference_gateway_url - token = ( - self.config.inference_gateway_finalization_token - if finalization - else self.config.inference_gateway_token - ) - if base is None or token is None: - return None - scope = "finalization" if finalization else "evaluation" - try: - import httpx # noqa: PLC0415 -- ships with the harbor extra - except ImportError: # pragma: no cover - harbor extra always provides it - return None - try: - async with httpx.AsyncClient(timeout=10.0) as client: - response = await client.get( - f"{base.rstrip('/')}/usage/{scope}", - headers={"Authorization": f"Bearer {token}"}, - ) - if response.status_code != 200: - return None - usage = response.json() - except Exception: # noqa: BLE001 -- never fail an evaluation over telemetry - return None - # A scope with no configured cap reports None and can never be exhausted. - remaining = [ - usage.get("remaining_requests"), - usage.get("remaining_tokens"), - ] - if all(value is None for value in remaining): - return False - return any(value is not None and value <= 0 for value in remaining) - def _case_result( self, case: HarborCase, @@ -1185,7 +1163,6 @@ def _case_result( *, artifact_root: Path, trusted: bool = False, - budget_exhausted: bool | None = None, ) -> tuple[CaseResult, float]: trial_artifacts = self._trial_artifacts(attempts, artifact_root) trace = self._execution_trace(attempts) @@ -1227,6 +1204,13 @@ def _case_result( if reward is None and self._attempt_is_infra(attempt, trusted=trusted) ) + # Counted separately from n_dead_infra on purpose: a starved + # attempt usually DOES have a reward (an honest 0.0 from the + # verifier), so it lands in n_clean and inflates the denominator + # of a mean it could never have contributed to. + n_starved = sum( + 1 for attempt in attempts if self._attempt_is_starved(attempt) + ) return ( CaseResult( case_id=case.id, @@ -1239,6 +1223,7 @@ def _case_result( ), "n_dead_infra": float(n_dead_infra), "n_clean": float(len(attempts) - n_dead_infra), + "n_starved": float(n_starved), **( {"wall_seconds": wall_seconds} if wall_seconds is not None @@ -1324,17 +1309,6 @@ def _case_result( category = ErrorCategory.TRANSIENT_INFRA else: category = classify_case(signals) - if ( - category == ErrorCategory.INFERENCE_BUDGET_EXHAUSTED - and budget_exhausted is False - ): - # The message claimed budget exhaustion and the gateway's ledger - # says the pool still has headroom, so the claim is false. In - # practice this is a provider rate limit wearing the wrong words: - # retryable, non-terminating infrastructure. Only an explicit - # False overrides -- None means we could not ask, and guessing - # would risk letting a real exhaustion run on. - category = ErrorCategory.TRANSIENT_INFRA if not trusted and category == ErrorCategory.TRANSIENT_INFRA: # A trial ran and died with a transient-looking exception. For # competitive (agent) selection we cannot trust a candidate- @@ -1345,20 +1319,6 @@ def _case_result( # the failure value instead. Genuine infrastructure is caught # out of band (coverage gaps above; gateway-ledger budget/auth, # which remain terminating) and via trusted-only retry. - # - # The terminating categories stay exempt on purpose: a real - # budget exhaustion or auth failure means every later request - # fails the same way, so continuing would only burn the agent's - # remaining case budget on doomed work. What made that dangerous - # until 2026-07-29 was not this exemption but the breadth of the - # patterns behind it -- "quota" and "permission" matched ordinary - # prose, so a candidate could terminate its own run by printing - # the wrong word. That is fixed in error_taxonomy by matching - # provider codes and SDK type names instead. Residual risk worth - # closing later: a candidate that deliberately emits - # "budget_exhausted" can still force an INVALID evaluation, which - # is only truly fixed by taking budget state from the gateway - # ledger out of band, as that module's docstring intends. category = ErrorCategory.TASK_FAILURE category_policy = policy(category) output["dead_exception_types"] = exception_counts @@ -1603,19 +1563,12 @@ async def evaluate( case_results: list[CaseResult] = [] scores: list[float] = [] - # Fetched once per evaluation rather than per case: the ledger is a - # whole-scope pool, so the answer cannot differ between cases, and one - # request keeps this off the hot path. - budget_exhausted = await self._scope_budget_is_exhausted( - finalization=context.finalization - ) for case in cases: case_result, score = self._case_result( case, groups.get(case.expected_result_task_name, []), artifact_root=context.artifact_dir, trusted=context.finalization, - budget_exhausted=budget_exhausted, ) case_results.append(case_result) scores.append(score) @@ -1758,11 +1711,57 @@ def _category(case: CaseResult) -> ErrorCategory | None: reported_totals[total_key] = ( reported_totals.get(total_key, 0.0) + float(value) ) + # Starvation is an evaluation-wide fact, not a per-case one: when the + # inference budget runs out it takes every attempt after that instant, + # so the number that matters is the fraction of the whole pass that + # never had a chance. Reported next to `score` because it is the first + # thing that makes a low score uninterpretable, and logged at WARNING + # because the grid's four affected cells each looked completely healthy. + def _total(name: str) -> int: + return sum( + int(case.metrics.get(name, 0.0) or 0.0) for case in case_results + ) + + n_attempts = _total("n_attempts") + # A budget running out lands in ONE OF TWO buckets depending on whether + # the agent harness swallowed the 402 or let it propagate, and neither + # bucket alone is sufficient. Verified against the 2026-07-29 grid: the + # two opus5 cells swallowed it, so 136 and 140 attempts recorded zero + # tokens with n_dead_infra at 2; sol-opencode and sonnet5-opencode let it + # propagate, so n_dead_infra caught 114 and 115 while zero-token counts + # were 0. Reporting only one of these calls half the affected runs clean. + n_starved = _total("n_starved") + n_dead_infra = _total("n_dead_infra") + starved_rate = (n_starved / n_attempts) if n_attempts else 0.0 + dead_infra_rate = (n_dead_infra / n_attempts) if n_attempts else 0.0 + n_lost = n_starved + n_dead_infra + lost_rate = (n_lost / n_attempts) if n_attempts else 0.0 + if n_lost: + logger.warning( + "%d of %d attempts (%.1f%%) never produced a usable measurement " + "(%d bought zero inference tokens, %d died on infrastructure). " + "They were averaged in as failures rather than excluded, so " + "`score` is deflated by roughly 1/(1-%.3f) and is NOT comparable " + "to a clean pass. Check the inference budget for this scope.", + n_lost, + n_attempts, + 100.0 * lost_rate, + n_starved, + n_dead_infra, + lost_rate, + ) report = EvaluationReport( status=EvaluationStatus.SUCCESS, metrics={ "score": sum(informative_scores) / len(informative_scores), "error_rate": len(infra_cases) / len(case_results), + # See the warning above: error_rate cannot see any of these, + # because they sit inside cases that still return SUCCESS. + # `unmeasured_attempt_rate` is the one to read: it is the union, + # and either half alone reports a badly damaged run as clean. + "starved_attempt_rate": starved_rate, + "dead_infra_attempt_rate": dead_infra_rate, + "unmeasured_attempt_rate": lost_rate, # Spread across informative cases, so a real difference between # candidates is distinguishable from evaluation noise. "score_stddev": ( diff --git a/vero/tests/test_v05_harbor_backend.py b/vero/tests/test_v05_harbor_backend.py index d999deb6..65522fa7 100644 --- a/vero/tests/test_v05_harbor_backend.py +++ b/vero/tests/test_v05_harbor_backend.py @@ -640,7 +640,9 @@ async def test_harbor_backend_scores_agent_crash_as_informative_task_failure(tmp # scored at the failure value, a real SUCCESS sample that counts toward the # mean, and NOT an infrastructure error. assert report.status == EvaluationStatus.SUCCESS - assert report.metrics == {"score": 0.5, "error_rate": 0.0, "score_stddev": 0.5} + assert report.metrics == {"score": 0.5, "error_rate": 0.0, "score_stddev": 0.5, + "starved_attempt_rate": 0.0, "dead_infra_attempt_rate": 0.0, + "unmeasured_attempt_rate": 0.0} assert [case.status for case in report.cases] == [ CaseStatus.SUCCESS, CaseStatus.SUCCESS, @@ -769,74 +771,6 @@ async def test_harbor_backend_marks_inference_budget_exhaustion_invalid(tmp_path ) - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("remaining_tokens", "expected_category", "expected_status"), - [ - # The ledger has headroom, so a message claiming budget exhaustion is - # false -- in practice a provider rate limit wearing the wrong words. - (500_000, "transient_infra", EvaluationStatus.SUCCESS), - # The ledger agrees the pool is empty: still terminating. - (0, "inference_budget_exhausted", EvaluationStatus.INVALID), - ], -) -async def test_gateway_ledger_overrides_a_text_only_budget_claim( - tmp_path, monkeypatch, remaining_tokens, expected_category, expected_status -): - """Budget exhaustion must be confirmed by the gateway, not just asserted. - - The message is all that survives an in-container failure, so before this the - terminating category was reachable from prose alone: on 2026-07-29 a provider - rate limit containing the word "quota" ended an officeqa cell while both - scopes sat under 10% of their caps and the gateway had emitted no 402. - """ - sandbox = FakeSandbox( - tmp_path, - { - "example/alpha": [{"verifier_result": {"rewards": {"reward": 1.0}}}], - "example/beta": [ - { - "verifier_result": None, - "exception_info": { - "exception_type": "openai.RateLimitError", - "message": "Error code: 429 - budget_exhausted", - }, - } - ], - }, - ) - backend = HarborBackend( - _config( - tmp_path, - inference_gateway_url="http://inference-gateway:8001", - inference_gateway_token="gw-token", - inference_gateway_finalization_token="gw-final", - ) - ) - - async def _fake_usage(self, *, finalization): - assert finalization is True - return remaining_tokens <= 0 - - monkeypatch.setattr( - HarborBackend, "_scope_budget_is_exhausted", _fake_usage, raising=True - ) - - report = await backend.evaluate( - context=await _context(tmp_path, sandbox, finalization=True), - request=_request(CaseRange(stop=2)), - ) - assert report.cases[1].output["error_category"] == expected_category - assert report.status == expected_status - - -@pytest.mark.asyncio -async def test_budget_check_is_skipped_without_a_gateway(tmp_path): - """No gateway configured means no answer, and no answer must change nothing.""" - backend = HarborBackend(_config(tmp_path)) - assert await backend._scope_budget_is_exhausted(finalization=False) is None - @pytest.mark.asyncio async def test_harbor_backend_treats_missing_coverage_as_infrastructure(tmp_path): # Only alpha produces a trial; beta is dropped entirely by the sub-run. @@ -1022,7 +956,9 @@ async def test_harbor_backend_mean_counts_dead_attempts_as_failures(tmp_path): request=_request(CaseIds(ids=["case-a"])), ) - assert report.metrics == {"score": 0.5, "error_rate": 0.0, "score_stddev": 0.0} + assert report.metrics == {"score": 0.5, "error_rate": 0.0, "score_stddev": 0.0, + "starved_attempt_rate": 0.0, "dead_infra_attempt_rate": 0.0, + "unmeasured_attempt_rate": 0.0} assert report.cases[0].metrics == { "score": 0.5, "n_attempts": 2.0, @@ -1032,9 +968,60 @@ async def test_harbor_backend_mean_counts_dead_attempts_as_failures(tmp_path): # zero-filled failure, not infra dilution. "n_dead_infra": 0.0, "n_clean": 2.0, + # Neither attempt reports token counters at all, so starvation is + # unknowable here and must not be asserted. See the starvation test below. + "n_starved": 0.0, } +@pytest.mark.asyncio +async def test_harbor_backend_surfaces_attempts_that_bought_no_inference(tmp_path): + """An attempt that ran but spent nothing must be visible in the report. + + This is the failure that cost the swe-bench-pro grid a day: once the + inference budget was gone, every remaining attempt still ran its sandbox, + agent and verifier, produced an honest 0.0 from the hidden suite, and was + averaged in. No exception was raised, so `n_dead_infra` stayed at 0, the + case still returned SUCCESS, and `error_rate` read a clean 0.0 while two + thirds of the pass was dead weight. Only the token counters showed it. + """ + sandbox = FakeSandbox( + tmp_path, + { + "example/alpha": [ + { + "verifier_result": {"rewards": {"pass": 1.0}}, + "agent_result": {"n_input_tokens": 4000, "n_output_tokens": 120}, + }, + # Ran, scored a real 0.0 from the verifier, bought nothing. + { + "verifier_result": {"rewards": {"pass": 0.0}}, + "agent_result": {"n_input_tokens": 0, "n_output_tokens": 0}, + }, + ] + }, + ) + backend = HarborBackend(_config(tmp_path, aggregate_attempts="mean")) + + report = await backend.evaluate( + context=await _context(tmp_path, sandbox), + request=_request(CaseIds(ids=["case-a"])), + ) + + case = report.cases[0] + assert case.metrics["n_starved"] == 1.0 + # The starved attempt looks perfectly healthy to every pre-existing signal. + assert case.metrics["n_dead_infra"] == 0.0 + assert case.metrics["n_clean"] == 2.0 + assert case.status == CaseStatus.SUCCESS + assert report.metrics["error_rate"] == 0.0 + # ... and this is the one number that gives it away. + assert report.metrics["starved_attempt_rate"] == 0.5 + assert report.metrics["unmeasured_attempt_rate"] == 0.5 + # The score is halved by an attempt that never had a chance to earn one. + assert report.metrics["score"] == 0.5 + + @pytest.mark.asyncio async def test_harbor_backend_fails_when_no_requested_trials_match(tmp_path): secret = "sensitive-token" @@ -1289,72 +1276,3 @@ async def test_case_result_carries_a_phase_execution_trace(tmp_path: Path): failed = by_id["case-b"].execution_trace assert [span["phase"] for span in failed] == ["environment_setup", "exception"] assert failed[-1]["detail"]["exception_type"] == "TimeoutError" - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("payload", "status_code", "expected"), - [ - # Both caps present and one is empty -> exhausted. - ({"remaining_requests": 40, "remaining_tokens": 0}, 200, True), - ({"remaining_requests": 0, "remaining_tokens": 900}, 200, True), - # Headroom on both -> not exhausted, which is what lets a false text - # claim be overridden. - ({"remaining_requests": 40, "remaining_tokens": 900}, 200, False), - # An uncapped scope reports None for both. That is "cannot be exhausted", - # not "unknown" -- returning None here would leave a bogus terminating - # claim standing on exactly the scopes that have no limit to hit. - ({"remaining_requests": None, "remaining_tokens": None}, 200, False), - # Anything unhealthy is unknown: never guess from a failed lookup. - ({}, 403, None), - ({}, 500, None), - ], -) -async def test_scope_budget_lookup_reads_the_ledger( - tmp_path, monkeypatch, payload, status_code, expected -): - import httpx - - captured: dict[str, object] = {} - - class _Response: - def __init__(self): - self.status_code = status_code - - def json(self): - return payload - - class _Client: - def __init__(self, *args, **kwargs): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *exc): - return False - - async def get(self, url, headers=None): - captured["url"] = url - captured["headers"] = headers - return _Response() - - monkeypatch.setattr(httpx, "AsyncClient", _Client) - backend = HarborBackend( - _config( - tmp_path, - inference_gateway_url="http://inference-gateway:8001/", - inference_gateway_token="gw-token", - inference_gateway_finalization_token="gw-final", - ) - ) - - assert await backend._scope_budget_is_exhausted(finalization=False) is expected - # The evaluation scope is asked with the evaluation token, and the trailing - # slash on the configured URL must not produce a doubled path separator. - assert captured["url"] == "http://inference-gateway:8001/usage/evaluation" - assert captured["headers"]["Authorization"] == "Bearer gw-token" - - await backend._scope_budget_is_exhausted(finalization=True) - assert captured["url"] == "http://inference-gateway:8001/usage/finalization" - assert captured["headers"]["Authorization"] == "Bearer gw-final"