From ec7ebc2b26bc0d9b7bffffbaa526bd4c6301f29f Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 12:26:20 -0700 Subject: [PATCH 1/9] =?UTF-8?q?refactor(results):=201/6=20=E2=80=94=20rena?= =?UTF-8?q?me=20the=20persisted=20turn-cap=20names=20to=20tool=20calls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FinalStatus/AgentEndStatus/TurnEndStatus TOOL_CALLS_EXHAUSTED, the tool_calls_exhausted flag (derived by the collector from the end status; AgentEndEvent.max_turns_exhausted deleted), run_limits.expected_tool_calls and the run.json keys. DialogStopReason gains TOOL_CALL_CAP. The evalboard reads both spellings; REPORT_SCHEMA names the historical ones. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 8 +-- .claude/notes/isolation.md | 10 +-- .claude/notes/orchestration.md | 12 ++-- docs/DIALOG_MODE.md | 4 +- docs/REPORT_SCHEMA.md | 19 ++++-- docs/TASK_DEFINITION_GUIDE.md | 12 ++-- docs/USER_GUIDE.md | 2 +- docs/agents/HARNESS_PARITY.md | 6 +- docs/agents/OPENCODE.md | 2 +- docs/agents/PI.md | 2 +- docs/tutorials/04-writing-a-task.md | 4 +- evalboard/app/_overview/efficiency-charts.tsx | 2 +- .../[...task]/__tests__/turns-stat.test.tsx | 6 +- .../app/runs/[id]/[...task]/turns-stat.tsx | 4 +- .../runs/[id]/__tests__/task-grid.test.tsx | 6 +- evalboard/app/runs/[id]/activation/page.tsx | 2 +- evalboard/app/runs/[id]/task-grid.tsx | 6 +- evalboard/app/trends/trends-view.tsx | 4 +- evalboard/lib/__tests__/overview.test.ts | 4 +- evalboard/lib/__tests__/pills.test.tsx | 8 +-- evalboard/lib/__tests__/runs.test.ts | 13 ++-- evalboard/lib/__tests__/status.test.ts | 2 + evalboard/lib/overview.ts | 6 +- evalboard/lib/pills.tsx | 4 +- evalboard/lib/runs.ts | 8 ++- evalboard/lib/status.ts | 2 +- evalboard/lib/turns.ts | 6 +- experiments/default.yaml | 8 +-- plugins/coder-eval/skills/analyze/SKILL.md | 4 +- src/coder_eval/agent.py | 2 +- src/coder_eval/agents/antigravity_agent.py | 3 +- src/coder_eval/agents/claude_code_agent.py | 3 +- src/coder_eval/agents/codex_agent.py | 3 +- src/coder_eval/agents/opencode_agent.py | 3 +- src/coder_eval/agents/pi_agent.py | 3 +- src/coder_eval/models/enums.py | 12 ++-- src/coder_eval/models/limits.py | 11 ++- src/coder_eval/models/results.py | 9 +-- src/coder_eval/orchestrator.py | 68 +++++++++---------- src/coder_eval/reports/html.py | 15 ++-- src/coder_eval/reports/markdown.py | 11 +-- src/coder_eval/resources/tags.yaml | 2 +- src/coder_eval/result_metrics.py | 8 +-- src/coder_eval/run_record.py | 18 ++--- src/coder_eval/simulation/termination.py | 1 + src/coder_eval/streaming/collector.py | 3 +- src/coder_eval/streaming/events.py | 5 +- src/coder_eval/streaming/renderers.py | 9 +-- tasks/fibonacci_with_template.yaml | 2 +- tasks/hello_date.yaml | 2 +- tasks/opencode_smoke_test.yaml | 2 +- tasks/pi_smoke_test.yaml | 2 +- tasks/run_limits/max_turns_cap.yaml | 2 +- tasks/smoke_negative_path.yaml | 2 +- .../antigravity_a_single_text_turn.json | 2 +- .../antigravity_b_tool_call_resolved.json | 2 +- ...y_c_thinking_and_tool_same_generation.json | 2 +- .../expected/antigravity_d_orphaned_tool.json | 2 +- .../antigravity_e_multi_generation.json | 2 +- .../expected/claude_a_single_text_turn.json | 2 +- .../expected/claude_b_tool_use_result.json | 2 +- .../claude_c_multi_emission_delta.json | 2 +- .../expected/claude_d_subagent_terminal.json | 2 +- .../claude_e_model_usage_and_backfill.json | 2 +- .../expected/claude_f_orphaned_tool.json | 2 +- .../claude_g_crash_format_placeholder.json | 2 +- .../claude_h1_timeout_process_error.json | 2 +- .../claude_h2_process_error_crash.json | 2 +- .../claude_i_in_loop_deadline_break.json | 2 +- .../expected/codex_a_agent_message_only.json | 2 +- .../expected/codex_b_command_execution.json | 2 +- .../codex_c_reasoning_placeholder.json | 2 +- .../codex_d_cross_flush_is_error.json | 2 +- .../expected/codex_e_orphan_tool.json | 2 +- .../expected/codex_f_collab_fallback.json | 2 +- .../expected/codex_g_items_rebuild.json | 2 +- .../codex_h_no_turn_completed_crash.json | 2 +- .../expected/opencode_a_single_text_turn.json | 2 +- .../opencode_b_tool_call_resolved.json | 2 +- .../opencode_c_multi_step_tiling.json | 2 +- .../expected/opencode_d_orphaned_tool.json | 2 +- .../opencode_e_error_after_generation.json | 2 +- .../expected/pi_a_single_text_turn.json | 2 +- .../expected/pi_b_tool_call_resolved.json | 2 +- .../expected/pi_c_multi_turn_tiling.json | 2 +- .../expected/pi_d_orphaned_tool.json | 2 +- .../expected/pi_e_error_after_generation.json | 2 +- .../expected/pi_f_duplicate_turn_end.json | 2 +- tests/_fixtures/report_snapshots/run_full.md | 4 +- .../harbor_e2e/fixtures/template_sources.yaml | 2 +- .../ce018_no_final_status_name_denylist.py | 2 +- tests/test_agent.py | 8 +-- tests/test_aggregate.py | 2 +- tests/test_antigravity_agent.py | 10 +-- tests/test_codex_agent.py | 10 +-- tests/test_custom_lint.py | 2 +- tests/test_detached_grading_boundaries.py | 4 +- tests/test_event_collector.py | 19 +++++- tests/test_execute_evaluate_loop.py | 18 ++--- tests/test_experiment_reports.py | 6 +- tests/test_lint_no_top_level_run_limits.py | 6 +- tests/test_opencode_agent.py | 14 ++-- tests/test_orchestrator.py | 16 ++--- tests/test_orchestrator_error_log_tail.py | 6 +- tests/test_orchestrator_telemetry.py | 2 +- tests/test_pi_agent.py | 14 ++-- tests/test_reports.py | 46 ++++++------- tests/test_reports_html.py | 34 +++++----- tests/test_reports_junit.py | 4 +- tests/test_result_metrics.py | 44 ++++++------ tests/test_run_limits_models.py | 22 +++--- tests/test_run_limits_orchestrator.py | 66 +++++++++--------- tests/test_run_metrics.py | 2 +- tests/test_run_record.py | 31 +++++---- tests/test_seed_from_prior_result.py | 22 ++++-- tests/test_streaming_renderers.py | 22 +++--- tests/test_verify_published_workflow.py | 2 +- 117 files changed, 472 insertions(+), 413 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 6c2a5e2f3..ae7c6b832 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -70,7 +70,7 @@ intentionally brief and out of scope; trimming for DISPLAY belongs in the render their CLIs stream a real multi-step loop per `communicate()` (`step_start`/`step_finish`, `turn_start`/`turn_end`). The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as - `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in + `tool_calls_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. The **known unfixed divergences** — which config fields each harness does and does not @@ -111,7 +111,7 @@ failure is not swallowed. `finalize` is total: an unmapped future member raises loudly instead of silently bucketing to COMPLETED. -Status precedence is the same everywhere: timeout > stopped_early > max_turns_exhausted > +Status precedence is the same everywhere: timeout > stopped_early > tool_calls_exhausted > completed. `stopped_early` outranks the cap because an armed criterion deciding the outcome is the more specific reason to have cut the run, and every loop checks it first. @@ -261,13 +261,13 @@ matter how much the run actually billed. So the CLI harnesses crash rather than Crashing routes it to `FinalStatus.ERROR`, which is excluded from outcomes. - **A CLI that closed its stream but would not exit** within the grace period. -Every arm is gated on `stopped_early` / `max_turns_exhausted`, because an intentional cut +Every arm is gated on `stopped_early` / `tool_calls_exhausted`, because an intentional cut can land before the clearing event arrives. Pi's error case shows why: `error_message` is set at an error `turn_end` and cleared only by a LATER non-error `turn_end`, but a `max_turns` / `should_stop` cut can fire at the next `turn_start`, leaving a stale error from a turn Pi was still retrying. Without the guard that clean, budget-exhausted cut would crash and burn retries, contradicting the documented "finalizes cleanly as -`max_turns_exhausted`, no crash" contract. +`tool_calls_exhausted`, no crash" contract. OpenCode has one escape hatch, `require_token_telemetry`, for a provider or auth mode that reports no usage at all — where crashing every turn makes the harness unusable rather than diff --git a/.claude/notes/isolation.md b/.claude/notes/isolation.md index c9e2957d5..6baaae2bb 100644 --- a/.claude/notes/isolation.md +++ b/.claude/notes/isolation.md @@ -26,18 +26,18 @@ `FinalStatus.is_execution_fact` (TIMEOUT / ERROR / BUILD_FAILED / the budget stops) is **preserved**, never overwritten: grading may only move `NOT_GRADED` to SUCCESS/FAILURE, since it neither repeated nor observed the agent phase. - **`MAX_TURNS_EXHAUSTED` is deliberately NOT one of them — anywhere**. + **`TOOL_CALLS_EXHAUSTED` is deliberately NOT one of them — anywhere**. `_EXECUTION_FACT_STATUSES` maps it to `False`, and the table and the chain that reads it must agree: it shipped as `True` while `_terminal_status`'s own docstring argued the opposite, and the disagreement pinned a re-graded max-turns row at - MAX_TURNS_EXHAUSTED *while holding `weighted_score` 1.000* and exit 1 — a combination + TOOL_CALLS_EXHAUSTED *while holding `weighted_score` 1.000* and exit 1 — a combination `run` can never produce for the same trajectory. Under `execute`: `_terminal_status` puts the `grade=False` arm ABOVE it, because on the graded path it is subordinate to the verdict — `run` returns SUCCESS for a max-turns trajectory whose criteria pass — so it is not knowable without grading. Consuming it first made it terminal AND permanent (the `is_execution_fact` arm then pinned it), so identical agent output - scored SUCCESS/1.0 under `run` and MAX_TURNS_EXHAUSTED under `execute` → `evaluate`. - The fact survives on `result.max_turns_exhausted`, which `_seed_from_prior_result` + scored SUCCESS/1.0 under `run` and TOOL_CALLS_EXHAUSTED under `execute` → `evaluate`. + The fact survives on `result.tool_calls_exhausted`, which `_seed_from_prior_result` carries, so the detached grade walks the identical chain. The CLI must also branch on WHERE a status came from, not on its value: a preserved TIMEOUT exited 0 under "All criteria passed" (a CI wrapper reading the exit code went green on a row run.json @@ -741,7 +741,7 @@ verdict for work it never looked at and billing the model for it. Nothing else c Both are keyed on EVIDENCE, not on the label. For `grade`, "did it grade" is `success_criteria_results` or a non-None `weighted_score`: exempting every execution-fact -status let a stale image return a fully graded MAX_TURNS_EXHAUSTED row — criteria vector, +status let a stale image return a fully graded TOOL_CALLS_EXHAUSTED row — criteria vector, weighted score and all — unchallenged, because that exemption exists for statuses a *fresh* image also produces, and a fresh one produces them with neither. For `regrade`, a container that honored the request seeds from `prior` and never runs the agent, so a DIFFERENT diff --git a/.claude/notes/orchestration.md b/.claude/notes/orchestration.md index 800adb40b..f52c7e75e 100644 --- a/.claude/notes/orchestration.md +++ b/.claude/notes/orchestration.md @@ -57,7 +57,7 @@ and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** - — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are + — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `TOOL_CALLS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog @@ -88,15 +88,15 @@ grading switch was threaded in. Its ORDER is load-bearing at every step. observed. Without that first arm, a crashed run re-graded against its half-finished workspace reports SUCCESS — with the original `error_message` still attached. -**The NOT_GRADED arm sits ABOVE `max_turns_exhausted`, and that order is what makes -`execute` + `evaluate` equal a single `run`.** MAX_TURNS_EXHAUSTED reads like an execution +**The NOT_GRADED arm sits ABOVE `tool_calls_exhausted`, and that order is what makes +`execute` + `evaluate` equal a single `run`.** TOOL_CALLS_EXHAUSTED reads like an execution fact but is not one: on the graded path it is subordinate to the verdict — `run` returns SUCCESS for a max-turns trajectory whose criteria pass, and only falls through to -MAX_TURNS_EXHAUSTED when they do not — so it is not knowable under `grade=False`. +TOOL_CALLS_EXHAUSTED when they do not — so it is not knowable under `grade=False`. Consuming it first made it terminal AND permanent, so the same agent output scored -SUCCESS/1.0 under `run` and MAX_TURNS_EXHAUSTED under `execute` → `evaluate`; being +SUCCESS/1.0 under `run` and TOOL_CALLS_EXHAUSTED under `execute` → `evaluate`; being category `failed`, `run --resume` then called the row complete and left it forever -unscored. Nothing is lost by deferring: the fact lives on `result.max_turns_exhausted`, +unscored. Nothing is lost by deferring: the fact lives on `result.tool_calls_exhausted`, which the seeding carries. The statuses that ARE execution facts differ in kind — they abort the run before a verdict is reachable, so preserving them overturns nothing. diff --git a/docs/DIALOG_MODE.md b/docs/DIALOG_MODE.md index 0f18bdd28..b0c9e476a 100644 --- a/docs/DIALOG_MODE.md +++ b/docs/DIALOG_MODE.md @@ -122,8 +122,8 @@ After each exchange the driver evaluates the stop conditions **in this order**, 2. **`stop_on_criteria_pass`** (`criteria_passed`) — every success criterion passes. Requires per-turn checking (`check_criteria: every_turn` or `both`); pairing it with the default `end_of_dialog` is rejected at load time, since there would be nothing to check against. -3. **`max_turns`** (`max_turns`) — the hard cap on exchanges. The agent exhausting its *own* inner - `max_turns` mid-exchange ends the dialog with the same reason. +3. **`max_turns`** (`max_turns`) — the hard cap on exchanges. The agent reaching its *own* tool-call + cap mid-exchange ends the dialog with its own reason, `tool_call_cap`. 4. **`max_total_tokens`** (`budget`) — the dialog-wide budget across simulator **and** agent. The dialog ends and the task is **still scored** — unlike [`run_limits.max_total_tokens`](TASK_DEFINITION_GUIDE.md#run-limits), which covers the subject diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index b1f1a8cd1..b008c29a6 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -83,12 +83,21 @@ including: `task_id`, `replicate_index`, `variant_id`, `status` `judge_cost_usd` / `simulator_cost_usd` slices and the `cost_complete` flag), `expected_commands`, `actual_commands`, `commands_efficiency`, `agent_config`, `sdk_options`, -`installed_tools`, turn accounting (`total_turns`, `visible_turns`, `expected_turns`, -`max_turns_exhausted`, `has_final_reply`), and early-stop fields (`stopped_early`, +`installed_tools`, turn accounting (`total_turns`, `visible_turns`, `expected_tool_calls`, +`expected_tool_calls_overage`, `tool_calls_exhausted`, `has_final_reply`), and early-stop fields (`stopped_early`, `early_stop_reason`, `turns_remaining_at_stop`). `iterations` here is a **reduced** turn digest (`{iteration, duration_seconds, command_count, assistant_turn_count, crashed, crash_reason}`) — the full transcript is in `task.json`. +> **Historical spellings.** Runs written before the tool-call rename carry +> `max_turns_exhausted`, `expected_turns`, `expected_turns_overage` and the status +> `MAX_TURNS_EXHAUSTED` instead of `tool_calls_exhausted`, `expected_tool_calls`, +> `expected_tool_calls_overage` and `TOOL_CALLS_EXHAUSTED`. The evalboard reads both. +> The Python side does not: there is no alias. A `task.json` with the old flag loads with +> the fact `false`; one whose `final_status` is `MAX_TURNS_EXHAUSTED`, or whose recorded +> config sets `run_limits.expected_turns`, does not load. `run --resume` then runs that +> row again, and `evaluate ` cannot re-grade it from its recorded config. + ### Missing cost is never fatal Pricing degrades; the evaluation does not. A model absent from the rate card, a turn @@ -128,7 +137,7 @@ The authoritative per-replicate record. | --- | --- | --- | | `final_status` | [`FinalStatus`](#finalstatus) | Terminal status. | | `weighted_score` | `float \| null` | Weighted average of criterion scores, 0.0–1.0. | -| `max_turns_exhausted` | `bool` | Ran out of turns. | +| `tool_calls_exhausted` | `bool` | The tool-call cap ended an iteration before the agent completed on its own. | | `iteration_count` | `int` | Number of turns. | | `success_criteria_results` | `list[CriterionResult]` | Per-criterion results — see [below](#criterionresult). | | `post_failure_criteria_results` | `list[CriterionResult]` | Diagnostic artifact evidence collected after a terminal agent failure. It does not affect `final_status`, `weighted_score`, gating, or suite aggregation. | @@ -211,7 +220,7 @@ canonical score remains 0.0. (`list[ProviderCallCost]` — one row per real upstream call with its ACTUAL cost + cache buckets, captured proxy-side on the LiteLLM open-weight backend and rendered by the evalboard as a per-call table; empty on every other -backend), `num_turns`, `max_turns_exhausted`, +backend), `num_turns`, `tool_calls_exhausted`, `result_summary` (`{is_error, subtype, stop_reason, result}`), `crashed`, `crash_reason`. @@ -320,7 +329,7 @@ String enum values and their reporting category: | `SUCCESS` | succeeded | `+` | | `FAILURE` | failed | `-` | | `TIMEOUT` | failed | `T` | -| `MAX_TURNS_EXHAUSTED` | failed | `M` | +| `TOOL_CALLS_EXHAUSTED` | failed | `C` | | `TOKEN_BUDGET_EXCEEDED` | failed | `#` | | `COST_BUDGET_EXCEEDED` | failed | `$` | | `ERROR` | error | `!` | diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index c4e1126fb..1ef8235e2 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -259,7 +259,7 @@ valid and an empty block is legal — every field defaults to "no limit". run_limits: # Structural caps max_turns: 20 # hard cap on agent inner-loop turns per iteration - expected_turns: 8 # SOFT efficiency budget (visible turns) — never aborts + expected_tool_calls: 8 # SOFT efficiency budget (visible tool calls) — never aborts task_timeout: 300 # wall-clock cap for the full run envelope, seconds turn_timeout: 300 # per-communicate() timeout, seconds @@ -274,7 +274,7 @@ run_limits: | Field | Default | Constraint | Description | |-------|---------|------------|-------------| | `max_turns` | *unset* | `> 0` | Hard cap on agent inner-loop turns per iteration. Unset uses the SDK default. | -| `expected_turns` | *unset* | `>= 1` | **Soft** target for cumulative visible turns. Exceeding it warns and badges the report; it never aborts. See [`expected_turns`](#expected_turns-soft-efficiency-budget). | +| `expected_tool_calls` | *unset* | `>= 1` | **Soft** target for cumulative visible tool calls. Exceeding it warns and badges the report; it never aborts. See [`expected_tool_calls`](#expected_tool_calls-soft-efficiency-budget). | | `task_timeout` | *unset* | `>= 30` | Max seconds for the full run envelope, including agent work, grading, and post-run work. | | `turn_timeout` | *unset* | `>= 10` | Max seconds for the agent's single `communicate()` iteration. | | `max_input_tokens` | *unset* | `>= 1` | Max cumulative input (prompt) tokens. | @@ -334,15 +334,15 @@ coder-eval run task.yaml -D run_limits.max_usd=2.50 -D run_limits.max_total_toke > They must live under `run_limits:`. (A deprecation shim hoisted them > automatically until it was removed on 2026-06-01.) -### `expected_turns` (soft efficiency budget) +### `expected_tool_calls` (soft efficiency budget) -`run_limits.expected_turns` is a **soft target**, not a cap: the run is never +`run_limits.expected_tool_calls` is a **soft target**, not a cap: the run is never aborted for exceeding it (use `max_turns` for a hard limit). It's the budget the dashboard's **"Within Expected Turns"** metric divides by — a task counts as "within budget" when it succeeds *and* its turn count stays within **1.5×** -`expected_turns`. The run-level headline reports the share of **budgeted** tasks +`expected_tool_calls`. The run-level headline reports the share of **budgeted** tasks that did: a budgeted task that failed counts as over budget, while tasks with no -`expected_turns` budget are excluded entirely (success or fail). +`expected_tool_calls` budget are excluded entirely (success or fail). The count compared against the budget is **visible turns** — one per tool call plus one for the agent's final reply — *not* the SDK's `total_turns` (which diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index c561e8107..3f0fe446e 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -84,7 +84,7 @@ exits non-zero, exactly as under `run`. Exhausting `max_turns` is the one fact that does *not* become a status here. Under `run` it decides the outcome only when the criteria fail — a max-turns trajectory whose criteria pass is `SUCCESS` — so it is not knowable without grading. `execute` -records `max_turns_exhausted: true` on the row and finalizes `NOT_GRADED`; the later +records `tool_calls_exhausted: true` on the row and finalizes `NOT_GRADED`; the later grade reads the flag and reaches exactly the status `run` would have. Rows like this are picked up by `run --resume`, which owes a grade to anything executed but never scored — including a row that also timed out or tripped a budget. diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index e641c7200..f59b4ec78 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -597,7 +597,7 @@ The cap is enforced on the same loop boundary as the cooperative early stop: the step or notification that reaches the cap is processed whole, and the next one is never pulled. The in-flight turn is then cancelled server-side (best effort) so the cap actually stops spend. A run cut this way finalizes cleanly as -`max_turns_exhausted` — it is not a crash, and it is not retried. +`tool_calls_exhausted` — it is not a crash, and it is not retried. **claude-code keeps its native SDK cap.** That is a real, honored cap, so it is left alone rather than reimplemented in a different unit. Its unit is the SDK's own @@ -633,9 +633,9 @@ The signals a capped run leaves behind, on every backend: - Criteria are still checked against whatever the agent produced, because the cap is an ordinary end-of-run rather than an error. So a capped run that nonetheless satisfies its criteria finishes as `SUCCESS`; one that does not finishes as - `MAX_TURNS_EXHAUSTED` (reporting category `failed`, icon `M`). Never `ERROR`, + `TOOL_CALLS_EXHAUSTED` (reporting category `failed`, icon `C`). Never `ERROR`, and never retried. -- `max_turns_exhausted: true` on the task record. +- `tool_calls_exhausted: true` on the task record. - On Codex and Antigravity, the count of *resolved* tool calls the model itself issued equals the cap. Two things can add a further *recorded* command, and neither means the cap leaked: diff --git a/docs/agents/OPENCODE.md b/docs/agents/OPENCODE.md index d06b39c05..fa74ad887 100644 --- a/docs/agents/OPENCODE.md +++ b/docs/agents/OPENCODE.md @@ -346,7 +346,7 @@ any other provider credential can be added via `sandbox.env_passthrough_extra`. - **`max_turns` counts OpenCode's native steps.** One step = one assistant generation (`step_start`/`step_finish`) and may carry several tool calls; `max_turns: N` allows N complete steps, then the run finalizes cleanly as - `max_turns_exhausted`. This is the claude-code-style native unit, not the + `tool_calls_exhausted`. This is the claude-code-style native unit, not the visible-turn unit Codex/Antigravity use — see [Run-Limit Parity](HARNESS_PARITY.md) before holding `max_turns` constant across harnesses. diff --git a/docs/agents/PI.md b/docs/agents/PI.md index eec02eb4d..fdca6b452 100644 --- a/docs/agents/PI.md +++ b/docs/agents/PI.md @@ -242,7 +242,7 @@ docker` whenever the task prompt or workspace is not fully trusted. instead — it is enforced via `--append-system-prompt`. - **`max_turns` counts Pi's native agent-loop turns.** One `turn_start` = one agent-loop step; `max_turns: N` allows N complete turns, then the run finalizes - cleanly as `max_turns_exhausted`. See + cleanly as `tool_calls_exhausted`. See [Run-Limit Parity](HARNESS_PARITY.md) before holding `max_turns` constant across harnesses. - **No sub-agent attribution.** Pi's CLI stream does not expose nested agent diff --git a/docs/tutorials/04-writing-a-task.md b/docs/tutorials/04-writing-a-task.md index 83d15c4e2..602c95ca3 100644 --- a/docs/tutorials/04-writing-a-task.md +++ b/docs/tutorials/04-writing-a-task.md @@ -38,7 +38,7 @@ agent: setting_sources: [] # isolate the sandbox from your own CLAUDE.md/settings run_limits: - expected_turns: 5 # soft target — warns when exceeded, never aborts + expected_tool_calls: 5 # soft target — warns when exceeded, never aborts success_criteria: - type: "file_exists" @@ -62,7 +62,7 @@ What each block does: settings out of the sandbox. Without it, a large host `CLAUDE.md` is injected into every API call, inflating cache-creation tokens and cost. Leave it out only when the task needs your MCP servers. -- **`run_limits.expected_turns`** — an efficiency target, not a cap: exceeding +- **`run_limits.expected_tool_calls`** — an efficiency target, not a cap: exceeding it logs a warning and adds a report badge but never aborts (use `run_limits.max_turns` for a hard cap). - **`success_criteria`** — each criterion scores 0.0–1.0 and supports `weight` diff --git a/evalboard/app/_overview/efficiency-charts.tsx b/evalboard/app/_overview/efficiency-charts.tsx index 002e74b9c..9af868b78 100644 --- a/evalboard/app/_overview/efficiency-charts.tsx +++ b/evalboard/app/_overview/efficiency-charts.tsx @@ -51,7 +51,7 @@ const TABS: Array<{ key: "turns", label: "Turns", heading: "Within Expected Turns (%)", - title: "Share of tasks carrying an expected_turns budget whose visible turns stayed within 1.5× it. A budgeted task that failed counts as over budget.", + title: "Share of tasks carrying an expected_tool_calls budget whose visible turns stayed within 1.5× it. A budgeted task that failed counts as over budget.", blurb: (scoped) => "% of budgeted tasks that stayed within 1.5× their expected turns (a budgeted task that failed counts as over budget) · runs with no budgeted task are omitted rather than plotted at 0" + (scoped ? " · scoped to the active filter" : ""), diff --git a/evalboard/app/runs/[id]/[...task]/__tests__/turns-stat.test.tsx b/evalboard/app/runs/[id]/[...task]/__tests__/turns-stat.test.tsx index 9599423f9..106b9ad27 100644 --- a/evalboard/app/runs/[id]/[...task]/__tests__/turns-stat.test.tsx +++ b/evalboard/app/runs/[id]/[...task]/__tests__/turns-stat.test.tsx @@ -17,7 +17,7 @@ describe("TurnsStat", () => { expect(dd.tagName).toBe("DD"); expect(dd.className).toContain("text-rose-700"); expect(dd.className).not.toContain("bg-"); - expect(dd).toHaveAttribute("title", "expected_turns target: 5"); + expect(dd).toHaveAttribute("title", "expected_tool_calls target: 5"); }); test("yellow text at +25%–+50% (ratio 1.4)", () => { @@ -38,7 +38,7 @@ describe("TurnsStat", () => { expect(dd.tagName).toBe("DD"); expect(dd.className).toContain("text-gray-900"); expect(dd.className).not.toMatch(/text-(rose|amber|emerald)-/); - expect(dd).toHaveAttribute("title", "no expected_turns target set"); + expect(dd).toHaveAttribute("title", "no expected_tool_calls target set"); }); test("both null renders em dash with default text", () => { @@ -46,7 +46,7 @@ describe("TurnsStat", () => { const dd = screen.getByText("—"); expect(dd.tagName).toBe("DD"); expect(dd.className).toContain("text-gray-900"); - expect(dd).toHaveAttribute("title", "no expected_turns target set"); + expect(dd).toHaveAttribute("title", "no expected_tool_calls target set"); }); }); diff --git a/evalboard/app/runs/[id]/[...task]/turns-stat.tsx b/evalboard/app/runs/[id]/[...task]/turns-stat.tsx index 2fbb39ce1..0de3d0c78 100644 --- a/evalboard/app/runs/[id]/[...task]/turns-stat.tsx +++ b/evalboard/app/runs/[id]/[...task]/turns-stat.tsx @@ -22,8 +22,8 @@ export function TurnsStat({ className={`mt-0.5 tabular-nums font-medium ${turnsCellClasses(tint)}`} title={ expectedTurns != null - ? `expected_turns target: ${expectedTurns}` - : "no expected_turns target set" + ? `expected_tool_calls target: ${expectedTurns}` + : "no expected_tool_calls target set" } > {fmtTurnsCount(turns)} diff --git a/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx b/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx index 27953ed3f..c894223c0 100644 --- a/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx @@ -223,7 +223,7 @@ describe("TaskGrid — Turns column", () => { expect(overCell).toHaveTextContent("10"); expect(overCell.className).toContain("text-rose-700"); expect(overCell.className).not.toContain("bg-"); - expect(overCell).toHaveAttribute("title", "expected_turns target: 5"); + expect(overCell).toHaveAttribute("title", "expected_tool_calls target: 5"); expect(turnsCellFor("mid").className).toContain("text-amber-700"); expect(turnsCellFor("under").className).toContain("text-emerald-700"); @@ -236,7 +236,7 @@ describe("TaskGrid — Turns column", () => { ); expect(noTargetCell).toHaveAttribute( "title", - "no expected_turns target set", + "no expected_tool_calls target set", ); }); @@ -304,7 +304,7 @@ describe("TaskGrid — column tooltips", () => { ); expect(header("Turns")).toHaveAttribute( "title", - expect.stringContaining("expected_turns"), + expect.stringContaining("expected_tool_calls"), ); // No ⓘ buttons anywhere: each header carries its sort toggle and nothing else. for (const h of screen.getAllByRole("columnheader")) { diff --git a/evalboard/app/runs/[id]/activation/page.tsx b/evalboard/app/runs/[id]/activation/page.tsx index 201753225..c7139be53 100644 --- a/evalboard/app/runs/[id]/activation/page.tsx +++ b/evalboard/app/runs/[id]/activation/page.tsx @@ -19,7 +19,7 @@ function pct(v: number | null): string { // A case is a mistake when the skill that fired isn't the one expected: a miss // (nothing fired), a false positive (something fired on a negative), or the wrong // skill. Tied to the two columns the table shows, not to the run status (which -// also flips on harness noise like MAX_TURNS_EXHAUSTED). A negative case expects +// also flips on harness noise like TOOL_CALLS_EXHAUSTED). A negative case expects // nothing — normalised to "". Unknown (no triggered signal) is never flagged. function isMistake(c: ActivationCaseRow): boolean { if (c.triggeredSkill == null) return false; diff --git a/evalboard/app/runs/[id]/task-grid.tsx b/evalboard/app/runs/[id]/task-grid.tsx index 6d3fb7b99..75ec92f67 100644 --- a/evalboard/app/runs/[id]/task-grid.tsx +++ b/evalboard/app/runs/[id]/task-grid.tsx @@ -53,7 +53,7 @@ type SortKey = // TOKEN_COLUMN_HELP; the rest is grid-specific. const COLUMN_HELP: Partial> = { ...TOKEN_COLUMN_HELP, - turns: "Visible turns: one per tool call plus one for the final reply. Tinted against the task's hand-written expected_turns budget (yellow past 1.25×, red past 1.5×); untinted when the task declares none.", + turns: "Visible turns: one per tool call plus one for the final reply. Tinted against the task's hand-written expected_tool_calls budget (yellow past 1.25×, red past 1.5×); untinted when the task declares none.", vsExp: "Duration ÷ the time this task is expected to need. The expected time is derived per task, per harness by the eval runner (its fastest passing run, or p10 once there are ten) and stamped into the run — never hand-written. Past 2× counts as slow; a task its harness has never passed shows —.", cost: "Total billed cost for this task, reported by the SDK (summed across turns).", variant: "Experiment arm this row was produced by. A run declaring `variants:` executes every task once per arm and keeps each arm's output in its own subtree, so the same task appears once per arm and the two rows are separate measurements — never collapsed together.", @@ -817,8 +817,8 @@ export function TaskGrid({ className={`py-3 px-4 text-right tabular-nums font-medium ${turnsCellClasses(turnsTint)}`} title={ t.expectedTurns != null - ? `expected_turns target: ${t.expectedTurns}` - : "no expected_turns target set" + ? `expected_tool_calls target: ${t.expectedTurns}` + : "no expected_tool_calls target set" } > {fmtTurnsCount( diff --git a/evalboard/app/trends/trends-view.tsx b/evalboard/app/trends/trends-view.tsx index dadda7edf..15cb07703 100644 --- a/evalboard/app/trends/trends-view.tsx +++ b/evalboard/app/trends/trends-view.tsx @@ -355,8 +355,8 @@ function HistoryTable({ e.matureSkipped ? MATURE_TOOLTIP : e.expectedTurns != null - ? `expected_turns target: ${e.expectedTurns}` - : "no expected_turns target set" + ? `expected_tool_calls target: ${e.expectedTurns}` + : "no expected_tool_calls target set" } > {e.matureSkipped diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index 38b20899e..0c21b68b3 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -114,7 +114,7 @@ describe("summarizeListing", () => { describe("turnBudgetRateForTasks", () => { test("null when no task in scope carries a budget", () => { - // No task carries an expected_turns budget, so none is eligible and the + // No task carries an expected_tool_calls budget, so none is eligible and the // final eligible>0 check returns null (the chart shows a gap). expect(turnBudgetRateForTasks([task({ visibleTurns: 5 })])).toBeNull(); }); @@ -161,7 +161,7 @@ describe("turnBudgetRateForTasks", () => { }); test("budget-less failures are excluded from the denominator", () => { - // Eligibility is symmetric: a failure with no expected_turns budget is + // Eligibility is symmetric: a failure with no expected_tool_calls budget is // excluded just like a budget-less success, so it cannot drag the rate // down. Only the budgeted within-budget SUCCESS counts → 100%. const rate = turnBudgetRateForTasks([ diff --git a/evalboard/lib/__tests__/pills.test.tsx b/evalboard/lib/__tests__/pills.test.tsx index a1d367b4f..61d00aed1 100644 --- a/evalboard/lib/__tests__/pills.test.tsx +++ b/evalboard/lib/__tests__/pills.test.tsx @@ -18,7 +18,7 @@ describe("StatusPill — colour by outcome", () => { "FAILURE", "ERROR", "TIMEOUT", - "MAX_TURNS_EXHAUSTED", + "TOOL_CALLS_EXHAUSTED", "TOKEN_BUDGET_EXCEEDED", "COST_BUDGET_EXCEEDED", ])("%s is red (not grey)", (status) => { @@ -34,10 +34,10 @@ describe("StatusPill — colour by outcome", () => { }); test("relabel keeps the specific status label but still colours red", () => { - // MAX_TURNS_EXHAUSTED keeps its raw label (informative) while being red. - const el = pill("MAX_TURNS_EXHAUSTED", true); + // TOOL_CALLS_EXHAUSTED keeps its raw label (informative) while being red. + const el = pill("TOOL_CALLS_EXHAUSTED", true); expect(el.className).toContain("text-red-700"); - expect(el.textContent).toBe("MAX_TURNS_EXHAUSTED"); + expect(el.textContent).toBe("TOOL_CALLS_EXHAUSTED"); // Generic FAILURE relabels to "Failed". expect(pill("FAILURE", true).textContent).toBe("Failed"); expect(pill("SUCCESS", true).textContent).toBe("Passed"); diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index d7736f8de..a293ed18b 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -64,24 +64,29 @@ describe("parseCriterionResults", () => { }); describe("toTaskRow", () => { - test("propagates total_turns and expected_turns", () => { + test("propagates total_turns and expected_tool_calls", () => { const row = toTaskRow({ task_id: "x", total_turns: 7, - expected_turns: 5, + expected_tool_calls: 5, }); expect(row.totalTurns).toBe(7); expect(row.expectedTurns).toBe(5); }); + test("a run written before the rename reads the historical expected_turns key", () => { + const row = toTaskRow({ task_id: "x", expected_turns: 4 }); + expect(row.expectedTurns).toBe(4); + }); + test("legacy raw shape (no new fields) yields null", () => { const row = toTaskRow({ task_id: "x" }); expect(row.totalTurns).toBeNull(); expect(row.expectedTurns).toBeNull(); }); - test("expected_turns explicitly null on raw yields null", () => { - const row = toTaskRow({ task_id: "x", expected_turns: null }); + test("expected_tool_calls explicitly null on raw yields null", () => { + const row = toTaskRow({ task_id: "x", expected_tool_calls: null }); expect(row.expectedTurns).toBeNull(); }); diff --git a/evalboard/lib/__tests__/status.test.ts b/evalboard/lib/__tests__/status.test.ts index a91db0a64..9f4b639e5 100644 --- a/evalboard/lib/__tests__/status.test.ts +++ b/evalboard/lib/__tests__/status.test.ts @@ -18,6 +18,8 @@ const EVERY_FINAL_STATUS: Record = { ERROR: "error", BUILD_FAILED: "error", TIMEOUT: "failed", + TOOL_CALLS_EXHAUSTED: "failed", + // Historical spelling, on runs written before the rename. MAX_TURNS_EXHAUSTED: "failed", TOKEN_BUDGET_EXCEEDED: "failed", COST_BUDGET_EXCEEDED: "failed", diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index bd32866cd..f47390832 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -31,7 +31,7 @@ export interface RunPoint { harness: string; successRate: number | null; // % of budgeted tasks whose visible turns stayed within 1.5× their - // expected_turns budget. Only tasks carrying a positive expected_turns + // expected_tool_calls budget. Only tasks carrying a positive expected_tool_calls // budget are eligible — both SUCCESS and non-SUCCESS. A budgeted task that // did not succeed counts as over budget (a failed run never "stayed within // budget"); a budgeted SUCCESS task is over budget only if its visible @@ -56,10 +56,10 @@ export interface RunPoint { } // The % of budgeted tasks whose visible turns stayed within 1.5× their -// expected_turns budget; null when no task in scope carries a budget. +// expected_tool_calls budget; null when no task in scope carries a budget. // // Eligibility is symmetric: a task counts iff it carries a positive -// expected_turns budget, whether or not it succeeded. A budgeted non-SUCCESS +// expected_tool_calls budget, whether or not it succeeded. A budgeted non-SUCCESS // task is treated as having exhausted its budget (infinite turns) and counts // against the rate — a failed run never "stayed within budget." A budgeted // SUCCESS task counts within budget only when its visible turns are within diff --git a/evalboard/lib/pills.tsx b/evalboard/lib/pills.tsx index 25cd715d5..d6720eba0 100644 --- a/evalboard/lib/pills.tsx +++ b/evalboard/lib/pills.tsx @@ -78,14 +78,14 @@ export function StatusPill({ const ok = status === "SUCCESS" || status === "Completed"; // Colour EVERY non-passing terminal status red, not just the enumerated // few. statusCategory maps all coder_eval failure statuses (FAILURE, ERROR, - // TIMEOUT, MAX_TURNS_EXHAUSTED, TOKEN_BUDGET_EXCEEDED, …) to failed/error; + // TIMEOUT, TOOL_CALLS_EXHAUSTED, TOKEN_BUDGET_EXCEEDED, …) to failed/error; // this catches the ones that previously fell through to a misleading grey. // Flow-execution failures (Faulted/Failed) land in statusCategory's "failed" // bucket too. Only null/unknown stays grey. const cat = statusCategory(status); const isFailure = !ok && isFailureCategory(cat); // Narrower list drives the relabel-to-"Failed" text so specific statuses - // (e.g. MAX_TURNS_EXHAUSTED) keep their raw label while still showing red. + // (e.g. TOOL_CALLS_EXHAUSTED) keep their raw label while still showing red. const fail = status === "FAILURE" || status === "ERROR" || diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 47b064709..82e9f6892 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -513,13 +513,15 @@ export interface RawTaskResult { // before the dashboard-expected-turns PR; both fields are optional and // null-fallback through the cell helpers in lib/turns.ts. total_turns?: number; + expected_tool_calls?: number | null; + // Historical spelling of expected_tool_calls, on runs written before the rename. expected_turns?: number | null; // Derived expected wall clock for this task, stamped by the eval runner // (see eval_runner/skills/timing.py). Absent on unscored tasks and on every // run predating the stamp, which read as unscored through lib/timing.ts. expected_seconds?: number | null; // Documented visible-turn count (tool calls + final reply) — the canonical - // metric the "within expected turns" chart compares against expected_turns. + // metric the "within expected turns" chart compares against expected_tool_calls. // Absent on runs predating this field; visibleTurnsFromRaw() then reconstructs // it from actual_commands + has_final_reply (the identical formula) so the // metric still populates for historical runs. @@ -894,7 +896,7 @@ export function toTaskRow(t: RawTaskResult): TaskResultSummary { totalCostUsd: t.total_cost_usd ?? null, actualCommands: t.actual_commands ?? null, totalTurns: t.total_turns ?? null, - expectedTurns: t.expected_turns ?? null, + expectedTurns: t.expected_tool_calls ?? t.expected_turns ?? null, expectedSeconds: t.expected_seconds ?? null, hasFinalReply: t.has_final_reply ?? false, inputTokens: t.input_tokens ?? null, @@ -1278,7 +1280,7 @@ export async function readRunOverview( weightedScore: t.weighted_score ?? null, actualCommands: t.actual_commands ?? null, totalTurns: t.total_turns ?? null, - expectedTurns: t.expected_turns ?? null, + expectedTurns: t.expected_tool_calls ?? t.expected_turns ?? null, expectedSeconds: t.expected_seconds ?? null, visibleTurns: visibleTurnsFromRaw(t), hasFinalReply: t.has_final_reply ?? false, diff --git a/evalboard/lib/status.ts b/evalboard/lib/status.ts index c79b9519d..dbdd00c8c 100644 --- a/evalboard/lib/status.ts +++ b/evalboard/lib/status.ts @@ -3,7 +3,7 @@ // SUCCESS -> passed // ERROR / BUILD_FAILED -> error (BUILD_FAILED is an environment/setup failure) // NOT_GRADED -> ungraded (`coder-eval execute`: ran, deliberately unscored) -// anything else (FAILURE, TIMEOUT, MAX_TURNS_EXHAUSTED, …) -> failed +// anything else (FAILURE, TIMEOUT, TOOL_CALLS_EXHAUSTED, …) -> failed // // "ungraded" is its OWN member rather than being folded into "unknown". Folding // it there looks safe — an ungraded row genuinely has no verdict — but every diff --git a/evalboard/lib/turns.ts b/evalboard/lib/turns.ts index 1eabd030d..d1437e8cc 100644 --- a/evalboard/lib/turns.ts +++ b/evalboard/lib/turns.ts @@ -17,7 +17,7 @@ export function getTurnRatioThresholds(): TurnRatioThresholds { export type TurnTint = "green" | "yellow" | "red" | null; -// Pure turn-efficiency ratio (turns ÷ expected_turns), used to tint per-task +// Pure turn-efficiency ratio (turns ÷ expected_tool_calls), used to tint per-task // "Turns" cells. This is deliberately blind to pass/fail: the cell answers // "was this task's turn usage efficient?", which is meaningful regardless of // outcome — a task that crashed at 2 turns should NOT read as "over budget" @@ -77,12 +77,12 @@ export function fmtTurnsCount(n: number | null): string { } // Fail a task's turn-budget check once its visible turns exceed the budget by -// more than this fraction (> 1.5× expected_turns). +// more than this fraction (> 1.5× expected_tool_calls). export const TURN_BUDGET_TOLERANCE = 0.5; // Whether a task stayed within (1 + tolerance) × its expected-turns budget, // using the documented visible-turn count. Returns null when the task is not -// eligible: no visible-turn count, or no positive expected_turns budget. +// eligible: no visible-turn count, or no positive expected_tool_calls budget. export function withinTurnBudget( visibleTurns: number | null, expectedTurns: number | null, diff --git a/experiments/default.yaml b/experiments/default.yaml index ae364b978..43c54cbb8 100644 --- a/experiments/default.yaml +++ b/experiments/default.yaml @@ -23,10 +23,10 @@ defaults: # trips on runaway loops; task_timeout / turn_timeout below are the practical # guards. Override per-task when a task legitimately needs more. max_turns: 100 - # Soft target: when cumulative SDK turns across all iterations of a task - # exceed this, the orchestrator logs a one-shot warning and the report - # surfaces a badge. Does NOT abort the run (max_turns is the hard cap). - # expected_turns: 15 + # Soft target: when cumulative visible tool calls across all iterations of a + # task exceed this, the orchestrator logs a one-shot warning and the report + # surfaces a badge. Does NOT abort the run. + # expected_tool_calls: 15 # Maximum total seconds for the entire task evaluation (all iterations combined). task_timeout: 600 # Maximum seconds per agent turn / communicate call (null = no limit). diff --git a/plugins/coder-eval/skills/analyze/SKILL.md b/plugins/coder-eval/skills/analyze/SKILL.md index 56b365b32..667b601ac 100644 --- a/plugins/coder-eval/skills/analyze/SKILL.md +++ b/plugins/coder-eval/skills/analyze/SKILL.md @@ -50,7 +50,7 @@ summary per task with `jq` (or `python3` if `jq` is missing): ``` { task_id, final_status, weighted_score, duration_seconds, - iteration_count, model_used, max_turns_exhausted, + iteration_count, model_used, tool_calls_exhausted, total_cost_usd: .total_token_usage.total_cost_usd, total_tokens: (.total_token_usage.input_tokens + .total_token_usage.output_tokens), assistant_turns: .total_assistant_turns, @@ -255,7 +255,7 @@ Write the report to `/analysis.md`. ## Score Breakdown | Metric | Value | |---|---| -| Tasks run / succeeded / failed / ERROR / MAX_TURNS_EXHAUSTED | ... | +| Tasks run / succeeded / failed / ERROR / TOOL_CALLS_EXHAUSTED | ... | | Success rate | ...% | | Mean weighted score | ... ± std | | Total cost / tokens | $... / ... | diff --git a/src/coder_eval/agent.py b/src/coder_eval/agent.py index 739f21d3e..14d4fc31b 100644 --- a/src/coder_eval/agent.py +++ b/src/coder_eval/agent.py @@ -228,7 +228,7 @@ async def communicate( some SDKs swallow it. max_turns: Hard cap on inner-loop turns within this single ``communicate()`` call. When the agent would exceed it, the - returned ``TurnRecord`` has ``max_turns_exhausted=True``. + returned ``TurnRecord`` has ``tool_calls_exhausted=True``. None defers to the underlying SDK default. should_stop: Cooperative early-stop poll. An implementation with ``contract.cooperative_stop`` calls it at each safe message diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index be76208c6..98a6bcf8d 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -654,7 +654,7 @@ def _on_turn_timeout() -> None: if state.stopped_early_hit: status = AgentEndStatus.STOPPED_EARLY elif state.max_turns_hit: - status = AgentEndStatus.MAX_TURNS_EXHAUSTED + status = AgentEndStatus.TOOL_CALLS_EXHAUSTED else: status = AgentEndStatus.COMPLETED state.finalize(status, crashed=False, crash_reason=None) @@ -1064,7 +1064,6 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso num_turns=self._assistant_turns, crashed=crashed, crash_reason=crash_reason, - max_turns_exhausted=status is AgentEndStatus.MAX_TURNS_EXHAUSTED, duration_seconds=time.monotonic() - self.turn_start_time, # One basis with the window bounds — see the AgentStartEvent site. timestamp=self.clock.now(), diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index f02c02ff2..9cbacea2b 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -642,7 +642,7 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso or (self.max_turns is not None and self.num_turns is not None and self.num_turns > self.max_turns) ) if max_turns_exhausted and status == AgentEndStatus.COMPLETED: - status = AgentEndStatus.MAX_TURNS_EXHAUSTED + status = AgentEndStatus.TOOL_CALLS_EXHAUSTED self.log.warning("Agent exhausted max_turns (%s); turn ended without completing", self.max_turns) if self.current_turn_id is not None: @@ -676,7 +676,6 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso assistant_turn_count=self.assistant_turn_count, messages=list(self.sdk_messages), num_turns=self.num_turns, - max_turns_exhausted=max_turns_exhausted, result_summary=self.sdk_result_summary, crashed=crashed, crash_reason=crash_reason, diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 77f2d90ba..a7742c104 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -739,7 +739,6 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso num_turns=1, crashed=crashed, crash_reason=crash_reason, - max_turns_exhausted=status is AgentEndStatus.MAX_TURNS_EXHAUSTED, duration_seconds=time.monotonic() - self.turn_start_time, ) ) @@ -1009,7 +1008,7 @@ def _on_turn_timeout() -> None: if state.stopped_early_hit: status = AgentEndStatus.STOPPED_EARLY elif state.max_turns_hit: - status = AgentEndStatus.MAX_TURNS_EXHAUSTED + status = AgentEndStatus.TOOL_CALLS_EXHAUSTED else: status = AgentEndStatus.COMPLETED state.finalize(status, crashed=False, crash_reason=None) diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index 5eb935631..84123b41e 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -733,7 +733,6 @@ def finalize( assistant_turn_count=self.step_count, messages=list(self.messages), num_turns=self.step_count, - max_turns_exhausted=self.max_turns_exhausted, result_summary=ResultSummary( is_error=crashed, subtype=status.value, @@ -1269,7 +1268,7 @@ async def _settle_turn( if stopped_early: return AgentEndStatus.STOPPED_EARLY if state.max_turns_exhausted: - return AgentEndStatus.MAX_TURNS_EXHAUSTED + return AgentEndStatus.TOOL_CALLS_EXHAUSTED return AgentEndStatus.COMPLETED def _crash_turn( diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 2c89bbadd..7bfbecdb7 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -652,7 +652,6 @@ def finalize( assistant_turn_count=self.turn_count, messages=list(self.messages), num_turns=self.turn_count, - max_turns_exhausted=self.max_turns_exhausted, result_summary=ResultSummary( is_error=crashed, subtype=status.value, @@ -1119,7 +1118,7 @@ async def _settle_turn( if stopped_early: return AgentEndStatus.STOPPED_EARLY if state.max_turns_exhausted: - return AgentEndStatus.MAX_TURNS_EXHAUSTED + return AgentEndStatus.TOOL_CALLS_EXHAUSTED return AgentEndStatus.COMPLETED def _crash_turn( diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index adcbfa7e8..3324dd532 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -12,7 +12,7 @@ class FinalStatus(StrEnum): ERROR = "ERROR" BUILD_FAILED = "BUILD_FAILED" TIMEOUT = "TIMEOUT" - MAX_TURNS_EXHAUSTED = "MAX_TURNS_EXHAUSTED" + TOOL_CALLS_EXHAUSTED = "TOOL_CALLS_EXHAUSTED" TOKEN_BUDGET_EXCEEDED = "TOKEN_BUDGET_EXCEEDED" COST_BUDGET_EXCEEDED = "COST_BUDGET_EXCEEDED" # No verdict to report. Distinct from FAILURE (criteria checked, did not pass) @@ -54,7 +54,7 @@ def is_execution_fact(self) -> bool: # An environment fault, not a task outcome the agent could have avoided. FinalStatus.BUILD_FAILED: "error", FinalStatus.TIMEOUT: "failed", - FinalStatus.MAX_TURNS_EXHAUSTED: "failed", + FinalStatus.TOOL_CALLS_EXHAUSTED: "failed", FinalStatus.TOKEN_BUDGET_EXCEEDED: "failed", FinalStatus.COST_BUDGET_EXCEEDED: "failed", # A FOURTH category, not a fold into one of the three. Excluded from both the @@ -71,7 +71,7 @@ def is_execution_fact(self) -> bool: FinalStatus.ERROR: "!", FinalStatus.BUILD_FAILED: "B", FinalStatus.TIMEOUT: "T", - FinalStatus.MAX_TURNS_EXHAUSTED: "M", + FinalStatus.TOOL_CALLS_EXHAUSTED: "C", FinalStatus.TOKEN_BUDGET_EXCEEDED: "#", FinalStatus.COST_BUDGET_EXCEEDED: "$", FinalStatus.NOT_GRADED: "?", @@ -90,11 +90,11 @@ def is_execution_fact(self) -> bool: FinalStatus.ERROR: True, FinalStatus.BUILD_FAILED: True, FinalStatus.TIMEOUT: True, - # HAZARD: False, and it must stay False. MAX_TURNS_EXHAUSTED is SUBORDINATE to + # HAZARD: False, and it must stay False. TOOL_CALLS_EXHAUSTED is SUBORDINATE to # the verdict, not a fact that outranks it; the fact itself lives on - # `EvaluationResult.max_turns_exhausted`, which the seeding carries. + # `EvaluationResult.tool_calls_exhausted`, which the seeding carries. # Rationale: .claude/notes/orchestration.md § The terminal-status chain - FinalStatus.MAX_TURNS_EXHAUSTED: False, + FinalStatus.TOOL_CALLS_EXHAUSTED: False, FinalStatus.TOKEN_BUDGET_EXCEEDED: True, FinalStatus.COST_BUDGET_EXCEEDED: True, } diff --git a/src/coder_eval/models/limits.py b/src/coder_eval/models/limits.py index b06029f18..199b884e7 100644 --- a/src/coder_eval/models/limits.py +++ b/src/coder_eval/models/limits.py @@ -36,16 +36,13 @@ class RunLimits(BaseModel): gt=0, description="Max agent inner-loop turns per iteration. None = SDK default.", ) - expected_turns: int | None = Field( + expected_tool_calls: int | None = Field( default=None, ge=1, description=( - "Soft target for cumulative visible turns across a task. A 'turn' is one " - "entry in the Turn timeline: each tool call contributes 1, plus 1 for the " - "final reply when present. " - "When the running total exceeds this, the orchestrator logs a one-shot " - "warning and the report renders a badge — the run is NOT aborted " - "(use max_turns for a hard cap). None disables the check." + "Soft target for cumulative visible tool calls across a task (each resolved tool call " + "counts 1, plus 1 for the final reply when present). Exceeding it logs a one-shot warning " + "and badges the report; the run is NOT aborted (use max_turns for a hard cap)." ), ) task_timeout: int | None = Field( diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 11f1c3780..692a5e730 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -398,9 +398,9 @@ class TurnRecord(BaseModel): "ResultMessage (e.g. crash partial before the final message arrived)." ), ) - max_turns_exhausted: bool = Field( + tool_calls_exhausted: bool = Field( default=False, - description="Whether the agent hit the max_turns limit without voluntarily completing", + description="Whether the tool-call cap ended this turn before the agent completed on its own", ) result_summary: ResultSummary | None = Field( default=None, @@ -455,6 +455,7 @@ class SimulationTelemetry(BaseModel): "criteria_passed", "stop_token", "max_turns", + "tool_call_cap", "budget", "error", "run_limit_exceeded", @@ -597,9 +598,9 @@ class EvaluationResult(BaseModel): # Results final_status: FinalStatus = Field(description="Final status of the evaluation") - max_turns_exhausted: bool = Field( + tool_calls_exhausted: bool = Field( default=False, - description="Whether any iteration hit the agent max_turns limit without the agent voluntarily completing", + description="Whether the tool-call cap ended any iteration before the agent completed on its own", ) weighted_score: float | None = Field( default=None, ge=0.0, le=1.0, description="Weighted average of criterion scores (0.0 to 1.0)" diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index f1a6a3f0d..fa1bcf050 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -479,9 +479,9 @@ def __init__( # exactly once per task even if _check_run_limits fires every turn. self._cost_budget_skipped_logged: bool = False - # One-shot flag: emit the expected_turns rollup warning exactly once per - # task run even though _check_expected_turns is called after every turn. - self._expected_turns_warning_emitted: bool = False + # One-shot flag: emit the expected_tool_calls rollup warning exactly once per + # task run even though _check_expected_tool_calls is called after every turn. + self._expected_tool_calls_warning_emitted: bool = False # One-shot flag: a resolved task may be inspected more than once during # setup, but its ineffective timeout relationship should be logged once. @@ -507,7 +507,7 @@ def _terminal_status(self, success: bool) -> FinalStatus: """The status a normally-completed evaluation loop lands on. ORDER MATTERS at every step: a detached grade may not overturn an - execution fact, and the NOT_GRADED arm sits ABOVE ``max_turns_exhausted`` + execution fact, and the NOT_GRADED arm sits ABOVE ``tool_calls_exhausted`` so that ``execute`` + ``evaluate`` equals a single ``run``. With ``grade=True`` and no prior result the chain is the original one. @@ -526,8 +526,8 @@ def _terminal_status(self, success: bool) -> FinalStatus: return FinalStatus.SUCCESS if not self.grade: return FinalStatus.NOT_GRADED - if self.result.max_turns_exhausted: - return FinalStatus.MAX_TURNS_EXHAUSTED + if self.result.tool_calls_exhausted: + return FinalStatus.TOOL_CALLS_EXHAUSTED return FinalStatus.FAILURE async def run(self) -> EvaluationResult: @@ -709,7 +709,7 @@ def _kill_agent_subprocess_sync() -> None: await self._cleanup() # AFTER teardown, so post-run and cleanup errors land in the # report, but BEFORE finalization so task.json includes it. An - # ALLOWLIST: SUCCESS, MAX_TURNS_EXHAUSTED and NOT_GRADED all skip + # ALLOWLIST: SUCCESS, TOOL_CALLS_EXHAUSTED and NOT_GRADED all skip # it, none being a diagnosis of something going wrong. if self.result.final_status in { FinalStatus.ERROR, @@ -759,7 +759,7 @@ def _seed_from_prior_result(self) -> None: self.result.early_stop = prior.early_stop # Execution facts that outlive the agent process. - self.result.max_turns_exhausted = prior.max_turns_exhausted + self.result.tool_calls_exhausted = prior.tool_calls_exhausted self.result.error_message = prior.error_message self.result.error_details = prior.error_details self.result.error_log_tail = prior.error_log_tail @@ -1227,35 +1227,33 @@ def _check_run_limits(self, *, iteration: int) -> None: iteration=iteration, ) - def _check_expected_turns(self, *, iteration: int) -> None: - """Emit a one-shot warning if visible turns exceed expected_turns. + def _check_expected_tool_calls(self, *, iteration: int) -> None: + """Emit a one-shot warning if visible tool calls exceed expected_tool_calls. - Soft sibling of ``_check_run_limits.max_turns``: never aborts the run. - ``max_turns`` remains the hard cap (enforced inside the SDK). A - "turn" here is one timeline entry: each tool call plus the final - reply when present — the same metric evalboard renders. Cumulative - across iterations so simulation/dialog tasks compare against the - budget the user set. + Soft sibling of the hard tool-call cap: never aborts the run. The count is + one timeline entry per tool call plus the final reply when present — the + same metric evalboard renders. Cumulative across iterations so dialog tasks + compare against the budget the user set. """ if self.result is None: return limits = self.task.run_limits - if limits is None or limits.expected_turns is None: + if limits is None or limits.expected_tool_calls is None: return - if self._expected_turns_warning_emitted: + if self._expected_tool_calls_warning_emitted: return total = visible_turn_count(self.result) - if total > limits.expected_turns: + if total > limits.expected_tool_calls: logger.warning( - "Visible turns (%d) exceeded expected_turns (%d) at iteration %d " - + "for task %s. Run continues — max_turns remains the hard cap.", + "Visible tool calls (%d) exceeded expected_tool_calls (%d) at iteration %d " + + "for task %s. Run continues — this target never aborts.", total, - limits.expected_turns, + limits.expected_tool_calls, iteration, self.task.task_id, ) - self._expected_turns_warning_emitted = True + self._expected_tool_calls_warning_emitted = True def _warn_on_ineffective_task_timeout(self) -> None: """Log resolved cross-field run-limit warnings once per task run.""" @@ -2209,18 +2207,18 @@ async def _evaluation_loop(self) -> bool: # Facts about the RUN, recorded BEFORE the grading switch: `execute` # withholds the verdict, never the facts. Recording the fact is not - # finalizing on it — max_turns decides the status only when the criteria + # finalizing on it — the tool-call cap decides the status only when the criteria # fail, so under grade=False this is carried into task.json for the # detached grade rather than turned into a terminal status. # Rationale: .claude/notes/orchestration.md § The four grading sites - if turn_record.max_turns_exhausted: - self.result.max_turns_exhausted = True + if turn_record.tool_calls_exhausted: + self.result.tool_calls_exhausted = True logger.warning( "Agent exhausted max_turns (%s).", self.task.run_limits.max_turns if self.task.run_limits else None, ) # Soft cumulative-turn check (logs once; never aborts). - self._check_expected_turns(iteration=iteration) + self._check_expected_tool_calls(iteration=iteration) # Grading site 2 of 4. The trajectory is captured and persisted exactly as # on a graded run, but nothing is scored; returning False keeps FinalStatus @@ -2608,16 +2606,16 @@ async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir: stop_reason = stop_decision.reason break - # Soft check (logs once, never aborts). BEFORE the max_turns - # break, so a turn tripping both still emits the expected_turns - # warning before the dialog terminates. - self._check_expected_turns(iteration=turns_completed) + # Soft check (logs once, never aborts). BEFORE the cap break, so a + # turn tripping both still emits the expected_tool_calls warning + # before the dialog terminates. + self._check_expected_tool_calls(iteration=turns_completed) - if turn_record.max_turns_exhausted: - self.result.max_turns_exhausted = True - stop_reason = DialogStopReason.MAX_TURNS + if turn_record.tool_calls_exhausted: + self.result.tool_calls_exhausted = True + stop_reason = DialogStopReason.TOOL_CALL_CAP logger.warning( - "Agent exhausted its inner max_turns during simulation turn %s; ending dialog.", + "Agent reached the tool-call cap during simulation turn %s; ending dialog.", turns_completed, ) break diff --git a/src/coder_eval/reports/html.py b/src/coder_eval/reports/html.py index 6b1bbd6a3..ddb2de28f 100644 --- a/src/coder_eval/reports/html.py +++ b/src/coder_eval/reports/html.py @@ -24,7 +24,7 @@ from ..analysis import calculate_command_statistics from ..durations import format_ms from ..models import FinalStatus, eval_result_total_cost, sum_costs -from ..result_metrics import expected_turns_overage, turn_time_buckets +from ..result_metrics import expected_tool_calls_overage, turn_time_buckets from ..stats import stddev, welch_t_test from .helpers import ( collect_variant_series, @@ -349,11 +349,13 @@ def _render_header(result: EvaluationResult) -> str: task_cost = eval_result_total_cost(result) if task_cost is not None: cost_badge = f'${task_cost:.4f}' - expected_turns_badge = "" - overage = expected_turns_overage(result) + expected_tool_calls_badge = "" + overage = expected_tool_calls_overage(result) if overage is not None: actual, expected = overage - expected_turns_badge = f'expected_turns exceeded ({actual}/{expected})' + expected_tool_calls_badge = ( + f'expected_tool_calls exceeded ({actual}/{expected})' + ) early_stop_badge = "" if result.early_stop is not None: title = early_stop_gate_note(result.early_stop.reason.value) @@ -373,7 +375,7 @@ def _render_header(result: EvaluationResult) -> str: {agent_type} · {model} {_esc(duration)} {cost_badge} - {expected_turns_badge} + {expected_tool_calls_badge} {early_stop_badge} Toggle theme @@ -766,7 +768,7 @@ def _render_turn( f"" ) duration_label = f'{_esc(_format_duration(turn.duration_seconds))}' - exhausted = 'max_turns exhausted' if turn.max_turns_exhausted else "" + exhausted = 'tool-call cap reached' if turn.tool_calls_exhausted else "" response_block = ( f"""
@@ -1069,6 +1071,7 @@ def _render_installed_tools(result: EvaluationResult) -> str: "criteria_passed": ("success", "criteria passed"), "stop_token": ("neutral", "simulator ended dialog"), "max_turns": ("failure", "turn cap reached"), + "tool_call_cap": ("failure", "tool-call cap reached"), "budget": ("failure", "token budget exhausted"), "error": ("failure", "simulator error"), } diff --git a/src/coder_eval/reports/markdown.py b/src/coder_eval/reports/markdown.py index 05c3f8581..11860bcd6 100644 --- a/src/coder_eval/reports/markdown.py +++ b/src/coder_eval/reports/markdown.py @@ -515,7 +515,7 @@ def _task_details_lines(summary: RunSummary) -> list[str]: @staticmethod def _runtime_notes_lines(summary: RunSummary) -> list[str]: - """The ``## Run-time Notes`` blockquotes (max_turns exhaustion + expected_turns + """The ``## Run-time Notes`` blockquotes (tool-call cap + expected_tool_calls overage). Returns ``[]`` when there are no notes so the caller adds nothing — preserving the "only render the section when notes exist" behavior. """ @@ -524,9 +524,9 @@ def _runtime_notes_lines(summary: RunSummary) -> list[str]: notes: list[str] = [] for t in summary.task_results: task_id = t.get("task_id", "?") - if t.get("max_turns_exhausted"): - notes.append(f"> **WARNING:** [{task_id}] max_turns exhausted") - overage_field = t.get("expected_turns_overage") + if t.get("tool_calls_exhausted"): + notes.append(f"> **WARNING:** [{task_id}] tool-call cap reached") + overage_field = t.get("expected_tool_calls_overage") if ( isinstance(overage_field, (list, tuple)) and len(overage_field) == 2 @@ -534,7 +534,8 @@ def _runtime_notes_lines(summary: RunSummary) -> list[str]: ): actual, expected = overage_field notes.append( - f"> **WARNING:** [{task_id}] expected_turns exceeded: {actual}/{expected} (cumulative SDK turns)" + f"> **WARNING:** [{task_id}] expected_tool_calls exceeded: {actual}/{expected}" + + " (cumulative visible tool calls)" ) if t.get("stopped_early"): reason = t.get("early_stop_reason") or "unknown" diff --git a/src/coder_eval/resources/tags.yaml b/src/coder_eval/resources/tags.yaml index c76df4077..51682396d 100644 --- a/src/coder_eval/resources/tags.yaml +++ b/src/coder_eval/resources/tags.yaml @@ -52,4 +52,4 @@ tags: - name: max-turns-too-low definition: Task ran out of turns before completing; max_turns budget is below what the task realistically needs. examples: - - "MAX_TURNS_EXHAUSTED: max_turns=50 but 86 turns used" + - "TOOL_CALLS_EXHAUSTED: cap of 50 tool calls reached" diff --git a/src/coder_eval/result_metrics.py b/src/coder_eval/result_metrics.py index 10be401ec..fbc29fba4 100644 --- a/src/coder_eval/result_metrics.py +++ b/src/coder_eval/result_metrics.py @@ -153,12 +153,12 @@ def visible_turn_count(result: EvaluationResult) -> int: return commands + (1 if has_final_reply(result) else 0) -def expected_turns_overage(result: EvaluationResult) -> tuple[int, int] | None: +def expected_tool_calls_overage(result: EvaluationResult) -> tuple[int, int] | None: """Return ``(visible_turns, expected)`` when the visible-events turn - count strictly exceeds ``run_limits.expected_turns``; else ``None``. + count strictly exceeds ``run_limits.expected_tool_calls``; else ``None``. Safe against missing ``task_config``, missing ``run_limits``, and - non-int ``expected_turns`` values. + non-int ``expected_tool_calls`` values. """ task_cfg = result.task_config if task_cfg is None: @@ -166,7 +166,7 @@ def expected_turns_overage(result: EvaluationResult) -> tuple[int, int] | None: run_limits = (task_cfg.resolved or {}).get("run_limits") or {} if not isinstance(run_limits, dict): return None - expected = run_limits.get("expected_turns") + expected = run_limits.get("expected_tool_calls") if not isinstance(expected, int) or expected < 1: return None actual = visible_turn_count(result) diff --git a/src/coder_eval/run_record.py b/src/coder_eval/run_record.py index 8f99719df..4a118aefd 100644 --- a/src/coder_eval/run_record.py +++ b/src/coder_eval/run_record.py @@ -17,7 +17,7 @@ from coder_eval.errors import truncate_crash_message from coder_eval.models import EvaluationResult, FinalStatus, judge_cost_usd, simulator_cost_usd, sum_costs -from coder_eval.result_metrics import expected_turns_overage, turn_time_buckets, visible_turn_count +from coder_eval.result_metrics import expected_tool_calls_overage, turn_time_buckets, visible_turn_count from coder_eval.result_metrics import has_final_reply as _has_final_reply @@ -84,7 +84,7 @@ def eval_result_to_task_dict( ref_similarity = cr.score break - overage = expected_turns_overage(result) + overage = expected_tool_calls_overage(result) total_turns = sum((t.num_turns or 0) for t in result.iterations) @@ -97,13 +97,13 @@ def eval_result_to_task_dict( simulator_cost = simulator_cost_usd(result) row_total_cost = sum_costs(agent_cost, judge_cost, simulator_cost) - expected_turns_value: int | None = None + expected_tool_calls_value: int | None = None if result.task_config is not None: rl = (result.task_config.resolved or {}).get("run_limits") or {} if isinstance(rl, dict): - raw = rl.get("expected_turns") + raw = rl.get("expected_tool_calls") if isinstance(raw, int) and raw >= 1: - expected_turns_value = raw + expected_tool_calls_value = raw _buckets = turn_time_buckets(result) @@ -175,13 +175,13 @@ def eval_result_to_task_dict( "agent_config": (result.agent_config.model_dump() if result.agent_config else None), "sdk_options": result.sdk_options, "installed_tools": result.environment_info.get("installed_tools"), - "max_turns_exhausted": result.max_turns_exhausted, - "expected_turns_overage": list(overage) if overage is not None else None, + "tool_calls_exhausted": result.tool_calls_exhausted, + "expected_tool_calls_overage": list(overage) if overage is not None else None, "total_turns": total_turns, # "Visible turns" (tool calls + final reply) -- what the "within expected - # turns" metric compares against. Distinct from total_turns (SDK num_turns). + # tool calls" metric compares against. Distinct from total_turns (SDK num_turns). "visible_turns": visible_turn_count(result), - "expected_turns": expected_turns_value, + "expected_tool_calls": expected_tool_calls_value, "has_final_reply": has_reply, # None/False on the default path, so downstream analysis never confuses a # truncated run with a full one. diff --git a/src/coder_eval/simulation/termination.py b/src/coder_eval/simulation/termination.py index d5610bda4..d00284480 100644 --- a/src/coder_eval/simulation/termination.py +++ b/src/coder_eval/simulation/termination.py @@ -14,6 +14,7 @@ class DialogStopReason(StrEnum): CRITERIA_PASSED = "criteria_passed" STOP_TOKEN = "stop_token" MAX_TURNS = "max_turns" + TOOL_CALL_CAP = "tool_call_cap" BUDGET = "budget" ERROR = "error" RUN_LIMIT_EXCEEDED = "run_limit_exceeded" diff --git a/src/coder_eval/streaming/collector.py b/src/coder_eval/streaming/collector.py index 61b033006..10b8f1149 100644 --- a/src/coder_eval/streaming/collector.py +++ b/src/coder_eval/streaming/collector.py @@ -27,6 +27,7 @@ ) from coder_eval.streaming.events import ( AgentEndEvent, + AgentEndStatus, AgentStartEvent, StreamEvent, ToolEndEvent, @@ -238,7 +239,7 @@ def build_turn_record(self) -> TurnRecord: assistant_turn_count=end.assistant_turn_count, messages=messages, num_turns=end.num_turns, - max_turns_exhausted=end.max_turns_exhausted, + tool_calls_exhausted=end.status is AgentEndStatus.TOOL_CALLS_EXHAUSTED, result_summary=end.result_summary, crashed=end.crashed, crash_reason=end.crash_reason, diff --git a/src/coder_eval/streaming/events.py b/src/coder_eval/streaming/events.py index 505a4cddb..4d4f6c677 100644 --- a/src/coder_eval/streaming/events.py +++ b/src/coder_eval/streaming/events.py @@ -59,7 +59,7 @@ class TurnEndStatus(StrEnum): COMPLETED = "completed" CRASHED = "crashed" TIMEOUT = "timeout" - MAX_TURNS_EXHAUSTED = "max_turns_exhausted" + TOOL_CALLS_EXHAUSTED = "tool_calls_exhausted" STOPPED_EARLY = "stopped_early" # cooperative early-stop-on-criterion (clean, non-crash) @@ -69,7 +69,7 @@ class AgentEndStatus(StrEnum): COMPLETED = "completed" CRASHED = "crashed" TIMEOUT = "timeout" - MAX_TURNS_EXHAUSTED = "max_turns_exhausted" + TOOL_CALLS_EXHAUSTED = "tool_calls_exhausted" STOPPED_EARLY = "stopped_early" # cooperative early-stop-on-criterion (clean, non-crash) @@ -167,7 +167,6 @@ class AgentEndEvent(StreamEvent): assistant_turn_count: int = 0 messages: _MessageList = Field(default_factory=list) num_turns: int | None = None - max_turns_exhausted: bool = False result_summary: ResultSummary | None = None crashed: bool = False crash_reason: str | None = None diff --git a/src/coder_eval/streaming/renderers.py b/src/coder_eval/streaming/renderers.py index 0829488d3..743ab68d4 100644 --- a/src/coder_eval/streaming/renderers.py +++ b/src/coder_eval/streaming/renderers.py @@ -10,6 +10,7 @@ from coder_eval.models import ResultSummary from coder_eval.streaming.events import ( AgentEndEvent, + AgentEndStatus, AgentStartEvent, CriteriaCheckEvent, CriterionSummary, @@ -125,8 +126,8 @@ def _format_event(self, event: StreamEvent) -> str | None: f"[bold]--- Turn complete: {len(event.messages)} msgs, " f"{event.duration_seconds:.1f}s, {usage_str} ---[/bold]" ) - if event.max_turns_exhausted: - line += " [yellow](max_turns exhausted)[/yellow]" + if event.status is AgentEndStatus.TOOL_CALLS_EXHAUSTED: + line += " [yellow](tool-call cap reached)[/yellow]" if event.crashed and event.crash_reason: line += f"\n[red] reason: {escape(event.crash_reason)}[/red]" error_detail = _format_result_error(event.result_summary) @@ -232,8 +233,8 @@ def _format_event(self, event: StreamEvent) -> str | None: f"[{event.task_id}] --- Agent complete [{event.status.value}]: " f"{len(event.messages)} msgs, {event.duration_seconds:.1f}s, {usage_str} ---" ) - if event.max_turns_exhausted: - line += " (max_turns exhausted)" + if event.status is AgentEndStatus.TOOL_CALLS_EXHAUSTED: + line += " (tool-call cap reached)" if event.crashed and event.crash_reason: line += f"\n[{event.task_id}] reason: {event.crash_reason}" error_detail = _format_result_error(event.result_summary) diff --git a/tasks/fibonacci_with_template.yaml b/tasks/fibonacci_with_template.yaml index 49417d120..4a52d8405 100644 --- a/tasks/fibonacci_with_template.yaml +++ b/tasks/fibonacci_with_template.yaml @@ -3,7 +3,7 @@ description: "Implement fibonacci function with starter code template" tags: [golden, basic, pure-python, template] run_limits: - expected_turns: 3 + expected_tool_calls: 3 agent: type: claude-code diff --git a/tasks/hello_date.yaml b/tasks/hello_date.yaml index 667e8aad7..a896efb41 100644 --- a/tasks/hello_date.yaml +++ b/tasks/hello_date.yaml @@ -7,7 +7,7 @@ tags: [smoke, smoke-pass, golden, basic, pure-python] # warning + report badge when exceeded; does NOT abort the task or # affect scoring. run_limits: - expected_turns: 5 + expected_tool_calls: 5 agent: type: "claude-code" diff --git a/tasks/opencode_smoke_test.yaml b/tasks/opencode_smoke_test.yaml index 61b3877d2..b229c296a 100644 --- a/tasks/opencode_smoke_test.yaml +++ b/tasks/opencode_smoke_test.yaml @@ -7,7 +7,7 @@ initial_prompt: "Create a Python file named app.py in the current working direct tags: [smoke, basic, pure-python, opencode] run_limits: - expected_turns: 5 + expected_tool_calls: 5 # The CLI drives a full agent loop per invocation; give it room but keep the # smoke bounded so a hung provider fails fast instead of stalling a suite. task_timeout: 600 diff --git a/tasks/pi_smoke_test.yaml b/tasks/pi_smoke_test.yaml index ad5c42f26..78bd87ae1 100644 --- a/tasks/pi_smoke_test.yaml +++ b/tasks/pi_smoke_test.yaml @@ -8,7 +8,7 @@ initial_prompt: "Create a Python file named app.py in the current working direct tags: [smoke, basic, pure-python, pi] run_limits: - expected_turns: 5 + expected_tool_calls: 5 # The CLI drives a full agent loop per invocation; give it room but keep the # smoke bounded so a hung provider fails fast instead of stalling a suite. task_timeout: 600 diff --git a/tasks/run_limits/max_turns_cap.yaml b/tasks/run_limits/max_turns_cap.yaml index 4d236db17..dfecdd66d 100644 --- a/tasks/run_limits/max_turns_cap.yaml +++ b/tasks/run_limits/max_turns_cap.yaml @@ -4,7 +4,7 @@ description: >- sequential tool calls than the cap allows, so every harness must stop at the cap rather than running the prompt to completion. Run it with --type claude-code / codex / antigravity and compare: the cap must produce a CLEAN - stop (max_turns_exhausted, criteria still checked), never a crash. + stop (tool_calls_exhausted, criteria still checked), never a crash. tags: - run-limits diff --git a/tasks/smoke_negative_path.yaml b/tasks/smoke_negative_path.yaml index c7b6be488..c3ccd050e 100644 --- a/tasks/smoke_negative_path.yaml +++ b/tasks/smoke_negative_path.yaml @@ -13,7 +13,7 @@ tags: [smoke, smoke-fail] # AgentConfig has extra="forbid", so do NOT put max_turns under agent:. # Set to 2 (not 1) so a model that narrates before calling Write still gets a # chance to produce out.txt — keeps the failure shape "agent succeeded at the -# prompt but the unsatisfiable criterion fired" rather than MAX_TURNS_EXHAUSTED. +# prompt but the unsatisfiable criterion fired" rather than TOOL_CALLS_EXHAUSTED. run_limits: max_turns: 2 diff --git a/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json index b29372c32..af711bd03 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -50,6 +49,7 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json index e346caafb..9b697ba3d 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -81,6 +80,7 @@ "total_cost_usd": "", "uncached_input_tokens": 120 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json b/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json index 89bcd10fb..0ddd04e5f 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -90,6 +89,7 @@ "total_cost_usd": "", "uncached_input_tokens": 200 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json index f5bcc178f..48cb5dac5 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -70,6 +69,7 @@ "total_cost_usd": "", "uncached_input_tokens": 90 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json b/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json index e9ac990bf..8a5f82535 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -104,6 +103,7 @@ "total_cost_usd": "", "uncached_input_tokens": 330 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json index dd99b9f5b..89f2177cb 100644 --- a/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -55,6 +54,7 @@ "total_cost_usd": "", "uncached_input_tokens": 50 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json b/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json index 42b675546..cefa15988 100644 --- a/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json +++ b/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -77,6 +76,7 @@ "total_cost_usd": "", "uncached_input_tokens": 80 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json b/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json index 80de2a507..c4026d4b3 100644 --- a/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json +++ b/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 1000, @@ -131,6 +130,7 @@ "total_cost_usd": "", "uncached_input_tokens": 237 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json b/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json index e9fa5477f..d60a77b0f 100644 --- a/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json +++ b/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -104,6 +103,7 @@ "total_cost_usd": "", "uncached_input_tokens": 390 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json b/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json index 7b46dede4..eec1d515f 100644 --- a/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json +++ b/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 5, @@ -63,6 +62,7 @@ "total_cost_usd": "", "uncached_input_tokens": 500 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json index f5f8cd9b2..ab23e17ce 100644 --- a/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json @@ -29,7 +29,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -78,6 +77,7 @@ "total_cost_usd": "", "uncached_input_tokens": 60 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json b/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json index 69b9cfeee..57a1dd93b 100644 --- a/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json +++ b/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json @@ -8,13 +8,13 @@ "harness_startup_ms": null, "harness_teardown_ms": null, "iteration": 1, - "max_turns_exhausted": false, "messages": [], "model_used": null, "num_turns": null, "result_summary": null, "timestamp": "", "token_usage": null, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json b/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json index 8eee6edfb..c976d71b6 100644 --- a/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json +++ b/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json @@ -8,13 +8,13 @@ "harness_startup_ms": null, "harness_teardown_ms": null, "iteration": 1, - "max_turns_exhausted": false, "messages": [], "model_used": null, "num_turns": null, "result_summary": null, "timestamp": "", "token_usage": null, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json b/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json index 997a23868..b8b3d4049 100644 --- a/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json +++ b/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json @@ -8,13 +8,13 @@ "harness_startup_ms": null, "harness_teardown_ms": null, "iteration": 1, - "max_turns_exhausted": false, "messages": [], "model_used": null, "num_turns": null, "result_summary": null, "timestamp": "", "token_usage": null, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json b/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json index f44f93cec..312298e97 100644 --- a/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json +++ b/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -43,6 +42,7 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json b/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json index c755ee0c8..2b610394e 100644 --- a/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json +++ b/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -50,6 +49,7 @@ "total_cost_usd": "", "uncached_input_tokens": 92 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json b/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json index b57169b84..1d172affe 100644 --- a/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json +++ b/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -81,6 +80,7 @@ "total_cost_usd": "", "uncached_input_tokens": 120 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json b/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json index 77a89092f..3f76c93ac 100644 --- a/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json +++ b/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -77,6 +76,7 @@ "total_cost_usd": "", "uncached_input_tokens": 92 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json b/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json index e239e5391..8371a9a84 100644 --- a/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json +++ b/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -72,6 +71,7 @@ "total_cost_usd": "", "uncached_input_tokens": 90 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json b/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json index dcaaf846c..663e1e363 100644 --- a/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json +++ b/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -65,6 +64,7 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json b/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json index 3d8031b04..7ca68b570 100644 --- a/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json +++ b/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json @@ -49,7 +49,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -123,6 +122,7 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json b/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json index c73dd11b0..7a6f5df8b 100644 --- a/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json +++ b/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json @@ -8,7 +8,6 @@ "harness_startup_ms": null, "harness_teardown_ms": null, "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -43,6 +42,7 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json b/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json index b0414c980..c286da2b4 100644 --- a/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json +++ b/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -50,6 +49,7 @@ "total_cost_usd": "", "uncached_input_tokens": 92 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json index 7a6758156..594b2e72b 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -55,6 +54,7 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json index 7953c6f08..0b4faf28b 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 5, @@ -104,6 +103,7 @@ "total_cost_usd": "", "uncached_input_tokens": 95 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json b/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json index 01eeb86d0..c486f259a 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json +++ b/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -104,6 +103,7 @@ "total_cost_usd": "", "uncached_input_tokens": 150 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json index b791f9ee7..9a2d261f4 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -86,6 +85,7 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json b/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json index 66059b14a..65a062a57 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json +++ b/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -55,6 +54,7 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json index d794274ae..94ef4899d 100644 --- a/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -55,6 +54,7 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json index 06e3816e5..da349e69c 100644 --- a/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json @@ -48,7 +48,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -153,6 +152,7 @@ "total_cost_usd": "", "uncached_input_tokens": 997 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json b/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json index deebf4700..d70291ca0 100644 --- a/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json +++ b/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -104,6 +103,7 @@ "total_cost_usd": "", "uncached_input_tokens": 150 }, + "tool_calls_exhausted": false, "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json index c5e8d127e..04b07999e 100644 --- a/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json @@ -28,7 +28,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -86,6 +85,7 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json b/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json index f860de537..4bec82746 100644 --- a/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json +++ b/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -72,6 +71,7 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json b/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json index db88ceeca..46962409d 100644 --- a/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json +++ b/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json @@ -8,7 +8,6 @@ "harness_startup_ms": "", "harness_teardown_ms": "", "iteration": 1, - "max_turns_exhausted": false, "messages": [ { "cache_creation_tokens": 0, @@ -72,6 +71,7 @@ "total_cost_usd": "", "uncached_input_tokens": 110 }, + "tool_calls_exhausted": false, "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/report_snapshots/run_full.md b/tests/_fixtures/report_snapshots/run_full.md index e576cf416..8a67360e8 100644 --- a/tests/_fixtures/report_snapshots/run_full.md +++ b/tests/_fixtures/report_snapshots/run_full.md @@ -27,8 +27,8 @@ ## Run-time Notes -> **WARNING:** [alpha] max_turns exhausted -> **WARNING:** [beta] expected_turns exceeded: 10/5 (cumulative SDK turns) +> **WARNING:** [alpha] tool-call cap reached +> **WARNING:** [beta] expected_tool_calls exceeded: 10/5 (cumulative visible tool calls) ## Generation Metrics diff --git a/tests/harbor_e2e/fixtures/template_sources.yaml b/tests/harbor_e2e/fixtures/template_sources.yaml index 5ada088ef..4a1087a56 100644 --- a/tests/harbor_e2e/fixtures/template_sources.yaml +++ b/tests/harbor_e2e/fixtures/template_sources.yaml @@ -7,7 +7,7 @@ description: > environment/task.yaml's sandbox block being field-merged (not replaced). run_limits: - expected_turns: 3 + expected_tool_calls: 3 agent: type: "claude-code" diff --git a/tests/lint/rules/ce018_no_final_status_name_denylist.py b/tests/lint/rules/ce018_no_final_status_name_denylist.py index 1a3b4f962..b58cb6be4 100644 --- a/tests/lint/rules/ce018_no_final_status_name_denylist.py +++ b/tests/lint/rules/ce018_no_final_status_name_denylist.py @@ -32,7 +32,7 @@ "ERROR", "BUILD_FAILED", "TIMEOUT", - "MAX_TURNS_EXHAUSTED", + "TOOL_CALLS_EXHAUSTED", "TOKEN_BUDGET_EXCEEDED", "COST_BUDGET_EXCEEDED", "NOT_GRADED", diff --git a/tests/test_agent.py b/tests/test_agent.py index 2b6cd8f25..55c876dcb 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -1438,7 +1438,7 @@ async def mock_query(prompt, options, transport=None): partial = agent.pending_turn assert partial is not None assert partial.crashed is True - assert partial.max_turns_exhausted is False + assert partial.tool_calls_exhausted is False # No ResultMessage arrived before the crash, so num_turns is None. assert partial.num_turns is None # The Skill invocation that happened before the crash is preserved. @@ -1645,7 +1645,7 @@ async def test_claude_agent_error_max_turns_is_clean_completion_not_crash(): retryable (max_retries=2) and resume the same prompt that just burned its turn budget — pure waste. Instead the agent falls through to the success path so the orchestrator's existing - ``max_turns_exhausted`` handling can stop iterating. + ``tool_calls_exhausted`` handling can stop iterating. """ config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") agent = ClaudeCodeAgent(config) @@ -1684,7 +1684,7 @@ async def mock_query(prompt, options, transport=None): turn_record = await agent.communicate("solve something hard") assert turn_record.crashed is False - assert turn_record.max_turns_exhausted is True + assert turn_record.tool_calls_exhausted is True # Iteration counter advances normally on a clean turn (no rollback). assert agent._iteration == 1 # The ResultMessage details are still captured for diagnostics. @@ -1737,7 +1737,7 @@ async def mock_query(prompt, options, transport=None): turn_record = await agent.communicate("solve something hard") assert turn_record.crashed is False - assert turn_record.max_turns_exhausted is True + assert turn_record.tool_calls_exhausted is True assert agent._iteration == 1 assert turn_record.result_summary is not None assert turn_record.result_summary.subtype == "error_max_turns" diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index b0d53e327..ca8d2ed48 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -90,7 +90,7 @@ def test_build_run_summary_buckets_every_status_by_category() -> None: assert summary.tasks_cost_budget_exceeded == 1 # The two the hand-picked test omitted both classify as "failed". assert FinalStatus.COST_BUDGET_EXCEEDED.category == "failed" - assert FinalStatus.MAX_TURNS_EXHAUSTED.category == "failed" + assert FinalStatus.TOOL_CALLS_EXHAUSTED.category == "failed" def test_build_run_summary_threads_replicate_index() -> None: diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index d7a2d368f..9f3017a3c 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -1522,7 +1522,7 @@ async def test_max_turns_caps_visible_turns(): record = await agent.communicate("go", max_turns=2) assert len(record.commands) == 2 - assert record.max_turns_exhausted is True + assert record.tool_calls_exhausted is True async def test_max_turns_keeps_the_deciding_step_whole(): @@ -1542,7 +1542,7 @@ async def test_under_the_cap_completes_normally(): record = await agent.communicate("go", max_turns=5) assert len(record.commands) == 2 - assert record.max_turns_exhausted is False + assert record.tool_calls_exhausted is False async def test_no_max_turns_is_uncapped(): @@ -1552,7 +1552,7 @@ async def test_no_max_turns_is_uncapped(): record = await agent.communicate("go") assert len(record.commands) == 4 - assert record.max_turns_exhausted is False + assert record.tool_calls_exhausted is False async def test_cooperative_stop_outranks_the_cap(): @@ -1561,7 +1561,7 @@ async def test_cooperative_stop_outranks_the_cap(): record = await agent.communicate("go", max_turns=1, should_stop=lambda: True) - assert record.max_turns_exhausted is False + assert record.tool_calls_exhausted is False assert len(record.commands) == 1 @@ -1606,7 +1606,7 @@ async def test_cap_reached_on_a_poll_redrain_stops_polling(monkeypatch): record = await agent.communicate("go", max_turns=2) - assert record.max_turns_exhausted is True + assert record.tool_calls_exhausted is True # The cap counts RESOLVED calls. The still-open bg2 is force-closed and recorded # as unresolved rather than dropped, so the trajectory shows what was interrupted. resolved = [c for c in record.commands if c.result_status != "unknown"] diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index d71bcce23..5e111bbc8 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -2045,7 +2045,7 @@ async def test_cap_stops_the_pump_at_the_limit(self): record = await agent.communicate("go", max_turns=2) assert len(record.commands) == 2 - assert record.max_turns_exhausted is True + assert record.tool_calls_exhausted is True async def test_cap_keeps_the_deciding_call_complete(self): """Counting COMPLETED calls means the one that reaches the cap keeps its result.""" @@ -2070,7 +2070,7 @@ async def test_under_the_cap_completes_normally(self): record = await agent.communicate("go", max_turns=5) assert len(record.commands) == 2 - assert record.max_turns_exhausted is False + assert record.tool_calls_exhausted is False async def test_no_cap_consumes_the_whole_stream(self): """None must preserve the pre-existing behavior exactly.""" @@ -2079,7 +2079,7 @@ async def test_no_cap_consumes_the_whole_stream(self): record = await agent.communicate("go") assert len(record.commands) == 4 - assert record.max_turns_exhausted is False + assert record.tool_calls_exhausted is False async def test_cooperative_stop_outranks_the_cap(self): """Both firing on the same notification reports STOPPED_EARLY.""" @@ -2087,7 +2087,7 @@ async def test_cooperative_stop_outranks_the_cap(self): record = await agent.communicate("go", max_turns=1, should_stop=lambda: True) - assert record.max_turns_exhausted is False + assert record.tool_calls_exhausted is False async def test_capped_turn_still_folds_sub_agent_tokens(self, monkeypatch, tmp_path): """A capped turn must not lose the child threads' spend. @@ -2124,7 +2124,7 @@ async def test_capped_turn_still_folds_sub_agent_tokens(self, monkeypatch, tmp_p record = await agent.communicate("delegate it", max_turns=2) - assert record.max_turns_exhausted is True + assert record.tool_calls_exhausted is True # The child's inner shell command was recovered despite the cap... assert [c for c in record.commands if c.tool_name == "Bash"] # ...and its generation nests under the spawn, carrying its own tokens... diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 5c94214c2..7c684cd71 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -316,7 +316,7 @@ def _run(src: str): '"ERROR" == s', # reversed 's in ("FAILURE", "TIMEOUT")', 's not in ("TOKEN_BUDGET_EXCEEDED", "COST_BUDGET_EXCEEDED")', - 's in ("succeeded", "MAX_TURNS_EXHAUSTED")', # tuple mixes a member name in + 's in ("succeeded", "TOOL_CALLS_EXHAUSTED")', # tuple mixes a member name in 's in ["SUCCESS", "FAILURE"]', # list literal 's in {"ERROR"}', # set literal ], diff --git a/tests/test_detached_grading_boundaries.py b/tests/test_detached_grading_boundaries.py index 8f8b327eb..a7d6d706e 100644 --- a/tests/test_detached_grading_boundaries.py +++ b/tests/test_detached_grading_boundaries.py @@ -161,13 +161,13 @@ def test_an_execution_fact_carrying_a_verdict_is_still_refused(self, tmp_path: P score, so `graded_anyway` was `False` in all four tests — replacing that whole expression with a literal `False` left the suite fully green, i.e. the defect it exists for could be reintroduced silently. A stale image - returning a fully graded MAX_TURNS_EXHAUSTED row is exactly the case the + returning a fully graded TOOL_CALLS_EXHAUSTED row is exactly the case the exemption must NOT cover: a fresh image reports that status with no verdict attached. """ from coder_eval.isolation.docker_runner import DockerRunError - graded = _result(FinalStatus.MAX_TURNS_EXHAUSTED) + graded = _result(FinalStatus.TOOL_CALLS_EXHAUSTED) graded.weighted_score = 1.0 graded.success_criteria_results = [ CriterionResult(criterion_type="file_exists", description="x", score=1.0, weight=1.0) diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index 9755120a5..89d9cf872 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -211,6 +211,8 @@ class TestFullFieldParity: "token_usage", "timestamp", "provider_call_costs", + # Derived from end.status: the end status is the one source of truth. + "tool_calls_exhausted", # Measured by the collector between the agent's own start/end event # stamps and the first/last generation window — not carried on # AgentEndEvent, because no agent computes them. @@ -241,7 +243,6 @@ def _full_agent_end(self) -> AgentEndEvent: assistant_turn_count=3, messages=[msg], num_turns=3, - max_turns_exhausted=True, result_summary=ResultSummary(is_error=False, subtype="success", result="all done"), crashed=True, crash_reason="boom", @@ -285,6 +286,22 @@ def test_every_verbatim_field_round_trips(self): else: assert record_value == event_value, f"{name}: record={record_value!r} event={event_value!r}" + @pytest.mark.parametrize( + ("status", "exhausted"), + [ + (AgentEndStatus.TOOL_CALLS_EXHAUSTED, True), + (AgentEndStatus.COMPLETED, False), + (AgentEndStatus.STOPPED_EARLY, False), + ], + ) + def test_tool_calls_exhausted_is_derived_from_the_end_status(self, status, exhausted): + collector = EventCollector() + _feed( + collector, + [AgentStartEvent(task_id=TASK_ID, prompt="p", iteration=1), AgentEndEvent(task_id=TASK_ID, status=status)], + ) + assert collector.build_turn_record().tool_calls_exhausted is exhausted + _GEN_BASE = datetime(2026, 9, 11, 9, 0, 0) _GEN_WINDOW_MS = 1.0 diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py index 88a8f59ef..933341ad2 100644 --- a/tests/test_execute_evaluate_loop.py +++ b/tests/test_execute_evaluate_loop.py @@ -419,12 +419,12 @@ def test_grading_the_same_run_twice_reaches_the_same_verdict(tmp_path: Path) -> # -------------------------------------------------------------------------- -def test_execute_records_max_turns_exhausted_exactly_as_run_does(tmp_path: Path) -> None: - """`max_turns_exhausted` is a fact about the RUN, not a verdict. +def test_execute_records_tool_calls_exhausted_exactly_as_run_does(tmp_path: Path) -> None: + """`tool_calls_exhausted` is a fact about the RUN, not a verdict. It used to be captured AFTER the grading switch's early return, so under `execute` it was never recorded at all: the row finalized NOT_GRADED and the - command exited 0 where `run` reported MAX_TURNS_EXHAUSTED and exited 1 — for + command exited 0 where `run` reported TOOL_CALLS_EXHAUSTED and exited 1 — for identical agent output. `_seed_from_prior_result` cannot restore a fact the execute phase never captured, so a later `evaluate` inherited the wrong terminal status too. @@ -436,7 +436,7 @@ def test_execute_records_max_turns_exhausted_exactly_as_run_does(tmp_path: Path) def _exhausted(self, *args: Any, **kwargs: Any): record = original(self, *args, **kwargs) - record.max_turns_exhausted = True + record.tool_calls_exhausted = True return record def _run(command: str, run_dir: Path) -> Any: @@ -451,15 +451,15 @@ def _run(command: str, run_dir: Path) -> Any: _run("execute", executed_dir) executed = _row(_task_dir(executed_dir)) - assert graded["max_turns_exhausted"] is True, "the fixture must actually exhaust turns under `run`" - assert executed["max_turns_exhausted"] is True, ( + assert graded["tool_calls_exhausted"] is True, "the fixture must actually exhaust turns under `run`" + assert executed["tool_calls_exhausted"] is True, ( "`execute` dropped a fact about the run. Only the verdict is withheld." ) # The FACT is recorded; the STATUS is not decided. `run` returns SUCCESS for # a max-turns trajectory whose criteria pass and only falls through to - # MAX_TURNS_EXHAUSTED when they fail — so the status is not knowable without + # TOOL_CALLS_EXHAUSTED when they fail — so the status is not knowable without # grading, and claiming it here made it both terminal and permanent - # (MAX_TURNS_EXHAUSTED is an execution fact, which the detached grade may + # (TOOL_CALLS_EXHAUSTED is an execution fact, which the detached grade may # never overturn). assert executed["final_status"] == FinalStatus.NOT_GRADED.value @@ -470,7 +470,7 @@ def _run(command: str, run_dir: Path) -> Any: assert regraded["final_status"] == graded["final_status"] assert regraded["weighted_score"] == graded["weighted_score"] - assert regraded["max_turns_exhausted"] is True, "the fact must survive the grade too" + assert regraded["tool_calls_exhausted"] is True, "the fact must survive the grade too" def test_a_detached_grade_keeps_the_runs_api_routing_not_the_graders(tmp_path: Path) -> None: diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index a6e72271b..4d0ea0bec 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -273,7 +273,7 @@ def test_experiment_report_variant_summary_shows_errors(self): assert "**Errors**: 1" in md def test_task_detail_table_shows_timeout_icon(self): - """TIMEOUT and MAX_TURNS_EXHAUSTED should get distinct icons, not '?'.""" + """TIMEOUT and TOOL_CALLS_EXHAUSTED should get distinct icons, not '?'.""" result = ExperimentResult( experiment_id="icon-test", description="Icon test", @@ -300,7 +300,7 @@ def test_task_detail_table_shows_timeout_icon(self): variant_id="v", task_id="t-exhausted", weighted_score=0.0, - final_status="MAX_TURNS_EXHAUSTED", + final_status="TOOL_CALLS_EXHAUSTED", duration_seconds=30.0, ), ], @@ -322,7 +322,7 @@ def test_task_detail_table_shows_timeout_icon(self): total_duration_seconds=90.0, ) md = ExperimentReportGenerator.generate_experiment_report(result) - # TIMEOUT and MAX_TURNS_EXHAUSTED should NOT show "?" — they should have real icons + # TIMEOUT and TOOL_CALLS_EXHAUSTED should NOT show "?" — they should have real icons assert "(?)" not in md def test_generate_task_summary_json(self, sample_result): diff --git a/tests/test_lint_no_top_level_run_limits.py b/tests/test_lint_no_top_level_run_limits.py index 6b40ecaab..c3d37a354 100644 --- a/tests/test_lint_no_top_level_run_limits.py +++ b/tests/test_lint_no_top_level_run_limits.py @@ -66,9 +66,9 @@ def test_does_not_flag_run_limits_max_turns_anywhere() -> None: assert not _violations("x = rl.task_timeout") -def test_does_not_flag_turn_max_turns_exhausted() -> None: - """`max_turns_exhausted` is a distinct TurnRecord field — must not collide.""" - assert not _violations("x = turn.max_turns_exhausted") +def test_does_not_flag_turn_tool_calls_exhausted() -> None: + """`tool_calls_exhausted` is a distinct TurnRecord field — must not collide.""" + assert not _violations("x = turn.tool_calls_exhausted") def test_synthetic_regression_in_orchestrator_fires() -> None: diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index d1604441a..75e1e173d 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -1468,15 +1468,15 @@ async def test_should_stop_ends_turn_cleanly(self, patch_exec, tmp_path): async def test_max_turns_marks_exhausted(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) record = await _run(_agent(), tmp_path, max_turns=1) - assert record.max_turns_exhausted is True + assert record.tool_calls_exhausted is True async def test_a_cap_the_run_stays_under_is_not_exhausted(self, patch_exec, tmp_path): """The OTHER direction, which decides `FinalStatus`. HAPPY_STREAM is exactly 2 steps, so `max_turns=2` is the boundary: an off-by-one here (`>` becoming `>=`, or counting finished steps instead of - started ones) reports MAX_TURNS_EXHAUSTED — orchestrator.py turns the flag - straight into `FinalStatus.MAX_TURNS_EXHAUSTED` — for a run that finished + started ones) reports TOOL_CALLS_EXHAUSTED — orchestrator.py turns the flag + straight into `FinalStatus.TOOL_CALLS_EXHAUSTED` — for a run that finished well inside its budget. A spurious exhaustion also suppresses the non-zero- exit and zero-telemetry crash guards, which are both conditioned on it, so the run would score silently instead of failing loudly. @@ -1484,7 +1484,7 @@ async def test_a_cap_the_run_stays_under_is_not_exhausted(self, patch_exec, tmp_ patch_exec(_FakeProcess(HAPPY_STREAM)) record = await _run(_agent(), tmp_path, max_turns=2) - assert record.max_turns_exhausted is False + assert record.tool_calls_exhausted is False assert record.assistant_turn_count == 2 # Both steps' telemetry is present — the cap did not truncate the stream. assert record.token_usage is not None @@ -1493,7 +1493,7 @@ async def test_a_cap_the_run_stays_under_is_not_exhausted(self, patch_exec, tmp_ async def test_no_cap_is_uncapped(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) record = await _run(_agent(), tmp_path) - assert record.max_turns_exhausted is False + assert record.tool_calls_exhausted is False assert record.assistant_turn_count == 2 async def test_the_deciding_step_is_kept_whole(self, patch_exec, tmp_path): @@ -1506,7 +1506,7 @@ async def test_the_deciding_step_is_kept_whole(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) record = await _run(_agent(), tmp_path, max_turns=1) - assert record.max_turns_exhausted is True + assert record.tool_calls_exhausted is True assert len(record.commands) == 1 # step 1's tool call usage = record.token_usage assert usage is not None @@ -1522,7 +1522,7 @@ async def test_the_step_past_the_cap_is_never_admitted(self, patch_exec, tmp_pat recorder = _EventRecorder() record = await _run(_agent(), tmp_path, max_turns=1, stream_callback=recorder) - assert record.max_turns_exhausted is True + assert record.tool_calls_exhausted is True assert record.assistant_turn_count == 1 assert len([e for e in recorder.events if isinstance(e, TurnStartEvent)]) == 1 assert len([e for e in recorder.events if isinstance(e, TurnEndEvent)]) == 1 diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 00a4c79f9..fe823be68 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -1762,9 +1762,9 @@ async def test_overrides_apply_max_turns_field_merge(tmp_path): from coder_eval.orchestration.overrides import apply_overrides task, _ = load_task(Path("tasks/hello_date.yaml")) - # hello_date.yaml ships a baseline run_limits.expected_turns; max_turns is + # hello_date.yaml ships a baseline run_limits.expected_tool_calls; max_turns is # the field this test exercises. The override must field-merge on top. - baseline_expected_turns = task.run_limits.expected_turns if task.run_limits else None + baseline_expected_tool_calls = task.run_limits.expected_tool_calls if task.run_limits else None assert task.run_limits is None or task.run_limits.max_turns is None apply_overrides(task, {"run_limits.max_turns": 42}) @@ -1772,7 +1772,7 @@ async def test_overrides_apply_max_turns_field_merge(tmp_path): assert task.run_limits is not None assert task.run_limits.max_turns == 42 # Field-merge must preserve other run_limits keys from the task YAML. - assert task.run_limits.expected_turns == baseline_expected_turns + assert task.run_limits.expected_tool_calls == baseline_expected_tool_calls # ==================== Duplicate Task ID Validation Tests ==================== @@ -1829,7 +1829,7 @@ def test_resolve_all_tasks_rejects_duplicate_task_ids(tmp_path): @pytest.mark.asyncio -async def test_evaluation_loop_breaks_on_max_turns_exhausted(tmp_path): +async def test_evaluation_loop_breaks_on_tool_calls_exhausted(tmp_path): """Orchestrator stops iterating when the agent exhausts max_turns without passing criteria.""" from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch @@ -1877,13 +1877,13 @@ async def test_evaluation_loop_breaks_on_max_turns_exhausted(tmp_path): environment_info={}, ) - # Agent returns a turn record with max_turns_exhausted=True + # Agent returns a turn record with tool_calls_exhausted=True exhausted_turn = TurnRecord( iteration=1, user_input="test prompt", agent_output="I ran out of turns", duration_seconds=5.0, - max_turns_exhausted=True, + tool_calls_exhausted=True, ) mock_agent = AsyncMock() mock_agent.communicate = AsyncMock(return_value=exhausted_turn) @@ -1911,8 +1911,8 @@ async def test_evaluation_loop_breaks_on_max_turns_exhausted(tmp_path): assert orchestrator.result.iteration_count == 1 # Agent communicate should have been called only once assert mock_agent.communicate.call_count == 1 - # max_turns_exhausted should be propagated to the result - assert orchestrator.result.max_turns_exhausted is True + # tool_calls_exhausted should be propagated to the result + assert orchestrator.result.tool_calls_exhausted is True @pytest.mark.asyncio diff --git a/tests/test_orchestrator_error_log_tail.py b/tests/test_orchestrator_error_log_tail.py index b1c18f913..8928d7719 100644 --- a/tests/test_orchestrator_error_log_tail.py +++ b/tests/test_orchestrator_error_log_tail.py @@ -136,12 +136,12 @@ async def fake_loop() -> bool: @pytest.mark.asyncio -async def test_error_log_tail_none_on_max_turns_exhausted(tmp_path): +async def test_error_log_tail_none_on_tool_calls_exhausted(tmp_path): orch = _build_orchestrator(tmp_path) async def fake_loop() -> bool: assert orch.result is not None - orch.result.max_turns_exhausted = True + orch.result.tool_calls_exhausted = True return False with ( @@ -153,7 +153,7 @@ async def fake_loop() -> bool: ): result = await orch.run() - assert result.final_status == FinalStatus.MAX_TURNS_EXHAUSTED + assert result.final_status == FinalStatus.TOOL_CALLS_EXHAUSTED assert result.error_log_tail is None diff --git a/tests/test_orchestrator_telemetry.py b/tests/test_orchestrator_telemetry.py index 623adcf72..e7899aefc 100644 --- a/tests/test_orchestrator_telemetry.py +++ b/tests/test_orchestrator_telemetry.py @@ -95,7 +95,7 @@ def test_success_emits_task_end(tmp_path): [ (FinalStatus.SUCCESS, "succeeded"), (FinalStatus.FAILURE, "failed"), - (FinalStatus.MAX_TURNS_EXHAUSTED, "failed"), + (FinalStatus.TOOL_CALLS_EXHAUSTED, "failed"), (FinalStatus.ERROR, "error"), (FinalStatus.TIMEOUT, "failed"), (FinalStatus.TOKEN_BUDGET_EXCEEDED, "failed"), diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 391e5048d..052fa0fd1 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -464,13 +464,13 @@ class TestMaxTurns: async def test_max_turns_marks_exhausted(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) record = await _run(_agent(), tmp_path, max_turns=1) - assert record.max_turns_exhausted is True + assert record.tool_calls_exhausted is True async def test_a_cap_the_run_stays_under_is_not_exhausted(self, patch_exec, tmp_path): """The fixture is exactly 3 turns, so max_turns=3 is the boundary.""" patch_exec(_FakeProcess(HAPPY_STREAM)) record = await _run(_agent(), tmp_path, max_turns=3) - assert record.max_turns_exhausted is False + assert record.tool_calls_exhausted is False assert record.assistant_turn_count == 3 async def test_the_deciding_turn_is_kept_whole(self, patch_exec, tmp_path): @@ -478,7 +478,7 @@ async def test_the_deciding_turn_is_kept_whole(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) record = await _run(_agent(), tmp_path, max_turns=1) - assert record.max_turns_exhausted is True + assert record.tool_calls_exhausted is True assert len(record.commands) == 1 # turn 1's write usage = record.token_usage assert usage is not None @@ -491,7 +491,7 @@ async def test_the_turn_past_the_cap_is_never_admitted(self, patch_exec, tmp_pat recorder = _EventRecorder() record = await _run(_agent(), tmp_path, max_turns=1, stream_callback=recorder) - assert record.max_turns_exhausted is True + assert record.tool_calls_exhausted is True assert record.assistant_turn_count == 1 starts = [e for e in recorder.events if isinstance(e, TurnStartEvent)] ends = [e for e in recorder.events if isinstance(e, TurnEndEvent)] @@ -501,7 +501,7 @@ async def test_the_turn_past_the_cap_is_never_admitted(self, patch_exec, tmp_pat async def test_no_cap_is_uncapped(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) record = await _run(_agent(), tmp_path) - assert record.max_turns_exhausted is False + assert record.tool_calls_exhausted is False assert record.assistant_turn_count == 3 @@ -851,7 +851,7 @@ async def test_terminal_error_crashes_the_turn(self, patch_exec, tmp_path): async def test_max_turns_cut_after_an_error_turn_finalizes_cleanly(self, patch_exec, tmp_path): """A max_turns cut landing right after an error turn_end (pi still retrying, so error_message is set but not yet cleared) must finalize as - max_turns_exhausted — NOT crash on the stale error. Guards the documented + tool_calls_exhausted — NOT crash on the stale error. Guards the documented 'no crash, no retry' contract; without the intentional-cut gate the error arm would fire on a clean budget exhaustion.""" # turn 1 errors; turn 2's turn_start trips max_turns=1 before any clean @@ -859,7 +859,7 @@ async def test_max_turns_cut_after_an_error_turn_finalizes_cleanly(self, patch_e stream = [_turn_start(), _turn_end_error("transient 429"), _turn_start(), _turn_end(inp=1, out=1)] patch_exec(_FakeProcess(stream)) record = await _run(_agent(), tmp_path, max_turns=1) - assert record.max_turns_exhausted is True + assert record.tool_calls_exhausted is True assert record.crashed is False def test_error_message_resets_on_a_recovered_turn(self): diff --git a/tests/test_reports.py b/tests/test_reports.py index 18e9cb47d..74d5decfe 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -52,7 +52,7 @@ def test_generate_markdown_snapshot_full(): Triggers: multiple distinct models (Models line), all P0 metrics (scores, latency, assistant turns, crashed partials, ground-truth similarity), all four dynamic table columns (model/tags/similarity/cmd-efficiency incl. expected/actual), - run-time notes (max_turns_exhausted + expected_turns_overage), generation metrics, + run-time notes (tool_calls_exhausted + expected_tool_calls_overage), generation metrics, token usage (incl. cache + cost), agent settings, installed tools, and a non-empty environment block. All inputs are fixed values so the output is deterministic. """ @@ -79,7 +79,7 @@ def test_generate_markdown_snapshot_full(): "commands_efficiency": 0.75, "expected_commands": 4, "actual_commands": 6, - "max_turns_exhausted": True, + "tool_calls_exhausted": True, "iterations": [ {"iteration": 1, "crashed": False, "assistant_turn_count": 3, "duration_seconds": 4.2}, ], @@ -104,7 +104,7 @@ def test_generate_markdown_snapshot_full(): "commands_efficiency": 1.0, "expected_commands": 3, "actual_commands": 3, - "expected_turns_overage": [10, 5], + "expected_tool_calls_overage": [10, 5], "iterations": [ {"iteration": 1, "crashed": True, "assistant_turn_count": 1, "duration_seconds": 2.0}, ], @@ -1245,12 +1245,12 @@ def test_markdown_summary_crashed_partials_breakdown(task_turns, expected_line): def _summary_with_notes( *, - max_turns_exhausted: bool = False, - expected_turns_overage: list[int] | None = None, + tool_calls_exhausted: bool = False, + expected_tool_calls_overage: list[int] | None = None, ) -> RunSummary: task = _make_task_result("t1", "SUCCESS", 1.0, 10.0) - task["max_turns_exhausted"] = max_turns_exhausted - task["expected_turns_overage"] = expected_turns_overage + task["tool_calls_exhausted"] = tool_calls_exhausted + task["expected_tool_calls_overage"] = expected_tool_calls_overage return RunSummary( run_id="r", start_time=datetime(2026, 5, 21, 12, 0, 0), @@ -1266,46 +1266,46 @@ def _summary_with_notes( ) -def test_generate_markdown_renders_expected_turns_marker_when_exceeded(): - summary = _summary_with_notes(expected_turns_overage=[7, 5]) +def test_generate_markdown_renders_expected_tool_calls_marker_when_exceeded(): + summary = _summary_with_notes(expected_tool_calls_overage=[7, 5]) report_md = ReportGenerator.generate_markdown(summary) assert "## Run-time Notes" in report_md - assert "expected_turns exceeded" in report_md + assert "expected_tool_calls exceeded" in report_md assert "7/5" in report_md -def test_generate_markdown_no_expected_turns_marker_when_under(): +def test_generate_markdown_no_expected_tool_calls_marker_when_under(): # Under-budget: the dict carries no overage field — emit nothing. - summary = _summary_with_notes(expected_turns_overage=None) + summary = _summary_with_notes(expected_tool_calls_overage=None) report_md = ReportGenerator.generate_markdown(summary) - assert "expected_turns exceeded" not in report_md + assert "expected_tool_calls exceeded" not in report_md -def test_generate_markdown_no_expected_turns_marker_when_unset(): +def test_generate_markdown_no_expected_tool_calls_marker_when_unset(): summary = _summary_with_notes() report_md = ReportGenerator.generate_markdown(summary) - assert "expected_turns exceeded" not in report_md + assert "expected_tool_calls exceeded" not in report_md assert "## Run-time Notes" not in report_md -def test_generate_markdown_renders_max_turns_exhausted_marker(): - summary = _summary_with_notes(max_turns_exhausted=True) +def test_generate_markdown_renders_tool_calls_exhausted_marker(): + summary = _summary_with_notes(tool_calls_exhausted=True) report_md = ReportGenerator.generate_markdown(summary) assert "## Run-time Notes" in report_md - assert "max_turns exhausted" in report_md + assert "tool-call cap reached" in report_md def test_generate_markdown_no_max_turns_marker_when_not_exhausted(): - summary = _summary_with_notes(max_turns_exhausted=False) + summary = _summary_with_notes(tool_calls_exhausted=False) report_md = ReportGenerator.generate_markdown(summary) - assert "max_turns exhausted" not in report_md + assert "tool-call cap reached" not in report_md def test_generate_markdown_renders_both_markers_when_both_fire(): - summary = _summary_with_notes(max_turns_exhausted=True, expected_turns_overage=[7, 5]) + summary = _summary_with_notes(tool_calls_exhausted=True, expected_tool_calls_overage=[7, 5]) report_md = ReportGenerator.generate_markdown(summary) - assert "max_turns exhausted" in report_md - assert "expected_turns exceeded" in report_md + assert "tool-call cap reached" in report_md + assert "expected_tool_calls exceeded" in report_md assert "7/5" in report_md diff --git a/tests/test_reports_html.py b/tests/test_reports_html.py index 7afcd31d0..d6f576ecb 100644 --- a/tests/test_reports_html.py +++ b/tests/test_reports_html.py @@ -892,14 +892,14 @@ def test_task_html_renders_cost_badge_in_header(): assert "$0.5000" in header -def _result_with_expected_turns( +def _result_with_expected_tool_calls( resolved_run_limits: dict | None, *, commands_per_turn: list[int] | None = None, final_reply: str | None = None, task_config: bool = True, ) -> EvaluationResult: - """Build an EvaluationResult that exercises expected_turns_overage. + """Build an EvaluationResult that exercises expected_tool_calls_overage. Visible turns = sum(commands_per_turn) + (1 if final_reply else 0). """ @@ -938,39 +938,39 @@ def _result_with_expected_turns( return result -def test_task_html_renders_expected_turns_badge_when_exceeded(): +def test_task_html_renders_expected_tool_calls_badge_when_exceeded(): # 6 tools + reply = 7 visible turns; budget 5 → 7/5 overage. - result = _result_with_expected_turns( - {"expected_turns": 5}, + result = _result_with_expected_tool_calls( + {"expected_tool_calls": 5}, commands_per_turn=[2, 3, 1], final_reply="done", ) html = HTMLReportGenerator.generate_task_html(result) - assert "expected_turns exceeded" in html + assert "expected_tool_calls exceeded" in html assert "7/5" in html -def test_task_html_no_expected_turns_badge_when_under(): +def test_task_html_no_expected_tool_calls_badge_when_under(): # 7 visible turns under budget 10 → no badge. - result = _result_with_expected_turns( - {"expected_turns": 10}, + result = _result_with_expected_tool_calls( + {"expected_tool_calls": 10}, commands_per_turn=[2, 3, 1], final_reply="done", ) html = HTMLReportGenerator.generate_task_html(result) - assert "expected_turns exceeded" not in html + assert "expected_tool_calls exceeded" not in html -def test_task_html_no_expected_turns_badge_when_unset(): - result = _result_with_expected_turns({"max_turns": 10}, commands_per_turn=[2, 3, 2]) +def test_task_html_no_expected_tool_calls_badge_when_unset(): + result = _result_with_expected_tool_calls({"max_turns": 10}, commands_per_turn=[2, 3, 2]) html = HTMLReportGenerator.generate_task_html(result) - assert "expected_turns exceeded" not in html + assert "expected_tool_calls exceeded" not in html -def test_task_html_no_expected_turns_badge_when_task_config_none(): - result = _result_with_expected_turns(None, commands_per_turn=[2, 3, 2], task_config=False) +def test_task_html_no_expected_tool_calls_badge_when_task_config_none(): + result = _result_with_expected_tool_calls(None, commands_per_turn=[2, 3, 2], task_config=False) html = HTMLReportGenerator.generate_task_html(result) - assert "expected_turns exceeded" not in html + assert "expected_tool_calls exceeded" not in html def test_task_html_omits_cost_badge_when_cost_is_none(): @@ -1232,7 +1232,7 @@ def test_generation_metrics_breaks_down_crashed_partials(): FinalStatus.ERROR: "error", FinalStatus.BUILD_FAILED: "error", FinalStatus.TIMEOUT: "failure", - FinalStatus.MAX_TURNS_EXHAUSTED: "failure", + FinalStatus.TOOL_CALLS_EXHAUSTED: "failure", FinalStatus.TOKEN_BUDGET_EXCEEDED: "failure", FinalStatus.COST_BUDGET_EXCEEDED: "failure", # Neutral on purpose — an ungraded row has no verdict to colour. See diff --git a/tests/test_reports_junit.py b/tests/test_reports_junit.py index 3d47ce425..0081f87a8 100644 --- a/tests/test_reports_junit.py +++ b/tests/test_reports_junit.py @@ -114,7 +114,7 @@ def test_happy_path_grouping_and_counts(write_run_json: Callable[..., Path], tmp def test_root_counts_equal_summed_children(write_run_json: Callable[..., Path], tmp_path: Path) -> None: run_dir = tmp_path / "run" - rows = [_row("a", "SUCCESS"), _row("b", "FAILURE"), _row("c", "MAX_TURNS_EXHAUSTED")] + rows = [_row("a", "SUCCESS"), _row("b", "FAILURE"), _row("c", "TOOL_CALLS_EXHAUSTED")] write_run_json(run_dir, rows) root = fromstring(generate_junit_xml(run_dir)) @@ -129,7 +129,7 @@ def test_root_counts_equal_summed_children(write_run_json: Callable[..., Path], assert int(root.get("tests")) == total_cases assert int(root.get("failures")) == total_failures assert int(root.get("errors")) == total_errors - # MAX_TURNS_EXHAUSTED is category "failed". + # TOOL_CALLS_EXHAUSTED is category "failed". assert total_failures == 2 assert total_errors == 0 diff --git a/tests/test_result_metrics.py b/tests/test_result_metrics.py index a818ae048..8b07a990a 100644 --- a/tests/test_result_metrics.py +++ b/tests/test_result_metrics.py @@ -21,7 +21,7 @@ ) from coder_eval.result_metrics import ( TurnTimeBuckets, - expected_turns_overage, + expected_tool_calls_overage, has_final_reply, turn_time_buckets, visible_turn_count, @@ -139,58 +139,64 @@ class TestExpectedTurnsOverage: def test_strict_greater_than(self): # 5 tools + reply = 6 visible turns. Budget 6 → no overage (equal). result = _make_result( - resolved={"run_limits": {"expected_turns": 6}}, + resolved={"run_limits": {"expected_tool_calls": 6}}, turns=[_turn_with_commands(commands=5, reply="done")], ) - assert expected_turns_overage(result) is None + assert expected_tool_calls_overage(result) is None # 5 tools + reply = 6 visible turns. Budget 5 → overage (6 > 5). result = _make_result( - resolved={"run_limits": {"expected_turns": 5}}, + resolved={"run_limits": {"expected_tool_calls": 5}}, turns=[_turn_with_commands(commands=5, reply="done")], ) - assert expected_turns_overage(result) == (6, 5) + assert expected_tool_calls_overage(result) == (6, 5) def test_missing_reply_skipped(self): # Tools across multiple iterations sum correctly; absent reply # contributes nothing (no +1). result = _make_result( - resolved={"run_limits": {"expected_turns": 5}}, + resolved={"run_limits": {"expected_tool_calls": 5}}, turns=[_turn_with_commands(commands=4), _turn_with_commands(commands=5)], ) - assert expected_turns_overage(result) == (9, 5) + assert expected_tool_calls_overage(result) == (9, 5) def test_task_config_none(self): result = _make_result(task_config=False, turns=[_turn_with_commands(commands=10)]) - assert expected_turns_overage(result) is None + assert expected_tool_calls_overage(result) is None def test_run_limits_missing(self): result = _make_result(resolved={}, turns=[_turn_with_commands(commands=10)]) - assert expected_turns_overage(result) is None + assert expected_tool_calls_overage(result) is None - def test_expected_turns_unset(self): + def test_expected_tool_calls_unset(self): result = _make_result(resolved={"run_limits": {"max_turns": 10}}, turns=[_turn_with_commands(commands=20)]) - assert expected_turns_overage(result) is None + assert expected_tool_calls_overage(result) is None + + def test_the_historical_expected_turns_key_is_not_read(self): + result = _make_result(resolved={"run_limits": {"expected_turns": 5}}, turns=[_turn_with_commands(commands=20)]) + assert expected_tool_calls_overage(result) is None def test_invalid_expected_type(self): result = _make_result( - resolved={"run_limits": {"expected_turns": "ten"}}, turns=[_turn_with_commands(commands=20)] + resolved={"run_limits": {"expected_tool_calls": "ten"}}, turns=[_turn_with_commands(commands=20)] ) - assert expected_turns_overage(result) is None + assert expected_tool_calls_overage(result) is None - def test_expected_turns_zero_treated_as_invalid(self): + def test_expected_tool_calls_zero_treated_as_invalid(self): # Defensive: the model enforces ge=1, but a hand-rolled task.json could # still inject 0 — the helper must treat it as a disabled check. - result = _make_result(resolved={"run_limits": {"expected_turns": 0}}, turns=[_turn_with_commands(commands=10)]) - assert expected_turns_overage(result) is None + result = _make_result( + resolved={"run_limits": {"expected_tool_calls": 0}}, turns=[_turn_with_commands(commands=10)] + ) + assert expected_tool_calls_overage(result) is None def test_run_limits_not_a_dict(self): result = _make_result(resolved={"run_limits": "not-a-dict"}, turns=[_turn_with_commands(commands=10)]) - assert expected_turns_overage(result) is None + assert expected_tool_calls_overage(result) is None def test_empty_turns(self): - result = _make_result(resolved={"run_limits": {"expected_turns": 1}}, turns=[]) - assert expected_turns_overage(result) is None + result = _make_result(resolved={"run_limits": {"expected_tool_calls": 1}}, turns=[]) + assert expected_tool_calls_overage(result) is None class TestTurnDefinitionMatchesDoc: diff --git a/tests/test_run_limits_models.py b/tests/test_run_limits_models.py index 12241a1c1..5c7478a20 100644 --- a/tests/test_run_limits_models.py +++ b/tests/test_run_limits_models.py @@ -96,7 +96,7 @@ def test_extra_forbid(self): def test_all_fields_roundtrip(self): rl = RunLimits( max_turns=20, - expected_turns=15, + expected_tool_calls=15, task_timeout=600, turn_timeout=120, max_input_tokens=1000, @@ -109,21 +109,21 @@ def test_all_fields_roundtrip(self): rebuilt = RunLimits.model_validate(dumped) assert rebuilt == rl - def test_expected_turns_default_none(self): - assert RunLimits().expected_turns is None + def test_expected_tool_calls_default_none(self): + assert RunLimits().expected_tool_calls is None - def test_expected_turns_lower_bound(self): + def test_expected_tool_calls_lower_bound(self): with pytest.raises(ValidationError, match="greater than or equal to 1"): - RunLimits(expected_turns=0) - assert RunLimits(expected_turns=1).expected_turns == 1 + RunLimits(expected_tool_calls=0) + assert RunLimits(expected_tool_calls=1).expected_tool_calls == 1 - def test_expected_turns_yaml_coercion(self): - assert RunLimits.model_validate({"expected_turns": "10"}).expected_turns == 10 + def test_expected_tool_calls_yaml_coercion(self): + assert RunLimits.model_validate({"expected_tool_calls": "10"}).expected_tool_calls == 10 - def test_expected_turns_greater_than_max_turns_allowed(self): - rl = RunLimits(max_turns=5, expected_turns=20) + def test_expected_tool_calls_greater_than_max_turns_allowed(self): + rl = RunLimits(max_turns=5, expected_tool_calls=20) assert rl.max_turns == 5 - assert rl.expected_turns == 20 + assert rl.expected_tool_calls == 20 def test_extra_forbid_still_rejects_unknowns(self): with pytest.raises(ValidationError): diff --git a/tests/test_run_limits_orchestrator.py b/tests/test_run_limits_orchestrator.py index 86760d485..266fce88a 100644 --- a/tests/test_run_limits_orchestrator.py +++ b/tests/test_run_limits_orchestrator.py @@ -421,82 +421,82 @@ async def test_dialog_aborts_with_run_limit_stop_reason(self, tmp_path): class TestCheckExpectedTurnsUnit: - """Direct unit tests of Orchestrator._check_expected_turns.""" + """Direct unit tests of Orchestrator._check_expected_tool_calls.""" def test_noop_when_run_limits_is_none(self, tmp_path, caplog): orch = _make_orchestrator(_make_task(), tmp_path) orch.result.iterations.append(_make_turn(commands=100)) with caplog.at_level(logging.WARNING): - orch._check_expected_turns(iteration=1) - assert "expected_turns" not in caplog.text.lower() - assert orch._expected_turns_warning_emitted is False + orch._check_expected_tool_calls(iteration=1) + assert "expected_tool_calls" not in caplog.text.lower() + assert orch._expected_tool_calls_warning_emitted is False - def test_noop_when_expected_turns_unset(self, tmp_path, caplog): + def test_noop_when_expected_tool_calls_unset(self, tmp_path, caplog): orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_turns=10)), tmp_path) orch.result.iterations.append(_make_turn(commands=20)) with caplog.at_level(logging.WARNING): - orch._check_expected_turns(iteration=1) - assert "expected_turns" not in caplog.text.lower() - assert orch._expected_turns_warning_emitted is False + orch._check_expected_tool_calls(iteration=1) + assert "expected_tool_calls" not in caplog.text.lower() + assert orch._expected_tool_calls_warning_emitted is False def test_no_warning_at_exact_equal(self, tmp_path, caplog): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(expected_turns=6)), tmp_path) + orch = _make_orchestrator(_make_task(run_limits=RunLimits(expected_tool_calls=6)), tmp_path) # 5 tools + reply = 6 visible turns; equal → no warning. orch.result.iterations.append(_make_turn(iteration=1, commands=3)) orch.result.iterations.append(_make_turn(iteration=2, commands=2, reply="done")) with caplog.at_level(logging.WARNING): - orch._check_expected_turns(iteration=2) + orch._check_expected_tool_calls(iteration=2) assert "Visible turns" not in caplog.text - assert orch._expected_turns_warning_emitted is False + assert orch._expected_tool_calls_warning_emitted is False def test_warning_fires_once_when_exceeded(self, tmp_path, caplog): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(expected_turns=5)), tmp_path) + orch = _make_orchestrator(_make_task(run_limits=RunLimits(expected_tool_calls=5)), tmp_path) # 2 + 2 = 4 visible turns, still under 5. orch.result.iterations.append(_make_turn(iteration=1, commands=2)) orch.result.iterations.append(_make_turn(iteration=2, commands=2)) with caplog.at_level(logging.WARNING): - orch._check_expected_turns(iteration=2) + orch._check_expected_tool_calls(iteration=2) assert "Visible turns" not in caplog.text # +3 tools = 7 visible turns, over 5 → fires. orch.result.iterations.append(_make_turn(iteration=3, commands=3)) with caplog.at_level(logging.WARNING): - orch._check_expected_turns(iteration=3) - assert "Visible turns (7) exceeded expected_turns (5)" in caplog.text - assert orch._expected_turns_warning_emitted is True + orch._check_expected_tool_calls(iteration=3) + assert "Visible tool calls (7) exceeded expected_tool_calls (5)" in caplog.text + assert orch._expected_tool_calls_warning_emitted is True # Re-firing on a later iteration is a no-op. caplog.clear() orch.result.iterations.append(_make_turn(iteration=4, commands=5)) with caplog.at_level(logging.WARNING): - orch._check_expected_turns(iteration=4) + orch._check_expected_tool_calls(iteration=4) assert "Visible turns" not in caplog.text def test_warning_counts_reply_as_one(self, tmp_path, caplog): """A pure-text iteration (0 tools, just a reply) contributes 1 visible turn.""" - orch = _make_orchestrator(_make_task(run_limits=RunLimits(expected_turns=3)), tmp_path) + orch = _make_orchestrator(_make_task(run_limits=RunLimits(expected_tool_calls=3)), tmp_path) # 2 tools, then a 2-tool turn that also emits a final reply. # Visible: 2 + 2 + 1(reply) = 5. Crosses 3 → warns. orch.result.iterations.append(_make_turn(iteration=1, commands=2)) orch.result.iterations.append(_make_turn(iteration=2, commands=2, reply="ok")) with caplog.at_level(logging.WARNING): - orch._check_expected_turns(iteration=2) - assert "Visible turns (5) exceeded expected_turns (3)" in caplog.text + orch._check_expected_tool_calls(iteration=2) + assert "Visible tool calls (5) exceeded expected_tool_calls (3)" in caplog.text def test_noop_when_result_is_none(self, tmp_path, caplog): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(expected_turns=1)), tmp_path) + orch = _make_orchestrator(_make_task(run_limits=RunLimits(expected_tool_calls=1)), tmp_path) orch.result = None with caplog.at_level(logging.WARNING): - orch._check_expected_turns(iteration=1) + orch._check_expected_tool_calls(iteration=1) assert "Visible turns" not in caplog.text @pytest.mark.asyncio class TestExpectedTurnsSingleShot: - """Drive the real _evaluation_loop and assert expected_turns warning never aborts.""" + """Drive the real _evaluation_loop and assert expected_tool_calls warning never aborts.""" async def test_warning_does_not_abort_run(self, tmp_path, caplog): - task = _make_task(run_limits=RunLimits(expected_turns=2)) + task = _make_task(run_limits=RunLimits(expected_tool_calls=2)) # 4 tools + reply = 5 visible turns, exceeds 2. turn = _make_turn(iteration=1, commands=4, reply="done") @@ -517,18 +517,18 @@ async def test_warning_does_not_abort_run(self, tmp_path, caplog): all_passed = await orch._evaluation_loop() assert all_passed is True - assert orch._expected_turns_warning_emitted is True - assert "Visible turns (5) exceeded expected_turns (2)" in caplog.text + assert orch._expected_tool_calls_warning_emitted is True + assert "Visible tool calls (5) exceeded expected_tool_calls (2)" in caplog.text @pytest.mark.asyncio class TestExpectedTurnsSimulation: - """Drive the simulation dialog loop and confirm expected_turns fires once.""" + """Drive the simulation dialog loop and confirm expected_tool_calls fires once.""" async def test_warning_fires_in_simulation_and_does_not_abort(self, tmp_path, caplog): """A simulation turn that trips the soft target logs once; the dialog continues until the simulator decides to stop. The warning must fire - before the max_turns_exhausted break so a turn that trips both still + before the tool_calls_exhausted break so a turn that trips both still emits the soft-target signal.""" from coder_eval.models import SimulationConfig from coder_eval.simulation.user_simulator import SimulatorResult @@ -540,7 +540,7 @@ async def test_warning_fires_in_simulation_and_does_not_abort(self, tmp_path, ca max_turns=5, check_criteria="end_of_dialog", ) - task = _make_task(run_limits=RunLimits(expected_turns=3)) + task = _make_task(run_limits=RunLimits(expected_tool_calls=3)) task = task.model_copy(update={"simulation": sim, "initial_prompt": "first message"}) orch = _make_orchestrator(task, tmp_path) @@ -578,7 +578,7 @@ async def test_warning_fires_in_simulation_and_does_not_abort(self, tmp_path, ca # The simulator-driven loop only sends one agent turn before hitting # the stop token, so cumulative is 1 (not over 3). Confirm no warning. assert "Visible turns" not in caplog.text - assert orch._expected_turns_warning_emitted is False + assert orch._expected_tool_calls_warning_emitted is False # Run completed cleanly. assert orch.result.simulation is not None assert orch.result.simulation.stop_reason == "stop_token" @@ -596,7 +596,7 @@ async def test_warning_fires_when_single_simulation_turn_exceeds(self, tmp_path, max_turns=5, check_criteria="end_of_dialog", ) - task = _make_task(run_limits=RunLimits(expected_turns=2)) + task = _make_task(run_limits=RunLimits(expected_tool_calls=2)) task = task.model_copy(update={"simulation": sim, "initial_prompt": "first message"}) orch = _make_orchestrator(task, tmp_path) @@ -629,8 +629,8 @@ async def test_warning_fires_when_single_simulation_turn_exceeds(self, tmp_path, ): await orch._simulation_dialog_loop("first message", tmp_path / "sandbox") - assert "Visible turns (5) exceeded expected_turns (2)" in caplog.text - assert orch._expected_turns_warning_emitted is True + assert "Visible tool calls (5) exceeded expected_tool_calls (2)" in caplog.text + assert orch._expected_tool_calls_warning_emitted is True class TestBuildSimulationTelemetry: diff --git a/tests/test_run_metrics.py b/tests/test_run_metrics.py index 6866a87f4..6869215ad 100644 --- a/tests/test_run_metrics.py +++ b/tests/test_run_metrics.py @@ -86,7 +86,7 @@ def test_timeout_and_budget_statuses_are_failures_not_errors(self): [ _row(FinalStatus.SUCCESS), _row(FinalStatus.TIMEOUT), - _row(FinalStatus.MAX_TURNS_EXHAUSTED), + _row(FinalStatus.TOOL_CALLS_EXHAUSTED), _row(FinalStatus.TOKEN_BUDGET_EXCEEDED), _row(FinalStatus.COST_BUDGET_EXCEEDED), ] diff --git a/tests/test_run_record.py b/tests/test_run_record.py index c2d0f82c3..116c79c7d 100644 --- a/tests/test_run_record.py +++ b/tests/test_run_record.py @@ -40,8 +40,8 @@ "error_category": None, "error_message": None, "expected_commands": None, - "expected_turns": None, - "expected_turns_overage": None, + "expected_tool_calls": None, + "expected_tool_calls_overage": None, "gate_threshold": None, "generation_ms": None, "has_final_reply": False, @@ -59,7 +59,7 @@ } ], "judge_cost_usd": None, - "max_turns_exhausted": False, + "tool_calls_exhausted": False, "model_used": "claude-haiku-4-5", "output_tokens": 200, "reference_similarity": None, @@ -229,11 +229,11 @@ def test_empty_turns(self): class TestExpectedTurnsKey: def test_emits_when_configured(self): result = _make_result( - resolved={"run_limits": {"expected_turns": 12}}, + resolved={"run_limits": {"expected_tool_calls": 12}}, turns=[_turn_with_expected(5)], ) d = eval_result_to_task_dict(result) - assert d["expected_turns"] == 12 + assert d["expected_tool_calls"] == 12 def test_none_when_unset(self): result = _make_result( @@ -241,28 +241,35 @@ def test_none_when_unset(self): turns=[_turn_with_expected(5)], ) d = eval_result_to_task_dict(result) - assert d["expected_turns"] is None + assert d["expected_tool_calls"] is None def test_none_when_task_config_none(self): result = _make_result(task_config=False, turns=[_turn_with_expected(5)]) d = eval_result_to_task_dict(result) - assert d["expected_turns"] is None + assert d["expected_tool_calls"] is None + + def test_row_carries_the_tool_call_keys_and_none_of_the_historical_ones(self): + result = _make_result(resolved={"run_limits": {"expected_turns": 12}}, turns=[_turn_with_expected(5)]) + d = eval_result_to_task_dict(result) + assert {"tool_calls_exhausted", "expected_tool_calls", "expected_tool_calls_overage"} <= d.keys() + assert not {"max_turns_exhausted", "expected_turns", "expected_turns_overage"} & d.keys() + assert d["expected_tool_calls"] is None def test_none_when_invalid_type(self): result = _make_result( - resolved={"run_limits": {"expected_turns": "ten"}}, + resolved={"run_limits": {"expected_tool_calls": "ten"}}, turns=[_turn_with_expected(5)], ) d = eval_result_to_task_dict(result) - assert d["expected_turns"] is None + assert d["expected_tool_calls"] is None def test_none_when_zero(self): result = _make_result( - resolved={"run_limits": {"expected_turns": 0}}, + resolved={"run_limits": {"expected_tool_calls": 0}}, turns=[_turn_with_expected(5)], ) d = eval_result_to_task_dict(result) - assert d["expected_turns"] is None + assert d["expected_tool_calls"] is None def test_none_when_run_limits_not_dict(self): result = _make_result( @@ -270,4 +277,4 @@ def test_none_when_run_limits_not_dict(self): turns=[_turn_with_expected(5)], ) d = eval_result_to_task_dict(result) - assert d["expected_turns"] is None + assert d["expected_tool_calls"] is None diff --git a/tests/test_seed_from_prior_result.py b/tests/test_seed_from_prior_result.py index 8edceecb6..fd0f48432 100644 --- a/tests/test_seed_from_prior_result.py +++ b/tests/test_seed_from_prior_result.py @@ -45,7 +45,7 @@ "iterations", "iteration_count", "early_stop", - "max_turns_exhausted", + "tool_calls_exhausted", "error_message", "error_details", "error_log_tail", @@ -117,7 +117,7 @@ def _prior() -> EvaluationResult: # its dialog record. iterations=[TurnRecord(iteration=1, user_input="prior prompt", agent_output="prior reply")], simulation=SimulationTelemetry(n_trials=3, replicate_index=2, stop_reason="stop_token", total_turns=4), - max_turns_exhausted=True, + tool_calls_exhausted=True, error_message="prior message", error_details={"where": "prior"}, error_log_tail="prior tail", @@ -291,7 +291,7 @@ def test_grading_cannot_overturn_an_execution_fact() -> None: assert status.is_execution_fact, f"{status} describes the run, so grading must preserve it" -def test_max_turns_exhausted_is_not_an_execution_fact() -> None: +def test_tool_calls_exhausted_is_not_an_execution_fact() -> None: """The one status that reads like an execution fact and is not one. It is SUBORDINATE to the verdict: `run` returns SUCCESS for a max-turns @@ -299,16 +299,26 @@ def test_max_turns_exhausted_is_not_an_execution_fact() -> None: they do not — which is why `_terminal_status` puts the `grade=False` arm above it. The table said True while that method's docstring argued the opposite, so a prior max-turns row re-graded through `evaluate` was written - back as MAX_TURNS_EXHAUSTED *holding weighted_score 1.000* and exited 1 — a + back as TOOL_CALLS_EXHAUSTED *holding weighted_score 1.000* and exited 1 — a combination `run` can never produce for the same trajectory. Its own test, not a line in the loop above, because the two statements ("grading may not launder a crash into a pass" and "grading decides this one") are different contracts that happened to share a fixture. """ - assert not FinalStatus.MAX_TURNS_EXHAUSTED.is_execution_fact + assert not FinalStatus.TOOL_CALLS_EXHAUSTED.is_execution_fact # The fact is not lost; it just lives somewhere a verdict cannot contradict. - assert "max_turns_exhausted" in EvaluationResult.model_fields + assert "tool_calls_exhausted" in EvaluationResult.model_fields + + +def test_a_prior_record_with_the_historical_flag_spelling_loads_with_the_fact_unset() -> None: + """A task.json written before the rename carries ``max_turns_exhausted``: ignored, not aliased.""" + raw = _prior().model_dump(mode="json") + del raw["tool_calls_exhausted"] + raw["max_turns_exhausted"] = True + loaded = EvaluationResult.model_validate(raw) + assert loaded.tool_calls_exhausted is False + assert not hasattr(loaded, "max_turns_exhausted") def test_seeding_is_a_no_op_without_a_prior_result(tmp_path: Path) -> None: diff --git a/tests/test_streaming_renderers.py b/tests/test_streaming_renderers.py index 46a531f7b..c917262ac 100644 --- a/tests/test_streaming_renderers.py +++ b/tests/test_streaming_renderers.py @@ -270,22 +270,21 @@ def test_rich_agent_start_includes_model(): assert "model=claude-opus-4-8" in buf.getvalue() -def test_rich_agent_end_surfaces_crash_and_max_turns(): - """AgentEndEvent on the console surfaces max_turns + crash reason + error detail.""" +def test_rich_agent_end_surfaces_crash_reason_and_error_detail(): + """AgentEndEvent on the console surfaces the crash reason + error detail, and no cap label.""" renderer, buf = _make_renderer() renderer.on_event( AgentEndEvent( task_id="t1", status=AgentEndStatus.CRASHED, duration_seconds=1.0, - max_turns_exhausted=True, crashed=True, crash_reason="CLI process failed (exit code 1)", result_summary=ResultSummary(is_error=True, subtype="error_during_execution", result="boom"), ) ) output = buf.getvalue() - assert "max_turns exhausted" in output + assert "tool-call cap reached" not in output assert "reason:" in output and "exit code 1" in output assert "detail:" in output and "error_during_execution" in output @@ -327,21 +326,26 @@ def test_logging_text_chunk_is_skipped(caplog): assert len(caplog.records) == 0 -def test_logging_agent_end_notes_max_turns(caplog): - """A clean max_turns-exhausted exit is annotated without a crash reason.""" +def test_rich_agent_end_labels_the_tool_call_cap_from_the_status(): + renderer, buf = _make_renderer() + renderer.on_event(AgentEndEvent(task_id="t1", status=AgentEndStatus.TOOL_CALLS_EXHAUSTED, duration_seconds=1.0)) + assert "tool-call cap reached" in buf.getvalue() + + +def test_logging_agent_end_notes_tool_call_cap(caplog): + """A clean tool-call-cap exit is annotated from the status, without a crash reason.""" renderer = LoggingStreamRenderer() with caplog.at_level(logging.DEBUG, logger="coder_eval.streaming.renderers"): renderer.on_event( AgentEndEvent( task_id="t1", - status=AgentEndStatus.MAX_TURNS_EXHAUSTED, + status=AgentEndStatus.TOOL_CALLS_EXHAUSTED, duration_seconds=2.0, - max_turns_exhausted=True, crashed=False, ) ) out = _logged_lines(caplog) - assert "max_turns exhausted" in out + assert "tool-call cap reached" in out assert "reason:" not in out diff --git a/tests/test_verify_published_workflow.py b/tests/test_verify_published_workflow.py index d3f25b4cf..286000de9 100644 --- a/tests/test_verify_published_workflow.py +++ b/tests/test_verify_published_workflow.py @@ -451,7 +451,7 @@ def test_gate_classifies_every_final_status(): assert not unknown, f"the gate compares `status` against non-FinalStatus values {unknown} — dead branches" tolerated = {s.value for s in FinalStatus} - named - assert tolerated == {FinalStatus.FAILURE.value, FinalStatus.MAX_TURNS_EXHAUSTED.value}, ( + assert tolerated == {FinalStatus.FAILURE.value, FinalStatus.TOOL_CALLS_EXHAUSTED.value}, ( f"the gate does not classify {tolerated}. Every FinalStatus must be either hard-failed " "or deliberately tolerated as a model-quality outcome; an unclassified one falls through " "to a GREEN unattended paid run." From dda40e4d79221eee892c14403e18bee0c1198ddc Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 12:58:32 -0700 Subject: [PATCH 2/9] =?UTF-8?q?feat(orchestration):=202/6=20=E2=80=94=20on?= =?UTF-8?q?e=20TurnMonitor=20answers=20should=5Fstop;=20run=5Flimits.max?= =?UTF-8?q?=5Ftool=5Fcalls=20replaces=20max=5Fturns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StopReason and end_status_for join the event protocol; communicate() loses max_turns and its should_stop returns a reason. TurnMonitor (from EarlyStopWatcher) is attached on every run and latches the armed early stop or the cumulative tool-call cap; every adapter drops its own cap and finalizes with end_status_for. sdk_options.max_turns is allowed on Claude Code and used by the simulator and the judge. SPI_VERSION 2. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 29 +- .claude/notes/contracts.md | 4 +- .claude/notes/isolation.md | 4 +- .claude/notes/orchestration.md | 35 +- .claude/notes/reporting.md | 12 +- .claude/notes/timing.md | 2 +- .github/workflows/verify-published-action.yml | 4 +- docs/AB_EXPERIMENTS.md | 12 +- docs/DIALOG_MODE.md | 15 +- docs/EXTENDING.md | 22 +- docs/REPORT_SCHEMA.md | 12 +- docs/TASK_DEFINITION_GUIDE.md | 25 +- docs/USER_GUIDE.md | 9 +- docs/agents/ANTIGRAVITY.md | 8 +- docs/agents/CLAUDE_CODE.md | 13 +- docs/agents/CODEX.md | 2 +- docs/agents/HARNESS_PARITY.md | 79 +- docs/agents/OPENCODE.md | 16 +- docs/agents/PI.md | 13 +- docs/tutorials/04-writing-a-task.md | 2 +- experiments/default.yaml | 9 +- experiments/early-stop-ab.yaml | 2 +- experiments/smoke_variants.yaml | 12 +- plugins/coder-eval/reference/criteria.md | 2 +- plugins/coder-eval/skills/analyze/SKILL.md | 8 +- src/coder_eval/agent.py | 19 +- src/coder_eval/agents/antigravity_agent.py | 91 +- src/coder_eval/agents/claude_code_agent.py | 53 +- src/coder_eval/agents/codex_agent.py | 81 +- src/coder_eval/agents/noop_agent.py | 4 +- src/coder_eval/agents/opencode_agent.py | 42 +- src/coder_eval/agents/pi_agent.py | 46 +- src/coder_eval/cli/execute_command.py | 2 +- src/coder_eval/cli/run_command.py | 2 +- src/coder_eval/config.py | 2 +- src/coder_eval/criteria/agent_judge.py | 8 +- src/coder_eval/evaluation/sub_agent.py | 8 +- src/coder_eval/harbor/packager.py | 4 +- src/coder_eval/models/agent_config.py | 3 +- src/coder_eval/models/criteria.py | 18 +- src/coder_eval/models/limits.py | 21 +- src/coder_eval/models/results.py | 13 +- src/coder_eval/orchestration/early_stop.py | 416 +-------- src/coder_eval/orchestration/turn_monitor.py | 463 ++++++++++ src/coder_eval/orchestrator.py | 85 +- src/coder_eval/resources/tags.yaml | 2 +- src/coder_eval/run_record.py | 4 +- src/coder_eval/simulation/user_simulator.py | 5 +- src/coder_eval/spi.py | 6 +- src/coder_eval/streaming/collector.py | 18 +- src/coder_eval/streaming/events.py | 27 + tasks/agents/antigravity_hello_world.yaml | 2 +- .../antigravity_hello_world_docker.yaml | 2 +- tasks/agents/claude_hello_world.yaml | 2 +- tasks/agents/claude_hello_world_docker.yaml | 2 +- tasks/agents/claude_parallel_single_gen.yaml | 2 +- tasks/agents/claude_subagent_test.yaml | 2 +- tasks/agents/codex_hello_world.yaml | 2 +- tasks/agents/codex_parallel_commands.yaml | 2 +- tasks/agents/codex_parallel_single_gen.yaml | 2 +- tasks/agents/codex_string_utils.yaml | 2 +- tasks/agents/codex_subagent_test.yaml | 2 +- tasks/agents/subagent_bash_long_input.yaml | 2 +- tasks/agents/subagent_merge_sort.yaml | 2 +- .../anti_cheat_reference.yaml | 2 +- .../dockerfile_build_example.yaml | 2 +- .../working_dir_auto_example.yaml | 2 +- .../working_dir_concrete_example.yaml | 2 +- .../early_stop_decision_budget_exceeded.yaml | 6 +- ...y_stop_weighted_high_weight_kills_run.yaml | 4 +- ...rly_stop_weighted_low_weight_absorbed.yaml | 2 +- tasks/internal/session_resumption.yaml | 2 +- .../echo_simulated_judged.yaml | 2 +- tasks/record_cli_responses.yaml | 4 +- ...turns_cap.yaml => max_tool_calls_cap.yaml} | 19 +- tasks/run_limits/turn_timeout.yaml | 2 +- .../3d-scan-calc/3d-scan-calc.yaml | 2 +- .../court-form-filling.yaml | 2 +- .../dialogue-parser/dialogue-parser.yaml | 2 +- tasks/smoke_agent_judge.yaml | 2 +- tasks/smoke_budget_exceeded.yaml | 2 +- tasks/smoke_cost_budget_exceeded.yaml | 2 +- tasks/smoke_llm_judge.yaml | 2 +- tasks/smoke_negative_path.yaml | 6 +- tasks/smoke_task_timeout.yaml | 2 +- tasks/smoke_variants.yaml | 2 +- tasks/token_check.yaml | 2 +- tests/fixtures/byoa_demo_plugin/byoa_demo.py | 2 +- tests/harbor_e2e/fixtures/llm_judge.yaml | 2 +- tests/lint/live_verdict_contract.py | 8 +- .../rules/no_top_level_run_limits_access.py | 6 +- tests/test_agent.py | 91 +- tests/test_agent_judge_criterion.py | 31 +- tests/test_agent_telemetry.py | 3 +- tests/test_agentless.py | 4 +- tests/test_antigravity_agent.py | 95 +- tests/test_cli_set_overrides.py | 14 +- tests/test_codex_agent.py | 157 ++-- tests/test_config_lineage.py | 2 +- tests/test_config_merge_engine.py | 4 +- tests/test_config_precedence.py | 16 +- tests/test_custom_lint.py | 6 +- tests/test_early_stop.py | 808 +++++++++--------- tests/test_event_collector.py | 6 +- tests/test_execute_evaluate_loop.py | 25 +- tests/test_experiment_models.py | 2 +- tests/test_harness_conformance.py | 220 ++++- tests/test_lint_no_top_level_run_limits.py | 4 + tests/test_merge_characterization.py | 16 +- tests/test_merge_unification.py | 6 +- tests/test_opencode_agent.py | 102 ++- tests/test_orchestrator.py | 257 ++++-- tests/test_overrides_engine.py | 14 +- tests/test_pi_agent.py | 108 ++- tests/test_reference_permissions.py | 2 + tests/test_run_limits_models.py | 34 +- tests/test_run_limits_orchestrator.py | 3 +- tests/test_run_limits_resolver.py | 70 +- tests/test_run_record.py | 37 +- tests/test_sdk_option_classification.py | 12 + tests/test_simulation_integration.py | 67 ++ tests/test_spi.py | 8 +- tests/test_sub_agent_runner.py | 36 +- tests/test_timeout_orchestrator.py | 5 +- tests/test_timing_identity_contract.py | 2 - tests/test_turn_monitor.py | 213 +++++ tests/test_user_simulator.py | 24 + tests/test_verify_published_workflow.py | 2 +- tests/test_visible_turn_cap.py | 68 -- tests/test_yaml_migration.py | 12 +- 130 files changed, 2660 insertions(+), 1839 deletions(-) create mode 100644 src/coder_eval/orchestration/turn_monitor.py rename tasks/run_limits/{max_turns_cap.yaml => max_tool_calls_cap.yaml} (75%) create mode 100644 tests/test_turn_monitor.py delete mode 100644 tests/test_visible_turn_cap.py diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index ae7c6b832..f937f35cd 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -61,17 +61,9 @@ intentionally brief and out of scope; trimming for DISPLAY belongs in the render - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. - **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool - calls, read live off the shared `EventCollector.visible_turn_count`, the same list - `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, - so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit - (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT - the same budget across harnesses. OpenCode and Pi each keep a native unit too, because - their CLIs stream a real multi-step loop per `communicate()` - (`step_start`/`step_finish`, `turn_start`/`turn_end`). The cap is enforced on the same - loop boundary as the cooperative early stop and finalizes cleanly as - `tool_calls_exhausted` (no crash, no retry); on Antigravity that boundary lives in - `_drain()`, so the background-work poll loop honors it too. + **`run_limits.max_tool_calls` is the `TurnMonitor`'s cap, in resolved tool calls, on + every harness**: no adapter counts it; each stops at its next `should_stop` poll and + finalizes cleanly as `tool_calls_exhausted` (no crash, no retry). The **known unfixed divergences** — which config fields each harness does and does not enforce, and the per-harness `agent.plugins[].path` depth (claude-code REQUIRES a @@ -113,15 +105,16 @@ bucketing to COMPLETED. Status precedence is the same everywhere: timeout > stopped_early > tool_calls_exhausted > completed. `stopped_early` outranks the cap because an armed criterion deciding the -outcome is the more specific reason to have cut the run, and every loop checks it first. +outcome is the more specific reason to have cut the run; the `TurnMonitor` evaluates the +armed criteria before the cap, so an armed stop wins a tie and the first latched reason is +final. ## Why a post-stop exception is not a crash -Once the loop has broken on purpose — a cooperative stop or the turn cap — an exception -raised while tearing the stream down must NOT be escalated. Escalating triggers the -orchestrator's retry with the watcher's decision still latched, so the retry stops at turn -0 having spent nothing useful; a cap-break is the same shape, where the retry burns the -budget again and re-hits the cap. `ended_cleanly` is the guard. +Once the loop has broken on purpose — any `should_stop` reason, including the tool-call +cap — an exception raised while tearing the stream down must NOT be escalated. Escalating +triggers the orchestrator's retry with the monitor's decision still latched, so the retry +stops at its first poll having spent nothing useful. `ended_cleanly` is the guard. ## Why the constructors declare every kwarg @@ -264,7 +257,7 @@ matter how much the run actually billed. So the CLI harnesses crash rather than Every arm is gated on `stopped_early` / `tool_calls_exhausted`, because an intentional cut can land before the clearing event arrives. Pi's error case shows why: `error_message` is set at an error `turn_end` and cleared only by a LATER non-error `turn_end`, but a -`max_turns` / `should_stop` cut can fire at the next `turn_start`, leaving a stale error +`should_stop` cut (an early stop or the tool-call cap) can fire at the next `turn_start`, leaving a stale error from a turn Pi was still retrying. Without the guard that clean, budget-exhausted cut would crash and burn retries, contradicting the documented "finalizes cleanly as `tool_calls_exhausted`, no crash" contract. diff --git a/.claude/notes/contracts.md b/.claude/notes/contracts.md index 980f3edba..e337dc7f3 100644 --- a/.claude/notes/contracts.md +++ b/.claude/notes/contracts.md @@ -41,7 +41,7 @@ answers pass or fail, it answers the same for every longer prefix); `undecided` verdict allowed to change on a later call. Both properties are stated at the definition site in `criteria/base.py`, because they are the contract an author has to satisfy. -`EarlyStopWatcher`'s deferred fail-stop and its pass/fail flip-attribution are correct ONLY +`TurnMonitor`'s deferred fail-stop and its pass/fail flip-attribution are correct ONLY because the two shipped implementations honor them. A non-monotonic or non-deterministic override compiles, passes CE025, and silently corrupts the stop logic. @@ -92,7 +92,7 @@ matches, but the same haystacks feed `exclude_pattern` and the `max_count` gate, normalized form can newly satisfy an exclusion or trip a cap — a command that counted on the raw text alone can stop counting. -It is memoized because the early-stop watcher re-scans the whole accumulated trajectory on +It is memoized because the `TurnMonitor` re-scans the whole accumulated trajectory on every tool-call event, normalizing the same command many times per run. The regex search window is capped to bound ReDoS on a large command string, and diff --git a/.claude/notes/isolation.md b/.claude/notes/isolation.md index 6baaae2bb..7e309dbc1 100644 --- a/.claude/notes/isolation.md +++ b/.claude/notes/isolation.md @@ -29,11 +29,11 @@ **`TOOL_CALLS_EXHAUSTED` is deliberately NOT one of them — anywhere**. `_EXECUTION_FACT_STATUSES` maps it to `False`, and the table and the chain that reads it must agree: it shipped as `True` while `_terminal_status`'s own docstring argued - the opposite, and the disagreement pinned a re-graded max-turns row at + the opposite, and the disagreement pinned a re-graded capped row at TOOL_CALLS_EXHAUSTED *while holding `weighted_score` 1.000* and exit 1 — a combination `run` can never produce for the same trajectory. Under `execute`: `_terminal_status` puts the `grade=False` arm ABOVE it, because on the graded path it is subordinate to - the verdict — `run` returns SUCCESS for a max-turns trajectory whose criteria pass — + the verdict — `run` returns SUCCESS for a capped trajectory whose criteria pass — so it is not knowable without grading. Consuming it first made it terminal AND permanent (the `is_execution_fact` arm then pinned it), so identical agent output scored SUCCESS/1.0 under `run` and TOOL_CALLS_EXHAUSTED under `execute` → `evaluate`. diff --git a/.claude/notes/orchestration.md b/.claude/notes/orchestration.md index f52c7e75e..bf22c2078 100644 --- a/.claude/notes/orchestration.md +++ b/.claude/notes/orchestration.md @@ -17,7 +17,7 @@ - **Generic CLI overrides (`-D`/`--set`)**: Layer 5 is a thin wrapper (`orchestration/overrides.py`) over the resolver above. `coder-eval run -D - agent.model=opus -D run_limits.max_turns=30` overrides any field on the resolved + agent.model=opus -D run_limits.max_tool_calls=30` overrides any field on the resolved `TaskDefinition` (`agent`/`run_limits`/`sandbox` roots), schema-validated with did-you-mean. Only `--model` (→ `agent.model`) and `--driver` (→ `sandbox.driver`) survive as active thin aliases that emit the equivalent `-D` entry; an alias and `-D` @@ -91,7 +91,7 @@ workspace reports SUCCESS — with the original `error_message` still attached. **The NOT_GRADED arm sits ABOVE `tool_calls_exhausted`, and that order is what makes `execute` + `evaluate` equal a single `run`.** TOOL_CALLS_EXHAUSTED reads like an execution fact but is not one: on the graded path it is subordinate to the verdict — `run` returns -SUCCESS for a max-turns trajectory whose criteria pass, and only falls through to +SUCCESS for a capped trajectory whose criteria pass, and only falls through to TOOL_CALLS_EXHAUSTED when they do not — so it is not knowable under `grade=False`. Consuming it first made it terminal AND permanent, so the same agent output scored SUCCESS/1.0 under `run` and TOOL_CALLS_EXHAUSTED under `execute` → `evaluate`; being @@ -279,8 +279,8 @@ silent. A missing stamp (a run predating the feature) is tolerated. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's - **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the - smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — + **armed** criteria decide the outcome, so a raised `max_tool_calls` isn't wasted on the + smoke flavor. The block's PRESENCE is the arming and alone arms the monitor — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and @@ -313,8 +313,9 @@ silent. A missing stamp (a run predating the feature) is tolerated. never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. - Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when - `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the + Driven by `orchestration/turn_monitor.py::TurnMonitor` (built by `_build_monitor` in + `_setup` on every run; its criteria are armed when `early_stop_active(task)`: ≥1 armed + criterion, kill switch not thrown, and grading on) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut @@ -359,11 +360,14 @@ is never assigned there, so an armed simulation task gates strict-AND on a possi truncated trajectory. Wiring the dialog path through it means also setting `early_stop` there; until then the limit is stated rather than implied. -The watcher is built ONCE, in `_setup`, so its turn/tool counters and wall-clock origin -accumulate across retry attempts. It is built before the evaluate-only early return, so an -armed evaluate-only re-grade builds an inert, never-fed watcher — harmless, and one -creation point. Under `execute` it is armed but stays disabled: there is no outcome to -decide and the trajectory is the deliverable, so an armed criterion must not truncate it. +The `TurnMonitor` is built ONCE, by `_build_monitor` in `_setup`, on every run, so its +tool-call counters and wall-clock origin accumulate across retry attempts and dialog turns +(that is what makes `run_limits.max_tool_calls` cumulative per task). It is built before +the evaluate-only early return, so an evaluate-only re-grade builds an inert, never-fed +monitor — harmless, and one creation point. Under `execute` its criteria are not armed +(`arm=self.grade`): there is no outcome to decide and the trajectory is the deliverable, so +an armed criterion must not truncate it. The tool-call cap still applies there, because it +is a run limit, not a verdict. ### Verdicts latch, and the decision happens on the CALL @@ -462,14 +466,15 @@ distractor rows (fail live, pass and timeout inert) without per-row conditionals why the validator carries NO per-instance polarity guards. Arming an unobservable criterion is structurally impossible, since the block exists only on `LiveSuccessCriterion`, so a `file_exists` criterion carrying one is an `extra='forbid'` error at load. An armed-but- -empty set needs no guard either: with no blocks present there is simply no watcher. +empty set needs no guard either: with no blocks present the monitor has nothing armed and +only its run-limit cap can stop the run. -The watcher keeps its OWN `EventCollector`, independent of the one the agent builds its +The monitor keeps its OWN `EventCollector`, independent of the one the agent builds its returned `TurnRecord` from, so each `live_verdict` sees a fresh single-element partial trajectory. -**Fail-open:** a `live_verdict` that raises disarms the watcher, logs loudly, and degrades -to a full run. Because live verdicts are triggers and not truth, this can never produce a +**Fail-open:** a `live_verdict` that raises disarms the armed criteria, logs loudly, and +degrades to a full run. The tool-call cap reads counters, so it keeps running. Because live verdicts are triggers and not truth, this can never produce a FALSE early stop — it only ever errs toward running more. ### Why the guardrails are not model validators diff --git a/.claude/notes/reporting.md b/.claude/notes/reporting.md index 5c0607cfa..823d940a1 100644 --- a/.claude/notes/reporting.md +++ b/.claude/notes/reporting.md @@ -15,16 +15,16 @@ - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — - `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / + `max_tool_calls` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D - run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D + run_limits.max_tool_calls=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead - (see [orchestration.md](orchestration.md) § Early stop on criterion) — the watcher + (see [orchestration.md](orchestration.md) § Early stop on criterion) — the monitor must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. @@ -332,10 +332,12 @@ means the whole bill. ### The claims the reports do NOT make -An early-stopped row does not advertise "N turns avoided". That derived from +An early-stopped row does not advertise "N turns avoided". That claim once derived from `max_turns - sdk_turn_index`, and on harnesses where one `communicate()` is a single SDK turn it advertised dozens of avoided turns when all that was cut was a tool-call tail. The -upper bound is still persisted, labelled as the bound it is. +upper bound is still persisted as `tool_calls_remaining_at_stop` +(`max_tool_calls - tool_call_index`, null when the cap is unset), labelled as the bound it +is. Missing spend is worded cause-agnostically, because an unpriced turn and a hard kill reach the same conclusion and the report cannot always tell which applied. diff --git a/.claude/notes/timing.md b/.claude/notes/timing.md index 49a3ad769..9d58a81c8 100644 --- a/.claude/notes/timing.md +++ b/.claude/notes/timing.md @@ -188,7 +188,7 @@ terminal event as `AgentEndEvent(messages=list(...))` — that copies the LIST, message objects — so writing in place would reach back into the agent's own live state from the collector, which is exactly the layering "the collector is the sole capture seam" exists to prevent. It is also unconditionally safe for a caller that builds a record -twice: `EarlyStopWatcher` holds one collector across a turn's tool-call rounds and calls +twice: `TurnMonitor` holds one collector across a task's tool-call rounds and calls `build_turn_record` on every one. Grouping by identical bounds rather than `message_id`: Codex splits one window across two diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml index 3079e354e..27540943a 100644 --- a/.github/workflows/verify-published-action.yml +++ b/.github/workflows/verify-published-action.yml @@ -344,10 +344,10 @@ jobs: # caps are ~10x what the one-file task needs; tripping the spend or wall-clock # one produces a COST_BUDGET_EXCEEDED / TIMEOUT row that the gate's run-limit # branch prints and fails on (both statuses report as "failed", so nothing else - # in the gate would). MAX_TURNS_EXHAUSTED is the one deliberate exception: it is + # in the gate would). TOOL_CALLS_EXHAUSTED is the one deliberate exception: it is # the classic model-quality outcome, tolerated like an unmet criterion. run_limits: - max_turns: 5 + max_tool_calls: 5 task_timeout: 300 max_usd: 0.25 diff --git a/docs/AB_EXPERIMENTS.md b/docs/AB_EXPERIMENTS.md index eee67bf81..44aeaac27 100644 --- a/docs/AB_EXPERIMENTS.md +++ b/docs/AB_EXPERIMENTS.md @@ -99,7 +99,7 @@ it (the unification invariant). The per-field strategy is: **most lists** (`allowed_tools`, `disallowed_tools`, `plugins`, …) — **replace** (last layer wins; a variant's `allowed_tools: ["Read"]` replaces the lower list entirely). `run_limits` is per-field replace, so a variant setting - `run_limits.max_turns` leaves the task's `task_timeout` intact. + `run_limits.max_tool_calls` leaves the task's `task_timeout` intact. - **nested models** (`sandbox.docker`, `python`, `node`, `limits`) and **free-form dicts** (`agent.sdk_options`) — **deep**-merge: a higher layer touching one sub-key (e.g. `docker.network`) preserves siblings set below it (e.g. @@ -152,7 +152,7 @@ From `ExperimentVariant` (`coder_eval/models/experiment.py`): | `prompt_mutations` | list | Ordered mutations applied to `initial_prompt` | | `initial_prompt` | str | Full prompt replacement (mutually exclusive with the two below) | | `initial_prompt_file` | str | Prompt replacement loaded from a file | -| `run_limits` | block | Per-key cap overrides (`max_turns`, `task_timeout`, token/USD budgets) | +| `run_limits` | block | Per-key cap overrides (`max_tool_calls`, `task_timeout`, token/USD budgets) | | `driver` | `tempdir`/`docker` | Sandbox driver — enables tempdir-vs-docker arms | | `checker_context` | dict | Backend/model override for the judge side (llm_judge, agent_judge) — has no bearing on the simulator; see [Checker Context](TASK_DEFINITION_GUIDE.md#checker-context); **not** currently `-D`-reachable | @@ -309,7 +309,7 @@ that define "the interesting thing happened" with `stop_early:` blocks in the task file; the `smoke` variant cuts off as soon as they're decided, while `e2e` runs to completion. Because the field merge is per-key, the variant sets only -`stop_early` (the run-level kill switch) without disturbing the task's `max_turns`. +`stop_early` (the run-level kill switch) without disturbing the task's `max_tool_calls`. ```yaml experiment_id: early-stop-ab @@ -324,7 +324,7 @@ variants: ``` The task file supplies the arming (`stop_early:` blocks on the criteria that gate the -flavor) and a `max_turns` generous enough for `e2e`; see +flavor) and a `max_tool_calls` generous enough for `e2e`; see [`stop_early`](TASK_DEFINITION_GUIDE.md#stop_early-opt-in-early-stop). This recipe ships as `experiments/early-stop-ab.yaml`. @@ -390,8 +390,8 @@ if any listed metric is below its minimum. | `--driver tempdir\|docker` | Override sandbox driver for all tasks. | | `-j, --max-parallel N` | Run up to N tasks concurrently. | | `-t, --tags` / `--exclude-tags` | Filter which tasks run. | -| `-D path=value` / `--set` | Generic layer-5 override of any resolved task-config field, applied to **every** variant — e.g. `-D agent.model=opus -D run_limits.max_turns=30`. Repeatable; schema-validated. | -| `--model`, `--driver` | Thin aliases for `-D` (`--model` ≡ `-D agent.model`, `--driver` ≡ `-D sandbox.driver`). All other task-config knobs (permission mode, turn/timeout limits, tools, plugins, SDK options) are set via `-D`. Layer-5 overrides apply to **every** variant (use sparingly — they erase the contrast between arms). | +| `-D path=value` / `--set` | Generic layer-5 override of any resolved task-config field, applied to **every** variant — e.g. `-D agent.model=opus -D run_limits.max_tool_calls=30`. Repeatable; schema-validated. | +| `--model`, `--driver` | Thin aliases for `-D` (`--model` ≡ `-D agent.model`, `--driver` ≡ `-D sandbox.driver`). All other task-config knobs (permission mode, tool-call/timeout limits, tools, plugins, SDK options) are set via `-D`. Layer-5 overrides apply to **every** variant (use sparingly — they erase the contrast between arms). | | `--type` | Dedicated flag for agent type, applied to every variant (re-parses the agent discriminated union). | Layer-5 flags win over variant config, so overriding the very thing you're diff --git a/docs/DIALOG_MODE.md b/docs/DIALOG_MODE.md index b0c9e476a..56a238319 100644 --- a/docs/DIALOG_MODE.md +++ b/docs/DIALOG_MODE.md @@ -122,8 +122,10 @@ After each exchange the driver evaluates the stop conditions **in this order**, 2. **`stop_on_criteria_pass`** (`criteria_passed`) — every success criterion passes. Requires per-turn checking (`check_criteria: every_turn` or `both`); pairing it with the default `end_of_dialog` is rejected at load time, since there would be nothing to check against. -3. **`max_turns`** (`max_turns`) — the hard cap on exchanges. The agent reaching its *own* tool-call - cap mid-exchange ends the dialog with its own reason, `tool_call_cap`. +3. **`max_turns`** (`max_turns`) — the hard cap on exchanges. The agent reaching + [`run_limits.max_tool_calls`](TASK_DEFINITION_GUIDE.md#run-limits) mid-exchange ends the dialog + with its own reason, `tool_call_cap`. That cap is cumulative across every dialog turn: it counts + the agent's resolved tool calls over the whole dialog, not per exchange. 4. **`max_total_tokens`** (`budget`) — the dialog-wide budget across simulator **and** agent. The dialog ends and the task is **still scored** — unlike [`run_limits.max_total_tokens`](TASK_DEFINITION_GUIDE.md#run-limits), which covers the subject @@ -131,8 +133,8 @@ After each exchange the driver evaluates the stop conditions **in this order**, 5. **`stop_token`** (`stop_token`) — only if none of the above fired is the simulator asked for another message; the sentinel token in *that fresh utterance* ends the dialog. This is the workhorse in practice — the simulator decides, in character, that it got what it wanted — but it - is evaluated **last**, so a turn that trips `max_turns` or the budget never gets the chance to - produce it. + is evaluated **last**, so a turn that trips `max_turns`, the tool-call cap or the budget never + gets the chance to produce it. A simulator call that raises ends the dialog with `error` (and increments `simulation.simulator_failures`). The reason is recorded as `simulation.stop_reason` on the result, @@ -188,8 +190,9 @@ with them. That guard is what keeps a chatty simulator from poisoning the grade. ## What it costs -Budget roughly `max_turns × n_trials` agent turns per (task, variant) — the worst case, since the -dialog usually stops on the stop token first. On top of that: +Budget roughly `simulation.max_turns × n_trials` agent turns per (task, variant) — the worst case, +since the dialog usually stops on the stop token first. `run_limits.max_tool_calls` bounds the tool +calls of one trial's whole dialog, not of each exchange. On top of that: - **Simulator tokens**, one generation per turn. Small next to the agent's, but not free, and they are reported separately as `simulation.simulator_input_tokens` / `simulator_output_tokens`. diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index c4307de56..22df940f1 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -48,7 +48,7 @@ signature. from coder_eval.spi import SPI_VERSION, AgentRegistry def register(registry: type[AgentRegistry]) -> None: - assert SPI_VERSION == 1, f"my-agent supports coder_eval SPI 1, not {SPI_VERSION}" + assert SPI_VERSION == 2, f"my-agent supports coder_eval SPI 2, not {SPI_VERSION}" # Bind type string → config class → agent class. registry.register("my-agent", MyAgentConfig)(MyAgent) # Optionally contribute pricing here too (see §3): @@ -129,8 +129,8 @@ class MyAgent(Agent[MyAgentConfig]): - `tool_names` is required exactly when a tool-list row is `ENFORCED`. It must map every canonical name; list a name your harness has no tool for in `no_equivalent`. - Set `cooperative_stop=True` only if your `communicate()` honors `should_stop` - (needed for criterion-level `stop_early:` arming). `False` means early stop is - rejected at resolution for your agent. + (needed for criterion-level `stop_early:` arming and for `run_limits.max_tool_calls` + to cut a turn). `False` means early stop is rejected at resolution for your agent. ### The `Agent` ABC — implementation checklist @@ -141,9 +141,19 @@ it on every LiteLLM route. Implement these three abstract methods: - [ ] `async def start(self, working_directory, *, env_path_prepend=None, plugin_tools_dir=None) -> None` -- [ ] `async def communicate(self, user_input, *, stream_callback=None, timeout=None, max_turns=None, should_stop=None) -> TurnRecord` +- [ ] `async def communicate(self, user_input, *, stream_callback=None, timeout=None, should_stop: Callable[[], StopReason | None] | None = None) -> TurnRecord` - [ ] `async def stop(self) -> None` +`should_stop` is the run's single stop poll. The `TurnMonitor` owns it: it reads your +event stream and decides every stop (armed criteria, the tool-call cap). Your agent +does not count or cap anything. With `cooperative_stop=True`: + +- [ ] Call `should_stop()` at each safe boundary (for example, after each resolved + tool call, before you pull the next unit of work). +- [ ] When it returns a `StopReason`, stop pulling work and remember the reason. +- [ ] Finalize the turn with `AgentEndStatus` `end_status_for(reason)` (both names + come from `coder_eval.spi`), with `crashed=False`. Do not raise. + Optional overrides (sensible defaults exist): `kill()`, `kill_sync()` (called from a non-asyncio watchdog thread — must **not** await), `discard_pending_turn()`. @@ -270,14 +280,14 @@ Notes: own fields — no `turn_records`, no checker instance), and override the checker's `live_verdict(...)`. `LiveSuccessCriterion` subclassing is the single source of truth for "is this criterion type live-observable" — - `validate_early_stop`/`EarlyStopWatcher` check `isinstance(c, + `validate_early_stop`/`TurnMonitor` check `isinstance(c, LiveSuccessCriterion)` directly, no separate checker-side flag. A lint rule (`tests/test_custom_lint.py::TestCE025LiveVerdictConsistency`) keeps the model subclassing and the checker's `live_verdict` override paired. - Your `live_verdict` must be **deterministic** (a pure function of the `turn_records` prefix — no wall-clock, randomness, or hidden instance state) and **monotonic** (once it returns `"pass"`/`"fail"` for some prefix, every - longer prefix returns that same verdict) — `EarlyStopWatcher`'s verdict + longer prefix returns that same verdict) — `TurnMonitor`'s verdict latching and deferred stops silently depend on both. Lint rule CE036 (`tests/lint/live_verdict_contract.py`) enforces this by replaying each live criterion against every prefix of recorded trajectories, and **fails until diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index b008c29a6..be83730d8 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -85,7 +85,7 @@ including: `task_id`, `replicate_index`, `variant_id`, `status` `actual_commands`, `commands_efficiency`, `agent_config`, `sdk_options`, `installed_tools`, turn accounting (`total_turns`, `visible_turns`, `expected_tool_calls`, `expected_tool_calls_overage`, `tool_calls_exhausted`, `has_final_reply`), and early-stop fields (`stopped_early`, -`early_stop_reason`, `turns_remaining_at_stop`). `iterations` here is a **reduced** +`early_stop_reason`, `tool_calls_remaining_at_stop`). `iterations` here is a **reduced** turn digest (`{iteration, duration_seconds, command_count, assistant_turn_count, crashed, crash_reason}`) — the full transcript is in `task.json`. @@ -97,6 +97,13 @@ crashed, crash_reason}`) — the full transcript is in `task.json`. > the fact `false`; one whose `final_status` is `MAX_TURNS_EXHAUSTED`, or whose recorded > config sets `run_limits.expected_turns`, does not load. `run --resume` then runs that > row again, and `evaluate ` cannot re-grade it from its recorded config. +> +> Runs written before the tool-call cap replaced the turn cap carry +> `turns_remaining_at_stop` instead of `tool_calls_remaining_at_stop`, in both +> `EarlyStopInfo` and the `run.json` row. No reader maps the old key. Their recorded +> config also sets `run_limits.max_turns`, which no longer validates, so +> `evaluate ` re-grades such a run from the source task YAML and prints its +> fallback warning. ### Missing cost is never fatal @@ -242,7 +249,8 @@ criterion timed out undecided past its `stop_early.decide_within`; it gates thro the same weighted armed gate as a native fail), `deciding_criterion_type`, `deciding_criterion_description`, `armed_criteria`, `sdk_turn_index`, `tool_call_index` (1-based, includes the in-flight call), -`elapsed_seconds`, `turns_remaining_at_stop`, `gate_threshold` (the +`elapsed_seconds`, `tool_calls_remaining_at_stop` (`max_tool_calls − tool_call_index`, +floored at `0`; `null` when `run_limits.max_tool_calls` is unset), `gate_threshold` (the `run_limits.stop_early_gate_threshold` in effect for this stop; default `1.0`). --- diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 1ef8235e2..10e7771a5 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -258,7 +258,7 @@ valid and an empty block is legal — every field defaults to "no limit". ```yaml run_limits: # Structural caps - max_turns: 20 # hard cap on agent inner-loop turns per iteration + max_tool_calls: 20 # hard cap on resolved tool calls across the whole task expected_tool_calls: 8 # SOFT efficiency budget (visible tool calls) — never aborts task_timeout: 300 # wall-clock cap for the full run envelope, seconds turn_timeout: 300 # per-communicate() timeout, seconds @@ -273,7 +273,7 @@ run_limits: | Field | Default | Constraint | Description | |-------|---------|------------|-------------| -| `max_turns` | *unset* | `> 0` | Hard cap on agent inner-loop turns per iteration. Unset uses the SDK default. | +| `max_tool_calls` | *unset* | `> 0` | Hard cap on resolved tool calls across the whole task: every retry attempt and every dialog turn count. The TurnMonitor enforces it at the agent's next poll boundary, on every harness. The round that reaches the cap is processed whole, so tool calls already in flight can still land after it. The run finalizes cleanly as `tool_calls_exhausted`, and the criteria are still checked. Unset means no cap. | | `expected_tool_calls` | *unset* | `>= 1` | **Soft** target for cumulative visible tool calls. Exceeding it warns and badges the report; it never aborts. See [`expected_tool_calls`](#expected_tool_calls-soft-efficiency-budget). | | `task_timeout` | *unset* | `>= 30` | Max seconds for the full run envelope, including agent work, grading, and post-run work. | | `turn_timeout` | *unset* | `>= 10` | Max seconds for the agent's single `communicate()` iteration. | @@ -317,7 +317,7 @@ model. `run_limits` without disturbing the task's other caps: ```bash -coder-eval run task.yaml -D run_limits.max_turns=30 -D run_limits.task_timeout=900 +coder-eval run task.yaml -D run_limits.max_tool_calls=30 -D run_limits.task_timeout=900 coder-eval run task.yaml -D run_limits.max_usd=2.50 -D run_limits.max_total_tokens=200000 ``` @@ -331,13 +331,16 @@ coder-eval run task.yaml -D run_limits.max_usd=2.50 -D run_limits.max_total_toke > **No longer supported:** `max_turns` / `turn_timeout` (and top-level > `task_timeout`) under `agent:` or at the task top level are rejected — > the agent model's `extra="forbid"` raises a clear validation error. -> They must live under `run_limits:`. (A deprecation shim hoisted them -> automatically until it was removed on 2026-06-01.) +> `turn_timeout` and `task_timeout` must live under `run_limits:`. (A +> deprecation shim hoisted them automatically until it was removed on +> 2026-06-01.) `max_turns` under `run_limits:` is rejected too: use +> `run_limits.max_tool_calls`, which counts resolved tool calls, not agent +> inner-loop turns. ### `expected_tool_calls` (soft efficiency budget) `run_limits.expected_tool_calls` is a **soft target**, not a cap: the run is never -aborted for exceeding it (use `max_turns` for a hard limit). It's the budget the +aborted for exceeding it (use `max_tool_calls` for a hard limit). It's the budget the dashboard's **"Within Expected Turns"** metric divides by — a task counts as "within budget" when it succeeds *and* its turn count stays within **1.5×** `expected_tool_calls`. The run-level headline reports the share of **budgeted** tasks @@ -356,7 +359,7 @@ default) to exclude a task from the metric entirely. ### `stop_early` (opt-in early stop) Early stop ends a single-shot run **early** once the run's **armed** criteria -decide the outcome — so you can raise `max_turns` for the full-run flavor +decide the outcome — so you can raise `max_tool_calls` for the full-run flavor without paying for turns the smoke flavor doesn't need. A criterion is *armed* by attaching a **`stop_early:` block** to it — the block's presence IS the arming, and it alone activates the run's watcher; there is **no run-level @@ -380,7 +383,7 @@ under the weighted ceiling rule — plus two knobs inside the block: ```yaml run_limits: - max_turns: 30 + max_tool_calls: 30 success_criteria: - type: skill_triggered skill_name: date-teller @@ -518,7 +521,7 @@ Semantics: cannot doom the gate is absorbed, and the run continues). The timeout is checked after the criterion's own verdict each round, so one that decides on that very step is never penalized. `None` (default) = no timeout; the run - relies solely on `run_limits.max_turns`. The step count is **cumulative + relies solely on `run_limits.max_tool_calls`. The step count is **cumulative across every retry attempt** of the turn — including an attempt that crashed or timed out before this criterion's own investigation even began — so size the budget with that headroom in mind. @@ -528,8 +531,8 @@ compares a truncated run against a full one): | Surface | Field / marker | |---------|----------------| -| `run.json` row | `stopped_early`, `early_stop_reason`, `turns_remaining_at_stop` | -| `run.md` | `> **NOTE:** […] stopped early (); <= N turn(s) avoided …` | +| `run.json` row | `stopped_early`, `early_stop_reason`, `tool_calls_remaining_at_stop` | +| `run.md` | `> **NOTE:** […] stopped early (); ` | | `task.html` | header badge `stopped early ()` + `advisory — not gated` markers | | Telemetry | `EarlyStopped` / `EarlyStopReason` dimensions on `CoderEval.Task.End` | diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 3f0fe446e..0867b5b4f 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -36,7 +36,7 @@ coder-eval run tasks/hello_date.yaml --stream full # live LLM output | `--max-parallel, -j` | Concurrent tasks (default: 1) | | `--preservation-mode` | Sandbox persistence: `NONE` / `MOVE_ON_WRITE` / `DIRECT_WRITE`. Default is driver-derived (docker → `DIRECT_WRITE`, else `MOVE_ON_WRITE`); explicit value always wins. | | `--run-dir` | Custom run directory (default: timestamped in `runs/`) | -| `-D path=value` / `--set` | Override any resolved task-config field (`agent`/`run_limits`/`sandbox` roots), e.g. `-D run_limits.max_turns=30 -D agent.permission_mode=plan -D agent.sdk_options.effort=high`. Repeatable; schema-validated. This is the way to set permission mode, turn/timeout limits, token/USD budget caps, tools, plugins, and SDK options. | +| `-D path=value` / `--set` | Override any resolved task-config field (`agent`/`run_limits`/`sandbox` roots), e.g. `-D run_limits.max_tool_calls=30 -D agent.permission_mode=plan -D agent.sdk_options.effort=high`. Repeatable; schema-validated. This is the way to set permission mode, tool-call/timeout limits, token/USD budget caps, tools, plugins, and SDK options. | | `--model, -m` | Shorthand alias for `-D agent.model=…` (e.g., `claude-sonnet-5`) | | `--driver` | Shorthand alias for `-D sandbox.driver=…` (`tempdir` or `docker`) | | `--type, -T` | Override agent type for all tasks (`claude-code`, `codex`, `antigravity`, `opencode`, `pi`, or a plugin kind). | @@ -81,8 +81,8 @@ you want to iterate on afterwards. Grade the results later with budget breach still reports `ERROR` / `TIMEOUT` / `TOKEN_BUDGET_EXCEEDED` and still exits non-zero, exactly as under `run`. -Exhausting `max_turns` is the one fact that does *not* become a status here. Under -`run` it decides the outcome only when the criteria fail — a max-turns trajectory +Exhausting `max_tool_calls` is the one fact that does *not* become a status here. Under +`run` it decides the outcome only when the criteria fail — a capped trajectory whose criteria pass is `SUCCESS` — so it is not knowable without grading. `execute` records `tool_calls_exhausted: true` on the row and finalizes `NOT_GRADED`; the later grade reads the flag and reaches exactly the status `run` would have. Rows like this @@ -189,6 +189,9 @@ expansion are already baked into `resolved`, so re-loading the source would silently grade a *different* task. The run's trajectory is restored too, so criteria that read the agent's tool calls (`command_executed`, `skill_triggered`, judges with trajectory) score exactly as they would have during the run. +A run recorded before `run_limits.max_turns` became `run_limits.max_tool_calls` carries +the old key, which no longer validates, so `evaluate` re-grades it from the source task +YAML and prints its fallback warning. It writes the verdict back into the run's `task.json` and keeps the pre-grade record beside it as `task.execute.json`. Writing back in place is what makes diff --git a/docs/agents/ANTIGRAVITY.md b/docs/agents/ANTIGRAVITY.md index b54543d03..583e12216 100644 --- a/docs/agents/ANTIGRAVITY.md +++ b/docs/agents/ANTIGRAVITY.md @@ -75,7 +75,7 @@ agent: path: "$SKILLS_PLUGIN_PATH" # a directory of skills (SKILL.md), env-expanded run_limits: - max_turns: 5 + max_tool_calls: 5 task_timeout: 360 turn_timeout: 300 @@ -201,9 +201,9 @@ as every other agent. happens on the subsequent async `stop()`. 4. **Denied tools stay visible.** A policy denial rejects the call after the model makes it, so a denied tool can still cost tokens on a retry. -5. **`max_turns` counts visible turns.** One `communicate()` is a single SDK turn here, - so the cap counts resolved tool calls instead, enforced on the step loop. See - [Run-Limit Parity](HARNESS_PARITY.md). +5. **`max_tool_calls` counts resolved tool calls.** The adapter counts nothing itself. + The TurnMonitor owns the cap, as on every harness, and the adapter stops at its + next `should_stop` poll on the step loop. See [Run-Limit Parity](HARNESS_PARITY.md). 6. **Shell commands over ~10s are moved to the background.** The localharness has a 10-second maximum synchronous wait; past it the command becomes a background task and the model gets a task id, not a result. The turn polls for that result instead diff --git a/docs/agents/CLAUDE_CODE.md b/docs/agents/CLAUDE_CODE.md index 12356ee90..81b8dbd90 100644 --- a/docs/agents/CLAUDE_CODE.md +++ b/docs/agents/CLAUDE_CODE.md @@ -109,9 +109,13 @@ agent: | `ignore_patterns` | `list[str] \| null` | Gitignore-style overrides for the workspace copy used by judge sub-agents (supports `!` negation). | > `sdk_options` is a deliberate escape hatch. Framework-owned keys (`model`, -> `permission_mode`, `allowed_tools`, `mcp_servers`, `resume`, `max_turns`, +> `permission_mode`, `allowed_tools`, `mcp_servers`, `resume`, > `setting_sources`, `include_partial_messages`, …) are rejected there — set those > through their typed fields or `-D run_limits.*`. MCP servers are not a YAML field. +> `sdk_options.max_turns` is allowed: it is the SDK's own agent-loop cap, and when it +> trips the turn ends `COMPLETED` with `result_summary.subtype == "error_max_turns"`, +> while `run_limits.max_tool_calls` is the framework's cap on resolved tool calls +> (the `TurnMonitor` enforces it, as on every harness); both apply. > **System-prompt reproducibility.** In `append` mode the preset's *dynamic > sections* (working directory, git status, auto-memory) are excluded so the system @@ -205,7 +209,7 @@ Claude Code supports the cooperative early-stop seam, as do the carries a `stop_early:` block, a single-shot run ends cleanly at the next tool-call boundary once its **armed** criteria (those carrying a `stop_early:` block) are -decided — so a raised `max_turns` isn't wasted on a smoke run. Early stop errors at +decided — so a raised `max_tool_calls` isn't wasted on a smoke run. Early stop errors at resolution for any agent whose contract does not declare `cooperative_stop`. See the [Task Definition Guide](../TASK_DEFINITION_GUIDE.md) for the full contract. @@ -215,7 +219,10 @@ Claude Code produces the richest telemetry of the agents: - **Authoritative billing** comes from the SDK's cumulative `model_usage` on the terminal result message and reconciles to `total_cost_usd` — this already includes - sub-agent consumption that the per-message stream under-reports. + sub-agent consumption that the per-message stream under-reports. A turn that the + `TurnMonitor` stops (early stop, the tool-call cap) ends before that message + arrives, so its usage and cost come from the per-message stream and the rate card, + and can be slightly lower than the bill. - **Sub-agent accounting** is derived by grouping `parent_tool_use_id`-tagged assistant messages; there is no separate per-sub-agent field. The terminal sub-agent generation (delivered as the Agent tool result, never streamed) is diff --git a/docs/agents/CODEX.md b/docs/agents/CODEX.md index 65f43052a..c511deb73 100644 --- a/docs/agents/CODEX.md +++ b/docs/agents/CODEX.md @@ -203,7 +203,7 @@ The Codex SDK is synchronous. The agent uses `_run_async()` helper to detect and | **System prompt** | `system_prompt` appended to the default prompt (SDK `claude_code` preset) | `system_prompt` passed as `developer_instructions` on top of the Codex base prompt | | **Session Resume** | `--resume {session_id}` | Via thread ID | | **Permissions** | `permission_mode` + `allowed_tools` + `disallowed_tools` | Not supported; always full-access | -| **`max_turns`** | Native SDK turn cap (assistant messages) | Visible-turn cap (tool calls), enforced on the notification pump | +| **`max_tool_calls`** | TurnMonitor cap on resolved tool calls, polled between messages | Same TurnMonitor cap, polled after each streamed notification | | **Early stop** | Supported (cooperative `should_stop`, polled between messages) | Supported — polled after each streamed notification; the in-flight turn is interrupted best-effort | Run-limit semantics per harness: [Run-Limit Parity](HARNESS_PARITY.md). diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index f59b4ec78..db0fe1380 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -1,9 +1,10 @@ # Run-Limit Parity -One task file, run on any harness, must be the same task. `run_limits.max_turns` -was the field that broke that promise hardest: Claude Code enforced it, and Codex and -Antigravity accepted it and never read it, so `max_turns: 6` ran capped on one -backend and unbounded on the other two. +One task file, run on any harness, must be the same task. The old +`run_limits.max_turns` was the field that broke that promise hardest: Claude Code +enforced it, and Codex and Antigravity accepted it and never read it, so +`max_turns: 6` ran capped on one backend and unbounded on the other two. Its +replacement, `run_limits.max_tool_calls`, is enforced centrally. This page is the contract for what each run limit means per harness, plus what each shared `agent` field means on each harness. @@ -12,7 +13,7 @@ shared `agent` field means on each harness. | Limit | claude-code | codex | antigravity | opencode | pi | |---|---|---|---|---|---| -| `run_limits.max_turns` | native SDK cap (agent-loop turns) | visible-turn cap (resolved tool calls) | visible-turn cap (resolved tool calls) | native step cap (the CLI's own agent-loop steps) | native turn cap (the CLI's own `turn_start` agent-loop steps) | +| `run_limits.max_tool_calls` | TurnMonitor cap (resolved tool calls) via `should_stop` | TurnMonitor cap (resolved tool calls) via `should_stop` | TurnMonitor cap (resolved tool calls) via `should_stop` | TurnMonitor cap (resolved tool calls) via `should_stop` | TurnMonitor cap (resolved tool calls) via `should_stop` | | `run_limits.turn_timeout` | watchdog, SIGKILL on the CLI subprocess | watchdog + cooperative interrupt | watchdog, plus an earlier internal poll deadline at 80% of it (see below) | deadline enforced in-loop and on the final reap; SIGTERM→SIGKILL on the CLI's whole process group | deadline enforced in-loop and on the final reap; SIGTERM→SIGKILL on the CLI's whole process group | | `run_limits.task_timeout` | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | | `run_limits.stop_early` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` (event granularity) | cooperative `should_stop` (event granularity — Pi streams incrementally) | @@ -572,8 +573,8 @@ needed to drive it. when a generation begins. Nothing in the timing accounting reads it — the head and tail are measured from the first and last `AssistantMessage` instead, which is uniform across all five — so this is recorded rather than - fixed. It is NOT a `max_turns` hazard: `EventCollector.visible_turn_count` is - `len(self._commands)`, derived from `ToolEndEvent`, and `_turn_starts` feeds + fixed. It is NOT a `max_tool_calls` hazard: the TurnMonitor counts resolved + `ToolEndEvent`s, and `_turn_starts` feeds only `assistant_turn_count` on the no-`AgentEndEvent` fallback path. The real cost of normalizing it is that the event drives the live renderers, so moving it changes the turn boundaries users watch during a run. @@ -581,50 +582,16 @@ needed to drive it. All three are deliberately deferred; see `c/time-bugs-audit.md` for the measurements. -## `max_turns` counts visible turns on Codex and Antigravity - -A "visible turn" is one entry in the run's timeline: one resolved tool call. It is -the unit `result_metrics.visible_turn_count` reports and the unit that lands in -`TurnRecord.commands`. Both backends count it live off the shared -`EventCollector.visible_turn_count`, so one `max_turns` value means one thing on -both. - -They need their own counter because a native one would be meaningless: Codex and -Antigravity each deliver exactly **one SDK turn per `communicate()` call**, so an -SDK-level cap would clamp at 1 no matter what the task asked for. - -The cap is enforced on the same loop boundary as the cooperative early stop: the -step or notification that reaches the cap is processed whole, and the next one is -never pulled. The in-flight turn is then cancelled server-side (best effort) so -the cap actually stops spend. A run cut this way finalizes cleanly as -`tool_calls_exhausted` — it is not a crash, and it is not retried. - -**claude-code keeps its native SDK cap.** That is a real, honored cap, so it is -left alone rather than reimplemented in a different unit. Its unit is the SDK's own -agent-loop turn, which absorbs an arbitrary number of *parallel* tool calls, so the -same number bounds very different amounts of work: under a prompt that encourages -batching, a cap of N here permits many more than N tool calls, where it buys exactly -N on the other two. - -**OpenCode also keeps a native unit — its stream's own steps.** Unlike Codex and -Antigravity, `opencode run` executes a real multi-step agent loop per invocation -and streams it (`step_start` / `step_finish`), so the natural agent-loop unit -exists and is honored: `max_turns: N` allows N complete steps and cuts the run -when step N+1 begins, with the completed steps' tokens intact. A step is one -assistant generation and may carry several tool calls — so, as with claude-code, -the same number is a looser tool-call budget than on the visible-turn backends. - -**Pi keeps a native unit too — its `turn_start` agent-loop steps.** Like OpenCode, -`pi -p --mode json` runs a real multi-step agent loop per invocation and streams it -(`turn_start` / `turn_end`), so `max_turns: N` allows N complete turns and cuts the -run when turn N+1 begins, with the completed turns' tokens intact. Pi streams -incrementally, so the cut genuinely stops spend mid-run. A Pi turn is one assistant -generation and may carry several tool calls — the same looser budget as claude-code -and OpenCode. - -**So holding `max_turns` constant across harnesses does not hold the budget -constant.** If you are A/B-ing across backends and the cap is close to binding, that -is the number to distrust. +## `max_tool_calls` is the TurnMonitor's cap on every harness + +`run_limits.max_tool_calls` counts resolved tool calls, cumulative across every retry +attempt and every dialog turn of a task. No adapter counts or caps anything itself. The +orchestrator's `TurnMonitor` reads the event stream, and when the resolved tool calls +reach the cap, its `should_stop` poll returns `tool_call_cap`. Each harness stops at its +next poll boundary: the round that reaches the cap is processed whole, and the next one +is never pulled. A run cut this way finalizes cleanly as `tool_calls_exhausted`. It is +not a crash, and it is not retried. One number therefore means the same budget on +every harness. ### What a capped run looks like @@ -636,8 +603,8 @@ The signals a capped run leaves behind, on every backend: `TOOL_CALLS_EXHAUSTED` (reporting category `failed`, icon `C`). Never `ERROR`, and never retried. - `tool_calls_exhausted: true` on the task record. -- On Codex and Antigravity, the count of *resolved* tool calls the model itself - issued equals the cap. Two things can add a further *recorded* command, and +- The count of *resolved* tool calls the model itself issued reaches the cap. Calls of + the round that reached it can still resolve after it. Two things can add a further *recorded* command, and neither means the cap leaked: - A tool call already in flight when the cap fires is force-closed and recorded with `result_status: unknown` rather than dropped, so the trajectory shows what @@ -677,9 +644,9 @@ a long `npm install` or build runs to completion here the way it does on the oth two, but a command that never finishes reads as an ordinary low score rather than a timeout. -## Timeouts are not turn caps +## Timeouts are not tool-call caps -A timeout is a *failure* (partial turn captured, error status); the turn cap is a +A timeout is a *failure* (partial turn captured, error status); the tool-call cap is a *clean stop*. Conflating them is the mistake this page exists to prevent: a task whose cap fires should not look like a task whose harness hung. @@ -754,7 +721,7 @@ both. See [OpenCode](OPENCODE.md) and [Pi § plugins](PI.md#known-limitations). ## Reproducing -`tasks/run_limits/` holds one fixture per limit: `max_turns_cap.yaml` asks for more +`tasks/run_limits/` holds one fixture per limit: `max_tool_calls_cap.yaml` asks for more sequential work than its cap allows, and `turn_timeout.yaml` runs a command that outlives its watchdog. Run either with `--type claude-code` / `--type codex` / `--type antigravity` / `--type opencode` / `--type pi` to check a backend against the diff --git a/docs/agents/OPENCODE.md b/docs/agents/OPENCODE.md index fa74ad887..6b711a48b 100644 --- a/docs/agents/OPENCODE.md +++ b/docs/agents/OPENCODE.md @@ -296,8 +296,9 @@ the run really billed. Two shapes reach it: counts** (a provider or auth mode that omits `tokens`) — the error reports the finished-step count and whether cost was present. -Intentional cuts (`should_stop`, `max_turns`) are exempt: both can land before -the first event, or between a step's start and its `step_finish`. +Intentional cuts (any `should_stop` reason, including the tool-call cap) are +exempt: a cut can land before the first event, or between a step's start and its +`step_finish`. For a provider or auth mode that genuinely reports no usage — where failing every turn would make the harness unusable rather than merely imprecise — set @@ -343,13 +344,10 @@ any other provider credential can be added via `sandbox.env_passthrough_extra`. - **Only the *skills* half of a `plugins:` entry is honored** (see below). A Claude plugin's agents, hooks, commands and MCP servers have no OpenCode equivalent and are still dropped. -- **`max_turns` counts OpenCode's native steps.** One step = one assistant - generation (`step_start`/`step_finish`) and may carry several tool calls; - `max_turns: N` allows N complete steps, then the run finalizes cleanly as - `tool_calls_exhausted`. This is the claude-code-style native unit, not the - visible-turn unit Codex/Antigravity use — see - [Run-Limit Parity](HARNESS_PARITY.md) before holding `max_turns` constant - across harnesses. +- **`max_tool_calls` counts resolved tool calls, not OpenCode steps.** The adapter + counts nothing itself. The TurnMonitor owns the cap, as on every harness; the + adapter stops at its next `should_stop` poll, and the run finalizes cleanly as + `tool_calls_exhausted`. See [Run-Limit Parity](HARNESS_PARITY.md). - **The `docker` sandbox driver is unsupported.** The CLI is not in the image (`OPENROUTER_API_KEY` is now forwarded by default, added for Pi, but the OpenCode CLI itself is still absent) — see [Running in Docker](#running-in-docker) for the diff --git a/docs/agents/PI.md b/docs/agents/PI.md index fdca6b452..b18f16802 100644 --- a/docs/agents/PI.md +++ b/docs/agents/PI.md @@ -155,7 +155,7 @@ Mapping from the CLI's event vocabulary onto `TurnRecord`: | Pi event | Becomes | |---|---| -| `turn_start` | `TurnStartEvent` (one inner turn; the unit `max_turns` counts) | +| `turn_start` | `TurnStartEvent` (one inner turn) | | `message_update` (`text_delta`) | `TextChunkEvent` + `agent_output` | | `tool_execution_start` | `ToolStartEvent` | | `tool_execution_end` | `ToolEndEvent` | @@ -197,7 +197,7 @@ merged. The internal retry is bounded by `turn_timeout` / `task_timeout`. A turn whose CLI exits cleanly but which captured **no recognized events** (an upgrade renamed the vocabulary) is failed rather than reported as a clean empty success — the error names the unrecognized event types it saw. Intentional cuts -(`should_stop`, `max_turns`) are exempt. +(any `should_stop` reason, including the tool-call cap) are exempt. > **Zero-usage turn.** A provider that reports no usage yields an all-zero > `token_usage`. Pi does **not** hard-fail such a turn (its multi-provider surface @@ -240,11 +240,10 @@ docker` whenever the task prompt or workspace is not fully trusted. assets (agents/hooks/commands/MCP) are not wired. - **`system_prompt_file` is not read by the adapter.** Use `system_prompt` (inline) instead — it is enforced via `--append-system-prompt`. -- **`max_turns` counts Pi's native agent-loop turns.** One `turn_start` = one - agent-loop step; `max_turns: N` allows N complete turns, then the run finalizes - cleanly as `tool_calls_exhausted`. See - [Run-Limit Parity](HARNESS_PARITY.md) before holding `max_turns` constant across - harnesses. +- **`max_tool_calls` counts resolved tool calls, not Pi turns.** The adapter counts + nothing itself. The TurnMonitor owns the cap, as on every harness; the adapter + stops at its next `should_stop` poll, and the run finalizes cleanly as + `tool_calls_exhausted`. See [Run-Limit Parity](HARNESS_PARITY.md). - **No sub-agent attribution.** Pi's CLI stream does not expose nested agent generations, so per-sub-agent token grouping (available for Claude and Codex) is not derivable. diff --git a/docs/tutorials/04-writing-a-task.md b/docs/tutorials/04-writing-a-task.md index 602c95ca3..a472ed8cb 100644 --- a/docs/tutorials/04-writing-a-task.md +++ b/docs/tutorials/04-writing-a-task.md @@ -64,7 +64,7 @@ What each block does: only when the task needs your MCP servers. - **`run_limits.expected_tool_calls`** — an efficiency target, not a cap: exceeding it logs a warning and adds a report badge but never aborts (use - `run_limits.max_turns` for a hard cap). + `run_limits.max_tool_calls` for a hard cap). - **`success_criteria`** — each criterion scores 0.0–1.0 and supports `weight` (default 1.0) and `pass_threshold` (default 0.9). `run_command` here only checks the exit code; it can also match stdout (`expected_stdout`) or read a diff --git a/experiments/default.yaml b/experiments/default.yaml index 43c54cbb8..195a51846 100644 --- a/experiments/default.yaml +++ b/experiments/default.yaml @@ -18,11 +18,12 @@ defaults: # Per-row dataset multiplication: a 100-row task with max_usd: 0.10 permits up to # $10 cumulative spend. run_limits: - # Hard safety ceiling on agent inner-loop turns per iteration (null = SDK - # default). Set well above the typical 3-18 turns a task needs so it only - # trips on runaway loops; task_timeout / turn_timeout below are the practical + # Hard safety ceiling on resolved tool calls per task, cumulative across every + # retry and dialog turn, in the tool-call unit, the same on every harness. + # Local runs show a median of 2 and a p99 of 31 visible turns, so 100 trips + # only on runaway loops; task_timeout / turn_timeout below are the practical # guards. Override per-task when a task legitimately needs more. - max_turns: 100 + max_tool_calls: 100 # Soft target: when cumulative visible tool calls across all iterations of a # task exceed this, the orchestrator logs a one-shot warning and the report # surfaces a badge. Does NOT abort the run. diff --git a/experiments/early-stop-ab.yaml b/experiments/early-stop-ab.yaml index 0edaefdb4..85269209c 100644 --- a/experiments/early-stop-ab.yaml +++ b/experiments/early-stop-ab.yaml @@ -1,7 +1,7 @@ experiment_id: early-stop-ab description: | Smoke vs. e2e flavors from ONE task file via opt-in early-stop. Both variants - share the same tasks, criteria, and max_turns. Arming is per-criterion (the + share the same tasks, criteria, and max_tool_calls. Arming is per-criterion (the `stop_early:` blocks in the task file), so the `smoke` variant needs NO override at all — the armed criteria decide when it cuts off. The `e2e` variant throws the run-level kill switch (`run_limits.stop_early: false`, diff --git a/experiments/smoke_variants.yaml b/experiments/smoke_variants.yaml index 77493cb72..d2b62425a 100644 --- a/experiments/smoke_variants.yaml +++ b/experiments/smoke_variants.yaml @@ -2,8 +2,8 @@ experiment_id: smoke-variants description: | Smoke test for the 5-layer config-merge resolver: ensures multi-variant fan-out + variant-level overrides flow correctly from the experiment YAML - through ResolvedTask. Two variants override `max_turns` to different values - so the resolver bug "variants share the task's max_turns" would surface as a + through ResolvedTask. Two variants override `run_limits.max_tool_calls` to different + values so the resolver bug "variants share the task's cap" would surface as a count drift or behavior drift in the run. defaults: @@ -14,11 +14,11 @@ defaults: variants: - variant_id: turns-low - description: "Variant overriding max_turns to a lower value." + description: "Variant overriding max_tool_calls to a lower value." run_limits: - max_turns: 1 + max_tool_calls: 1 - variant_id: turns-mid - description: "Variant overriding max_turns to a higher value." + description: "Variant overriding max_tool_calls to a higher value." run_limits: - max_turns: 2 + max_tool_calls: 2 diff --git a/plugins/coder-eval/reference/criteria.md b/plugins/coder-eval/reference/criteria.md index a946bc07e..d0c4ca15d 100644 --- a/plugins/coder-eval/reference/criteria.md +++ b/plugins/coder-eval/reference/criteria.md @@ -32,7 +32,7 @@ Optional: | Field | What it is | | --- | --- | -| `stop_early` | Opt-in early-stop policy block; its PRESENCE arms this criterion for the run's early-stop watcher — the block alone activates the watcher, there is no run-level master switch (run_limits.stop_early: false is the run-level veto). An armed criterion's definitive effective FAIL — a native live-fail, or the decide_within timeout expiring — may end the run under the weighted ceiling rule (deferred while any pass-capable armed criterion is still undecided); set on_pass: stop to also end the run on a live PASS. An empty block (stop_early: {}) is the idiomatic distractor arming: fail-stop on misfire, nothing else. Triggers whose polarity this instance cannot decide are inert by design (dataset fan-out support). Unarmed criteria stay advisory on an early-stopped run. Only exists on live-observable criteria, so arming anything else is a schema error. | +| `stop_early` | Opt-in early-stop policy block; its PRESENCE arms this criterion for the run's TurnMonitor — the block alone arms it, there is no run-level master switch (run_limits.stop_early: false is the run-level veto). An armed criterion's definitive effective FAIL — a native live-fail, or the decide_within timeout expiring — may end the run under the weighted ceiling rule (deferred while any pass-capable armed criterion is still undecided); set on_pass: stop to also end the run on a live PASS. An empty block (stop_early: {}) is the idiomatic distractor arming: fail-stop on misfire, nothing else. Triggers whose polarity this instance cannot decide are inert by design (dataset fan-out support). Unarmed criteria stay advisory on an early-stopped run. Only exists on live-observable criteria, so arming anything else is a schema error. | ## Criterion types diff --git a/plugins/coder-eval/skills/analyze/SKILL.md b/plugins/coder-eval/skills/analyze/SKILL.md index 667b601ac..7527b727e 100644 --- a/plugins/coder-eval/skills/analyze/SKILL.md +++ b/plugins/coder-eval/skills/analyze/SKILL.md @@ -54,7 +54,7 @@ summary per task with `jq` (or `python3` if `jq` is missing): total_cost_usd: .total_token_usage.total_cost_usd, total_tokens: (.total_token_usage.input_tokens + .total_token_usage.output_tokens), assistant_turns: .total_assistant_turns, - max_turns: .task_config.resolved.run_limits.max_turns, + max_tool_calls: .task_config.resolved.run_limits.max_tool_calls, criteria_count: (.success_criteria_results | length), all_criteria_perfect: (.success_criteria_results | length > 0 and all(.[]; .score == 1.0)), @@ -73,7 +73,7 @@ out means running `jq 'keys' ` and reading the result**, not assu stale path yields a table of nulls that reads like a run with no data instead of an error. -There is no top-level `total_tokens`, `total_cost_usd`, `max_turns` or `criteria_count` +There is no top-level `total_tokens`, `total_cost_usd`, `max_tool_calls` or `criteria_count` in any generation: token and cost figures live under `total_token_usage`, and a criterion's type is `criterion_type`. A criterion passes when `score >= pass_threshold` — there is no `passed` boolean. @@ -83,7 +83,7 @@ Two names *did* change between generations, which is what the `keys` check is fo | Current runs | Older runs | Where | | --- | --- | --- | | `iterations` | `turns` | top-level record key | -| `task_config.resolved.run_limits.max_turns` | `task_config.resolved.max_iterations` | inside the free-form `task_config` dict | +| `task_config.resolved.run_limits.max_tool_calls` | `task_config.resolved.run_limits.max_turns` (a turn cap, not a tool-call cap), and before that `task_config.resolved.max_iterations` | inside the free-form `task_config` dict | Extract whichever the file actually has. The loader still accepts the older top-level name when reading, so an old run is not broken — but current runs do not write it, and @@ -159,7 +159,7 @@ passes: variant/run scope. 4. **Criteria** — sensitivity `weight × (threshold − score)`; fragile passes sitting on the threshold; redundant criteria and coverage gaps. -5. **Configuration** — lineage conflicts (`source != "task"`), `max_turns` hit or +5. **Configuration** — lineage conflicts (`source != "task"`), tool-call cap (`max_tool_calls`) hit or wildly excessive, model fit, `allowed_tools` alignment with what the task needs. 6. **Environment** — infrastructure errors, missing services, expired credentials, CLI tool errors. Also **idempotency and cross-run contamination**: a criterion that passed on diff --git a/src/coder_eval/agent.py b/src/coder_eval/agent.py index 14d4fc31b..8c511401f 100644 --- a/src/coder_eval/agent.py +++ b/src/coder_eval/agent.py @@ -14,7 +14,7 @@ from .models import ApiRoute, BaseAgentConfig, HarnessContract, ToolNameMap, TurnRecord from .streaming.callbacks import StreamCallback from .streaming.collector import EventCollector -from .streaming.events import AgentEndStatus +from .streaming.events import AgentEndStatus, StopReason logger = logging.getLogger(__name__) @@ -214,8 +214,7 @@ async def communicate( *, stream_callback: StreamCallback | None = None, timeout: float | None = None, - max_turns: int | None = None, - should_stop: Callable[[], bool] | None = None, + should_stop: Callable[[], StopReason | None] | None = None, ) -> TurnRecord: """Send a message to the agent and receive its response. @@ -226,15 +225,11 @@ async def communicate( agent must force-terminate any in-flight subprocess and raise TurnTimeoutError. Do not rely solely on asyncio cancellation -- some SDKs swallow it. - max_turns: Hard cap on inner-loop turns within this single - ``communicate()`` call. When the agent would exceed it, the - returned ``TurnRecord`` has ``tool_calls_exhausted=True``. - None defers to the underlying SDK default. - should_stop: Cooperative early-stop poll. An implementation with - ``contract.cooperative_stop`` calls it at each safe message - boundary and, when it returns True, stops pulling further work and - finalizes the turn cleanly (``crashed=False``, no raise). Agents - that do not support it accept and ignore the argument. + should_stop: The run's single stop poll. An implementation with + ``contract.cooperative_stop`` calls it at each safe boundary; a + non-None reason means stop pulling work, remember the reason, and + finalize with ``end_status_for(reason)`` (``crashed=False``, no + raise). Agents that do not support it accept and ignore it. Returns: TurnRecord containing the complete interaction diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 98a6bcf8d..24b408042 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -61,6 +61,7 @@ AgentEndEvent, AgentEndStatus, AgentStartEvent, + StopReason, TextChunkEvent, ToolEndEvent, ToolEndStatus, @@ -68,6 +69,7 @@ TurnEndEvent, TurnEndStatus, TurnStartEvent, + end_status_for, ) from coder_eval.timing import TurnClock, close_window from coder_eval.utils import expand_env_vars @@ -415,7 +417,7 @@ async def _drain( self, conversation: Any, state: "_AntigravityTurnState", - should_stop: Callable[[], bool] | None, + should_stop: Callable[[], StopReason | None] | None, ) -> None: """Consume one ``receive_steps()`` cycle onto ``state``, honoring a cooperative stop mid-stream. Shared by the initial drain and each poll @@ -437,19 +439,10 @@ async def _drain( async for step in steps: pulled = True state.process_step(step) - if should_stop is not None and should_stop(): - state.stopped_early_hit = True - self._log.debug("Cooperative stop requested; ending step loop at this boundary") - break - # The turn cap shares this boundary: the step that reached - # the cap is kept whole, the next is never pulled. After the - # cooperative stop, so an armed early-stop wins a tie. - if state.max_turns_reached(): - state.max_turns_hit = True - self._log.debug( - "max_turns (%s visible turns) reached; ending step loop", - state.max_turns, - ) + reason = should_stop() if should_stop is not None else None + if reason is not None: + state.stop_reason = reason + self._log.debug("Stop requested (%s); ending step loop at this boundary", reason.value) break return except RuntimeError: @@ -467,20 +460,14 @@ async def communicate( *, stream_callback: StreamCallback | None = None, timeout: float | None = None, - max_turns: int | None = None, - should_stop: Callable[[], bool] | None = None, + should_stop: Callable[[], StopReason | None] | None = None, ) -> TurnRecord: """Send a message to the Antigravity agent and receive its response. - ``should_stop`` is the cooperative early-stop callback, polled after each - processed step. When it returns True the step loop breaks, the - conversation is cancelled (best-effort) and the turn finalizes cleanly as - ``STOPPED_EARLY`` (``crashed=False``). - - ``max_turns`` caps VISIBLE turns — resolved tool calls — enforced in-stream - on the same boundary as the cooperative stop: one ``communicate()`` here is - a single SDK turn, so a native counter would cap at 1 and mean nothing. - See docs/agents/HARNESS_PARITY.md. + ``should_stop`` is the run's stop poll, called after each processed step. + On a reason the step loop breaks, the conversation is cancelled + (best-effort) and the turn finalizes cleanly with ``end_status_for(reason)`` + (``crashed=False``). Drives one logical turn: ``conversation.send(prompt)`` then iterate ``receive_steps()`` until the turn goes idle. @@ -520,7 +507,6 @@ async def communicate( model=model, turn_start_time=turn_start_time, clock=clock, - max_turns=max_turns, ) try: @@ -556,7 +542,7 @@ def _on_turn_timeout() -> None: poll_deadline = turn_start_time + timeout * _POLL_DEADLINE_TIMEOUT_FRACTION if timeout else None try: await conversation.send(user_input) - # should_stop runs AFTER process_step (the emission the watcher + # should_stop runs AFTER process_step (the emission the monitor # latches on) and BEFORE the next step is pulled. await self._drain(conversation, state, should_stop) @@ -566,8 +552,7 @@ def _on_turn_timeout() -> None: # zero times. # Rationale: .claude/notes/agents.md § Antigravity Step interleaving and the background poll while ( - not state.stopped_early_hit - and not state.max_turns_hit + state.stop_reason is None and not state.timeout_hit and state.has_orphaned_tool_call() and ( @@ -583,19 +568,13 @@ def _on_turn_timeout() -> None: # Skip the re-drain, which could itself await # indefinitely on genuinely non-idle work. break - if should_stop is not None and should_stop(): - state.stopped_early_hit = True + reason = should_stop() if should_stop is not None else None + if reason is not None: + state.stop_reason = reason break - # A re-drain honors the turn cap too (the check lives in - # _drain), so a poll cycle can be the one that reaches it. await self._drain(conversation, state, should_stop) - if ( - state.has_orphaned_tool_call() - and not state.stopped_early_hit - and not state.max_turns_hit - and not state.timeout_hit - ): + if state.has_orphaned_tool_call() and state.stop_reason is None and not state.timeout_hit: # Exited via this loop's OWN bound, not an external # stop/timeout: the call is force-closed as unresolved and # the turn is still graded normally on everything else. @@ -607,7 +586,7 @@ def _on_turn_timeout() -> None: msg = "Poll budget exhausted (%s, poll_count=%d) with a tool call still ACTIVE." self._log.warning(msg, bound, poll_count) - if state.stopped_early_hit or state.max_turns_hit: + if state.stop_reason is not None: # Best-effort server-side cancel. One check point, so it # fires exactly once whichever drain stopped. with contextlib.suppress(Exception): @@ -649,14 +628,9 @@ def _on_turn_timeout() -> None: self._state = AgentState.WORKING self._end_turn_ok() - # Precedence: timeout (raised above) > stopped_early > max_turns > done. + # Precedence: timeout (raised above) > the stop reason > done. # Rationale: .claude/notes/agents.md § Shared turn lifecycle - if state.stopped_early_hit: - status = AgentEndStatus.STOPPED_EARLY - elif state.max_turns_hit: - status = AgentEndStatus.TOOL_CALLS_EXHAUSTED - else: - status = AgentEndStatus.COMPLETED + status = end_status_for(state.stop_reason) if state.stop_reason is not None else AgentEndStatus.COMPLETED state.finalize(status, crashed=False, crash_reason=None) return collector.build_turn_record() @@ -745,7 +719,6 @@ def __init__( model: str, turn_start_time: float, clock: TurnClock, - max_turns: int | None = None, ) -> None: self._agent = agent self.emit = emit @@ -760,10 +733,8 @@ def __init__( # instead of monkeypatching `datetime` out from under the reducer. self.clock = clock - self.max_turns = max_turns self.timeout_hit = False - self.stopped_early_hit = False - self.max_turns_hit = False + self.stop_reason: StopReason | None = None self.finalized = False self.total_usage = TokenUsage() @@ -794,22 +765,12 @@ def __init__( @property def ended_cleanly(self) -> bool: - """True once the loop broke on purpose (cooperative stop or the turn cap). - - Both are non-crash terminations, so a stray exception raised while - unwinding the step generator afterwards must not be escalated. - """ - return self.stopped_early_hit or self.max_turns_hit - - def max_turns_reached(self) -> bool: - """True once this turn has produced ``max_turns`` visible turns. + """True once the loop broke on a ``should_stop`` reason. - Delegates to ``EventCollector.visible_turn_count``, the single - agent-agnostic capture path, so one ``max_turns`` means the same thing here - and on Codex. It counts RESOLVED tool calls, so the call that reaches the - cap keeps its result instead of being force-closed as unresolved. + A non-crash termination, so a stray exception raised while unwinding the + step generator afterwards must not be escalated. """ - return self.max_turns is not None and self.collector.visible_turn_count >= self.max_turns + return self.stop_reason is not None def _seed_first_generation_window(self, source: Any) -> None: """Move the first window's mark to the first observed MODEL output. diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 9cbacea2b..a56eb479e 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -73,6 +73,7 @@ AgentEndEvent, AgentEndStatus, AgentStartEvent, + StopReason, TextChunkEvent, ToolEndEvent, ToolEndStatus, @@ -80,6 +81,7 @@ TurnEndEvent, TurnEndStatus, TurnStartEvent, + end_status_for, ) from coder_eval.timing import TurnClock, close_window from coder_eval.utils import dump_dataclass, process_plugins @@ -207,7 +209,6 @@ def __init__( task_id: str, user_input: str, iteration: int, - max_turns: int | None, log: PrefixedAdapter, turn_start_time: float, deadline: float | None, @@ -219,15 +220,14 @@ def __init__( self.task_id = task_id self.user_input = user_input self.iteration = iteration - self.max_turns = max_turns self.log = log self.turn_start_time = turn_start_time self.deadline = deadline # Set True by the in-loop deadline break OR the watchdog callback. self.timeout_hit = False - # Set True by the in-loop cooperative-stop break (early-stop-on-criterion). - # Distinct from timeout_hit: a clean, non-crash stop that must NOT raise. - self.stopped_early_hit = False + # Set by the in-loop should_stop break. Distinct from timeout_hit: a clean, + # non-crash stop that must NOT raise. + self.stop_reason: StopReason | None = None # Resolved by _build_claude_query, set on the state before any finalize # path. Stays None if we crash before setup (finalize reads it for cost # backfill). @@ -637,14 +637,6 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso ) ) - max_turns_exhausted = not crashed and ( - self._agent._is_max_turns_result(self.sdk_result_summary) - or (self.max_turns is not None and self.num_turns is not None and self.num_turns > self.max_turns) - ) - if max_turns_exhausted and status == AgentEndStatus.COMPLETED: - status = AgentEndStatus.TOOL_CALLS_EXHAUSTED - self.log.warning("Agent exhausted max_turns (%s); turn ended without completing", self.max_turns) - if self.current_turn_id is not None: self.emit.on_event( TurnEndEvent( @@ -922,8 +914,7 @@ async def communicate( *, stream_callback: StreamCallback | None = None, timeout: float | None = None, - max_turns: int | None = None, - should_stop: Callable[[], bool] | None = None, + should_stop: Callable[[], StopReason | None] | None = None, ) -> TurnRecord: """Send a message to Claude and receive its response. @@ -934,10 +925,9 @@ async def communicate( the CLI subprocess when it elapses — the SDK's anyio task groups suppress cooperative cancellation, so `asyncio.wait_for` is not sufficient. - max_turns: Hard cap on inner-loop turns. None defers to the SDK. - should_stop: Cooperative early-stop poll, checked after each dispatched - message; the first True finalizes cleanly as STOPPED_EARLY - (``crashed=False``, no raise) at the next boundary. + should_stop: The run's stop poll, checked after each dispatched message; + the first reason finalizes cleanly with ``end_status_for(reason)`` + (``crashed=False``, no raise) at that boundary. Returns: TurnRecord containing the complete interaction @@ -976,7 +966,6 @@ async def communicate( task_id=task_id, user_input=user_input, iteration=self._iteration, - max_turns=max_turns, log=self._log, turn_start_time=turn_start_time, deadline=deadline, @@ -990,9 +979,7 @@ def capture_stderr(line: str) -> None: stderr_lines.append(line) try: - options, transport, effective_model = self._build_claude_query( - user_input, timeout, max_turns, capture_stderr - ) + options, transport, effective_model = self._build_claude_query(user_input, timeout, capture_stderr) # Set on the state BEFORE the AgentStart emit and any finalize path # (finalize reads it for cost backfill); stays None if setup crashed. state.effective_model = effective_model @@ -1090,10 +1077,9 @@ def _on_turn_timeout() -> None: if state.timeout_hit: assert timeout is not None state.finalize(AgentEndStatus.TIMEOUT, crashed=True, crash_reason=format_timeout_reason(timeout)) - elif state.stopped_early_hit: - # NOT a crash, NOT a timeout. The max_turns promotion in - # finalize() only fires for COMPLETED, so this survives. - state.finalize(AgentEndStatus.STOPPED_EARLY, crashed=False, crash_reason=None) + elif state.stop_reason is not None: + # NOT a crash, NOT a timeout. + state.finalize(end_status_for(state.stop_reason), crashed=False, crash_reason=None) else: state.finalize(AgentEndStatus.COMPLETED, crashed=False, crash_reason=None) self._active_transport = None @@ -1117,7 +1103,7 @@ async def _pump_messages( state: _ClaudeTurnState, query_kwargs: dict[str, Any], deadline: float | None, - should_stop: Callable[[], bool] | None, + should_stop: Callable[[], StopReason | None] | None, ) -> None: """Drive the SDK message stream for one turn (extracted from ``communicate``). @@ -1129,7 +1115,7 @@ async def _pump_messages( - The wall-clock guard runs at the TOP, so an over-deadline message is DISCARDED — no append, no events. Do NOT move it to a post-loop check. - - The cooperative stop runs AFTER ``state.dispatch(message)``, so a watcher + - The cooperative stop runs AFTER ``state.dispatch(message)``, so the monitor can flip its flag on THIS message and the next is never pulled. """ async for message in query(**query_kwargs): @@ -1138,16 +1124,16 @@ async def _pump_messages( self._log.warning("Turn timeout reached mid-stream; breaking out of message loop") break state.dispatch(message) - if should_stop is not None and should_stop(): - state.stopped_early_hit = True - self._log.debug("Cooperative stop requested; ending message loop at this boundary") + reason = should_stop() if should_stop is not None else None + if reason is not None: + state.stop_reason = reason + self._log.debug("Stop requested (%s); ending message loop at this boundary", reason.value) break def _build_claude_query( self, user_input: str, timeout: float | None, - max_turns: int | None, stderr_callback: Callable[[str], None], ) -> tuple[ClaudeAgentOptions, SubprocessCLITransport | None, str | None]: """Build the SDK options (+ a timeout-only transport) for one turn. @@ -1197,7 +1183,6 @@ def _build_claude_query( allowed_tools=self.config.allowed_tools or [], disallowed_tools=disallowed_tools, model=effective_model, - max_turns=max_turns, plugins=plugins, # type: ignore[arg-type] stderr=stderr_callback, # Capture stderr for better error messages env=env, diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index a7742c104..8473ca7b7 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -46,6 +46,7 @@ AgentEndEvent, AgentEndStatus, AgentStartEvent, + StopReason, TextChunkEvent, ToolEndEvent, ToolEndStatus, @@ -53,6 +54,7 @@ TurnEndEvent, TurnEndStatus, TurnStartEvent, + end_status_for, ) from coder_eval.timing import close_window from coder_eval.utils import expand_env_vars @@ -305,7 +307,6 @@ def __init__( user_input: str, iteration: int, turn_start_time: float, - max_turns: int | None = None, ) -> None: self._agent = agent self.emit = emit @@ -317,10 +318,8 @@ def __init__( self.user_input = user_input self.iteration = iteration self.turn_start_time = turn_start_time - self.max_turns = max_turns self.timeout_hit = False - self.stopped_early_hit = False - self.max_turns_hit = False + self.stop_reason: StopReason | None = None self.finalized = False # Live pump scratch (set during streaming). @@ -488,23 +487,12 @@ def _flush_message(self, last: Any) -> None: @property def ended_cleanly(self) -> bool: - """True once the pump broke on purpose (cooperative stop or the turn cap). + """True once the pump broke on a ``should_stop`` reason. - Both are non-crash terminations, so an exception raised while tearing the - stream down afterwards must not be escalated into a retry. + A non-crash termination, so an exception raised while tearing the stream + down afterwards must not be escalated into a retry. """ - return self.stopped_early_hit or self.max_turns_hit - - def max_turns_reached(self) -> bool: - """True once this turn has produced ``max_turns`` visible turns. - - Delegates to ``EventCollector.visible_turn_count`` rather than - ``self.commands``, which SKIPS items whose telemetry the SDK does not - resolve; the collector counts every emitted tool end, which is what lands - in ``TurnRecord.commands``. Codex delivers one SDK turn per - ``communicate()``, so a native counter would cap at 1. - """ - return self.max_turns is not None and self.collector.visible_turn_count >= self.max_turns + return self.stop_reason is not None def dispatch(self, notification: Any) -> bool: """Route a notification to its handler. Returns True on ``turn/completed`` @@ -858,8 +846,7 @@ async def communicate( *, stream_callback: StreamCallback | None = None, timeout: float | None = None, - max_turns: int | None = None, - should_stop: Callable[[], bool] | None = None, + should_stop: Callable[[], StopReason | None] | None = None, ) -> TurnRecord: """Send a message to Codex and receive its response. @@ -867,15 +854,10 @@ async def communicate( user_input: The message/prompt to send stream_callback: Optional callback for real-time event streaming timeout: Hard wall-clock deadline in seconds - max_turns: Hard cap on VISIBLE turns — tool calls, the unit - ``result_metrics.visible_turn_count`` counts — enforced in-stream on - the same pump boundary as the cooperative stop. Codex delivers one - SDK turn per ``communicate()``, so a native turn counter would cap - at 1; see docs/agents/HARNESS_PARITY.md. - should_stop: Cooperative early-stop callback, polled after each - dispatched notification. When it returns True the pump breaks, - the in-flight turn is interrupted (best-effort) and the turn - finalizes cleanly as ``STOPPED_EARLY`` (``crashed=False``). + should_stop: The run's stop poll, called after each dispatched + notification. On a reason the pump breaks, the in-flight turn is + interrupted (best-effort) and the turn finalizes cleanly with + ``end_status_for(reason)`` (``crashed=False``). Returns: TurnRecord containing the complete interaction @@ -916,7 +898,6 @@ async def communicate( user_input=user_input, iteration=self._iteration, turn_start_time=turn_start_time, - max_turns=max_turns, ) try: @@ -1003,14 +984,9 @@ def _on_turn_timeout() -> None: self._state = AgentState.WORKING self._end_turn_ok() - # Precedence: timeout (raised above) > stopped_early > max_turns > done. + # Precedence: timeout (raised above) > the stop reason > done. # Rationale: .claude/notes/agents.md § Shared turn lifecycle - if state.stopped_early_hit: - status = AgentEndStatus.STOPPED_EARLY - elif state.max_turns_hit: - status = AgentEndStatus.TOOL_CALLS_EXHAUSTED - else: - status = AgentEndStatus.COMPLETED + status = end_status_for(state.stop_reason) if state.stop_reason is not None else AgentEndStatus.COMPLETED state.finalize(status, crashed=False, crash_reason=None) return collector.build_turn_record() @@ -1428,7 +1404,7 @@ def _format_turn_result(self, turn_result: Any) -> str: return str(turn_result) async def _run_turn_with_streaming( - self, state: _CodexTurnState, should_stop: Callable[[], bool] | None = None + self, state: _CodexTurnState, should_stop: Callable[[], StopReason | None] | None = None ) -> tuple[Any, Any, str]: """Drive ``turn.stream()`` through the per-turn state, emitting the standard event protocol; returns ``(turn_result, latest_token_usage, agent_text)``. @@ -1437,7 +1413,7 @@ async def _run_turn_with_streaming( drives the inner pump. ``state`` is mutated IN PLACE, so a mid-turn crash keeps the partial. - ``should_stop`` runs AFTER ``state.dispatch`` (the emission the watcher + ``should_stop`` runs AFTER ``state.dispatch`` (the emission the monitor latches on) and BEFORE the next notification is pulled. """ # Starts the turn without blocking, and opens the event stream. @@ -1455,17 +1431,10 @@ async def _run_turn_with_streaming( break if state.dispatch(notification): # True on a valid turn/completed break - if should_stop is not None and should_stop(): - state.stopped_early_hit = True - self._log.debug("Cooperative stop requested; ending notification pump at this boundary") - self._interrupt_active_turn() # best-effort; stops server-side spend - break - # The cap shares this boundary: the notification that reached it is - # dispatched whole, the next is never pulled. After the cooperative - # stop, so an armed early-stop wins a tie. - if state.max_turns_reached(): - state.max_turns_hit = True - self._log.debug("max_turns (%s visible turns) reached; ending notification pump", state.max_turns) + reason = should_stop() if should_stop is not None else None + if reason is not None: + state.stop_reason = reason + self._log.debug("Stop requested (%s); ending notification pump at this boundary", reason.value) self._interrupt_active_turn() # best-effort; stops server-side spend break finally: @@ -1485,13 +1454,13 @@ async def _run_turn_with_streaming( if not state.messages: state.messages.extend(self._messages_from_items(getattr(state.turn_result, "items", None), state.turn_id)) - # RUNS on a turn-cap stop, because recovery is also the only writer of the - # `parent_tool_use_id`-tagged messages `_fold_subagent_tokens` sums — so + # RUNS on a cap or budget stop, because recovery is also the only writer of + # the `parent_tool_use_id`-tagged messages `_fold_subagent_tokens` sums — so # skipping it drops the child threads' spend from the run's cost entirely. - # Still SKIPPED on a cooperative stop: an armed gate has already decided - # the run, and children may have no rollout yet. + # Still SKIPPED on an early-criterion stop: an armed gate has already + # decided the run, and children may have no rollout yet. # Rationale: .claude/notes/agents.md § Codex rollout rebuild - if state.spawned_children and not state.stopped_early_hit: + if state.spawned_children and state.stop_reason is not StopReason.EARLY_CRITERION: await self._recover_subagent_tool_calls( state.spawned_children, state.collab_results, diff --git a/src/coder_eval/agents/noop_agent.py b/src/coder_eval/agents/noop_agent.py index 47035f613..fb3c0ef4a 100644 --- a/src/coder_eval/agents/noop_agent.py +++ b/src/coder_eval/agents/noop_agent.py @@ -26,6 +26,7 @@ AgentEndEvent, AgentEndStatus, AgentStartEvent, + StopReason, TurnEndEvent, TurnEndStatus, TurnStartEvent, @@ -74,8 +75,7 @@ async def communicate( *, stream_callback: StreamCallback | None = None, timeout: float | None = None, - max_turns: int | None = None, - should_stop: Callable[[], bool] | None = None, + should_stop: Callable[[], StopReason | None] | None = None, ) -> TurnRecord: """Return an empty turn without contacting any model. diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index 84123b41e..5c0249d3a 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -59,6 +59,7 @@ AgentEndEvent, AgentEndStatus, AgentStartEvent, + StopReason, StreamEvent, TextChunkEvent, ToolEndEvent, @@ -67,6 +68,7 @@ TurnEndEvent, TurnEndStatus, TurnStartEvent, + end_status_for, ) from coder_eval.timing import close_window @@ -280,7 +282,6 @@ def __init__(self, *, task_id: str, iteration: int, user_input: str, model: str self.sequence = 0 self.stop_reason: str | None = None self.error_message: str | None = None - self.max_turns_exhausted = False # Guards the one-terminal-event rule; see finalize(). self.finalized = False # Guards _warn_token_shape: one report per turn, not one per step. @@ -1026,8 +1027,7 @@ async def communicate( *, stream_callback: StreamCallback | None = None, timeout: float | None = None, - max_turns: int | None = None, - should_stop: Callable[[], bool] | None = None, + should_stop: Callable[[], StopReason | None] | None = None, ) -> TurnRecord: if self.working_directory is None: raise RuntimeError("OpenCodeAgent.start() must be called before communicate()") @@ -1058,7 +1058,7 @@ def emit(event: StreamEvent) -> None: ) deadline = None if timeout is None else time.monotonic() + timeout - stopped_early = False + requested_stop: StopReason | None = None stderr_drain: asyncio.Future[bytes] | None = None # Bound OUTSIDE the try so `finally` can tell "never spawned" from # "spawned and possibly still running". @@ -1119,13 +1119,10 @@ def emit(event: StreamEvent) -> None: if not line: break - self._handle_line(line, state, max_turns=max_turns) + self._handle_line(line, state) - if state.max_turns_exhausted: - await self.kill() - break - if should_stop is not None and should_stop(): - stopped_early = True + requested_stop = should_stop() if should_stop is not None else None + if requested_stop is not None: await self.kill() break finally: @@ -1138,7 +1135,7 @@ def emit(event: StreamEvent) -> None: state, collector, stderr_drain, - stopped_early=stopped_early, + requested_stop=requested_stop, deadline=deadline, timeout=timeout, ) @@ -1193,7 +1190,7 @@ async def _settle_turn( collector: EventCollector, stderr_drain: asyncio.Future[bytes] | None, *, - stopped_early: bool, + requested_stop: StopReason | None, deadline: float | None, timeout: float | None, ) -> AgentEndStatus: @@ -1230,7 +1227,7 @@ async def _settle_turn( # A non-zero exit with no structured error still means the turn died: # surface stderr rather than reporting a silent empty success. - if proc.returncode not in (0, None) and not stopped_early and not state.max_turns_exhausted: + if proc.returncode not in (0, None) and requested_stop is None: detail = stderr_bytes.decode("utf-8", "replace").strip() or f"exit code {proc.returncode}" self._crash_turn(state, collector, f"OpenCode exited non-zero: {detail}") @@ -1243,7 +1240,7 @@ async def _settle_turn( # Rationale: .claude/notes/agents.md § Why a clean exit can still be a crash nothing_recognized = state.recognized_events == 0 finished_without_tokens = state.steps_finished > 0 and state.usage.is_empty() - if not stopped_early and not state.max_turns_exhausted and (nothing_recognized or finished_without_tokens): + if requested_stop is None and (nothing_recognized or finished_without_tokens): if nothing_recognized: seen = ", ".join(sorted(state.unrecognized_types)) or "none (stdout carried no JSON events)" detail = f"It emitted no recognized events at all. Unrecognized event types seen: {seen}." @@ -1265,11 +1262,7 @@ async def _settle_turn( else: self._crash_turn(state, collector, message) - if stopped_early: - return AgentEndStatus.STOPPED_EARLY - if state.max_turns_exhausted: - return AgentEndStatus.TOOL_CALLS_EXHAUSTED - return AgentEndStatus.COMPLETED + return end_status_for(requested_stop) if requested_stop is not None else AgentEndStatus.COMPLETED def _crash_turn( self, @@ -1307,11 +1300,8 @@ async def _timeout_turn( finally: self._capture_partial_turn(collector) - def _handle_line(self, line: bytes, state: _OpenCodeTurnState, *, max_turns: int | None = None) -> None: - """Parse one nd-JSON line and dispatch it. Never raises on bad input. - - A ``step_start`` past ``max_turns`` sets ``state.max_turns_exhausted`` instead of opening a step. - """ + def _handle_line(self, line: bytes, state: _OpenCodeTurnState) -> None: + """Parse one nd-JSON line and dispatch it. Never raises on bad input.""" raw = line.decode("utf-8", "replace").strip() if not raw: return @@ -1339,9 +1329,7 @@ def _handle_line(self, line: bytes, state: _OpenCodeTurnState, *, max_turns: int state.thread_id = session_id self._session_id = session_id - if event_type == _STEP_START and max_turns is not None and state.step_count >= max_turns: - state.max_turns_exhausted = True - elif event_type == _STEP_START: + if event_type == _STEP_START: state.on_step_start(part) elif event_type == _TEXT: state.on_text(part) diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 7bfbecdb7..60c956ca6 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -11,7 +11,7 @@ transient provider error internally — and ``agent_end`` is therefore NOT terminal. ``agent_settled`` (or EOF) is; the single ``AgentEndEvent`` is emitted there. -- ``turn_start`` is one per agent-loop step, and is the unit ``max_turns`` counts. +- ``turn_start`` is one per agent-loop step (``num_turns`` on the record). - ``message_end`` is ignored for token accounting: ``turn_end`` echoes the same assistant usage once per step, so reading both would double-count. @@ -67,6 +67,7 @@ AgentEndEvent, AgentEndStatus, AgentStartEvent, + StopReason, StreamEvent, TextChunkEvent, ToolEndEvent, @@ -75,6 +76,7 @@ TurnEndEvent, TurnEndStatus, TurnStartEvent, + end_status_for, ) from coder_eval.timing import TurnClock, close_window @@ -239,7 +241,7 @@ def __init__( self.messages: list[TranscriptMessage] = [] self.text_parts: list[str] = [] - # Pi `turn_start` events counted; this is what max_turns caps. + # Pi `turn_start` events counted. self.turn_count = 0 self.turn_id: str = "" # True between a step's `turn_start` and its `turn_end`. `finalize` needs @@ -259,7 +261,6 @@ def __init__( self.sequence = 0 self.stop_reason: str | None = None self.error_message: str | None = None - self.max_turns_exhausted = False # Guards the one-terminal-event rule; see finalize(). self.finalized = False # Count of events matched against the recognized Pi vocabulary (drift check), @@ -897,8 +898,7 @@ async def communicate( *, stream_callback: StreamCallback | None = None, timeout: float | None = None, - max_turns: int | None = None, - should_stop: Callable[[], bool] | None = None, + should_stop: Callable[[], StopReason | None] | None = None, ) -> TurnRecord: if self.working_directory is None: raise RuntimeError("PiAgent.start() must be called before communicate()") @@ -935,7 +935,7 @@ def emit(event: StreamEvent) -> None: # Deadlines stay on `time.monotonic()`, deliberately NOT the turn clock: # a deadline must not move when the wall clock steps. deadline = None if timeout is None else time.monotonic() + timeout - stopped_early = False + requested_stop: StopReason | None = None stderr_drain: asyncio.Future[bytes] | None = None # Bound OUTSIDE the try so `finally` can tell "never spawned" from # "spawned and possibly still running". @@ -994,13 +994,10 @@ def emit(event: StreamEvent) -> None: if not line: break - self._handle_line(line, state, max_turns=max_turns) + self._handle_line(line, state) - if state.max_turns_exhausted: - await self.kill() - break - if should_stop is not None and should_stop(): - stopped_early = True + requested_stop = should_stop() if should_stop is not None else None + if requested_stop is not None: await self.kill() break finally: @@ -1013,7 +1010,7 @@ def emit(event: StreamEvent) -> None: state, collector, stderr_drain, - stopped_early=stopped_early, + requested_stop=requested_stop, deadline=deadline, timeout=timeout, ) @@ -1063,7 +1060,7 @@ async def _settle_turn( collector: EventCollector, stderr_drain: asyncio.Future[bytes] | None, *, - stopped_early: bool, + requested_stop: StopReason | None, deadline: float | None, timeout: float | None, ) -> AgentEndStatus: @@ -1095,17 +1092,17 @@ async def _settle_turn( # intentional cuts: a cut can fire before the clearing `turn_end` arrives, # leaving a stale error from a turn pi was still retrying. # Rationale: .claude/notes/agents.md § Why a clean exit can still be a crash - if state.error_message is not None and not stopped_early and not state.max_turns_exhausted: + if state.error_message is not None and requested_stop is None: self._crash_turn(state, collector, f"Pi error: {state.error_message}") # A non-zero exit with no intentional cut means the turn died. - if proc.returncode not in (0, None) and not stopped_early and not state.max_turns_exhausted: + if proc.returncode not in (0, None) and requested_stop is None: detail = stderr_bytes.decode("utf-8", "replace").strip() or f"exit code {proc.returncode}" self._crash_turn(state, collector, f"Pi exited non-zero: {detail}") # A clean exit that recognized NO events is vocabulary drift. Intentional # cuts are exempt: a stop can land before the first event. - if not stopped_early and not state.max_turns_exhausted and state.recognized_events == 0: + if requested_stop is None and state.recognized_events == 0: seen = ", ".join(sorted(state.unrecognized_types)) or "none (stdout carried no JSON events)" self._crash_turn( state, @@ -1115,11 +1112,7 @@ async def _settle_turn( + "run from this CLI version.", ) - if stopped_early: - return AgentEndStatus.STOPPED_EARLY - if state.max_turns_exhausted: - return AgentEndStatus.TOOL_CALLS_EXHAUSTED - return AgentEndStatus.COMPLETED + return end_status_for(requested_stop) if requested_stop is not None else AgentEndStatus.COMPLETED def _crash_turn( self, @@ -1150,12 +1143,11 @@ async def _timeout_turn( finally: self._capture_partial_turn(collector) - def _handle_line(self, line: bytes, state: _PiTurnState, *, max_turns: int | None = None) -> None: + def _handle_line(self, line: bytes, state: _PiTurnState) -> None: """Parse one nd-JSON line and dispatch it. Never raises on bad input. ``agent_end`` is NOT terminal — only ``agent_settled`` / stdout EOF is — so - it is recognized, ignored, and the read loop keeps going. A ``turn_start`` - past ``max_turns`` sets ``state.max_turns_exhausted`` instead of opening a turn. + it is recognized, ignored, and the read loop keeps going. """ raw = line.decode("utf-8", "replace").strip() if not raw: @@ -1176,9 +1168,7 @@ def _handle_line(self, line: bytes, state: _PiTurnState, *, max_turns: int | Non elif len(state.unrecognized_types) < _MAX_UNRECOGNIZED_TYPES: state.unrecognized_types.add(event_type or "") - if event_type == "turn_start" and max_turns is not None and state.turn_count >= max_turns: - state.max_turns_exhausted = True - elif event_type == "turn_start": + if event_type == "turn_start": state.on_turn_start() elif event_type == "message_update": state.on_message_update(obj) diff --git a/src/coder_eval/cli/execute_command.py b/src/coder_eval/cli/execute_command.py index 51df9af06..f22da4124 100644 --- a/src/coder_eval/cli/execute_command.py +++ b/src/coder_eval/cli/execute_command.py @@ -165,7 +165,7 @@ def execute_command( metavar="PATH=VALUE", help=( "Override any resolved task-config field under agent/run_limits/sandbox, " - "e.g. -D run_limits.max_turns=30 -D agent.permission_mode=plan " + "e.g. -D run_limits.max_tool_calls=30 -D agent.permission_mode=plan " "-D agent.sdk_options.effort=high -D sandbox.docker.network=none. " "Repeatable. Validated against the schema. A path set by both an alias " "and -D is an error; values are YAML-parsed (on/off/yes/no stay strings). " diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index cad6cff0a..653a50f53 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -341,7 +341,7 @@ def run_command( metavar="PATH=VALUE", help=( "Override any resolved task-config field under agent/run_limits/sandbox, " - "e.g. -D run_limits.max_turns=30 -D agent.permission_mode=plan " + "e.g. -D run_limits.max_tool_calls=30 -D agent.permission_mode=plan " "-D agent.sdk_options.effort=high -D sandbox.docker.network=none. " "Repeatable. Validated against the schema. A path set by both an alias " "and -D is an error; values are YAML-parsed (on/off/yes/no stay strings). " diff --git a/src/coder_eval/config.py b/src/coder_eval/config.py index 946e6d141..9b2897154 100644 --- a/src/coder_eval/config.py +++ b/src/coder_eval/config.py @@ -43,7 +43,7 @@ _REMOVED_DEFAULT_KNOBS = { "DEFAULT_AGENT_MODEL": "agent.by_type.claude-code.model", "DEFAULT_PERMISSION_MODE": "agent.by_type.claude-code.permission_mode", - "DEFAULT_MAX_TURNS": "run_limits.max_turns", + "DEFAULT_MAX_TURNS": "run_limits.max_tool_calls", } diff --git a/src/coder_eval/criteria/agent_judge.py b/src/coder_eval/criteria/agent_judge.py index 896b39992..5f0d7b9bf 100644 --- a/src/coder_eval/criteria/agent_judge.py +++ b/src/coder_eval/criteria/agent_judge.py @@ -186,11 +186,7 @@ async def _check_impl_async( ) try: - turn = await runner.run_async( - user_msg, - max_turns=criterion.max_turns, - turn_timeout=float(criterion.turn_timeout), - ) + turn = await runner.run_async(user_msg, turn_timeout=float(criterion.turn_timeout)) except (TurnTimeoutError, AgentCrashError) as e: # Two pre-output failure modes share one return path (watchdog timeout, # SDK error result). No turn was produced, so no token usage is @@ -269,11 +265,11 @@ def _build_agent_config( deep=True, ) # SECURITY: the floor is present even when the user supplied their own list. - # Set-union is idempotent and order-independent. config.ignore_patterns = list({*config.ignore_patterns, *JUDGE_SECURITY_IGNORE_FLOOR}) # SECURITY/contract: the judge MUST be able to call its verdict tool, so the # tool name is forced in regardless of the user's override. config.allowed_tools = list({*(config.allowed_tools or []), SUBMIT_VERDICT_MCP_TOOL_NAME}) + config.sdk_options = {**config.sdk_options, "max_turns": criterion.max_turns} return config diff --git a/src/coder_eval/evaluation/sub_agent.py b/src/coder_eval/evaluation/sub_agent.py index f9e17425c..e1940a600 100644 --- a/src/coder_eval/evaluation/sub_agent.py +++ b/src/coder_eval/evaluation/sub_agent.py @@ -94,7 +94,7 @@ def __init__( # when the caller passed ``capture=None``. self.capture = capture - async def run_async(self, user_msg: str, *, max_turns: int | None, turn_timeout: float) -> TurnRecord: + async def run_async(self, user_msg: str, *, turn_timeout: float) -> TurnRecord: """Copy sandbox → start agent → communicate → stop. Kill on any exception. Async so a genuine network/subprocess wait yields the event loop instead of @@ -155,14 +155,13 @@ async def run_async(self, user_msg: str, *, max_turns: int | None, turn_timeout: logger.info( "sub_agent: starting (model=%s, max_turns=%s, allowed_tools=%s)", self._agent_config.model, - max_turns, + self._agent_config.sdk_options.get("max_turns"), self._agent_config.allowed_tools, ) turn = await self._run_agent( agent, judge_dir, user_msg, - max_turns, turn_timeout, plugin_tools_dir=self._sandbox.plugin_tools_dir, ) @@ -203,7 +202,6 @@ async def _run_agent( agent: ClaudeCodeAgent, judge_dir: Path, user_msg: str, - max_turns: int | None, turn_timeout: float, *, plugin_tools_dir: str | None = None, @@ -215,7 +213,7 @@ async def _run_agent( """ try: await agent.start(str(judge_dir), plugin_tools_dir=plugin_tools_dir) - return await agent.communicate(user_msg, timeout=turn_timeout, max_turns=max_turns) + return await agent.communicate(user_msg, timeout=turn_timeout) except BaseException: with contextlib.suppress(Exception): await agent.kill() diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py index 125906932..b5f196f0a 100644 --- a/src/coder_eval/harbor/packager.py +++ b/src/coder_eval/harbor/packager.py @@ -601,10 +601,10 @@ def _write_agent_phase_task_yaml( payload["pre_run"] = [c.model_dump(mode="json", exclude_none=True) for c in task.pre_run] if task.run_limits is not None: # `CoderEvalAgent.run()` invokes `coder-eval execute` against this - # file, which enforces `max_turns`/`turn_timeout`/`task_timeout`/the + # file, which enforces `max_tool_calls`/`turn_timeout`/`task_timeout`/the # token+USD budget caps during the agent phase itself -- dropping this # silently replaced a declared cap with the packaged default - # experiment's (`max_turns: 100`, `turn_timeout: 300`, no `max_usd` / + # experiment's (`max_tool_calls: 100`, `turn_timeout: 300`, no `max_usd` / # token ceiling at all). payload["run_limits"] = task.run_limits.model_dump(mode="json", exclude_none=True) (env_dir / "task.yaml").write_text(yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8") diff --git a/src/coder_eval/models/agent_config.py b/src/coder_eval/models/agent_config.py index c16b9916d..f8df2d64c 100644 --- a/src/coder_eval/models/agent_config.py +++ b/src/coder_eval/models/agent_config.py @@ -78,7 +78,6 @@ class LocalPluginConfig(TypedDict): "stderr", "debug_stderr", "resume", - "max_turns", "session_id", "session_store", "session_store_flush", @@ -88,6 +87,8 @@ class LocalPluginConfig(TypedDict): "fork_session", # budgeting -- overlaps RunLimits, which the orchestrator enforces with # explicit FinalStatus codes. Two guards would disagree on counts. + # (`max_turns` is the SDK's own agent-loop cap; the framework cap is + # `run_limits.max_tool_calls`.) "max_budget_usd", "task_budget", # security-critical: arbitrary code injection or settings-bypass diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index 4a613e6a7..21499196c 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -145,8 +145,8 @@ def is_stop_armed(self) -> bool: Always ``False`` on the base: only live-observable criteria (``LiveSuccessCriterion`` subclasses) carry stop triggers, so arming an unobservable criterion is unrepresentable rather than a validation - error. The armed set drives both the runtime watcher - (``EarlyStopWatcher``) and the weighted armed gate + error. The armed set drives both the runtime TurnMonitor + (``TurnMonitor``) and the weighted armed gate (``EvaluationResult.armed_criteria_passed``). """ return False @@ -210,8 +210,8 @@ class StopEarlyPolicy(BaseModel): """Per-criterion early-stop policy — presence of the block IS the arming. Attaching ``stop_early:`` to a live-observable criterion arms it for the - run's early-stop watcher — there is no run-level master switch; the block - alone activates the watcher (``run_limits.stop_early: false`` is the + run's TurnMonitor — there is no run-level master switch; the block + alone arms the TurnMonitor (``run_limits.stop_early: false`` is the run-level veto). Arming carries ONE implicit trigger — a definitive *effective* fail (a native live-fail, or the ``decide_within`` timeout expiring) may end the run under the weighted ceiling rule — plus the two knobs below. A trigger @@ -244,7 +244,7 @@ class StopEarlyPolicy(BaseModel): "live-fail, reported as reason 'decision_budget_exceeded'. Inert on an instance " "that can only ever live-fail (a distractor/guard, whose 'undecided' is its " "success state). None (default) = no timeout. The step count is CUMULATIVE across " - "every retry attempt of the turn (the same EarlyStopWatcher instance, and its " + "every retry attempt of the turn (the same TurnMonitor instance, and its " "counters, persist across retries) — including attempts that ultimately crashed or " "timed out before this criterion's own investigation even began. Size it with that " "headroom in mind." @@ -268,7 +268,7 @@ class LiveSuccessCriterion(BaseSuccessCriterion): Only ``SkillTriggeredCriterion`` / ``CommandExecutedCriterion`` subclass this today; a criterion type is "live-observable" iff it is a ``LiveSuccessCriterion`` subclass — the single source of truth - ``validate_early_stop`` / ``EarlyStopWatcher`` consult (no separate + ``validate_early_stop`` / ``TurnMonitor`` consult (no separate checker-side flag to keep in sync). """ @@ -276,7 +276,7 @@ class LiveSuccessCriterion(BaseSuccessCriterion): default=None, description=( "Opt-in early-stop policy block; its PRESENCE arms this criterion for the run's " - "early-stop watcher — the block alone activates the watcher, there is no run-level " + "TurnMonitor — the block alone arms it, there is no run-level " "master switch (run_limits.stop_early: false is the run-level veto). An armed " "criterion's definitive effective FAIL — a native live-fail, or the decide_within " "timeout expiring — may end the run under the weighted ceiling rule (deferred " @@ -308,7 +308,7 @@ def live_decidable_polarities(self) -> frozenset[LivePolarity]: Must return a subset of the polarities the corresponding checker's ``live_verdict`` can ever emit for this criterion type. Used by - ``EarlyStopWatcher`` to decide which triggers of an armed criterion's + ``TurnMonitor`` to decide which triggers of an armed criterion's ``stop_early`` block are live for this instance: ``on_pass: stop`` needs ``"pass"``, the implicit fail trigger needs ``"fail"``, and ``decide_within`` needs ``"pass"`` (a fail-only instance's 'undecided' @@ -1044,7 +1044,7 @@ def live_decidable_polarities(self) -> frozenset[LivePolarity]: live-``fail`` (a wrong skill engaging is a decidable miss; its absence is not). - ``EarlyStopWatcher`` consults this set to decide which triggers are + ``TurnMonitor`` consults this set to decide which triggers are live per instance: on a positive row ``on_pass: stop`` and ``decide_within`` are live while the implicit fail trigger is inert; on a distractor row the reverse. That per-row adaptivity is what lets diff --git a/src/coder_eval/models/limits.py b/src/coder_eval/models/limits.py index 199b884e7..188e720ce 100644 --- a/src/coder_eval/models/limits.py +++ b/src/coder_eval/models/limits.py @@ -10,7 +10,7 @@ DEFAULT_STOP_EARLY_GATE_THRESHOLD: Final[float] = 1.0 """Default ``stop_early_gate_threshold``: reproduces strict-AND gating exactly. -Single-sourced here so the field default below, the watcher's ``for_task`` +Single-sourced here so the field default below, the monitor's ``for_task`` fallback, and the orchestrator's finalize fallback can never drift apart. """ @@ -18,8 +18,9 @@ class RunLimits(BaseModel): """Run-time caps on a task. - Unifies structural caps (max_turns, task_timeout, turn_timeout) and - budget caps (tokens, USD). Structural caps stop the task. Budget caps are + Unifies structural caps (max_tool_calls, task_timeout, turn_timeout) and + budget caps (tokens, USD). The tool-call cap stops the task at the agent's + next poll boundary and is cumulative across every turn. Budget caps are checked after each completed agent turn and are cumulative across all turns of a single task: a single-iteration task finishes and is then marked over budget, and a dialog stops after the turn that crossed the @@ -31,10 +32,16 @@ class RunLimits(BaseModel): model_config = ConfigDict(extra="forbid") - max_turns: int | None = Field( + max_tool_calls: int | None = Field( default=None, gt=0, - description="Max agent inner-loop turns per iteration. None = SDK default.", + description=( + "Hard cap on resolved tool calls across the whole task (every retry attempt and every " + "dialog turn). Enforced by the TurnMonitor at the agent's next poll boundary on every " + "harness: the round that reaches the cap is processed whole, so tool calls already in " + "flight can still land after it. The run finalizes cleanly as tool_calls_exhausted; " + "criteria are still checked. None = no cap." + ), ) expected_tool_calls: int | None = Field( default=None, @@ -42,7 +49,7 @@ class RunLimits(BaseModel): description=( "Soft target for cumulative visible tool calls across a task (each resolved tool call " "counts 1, plus 1 for the final reply when present). Exceeding it logs a one-shot warning " - "and badges the report; the run is NOT aborted (use max_turns for a hard cap)." + "and badges the report; the run is NOT aborted (use max_tool_calls for a hard cap)." ), ) task_timeout: int | None = Field( @@ -101,7 +108,7 @@ class RunLimits(BaseModel): description=( "Run-level early-stop KILL SWITCH — there is no run-level master arm. Arming is " "per-criterion: a live-observable criterion's stop_early: block alone activates " - "the run's early-stop watcher. None (default): armed criteria decide; the run may " + "the run's TurnMonitor. None (default): armed criteria decide; the run may " "end early once they resolve mid-run (pass-stop when the on_pass=stop subset's " "weighted score is GUARANTEED to reach stop_early_gate_threshold regardless of " "any criterion still undecided, fail-stop when the armed set's weighted score is " diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 692a5e730..464d643e9 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -485,7 +485,7 @@ class EarlyStopReason(StrEnum): through ``armed_criteria_passed``'s weighted gate. ``DECISION_BUDGET_EXCEEDED`` is a reporting label only: it marks a fail-stop whose deciding criterion timed out undecided past its - ``stop_early.decide_within`` (an *effective* fail latched by the watcher) + ``stop_early.decide_within`` (an *effective* fail latched by the monitor) rather than live-failing natively. """ @@ -497,7 +497,7 @@ class EarlyStopReason(StrEnum): class EarlyStopInfo(BaseModel): """Records why and when a run stopped early (``None`` when it ran to completion). - Populated by the orchestrator's ``EarlyStopWatcher`` at the moment the armed + Populated by the orchestrator's ``TurnMonitor`` at the moment the armed criteria are decided mid-run. ``early_stop is not None`` is itself the "stopped early" flag — no separate bool. Serialized as part of ``EvaluationResult`` to ``task.json``; defaults to ``None`` on old files, so @@ -523,7 +523,7 @@ class EarlyStopInfo(BaseModel): + "advisory without re-deriving from task_config.", ) sdk_turn_index: int = Field( - description="SDK inner-turn count at the stop (watcher counts TurnStartEvents). NOT the " + description="SDK inner-turn count at the stop (the monitor counts TurnStartEvents). NOT the " + "orchestrator iteration, which is always 1 in single-shot." ) tool_call_index: int = Field( @@ -533,10 +533,9 @@ class EarlyStopInfo(BaseModel): + "stop. Read it as 'which call decided', not as a count of fully-completed tool calls." ) elapsed_seconds: float = Field(description="Wall-clock seconds from the first agent-start event to the stop.") - turns_remaining_at_stop: int | None = Field( + tool_calls_remaining_at_stop: int | None = Field( default=None, - description="max_turns - sdk_turn_index (an upper bound on turns avoided, not a measured " - + "saving); None when max_turns is unset.", + description="max_tool_calls - tool_call_index; None when the cap is unset.", ) gate_threshold: float = Field( default=DEFAULT_STOP_EARLY_GATE_THRESHOLD, @@ -715,7 +714,7 @@ class EvaluationResult(BaseModel): ) # Early-stop telemetry (only populated when the run was cut short by the - # armed-criteria watcher; None on a full run). See EarlyStopInfo. + # armed-criteria monitor; None on a full run). See EarlyStopInfo. early_stop: EarlyStopInfo | None = Field( default=None, description=( diff --git a/src/coder_eval/orchestration/early_stop.py b/src/coder_eval/orchestration/early_stop.py index d74e2c310..f94c1549e 100644 --- a/src/coder_eval/orchestration/early_stop.py +++ b/src/coder_eval/orchestration/early_stop.py @@ -1,8 +1,8 @@ -"""Early-stop-on-criterion: resolution-time validation + runtime watcher. +"""Early-stop-on-criterion: resolution-time validation. Opt-in. Arming lives ENTIRELY on the criterion — there is no run-level master switch. A ``stop_early:`` block on a ``LiveSuccessCriterion`` (so arming an -unobservable criterion is unrepresentable) alone activates the run's watcher; +unobservable criterion is unrepresentable) alone arms the run's ``TurnMonitor``; ``run_limits.stop_early: false`` is the run-level KILL SWITCH, and ``stop_early: true`` — the removed master arm — is rejected at resolution. @@ -14,8 +14,9 @@ A trigger whose polarity an instance can never decide is INERT BY DESIGN. -This module owns the whole feature: ``validate_early_stop`` (resolution-time -guardrails) and ``EarlyStopWatcher`` (the runtime ``StreamCallback``). +This module owns the resolution-time guardrails (``validate_early_stop``) and the +arming predicate; ``orchestration.turn_monitor.TurnMonitor`` evaluates the armed +criteria at run time. Rationale: .claude/notes/orchestration.md § Early stop on criterion """ @@ -23,36 +24,13 @@ from __future__ import annotations import logging -import time -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING -from coder_eval.models import ( - DEFAULT_STOP_EARLY_GATE_THRESHOLD, - EarlyStopInfo, - EarlyStopReason, - LivePolarity, - LiveSuccessCriterion, - StopEarlyPolicy, -) from coder_eval.orchestration.harness_contract import TaskResolutionError, registration_for -from coder_eval.streaming.collector import EventCollector -from coder_eval.streaming.events import ( - AgentStartEvent, - StreamEvent, - ToolEndEvent, - ToolEndStatus, - ToolStartEvent, - TurnStartEvent, -) if TYPE_CHECKING: - from coder_eval.criteria.base import BaseCriterion, LiveVerdict - from coder_eval.models import CommandTelemetry, TaskDefinition - - # In the TYPE_CHECKING block (only lazy annotations reference it), so the - # names are real references rather than strings analyzers cannot resolve. - _ArmedPair = tuple[LiveSuccessCriterion, BaseCriterion[Any]] + from coder_eval.models import TaskDefinition logger = logging.getLogger(__name__) @@ -68,11 +46,11 @@ class EarlyStopConfigError(TaskResolutionError): def early_stop_active(task: TaskDefinition) -> bool: - """True iff this run should build a watcher: >= 1 armed criterion, kill switch not thrown. + """True iff this run arms the monitor's criteria: >= 1 armed criterion, kill switch not thrown. The single arming predicate the orchestrator consults. Deliberately ignores ``run_limits.stop_early is True`` (the removed master arm) — that value is - rejected by ``validate_early_stop``, which every path runs before watcher + rejected by ``validate_early_stop``, which every path runs before monitor creation, so it can never reach a live run. """ limits = task.run_limits @@ -166,379 +144,3 @@ def validate_early_stop(task: TaskDefinition) -> None: + "> 0.0 on an armed task (a threshold of 0 trivially passes the armed gate " + "regardless of whether any armed criterion actually decided)." ) - - -class EarlyStopWatcher: - """Observes the agent event stream and trips the cooperative interrupt. - - A ``StreamCallback`` composed into the agent's callback chain. It maintains its - OWN ``EventCollector``, so each ``live_verdict`` sees a fresh single-element - partial trajectory. On every tool call it evaluates the armed criteria still - undecided and applies the stop rule; once a stop fires the decision is LATCHED - and further events are ignored. - - The orchestrator polls ``should_stop`` and afterwards reads ``info``. - - FAIL-OPEN: a raising ``live_verdict`` disarms the watcher and degrades to a - full run, which can never produce a FALSE early stop. - - Rationale: .claude/notes/orchestration.md § Inert triggers are by design, and the watcher fails open - """ - - def __init__( - self, - task_id: str, - armed: list[_ArmedPair], - *, - max_turns: int | None, - gate_threshold: float = DEFAULT_STOP_EARLY_GATE_THRESHOLD, - ) -> None: - self._task_id = task_id - self._armed = armed - self._gate_threshold = gate_threshold - self._armed_weight = sum(c.weight for c, _ in armed) - # Per-instance decidable polarities, aligned with `_armed`, static for the - # run. A trigger whose polarity the instance cannot decide is inert. - self._decidable: list[frozenset[LivePolarity]] = [ - criterion.live_decidable_polarities() for criterion, _checker in armed - ] - # Effective per-instance triggers (inert ones already resolved away). - # Every armed pair carries a stop_early block by construction - # (is_stop_armed == block presence); an explicit raise (not an assert, - # which -O strips) keeps a blockless pair from slipping through. - blocks: list[StopEarlyPolicy] = [] - for criterion, _checker in armed: - if criterion.stop_early is None: - raise ValueError(f"criterion {criterion.type!r} passed to EarlyStopWatcher without a stop_early block") - blocks.append(criterion.stop_early) - self._pass_trigger: list[bool] = [ - block.on_pass == "stop" and "pass" in pol for block, pol in zip(blocks, self._decidable, strict=True) - ] - # IMPLICIT in arming: an armed criterion's native live-fail may always - # stop the run (ceiling-gated). Inert when it cannot live-fail. - self._fail_trigger: list[bool] = ["fail" in pol for pol in self._decidable] - # A timeout only means anything for an instance waiting to observe a - # PASS: a fail-only instance's 'undecided' IS its success state. - self._budget: list[int | None] = [ - block.decide_within if "pass" in pol else None for block, pol in zip(blocks, self._decidable, strict=True) - ] - if not any(self._pass_trigger) and not any(self._fail_trigger) and all(b is None for b in self._budget): - # Legal for a fanned row whose armed lines are all inert for its role, - # but on a non-fanned task this is dead config. - logger.warning("[%s] all armed stop triggers are inert for this row; run cannot stop early", task_id) - self._max_turns = max_turns - self._collector = EventCollector() - self._sdk_turn_index = 0 - self._tool_call_index = 0 - self._started_monotonic: float | None = None - # Once an entry leaves "undecided" on a RESOLVED round its checker is - # never polled again. `_budget_expired` marks a latched fail as - # timeout-driven, reported as DECISION_BUDGET_EXCEEDED. - # Rationale: .claude/notes/orchestration.md § Verdicts latch, and the decision happens on the CALL - self._latched: list[LiveVerdict] = ["undecided"] * len(armed) - self._budget_expired: list[bool] = [False] * len(armed) - # For the "which criterion flipped to pass" attribution. Reassigned ONLY - # at the end of a non-firing evaluation, so it always holds the PREVIOUS - # round when a stop fires. - self._prev_verdicts: list[LiveVerdict] = ["undecided"] * len(armed) - self._info: EarlyStopInfo | None = None - self._disarmed = False - - @classmethod - def for_task(cls, task: TaskDefinition) -> EarlyStopWatcher: - """Build a watcher for an armed task (instantiates the armed criteria's checkers). - - The criteria registry is imported lazily here — it is not initialized at - module import time. Checker classes take no ctor args. Only - ``LiveSuccessCriterion`` instances can be armed (the trigger fields - exist nowhere else), so the ``isinstance`` filter is a pyright - narrowing aid, not a behavioral guard. - """ - from coder_eval.criteria import CriterionRegistry, init_criteria - - init_criteria(validate=False) - armed: list[_ArmedPair] = [ - (c, CriterionRegistry.get_checker(c.type)()) - for c in task.success_criteria - if isinstance(c, LiveSuccessCriterion) and c.is_stop_armed - ] - max_turns = task.run_limits.max_turns if task.run_limits is not None else None - gate_threshold = ( - task.run_limits.stop_early_gate_threshold - if task.run_limits is not None - else DEFAULT_STOP_EARLY_GATE_THRESHOLD - ) - return cls(task.task_id, armed, max_turns=max_turns, gate_threshold=gate_threshold) - - def on_event(self, event: StreamEvent) -> None: - """Fail-open wrapper around ``_on_event_impl``: any unexpected exception - anywhere in the round — the collector reduction included, not just the - verdict-collection loop — disarms the watcher and degrades to a full - run. The agent-side ``safe_emit`` swallows callback exceptions, so - without disarming here a raising collector would leave the watcher - silently evaluating a corrupted partial trajectory on every subsequent - event with ``_disarmed`` still False. - """ - if self._info is not None or self._disarmed: - return - try: - self._on_event_impl(event) - except Exception: - self._disarmed = True - logger.error( - "[%s] early-stop event handling raised unexpectedly; disarming watcher, run degrades to a full run", - self._task_id, - exc_info=True, - ) - - def _on_event_impl(self, event: StreamEvent) -> None: - """Forward the event to the internal collector; evaluate on each tool call. - - Counts ``TurnStartEvent`` for ``sdk_turn_index`` and each dispatched call - for the 1-based ``tool_call_index``, stamping the wall-clock origin at the - FIRST ``AgentStartEvent`` only, so a retry does not reset it. - - The decision is evaluated on the tool CALL. It is not in the collector yet - (which reduces commands from ``ToolEndEvent``), so it is passed in as the - in-flight command; ``tool_call_index`` increments on the resolved end, so - it stays a count of COMPLETED calls. - - UNRESOLVED tool ends are RECORDED but never counted or evaluated on — they - must still land in the collector, or the watcher would reduce a strictly - smaller command set than the authoritative check. - - Rationale: .claude/notes/orchestration.md § Verdicts latch, and the decision happens on the CALL - """ - if isinstance(event, AgentStartEvent): - if self._started_monotonic is None: - self._started_monotonic = time.monotonic() - elif isinstance(event, TurnStartEvent): - self._sdk_turn_index += 1 - elif isinstance(event, ToolStartEvent): - # Decide on the call, evaluating with it appended as the in-flight - # command (it has no ToolEnd to count yet, so report it as +1). - self._evaluate_impl(in_flight=event.tool) - return - elif isinstance(event, ToolEndEvent): - if event.status == ToolEndStatus.UNRESOLVED: - # Trajectory parity with the agent's collector: record, but do - # not count a round or evaluate. - self._collector.on_event(event) - return - self._tool_call_index += 1 - self._collector.on_event(event) - self._evaluate_impl() - return - self._collector.on_event(event) - - def should_stop(self) -> bool: - """The cooperative interrupt the agent polls after each dispatched message.""" - return self._info is not None - - @property - def info(self) -> EarlyStopInfo | None: - """The recorded stop info, or ``None`` if no stop fired (incl. after disarm).""" - return self._info - - @property - def disarmed(self) -> bool: - """True once a ``live_verdict`` raised and the watcher degraded to a full run.""" - return self._disarmed - - def _ceiling(self, verdicts: list[LiveVerdict]) -> float: - """Best-case weighted score over the WHOLE armed set, given current verdicts. - - Every already-failed criterion (native live-fail or expired budget) is - pinned at 0 (a monotonic ``live_verdict`` guarantees it stays failed); - every ``pass`` or still ``undecided`` criterion is credited its full - weight (the optimistic assumption that it could still end up scoring - 1.0). This is the same weighting ``EvaluationResult. - armed_criteria_passed`` uses for the real, final gate, so ``ceiling < - gate_threshold`` means the gate is mathematically guaranteed to fail no - matter how the trajectory continues. - """ - if self._armed_weight <= 0.0: - # Fails closed, as `armed_criteria_passed` does for the same unreachable case. - return 0.0 - return sum(c.weight for (c, _checker), v in zip(self._armed, verdicts, strict=True) if v != "fail") / ( - self._armed_weight - ) - - def _floor(self, verdicts: list[LiveVerdict], indices: list[int]) -> float | None: - """Worst-case weighted score over the given armed-index subset, given current verdicts. - - Mirrors ``_ceiling`` for the opposite direction: every still-undecided - (or already-``fail``) criterion in ``indices`` is credited nothing (the - pessimistic assumption that it could still end up scoring 0); only an - already-``pass`` criterion contributes its weight. Returns ``None`` - when the subset's total weight is 0 (the vacuous case — nothing to - bound), so callers don't have to special-case an empty numerator over - an empty denominator. - """ - weight = sum(self._armed[i][0].weight for i in indices) - if weight <= 0.0: - return None - return sum(self._armed[i][0].weight for i in indices if verdicts[i] == "pass") / weight - - def _collect_verdicts(self, in_flight: CommandTelemetry | None, tool_call_index: int) -> list[LiveVerdict]: - """One round of effective verdicts, latching decided ones on resolved rounds. - - A latched (non-``undecided``) verdict is returned as-is — its checker is - never polled again (the checkers' monotonicity contract makes re-polling - pure waste). A fresh ``undecided`` on a pass-capable instance whose - ``decide_within`` budget has expired becomes an *effective* - ``fail`` (marked in ``_budget_expired`` for reason attribution). - - Latching only happens on RESOLVED rounds (``in_flight is None``): an - in-flight round's verdict may fire a stop this round, but is not - persisted — a dispatched call that never resolves (crashed attempt) - must not leave a stale verdict behind across retries. The verdict is - recomputed from the collector's resolved commands on the next round. - """ - record = self._collector.build_turn_record() - if in_flight is not None: - # No ToolEnd yet, so the collector has not captured it. Append and - # re-sort by sequence to keep the partial trajectory in order. - record.commands = sorted([*record.commands, in_flight], key=lambda c: c.sequence_number) - records = [record] - verdicts: list[LiveVerdict] = [] - for i, (criterion, checker) in enumerate(self._armed): - if self._latched[i] != "undecided": - verdicts.append(self._latched[i]) - continue - try: - verdict: LiveVerdict = checker.live_verdict(criterion, records) - except Exception: - # Log WHICH criterion raised before re-raising to on_event's - # generic handler, where that context would be lost. - logger.error( - "[%s] early-stop live_verdict raised for criterion %r", - self._task_id, - criterion.type, - exc_info=True, - ) - raise - budget = self._budget[i] - budget_expired = verdict == "undecided" and budget is not None and tool_call_index >= budget - if budget_expired: - verdict = "fail" - if in_flight is None and verdict != "undecided": - self._latched[i] = verdict - self._budget_expired[i] = budget_expired - verdicts.append(verdict) - return verdicts - - def _budget_drove(self, index: int, verdicts: list[LiveVerdict], tool_call_index: int) -> bool: - """True when ``index``'s ``fail`` is timeout-driven rather than a native live-fail. - - Reads the persistent ``_budget_expired`` latch when set; for a - transient (in-flight, not-yet-latched) fail it re-derives: a fail on an - instance that cannot natively live-fail, with an expired budget, can - only have come from the timeout. A native live-fail on an instance - whose budget also happens to be expired reports as a native fail — - ``_collect_verdicts`` only converts the verdict when the checker itself - returned ``undecided``. - """ - if self._budget_expired[index]: - return True - budget = self._budget[index] - return ( - verdicts[index] == "fail" - and self._latched[index] == "undecided" - and budget is not None - and tool_call_index >= budget - and "fail" not in self._decidable[index] - ) - - def _evaluate_impl(self, in_flight: CommandTelemetry | None = None) -> None: - # An in-flight call has not been counted by a ToolEnd yet, so report it as - # the next (1-based) tool call. - tool_call_index = self._tool_call_index + (1 if in_flight is not None else 0) - verdicts = self._collect_verdicts(in_flight, tool_call_index) - - # RECALL DEFERRAL: a fail-stop is HELD while any pass-capable armed - # criterion is still undecided and within budget. A row with zero - # pass-capable criteria defers nothing. - # Rationale: .claude/notes/orchestration.md § Precision is traded, recall is not - pass_capable_undecided = any( - v == "undecided" and "pass" in pol for v, pol in zip(verdicts, self._decidable, strict=True) - ) - - # A criterion whose effective verdict is "fail" is a CANDIDATE; the stop - # fires only once the ceiling bound can no longer reach `gate_threshold`. - # Rationale: .claude/notes/orchestration.md § The ceiling and floor bounds - if not pass_capable_undecided: - # Deterministic precedence: a native live-fail candidate always wins - # over a budget-driven one, so the persisted/telemetry reason cannot - # flip between CRITERION_FAILED and DECISION_BUDGET_EXCEEDED on a - # mere reorder of ``success_criteria`` when both resolve on the same - # round. Within each class, first criteria-order match wins. - native_fails = [ - i - for i, v in enumerate(verdicts) - if v == "fail" and self._fail_trigger[i] and not self._budget_drove(i, verdicts, tool_call_index) - ] - budget_fails = [ - i for i, v in enumerate(verdicts) if v == "fail" and self._budget_drove(i, verdicts, tool_call_index) - ] - candidate_index = native_fails[0] if native_fails else (budget_fails[0] if budget_fails else None) - if candidate_index is not None and self._ceiling(verdicts) < self._gate_threshold: - reason = EarlyStopReason.CRITERION_FAILED if native_fails else EarlyStopReason.DECISION_BUDGET_EXCEEDED - self._fire(reason, self._armed[candidate_index][0], tool_call_index=tool_call_index) - return - - # Pass-stop: the on_pass=stop subset's FLOOR already meets - # ``gate_threshold``. Distractors are excluded from both the numerator and - # the denominator; no on_pass=stop criteria at all returns None. HELD while - # any pass-capable armed criterion OUTSIDE the subset is undecided -- - # cutting there would freeze a sibling's expected signal out of the run. - # Rationale: .claude/notes/orchestration.md § The ceiling and floor bounds - pass_stop_indices = [i for i, armed_pass in enumerate(self._pass_trigger) if armed_pass] - outside_pass_capable_undecided = any( - v == "undecided" and "pass" in pol and not armed_pass - for v, pol, armed_pass in zip(verdicts, self._decidable, self._pass_trigger, strict=True) - ) - if not outside_pass_capable_undecided: - floor = self._floor(verdicts, pass_stop_indices) - if floor is not None and floor >= self._gate_threshold: - # Deciding criterion = the last on_pass=stop (criteria order) whose - # verdict flipped vs the previous round; fall back to the last one. - deciding = self._armed[pass_stop_indices[-1]][0] - for i in pass_stop_indices: - if verdicts[i] != self._prev_verdicts[i]: - deciding = self._armed[i][0] - self._fire(EarlyStopReason.CRITERION_PASSED, deciding, tool_call_index=tool_call_index) - return - - # No stop this round — record the verdicts so the next round can detect - # flips. Resolved rounds only: an in-flight round's verdicts are - # deliberately not latched (the call may never resolve), so persisting - # them here would let a transient round mask the real flip attribution. - if in_flight is None: - self._prev_verdicts = verdicts - - def _fire(self, reason: EarlyStopReason, criterion: LiveSuccessCriterion, *, tool_call_index: int) -> None: - elapsed = 0.0 - if self._started_monotonic is not None: - elapsed = max(time.monotonic() - self._started_monotonic, 0.0) - turns_remaining = None if self._max_turns is None else max(self._max_turns - self._sdk_turn_index, 0) - self._info = EarlyStopInfo( - reason=reason, - deciding_criterion_type=criterion.type, - deciding_criterion_description=criterion.description, - armed_criteria=[f"{c.type}: {c.description}" for c, _ in self._armed], - sdk_turn_index=self._sdk_turn_index, - tool_call_index=tool_call_index, - elapsed_seconds=elapsed, - turns_remaining_at_stop=turns_remaining, - gate_threshold=self._gate_threshold, - ) - logger.info( - "[%s] early-stop fired: reason=%s deciding=%s sdk_turn=%d tool_call=%d elapsed=%.2fs", - self._task_id, - reason.value, - criterion.type, - self._sdk_turn_index, - tool_call_index, - elapsed, - ) diff --git a/src/coder_eval/orchestration/turn_monitor.py b/src/coder_eval/orchestration/turn_monitor.py new file mode 100644 index 000000000..776bc338a --- /dev/null +++ b/src/coder_eval/orchestration/turn_monitor.py @@ -0,0 +1,463 @@ +"""The run's single ``should_stop`` answerer: armed early stop and the tool-call cap. + +``TurnMonitor`` is a ``StreamCallback`` composed into the agent's callback chain +for the whole task. It owns ONE ``EventCollector`` across every retry attempt and +every dialog turn, so every count it answers from is cumulative per task. The agent +polls ``should_stop`` at its safe boundaries; the first non-None ``StopReason`` is +latched and final. + +Precedence on one round: ``EARLY_CRITERION`` then ``TOOL_CALL_CAP``. + +FAIL-OPEN covers the armed criteria only: any exception while reducing an event or +evaluating them disarms them and the run degrades to a full run. The cap reads +counters and is checked on every resolved call regardless, so it never disarms. + +Rationale: .claude/notes/orchestration.md § Early stop on criterion +""" + +from __future__ import annotations + +import logging +import time +from typing import TYPE_CHECKING, Any + +from coder_eval.models import ( + DEFAULT_STOP_EARLY_GATE_THRESHOLD, + EarlyStopInfo, + EarlyStopReason, + LivePolarity, + LiveSuccessCriterion, + RunLimits, + StopEarlyPolicy, + TokenUsage, +) +from coder_eval.orchestration.early_stop import early_stop_active +from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentStartEvent, + StopReason, + StreamEvent, + ToolEndEvent, + ToolEndStatus, + ToolStartEvent, + TurnEndEvent, + TurnStartEvent, +) + + +if TYPE_CHECKING: + from coder_eval.criteria.base import BaseCriterion, LiveVerdict + from coder_eval.models import CommandTelemetry, TaskDefinition + + _ArmedPair = tuple[LiveSuccessCriterion, BaseCriterion[Any]] + + +logger = logging.getLogger(__name__) + +_DISARMED = "armed criteria disarmed, run degrades to a full run" + + +class TurnMonitor: + """Observes the agent event stream and answers the cooperative ``should_stop`` poll. + + The orchestrator builds one per task, hands the same instance to every + ``communicate()`` call, polls nothing itself, and afterwards reads + ``stop_reason`` and ``info``. + + Rationale: .claude/notes/orchestration.md § Inert triggers are by design, and the watcher fails open + """ + + def __init__( + self, + task_id: str, + armed: list[_ArmedPair], + *, + limits: RunLimits | None, + gate_threshold: float = DEFAULT_STOP_EARLY_GATE_THRESHOLD, + ) -> None: + self._task_id = task_id + self._armed = armed + self._gate_threshold = gate_threshold + self._armed_weight = sum(c.weight for c, _ in armed) + # Per-instance decidable polarities, aligned with `_armed`, static for the + # run. A trigger whose polarity the instance cannot decide is inert. + self._decidable: list[frozenset[LivePolarity]] = [ + criterion.live_decidable_polarities() for criterion, _checker in armed + ] + # Effective per-instance triggers (inert ones already resolved away). + # Every armed pair carries a stop_early block by construction + # (is_stop_armed == block presence); an explicit raise (not an assert, + # which -O strips) keeps a blockless pair from slipping through. + blocks: list[StopEarlyPolicy] = [] + for criterion, _checker in armed: + if criterion.stop_early is None: + raise ValueError(f"criterion {criterion.type!r} passed to TurnMonitor without a stop_early block") + blocks.append(criterion.stop_early) + self._pass_trigger: list[bool] = [ + block.on_pass == "stop" and "pass" in pol for block, pol in zip(blocks, self._decidable, strict=True) + ] + # IMPLICIT in arming: an armed criterion's native live-fail may always + # stop the run (ceiling-gated). Inert when it cannot live-fail. + self._fail_trigger: list[bool] = ["fail" in pol for pol in self._decidable] + # A timeout only means anything for an instance waiting to observe a + # PASS: a fail-only instance's 'undecided' IS its success state. + self._budget: list[int | None] = [ + block.decide_within if "pass" in pol else None for block, pol in zip(blocks, self._decidable, strict=True) + ] + if ( + armed + and not any(self._pass_trigger) + and not any(self._fail_trigger) + and all(b is None for b in self._budget) + ): + # Legal for a fanned row whose armed lines are all inert for its role, + # but on a non-fanned task this is dead config. + logger.warning("[%s] all armed stop triggers are inert for this row; run cannot stop early", task_id) + self._limits = limits + self._collector = EventCollector() + self._resolved_tool_ids: set[str] = set() + self._sdk_turn_index = 0 + self._tool_call_index = 0 + self._started_monotonic: float | None = None + self._committed = TokenUsage() + self._in_flight = TokenUsage() + # Once an entry leaves "undecided" on a RESOLVED round its checker is + # never polled again. `_budget_expired` marks a latched fail as + # timeout-driven, reported as DECISION_BUDGET_EXCEEDED. + # Rationale: .claude/notes/orchestration.md § Verdicts latch, and the decision happens on the CALL + self._latched: list[LiveVerdict] = ["undecided"] * len(armed) + self._budget_expired: list[bool] = [False] * len(armed) + # For the "which criterion flipped to pass" attribution. Reassigned ONLY + # at the end of a non-firing evaluation, so it always holds the PREVIOUS + # round when a stop fires. + self._prev_verdicts: list[LiveVerdict] = ["undecided"] * len(armed) + self._info: EarlyStopInfo | None = None + self._stop_reason: StopReason | None = None + self._disarmed = False + + @classmethod + def for_task(cls, task: TaskDefinition, *, arm: bool) -> TurnMonitor: + """Build the task's monitor; its criteria are armed only when ``arm`` and the task arms any. + + ``arm`` is the grading switch: under ``execute`` the trajectory is the + deliverable, so no criterion may truncate it, but the cap still applies. + The criteria registry is imported lazily; only ``LiveSuccessCriterion`` + instances can be armed, so the ``isinstance`` filter only narrows types. + """ + armed: list[_ArmedPair] = [] + if arm and early_stop_active(task): + from coder_eval.criteria import CriterionRegistry, init_criteria + + init_criteria(validate=False) + armed = [ + (c, CriterionRegistry.get_checker(c.type)()) + for c in task.success_criteria + if isinstance(c, LiveSuccessCriterion) and c.is_stop_armed + ] + limits = task.run_limits + gate_threshold = limits.stop_early_gate_threshold if limits is not None else DEFAULT_STOP_EARLY_GATE_THRESHOLD + return cls(task.task_id, armed, limits=limits, gate_threshold=gate_threshold) + + def on_event(self, event: StreamEvent) -> None: + """Reduce one event; an unexpected exception disarms the criteria and never stops the counters.""" + try: + self._on_event_impl(event) + except Exception: + self._disarmed = True + logger.error("[%s] turn monitor event handling raised; %s", self._task_id, _DISARMED, exc_info=True) + + def _on_event_impl(self, event: StreamEvent) -> None: + """Forward the event to the collector, count, and evaluate the stop conditions. + + The armed criteria are evaluated on the tool CALL, with the call passed in + as the in-flight command, and again on its resolved end. ``tool_call_index`` + increments on each resolved end. UNRESOLVED tool ends are RECORDED but never + counted or evaluated on — they must still land in the collector, or the + monitor would reduce a strictly smaller command set than the authoritative + check. The cap counts distinct resolved tool ids. + + Rationale: .claude/notes/orchestration.md § Verdicts latch, and the decision happens on the CALL + """ + if event.parent_thread_id is not None: + return + if isinstance(event, AgentStartEvent): + if self._started_monotonic is None: + self._started_monotonic = time.monotonic() + self._in_flight = TokenUsage() + elif isinstance(event, TurnStartEvent): + self._sdk_turn_index += 1 + elif isinstance(event, TurnEndEvent): + if event.tokens is not None: + self._in_flight += event.tokens + elif isinstance(event, AgentEndEvent): + self._committed += event.usage + self._in_flight = TokenUsage() + elif isinstance(event, ToolStartEvent): + self._evaluate_armed(in_flight=event.tool) + return + elif isinstance(event, ToolEndEvent): + self._collector.on_event(event) + if event.status == ToolEndStatus.UNRESOLVED: + return + self._tool_call_index += 1 + self._resolved_tool_ids.add(event.tool.tool_id) + self._evaluate_armed() + self._evaluate_cap() + return + self._collector.on_event(event) + + def should_stop(self) -> StopReason | None: + """The cooperative poll the agent calls at each safe boundary.""" + return self._stop_reason + + @property + def stop_reason(self) -> StopReason | None: + """The latched reason, or ``None`` while nothing asked the agent to stop.""" + return self._stop_reason + + @property + def info(self) -> EarlyStopInfo | None: + """The early-stop record; set only when the latched reason is ``EARLY_CRITERION``.""" + return self._info + + @property + def armed(self) -> bool: + """True when the task armed at least one criterion for this run.""" + return bool(self._armed) + + @property + def disarmed(self) -> bool: + """True once event handling raised and the armed criteria degraded to a full run.""" + return self._disarmed + + @property + def tool_calls(self) -> int: + """Distinct resolved tool calls across the whole task.""" + return len(self._resolved_tool_ids) + + @property + def usage(self) -> TokenUsage: + """Committed usage from every finished ``communicate()`` plus the in-flight deltas.""" + return self._committed + self._in_flight + + def _latch(self, reason: StopReason) -> None: + if self._stop_reason is None: + self._stop_reason = reason + + def _evaluate_armed(self, in_flight: CommandTelemetry | None = None) -> None: + if not self._armed or self._disarmed or self._stop_reason is not None: + return + try: + self._evaluate_impl(in_flight) + except Exception: + self._disarmed = True + logger.error("[%s] early-stop evaluation raised; %s", self._task_id, _DISARMED, exc_info=True) + + def _evaluate_cap(self) -> None: + cap = self._limits.max_tool_calls if self._limits is not None else None + if cap is not None and self.tool_calls >= cap: + if self._stop_reason is None: + logger.info( + "[%s] tool-call cap reached: %d resolved tool calls (cap %d)", self._task_id, self.tool_calls, cap + ) + self._latch(StopReason.TOOL_CALL_CAP) + + def _ceiling(self, verdicts: list[LiveVerdict]) -> float: + """Best-case weighted score over the WHOLE armed set, given current verdicts. + + Every already-failed criterion (native live-fail or expired budget) is + pinned at 0 (a monotonic ``live_verdict`` guarantees it stays failed); + every ``pass`` or still ``undecided`` criterion is credited its full + weight (the optimistic assumption that it could still end up scoring + 1.0). This is the same weighting ``EvaluationResult. + armed_criteria_passed`` uses for the real, final gate, so ``ceiling < + gate_threshold`` means the gate is mathematically guaranteed to fail no + matter how the trajectory continues. + """ + if self._armed_weight <= 0.0: + # Fails closed, as `armed_criteria_passed` does for the same unreachable case. + return 0.0 + return sum(c.weight for (c, _checker), v in zip(self._armed, verdicts, strict=True) if v != "fail") / ( + self._armed_weight + ) + + def _floor(self, verdicts: list[LiveVerdict], indices: list[int]) -> float | None: + """Worst-case weighted score over the given armed-index subset, given current verdicts. + + Mirrors ``_ceiling`` for the opposite direction: every still-undecided + (or already-``fail``) criterion in ``indices`` is credited nothing (the + pessimistic assumption that it could still end up scoring 0); only an + already-``pass`` criterion contributes its weight. Returns ``None`` + when the subset's total weight is 0 (the vacuous case — nothing to + bound), so callers don't have to special-case an empty numerator over + an empty denominator. + """ + weight = sum(self._armed[i][0].weight for i in indices) + if weight <= 0.0: + return None + return sum(self._armed[i][0].weight for i in indices if verdicts[i] == "pass") / weight + + def _collect_verdicts(self, in_flight: CommandTelemetry | None, tool_call_index: int) -> list[LiveVerdict]: + """One round of effective verdicts, latching decided ones on resolved rounds. + + A latched (non-``undecided``) verdict is returned as-is — its checker is + never polled again (the checkers' monotonicity contract makes re-polling + pure waste). A fresh ``undecided`` on a pass-capable instance whose + ``decide_within`` budget has expired becomes an *effective* + ``fail`` (marked in ``_budget_expired`` for reason attribution). + + Latching only happens on RESOLVED rounds (``in_flight is None``): an + in-flight round's verdict may fire a stop this round, but is not + persisted — a dispatched call that never resolves (crashed attempt) + must not leave a stale verdict behind across retries. The verdict is + recomputed from the collector's resolved commands on the next round. + """ + record = self._collector.build_turn_record() + if in_flight is not None: + # No ToolEnd yet, so the collector has not captured it. Append and + # re-sort by sequence to keep the partial trajectory in order. + record.commands = sorted([*record.commands, in_flight], key=lambda c: c.sequence_number) + records = [record] + verdicts: list[LiveVerdict] = [] + for i, (criterion, checker) in enumerate(self._armed): + if self._latched[i] != "undecided": + verdicts.append(self._latched[i]) + continue + try: + verdict: LiveVerdict = checker.live_verdict(criterion, records) + except Exception: + # Log WHICH criterion raised before re-raising to on_event's + # generic handler, where that context would be lost. + logger.error( + "[%s] early-stop live_verdict raised for criterion %r", + self._task_id, + criterion.type, + exc_info=True, + ) + raise + budget = self._budget[i] + budget_expired = verdict == "undecided" and budget is not None and tool_call_index >= budget + if budget_expired: + verdict = "fail" + if in_flight is None and verdict != "undecided": + self._latched[i] = verdict + self._budget_expired[i] = budget_expired + verdicts.append(verdict) + return verdicts + + def _budget_drove(self, index: int, verdicts: list[LiveVerdict], tool_call_index: int) -> bool: + """True when ``index``'s ``fail`` is timeout-driven rather than a native live-fail. + + Reads the persistent ``_budget_expired`` latch when set; for a + transient (in-flight, not-yet-latched) fail it re-derives: a fail on an + instance that cannot natively live-fail, with an expired budget, can + only have come from the timeout. A native live-fail on an instance + whose budget also happens to be expired reports as a native fail — + ``_collect_verdicts`` only converts the verdict when the checker itself + returned ``undecided``. + """ + if self._budget_expired[index]: + return True + budget = self._budget[index] + return ( + verdicts[index] == "fail" + and self._latched[index] == "undecided" + and budget is not None + and tool_call_index >= budget + and "fail" not in self._decidable[index] + ) + + def _evaluate_impl(self, in_flight: CommandTelemetry | None = None) -> None: + # An in-flight call has not been counted by a ToolEnd yet, so report it as + # the next (1-based) tool call. + tool_call_index = self._tool_call_index + (1 if in_flight is not None else 0) + verdicts = self._collect_verdicts(in_flight, tool_call_index) + + # RECALL DEFERRAL: a fail-stop is HELD while any pass-capable armed + # criterion is still undecided and within budget. A row with zero + # pass-capable criteria defers nothing. + # Rationale: .claude/notes/orchestration.md § Precision is traded, recall is not + pass_capable_undecided = any( + v == "undecided" and "pass" in pol for v, pol in zip(verdicts, self._decidable, strict=True) + ) + + # A criterion whose effective verdict is "fail" is a CANDIDATE; the stop + # fires only once the ceiling bound can no longer reach `gate_threshold`. + # Rationale: .claude/notes/orchestration.md § The ceiling and floor bounds + if not pass_capable_undecided: + # Deterministic precedence: a native live-fail candidate always wins + # over a budget-driven one, so the persisted/telemetry reason cannot + # flip between CRITERION_FAILED and DECISION_BUDGET_EXCEEDED on a + # mere reorder of ``success_criteria`` when both resolve on the same + # round. Within each class, first criteria-order match wins. + native_fails = [ + i + for i, v in enumerate(verdicts) + if v == "fail" and self._fail_trigger[i] and not self._budget_drove(i, verdicts, tool_call_index) + ] + budget_fails = [ + i for i, v in enumerate(verdicts) if v == "fail" and self._budget_drove(i, verdicts, tool_call_index) + ] + candidate_index = native_fails[0] if native_fails else (budget_fails[0] if budget_fails else None) + if candidate_index is not None and self._ceiling(verdicts) < self._gate_threshold: + reason = EarlyStopReason.CRITERION_FAILED if native_fails else EarlyStopReason.DECISION_BUDGET_EXCEEDED + self._fire(reason, self._armed[candidate_index][0], tool_call_index=tool_call_index) + return + + # Pass-stop: the on_pass=stop subset's FLOOR already meets + # ``gate_threshold``. Distractors are excluded from both the numerator and + # the denominator; no on_pass=stop criteria at all returns None. HELD while + # any pass-capable armed criterion OUTSIDE the subset is undecided -- + # cutting there would freeze a sibling's expected signal out of the run. + # Rationale: .claude/notes/orchestration.md § The ceiling and floor bounds + pass_stop_indices = [i for i, armed_pass in enumerate(self._pass_trigger) if armed_pass] + outside_pass_capable_undecided = any( + v == "undecided" and "pass" in pol and not armed_pass + for v, pol, armed_pass in zip(verdicts, self._decidable, self._pass_trigger, strict=True) + ) + if not outside_pass_capable_undecided: + floor = self._floor(verdicts, pass_stop_indices) + if floor is not None and floor >= self._gate_threshold: + # Deciding criterion = the last on_pass=stop (criteria order) whose + # verdict flipped vs the previous round; fall back to the last one. + deciding = self._armed[pass_stop_indices[-1]][0] + for i in pass_stop_indices: + if verdicts[i] != self._prev_verdicts[i]: + deciding = self._armed[i][0] + self._fire(EarlyStopReason.CRITERION_PASSED, deciding, tool_call_index=tool_call_index) + return + + # No stop this round — record the verdicts so the next round can detect + # flips. Resolved rounds only: an in-flight round's verdicts are + # deliberately not latched (the call may never resolve), so persisting + # them here would let a transient round mask the real flip attribution. + if in_flight is None: + self._prev_verdicts = verdicts + + def _fire(self, reason: EarlyStopReason, criterion: LiveSuccessCriterion, *, tool_call_index: int) -> None: + elapsed = 0.0 + if self._started_monotonic is not None: + elapsed = max(time.monotonic() - self._started_monotonic, 0.0) + cap = self._limits.max_tool_calls if self._limits is not None else None + self._info = EarlyStopInfo( + reason=reason, + deciding_criterion_type=criterion.type, + deciding_criterion_description=criterion.description, + armed_criteria=[f"{c.type}: {c.description}" for c, _ in self._armed], + sdk_turn_index=self._sdk_turn_index, + tool_call_index=tool_call_index, + elapsed_seconds=elapsed, + tool_calls_remaining_at_stop=None if cap is None else max(cap - tool_call_index, 0), + gate_threshold=self._gate_threshold, + ) + self._latch(StopReason.EARLY_CRITERION) + logger.info( + "[%s] early-stop fired: reason=%s deciding=%s sdk_turn=%d tool_call=%d elapsed=%.2fs", + self._task_id, + reason.value, + criterion.type, + self._sdk_turn_index, + tool_call_index, + elapsed, + ) diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index fa1bcf050..1f9356ab1 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -65,10 +65,11 @@ resolve_evaluation_route, resolve_route, ) -from .orchestration.early_stop import EarlyStopWatcher, early_stop_active, validate_early_stop +from .orchestration.early_stop import early_stop_active, validate_early_stop from .orchestration.evaluation import resolve_reference_dir, stage_reference_dir from .orchestration.harness_contract import validate_harness_contract from .orchestration.run_limits import validate_run_limits +from .orchestration.turn_monitor import TurnMonitor from .path_utils import ( TASK_JSON_FILENAME, digest_tree, @@ -471,9 +472,9 @@ def __init__( # overwrite the reference and drive `reference_comparison` to 1.0. self._reference_digest: str | None = None - # Created in _setup only when armed; None otherwise, so the default path - # is entirely unaffected. - self._early_stop_watcher: EarlyStopWatcher | None = None + # Built once in _setup and handed to every communicate() call, so every + # count it answers the should_stop poll from is cumulative per task. + self._monitor: TurnMonitor | None = None # One-shot flag: emit the "cost budget configured but no cost data" warning # exactly once per task even if _check_run_limits fires every turn. @@ -1428,24 +1429,21 @@ async def _verify_reference_integrity(self) -> None: + f"({current[:12]}...). Refusing to grade against a reference the agent may have written." ) - def _arm_early_stop(self) -> None: - """Build the early-stop watcher, once, when the task arms one. + def _build_monitor(self) -> None: + """Build the task's ``TurnMonitor``, once. - Sits BEFORE `_setup`'s evaluate-only early return, so an armed - evaluate-only re-grade builds an inert (never-fed) watcher — harmless, - and keeps a single creation point. + Sits BEFORE `_setup`'s evaluate-only early return, so an evaluate-only + re-grade builds an inert (never-fed) monitor — harmless, and keeps a single + creation point. """ - if not early_stop_active(self.task): - return - if self.grade: - self._early_stop_watcher = EarlyStopWatcher.for_task(self.task) - return - # Under `execute` there is no outcome to decide and the trajectory IS the - # deliverable, so an armed criterion must not truncate it. - logger.info( - "Grading disabled (execute mode): early-stop is armed but stays disabled; " - + "the full trajectory is the deliverable." - ) + self._monitor = TurnMonitor.for_task(self.task, arm=self.grade) + if not self.grade and early_stop_active(self.task): + # Under `execute` there is no outcome to decide and the trajectory IS + # the deliverable, so an armed criterion must not truncate it. + logger.info( + "Grading disabled (execute mode): early-stop is armed but stays disabled; " + + "the full trajectory is the deliverable." + ) def _restore_recorded_command_path(self) -> None: """Re-apply the graded run's own PATH before its criteria run. @@ -1474,10 +1472,10 @@ async def _setup(self) -> None: self._warn_on_ineffective_task_timeout() # ONCE, up front, and BEFORE the evaluate-only early return: an armed - # evaluate-only re-grade builds an inert watcher, which is harmless and + # evaluate-only re-grade builds an inert monitor, which is harmless and # keeps a single creation point. # Rationale: .claude/notes/orchestration.md § Gate selection is fired-only - self._arm_early_stop() + self._build_monitor() # BEFORE either branch returns: judge criteria with include_reference # expect it populated in evaluate-only re-grades too. @@ -1894,19 +1892,16 @@ async def _communicate_with_retry( result = self.result run_limits = self.task.run_limits turn_timeout = run_limits.turn_timeout if run_limits else None - max_turns = run_limits.max_turns if run_limits else None - - agent_callback: StreamCallback | None = None - if self.stream_callback is not None: - agent_callback = TaskScopedCallback(self.stream_callback, self._log_task_id) + monitor = self._monitor + assert monitor is not None, "TurnMonitor not built" # The sole callback when --stream is off, else alongside the - # TaskScopedCallback. The same instance persists across retry attempts, so - # its counters and wall-clock origin accumulate. - watcher = self._early_stop_watcher - if watcher is not None: - agent_callback = ( - CompositeStreamCallback([watcher, agent_callback]) if agent_callback is not None else watcher + # TaskScopedCallback. The same instance persists across retry attempts and + # dialog turns, so its counters and wall-clock origin accumulate. + agent_callback: StreamCallback = monitor + if self.stream_callback is not None: + agent_callback = CompositeStreamCallback( + [monitor, TaskScopedCallback(self.stream_callback, self._log_task_id)] ) def _drain_pending_turn(*, attempt: int) -> None: @@ -1951,8 +1946,7 @@ async def _communicate_attempt() -> TurnRecord: prompt, stream_callback=agent_callback, timeout=turn_timeout, - max_turns=max_turns, - should_stop=watcher.should_stop if watcher is not None else None, + should_stop=monitor.should_stop, ) if turn_timeout is None: return await coro @@ -2086,7 +2080,7 @@ def _select_gate(self) -> bool: """Apply the verdict gate to the criteria results already on ``self.result``. Gate selection is FIRED-ONLY: the weighted armed gate applies IFF the - watcher actually cut the run. BOTH single-shot grading paths must call + monitor actually cut the run. BOTH single-shot grading paths must call this — the live one and the evaluate-only one — or a re-graded early-stopped run is scored under the full-run gate and flips its verdict. @@ -2112,9 +2106,9 @@ def _select_gate(self) -> bool: ) return self.result.armed_criteria_passed(self.task.success_criteria, gate_threshold) - if self._early_stop_watcher is not None: - if self._early_stop_watcher.disarmed: - logger.info("early-stop watcher disarmed fail-open (verdict error): gating on the full set.") + if self._monitor is not None and self._monitor.armed: + if self._monitor.disarmed: + logger.info("early-stop criteria disarmed fail-open (verdict error): gating on the full set.") else: logger.info("early-stop armed but never fired (run completed naturally): gating on the full set.") return self.result.all_criteria_passed(self.task.success_criteria) @@ -2199,9 +2193,10 @@ async def _evaluation_loop(self) -> bool: self.result.iterations.append(turn_record) self._sync_sandbox_command_path_with_agent() - # Record early-stop info (if the watcher tripped) BEFORE check_all_async, so it + # Record early-stop info (if the monitor tripped) BEFORE check_all_async, so it # survives even if a checker raises. None on a full run or when unarmed. - self.result.early_stop = self._early_stop_watcher.info if self._early_stop_watcher is not None else None + assert self._monitor is not None + self.result.early_stop = self._monitor.info logger.debug(f"Agent response received ({len(turn_record.agent_output)} chars)") @@ -2209,13 +2204,15 @@ async def _evaluation_loop(self) -> bool: # withholds the verdict, never the facts. Recording the fact is not # finalizing on it — the tool-call cap decides the status only when the criteria # fail, so under grade=False this is carried into task.json for the - # detached grade rather than turned into a terminal status. + # detached grade rather than turned into a terminal status. Read from the + # turn's end status, not the monitor's latch: a cap latched after the + # agent's last poll did not stop anything. # Rationale: .claude/notes/orchestration.md § The four grading sites if turn_record.tool_calls_exhausted: self.result.tool_calls_exhausted = True logger.warning( - "Agent exhausted max_turns (%s).", - self.task.run_limits.max_turns if self.task.run_limits else None, + "Agent reached the tool-call cap (%d resolved tool calls).", + self._monitor.tool_calls, ) # Soft cumulative-turn check (logs once; never aborts). self._check_expected_tool_calls(iteration=iteration) diff --git a/src/coder_eval/resources/tags.yaml b/src/coder_eval/resources/tags.yaml index 51682396d..3306f0566 100644 --- a/src/coder_eval/resources/tags.yaml +++ b/src/coder_eval/resources/tags.yaml @@ -50,6 +50,6 @@ tags: examples: - "300s turn_timeout exceeded at 80 turns; analysis recommends 600s" - name: max-turns-too-low - definition: Task ran out of turns before completing; max_turns budget is below what the task realistically needs. + definition: Task reached the tool-call cap before completing; max_tool_calls is below what the task realistically needs. examples: - "TOOL_CALLS_EXHAUSTED: cap of 50 tool calls reached" diff --git a/src/coder_eval/run_record.py b/src/coder_eval/run_record.py index 4a118aefd..5234bb6ee 100644 --- a/src/coder_eval/run_record.py +++ b/src/coder_eval/run_record.py @@ -187,8 +187,8 @@ def eval_result_to_task_dict( # truncated run with a full one. "stopped_early": result.early_stop is not None, "early_stop_reason": (result.early_stop.reason.value if result.early_stop is not None else None), - "turns_remaining_at_stop": ( - result.early_stop.turns_remaining_at_stop if result.early_stop is not None else None + "tool_calls_remaining_at_stop": ( + result.early_stop.tool_calls_remaining_at_stop if result.early_stop is not None else None ), # The threshold in effect for this stop, so a sweep that varies it can tell # which weighted-gate value produced a given verdict. diff --git a/src/coder_eval/simulation/user_simulator.py b/src/coder_eval/simulation/user_simulator.py index 0696583f0..c119c759b 100644 --- a/src/coder_eval/simulation/user_simulator.py +++ b/src/coder_eval/simulation/user_simulator.py @@ -209,6 +209,8 @@ def __init__( # preset must not prefix it. # Rationale: .claude/notes/contracts.md § The judge's identity is its system prompt system_prompt_mode="replace", + # One user utterance per call, so the SDK's own agent loop stops after one turn. + sdk_options={"max_turns": 1}, ) # parse_agent_config returns a union, but type=CLAUDE_CODE guarantees ClaudeCodeAgentConfig assert isinstance(agent_config, ClaudeCodeAgentConfig) @@ -330,8 +332,7 @@ async def next_user_message(self, dialog_pairs: list[tuple[str, str]]) -> Simula assert self._agent is not None, "UserSimulator.start() must be called before next_user_message()" prompt = dialog_pairs[-1][1] if dialog_pairs else _OPENER_NUDGE - # Simulator emits one user utterance per call, so cap the inner loop at 1 turn. - turn = await self._agent.communicate(prompt, max_turns=1) + turn = await self._agent.communicate(prompt) raw = turn.agent_output or "" usage = turn.token_usage input_tokens = usage.uncached_input_tokens if usage is not None else None diff --git a/src/coder_eval/spi.py b/src/coder_eval/spi.py index 0c918f397..c3cdec7ca 100644 --- a/src/coder_eval/spi.py +++ b/src/coder_eval/spi.py @@ -35,6 +35,7 @@ AgentEndEvent, AgentEndStatus, AgentStartEvent, + StopReason, TextChunkEvent, ToolEndEvent, ToolEndStatus, @@ -42,11 +43,12 @@ TurnEndEvent, TurnEndStatus, TurnStartEvent, + end_status_for, ) from coder_eval.timing import TurnClock, close_window -SPI_VERSION: Final[int] = 1 +SPI_VERSION: Final[int] = 2 __all__ = [ # noqa: RUF022 - plain sort, pinned by tests/test_spi.py "Agent", @@ -71,6 +73,7 @@ "READ_ONLY_DENIED_TOOLS", "ResultSummary", "SPI_VERSION", + "StopReason", "StreamCallback", "SystemPromptMode", "TextChunkEvent", @@ -87,5 +90,6 @@ "TurnStartEvent", "TurnTimeoutError", "close_window", + "end_status_for", "register_pricing", ] diff --git a/src/coder_eval/streaming/collector.py b/src/coder_eval/streaming/collector.py index 10b8f1149..27971d639 100644 --- a/src/coder_eval/streaming/collector.py +++ b/src/coder_eval/streaming/collector.py @@ -67,7 +67,7 @@ def on_event(self, event: StreamEvent) -> None: self._iteration = event.iteration self._user_input = event.prompt self._agent_start_at = event.timestamp - # EarlyStopWatcher keeps ONE collector across retries: left stale, + # TurnMonitor keeps ONE collector across retries: left stale, # this would pair the new start with the last attempt's end and # publish the clamped inversion as a measured 0.0. self._agent_end = None @@ -82,22 +82,6 @@ def on_event(self, event: StreamEvent) -> None: elif isinstance(event, AgentEndEvent): self._agent_end = event - @property - def visible_turn_count(self) -> int: - """Visible timeline entries observed so far — one per resolved tool call. - - The live, in-stream counterpart of ``result_metrics.visible_turn_count``, - which counts the very same list once the turn is a finished - ``TurnRecord`` (minus its trailing final-reply entry, which cannot exist - while the turn is still running). - - Agents whose SDK has no meaningful native turn counter (Codex, - Antigravity) enforce ``run_limits.max_turns`` against this, so the cap - means the same thing on both. Keying on ``tool_id`` means a re-emitted - end event cannot double-count. - """ - return len(self._commands) - def _ordered_commands(self) -> list[CommandTelemetry]: return sorted(self._commands.values(), key=lambda c: c.sequence_number) diff --git a/src/coder_eval/streaming/events.py b/src/coder_eval/streaming/events.py index 4d4f6c677..93a92e9f4 100644 --- a/src/coder_eval/streaming/events.py +++ b/src/coder_eval/streaming/events.py @@ -60,6 +60,8 @@ class TurnEndStatus(StrEnum): CRASHED = "crashed" TIMEOUT = "timeout" TOOL_CALLS_EXHAUSTED = "tool_calls_exhausted" + TOKEN_BUDGET_EXCEEDED = "token_budget_exceeded" + COST_BUDGET_EXCEEDED = "cost_budget_exceeded" STOPPED_EARLY = "stopped_early" # cooperative early-stop-on-criterion (clean, non-crash) @@ -70,9 +72,34 @@ class AgentEndStatus(StrEnum): CRASHED = "crashed" TIMEOUT = "timeout" TOOL_CALLS_EXHAUSTED = "tool_calls_exhausted" + TOKEN_BUDGET_EXCEEDED = "token_budget_exceeded" + COST_BUDGET_EXCEEDED = "cost_budget_exceeded" STOPPED_EARLY = "stopped_early" # cooperative early-stop-on-criterion (clean, non-crash) +class StopReason(StrEnum): + """Why ``should_stop`` asked the agent to stop; the agent finalizes with ``end_status_for(reason)``.""" + + EARLY_CRITERION = "early_criterion" + TOOL_CALL_CAP = "tool_call_cap" + TOKEN_BUDGET = "token_budget" + USD_BUDGET = "usd_budget" + + +_END_STATUS_FOR_STOP: dict[StopReason, AgentEndStatus] = { + StopReason.EARLY_CRITERION: AgentEndStatus.STOPPED_EARLY, + StopReason.TOOL_CALL_CAP: AgentEndStatus.TOOL_CALLS_EXHAUSTED, + StopReason.TOKEN_BUDGET: AgentEndStatus.TOKEN_BUDGET_EXCEEDED, + StopReason.USD_BUDGET: AgentEndStatus.COST_BUDGET_EXCEEDED, +} +assert set(_END_STATUS_FOR_STOP) == set(StopReason), "Missing end status for StopReason member" + + +def end_status_for(reason: StopReason) -> AgentEndStatus: + """The clean ``AgentEndStatus`` a turn stopped for ``reason`` finalizes with.""" + return _END_STATUS_FOR_STOP[reason] + + # The canonical TranscriptMessage union, so AgentEndEvent carries per-message # telemetry losslessly and stays in lock-step with TurnRecord.messages. _MessageList = list[TranscriptMessage] diff --git a/tasks/agents/antigravity_hello_world.yaml b/tasks/agents/antigravity_hello_world.yaml index 5de57e5ad..d4167e7d8 100644 --- a/tasks/agents/antigravity_hello_world.yaml +++ b/tasks/agents/antigravity_hello_world.yaml @@ -38,6 +38,6 @@ success_criteria: weight: 0.3 run_limits: - max_turns: 5 + max_tool_calls: 5 task_timeout: 360 turn_timeout: 300 diff --git a/tasks/agents/antigravity_hello_world_docker.yaml b/tasks/agents/antigravity_hello_world_docker.yaml index 856bc3957..4293899f4 100644 --- a/tasks/agents/antigravity_hello_world_docker.yaml +++ b/tasks/agents/antigravity_hello_world_docker.yaml @@ -47,6 +47,6 @@ success_criteria: weight: 0.3 run_limits: - max_turns: 5 + max_tool_calls: 5 task_timeout: 360 turn_timeout: 300 diff --git a/tasks/agents/claude_hello_world.yaml b/tasks/agents/claude_hello_world.yaml index 9f864c993..1747a3c0f 100644 --- a/tasks/agents/claude_hello_world.yaml +++ b/tasks/agents/claude_hello_world.yaml @@ -39,6 +39,6 @@ success_criteria: weight: 0.3 run_limits: - max_turns: 5 + max_tool_calls: 5 task_timeout: 300 turn_timeout: 60 diff --git a/tasks/agents/claude_hello_world_docker.yaml b/tasks/agents/claude_hello_world_docker.yaml index 5254a8464..66c0ae204 100644 --- a/tasks/agents/claude_hello_world_docker.yaml +++ b/tasks/agents/claude_hello_world_docker.yaml @@ -47,6 +47,6 @@ success_criteria: weight: 0.3 run_limits: - max_turns: 5 + max_tool_calls: 5 task_timeout: 300 turn_timeout: 60 diff --git a/tasks/agents/claude_parallel_single_gen.yaml b/tasks/agents/claude_parallel_single_gen.yaml index fc22ff1ac..e491473b8 100644 --- a/tasks/agents/claude_parallel_single_gen.yaml +++ b/tasks/agents/claude_parallel_single_gen.yaml @@ -44,6 +44,6 @@ success_criteria: weight: 0.5 run_limits: - max_turns: 8 + max_tool_calls: 8 task_timeout: 180 turn_timeout: 90 diff --git a/tasks/agents/claude_subagent_test.yaml b/tasks/agents/claude_subagent_test.yaml index 47490a2ae..757aea450 100644 --- a/tasks/agents/claude_subagent_test.yaml +++ b/tasks/agents/claude_subagent_test.yaml @@ -37,6 +37,6 @@ success_criteria: description: "answer.txt must contain the sub-agent's computed sum (5050)." run_limits: - max_turns: 12 + max_tool_calls: 12 task_timeout: 240 turn_timeout: 120 diff --git a/tasks/agents/codex_hello_world.yaml b/tasks/agents/codex_hello_world.yaml index ae91c1a68..d652df186 100644 --- a/tasks/agents/codex_hello_world.yaml +++ b/tasks/agents/codex_hello_world.yaml @@ -36,6 +36,6 @@ success_criteria: weight: 0.3 run_limits: - max_turns: 5 + max_tool_calls: 5 task_timeout: 300 turn_timeout: 60 diff --git a/tasks/agents/codex_parallel_commands.yaml b/tasks/agents/codex_parallel_commands.yaml index 2123c7b44..bb74c2dda 100644 --- a/tasks/agents/codex_parallel_commands.yaml +++ b/tasks/agents/codex_parallel_commands.yaml @@ -40,6 +40,6 @@ success_criteria: weight: 0.5 run_limits: - max_turns: 8 + max_tool_calls: 8 task_timeout: 300 turn_timeout: 120 diff --git a/tasks/agents/codex_parallel_single_gen.yaml b/tasks/agents/codex_parallel_single_gen.yaml index 668c363a3..6a017bc0e 100644 --- a/tasks/agents/codex_parallel_single_gen.yaml +++ b/tasks/agents/codex_parallel_single_gen.yaml @@ -39,6 +39,6 @@ success_criteria: weight: 0.5 run_limits: - max_turns: 8 + max_tool_calls: 8 task_timeout: 180 turn_timeout: 90 diff --git a/tasks/agents/codex_string_utils.yaml b/tasks/agents/codex_string_utils.yaml index 9ea147483..e7564ab46 100644 --- a/tasks/agents/codex_string_utils.yaml +++ b/tasks/agents/codex_string_utils.yaml @@ -56,6 +56,6 @@ success_criteria: weight: 0.2 run_limits: - max_turns: 8 + max_tool_calls: 8 task_timeout: 600 turn_timeout: 120 diff --git a/tasks/agents/codex_subagent_test.yaml b/tasks/agents/codex_subagent_test.yaml index 8534ebe26..7c99a3ab3 100644 --- a/tasks/agents/codex_subagent_test.yaml +++ b/tasks/agents/codex_subagent_test.yaml @@ -31,6 +31,6 @@ success_criteria: description: "answer.txt must contain the sub-agent's computed sum (5050)." run_limits: - max_turns: 12 + max_tool_calls: 12 task_timeout: 240 turn_timeout: 120 diff --git a/tasks/agents/subagent_bash_long_input.yaml b/tasks/agents/subagent_bash_long_input.yaml index d1840b94c..f1842c466 100644 --- a/tasks/agents/subagent_bash_long_input.yaml +++ b/tasks/agents/subagent_bash_long_input.yaml @@ -78,6 +78,6 @@ success_criteria: pass_threshold: 1.0 run_limits: - max_turns: 20 + max_tool_calls: 20 task_timeout: 600 turn_timeout: 180 diff --git a/tasks/agents/subagent_merge_sort.yaml b/tasks/agents/subagent_merge_sort.yaml index a7100d1ef..6a28422de 100644 --- a/tasks/agents/subagent_merge_sort.yaml +++ b/tasks/agents/subagent_merge_sort.yaml @@ -51,6 +51,6 @@ success_criteria: pass_threshold: 1.0 run_limits: - max_turns: 20 + max_tool_calls: 20 task_timeout: 300 turn_timeout: 120 diff --git a/tasks/anti_cheat_reference/anti_cheat_reference.yaml b/tasks/anti_cheat_reference/anti_cheat_reference.yaml index 0e34de35b..9603ae3c3 100644 --- a/tasks/anti_cheat_reference/anti_cheat_reference.yaml +++ b/tasks/anti_cheat_reference/anti_cheat_reference.yaml @@ -181,6 +181,6 @@ success_criteria: weight: 1.0 run_limits: - max_turns: 6 + max_tool_calls: 6 task_timeout: 300 turn_timeout: 150 diff --git a/tasks/dockerfile_build_example/dockerfile_build_example.yaml b/tasks/dockerfile_build_example/dockerfile_build_example.yaml index bb17d5f66..d14b4cc41 100644 --- a/tasks/dockerfile_build_example/dockerfile_build_example.yaml +++ b/tasks/dockerfile_build_example/dockerfile_build_example.yaml @@ -49,7 +49,7 @@ initial_prompt: | this example primarily exercises the custom Docker build, not the agent. run_limits: - max_turns: 3 + max_tool_calls: 3 task_timeout: 300 turn_timeout: 120 diff --git a/tasks/dockerfile_build_example/working_dir_auto_example.yaml b/tasks/dockerfile_build_example/working_dir_auto_example.yaml index 54875002e..921f4036b 100644 --- a/tasks/dockerfile_build_example/working_dir_auto_example.yaml +++ b/tasks/dockerfile_build_example/working_dir_auto_example.yaml @@ -41,7 +41,7 @@ initial_prompt: | exact contents into a new file named result.txt in the same directory. run_limits: - max_turns: 3 + max_tool_calls: 3 task_timeout: 300 turn_timeout: 120 diff --git a/tasks/dockerfile_build_example/working_dir_concrete_example.yaml b/tasks/dockerfile_build_example/working_dir_concrete_example.yaml index 17706afcd..9579a3d1e 100644 --- a/tasks/dockerfile_build_example/working_dir_concrete_example.yaml +++ b/tasks/dockerfile_build_example/working_dir_concrete_example.yaml @@ -41,7 +41,7 @@ initial_prompt: | exact contents into a new file named result.txt in the same directory. run_limits: - max_turns: 3 + max_tool_calls: 3 task_timeout: 300 turn_timeout: 120 diff --git a/tasks/early_stop_decision_budget_exceeded.yaml b/tasks/early_stop_decision_budget_exceeded.yaml index 5d985802b..f47f4f014 100644 --- a/tasks/early_stop_decision_budget_exceeded.yaml +++ b/tasks/early_stop_decision_budget_exceeded.yaml @@ -2,10 +2,10 @@ task_id: "early_stop_decision_budget_exceeded" description: > Decision-step timeout (GitHub issue #61, item 3): the armed command_executed criterion sets stop_early.decide_within: 3 — if the agent hasn't run the script - within its first 3 tool calls, EarlyStopWatcher latches an effective FAIL + within its first 3 tool calls, the TurnMonitor latches an effective FAIL for it (reported as reason=decision_budget_exceeded) and, since that fail drops the armed ceiling below the default gate threshold of 1.0, the run - fail-stops immediately instead of burning the rest of run_limits.max_turns. + fail-stops immediately instead of burning the rest of run_limits.max_tool_calls. The timeout gates through the same weighted armed gate as a native live-fail — no special force-fail path. Note the block's on_pass stays 'continue', so a live PASS never stops the run @@ -28,7 +28,7 @@ agent: allowed_tools: ["Read", "Write", "Bash", "Glob"] run_limits: - max_turns: 20 + max_tool_calls: 20 success_criteria: - type: "command_executed" diff --git a/tasks/early_stop_weighted_high_weight_kills_run.yaml b/tasks/early_stop_weighted_high_weight_kills_run.yaml index db75b81ec..c0c4e6cdd 100644 --- a/tasks/early_stop_weighted_high_weight_kills_run.yaml +++ b/tasks/early_stop_weighted_high_weight_kills_run.yaml @@ -12,7 +12,7 @@ description: > truncate recall) — with the prompt ordering curl before python app.py, the fail-stop actually fires as soon as the positive resolves (not "immediately" on the curl misfire itself), rather than burning the rest of - run_limits.max_turns on a doomed run. This is a NON-CI example task + run_limits.max_tool_calls on a doomed run. This is a NON-CI example task (deliberately untagged for smoke-pass/smoke-fail): whether the agent actually calls curl as instructed is not deterministic enough for a live-agent CI assertion — run it manually with `coder-eval run` to observe @@ -30,7 +30,7 @@ agent: allowed_tools: ["Read", "Write", "Bash"] run_limits: - max_turns: 20 + max_tool_calls: 20 stop_early_gate_threshold: 0.7 success_criteria: diff --git a/tasks/early_stop_weighted_low_weight_absorbed.yaml b/tasks/early_stop_weighted_low_weight_absorbed.yaml index 458dad9e3..d509c25d9 100644 --- a/tasks/early_stop_weighted_low_weight_absorbed.yaml +++ b/tasks/early_stop_weighted_low_weight_absorbed.yaml @@ -30,7 +30,7 @@ agent: allowed_tools: ["Read", "Write", "Bash"] run_limits: - max_turns: 20 + max_tool_calls: 20 stop_early_gate_threshold: 0.7 success_criteria: diff --git a/tasks/internal/session_resumption.yaml b/tasks/internal/session_resumption.yaml index 9187ce44e..c7aeada97 100644 --- a/tasks/internal/session_resumption.yaml +++ b/tasks/internal/session_resumption.yaml @@ -8,7 +8,7 @@ description: > tags: [internal, pure-python] run_limits: - max_turns: 10 + max_tool_calls: 10 agent: type: claude-code permission_mode: acceptEdits diff --git a/tasks/python_cli_simulated_judged/echo_simulated_judged.yaml b/tasks/python_cli_simulated_judged/echo_simulated_judged.yaml index 9095e0c02..0ae6b4a7b 100644 --- a/tasks/python_cli_simulated_judged/echo_simulated_judged.yaml +++ b/tasks/python_cli_simulated_judged/echo_simulated_judged.yaml @@ -6,7 +6,7 @@ description: > agent reproduced the string exactly without paraphrasing or inventing. tags: [python, simulation, llm-judge] run_limits: - max_turns: 4 + max_tool_calls: 4 task_timeout: 900 turn_timeout: 120 diff --git a/tasks/record_cli_responses.yaml b/tasks/record_cli_responses.yaml index b3c0943d9..d5e2afa72 100644 --- a/tasks/record_cli_responses.yaml +++ b/tasks/record_cli_responses.yaml @@ -86,7 +86,7 @@ agent: # Mirrors anti_cheat_reference. Read is kept deliberately: Claude Code's Write # tool refuses to overwrite a file the session has not read, so an agent that # wants to correct or extend captured.txt would otherwise burn turns against - # max_turns discovering the Bash heredoc fallback. Restricting tools would NOT + # max_tool_calls discovering the Bash heredoc fallback. Restricting tools would NOT # make the response strings unreachable anyway -- see the description block. allowed_tools: - Bash @@ -201,6 +201,6 @@ success_criteria: # model may skip after two successes is a pure flake source in a blocking bucket. run_limits: - max_turns: 6 + max_tool_calls: 6 task_timeout: 300 turn_timeout: 150 diff --git a/tasks/run_limits/max_turns_cap.yaml b/tasks/run_limits/max_tool_calls_cap.yaml similarity index 75% rename from tasks/run_limits/max_turns_cap.yaml rename to tasks/run_limits/max_tool_calls_cap.yaml index dfecdd66d..79981961e 100644 --- a/tasks/run_limits/max_turns_cap.yaml +++ b/tasks/run_limits/max_tool_calls_cap.yaml @@ -1,14 +1,15 @@ -task_id: run-limits-max-turns-cap +task_id: run-limits-max-tool-calls-cap description: >- - Parity fixture for run_limits.max_turns. The prompt asks for far more + Parity fixture for run_limits.max_tool_calls. The prompt asks for far more sequential tool calls than the cap allows, so every harness must stop at the cap rather than running the prompt to completion. Run it with --type - claude-code / codex / antigravity and compare: the cap must produce a CLEAN - stop (tool_calls_exhausted, criteria still checked), never a crash. + claude-code / codex / antigravity / pi / opencode and compare: the cap must + produce a CLEAN stop (`tool_calls_exhausted`, criteria still checked) on every + harness, never a crash. tags: - run-limits - - max-turns + - max-tool-calls - parity initial_prompt: | @@ -23,7 +24,7 @@ initial_prompt: | run_limits: # Far below the 12 the prompt asks for, so the cap always decides the ending. - max_turns: 4 + max_tool_calls: 4 # Generous: this fixture must fail on the cap, never on the clock. turn_timeout: 300 task_timeout: 600 @@ -40,11 +41,11 @@ success_criteria: weight: 1.0 # And this is the half that actually tests the cap. Without it the fixture - # passes on a harness that ignores max_turns entirely — the exact bug it exists + # passes on a harness that ignores the cap entirely — the exact bug it exists # to catch — because step-01.txt gets written either way. The chained contents # in the prompt make each step depend on reading the one before it, so no amount - # of batching within a single agent-loop turn can reach step 12 inside a cap of - # 4; a run that produced the last file therefore ran uncapped. + # of batching can reach step 12 inside a cap of 4 tool calls; a run that + # produced the last file therefore ran uncapped. - type: run_command command: "test ! -f step-12.txt" description: "The cap bound the run: the agent never reached the last file" diff --git a/tasks/run_limits/turn_timeout.yaml b/tasks/run_limits/turn_timeout.yaml index 9f9a78630..0e359963f 100644 --- a/tasks/run_limits/turn_timeout.yaml +++ b/tasks/run_limits/turn_timeout.yaml @@ -2,7 +2,7 @@ task_id: run-limits-turn-timeout description: >- Parity fixture for run_limits.turn_timeout. The prompt blocks far longer than the timeout allows, so every harness must abort the turn on the watchdog. The - contrast with the max_turns fixture is the point: a timeout is a FAILURE with a + contrast with the max_tool_calls fixture is the point: a timeout is a FAILURE with a partial turn captured, while the turn cap is a clean stop. tags: diff --git a/tasks/samples/skillsbench/3d-scan-calc/3d-scan-calc.yaml b/tasks/samples/skillsbench/3d-scan-calc/3d-scan-calc.yaml index 46cfef5b5..56b2d2f22 100644 --- a/tasks/samples/skillsbench/3d-scan-calc/3d-scan-calc.yaml +++ b/tasks/samples/skillsbench/3d-scan-calc/3d-scan-calc.yaml @@ -53,7 +53,7 @@ pre_run: timeout: 30 run_limits: - max_turns: 50 + max_tool_calls: 50 task_timeout: 1200 turn_timeout: 300 diff --git a/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml b/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml index 5a104ef56..4f484aeb8 100644 --- a/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml +++ b/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml @@ -41,7 +41,7 @@ pre_run: timeout: 30 run_limits: - max_turns: 50 + max_tool_calls: 50 task_timeout: 1200 turn_timeout: 300 diff --git a/tasks/samples/skillsbench/dialogue-parser/dialogue-parser.yaml b/tasks/samples/skillsbench/dialogue-parser/dialogue-parser.yaml index 45f8344e3..af1889970 100644 --- a/tasks/samples/skillsbench/dialogue-parser/dialogue-parser.yaml +++ b/tasks/samples/skillsbench/dialogue-parser/dialogue-parser.yaml @@ -54,7 +54,7 @@ pre_run: timeout: 30 run_limits: - max_turns: 50 + max_tool_calls: 50 task_timeout: 1200 turn_timeout: 300 diff --git a/tasks/smoke_agent_judge.yaml b/tasks/smoke_agent_judge.yaml index 3e1dae876..dc336bb89 100644 --- a/tasks/smoke_agent_judge.yaml +++ b/tasks/smoke_agent_judge.yaml @@ -17,7 +17,7 @@ tags: [smoke, smoke-pass, judge] # Capped low; agent only needs to write one short file. run_limits: - max_turns: 3 + max_tool_calls: 3 agent: type: "claude-code" diff --git a/tasks/smoke_budget_exceeded.yaml b/tasks/smoke_budget_exceeded.yaml index bd372c7c6..2a8f8f71b 100644 --- a/tasks/smoke_budget_exceeded.yaml +++ b/tasks/smoke_budget_exceeded.yaml @@ -13,7 +13,7 @@ initial_prompt: | # Unsatisfiable budget — any real prompt blows this in a single turn. run_limits: max_input_tokens: 1 - max_turns: 2 + max_tool_calls: 2 tags: [smoke-fail] diff --git a/tasks/smoke_cost_budget_exceeded.yaml b/tasks/smoke_cost_budget_exceeded.yaml index 75acd6e77..acd3a4c54 100644 --- a/tasks/smoke_cost_budget_exceeded.yaml +++ b/tasks/smoke_cost_budget_exceeded.yaml @@ -19,7 +19,7 @@ initial_prompt: | run_limits: max_usd: 0.0001 - max_turns: 2 + max_tool_calls: 2 tags: [smoke-cost-budget] diff --git a/tasks/smoke_llm_judge.yaml b/tasks/smoke_llm_judge.yaml index ea23566f2..425f32d49 100644 --- a/tasks/smoke_llm_judge.yaml +++ b/tasks/smoke_llm_judge.yaml @@ -12,7 +12,7 @@ initial_prompt: | tags: [smoke, smoke-pass, judge] run_limits: - max_turns: 2 + max_tool_calls: 2 agent: type: "claude-code" diff --git a/tasks/smoke_negative_path.yaml b/tasks/smoke_negative_path.yaml index c3ccd050e..bc118953a 100644 --- a/tasks/smoke_negative_path.yaml +++ b/tasks/smoke_negative_path.yaml @@ -9,13 +9,13 @@ initial_prompt: | Do not include any other content. Do not write the words 'sentinel' or 'zzz'. tags: [smoke, smoke-fail] -# max_turns lives at top level on TaskDefinition (PR #225, May 2026). -# AgentConfig has extra="forbid", so do NOT put max_turns under agent:. +# The cap lives under run_limits (max_tool_calls). AgentConfig has +# extra="forbid", so do NOT put it under agent:. # Set to 2 (not 1) so a model that narrates before calling Write still gets a # chance to produce out.txt — keeps the failure shape "agent succeeded at the # prompt but the unsatisfiable criterion fired" rather than TOOL_CALLS_EXHAUSTED. run_limits: - max_turns: 2 + max_tool_calls: 2 agent: type: "claude-code" diff --git a/tasks/smoke_task_timeout.yaml b/tasks/smoke_task_timeout.yaml index 0fb6a3e3f..568e322e2 100644 --- a/tasks/smoke_task_timeout.yaml +++ b/tasks/smoke_task_timeout.yaml @@ -15,7 +15,7 @@ initial_prompt: | # Sleep is 300s (10× task_timeout) so a regression that lets the agent run # through but kills it on the SDK-level turn timeout would also miscount. run_limits: - max_turns: 2 + max_tool_calls: 2 task_timeout: 30 tags: [smoke-fail] diff --git a/tasks/smoke_variants.yaml b/tasks/smoke_variants.yaml index 48ba75d21..b181b1a86 100644 --- a/tasks/smoke_variants.yaml +++ b/tasks/smoke_variants.yaml @@ -11,7 +11,7 @@ initial_prompt: | tags: [smoke-variants] run_limits: - max_turns: 4 + max_tool_calls: 4 agent: type: "claude-code" diff --git a/tasks/token_check.yaml b/tasks/token_check.yaml index 4212c44a5..3f5a263b5 100644 --- a/tasks/token_check.yaml +++ b/tasks/token_check.yaml @@ -29,5 +29,5 @@ success_criteria: weight: 1.0 run_limits: - max_turns: 15 + max_tool_calls: 15 task_timeout: 120 diff --git a/tests/fixtures/byoa_demo_plugin/byoa_demo.py b/tests/fixtures/byoa_demo_plugin/byoa_demo.py index d62532f50..d5ff8ca4d 100644 --- a/tests/fixtures/byoa_demo_plugin/byoa_demo.py +++ b/tests/fixtures/byoa_demo_plugin/byoa_demo.py @@ -44,5 +44,5 @@ def register(registry: type[AgentRegistry]) -> None: ``registry`` is the ``AgentRegistry`` class (not an instance). """ - assert SPI_VERSION == 1, f"byoa_demo supports coder_eval SPI 1, not {SPI_VERSION}" + assert SPI_VERSION == 2, f"byoa_demo supports coder_eval SPI 2, not {SPI_VERSION}" registry.register(DEMO_KIND, DemoAgentConfig)(DemoAgent) diff --git a/tests/harbor_e2e/fixtures/llm_judge.yaml b/tests/harbor_e2e/fixtures/llm_judge.yaml index d8f1cbb72..ed4c770c5 100644 --- a/tests/harbor_e2e/fixtures/llm_judge.yaml +++ b/tests/harbor_e2e/fixtures/llm_judge.yaml @@ -7,7 +7,7 @@ description: > coder-eval's own `run`/`evaluate`. run_limits: - max_turns: 2 + max_tool_calls: 2 agent: type: "claude-code" diff --git a/tests/lint/live_verdict_contract.py b/tests/lint/live_verdict_contract.py index 75ab8c5ef..5323315f1 100644 --- a/tests/lint/live_verdict_contract.py +++ b/tests/lint/live_verdict_contract.py @@ -1,6 +1,6 @@ """CE036 — every live-observable criterion must honor the ``live_verdict`` contract. -``EarlyStopWatcher``'s deferred fail-stop, verdict latching, and ``_prev_verdicts`` +``TurnMonitor``'s deferred fail-stop, verdict latching, and ``_prev_verdicts`` flip-attribution (``orchestration/early_stop.py``) are correct ONLY because every armed criterion's ``live_verdict`` is: @@ -340,7 +340,7 @@ def verdict_at( """``live_verdict`` over the first ``prefix_len`` commands. Wraps the prefix in a SINGLE ``TurnRecord``, which is exactly how - ``EarlyStopWatcher._collect_verdicts`` calls it (``records = [record]``) — the + ``TurnMonitor._collect_verdicts`` calls it (``records = [record]``) — the watcher rebuilds one record from its own collector on every round rather than accumulating a list. """ @@ -447,7 +447,7 @@ def contract_violations(checker: BaseCriterion[Any], case: ContractCase) -> list if final not in claimed: violations.append( f"{case.label!r}: live_verdict decided {final!r}, but this instance's " - + f"live_decidable_polarities() claims only {set(claimed) or '{}'}. EarlyStopWatcher " + + f"live_decidable_polarities() claims only {set(claimed) or '{}'}. TurnMonitor " + "would treat that trigger as inert while the checker actually decides it." ) @@ -477,7 +477,7 @@ def permuted_violations( Each shuffle is RENUMBERED (``sequence_number`` reassigned 0..N-1 in the new order) so the permuted trajectory is one the runtime could actually produce: - ``EarlyStopWatcher._collect_verdicts`` keeps its partial trajectory sorted by + ``TurnMonitor._collect_verdicts`` keeps its partial trajectory sorted by ``sequence_number``, so ``live_verdict`` never sees a list whose order contradicts those numbers. Without the renumber this layer would (a) report breaches on inputs the watcher cannot construct, and (b) degrade to a silent diff --git a/tests/lint/rules/no_top_level_run_limits_access.py b/tests/lint/rules/no_top_level_run_limits_access.py index 02ce61aca..483dc07f0 100644 --- a/tests/lint/rules/no_top_level_run_limits_access.py +++ b/tests/lint/rules/no_top_level_run_limits_access.py @@ -1,4 +1,4 @@ -"""CE007: ``.max_turns`` / ``.task_timeout`` / ``.turn_timeout`` are no longer top-level fields. +"""CE007: ``.max_turns`` / ``.max_tool_calls`` / ``.task_timeout`` / ``.turn_timeout`` are not top-level fields. Phase 1 of the unify-run-limits refactor (2026-05-12) removed these from ``TaskDefinition``, ``ExperimentDefaults``, and ``ExperimentVariant``. They @@ -6,7 +6,7 @@ ``.`` attribute access in core code. Pattern matched: an ``Attribute`` read or write whose attribute name is one of -the three banned names AND whose immediate prefix matches a known task-config +the banned names AND whose immediate prefix matches a known task-config identifier (``task``, ``self.task``, ``resolved_task``, ``expanded_task``, ``variant``, ``defaults``, ``experiment.defaults``). @@ -28,7 +28,7 @@ from tests.lint.rules.base import BaseRule -_BANNED_FIELDS = {"max_turns", "task_timeout", "turn_timeout"} +_BANNED_FIELDS = {"max_turns", "max_tool_calls", "task_timeout", "turn_timeout"} # Prefixes (last attribute name OR variable id) that mean "this is a task / # experiment-layer config object" and we should flag the access. diff --git a/tests/test_agent.py b/tests/test_agent.py index 55c876dcb..7aa151a68 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -13,6 +13,7 @@ from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.errors import AgentCrashError, TurnTimeoutError from coder_eval.models import AgentKind, parse_agent_config +from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus, StopReason def test_claude_agent_initialization(): @@ -181,7 +182,6 @@ async def _capture_sdk_options( agent: ClaudeCodeAgent, *, env_path_prepend: list[str] | None = None, - max_turns: int | None = None, ) -> list[ClaudeAgentOptions]: """Run one communicate() turn with a mocked query() and return captured options list.""" import tempfile @@ -210,28 +210,27 @@ async def mock_query(prompt, options): with tempfile.TemporaryDirectory() as tmpdir: await agent.start(tmpdir, env_path_prepend=env_path_prepend) with patch("coder_eval.agents.claude_code_agent.query", mock_query): - await agent.communicate("hello", max_turns=max_turns) + await agent.communicate("hello") return captured_options @pytest.mark.asyncio -async def test_claude_agent_max_turns_kwarg_reaches_sdk_options(): - """`communicate(max_turns=N)` propagates N to ClaudeAgentOptions.max_turns. - - Regression-guard for the Phase-1 refactor: max_turns is a per-call argument - (mirrors `timeout`), not a stored field on the agent. - """ - config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") +async def test_sdk_options_max_turns_reaches_claude_agent_options(): + """`sdk_options={"max_turns": N}` is the only way to set ClaudeAgentOptions.max_turns.""" + config = parse_agent_config(type=AgentKind.CLAUDE_CODE, sdk_options={"max_turns": 3}) agent = ClaudeCodeAgent(config) - captured_options = await _capture_sdk_options(agent, max_turns=42) - assert captured_options[0].max_turns == 42 + with tempfile.TemporaryDirectory() as tmpdir: + await agent.start(tmpdir) + options, _transport, _model = agent._build_claude_query("hello", None, lambda _line: None) + + assert options.max_turns == 3 @pytest.mark.asyncio -async def test_claude_agent_max_turns_default_is_none(): - """Without an explicit max_turns kwarg, ClaudeAgentOptions.max_turns is None (SDK default).""" +async def test_claude_agent_options_max_turns_default_is_none(): + """Without sdk_options.max_turns, ClaudeAgentOptions.max_turns is None (SDK default).""" config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") agent = ClaudeCodeAgent(config) @@ -1644,8 +1643,8 @@ async def test_claude_agent_error_max_turns_is_clean_completion_not_crash(): outcome, not a crash. Treating it as AGENT_CRASH would make it retryable (max_retries=2) and resume the same prompt that just burned its turn budget — pure waste. Instead the agent falls - through to the success path so the orchestrator's existing - ``tool_calls_exhausted`` handling can stop iterating. + through to the success path as a COMPLETED turn. It is NOT + TOOL_CALLS_EXHAUSTED: that status is reachable only through ``should_stop``. """ config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") agent = ClaudeCodeAgent(config) @@ -1665,7 +1664,7 @@ def __init__(self): self.session_id = "s-1" self.usage = {"input_tokens": 100, "output_tokens": 50} self.total_cost_usd = 0.01 - self.num_turns = 11 # > max_turns=10 + self.num_turns = 11 self.is_error = True self.subtype = "error_max_turns" self.stop_reason = "tool_use" @@ -1676,15 +1675,18 @@ async def mock_query(prompt, options, transport=None): yield ResultMessage() raise ProcessError("Command failed with exit code 1", exit_code=1, stderr="") + recorder = _EventRecorder() + with tempfile.TemporaryDirectory() as tmpdir: await agent.start(tmpdir) with patch("coder_eval.agents.claude_code_agent.query", mock_query): # Must NOT raise: error_max_turns is a clean completion path. - turn_record = await agent.communicate("solve something hard") + turn_record = await agent.communicate("solve something hard", stream_callback=recorder) assert turn_record.crashed is False - assert turn_record.tool_calls_exhausted is True + assert turn_record.tool_calls_exhausted is False + assert [e.status for e in recorder.events if isinstance(e, AgentEndEvent)] == [AgentEndStatus.COMPLETED] # Iteration counter advances normally on a clean turn (no rollback). assert agent._iteration == 1 # The ResultMessage details are still captured for diagnostics. @@ -1730,19 +1732,67 @@ async def mock_query(prompt, options, transport=None): # rather than ProcessError — exercise the except-Exception branch. raise RuntimeError("SDK stream wrapped the CLI exit-1 as a bare Exception") + recorder = _EventRecorder() + with tempfile.TemporaryDirectory() as tmpdir: await agent.start(tmpdir) with patch("coder_eval.agents.claude_code_agent.query", mock_query): - turn_record = await agent.communicate("solve something hard") + turn_record = await agent.communicate("solve something hard", stream_callback=recorder) assert turn_record.crashed is False - assert turn_record.tool_calls_exhausted is True + assert turn_record.tool_calls_exhausted is False + assert [e.status for e in recorder.events if isinstance(e, AgentEndEvent)] == [AgentEndStatus.COMPLETED] assert agent._iteration == 1 assert turn_record.result_summary is not None assert turn_record.result_summary.subtype == "error_max_turns" +class _EventRecorder: + def __init__(self) -> None: + self.events: list = [] + + def on_event(self, event) -> None: + self.events.append(event) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("reason", "status", "exhausted"), + [ + (StopReason.TOOL_CALL_CAP, AgentEndStatus.TOOL_CALLS_EXHAUSTED, True), + (StopReason.TOKEN_BUDGET, AgentEndStatus.TOKEN_BUDGET_EXCEEDED, False), + (StopReason.EARLY_CRITERION, AgentEndStatus.STOPPED_EARLY, False), + ], +) +async def test_claude_agent_should_stop_ends_turn_with_the_reason_status(reason, status, exhausted): + """A should_stop reason ends the turn at that boundary, cleanly, with end_status_for(reason).""" + config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") + agent = ClaudeCodeAgent(config) + recorder = _EventRecorder() + dispatched: list[str] = [] + + class AssistantMessage: + def __init__(self, text): + self.content = text + self.model = "mock-model" + + async def mock_query(prompt, options, transport=None): + for text in ("first", "second", "third"): + dispatched.append(text) + yield AssistantMessage(text) + + with tempfile.TemporaryDirectory() as tmpdir: + await agent.start(tmpdir) + with patch("coder_eval.agents.claude_code_agent.query", mock_query): + turn_record = await agent.communicate("go", stream_callback=recorder, should_stop=lambda: reason) + + assert dispatched == ["first"] + assert turn_record.crashed is False + assert turn_record.tool_calls_exhausted is exhausted + assert [e.status for e in recorder.events if isinstance(e, AgentEndEvent)] == [status] + + def test_setting_sources_default_is_project(): """When config.setting_sources is None, it defaults to ['project'] at runtime.""" config = parse_agent_config( @@ -1811,7 +1861,6 @@ def on_event(self, event): task_id="claude_code", user_input="hi", iteration=1, - max_turns=None, log=agent._log, turn_start_time=time.monotonic(), deadline=None, diff --git a/tests/test_agent_judge_criterion.py b/tests/test_agent_judge_criterion.py index 3aa2b37b3..fa73ea6e7 100644 --- a/tests/test_agent_judge_criterion.py +++ b/tests/test_agent_judge_criterion.py @@ -77,8 +77,8 @@ def _make_mock_agent(agent_output: str) -> MagicMock: _orig_run_async = SubAgentRunner.run_async -async def _run_with_capture_simulation(self, user_msg, *, max_turns, turn_timeout): - turn = await _orig_run_async(self, user_msg, max_turns=max_turns, turn_timeout=turn_timeout) +async def _run_with_capture_simulation(self, user_msg, *, turn_timeout): + turn = await _orig_run_async(self, user_msg, turn_timeout=turn_timeout) if self.capture is not None and self.capture.verdict is None and self.capture.error is None: try: data = _json.loads(turn.agent_output) @@ -259,13 +259,11 @@ def test_agent_judge_config_propagates(sandbox: Sandbox, direct_route: DirectRou assert set(agent_config.allowed_tools or []) == {"Read", "Grep", "mcp__coder_eval_judge__submit_verdict"} assert agent_config.disallowed_tools == ["Bash"] assert agent_config.setting_sources == [] - assert agent_config.sdk_options == {"effort": "low"} - # max_turns and turn_timeout are passed as call-time args to communicate(), - # not stored on AgentConfig. + assert agent_config.sdk_options == {"effort": "low", "max_turns": 3} mock_agent.communicate.assert_awaited_once() kwargs = mock_agent.communicate.call_args.kwargs assert kwargs["timeout"] == 45.0 - assert kwargs["max_turns"] == 3 + assert "max_turns" not in kwargs def test_agent_judge_rejects_turn_timeout_below_ten() -> None: @@ -442,9 +440,24 @@ def _default_with_sdk_options() -> AgentConfig: "max_thinking_tokens": 1024, "effort": "high", "fallback_model": "claude-haiku-4-5-20251001", + "max_turns": criterion.max_turns, } +def test_agent_judge_criterion_max_turns_overrides_sdk_options_max_turns() -> None: + """``criterion.max_turns`` is the judge's turn cap; an ``agent.sdk_options.max_turns`` cannot replace it.""" + from coder_eval.criteria.agent_judge import _build_agent_config + + criterion = AgentJudgeCriterion( + description="x", + prompt="grade", + max_turns=7, + agent=parse_agent_config(type="claude-code", sdk_options={"max_turns": 99}), + ) + config = _build_agent_config(criterion, system_prompt="sys") + assert config.sdk_options["max_turns"] == 7 + + def test_agent_judge_security_ignore_patterns_floor_enforced(sandbox: Sandbox, direct_route: DirectRoute) -> None: """User-supplied ignore_patterns are merged with the security floor, never replace it.""" @@ -1157,7 +1170,7 @@ def _patch_runner_with_capture(verdict_payload: dict | None, agent_output: str = from coder_eval.evaluation.sub_agent import SubAgentRunner from coder_eval.models import JudgeVerdict - async def _stub_run(self, user_msg, *, max_turns, turn_timeout): + async def _stub_run(self, user_msg, *, turn_timeout): if verdict_payload is not None: self.capture.verdict = JudgeVerdict.model_validate(verdict_payload) self.capture.error = None @@ -1189,7 +1202,7 @@ def test_agent_judge_tool_channel_overwrites_on_retry(sandbox: Sandbox, direct_r """LAST-call discipline at the criterion layer — the final verdict wins.""" criterion = AgentJudgeCriterion(description="x", prompt="grade") - async def _stub_run(self, user_msg, *, max_turns, turn_timeout): + async def _stub_run(self, user_msg, *, turn_timeout): # Simulate two calls — final one wins. self.capture.verdict = JudgeVerdict(score=0.2, rationale="first") self.capture.called_count += 1 @@ -1245,7 +1258,7 @@ def test_agent_judge_timeout_returns_judge_criterion_result(sandbox: Sandbox, di criterion = AgentJudgeCriterion(description="x", prompt="grade", turn_timeout=10) - async def _raise(self, user_msg, *, max_turns, turn_timeout): + async def _raise(self, user_msg, *, turn_timeout): raise TurnTimeoutError(10.0, task_id="t", iteration=1) with patch.object(SubAgentRunner, "run_async", _raise): diff --git a/tests/test_agent_telemetry.py b/tests/test_agent_telemetry.py index adcf59108..4bc331068 100644 --- a/tests/test_agent_telemetry.py +++ b/tests/test_agent_telemetry.py @@ -1334,7 +1334,7 @@ def _build_ms(**config_kwargs) -> float: samples = [] for _ in range(5): started = time.perf_counter() - agent._build_claude_query("hi", 60, 10, lambda _line: None) + agent._build_claude_query("hi", 60, lambda _line: None) samples.append((time.perf_counter() - started) * 1000.0) return min(samples) @@ -1399,7 +1399,6 @@ def _state(self, monkeypatch): task_id="t", user_input="go", iteration=1, - max_turns=None, log=agent._log, turn_start_time=0.0, deadline=None, diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 2001dcd66..5db1f5580 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -213,13 +213,13 @@ def test_cli_overrides_apply_to_none_task_without_breaking_contract(self) -> Non resolved = self._resolve(_none_task()) config = BatchRunConfig( run_dir=Path("."), - overrides={"agent.model": "some-model", "run_limits.max_turns": 5}, + overrides={"agent.model": "some-model", "run_limits.max_tool_calls": 5}, ) _apply_cli_overrides(resolved, config) assert resolved.is_none_agent is True # contract intact: still no-op assert resolved.agent is not None and resolved.agent.model == "some-model" - assert resolved.run_limits is not None and resolved.run_limits.max_turns == 5 + assert resolved.run_limits is not None and resolved.run_limits.max_tool_calls == 5 def test_explicit_type_override_replaces_none(self) -> None: """`--type ` is highest-precedence and replaces `type: none` like for any task. diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index 9f3017a3c..b4df23445 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -25,9 +25,11 @@ _to_token_usage, ) from coder_eval.agents.registry import AgentRegistry -from coder_eval.models import AgentKind, AntigravityAgentConfig, AssistantMessage, parse_agent_config +from coder_eval.models import AgentKind, AntigravityAgentConfig, AssistantMessage, RunLimits, parse_agent_config +from coder_eval.orchestration.turn_monitor import TurnMonitor from coder_eval.plugins import ensure_plugins_loaded from coder_eval.pricing import calculate_cost +from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus, StopReason from tests._bracket_clock import AnchoredClock, assert_bracket_on_the_clock, assert_overhead_is_measured from tests._fixtures.golden_streams._scrub import assert_reconciliation from tests._fixtures.golden_streams.antigravity_fixtures import ( @@ -983,10 +985,11 @@ async def test_communicate_respects_should_stop_during_poll(monkeypatch): call_count = 0 - def should_stop() -> bool: + def should_stop() -> StopReason | None: nonlocal call_count call_count += 1 - return call_count > 2 # False for batch1's 2 steps; True on the post-sleep check + # None for batch1's 2 steps; a reason on the post-sleep check + return StopReason.EARLY_CRITERION if call_count > 2 else None await agent.communicate("do it", should_stop=should_stop) @@ -1077,7 +1080,7 @@ async def test_communicate_recovers_from_transient_reentrancy_after_cooperative_ agent.working_directory = Path("/tmp") agent._sdk_agent = SimpleNamespace(conversation=conversation, is_started=True) - await agent.communicate("do it", should_stop=lambda: True) # breaks after the first step + await agent.communicate("do it", should_stop=lambda: StopReason.EARLY_CRITERION) # breaks after the first step # Without the retry, this second call raises AgentCrashError wrapping the # fake's RuntimeError (verified live before the fix landed). With it, the @@ -1496,12 +1499,10 @@ def test_tool_names_cover_the_canonical_vocabulary(): assert AntigravityAgent.tool_names.names["Bash"] == ("run_command",) -# --- max_turns visible-turn cap ----------------------------------------------------- +# --- should_stop reasons ------------------------------------------------------------- # -# max_turns was accepted and never read on this backend, so a task capping turns ran -# uncapped here while the same file capped on Claude Code. The cap counts VISIBLE -# turns (tool calls — result_metrics.visible_turn_count's unit), enforced on the same -# step-loop boundary as the cooperative stop. +# The adapter owns no cap. A `should_stop` reason ends the step loop at that boundary, +# and the reason picks the end status through `end_status_for`; the turn ends clean. def _tool_steps(count: int) -> list: @@ -1515,56 +1516,65 @@ def _tool_steps(count: int) -> list: return steps -async def test_max_turns_caps_visible_turns(): - """The stream offers 5 tool calls; max_turns=2 keeps 2 and never pulls the rest.""" - agent = _agent_with_steps(_tool_steps(5)) +class _EndCapture: + """Stream callback that keeps the ``AgentEndEvent``.""" - record = await agent.communicate("go", max_turns=2) + def __init__(self) -> None: + self.end: AgentEndEvent | None = None - assert len(record.commands) == 2 - assert record.tool_calls_exhausted is True + def on_event(self, event: object) -> None: + if isinstance(event, AgentEndEvent): + self.end = event -async def test_max_turns_keeps_the_deciding_step_whole(): - """The tool call that reaches the cap is completed, not cut mid-flight.""" - agent = _agent_with_steps(_tool_steps(3)) +@pytest.mark.parametrize( + ("reason", "status", "exhausted"), + [ + (StopReason.TOOL_CALL_CAP, AgentEndStatus.TOOL_CALLS_EXHAUSTED, True), + (StopReason.TOKEN_BUDGET, AgentEndStatus.TOKEN_BUDGET_EXCEEDED, False), + ], +) +async def test_should_stop_reason_ends_the_turn_with_its_status(reason, status, exhausted): + """A reason after the first processed step ends the loop; nothing further is pulled.""" + agent = _agent_with_steps(_tool_steps(5)) + capture = _EndCapture() - record = await agent.communicate("go", max_turns=1) + record = await agent.communicate("go", stream_callback=capture, should_stop=lambda: reason) + assert capture.end is not None + assert capture.end.status is status + assert record.crashed is False + assert record.tool_calls_exhausted is exhausted assert len(record.commands) == 1 - assert record.commands[0].result_status == "success" - assert record.commands[0].result_summary == "0" + assert agent._sdk_agent.conversation.cancel_call_count == 1 + +async def test_stop_after_a_done_step_keeps_the_deciding_call_whole(): + """A stop polled after the call's DONE step keeps its result.""" + agent = _agent_with_steps(_tool_steps(3)) + polls = 0 -async def test_under_the_cap_completes_normally(): - agent = _agent_with_steps(_tool_steps(2)) + def should_stop() -> StopReason | None: + nonlocal polls + polls += 1 + return StopReason.TOOL_CALL_CAP if polls >= 2 else None - record = await agent.communicate("go", max_turns=5) + record = await agent.communicate("go", should_stop=should_stop) - assert len(record.commands) == 2 - assert record.tool_calls_exhausted is False + assert len(record.commands) == 1 + assert record.commands[0].result_status == "success" + assert record.commands[0].result_summary == "0" -async def test_no_max_turns_is_uncapped(): - """None must preserve the pre-existing behavior exactly.""" +async def test_no_reason_consumes_every_step(): agent = _agent_with_steps(_tool_steps(4)) - record = await agent.communicate("go") + record = await agent.communicate("go", should_stop=lambda: None) assert len(record.commands) == 4 assert record.tool_calls_exhausted is False -async def test_cooperative_stop_outranks_the_cap(): - """Both firing on the same step reports STOPPED_EARLY — the more specific reason.""" - agent = _agent_with_steps(_tool_steps(5)) - - record = await agent.communicate("go", max_turns=1, should_stop=lambda: True) - - assert record.tool_calls_exhausted is False - assert len(record.commands) == 1 - - async def test_cap_reached_on_a_poll_redrain_stops_polling(monkeypatch): """The cap and the background-poll loop share a boundary. @@ -1581,8 +1591,7 @@ async def test_cap_reached_on_a_poll_redrain_stops_polling(monkeypatch): batch1 = [_step("TOOL_CALL", "ACTIVE", target="TARGET_ENVIRONMENT", tool_calls=[bg])] # The re-drain kicks off a SECOND background job, then closes the first and runs # one more call — reaching the cap (2) with an orphan still ACTIVE. Both exit - # conditions are live at once, and the cap has to win: otherwise the loop keeps - # polling out a background job on a run that is already over. + # conditions are live at once, and the cap has to win. batch2 = [ _step( "TOOL_CALL", @@ -1603,9 +1612,11 @@ async def test_cap_reached_on_a_poll_redrain_stops_polling(monkeypatch): batch3 = _tool_steps(2) # must never be drained agent = _agent_with_steps([batch1, batch2, batch3]) conv = agent._sdk_agent.conversation + monitor = TurnMonitor("t", [], limits=RunLimits(max_tool_calls=2)) - record = await agent.communicate("go", max_turns=2) + record = await agent.communicate("go", stream_callback=monitor, should_stop=monitor.should_stop) + assert monitor.stop_reason is StopReason.TOOL_CALL_CAP assert record.tool_calls_exhausted is True # The cap counts RESOLVED calls. The still-open bg2 is force-closed and recorded # as unresolved rather than dropped, so the trajectory shows what was interrupted. diff --git a/tests/test_cli_set_overrides.py b/tests/test_cli_set_overrides.py index 445496764..352220eea 100644 --- a/tests/test_cli_set_overrides.py +++ b/tests/test_cli_set_overrides.py @@ -60,7 +60,7 @@ def test_alias_and_dash_d_collision_hard_errors(self): assert "--model" in output def test_two_dash_d_collision_hard_errors(self): - result = runner.invoke(app, ["run", "-D", "run_limits.max_turns=30", "-D", "run_limits.max_turns=40"]) + result = runner.invoke(app, ["run", "-D", "run_limits.max_tool_calls=30", "-D", "run_limits.max_tool_calls=40"]) assert result.exit_code != 0 assert "more than once" in _strip_ansi(result.output) @@ -139,7 +139,7 @@ def test_driver_alias_matches_dash_d(self): def test_dash_d_value_coercion(self): # YAML-typed values: int stays int, truthy-alias stays string. - assert _overrides(set_overrides=["run_limits.max_turns=30"])["run_limits.max_turns"] == 30 + assert _overrides(set_overrides=["run_limits.max_tool_calls=30"])["run_limits.max_tool_calls"] == 30 assert _overrides(set_overrides=["agent.model=on"])["agent.model"] == "on" def test_collision_with_model_alias(self): @@ -170,15 +170,19 @@ def test_sdk_option_merges_without_clobbering(self): apply_overrides(task, _overrides(set_overrides=["agent.sdk_options.effort=high"])) assert task.agent.sdk_options == {"max_thinking_tokens": 1024, "effort": "high"} - def test_max_turns_leaves_task_timeout_intact(self): + def test_max_tool_calls_leaves_task_timeout_intact(self): from coder_eval.orchestration.overrides import apply_overrides task = self._task(run_limits={"task_timeout": 600}) - apply_overrides(task, _overrides(set_overrides=["run_limits.max_turns=5"])) + apply_overrides(task, _overrides(set_overrides=["run_limits.max_tool_calls=5"])) assert task.run_limits is not None - assert task.run_limits.max_turns == 5 + assert task.run_limits.max_tool_calls == 5 assert task.run_limits.task_timeout == 600 + def test_dash_d_run_limits_max_turns_is_rejected(self): + with pytest.raises(typer.BadParameter, match=r"unknown field 'max_turns' under 'run_limits'"): + _overrides(set_overrides=["run_limits.max_turns=5"]) + def test_docker_working_dir_override(self): from coder_eval.orchestration.overrides import apply_overrides diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index 5e111bbc8..e5948d40d 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -482,12 +482,14 @@ def test_get_state_returns_current_state(): import subprocess # noqa: E402 import tempfile # noqa: E402 import time # noqa: E402 +from collections.abc import Callable # noqa: E402 from pathlib import Path # noqa: E402 from types import SimpleNamespace # noqa: E402 from openai_codex.generated.v2_all import Turn, TurnCompletedNotification # noqa: E402 from coder_eval.errors import AgentCrashError, TurnTimeoutError # noqa: E402 +from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus, StopReason # noqa: E402 def _item_notification( @@ -2012,13 +2014,34 @@ def test_zsh_login_shell_restores_mock_prepend_end_to_end(self, monkeypatch, tmp agent._cleanup_login_shell_home() -class TestMaxTurnsVisibleTurnCap: - """``max_turns`` was documented as "unused for Codex single-turn" and dropped. +class _EndCapture: + """Stream callback that keeps the ``AgentEndEvent``.""" - Codex delivers one SDK turn per ``communicate()``, so a native turn counter would - cap at 1 and mean nothing; the cap therefore counts VISIBLE turns (completed tool - calls — the unit ``result_metrics.visible_turn_count`` sums) and is enforced on the - same pump boundary as the cooperative stop. + def __init__(self) -> None: + self.end: AgentEndEvent | None = None + + def on_event(self, event: object) -> None: + if isinstance(event, AgentEndEvent): + self.end = event + + +def _stop_on_call(n: int, reason: StopReason) -> Callable[[], StopReason | None]: + """A ``should_stop`` that returns ``reason`` on its ``n``-th poll and every poll after.""" + calls = 0 + + def should_stop() -> StopReason | None: + nonlocal calls + calls += 1 + return reason if calls >= n else None + + return should_stop + + +class TestShouldStopReasons: + """The adapter owns no cap: a ``should_stop`` reason ends the pump at that boundary. + + The reason picks the end status through ``end_status_for``, and the turn ends clean + (``crashed=False``) whichever reason fired. """ @staticmethod @@ -2039,68 +2062,60 @@ def _cmd_notifications(count: int) -> list: notifications.append(_turn_completed()) return notifications - async def test_cap_stops_the_pump_at_the_limit(self): + async def test_tool_call_cap_ends_tool_calls_exhausted(self): agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(5)) + capture = _EndCapture() - record = await agent.communicate("go", max_turns=2) + record = await agent.communicate( + "go", stream_callback=capture, should_stop=_stop_on_call(1, StopReason.TOOL_CALL_CAP) + ) - assert len(record.commands) == 2 + assert capture.end is not None + assert capture.end.status is AgentEndStatus.TOOL_CALLS_EXHAUSTED + assert record.crashed is False assert record.tool_calls_exhausted is True + assert len(record.commands) == 1 + + async def test_token_budget_ends_token_budget_exceeded(self): + agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(5)) + capture = _EndCapture() + + record = await agent.communicate( + "go", stream_callback=capture, should_stop=_stop_on_call(1, StopReason.TOKEN_BUDGET) + ) + + assert capture.end is not None + assert capture.end.status is AgentEndStatus.TOKEN_BUDGET_EXCEEDED + assert record.crashed is False + assert record.tool_calls_exhausted is False - async def test_cap_keeps_the_deciding_call_complete(self): - """Counting COMPLETED calls means the one that reaches the cap keeps its result.""" + async def test_stop_keeps_the_deciding_call_complete(self): + """A stop polled after a call's completion keeps that call's result.""" agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(3)) - record = await agent.communicate("go", max_turns=1) + record = await agent.communicate("go", should_stop=_stop_on_call(2, StopReason.TOOL_CALL_CAP)) assert len(record.commands) == 1 assert record.commands[0].result_status == "success" - async def test_cap_interrupts_the_in_flight_turn(self): - """Best-effort server-side interrupt, so the cap actually stops spend.""" + async def test_stop_interrupts_the_in_flight_turn(self): + """Best-effort server-side interrupt, so the stop actually ends spend.""" agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(5)) - await agent.communicate("go", max_turns=1) + await agent.communicate("go", should_stop=_stop_on_call(2, StopReason.TOOL_CALL_CAP)) assert agent.thread.last_handle.interrupted is True - async def test_under_the_cap_completes_normally(self): - agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(2)) - - record = await agent.communicate("go", max_turns=5) - - assert len(record.commands) == 2 - assert record.tool_calls_exhausted is False - - async def test_no_cap_consumes_the_whole_stream(self): - """None must preserve the pre-existing behavior exactly.""" + async def test_no_reason_consumes_the_whole_stream(self): agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(4)) - record = await agent.communicate("go") + record = await agent.communicate("go", should_stop=lambda: None) assert len(record.commands) == 4 assert record.tool_calls_exhausted is False - async def test_cooperative_stop_outranks_the_cap(self): - """Both firing on the same notification reports STOPPED_EARLY.""" - agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(5)) - - record = await agent.communicate("go", max_turns=1, should_stop=lambda: True) - - assert record.tool_calls_exhausted is False - - async def test_capped_turn_still_folds_sub_agent_tokens(self, monkeypatch, tmp_path): - """A capped turn must not lose the child threads' spend. - - Codex bills sub-agents on separate threads the parent total never sees, and - ``_recover_subagent_tool_calls`` is the ONLY writer of the - ``parent_tool_use_id`` messages ``_fold_subagent_tokens`` sums. So skipping - recovery because the pump was cut short does not just drop telemetry rows — - it silently removes the child's tokens and cost from the run. The cap is a - routine ending, so recovery still runs; only a cooperative stop skips it. - """ - monkeypatch.setenv("CODEX_HOME", str(tmp_path)) - child = "019e0000-eeee-7000-8000-000000000005" + @staticmethod + def _delegation(tmp_path, child: str) -> list: _write_child_rollout( tmp_path, child, @@ -2112,56 +2127,44 @@ async def test_capped_turn_still_folds_sub_agent_tokens(self, monkeypatch, tmp_p ) spawn = _collab_call("spawnAgent", call_id="call_spawn", model="gpt-5.5", child_thread=child) wait = _collab_call("wait", call_id="call_wait", result="5050", child_thread=child) - # The cap fires on the wait, before turn/completed is ever dispatched. - notifications = [ + return [ _item_notification("item/started", spawn), _item_notification("item/completed", spawn), _item_notification("item/started", wait), _item_notification("item/completed", wait), - *self._cmd_notifications(3), + *TestShouldStopReasons._cmd_notifications(3), ] - agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("delegate it", max_turns=2) + async def test_cap_stop_still_folds_sub_agent_tokens(self, monkeypatch, tmp_path): + """A cap stop must not lose the child threads' spend. + + ``_recover_subagent_tool_calls`` is the ONLY writer of the + ``parent_tool_use_id`` messages ``_fold_subagent_tokens`` sums, so skipping it + on a cap stop would remove the child's tokens and cost from the run. + """ + monkeypatch.setenv("CODEX_HOME", str(tmp_path)) + child = "019e0000-eeee-7000-8000-000000000005" + agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._delegation(tmp_path, child)) + + record = await agent.communicate("delegate it", should_stop=_stop_on_call(4, StopReason.TOOL_CALL_CAP)) assert record.tool_calls_exhausted is True - # The child's inner shell command was recovered despite the cap... assert [c for c in record.commands if c.tool_name == "Bash"] - # ...and its generation nests under the spawn, carrying its own tokens... nested = [m for m in record.messages if getattr(m, "parent_tool_use_id", None) == "call_spawn"] assert sum(m.output_tokens for m in nested) == 96 - # ...which is what makes the turn total (and therefore the run cost) - # include the sub-agent instead of silently under-reporting it. assert record.token_usage is not None assert record.token_usage.output_tokens >= 96 assert record.token_usage.cache_read_input_tokens >= 15104 - async def test_cooperative_stop_still_skips_sub_agent_recovery(self, monkeypatch, tmp_path): - """The early-stop path keeps its pre-existing skip: an armed gate already decided.""" + async def test_early_criterion_stop_skips_sub_agent_recovery(self, monkeypatch, tmp_path): + """Same stop point as the cap test: only an early-criterion stop skips recovery.""" monkeypatch.setenv("CODEX_HOME", str(tmp_path)) child = "019e0000-ffff-7000-8000-000000000006" - _write_child_rollout( - tmp_path, - child, - [ - {"type": "function_call", "name": "exec_command", "call_id": "c_py", "arguments": '{"cmd":"x"}'}, - {"type": "function_call_output", "call_id": "c_py", "output": "5050"}, - _token_count_event(inp=23859, cached=15104, out=96, tot_in=23859, tot_cached=15104, tot_out=96), - ], - ) - spawn = _collab_call("spawnAgent", call_id="call_spawn", model="gpt-5.5", child_thread=child) - wait = _collab_call("wait", call_id="call_wait", result="5050", child_thread=child) - notifications = [ - _item_notification("item/started", spawn), - _item_notification("item/completed", spawn), - _item_notification("item/started", wait), - _item_notification("item/completed", wait), - _turn_completed(), - ] - agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) + agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._delegation(tmp_path, child)) - record = await agent.communicate("delegate it", should_stop=lambda: True) + record = await agent.communicate("delegate it", should_stop=_stop_on_call(4, StopReason.EARLY_CRITERION)) + assert record.tool_calls_exhausted is False assert not [c for c in record.commands if c.tool_name == "Bash"] diff --git a/tests/test_config_lineage.py b/tests/test_config_lineage.py index c54de4db4..c47bc9d73 100644 --- a/tests/test_config_lineage.py +++ b/tests/test_config_lineage.py @@ -44,7 +44,7 @@ def _make_default_experiment() -> ExperimentDefinition: experiment_id="default", defaults=ExperimentDefaults( agent={"type": "claude-code", "permission_mode": "acceptEdits"}, - run_limits=RunLimits(max_turns=3), + run_limits=RunLimits(max_tool_calls=3), ), variants=[ExperimentVariant(variant_id="default")], ) diff --git a/tests/test_config_merge_engine.py b/tests/test_config_merge_engine.py index e44d7e1c3..3afad72e4 100644 --- a/tests/test_config_merge_engine.py +++ b/tests/test_config_merge_engine.py @@ -81,7 +81,7 @@ def test_unannotated_free_form_dict_is_deep(self): assert merge_strategy_of(ClaudeCodeAgentConfig.model_fields["sdk_options"]) == "deep" def test_unannotated_list_is_replace(self): - assert merge_strategy_of(RunLimits.model_fields["max_turns"]) == "replace" # scalar + assert merge_strategy_of(RunLimits.model_fields["max_tool_calls"]) == "replace" # scalar # an unannotated list field falls back to replace assert merge_strategy_of(ClaudeCodeAgentConfig.model_fields["allowed_tools"]) == "replace" @@ -435,7 +435,7 @@ class TestValidatePaths: [ "agent.model", "agent.permission_mode", - "run_limits.max_turns", + "run_limits.max_tool_calls", "sandbox.driver", "sandbox.docker.network", "agent.sdk_options.effort", diff --git a/tests/test_config_precedence.py b/tests/test_config_precedence.py index d309e0e36..6389548f8 100644 --- a/tests/test_config_precedence.py +++ b/tests/test_config_precedence.py @@ -232,7 +232,7 @@ def _precedence_task(): description="Test precedence", initial_prompt="test", agent=parse_agent_config(type=AgentKind.CLAUDE_CODE, model="yaml-model", permission_mode="default"), - run_limits=RunLimits(max_turns=10), + run_limits=RunLimits(max_tool_calls=10), sandbox=SandboxConfig(driver="tempdir"), success_criteria=[{"type": "file_exists", "path": "test.py", "description": "test"}], ) @@ -249,7 +249,7 @@ def test_agent_override_precedence_cli_over_yaml(): overrides={ "agent.model": "cli-model", "agent.permission_mode": "bypassPermissions", - "run_limits.max_turns": 99, + "run_limits.max_tool_calls": 99, }, ) @@ -258,24 +258,24 @@ def test_agent_override_precedence_cli_over_yaml(): assert task.agent.model == "cli-model" assert task.agent.permission_mode == "bypassPermissions" assert task.run_limits is not None - assert task.run_limits.max_turns == 99 + assert task.run_limits.max_tool_calls == 99 def test_cli_override_applies_with_lineage_detail(): - """A -D run_limits.max_turns override applies and records `-D` lineage detail.""" + """A -D run_limits.max_tool_calls override applies and records `-D` lineage detail.""" from coder_eval.models import ConfigLineageEntry from coder_eval.orchestration.config import BatchRunConfig from coder_eval.orchestration.experiment import _apply_cli_overrides task = _precedence_task() - config = BatchRunConfig(run_dir=Path("runs/test"), overrides={"run_limits.max_turns": 7}) + config = BatchRunConfig(run_dir=Path("runs/test"), overrides={"run_limits.max_tool_calls": 7}) lineage: dict[str, ConfigLineageEntry] = {} _apply_cli_overrides(task, config, lineage=lineage) assert task.run_limits is not None - assert task.run_limits.max_turns == 7 - assert lineage["run_limits.max_turns"].source_detail == "-D run_limits.max_turns" + assert task.run_limits.max_tool_calls == 7 + assert lineage["run_limits.max_tool_calls"].source_detail == "-D run_limits.max_tool_calls" def test_api_backend_enum_values(): @@ -401,7 +401,7 @@ def test_resolve_route_bedrock_missing_token_asserts(): [ ("DEFAULT_AGENT_MODEL", "agent.by_type.claude-code.model"), ("DEFAULT_PERMISSION_MODE", "agent.by_type.claude-code.permission_mode"), - ("DEFAULT_MAX_TURNS", "run_limits.max_turns"), + ("DEFAULT_MAX_TURNS", "run_limits.max_tool_calls"), ], ) def test_stale_default_env_var_raises(monkeypatch, var_name: str, replacement: str): diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 7c684cd71..699f3de26 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -3361,7 +3361,7 @@ def test_analyze_does_not_deny_the_legacy_key_absolutely(self): assert "`iterations`" in text, "analyze no longer names `iterations` as the current key" denial = text.partition("There is no top-level")[2].partition(".")[0] assert denial, "analyze lost the never-top-level denial sentence entirely" - for name in ("`total_tokens`", "`total_cost_usd`", "`max_turns`", "`criteria_count`"): + for name in ("`total_tokens`", "`total_cost_usd`", "`max_tool_calls`", "`criteria_count`"): assert name in denial, ( f"analyze stopped denying a top-level {name}, which is absent in EVERY " "generation — collateral damage from making the `turns` clause conditional" @@ -3632,7 +3632,7 @@ class TestCE036LiveVerdictContract: """CE036 — every live-observable criterion's `live_verdict` must be deterministic and monotonic (GitHub issue #61 item 2). - `EarlyStopWatcher` latches verdicts, defers the fail-stop, and attributes pass-stop + `TurnMonitor` latches verdicts, defers the fail-stop, and attributes pass-stop flips against the previous round — all correct only while `live_verdict` never contradicts an earlier decision and never varies for identical input. That contract was documented on `LiveVerdict`/`BaseCriterion.live_verdict` but unenforced: a third @@ -3861,7 +3861,7 @@ def recency_verdict(records): def test_permutation_renumbers_so_a_sequence_sorting_checker_is_still_probed(self): """The watcher hands `live_verdict` a trajectory sorted by `sequence_number` - (`EarlyStopWatcher._collect_verdicts`), so a checker may legitimately sort by it + (`TurnMonitor._collect_verdicts`), so a checker may legitimately sort by it too. If the shuffle left the original numbers attached, that sort would undo every permutation and this layer would silently probe nothing. Renumbering keeps the same recency bug detectable through the sort.""" diff --git a/tests/test_early_stop.py b/tests/test_early_stop.py index a5359ad42..869c35856 100644 --- a/tests/test_early_stop.py +++ b/tests/test_early_stop.py @@ -14,9 +14,9 @@ end of this file. Phase 3 (feature live): the ``EarlyStopReason`` / ``EarlyStopInfo`` models and -the ``armed_criteria_passed`` gate; the ``EarlyStopWatcher`` runtime observer +the ``armed_criteria_passed`` gate; the ``TurnMonitor`` runtime observer (stop rule, fail-open, latching, attribution); and the orchestrator wiring -(watcher composed into the stream, ``result.early_stop`` populated, armed-subset +(monitor composed into the stream, ``result.early_stop`` populated, armed-subset gate on an early-stopped run vs the full gate on a completed run). """ @@ -71,12 +71,12 @@ from coder_eval.orchestration.config import BatchRunConfig from coder_eval.orchestration.early_stop import ( EarlyStopConfigError, - EarlyStopWatcher, early_stop_active, validate_early_stop, ) from coder_eval.orchestration.experiment import load_experiment, resolve_all_tasks from coder_eval.orchestration.harness_contract import HarnessContractError +from coder_eval.orchestration.turn_monitor import TurnMonitor from coder_eval.orchestrator import Orchestrator, build_task_event from coder_eval.reports import ReportGenerator from coder_eval.reports.html import _render_criteria, _render_header @@ -85,6 +85,7 @@ AgentEndEvent, AgentEndStatus, AgentStartEvent, + StopReason, ToolEndEvent, ToolEndStatus, ToolStartEvent, @@ -140,7 +141,7 @@ def _task( agent=parse_agent_config(type=agent_type), sandbox=SandboxConfig(driver="tempdir"), success_criteria=criteria, - run_limits=RunLimits(stop_early=stop_early, max_turns=20, stop_early_gate_threshold=gate_threshold), + run_limits=RunLimits(stop_early=stop_early, max_tool_calls=20, stop_early_gate_threshold=gate_threshold), simulation=simulation, ) @@ -258,7 +259,7 @@ def _info(**overrides: Any) -> EarlyStopInfo: sdk_turn_index=1, tool_call_index=1, elapsed_seconds=1.0, - turns_remaining_at_stop=14, + tool_calls_remaining_at_stop=14, ) base.update(overrides) return EarlyStopInfo(**base) @@ -708,7 +709,7 @@ def test_max_steps_to_decide_inert_on_fail_only_criterion(self) -> None: # timeout is inert on it (its "undecided" is its success state). This # tolerance is what lets one dataset-fanned YAML line carry a timeout # for both positive rows (applies) and distractor rows (ignored). The - # runtime inertness itself is asserted in TestEarlyStopWatcher. + # runtime inertness itself is asserted in TestTurnMonitorEarlyStop. task = _task( criteria=[_skill_crit("weather-teller", "date-teller", stop_on_fail=True, max_steps_to_decide=3)], ) @@ -776,7 +777,7 @@ def test_inert_fail_trigger_without_max_count_accepted(self) -> None: def test_all_triggers_inert_still_accepted(self) -> None: # min_count=0, max_count=None decides NEITHER polarity — every trigger - # is inert, the run can never stop early, and the watcher just logs a + # is inert, the run can never stop early, and the monitor just logs a # debug breadcrumb. Accepted: on a fanned dataset some rows # legitimately end up with all-inert triggers. task = _task(criteria=[_cmd_crit(stop_on_pass=True, stop_on_fail=True, min_count=0, max_count=None)]) @@ -826,7 +827,7 @@ def test_guardrail1_armed_antigravity_accepts(self) -> None: validate_early_stop(task) # no raise def test_unarmed_task_is_plain_noop(self) -> None: - # No blocks -> no watcher, byte-for-byte default behavior. The old + # No blocks -> no armed criteria, byte-for-byte default behavior. The old # "at least one criterion" guard is gone with the master arm: there is # nothing left to arm a task that has no blocks. task = _task(criteria=[_skill_crit("s", "s")]) @@ -934,7 +935,7 @@ def _write_task_yaml(tmp_path: Path, *, criterion_yaml: str, stop_early: bool | + "sandbox:\n" + " driver: tempdir\n" + "run_limits:\n" - + " max_turns: 20\n" + + " max_tool_calls: 20\n" + stop_early_line + "success_criteria:\n" + criterion_yaml @@ -1013,7 +1014,7 @@ def test_run_surface_variant_inherits_task_threshold_with_stop_early_false(self, + "sandbox:\n" + " driver: tempdir\n" + "run_limits:\n" - + " max_turns: 20\n" + + " max_tool_calls: 20\n" + " stop_early_gate_threshold: 0.7\n" + "success_criteria:\n" + _ARMED_OBSERVABLE_CRITERION @@ -1102,8 +1103,7 @@ def test_early_stop_ab_variants_disarm_and_arm(self, tmp_path: Path) -> None: # --------------------------------------------------------------------------- # -# Cooperative should_stop seam on ClaudeCodeAgent — still UNWIRED: the -# orchestrator does not pass should_stop yet, so these drive the agent directly. +# Cooperative should_stop seam on ClaudeCodeAgent, driven directly. # --------------------------------------------------------------------------- # @@ -1136,25 +1136,25 @@ async def _run_claude_communicate( """Drive ``ClaudeCodeAgent.communicate`` over a mocked ``query`` yielding ``n_messages`` dummy messages. - ``stop_after``: build a should_stop that returns True once that many messages - have been pulled (checked after each dispatch). ``never``: pass an explicit - always-False should_stop. Neither: pass ``should_stop=None``. Returns + ``stop_after``: build a should_stop that returns ``EARLY_CRITERION`` once that + many messages have been pulled (checked after each dispatch). ``never``: pass + an explicit always-None should_stop. Neither: pass ``should_stop=None``. Returns ``(agent, record, sink, pulled_count)``. """ config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") agent = ClaudeCodeAgent(config) pulled = {"n": 0} - should_stop: Callable[[], bool] | None + should_stop: Callable[[], StopReason | None] | None if stop_after is not None: - def should_stop() -> bool: - return pulled["n"] >= stop_after + def should_stop() -> StopReason | None: + return StopReason.EARLY_CRITERION if pulled["n"] >= stop_after else None elif never: - def should_stop() -> bool: - return False + def should_stop() -> StopReason | None: + return None else: should_stop = None @@ -1192,7 +1192,7 @@ def __exit__(self, *args: Any) -> bool: async def _run_claude_communicate_timeout() -> tuple[ClaudeCodeAgent, _EventSink, BaseException | None]: """Drive ``communicate`` with a slow query (50ms) against a 10ms deadline AND - ``should_stop=True`` — the deadline guard must win. Returns + a should_stop returning ``EARLY_CRITERION`` — the deadline guard must win. Returns ``(agent, sink, raised_exception)``.""" config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") agent = ClaudeCodeAgent(config) @@ -1210,7 +1210,9 @@ async def slow_query(prompt: Any, options: Any, transport: Any = None) -> Any: patch("coder_eval.agents.claude_code_agent.ThreadedWatchdog", _NoopWatchdog), ): try: - await agent.communicate("p", stream_callback=sink, timeout=0.01, should_stop=lambda: True) + await agent.communicate( + "p", stream_callback=sink, timeout=0.01, should_stop=lambda: StopReason.EARLY_CRITERION + ) except TurnTimeoutError as exc: raised = exc return agent, sink, raised @@ -1273,7 +1275,7 @@ async def test_should_stop_none_consumes_full_stream(self) -> None: assert len(ends) == 1 assert ends[0].status == AgentEndStatus.COMPLETED - async def test_should_stop_false_consumes_full_stream(self) -> None: + async def test_should_stop_returning_none_consumes_full_stream(self) -> None: _agent, _record, sink, pulled = await _run_claude_communicate(never=True, n_messages=3) assert pulled == 3 assert _agent_end_events(sink)[0].status == AgentEndStatus.COMPLETED @@ -1315,7 +1317,7 @@ def test_info_defaults(self) -> None: elapsed_seconds=1.5, ) assert info.armed_criteria == [] - assert info.turns_remaining_at_stop is None + assert info.tool_calls_remaining_at_stop is None assert info.gate_threshold == 1.0 def test_gate_threshold_bounds_enforced(self) -> None: @@ -1476,80 +1478,80 @@ def test_armed_criteria_passed_weighted_gate_with_command_executed(self) -> None # --------------------------------------------------------------------------- # -# Phase 3: EarlyStopWatcher +# Phase 3: TurnMonitor early stop # --------------------------------------------------------------------------- # -def _watcher(criteria: list[Any], *, max_turns: int | None = 20, gate_threshold: float = 1.0) -> EarlyStopWatcher: +def _monitor_for(criteria: list[Any], *, max_tool_calls: int | None = 20, gate_threshold: float = 1.0) -> TurnMonitor: task = _task(criteria=criteria) assert task.run_limits is not None - task.run_limits.max_turns = max_turns + task.run_limits.max_tool_calls = max_tool_calls task.run_limits.stop_early_gate_threshold = gate_threshold - return EarlyStopWatcher.for_task(task) + return TurnMonitor.for_task(task, arm=True) -def _feed(watcher: EarlyStopWatcher, events: list[Any]) -> None: +def _feed(monitor: TurnMonitor, events: list[Any]) -> None: for event in events: - watcher.on_event(event) + monitor.on_event(event) -class TestEarlyStopWatcher: +class TestTurnMonitorEarlyStop: def test_for_task_arms_only_stop_criteria(self) -> None: - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True), FileExistsCriterion(path="x", description="x must exist"), ] ) # Only the armed criterion is tracked; the unarmed file_exists is ignored. - assert len(watcher._armed) == 1 + assert len(monitor._armed) == 1 def test_undecided_before_engagement_no_stop(self) -> None: - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, [_agent_start(), _turn_start()]) - assert watcher.should_stop() is False - assert watcher.info is None + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, [_agent_start(), _turn_start()]) + assert monitor.should_stop() is None + assert monitor.info is None def test_pass_stop_fires_on_expected_skill(self) -> None: - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_fail_stop_fires_on_distractor_skill(self) -> None: # A distractor criterion (its skill != the expected skill) fail-stops the # instant its skill is engaged — the per-skill precision signal. - watcher = _watcher([_skill_crit("weather-teller", "date-teller", stop_on_fail=True)]) - _feed(watcher, _skill_events("weather-teller")) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_FAILED + monitor = _monitor_for([_skill_crit("weather-teller", "date-teller", stop_on_fail=True)]) + _feed(monitor, _skill_events("weather-teller")) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_FAILED def test_wrong_skill_does_not_stop_positive_row(self) -> None: # Item 1: a positive row (armed pass) engaging the WRONG skill must NOT # stop — the run keeps going so the expected skill can still load later. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, _skill_events("weather-teller")) - assert watcher.should_stop() is False - assert watcher.info is None + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, _skill_events("weather-teller")) + assert monitor.should_stop() is None + assert monitor.info is None def test_stacked_pass_stop_requires_all(self) -> None: # Pass-stop needs EVERY armed criterion to live-pass. Two positives for # different skills: engaging only the first does not stop; engaging the # second (both now passed) fires the pass-stop. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True), _skill_crit("weather-teller", "weather-teller", stop_on_pass=True), ] ) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is False # only one of two has passed - _feed(watcher, [_tool_end(_skill_cmd("weather-teller", tool_id="w"))]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is None # only one of two has passed + _feed(monitor, [_tool_end(_skill_cmd("weather-teller", tool_id="w"))]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_stacked_wrong_skill_defers_fail_stop_until_positive_decides(self) -> None: # The recall guard: a positive (armed pass) + a distractor (armed fail). @@ -1557,20 +1559,20 @@ def test_stacked_wrong_skill_defers_fail_stop_until_positive_decides(self) -> No # the would-be TP as an FN and deflate suite recall. The misfire is latched # by the criterion's monotone semantics, so once the expected skill engages # (no pass-armed criterion left undecided) the deferred fail-stop fires. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True), _skill_crit("weather-teller", "date-teller", stop_on_fail=True), ] ) - _feed(watcher, _skill_events("weather-teller")) - assert watcher.should_stop() is False # positive undecided -> fail deferred - assert watcher.info is None - _feed(watcher, [_tool_end(_skill_cmd("date-teller", tool_id="d"))]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_FAILED - assert watcher.info.deciding_criterion_description == "weather-teller activation" + _feed(monitor, _skill_events("weather-teller")) + assert monitor.should_stop() is None # positive undecided -> fail deferred + assert monitor.info is None + _feed(monitor, [_tool_end(_skill_cmd("date-teller", tool_id="d"))]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_FAILED + assert monitor.info.deciding_criterion_description == "weather-teller activation" def test_fail_stop_precedes_pass_stop_same_round(self) -> None: # Precedence pin (kills the block-swap mutation): ONE tool call engages @@ -1578,7 +1580,7 @@ def test_fail_stop_precedes_pass_stop_same_round(self) -> None: # live-passes and the distractor live-fails in the SAME evaluation round # with no pass-armed criterion left undecided. Fail-stop is evaluated # before pass-stop, so the round must record CRITERION_FAILED. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True, stop_on_fail=True), # positive -> pass _skill_crit( @@ -1587,18 +1589,18 @@ def test_fail_stop_precedes_pass_stop_same_round(self) -> None: ] ) both = _cmd("Bash", {"command": "cat skills/date-teller/SKILL.md skills/weather-teller/SKILL.md"}) - _feed(watcher, [_agent_start(), _turn_start(), _tool_end(both)]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_FAILED - assert watcher.info.deciding_criterion_description == "weather-teller activation" + _feed(monitor, [_agent_start(), _turn_start(), _tool_end(both)]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_FAILED + assert monitor.info.deciding_criterion_description == "weather-teller activation" def test_auto_positive_row_misfire_alone_never_stops(self) -> None: # A positive row armed `auto` whose agent only ever touches wrong skills: # the fail-stop stays deferred for the whole run (the positive never # decides), so the run continues to the cap and full-trajectory scoring — # never a truncated FN. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True, stop_on_fail=True), # positive -> pass _skill_crit( @@ -1606,19 +1608,19 @@ def test_auto_positive_row_misfire_alone_never_stops(self) -> None: ), # distractor -> fail ] ) - _feed(watcher, _skill_events("weather-teller")) - _feed(watcher, [_turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) - assert watcher.should_stop() is False - assert watcher.info is None + _feed(monitor, _skill_events("weather-teller")) + _feed(monitor, [_turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) + assert monitor.should_stop() is None + assert monitor.info is None def test_auto_positive_pass_stops(self) -> None: # `auto` on a positive resolves to pass-armed: engaging the expected skill # pass-stops, identically to an explicit stop_on_pass=True. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True, stop_on_fail=True)]) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True, stop_on_fail=True)]) + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_auto_mixed_pass_stops_ignoring_undecided_distractors(self) -> None: # THE mixed-arming fix: one positive + two distractors, all armed `auto`. @@ -1626,7 +1628,7 @@ def test_auto_mixed_pass_stops_ignoring_undecided_distractors(self) -> None: # distractors are still "undecided" — fail-armed criteria are not required # to live-pass. (Under the old "every armed must pass" rule this could never # fire, since a distractor can never live-pass.) - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True, stop_on_fail=True), # positive -> pass _skill_crit( @@ -1635,114 +1637,116 @@ def test_auto_mixed_pass_stops_ignoring_undecided_distractors(self) -> None: _skill_crit("news-teller", "date-teller", stop_on_pass=True, stop_on_fail=True), # distractor -> fail ] ) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED # The deciding criterion is the positive that flipped to pass. - assert watcher.info.deciding_criterion_description == "date-teller activation" + assert monitor.info.deciding_criterion_description == "date-teller activation" def test_auto_negative_row_no_pass_stop_on_benign_call(self) -> None: # THE vacuous guard: a negative row (expected_skill == "") stacks only # distractors, so there are ZERO pass-armed criteria. A benign non-skill # tool call must NOT pass-stop on turn 0 (empty all() would be vacuously # True); the run continues to the cap as intended. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "", stop_on_pass=True, stop_on_fail=True), # distractor -> fail _skill_crit("weather-teller", "", stop_on_pass=True, stop_on_fail=True), # distractor -> fail ] ) - _feed(watcher, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) - assert watcher.should_stop() is False - assert watcher.info is None + _feed(monitor, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) + assert monitor.should_stop() is None + assert monitor.info is None def test_auto_negative_row_misfire_fail_stops(self) -> None: # The other half of the asymmetry: a negative row that DOES engage a skill # is a misfire and fail-stops (the precision signal), even though it can # never pass-stop. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "", stop_on_pass=True, stop_on_fail=True), # distractor -> fail _skill_crit("weather-teller", "", stop_on_pass=True, stop_on_fail=True), # distractor -> fail ] ) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_FAILED + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_FAILED def test_mixed_static_arming_pass_stops_ignoring_fail_armed(self) -> None: # The pass-armed-subset rule is not `auto`-specific: an explicit # pass-positive + fail-distractor mix also pass-stops on the positive alone. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True), # pass-armed _skill_crit("weather-teller", "date-teller", stop_on_fail=True), # fail-armed ] ) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_ceiling_bound_defers_fail_stop_below_default_gate_threshold(self) -> None: # The user's worked example on the trigger side: weights 0.8/0.2, # gate_threshold 0.7. The LOW-weight (0.2) criterion misfiring leaves a # ceiling of 0.8 (>= 0.7) — the gate could still pass if the high-weight # positive comes through, so the run must NOT stop yet. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True, stop_on_fail=True, weight=0.8), _skill_crit("weather-teller", "date-teller", stop_on_pass=True, stop_on_fail=True, weight=0.2), ], gate_threshold=0.7, ) - _feed(watcher, _skill_events("weather-teller")) - assert watcher.should_stop() is False - assert watcher.info is None + _feed(monitor, _skill_events("weather-teller")) + assert monitor.should_stop() is None + assert monitor.info is None def test_ceiling_bound_fires_fail_stop_when_high_weight_criterion_fails(self) -> None: # Mirror case: the HIGH-weight (0.8) positive misfiring as a distractor # leaves a ceiling of 0.2 (< 0.7) — the gate can never reach 0.7 no # matter what the low-weight criterion does, so the fail-stop must fire # even though it's the "small" criterion still undecided. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("weather-teller", "date-teller", stop_on_pass=True, stop_on_fail=True, weight=0.8), _skill_crit("news-teller", "date-teller", stop_on_pass=True, stop_on_fail=True, weight=0.2), ], gate_threshold=0.7, ) - _feed(watcher, _skill_events("weather-teller")) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_FAILED + _feed(monitor, _skill_events("weather-teller")) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_FAILED def test_zero_armed_weight_fails_closed_instead_of_dividing_by_zero(self) -> None: """The model rejects weight=0 on an armed criterion; a copy that skips validation - must still not crash the watcher, and must agree with the final gate (closed).""" + must still not crash the monitor, and must agree with the final gate (closed).""" armed = _skill_crit("date-teller", "date-teller", stop_on_fail=True).model_copy(update={"weight": 0.0}) - watcher = EarlyStopWatcher( - "t", [(armed, _watcher([_skill_crit("x", "x", stop_on_fail=True)])._armed[0][1])], max_turns=20 + monitor = TurnMonitor( + "t", + [(armed, _monitor_for([_skill_crit("x", "x", stop_on_fail=True)])._armed[0][1])], + limits=RunLimits(max_tool_calls=20), ) - assert watcher._ceiling(["undecided"]) == 0.0 + assert monitor._ceiling(["undecided"]) == 0.0 def test_default_gate_threshold_fires_fail_stop_on_any_weight(self) -> None: # At the default gate_threshold=1.0, even the low-weight criterion's # failure alone must still fire — byte-for-byte the pre-weighting rule. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True, stop_on_fail=True, weight=0.8), _skill_crit("weather-teller", "date-teller", stop_on_pass=True, stop_on_fail=True, weight=0.2), ] ) - _feed(watcher, _skill_events("weather-teller")) - assert watcher.should_stop() is False # deferred: positive still undecided - _feed(watcher, [_tool_end(_skill_cmd("date-teller", tool_id="d"))]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_FAILED + _feed(monitor, _skill_events("weather-teller")) + assert monitor.should_stop() is None # deferred: positive still undecided + _feed(monitor, [_tool_end(_skill_cmd("date-teller", tool_id="d"))]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_FAILED def test_floor_bound_pass_stops_before_low_weight_distractor_decides(self) -> None: # Floor generalization on the pass side: a high-weight (0.9) positive @@ -1750,45 +1754,45 @@ def test_floor_bound_pass_stops_before_low_weight_distractor_decides(self) -> No # there is no OTHER pass-armed criterion whose weight it needs to share # the floor with (fail-armed distractors are excluded from the # pass-armed floor by design either way). - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True, weight=0.9), ], gate_threshold=0.7, ) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_floor_bound_pass_stop_requires_full_pass_armed_subset_below_default(self) -> None: # Below the default threshold, a partially-decided pass-armed subset # (one of two passed) must NOT pass-stop yet if the still-undecided # one's weight share would drop the floor below the threshold. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True, weight=0.5), _skill_crit("weather-teller", "weather-teller", stop_on_pass=True, weight=0.5), ], gate_threshold=0.7, ) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is False - assert watcher.info is None + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is None + assert monitor.info is None def test_decision_budget_exceeded_when_still_undecided(self) -> None: # An armed criterion capped at max_steps_to_decide=1 that is still # "undecided" after its first tool call forces a budget-exceeded stop. # Full-field EarlyStopInfo parity, matching every other stop-reason test. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True, max_steps_to_decide=1)]) - _feed(watcher, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED - assert watcher.info.deciding_criterion_type == "skill_triggered" - assert watcher.info.deciding_criterion_description == "date-teller activation" - assert watcher.info.sdk_turn_index == 1 - assert watcher.info.tool_call_index == 1 + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True, max_steps_to_decide=1)]) + _feed(monitor, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED + assert monitor.info.deciding_criterion_type == "skill_triggered" + assert monitor.info.deciding_criterion_description == "date-teller activation" + assert monitor.info.sdk_turn_index == 1 + assert monitor.info.tool_call_index == 1 def test_decision_budget_exceeded_names_the_right_criterion_among_several(self) -> None: # Two armed criteria; the first resolves (pass) on the very call that @@ -1796,53 +1800,53 @@ def test_decision_budget_exceeded_names_the_right_criterion_among_several(self) # must be the one whose budget actually tripped — not just the first # armed criterion in list order — and the timeout-driven fail-stop # wins over the first criterion's pass (fail-stop is evaluated first). - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True, max_steps_to_decide=5), _skill_crit("weather-teller", "weather-teller", stop_on_pass=True, max_steps_to_decide=1), ] ) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED - assert watcher.info.deciding_criterion_description == "weather-teller activation" + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED + assert monitor.info.deciding_criterion_description == "weather-teller activation" def test_decision_budget_exceeded_with_command_executed(self) -> None: # The other LiveSuccessCriterion subclass: a command_executed pass-armed # criterion (min_count=1, no upper bound) capped at max_steps_to_decide=1 # that never sees a matching command force-fails identically. - watcher = _watcher([_cmd_crit(min_count=1, max_count=None, stop_on_pass=True, max_steps_to_decide=1)]) - _feed(watcher, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED - assert watcher.info.deciding_criterion_type == "command_executed" + monitor = _monitor_for([_cmd_crit(min_count=1, max_count=None, stop_on_pass=True, max_steps_to_decide=1)]) + _feed(monitor, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED + assert monitor.info.deciding_criterion_type == "command_executed" def test_decision_budget_not_exceeded_below_cap(self) -> None: # Same cap, but only reached on the FIRST tool call (index 1) — a cap of # 2 must not fire yet. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True, max_steps_to_decide=2)]) - _feed(watcher, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) - assert watcher.should_stop() is False - assert watcher.info is None + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True, max_steps_to_decide=2)]) + _feed(monitor, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) + assert monitor.should_stop() is None + assert monitor.info is None def test_real_decision_within_budget_wins_over_budget_check(self) -> None: # The criterion decides (pass-stops) on the SAME tool call that would # otherwise have tripped its budget — the real decision takes priority. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True, max_steps_to_decide=1)]) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True, max_steps_to_decide=1)]) + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_decision_budget_ignored_when_unset(self) -> None: # No max_steps_to_decide -> no budget check, run continues indefinitely - # (up to run_limits.max_turns) while undecided. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) - assert watcher.should_stop() is False - assert watcher.info is None + # (up to run_limits.max_tool_calls) while undecided. + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) + assert monitor.should_stop() is None + assert monitor.info is None def test_timeout_only_arming_pass_within_budget_never_stops(self) -> None: # THE fail-fast-without-success-stop intent: max_steps_to_decide alone @@ -1851,20 +1855,20 @@ def test_timeout_only_arming_pass_within_budget_never_stops(self) -> None: # untouched: no pass-stop (not armed for one), and the timeout can # never fire again (the verdict is no longer undecided). Extra calls # beyond the budget prove the latch holds. - watcher = _watcher([_skill_crit("date-teller", "date-teller", max_steps_to_decide=3)]) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is False + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", max_steps_to_decide=3)]) + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is None for i in range(4): # sail past the budget — still no stop - watcher.on_event(_tool_end(_cmd("Bash", {"command": f"echo {i}"}))) - assert watcher.should_stop() is False - assert watcher.info is None + monitor.on_event(_tool_end(_cmd("Bash", {"command": f"echo {i}"}))) + assert monitor.should_stop() is None + assert monitor.info is None def test_timeout_only_arming_undecided_past_budget_stops(self) -> None: # The other half of the same intent: not engaged within the budget → # effective fail → fail-stop (default gate threshold 1.0). - watcher = _watcher([_skill_crit("date-teller", "date-teller", max_steps_to_decide=2)]) + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", max_steps_to_decide=2)]) _feed( - watcher, + monitor, [ _agent_start(), _turn_start(), @@ -1872,17 +1876,17 @@ def test_timeout_only_arming_undecided_past_budget_stops(self) -> None: _tool_end(_cmd("Bash", {"command": "cat x"})), ], ) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED def test_timeout_inert_on_fail_only_distractor(self) -> None: # A distractor (fail-only decidable) carrying a timeout — the fanned # line case. Its "undecided" is its success state: sailing past the # budget with no misfire must NOT stop the run. - watcher = _watcher([_skill_crit("weather-teller", "date-teller", stop_on_fail=True, max_steps_to_decide=1)]) + monitor = _monitor_for([_skill_crit("weather-teller", "date-teller", stop_on_fail=True, max_steps_to_decide=1)]) _feed( - watcher, + monitor, [ _agent_start(), _turn_start(), @@ -1890,8 +1894,8 @@ def test_timeout_inert_on_fail_only_distractor(self) -> None: _tool_end(_cmd("Bash", {"command": "cat x"})), ], ) - assert watcher.should_stop() is False - assert watcher.info is None + assert monitor.should_stop() is None + assert monitor.info is None def test_low_weight_timeout_absorbed_below_threshold(self) -> None: # A timeout is an ORDINARY weighted fail: a low-weight (0.2) criterion @@ -1899,38 +1903,38 @@ def test_low_weight_timeout_absorbed_below_threshold(self) -> None: # so the run continues — the timeout is absorbed exactly like a # low-weight native fail. (The high-weight positive resolves first so # the deferral is not what's holding the stop.) - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True, weight=0.8), _skill_crit("todo-lister", "todo-lister", weight=0.2, max_steps_to_decide=1), ], gate_threshold=0.7, ) - _feed(watcher, _skill_events("date-teller")) + _feed(monitor, _skill_events("date-teller")) # date-teller passed (0.8 locked in); todo-lister timed out (0.2 lost). # Ceiling = 0.8 >= 0.7 → no fail-stop. Pass-stop floor over the # stop_on_pass subset = 0.8/0.8 = 1.0 >= 0.7 → pass-stop fires instead. - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_low_weight_timeout_absorbed_no_pass_stop_continues(self) -> None: # Same absorption, but with no stop_on_pass anywhere (both criteria # armed via timeouts only): the low-weight timeout alone cannot doom # the 0.7 gate — ceiling 0.8/1.0 after the high-weight positive # latches pass — and nothing else can stop, so the run continues. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", weight=0.8, max_steps_to_decide=50), _skill_crit("todo-lister", "todo-lister", weight=0.2, max_steps_to_decide=1), ], gate_threshold=0.7, ) - _feed(watcher, _skill_events("date-teller")) + _feed(monitor, _skill_events("date-teller")) for i in range(3): - watcher.on_event(_tool_end(_cmd("Bash", {"command": f"echo {i}"}))) - assert watcher.should_stop() is False - assert watcher.info is None + monitor.on_event(_tool_end(_cmd("Bash", {"command": f"echo {i}"}))) + assert monitor.should_stop() is None + assert monitor.info is None def test_pass_stop_deferred_while_outside_pass_capable_undecided(self) -> None: # Recall deferral on the PASS side (mixed arming): A (on_pass: stop) @@ -1938,21 +1942,21 @@ def test_pass_stop_deferred_while_outside_pass_capable_undecided(self) -> None: # via decide_within — is still undecided and within budget. Firing the # pass-stop here would truncate B's expected signal out of the # trajectory, so the stop is HELD. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True), _skill_crit("todo-lister", "todo-lister", max_steps_to_decide=5), ] ) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is False # deferred: todo-lister undecided - assert watcher.info is None + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is None # deferred: todo-lister undecided + assert monitor.info is None # Once B decides (pass), the on_pass=stop floor (over A alone) still # holds, so the deferred pass-stop fires on that round. - _feed(watcher, [_tool_end(_skill_cmd("todo-lister", tool_id="td"))]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + _feed(monitor, [_tool_end(_skill_cmd("todo-lister", tool_id="td"))]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_pass_stop_fires_after_outside_criterion_fails_below_threshold(self) -> None: # The other resolution of the deferral: B (0.2, decide_within=2) times @@ -1960,57 +1964,57 @@ def test_pass_stop_fires_after_outside_criterion_fails_below_threshold(self) -> # the low-weight fail cannot doom the ceiling (0.8 >= 0.7), so no # fail-stop — and with B decided, the deferral clears and the floor # (1.0 over the on_pass=stop subset) fires the pass-stop. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True, weight=0.8), _skill_crit("todo-lister", "todo-lister", weight=0.2, max_steps_to_decide=2), ], gate_threshold=0.7, ) - _feed(watcher, _skill_events("date-teller")) # call 1: A passes, B undecided (budget 2) - assert watcher.should_stop() is False # deferred while B is in budget - watcher.on_event(_tool_end(_cmd("Bash", {"command": "ls"}))) # call 2: B's budget expires - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + _feed(monitor, _skill_events("date-teller")) # call 1: A passes, B undecided (budget 2) + assert monitor.should_stop() is None # deferred while B is in budget + monitor.on_event(_tool_end(_cmd("Bash", {"command": "ls"}))) # call 2: B's budget expires + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_decision_budget_exceeded_on_in_flight_call(self) -> None: # The budget expires on the in-flight round: an AgentStart + TurnStart + # a dispatched ToolStart with NO ToolEnd. The in-flight call reports as # tool call 1, which meets decide_within=1 — the timeout fail-stop must # fire on the call itself, before any result resolves. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True, max_steps_to_decide=1)]) + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True, max_steps_to_decide=1)]) start = ToolStartEvent(task_id="t", tool=_cmd("Bash", {"command": "echo hi"})) - _feed(watcher, [_agent_start(), _turn_start(), start]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED - assert watcher.info.tool_call_index == 1 + _feed(monitor, [_agent_start(), _turn_start(), start]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED + assert monitor.info.tool_call_index == 1 def test_timeout_fail_deferred_while_sibling_positive_in_budget(self) -> None: # Criterion B times out (budget 1) while criterion A — pass-capable, # no budget — is still undecided: the fail-stop is DEFERRED (recall # protection). It fires the moment A decides. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True), _skill_crit("todo-lister", "todo-lister", max_steps_to_decide=1), ] ) - _feed(watcher, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "ls"}))]) - assert watcher.should_stop() is False # deferred: date-teller undecided - watcher.on_event(_tool_end(_skill_cmd("date-teller", tool_id="sk-9"))) + _feed(monitor, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "ls"}))]) + assert monitor.should_stop() is None # deferred: date-teller undecided + monitor.on_event(_tool_end(_skill_cmd("date-teller", tool_id="sk-9"))) # A resolved (pass) → deferral clears → B's latched timeout fail fires # (ceiling 0.5 < 1.0). Fail-stop precedes pass-stop in the same round. - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED def test_verdicts_latch_and_are_not_repolled(self) -> None: # Once a criterion decides on a resolved round, its live_verdict is # never called again — count the checker's calls directly. - watcher = _watcher([_skill_crit("date-teller", "date-teller", max_steps_to_decide=10)]) - checker = watcher._armed[0][1] + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", max_steps_to_decide=10)]) + checker = monitor._armed[0][1] calls = {"n": 0} original = type(checker).live_verdict @@ -2020,92 +2024,92 @@ def counting(self_, criterion, records): type(checker).live_verdict = counting # type: ignore[method-assign] try: - _feed(watcher, _skill_events("date-teller")) # decides pass on call 1 + _feed(monitor, _skill_events("date-teller")) # decides pass on call 1 decided_at = calls["n"] for i in range(5): - watcher.on_event(_tool_end(_cmd("Bash", {"command": f"echo {i}"}))) + monitor.on_event(_tool_end(_cmd("Bash", {"command": f"echo {i}"}))) assert calls["n"] == decided_at # latched: zero further polls finally: type(checker).live_verdict = original # type: ignore[method-assign] - assert watcher.should_stop() is False # and still no stop (no stop_on_pass) + assert monitor.should_stop() is None # and still no stop (no stop_on_pass) def test_pass_without_stop_on_pass_never_stops(self) -> None: # A stop_on_fail-armed positive... cannot exist (fail is inert on a # positive); the realistic shape is both-trigger fanning. On a positive # row with only stop_on_fail, NOTHING can ever fire — engaging the # skill latches pass silently and the run continues. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_fail=True)]) - _feed(watcher, _skill_events("date-teller")) - assert watcher.should_stop() is False - assert watcher.info is None + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_fail=True)]) + _feed(monitor, _skill_events("date-teller")) + assert monitor.should_stop() is None + assert monitor.info is None def test_records_turn_and_tool_index(self) -> None: - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, _skill_events("date-teller")) - assert watcher.info is not None - assert watcher.info.sdk_turn_index == 1 - assert watcher.info.tool_call_index == 1 - - def test_turns_remaining_from_max_turns(self) -> None: - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)], max_turns=15) - _feed(watcher, _skill_events("date-teller")) - assert watcher.info is not None - assert watcher.info.turns_remaining_at_stop == 14 # 15 - sdk_turn_index(1) - - def test_turns_remaining_none_when_max_turns_unset(self) -> None: - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)], max_turns=None) - _feed(watcher, _skill_events("date-teller")) - assert watcher.info is not None - assert watcher.info.turns_remaining_at_stop is None + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, _skill_events("date-teller")) + assert monitor.info is not None + assert monitor.info.sdk_turn_index == 1 + assert monitor.info.tool_call_index == 1 + + def test_tool_calls_remaining_from_max_tool_calls(self) -> None: + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)], max_tool_calls=15) + _feed(monitor, _skill_events("date-teller")) + assert monitor.info is not None + assert monitor.info.tool_calls_remaining_at_stop == 14 # 15 - tool_call_index(1) + + def test_tool_calls_remaining_none_when_max_tool_calls_unset(self) -> None: + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)], max_tool_calls=None) + _feed(monitor, _skill_events("date-teller")) + assert monitor.info is not None + assert monitor.info.tool_calls_remaining_at_stop is None def test_fail_open_on_raising_verdict(self) -> None: - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) with patch.object(SkillTriggeredChecker, "live_verdict", side_effect=RuntimeError("boom")): - _feed(watcher, _skill_events("date-teller")) + _feed(monitor, _skill_events("date-teller")) # Fail-open: disarmed, no false stop, degrades to a full run. - assert watcher.disarmed is True - assert watcher.should_stop() is False - assert watcher.info is None + assert monitor.disarmed is True + assert monitor.should_stop() is None + assert monitor.info is None def test_unresolved_tool_end_does_not_latch(self) -> None: # finalize() force-closes orphaned tools as UNRESOLVED AFTER the message # loop ends and the terminal status is chosen. Such an orphan Skill # engagement must NOT trip a stop, else a naturally-completed (or # timed-out / crashed) run gets recorded as early-stopped. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, [_agent_start(), _turn_start(), _unresolved_skill_end("date-teller")]) - assert watcher.should_stop() is False - assert watcher.info is None - assert watcher._tool_call_index == 0 # the unresolved end is not even counted + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, [_agent_start(), _turn_start(), _unresolved_skill_end("date-teller")]) + assert monitor.should_stop() is None + assert monitor.info is None + assert monitor._tool_call_index == 0 # the unresolved end is not even counted def test_resolved_after_unresolved_still_decides(self) -> None: # An UNRESOLVED end never evaluates, but a later RESOLVED engagement still # fires the stop (skipping orphan rounds never suppresses a real stop). - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, [_agent_start(), _turn_start(), _unresolved_skill_end("date-teller")]) - assert watcher.info is None - _feed(watcher, [_tool_end(_skill_cmd("date-teller", tool_id="sk-real"))]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, [_agent_start(), _turn_start(), _unresolved_skill_end("date-teller")]) + assert monitor.info is None + _feed(monitor, [_tool_end(_skill_cmd("date-teller", tool_id="sk-real"))]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_unresolved_end_recorded_for_trajectory_parity(self) -> None: # TRAJECTORY PARITY: the agent's EventCollector records force-closed # (UNRESOLVED) commands into the TurnRecord that check_all_async later - # scores — e.g. a crashed attempt's drained partial turn. The watcher + # scores — e.g. a crashed attempt's drained partial turn. The monitor # must reduce the SAME trajectory: the orphan is recorded (visible to # the next evaluation round), just never counted or evaluated on. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, [_agent_start(), _turn_start(), _unresolved_skill_end("date-teller")]) - assert watcher._tool_call_index == 0 # no round counted - assert watcher.info is None # no stop fired on the orphan itself - record = watcher._collector.build_turn_record() + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, [_agent_start(), _turn_start(), _unresolved_skill_end("date-teller")]) + assert monitor._tool_call_index == 0 # no round counted + assert monitor.info is None # no stop fired on the orphan itself + record = monitor._collector.build_turn_record() assert any(c.tool_name == "Skill" for c in record.commands) # ...but it IS in the trajectory # The next real round evaluates over the parity trajectory: an unrelated # Bash call decides the criterion pass from the recorded orphan. - _feed(watcher, [_tool_end(_cmd("Bash", {"command": "echo hi"}))]) - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + _feed(monitor, [_tool_end(_cmd("Bash", {"command": "echo hi"}))]) + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_budget_timeout_not_latched_when_orphan_already_decided(self) -> None: # The verdict-preserving half of trajectory parity: a decide_within @@ -2113,13 +2117,13 @@ def test_budget_timeout_not_latched_when_orphan_already_decided(self) -> None: # trajectory scores as a pass. The deciding engagement arrived as a # force-closed orphan (recorded, not evaluated); the budget expiring on # the next round must see it as a live-pass, not fabricate a fail. - watcher = _watcher([_skill_crit("date-teller", "date-teller", max_steps_to_decide=1)]) - _feed(watcher, [_agent_start(), _turn_start(), _unresolved_skill_end("date-teller")]) + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", max_steps_to_decide=1)]) + _feed(monitor, [_agent_start(), _turn_start(), _unresolved_skill_end("date-teller")]) # Round 1 (tool_call_index == 1 >= decide_within): without parity this # would latch a synthetic fail and fire DECISION_BUDGET_EXCEEDED. - _feed(watcher, [_tool_end(_cmd("Bash", {"command": "echo hi"}))]) - assert watcher.should_stop() is False - assert watcher.info is None + _feed(monitor, [_tool_end(_cmd("Bash", {"command": "echo hi"}))]) + assert monitor.should_stop() is None + assert monitor.info is None def test_pass_stop_cuts_undecided_fail_only_sibling_documented_gap(self) -> None: # KNOWN one-sided trade, pinned so a future deferral redesign flips it @@ -2131,120 +2135,120 @@ def test_pass_stop_cuts_undecided_fail_only_sibling_documented_gap(self) -> None # minimum count is reached and the armed gate scores it 0. Documented # in TASK_DEFINITION_GUIDE.md § stop_early: authoritative scoring for # such combinations belongs on the kill-switched run. - watcher = _watcher( + monitor = _monitor_for( [ _skill_crit("date-teller", "date-teller", stop_on_pass=True), _cmd_crit(min_count=1, max_count=3, stop_on_fail=True), ] ) - _feed(watcher, _skill_events("date-teller")) - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + _feed(monitor, _skill_events("date-teller")) + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_fail_stop_reason_precedence_is_criteria_order_invariant(self) -> None: # A native live-fail (distractor misfire) and a decide_within timeout # resolving on the SAME round must report the same persisted/telemetry # reason in either YAML order: the native fail always wins. - def build(order: str) -> EarlyStopWatcher: + def build(order: str) -> TurnMonitor: distractor = _skill_crit("weather-teller", "date-teller", stop_on_fail=True) timed = _skill_crit("date-teller", "date-teller", max_steps_to_decide=1) criteria = [distractor, timed] if order == "distractor-first" else [timed, distractor] - return _watcher(criteria) + return _monitor_for(criteria) for order in ("distractor-first", "timed-first"): - watcher = build(order) + monitor = build(order) # One resolved misfire round: the distractor natively fails AND the # timed criterion's budget (1) expires on the same tool call. - _feed(watcher, [_agent_start(), _turn_start(), _tool_end(_skill_cmd("weather-teller", tool_id="w1"))]) - assert watcher.info is not None, order - assert watcher.info.reason == EarlyStopReason.CRITERION_FAILED, order + _feed(monitor, [_agent_start(), _turn_start(), _tool_end(_skill_cmd("weather-teller", tool_id="w1"))]) + assert monitor.info is not None, order + assert monitor.info.reason == EarlyStopReason.CRITERION_FAILED, order def test_decision_latched_after_fire(self) -> None: - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, _skill_events("date-teller")) - fired = watcher.info + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, _skill_events("date-teller")) + fired = monitor.info # A subsequent (wrong-skill) engagement must not overwrite the latched decision. - _feed(watcher, [_tool_end(_skill_cmd("weather-teller", tool_id="sk-2"))]) - assert watcher.info is fired - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + _feed(monitor, [_tool_end(_skill_cmd("weather-teller", tool_id="sk-2"))]) + assert monitor.info is fired + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_tool_call_fires_before_result(self) -> None: # The decision latches on the tool CALL (ToolStartEvent): a Skill call # whose result never arrives (a cut-short turn would strip it) still stops. # No ToolEndEvent is ever fed. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, [_agent_start(), _turn_start(), _skill_start("date-teller")]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, [_agent_start(), _turn_start(), _skill_start("date-teller")]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED # The in-flight call reports as the 1st tool call even without a ToolEnd. - assert watcher.info.tool_call_index == 1 + assert monitor.info.tool_call_index == 1 def test_tool_call_distractor_fail_fires(self) -> None: # A distractor (armed fail) fail-stops on the tool CALL that engages its # skill, before any result arrives. - watcher = _watcher([_skill_crit("weather-teller", "date-teller", stop_on_fail=True)]) - _feed(watcher, [_agent_start(), _turn_start(), _skill_start("weather-teller")]) - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_FAILED + monitor = _monitor_for([_skill_crit("weather-teller", "date-teller", stop_on_fail=True)]) + _feed(monitor, [_agent_start(), _turn_start(), _skill_start("weather-teller")]) + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_FAILED def test_tool_call_latches_on_file_read_engagement(self) -> None: # Off-Claude agents (antigravity/codex) engage a skill by READING its files - # (skills//...), not via a Skill tool call. The watcher must latch on + # (skills//...), not via a Skill tool call. The monitor must latch on # that Read ToolStart — the file-path parameter carries the signal on the # call itself, so early-stop fires off-Claude just as it does for Claude. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) read = CommandTelemetry( tool_name="Read", tool_id="r1", timestamp=_TS, parameters={"file_path": "/repo/skills/date-teller/SKILL.md"}, ) - _feed(watcher, [_agent_start(), _turn_start(), ToolStartEvent(task_id="t", tool=read)]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.CRITERION_PASSED + _feed(monitor, [_agent_start(), _turn_start(), ToolStartEvent(task_id="t", tool=read)]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.CRITERION_PASSED def test_tool_call_latches_before_unresolved_end(self) -> None: # The call fires the stop in-loop; a later finalize() UNRESOLVED end for # the SAME call is short-circuited (decision already latched) — no relabel, # no double count. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, [_agent_start(), _turn_start(), _skill_start("date-teller", tool_id="sk-1")]) - fired = watcher.info - _feed(watcher, [_unresolved_skill_end("date-teller", tool_id="sk-1")]) - assert watcher.info is fired - assert watcher.info is not None - assert watcher.info.tool_call_index == 1 + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, [_agent_start(), _turn_start(), _skill_start("date-teller", tool_id="sk-1")]) + fired = monitor.info + _feed(monitor, [_unresolved_skill_end("date-teller", tool_id="sk-1")]) + assert monitor.info is fired + assert monitor.info is not None + assert monitor.info.tool_call_index == 1 def test_tool_call_index_counts_prior_resolved_calls(self) -> None: # A prior resolved, non-deciding tool is counted at its ToolEnd; the # deciding in-flight call is then reported as the next (2nd) call. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) prior = _cmd("Bash", {"command": "ls"}) # not a skill engagement - _feed(watcher, [_agent_start(), _turn_start(), _tool_end(prior)]) - assert watcher.info is None - _feed(watcher, [_skill_start("date-teller", tool_id="sk-1", sequence_number=1)]) - assert watcher.info is not None - assert watcher.info.tool_call_index == 2 + _feed(monitor, [_agent_start(), _turn_start(), _tool_end(prior)]) + assert monitor.info is None + _feed(monitor, [_skill_start("date-teller", tool_id="sk-1", sequence_number=1)]) + assert monitor.info is not None + assert monitor.info.tool_call_index == 2 def test_second_agent_start_does_not_reset_origin(self) -> None: # The wall-clock origin is stamped at the FIRST AgentStartEvent only; a # retry's second AgentStart must NOT reset it (the documented no-op branch # in on_event). Exercised deterministically via _started_monotonic rather # than the time-based elapsed_seconds field. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) - _feed(watcher, [_agent_start()]) - origin = watcher._started_monotonic + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True)]) + _feed(monitor, [_agent_start()]) + origin = monitor._started_monotonic assert origin is not None # A second AgentStart (as on a retry) must leave the origin untouched. - _feed(watcher, [_agent_start(), _turn_start()]) - assert watcher._started_monotonic == origin + _feed(monitor, [_agent_start(), _turn_start()]) + assert monitor._started_monotonic == origin # The stop that follows anchors elapsed_seconds to that first origin. - _feed(watcher, [_skill_start("date-teller")]) - assert watcher.info is not None - assert watcher.info.elapsed_seconds >= 0.0 + _feed(monitor, [_skill_start("date-teller")]) + assert monitor.info is not None + assert monitor.info.elapsed_seconds >= 0.0 def test_decision_budget_accumulates_across_retry_attempts(self) -> None: # Pins the documented contract (max_steps_to_decide's field @@ -2253,15 +2257,15 @@ def test_decision_budget_accumulates_across_retry_attempts(self) -> None: # AgentStartEvent (as on a retry) must NOT reset tool_call_index. A # future per-attempt reset would silently change scoring with this # test catching it. - watcher = _watcher([_skill_crit("date-teller", "date-teller", stop_on_pass=True, max_steps_to_decide=2)]) - _feed(watcher, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) - assert watcher.should_stop() is False # 1 call so far, budget is 2 + monitor = _monitor_for([_skill_crit("date-teller", "date-teller", stop_on_pass=True, max_steps_to_decide=2)]) + _feed(monitor, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))]) + assert monitor.should_stop() is None # 1 call so far, budget is 2 # A retry: a second AgentStartEvent must not reset the counter. - _feed(watcher, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo bye"}))]) - assert watcher.should_stop() is True - assert watcher.info is not None - assert watcher.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED - assert watcher.info.tool_call_index == 2 + _feed(monitor, [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo bye"}))]) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason == EarlyStopReason.DECISION_BUDGET_EXCEEDED + assert monitor.info.tool_call_index == 2 # --------------------------------------------------------------------------- # @@ -2271,7 +2275,7 @@ def test_decision_budget_accumulates_across_retry_attempts(self) -> None: class _ScriptedAgent: """Duck-typed agent: replays scripted events through the callback, polling - ``should_stop`` after each and breaking when it flips (mirrors the real + ``should_stop`` after each and breaking on a reason (mirrors the real message-boundary cut). Returns a fixed ``TurnRecord``.""" def __init__(self, events: list[Any], turn: TurnRecord) -> None: @@ -2289,8 +2293,7 @@ async def communicate( *, stream_callback: Any = None, timeout: float | None = None, - max_turns: int | None = None, - should_stop: Callable[[], bool] | None = None, + should_stop: Callable[[], StopReason | None] | None = None, ) -> TurnRecord: for event in self._events: if stream_callback is not None: @@ -2313,7 +2316,7 @@ async def _run_wiring( """Drive ``Orchestrator._evaluation_loop`` with a scripted agent + mock checker. ``scores`` are positional CriterionResult scores matching ``criteria``. - The early-stop watcher is built directly (_setup is not invoked here). + The turn monitor is built directly (_setup is not invoked here). """ task = _task(criteria=criteria, agent_type=agent_type, gate_threshold=gate_threshold) run_dir = tmp_path / "run" @@ -2340,8 +2343,7 @@ async def _run_wiring( ) orch.success_checker = checker - if early_stop_active(task): - orch._early_stop_watcher = EarlyStopWatcher.for_task(task) + orch._monitor = TurnMonitor.for_task(task, arm=True) turn = TurnRecord(iteration=1, user_input="p", agent_output="done") agent = _ScriptedAgent(events, turn) @@ -2372,7 +2374,7 @@ def _distractor_criteria(self) -> list[Any]: ] async def test_default_off_full_gate_no_early_stop(self, tmp_path) -> None: - # Unarmed: no watcher, all criteria gate, advisory 0.0 drags to FAILURE. + # Unarmed: no armed criteria, all criteria gate, advisory 0.0 drags to FAILURE. result, agent, _success = await _run_wiring( criteria=self._criteria(armed=False), events=_skill_events(self._SKILL), @@ -2380,7 +2382,7 @@ async def test_default_off_full_gate_no_early_stop(self, tmp_path) -> None: tmp_path=tmp_path, ) assert result.early_stop is None - assert agent.delivered == 3 # full stream consumed (should_stop=None) + assert agent.delivered == 3 # full stream consumed (should_stop never returns a reason) async def test_pass_stop_cuts_the_stream(self, tmp_path) -> None: # A trailing event AFTER the deciding ToolEnd proves the cut: delivered == 3. @@ -2464,7 +2466,7 @@ async def test_decision_budget_exceeded_gates_through_armed_gate(self, tmp_path) checker = MagicMock() checker.check_all_async = AsyncMock(return_value=[_crit_result(c.type, 1.0) for c in criteria]) orch.success_checker = checker - orch._early_stop_watcher = EarlyStopWatcher.for_task(task) + orch._monitor = TurnMonitor.for_task(task, arm=True) turn = TurnRecord(iteration=1, user_input="p", agent_output="done") events = [_agent_start(), _turn_start(), _tool_end(_cmd("Bash", {"command": "echo hi"}))] agent = _ScriptedAgent(events, turn) @@ -2521,7 +2523,7 @@ async def test_completed_naturally_weighted_armed_gate_does_not_run(self, tmp_pa # FIRED-ONLY gating, diverging in the other direction from the sibling # test below: two ARMED criteria (0.8 passing / 0.2 failing) under a # 0.7 threshold. The weighted armed gate WOULD pass (0.8 >= 0.7), but - # the run completed naturally (watcher never fired), so the strict + # the run completed naturally (monitor never fired), so the strict # full-set gate applies and the failing 0.2 criterion drags the run to # failure — proving the armed gate did not run. criteria = [ @@ -2530,7 +2532,7 @@ async def test_completed_naturally_weighted_armed_gate_does_not_run(self, tmp_pa ] result, agent, success = await _run_wiring( criteria=criteria, - events=[_agent_start(), _turn_start()], # no skill engagement -> watcher never fires + events=[_agent_start(), _turn_start()], # no skill engagement -> monitor never fires scores=[1.0, 0.0], tmp_path=tmp_path, gate_threshold=0.7, @@ -2541,16 +2543,16 @@ async def test_completed_naturally_weighted_armed_gate_does_not_run(self, tmp_pa assert success is False # the strict full-set gate is what decided this run async def test_completed_naturally_full_gate_applies_even_when_armed(self, tmp_path) -> None: - # FIRED-ONLY gating: an armed run whose watcher never fires (the agent + # FIRED-ONLY gating: an armed run whose monitor never fires (the agent # completed naturally) has a FULL trajectory, so the strict full-set # gate applies — the advisory 0.0 drags it to FAILURE exactly as it # would on an unarmed run. Arming a criterion (e.g. adding a # decide_within fail-fast timeout) must never change the verdict of a # run it didn't cut; the weighted armed gate is reserved for runs the - # watcher actually truncated. + # monitor actually truncated. result, agent, success = await _run_wiring( criteria=self._criteria(), - events=[_agent_start(), _turn_start()], # no skill engagement -> watcher never fires + events=[_agent_start(), _turn_start()], # no skill engagement -> monitor never fires scores=[1.0, 0.0], tmp_path=tmp_path, ) @@ -2563,7 +2565,7 @@ async def test_gate_threshold_plumbing_end_to_end(self, tmp_path) -> None: # Mutation-resistant pin for the plumbing hop: YAML # stop_early_gate_threshold -> the final gate (orchestrator.py) -> # _evaluation_loop's real return value. Weighted criteria (0.8/0.2); - # the positive engages its skill so the watcher FIRES a pass-stop + # the positive engages its skill so the monitor FIRES a pass-stop # (fired-only gating means the armed gate only ever applies to a fired # run), and the mocked frozen-trajectory scores fail the low-weight # distractor — so the threshold alone decides the verdict. @@ -2591,7 +2593,7 @@ async def test_gate_threshold_plumbing_end_to_end(self, tmp_path) -> None: assert success_low is True # 0.8 >= 0.7 — a mutation to a literal 1.0 would flip this async def test_gate_threshold_persisted_on_early_stop_info(self, tmp_path) -> None: - # The second plumbing hop: the fired watcher's own EarlyStopInfo + # The second plumbing hop: the fired monitor's own EarlyStopInfo # carries the threshold that was actually in effect. result, _agent, _success = await _run_wiring( criteria=self._criteria(), @@ -2648,22 +2650,22 @@ async def test_fail_open_wiring_degrades_to_full_run(self, tmp_path) -> None: class TestOrchestratorSetupActivation: - """The REAL ``Orchestrator._setup`` builds (or withholds) the watcher. + """The REAL ``Orchestrator._setup`` builds the turn monitor and arms (or withholds) its criteria. - The wiring tests above inject the watcher by hand; these drive ``_setup`` + The wiring tests above inject the monitor by hand; these drive ``_setup`` itself on its evaluate-only path (sandbox pre-set, so no agent/sandbox - creation is reached) to pin the activation seam: armed -> watcher built, - kill-switched -> watcher stays None. + creation is reached) to pin the activation seam: armed -> criteria armed, + kill-switched or execute mode -> monitor built but unarmed. """ - def _orchestrator(self, tmp_path: Path, *, stop_early: bool | None) -> Orchestrator: + def _orchestrator(self, tmp_path: Path, *, stop_early: bool | None, grade: bool = True) -> Orchestrator: task = _task( criteria=[_skill_crit("date-teller", "date-teller", stop_on_pass=True)], stop_early=stop_early, ) run_dir = tmp_path / "run" run_dir.mkdir(parents=True) - orch = Orchestrator(task=task, run_dir=run_dir, variant_id="default") + orch = Orchestrator(task=task, run_dir=run_dir, variant_id="default", grade=grade) orch.result = EvaluationResult( task_id=task.task_id, task_description=task.description, @@ -2680,18 +2682,32 @@ def _orchestrator(self, tmp_path: Path, *, stop_early: bool | None) -> Orchestra orch.sandbox = sandbox # evaluate-only: _setup skips agent/sandbox creation return orch - async def test_setup_builds_watcher_when_armed(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + async def test_setup_arms_monitor_when_armed(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "api_backend", ApiBackend.DIRECT) orch = self._orchestrator(tmp_path, stop_early=None) await orch._setup() - assert orch._early_stop_watcher is not None - assert len(orch._early_stop_watcher._armed) == 1 + assert orch._monitor is not None + assert orch._monitor.armed is True + assert len(orch._monitor._armed) == 1 - async def test_setup_kill_switch_leaves_watcher_none(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + async def test_setup_kill_switch_builds_unarmed_monitor( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setattr(settings, "api_backend", ApiBackend.DIRECT) orch = self._orchestrator(tmp_path, stop_early=False) await orch._setup() - assert orch._early_stop_watcher is None + assert orch._monitor is not None + assert orch._monitor.armed is False + + def test_build_monitor_execute_mode_leaves_criteria_unarmed( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + orch = self._orchestrator(tmp_path, stop_early=None, grade=False) + with caplog.at_level("INFO", logger="coder_eval.orchestrator"): + orch._build_monitor() + assert orch._monitor is not None + assert orch._monitor.armed is False + assert "execute mode" in caplog.text # --------------------------------------------------------------------------- # @@ -2706,7 +2722,7 @@ def _stopped_result( criteria_results: list[CriterionResult] | None = None, ) -> EvaluationResult: result = _result(criteria_results=criteria_results) - result.early_stop = _info(reason=reason, turns_remaining_at_stop=turns_remaining) + result.early_stop = _info(reason=reason, tool_calls_remaining_at_stop=turns_remaining) return result @@ -2732,14 +2748,14 @@ def test_task_dict_keys_present_when_early_stopped(self) -> None: d = eval_result_to_task_dict(_stopped_result()) assert d["stopped_early"] is True assert d["early_stop_reason"] == "criterion_passed" - assert d["turns_remaining_at_stop"] == 14 + assert d["tool_calls_remaining_at_stop"] == 14 assert d["gate_threshold"] == 1.0 def test_task_dict_keys_defaulted_when_not_early_stopped(self) -> None: d = eval_result_to_task_dict(_result()) assert d["stopped_early"] is False assert d["early_stop_reason"] is None - assert d["turns_remaining_at_stop"] is None + assert d["tool_calls_remaining_at_stop"] is None assert d["gate_threshold"] is None def test_task_dict_reflects_decision_budget_exceeded(self) -> None: @@ -2747,13 +2763,9 @@ def test_task_dict_reflects_decision_budget_exceeded(self) -> None: assert d["early_stop_reason"] == "decision_budget_exceeded" def test_runtime_note_omits_the_turns_avoided_claim(self) -> None: - """The note states the reason and the gate, and claims no turn saving. + """The note states the reason and the gate, and claims no saving. - It used to render ``<= N turn(s) avoided`` from ``max_turns - sdk_turn_index``. - On Codex and Antigravity one ``communicate()`` is a single SDK turn, so that - subtraction advertised the entire max_turns budget as saved when all that was - actually cut was a tool-call tail. ``turns_remaining_at_stop`` is still - persisted on ``EarlyStopInfo``, where its docstring calls it an upper bound. + ``tool_calls_remaining_at_stop`` is persisted on the row, not rendered. """ lines = ReportGenerator._runtime_notes_lines(_run_summary([eval_result_to_task_dict(_stopped_result())])) blob = "\n".join(lines) @@ -2761,7 +2773,7 @@ def test_runtime_note_omits_the_turns_avoided_claim(self) -> None: assert "gated on armed criteria only; other criteria are advisory" in blob assert "avoided" not in blob # Still recorded on the row for anyone who wants the bound. - assert eval_result_to_task_dict(_stopped_result())["turns_remaining_at_stop"] == 14 + assert eval_result_to_task_dict(_stopped_result())["tool_calls_remaining_at_stop"] == 14 def test_runtime_note_for_decision_budget_exceeded_names_the_timeout(self) -> None: # The budget-exceeded reason is an effective fail gated through the @@ -2903,20 +2915,20 @@ async def _run_codex_communicate( ) -> tuple[CodexAgent, TurnRecord, _EventSink, _FakeCodexStream, _FakeCodexTurnHandle]: """Drive ``CodexAgent.communicate`` over a fake notification stream. - ``stop_after``: should_stop returns True once that many notifications have - been pulled (checked after each dispatch). ``never``: an always-False - should_stop. Neither: ``should_stop=None``. + ``stop_after``: should_stop returns ``EARLY_CRITERION`` once that many + notifications have been pulled (checked after each dispatch). ``never``: an + always-None should_stop. Neither: ``should_stop=None``. """ agent = _codex_agent() stream = _FakeCodexStream(notifications) handle = _FakeCodexTurnHandle(stream) agent.thread = SimpleNamespace(turn=lambda _prompt: handle) - should_stop: Callable[[], bool] | None + should_stop: Callable[[], StopReason | None] | None if stop_after is not None: - should_stop = lambda: stream.iter.pulled >= stop_after # noqa: E731 + should_stop = lambda: StopReason.EARLY_CRITERION if stream.iter.pulled >= stop_after else None # noqa: E731 elif never: - should_stop = lambda: False # noqa: E731 + should_stop = lambda: None # noqa: E731 else: should_stop = None @@ -2951,7 +2963,7 @@ async def test_should_stop_none_consumes_full_stream(self) -> None: assert record.crashed is False assert _agent_end_events(sink)[0].status == AgentEndStatus.COMPLETED - async def test_should_stop_false_consumes_full_stream(self) -> None: + async def test_should_stop_returning_none_consumes_full_stream(self) -> None: notifications = [_codex_delta(0), _codex_delta(1), _codex_completed()] _agent, _record, sink, stream, _handle = await _run_codex_communicate(notifications=notifications, never=True) assert stream.iter.pulled == 3 @@ -2979,7 +2991,7 @@ async def test_stream_dying_without_stop_still_raises(self) -> None: async def test_timeout_beats_stop_precedence(self, monkeypatch: pytest.MonkeyPatch) -> None: # Both signals in one turn: the watchdog fires (timeout_hit) AND should_stop - # is True. The post-pump timeout check must win — TIMEOUT, crashed=True. + # returns a reason. The post-pump timeout check must win — TIMEOUT, crashed=True. class _FiringWatchdog: def __init__(self, *, on_timeout: Callable[[], None], **_kwargs: Any) -> None: self._on_timeout = on_timeout @@ -2997,7 +3009,9 @@ def __exit__(self, *_exc: Any) -> bool: agent.thread = SimpleNamespace(turn=lambda _prompt: _FakeCodexTurnHandle(stream)) sink = _EventSink() with pytest.raises(TurnTimeoutError): - await agent.communicate("prompt", stream_callback=sink, timeout=30.0, should_stop=lambda: True) + await agent.communicate( + "prompt", stream_callback=sink, timeout=30.0, should_stop=lambda: StopReason.EARLY_CRITERION + ) ends = _agent_end_events(sink) assert len(ends) == 1 assert ends[0].status == AgentEndStatus.TIMEOUT @@ -3010,7 +3024,7 @@ def __exit__(self, *_exc: Any) -> bool: async def test_post_stop_exception_stays_clean(self, monkeypatch: pytest.MonkeyPatch) -> None: # The retry-poisoning gap: an exception AFTER the cooperative break (here: # the pump's finally-side cleanup) must NOT crash-finalize the turn — a - # crash would trigger the orchestrator retry with the watcher's decision + # crash would trigger the orchestrator retry with the monitor's decision # still latched, stopping the retry at turn 0. def _boom(self: Any) -> None: raise RuntimeError("post-stop cleanup boom") @@ -3062,8 +3076,12 @@ def _capturing_init(self: Any, *args: Any, **kwargs: Any) -> None: patch.object(_CodexTurnState, "__init__", _capturing_init), patch.object(CodexAgent, "_recover_subagent_tool_calls", recover), ): - await agent.communicate("prompt", stream_callback=_EventSink(), should_stop=lambda: stream.iter.pulled >= 1) - assert captured["state"].stopped_early_hit is True + await agent.communicate( + "prompt", + stream_callback=_EventSink(), + should_stop=lambda: StopReason.EARLY_CRITERION if stream.iter.pulled >= 1 else None, + ) + assert captured["state"].stop_reason is StopReason.EARLY_CRITERION recover.assert_not_awaited() @@ -3133,11 +3151,11 @@ async def _run_antigravity_communicate( conversation = _CountingConversation([_ag_step(i) for i in range(n_steps)], cancel_raises=cancel_raises) agent = _antigravity_agent(conversation) - should_stop: Callable[[], bool] | None + should_stop: Callable[[], StopReason | None] | None if stop_after is not None: - should_stop = lambda: conversation.yielded >= stop_after # noqa: E731 + should_stop = lambda: StopReason.EARLY_CRITERION if conversation.yielded >= stop_after else None # noqa: E731 elif never: - should_stop = lambda: False # noqa: E731 + should_stop = lambda: None # noqa: E731 else: should_stop = None @@ -3168,7 +3186,7 @@ async def test_should_stop_none_consumes_full_stream(self) -> None: assert record.crashed is False assert _agent_end_events(sink)[0].status == AgentEndStatus.COMPLETED - async def test_should_stop_false_consumes_full_stream(self) -> None: + async def test_should_stop_returning_none_consumes_full_stream(self) -> None: _agent, _record, sink, conversation = await _run_antigravity_communicate(never=True, n_steps=3) assert conversation.yielded == 3 assert _agent_end_events(sink)[0].status == AgentEndStatus.COMPLETED @@ -3199,7 +3217,9 @@ def __exit__(self, *_exc: Any) -> bool: agent = _antigravity_agent(conversation) sink = _EventSink() with pytest.raises(TurnTimeoutError): - await agent.communicate("prompt", stream_callback=sink, timeout=30.0, should_stop=lambda: True) + await agent.communicate( + "prompt", stream_callback=sink, timeout=30.0, should_stop=lambda: StopReason.EARLY_CRITERION + ) ends = _agent_end_events(sink) assert len(ends) == 1 assert ends[0].status == AgentEndStatus.TIMEOUT @@ -3260,7 +3280,7 @@ def __exit__(self, exc_type: Any, *_exc: Any) -> bool: # --------------------------------------------------------------------------- # -# Orchestrator-level wiring on a non-Claude agent type: the watcher, gating and +# Orchestrator-level wiring on a non-Claude agent type: the monitor, gating and # report row are agent-agnostic — an armed codex task flows end to end. # --------------------------------------------------------------------------- # diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index 89d9cf872..b51609e45 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -747,7 +747,7 @@ def test_a_tool_running_before_the_first_window_is_not_counted_twice(self): assert rec.harness_startup_ms == pytest.approx(3000.0) def test_a_new_turn_clears_the_previous_turn_terminal_event(self): - """EarlyStopWatcher keeps ONE collector across retries. Left stale, the + """TurnMonitor keeps ONE collector across retries. Left stale, the next attempt's start pairs with the last attempt's end and the clamped inversion publishes as a measured 0.0.""" t0 = datetime(2026, 1, 1, 12, 0, 0) @@ -1132,7 +1132,7 @@ def test_the_span_set_is_computed_exactly_once(self): assert parameter.default is inspect.Parameter.empty def test_two_builds_agree(self): - """`EarlyStopWatcher` holds one collector across a turn's rounds.""" + """`TurnMonitor` holds one collector across a turn's rounds.""" collector = EventCollector() _feed( collector, @@ -1153,7 +1153,7 @@ def test_two_builds_agree(self): class TestBuildTurnRecordIsIdempotent: """Building the record twice must give the same numbers. - `EventCollector` is not built once and read once. `EarlyStopWatcher` holds + `EventCollector` is not built once and read once. `TurnMonitor` holds ONE across a turn's tool-call rounds and calls `build_turn_record()` on every one, and the crash path builds it again from `Agent._finalize`. The tool subtraction now happens inside that method, so a version of it that diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py index 933341ad2..27135bf0d 100644 --- a/tests/test_execute_evaluate_loop.py +++ b/tests/test_execute_evaluate_loop.py @@ -431,7 +431,7 @@ def test_execute_records_tool_calls_exhausted_exactly_as_run_does(tmp_path: Path """ from coder_eval.streaming.collector import EventCollector - # One turn that reports the cap was hit, on both paths. + # One turn that reports the cap ended it, on both paths. original = EventCollector.build_turn_record def _exhausted(self, *args: Any, **kwargs: Any): @@ -451,12 +451,12 @@ def _run(command: str, run_dir: Path) -> Any: _run("execute", executed_dir) executed = _row(_task_dir(executed_dir)) - assert graded["tool_calls_exhausted"] is True, "the fixture must actually exhaust turns under `run`" + assert graded["tool_calls_exhausted"] is True, "the fixture must actually exhaust tool calls under `run`" assert executed["tool_calls_exhausted"] is True, ( "`execute` dropped a fact about the run. Only the verdict is withheld." ) # The FACT is recorded; the STATUS is not decided. `run` returns SUCCESS for - # a max-turns trajectory whose criteria pass and only falls through to + # a tool-call-capped trajectory whose criteria pass and only falls through to # TOOL_CALLS_EXHAUSTED when they fail — so the status is not knowable without # grading, and claiming it here made it both terminal and permanent # (TOOL_CALLS_EXHAUSTED is an execution fact, which the detached grade may @@ -473,6 +473,25 @@ def _run(command: str, run_dir: Path) -> Any: assert regraded["tool_calls_exhausted"] is True, "the fact must survive the grade too" +def test_a_recorded_run_limits_max_turns_re_grades_from_the_source_yaml_loudly(tmp_path: Path) -> None: + """A run recorded before `run_limits.max_turns` became `max_tool_calls` no longer + validates. The grade must fall back to the source YAML and say so, not refuse.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + + row = _row(task_dir) + run_limits = row["task_config"]["resolved"]["run_limits"] + run_limits["max_turns"] = run_limits.pop("max_tool_calls") + (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") + + output = _invoke(["evaluate", str(task_dir)]).output + + assert "falling back to" in output + assert "NOT reapplied" in output + assert _row(task_dir)["final_status"] == FinalStatus.SUCCESS.value + + def test_a_detached_grade_keeps_the_runs_api_routing_not_the_graders(tmp_path: Path) -> None: """`_seed_from_prior_result`'s contract is that the PRIOR run wins on environment_info. The route recorder ran after the seeding and overwrote diff --git a/tests/test_experiment_models.py b/tests/test_experiment_models.py index b6707dd95..f202bb132 100644 --- a/tests/test_experiment_models.py +++ b/tests/test_experiment_models.py @@ -36,7 +36,7 @@ def test_variant_with_all_fields(self): variant = ExperimentVariant( variant_id="fast", agent={"model": "claude-sonnet-4-20250514"}, - run_limits=RunLimits(max_turns=5, task_timeout=120, turn_timeout=30), + run_limits=RunLimits(max_tool_calls=5, task_timeout=120, turn_timeout=30), ) assert variant.run_limits is not None assert variant.run_limits.task_timeout == 120 diff --git a/tests/test_harness_conformance.py b/tests/test_harness_conformance.py index b633c0e8a..a6b76d142 100644 --- a/tests/test_harness_conformance.py +++ b/tests/test_harness_conformance.py @@ -11,7 +11,7 @@ from __future__ import annotations import json -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Iterator from pathlib import Path from types import SimpleNamespace from typing import Any @@ -37,6 +37,7 @@ ) from coder_eval.orchestration.harness_contract import HarnessContractError, validate_harness_contract from coder_eval.plugins import ensure_plugins_loaded +from coder_eval.streaming.events import AgentEndEvent, StopReason, ToolEndEvent, end_status_for from tests.test_antigravity_agent import _install_fake_sdk @@ -414,3 +415,220 @@ def test_every_enforced_cell_has_exactly_one_probe() -> None: @pytest.mark.parametrize("cell", sorted(_PROBES)) async def test_probe(cell: tuple[str, str], tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: await _PROBES[cell](tmp_path, monkeypatch) + + +# --- cooperative stop: every StopReason ends the turn at the boundary --------------- + + +class _StopAfterFirstTool: + """``should_stop`` stub: ``reason`` once one tool call has resolved; also keeps the end event.""" + + def __init__(self, reason: StopReason) -> None: + self.reason = reason + self.tool_ends = 0 + self.end: AgentEndEvent | None = None + + def on_event(self, event: object) -> None: + if isinstance(event, ToolEndEvent): + self.tool_ends += 1 + elif isinstance(event, AgentEndEvent): + self.end = event + + def __call__(self) -> StopReason | None: + return self.reason if self.tool_ends else None + + +type StopProbe = Callable[[Path, pytest.MonkeyPatch, _StopAfterFirstTool], Awaitable[list[Any]]] +_SECOND = "second-call" + + +def _recording(items: list[Any], pulled: list[Any]) -> Iterator[Any]: + for item in items: + pulled.append(item) + yield item + + +async def _stop_claude(tmp_path: Path, _mp: pytest.MonkeyPatch, stop: _StopAfterFirstTool) -> list[Any]: + from tests._fixtures.golden_streams.claude_fixtures import ( + AssistantMessage, + ResultMessage, + ToolUseBlock, + UserMessage, + ) + + pulled: list[Any] = [] + events = [ + AssistantMessage([ToolUseBlock("first", "Bash", {"command": "ls"})], message_id="m1"), + UserMessage("first", False, "ok"), + AssistantMessage([ToolUseBlock(_SECOND, "Bash", {"command": "ls"})], message_id="m2"), + UserMessage(_SECOND, False, "ok"), + ResultMessage(), + ] + + async def fake_query(prompt: Any, options: Any, transport: Any = None): + for event in _recording(events, pulled): + yield event + + claude = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE)) + await claude.start(str(tmp_path)) + with patch("coder_eval.agents.claude_code_agent.query", fake_query): + await claude.communicate(USER_TURN, stream_callback=stop, should_stop=stop) + return [getattr(e.content[0], "id", None) for e in pulled if hasattr(e, "content")] + + +async def _stop_codex(_tmp: Path, _mp: pytest.MonkeyPatch, stop: _StopAfterFirstTool) -> list[Any]: + from tests.test_codex_agent import _FakeThread, _FakeTurnHandle, _item_notification, _started_agent + + pulled: list[Any] = [] + + def command(item_id: str) -> SimpleNamespace: + return SimpleNamespace( + type="commandExecution", id=item_id, command="ls", exit_code=0, aggregated_output="ok", duration_ms=1 + ) + + notifications = [ + _item_notification(method, command(item_id)) + for item_id in ("first", _SECOND) + for method in ("item/started", "item/completed") + ] + + class _RecordingHandle(_FakeTurnHandle): + def stream(self): # type: ignore[override] + return _recording(notifications, pulled) + + class _RecordingThread(_FakeThread): + def turn(self, _user_input: str): # type: ignore[override] + self.last_handle = _RecordingHandle(notifications) + return self.last_handle + + codex = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) + codex.thread = _RecordingThread(notifications) + await codex.communicate(USER_TURN, stream_callback=stop, should_stop=stop) + return [n.payload.item.root.id for n in pulled] + + +async def _stop_antigravity(tmp_path: Path, _mp: pytest.MonkeyPatch, stop: _StopAfterFirstTool) -> list[Any]: + from tests._fixtures.golden_streams.antigravity_fixtures import _FakeConversation, _step, _tc + + pulled: list[Any] = [] + + def call(tool_id: str) -> list[Any]: + args = {"command_line": "ls"} + return [ + _step("TOOL_CALL", "ACTIVE", target="TARGET_ENVIRONMENT", tool_calls=[_tc("run_command", tool_id, args)]), + _step( + "TOOL_CALL", + "DONE", + target="TARGET_ENVIRONMENT", + tool_calls=[_tc("run_command", tool_id, {**args, "exit_code": 0, "combined_output": "ok"})], + ), + ] + + steps = [*call("first"), *call(_SECOND)] + + class _RecordingConversation(_FakeConversation): + async def receive_steps(self): + self.receive_steps_call_count += 1 + for step in _recording(steps if self.receive_steps_call_count == 1 else [], pulled): + yield step + + agent = AntigravityAgent(parse_agent_config(type=AgentKind.ANTIGRAVITY)) + agent.working_directory = tmp_path + agent._sdk_agent = SimpleNamespace(conversation=_RecordingConversation([]), is_started=True) + await agent.communicate(USER_TURN, stream_callback=stop, should_stop=stop) + return [s.tool_calls[0].id for s in pulled] + + +async def _stop_cli( + cls: type, kind: AgentKind, lines: list[str], tmp_path: Path, monkeypatch: pytest.MonkeyPatch, stop: Any +) -> list[Any]: + from tests._fixtures.golden_streams.pi_fixtures import _FakeProcess + + pulled: list[Any] = [] + + class _RecordingProcess(_FakeProcess): + async def readline(self) -> bytes: + line = await super().readline() + if line: + pulled.append(json.loads(line)) + return line + + proc = _RecordingProcess(lines) + + async def fake_exec(*_argv: str, **_kwargs: Any) -> _RecordingProcess: + proc.stderr = proc # type: ignore[assignment] + return proc + + monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec) + monkeypatch.setattr("os.killpg", lambda _pgid, _sig: None, raising=False) + cli = await _cli_agent(cls, kind, tmp_path, monkeypatch) + try: + await cli.communicate(USER_TURN, stream_callback=stop, should_stop=stop) + finally: + await cli.stop() + return [tool_id for tool_id in ("first", _SECOND) if any(tool_id in json.dumps(p) for p in pulled)] + + +async def _stop_pi(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, stop: _StopAfterFirstTool) -> list[Any]: + from tests._fixtures.golden_streams.pi_fixtures import _tool_end, _tool_start, _turn_end, _turn_start + + lines = [_turn_start()] + for tool_id in ("first", _SECOND): + lines += [_tool_start(tool_id, "bash", {"command": "ls"}), _tool_end(tool_id, "bash", "ok")] + lines.append(_turn_end(inp=1, out=1)) + return await _stop_cli(PiAgent, AgentKind.PI, lines, tmp_path, monkeypatch, stop) + + +async def _stop_opencode(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, stop: _StopAfterFirstTool) -> list[Any]: + from tests._fixtures.golden_streams.opencode_fixtures import _evt + + def tool_use(tool_id: str) -> str: + state = {"status": "completed", "input": {"command": "ls"}, "output": "ok"} + return _evt( + "tool_use", + { + "id": f"prt_{tool_id}", + "messageID": "msg_1", + "type": "tool", + "tool": "bash", + "callID": tool_id, + "state": state, + }, + ) + + monkeypatch.delenv("OPENCODE_CONFIG_CONTENT", raising=False) + lines = [ + _evt("step_start", {"id": "prt_0", "messageID": "msg_1", "type": "step-start"}), + tool_use("first"), + tool_use(_SECOND), + ] + return await _stop_cli(OpenCodeAgent, AgentKind.OPENCODE, lines, tmp_path, monkeypatch, stop) + + +_STOP_PROBES: dict[str, StopProbe] = { + "claude-code": _stop_claude, + "codex": _stop_codex, + "antigravity": _stop_antigravity, + "pi": _stop_pi, + "opencode": _stop_opencode, +} + + +def test_every_cooperative_kind_has_a_stop_probe() -> None: + assert set(_STOP_PROBES) == {k.value for k in _KINDS if _contract(k).cooperative_stop} + + +@pytest.mark.parametrize("reason", list(StopReason)) +@pytest.mark.parametrize("kind", sorted(_STOP_PROBES)) +async def test_stop_reason_ends_the_turn_before_the_next_call( + kind: str, reason: StopReason, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + stop = _StopAfterFirstTool(reason) + + pulled = await _STOP_PROBES[kind](tmp_path, monkeypatch, stop) + + assert stop.end is not None + assert stop.end.status is end_status_for(reason) + assert stop.end.crashed is False + assert "first" in pulled + assert _SECOND not in pulled diff --git a/tests/test_lint_no_top_level_run_limits.py b/tests/test_lint_no_top_level_run_limits.py index c3d37a354..3121850e7 100644 --- a/tests/test_lint_no_top_level_run_limits.py +++ b/tests/test_lint_no_top_level_run_limits.py @@ -25,6 +25,10 @@ def test_flags_task_max_turns() -> None: assert _violations("x = task.max_turns") +def test_flags_task_max_tool_calls() -> None: + assert _violations("x = task.max_tool_calls") + + def test_flags_self_task_max_turns() -> None: assert _violations("class C:\n def m(self): x = self.task.max_turns") diff --git a/tests/test_merge_characterization.py b/tests/test_merge_characterization.py index bdb27b252..f97fdc4a3 100644 --- a/tests/test_merge_characterization.py +++ b/tests/test_merge_characterization.py @@ -142,13 +142,13 @@ def test_run_limits_field_merge_keeps_unset_lower_keys(self): task = _make_task(agent={"type": "claude-code"}, run_limits=RunLimits(task_timeout=300)) experiment = ExperimentDefinition( experiment_id="test", - variants=[ExperimentVariant(variant_id="v", run_limits=RunLimits(max_turns=5))], + variants=[ExperimentVariant(variant_id="v", run_limits=RunLimits(max_tool_calls=5))], ) resolved, _lineage, _ = resolve_task_for_variant(default_exp, task, experiment, experiment.variants[0]) assert resolved.run_limits is not None assert resolved.run_limits.turn_timeout == 60 # from default experiment assert resolved.run_limits.task_timeout == 300 # from task - assert resolved.run_limits.max_turns == 5 # from variant + assert resolved.run_limits.max_tool_calls == 5 # from variant def test_env_passthrough_extra_appends(self): default_exp = _default_exp() @@ -273,8 +273,8 @@ def test_sdk_options_recursive_merge(self): def test_run_limits_sibling_keys_preserved(self): task = _live_task(run_limits=RunLimits(task_timeout=600)) - apply_overrides(task, {"run_limits.max_turns": 30}) - assert task.run_limits.max_turns == 30 + apply_overrides(task, {"run_limits.max_tool_calls": 30}) + assert task.run_limits.max_tool_calls == 30 assert task.run_limits.task_timeout == 600 def test_list_field_replaces(self): @@ -316,10 +316,10 @@ def test_lineage_cli_source_for_touched_paths_only(self): task = _live_task() lineage: dict[str, ConfigLineageEntry] = { "agent.model": ConfigLineageEntry(value="yaml-model", source="task"), - "run_limits.max_turns": ConfigLineageEntry(value=10, source="variant"), + "run_limits.max_tool_calls": ConfigLineageEntry(value=10, source="variant"), } - apply_overrides(task, {"run_limits.max_turns": 20}, lineage=lineage) + apply_overrides(task, {"run_limits.max_tool_calls": 20}, lineage=lineage) # touched path relabeled to cli; untouched task entry preserved. - assert lineage["run_limits.max_turns"].source == "cli" - assert lineage["run_limits.max_turns"].source_detail == "-D run_limits.max_turns" + assert lineage["run_limits.max_tool_calls"].source == "cli" + assert lineage["run_limits.max_tool_calls"].source_detail == "-D run_limits.max_tool_calls" assert lineage["agent.model"].source == "task" diff --git a/tests/test_merge_unification.py b/tests/test_merge_unification.py index a19e39c15..55b267f64 100644 --- a/tests/test_merge_unification.py +++ b/tests/test_merge_unification.py @@ -81,10 +81,10 @@ def test_agent_system_prompt_file_clears_sibling(self): assert a.agent.system_prompt is None assert a.agent.system_prompt_file == "p.txt" - def test_run_limits_max_turns(self): - a, _ = _resolve(_task(), variant=ExperimentVariant(variant_id="v", run_limits=RunLimits(max_turns=5))) + def test_run_limits_max_tool_calls(self): + a, _ = _resolve(_task(), variant=ExperimentVariant(variant_id="v", run_limits=RunLimits(max_tool_calls=5))) b, _ = _resolve(_task()) - apply_overrides(b, {"run_limits.max_turns": 5}) + apply_overrides(b, {"run_limits.max_tool_calls": 5}) assert a.run_limits.model_dump() == b.run_limits.model_dump() def test_sandbox_driver(self): diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 75e1e173d..3720b2169 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -40,6 +40,7 @@ AgentEndEvent, AgentEndStatus, AgentStartEvent, + StopReason, ToolEndEvent, ToolEndStatus, ToolStartEvent, @@ -1219,7 +1220,7 @@ async def test_intentional_cuts_are_exempt(self, patch_exec, tmp_path): stream = [json.dumps({"id": "evt_1", "type": "session.next.idle", "properties": {"sessionID": SESSION}})] proc = _RunningProcess(stream) patch_exec(proc) - record = await _run(_agent(), tmp_path, should_stop=lambda: True) + record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION) assert record.crashed is False @staticmethod @@ -1278,7 +1279,7 @@ async def test_a_cut_before_any_step_finished_is_exempt(self, patch_exec, tmp_pa a step's start and its `step_finish` is an intentional cut, not drift.""" proc = _RunningProcess([_evt("step_start", {"id": "prt_1", "messageID": "msg_1", "type": "step-start"})]) patch_exec(proc) - record = await _run(_agent(), tmp_path, should_stop=lambda: True) + record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION) assert record.crashed is False async def test_real_tokens_are_never_condemned(self, patch_exec, tmp_path): @@ -1450,63 +1451,84 @@ async def test_turn_completes_under_stderr_backpressure(self, patch_exec, tmp_pa assert record.crashed is False +def _stop_after(calls: int, reason: StopReason): + """A ``should_stop`` that returns ``reason`` from its ``calls``-th check on (one check per dispatched line).""" + seen = 0 + + def should_stop() -> StopReason | None: + nonlocal seen + seen += 1 + return reason if seen >= calls else None + + return should_stop + + class TestCooperativeStop: def test_capability_flag_is_declared(self): assert OpenCodeAgent.contract.cooperative_stop is True - async def test_should_stop_ends_turn_cleanly(self, patch_exec, tmp_path): + async def test_early_criterion_ends_turn_stopped_early(self, patch_exec, tmp_path): """A live subprocess must be torn down, and the turn must not be a crash.""" proc = _RunningProcess(HAPPY_STREAM) patch_exec(proc) - record = await _run(_agent(), tmp_path, should_stop=lambda: True) + recorder = _EventRecorder() + record = await _run( + _agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION, stream_callback=recorder + ) assert record.crashed is False assert proc.terminated is True # Stopped at the first event boundary rather than draining the stream. assert record.assistant_turn_count < 2 + ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] + assert [e.status for e in ends] == [AgentEndStatus.STOPPED_EARLY] - async def test_max_turns_marks_exhausted(self, patch_exec, tmp_path): - patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path, max_turns=1) - assert record.tool_calls_exhausted is True + async def test_tool_call_cap_ends_turn_tool_calls_exhausted(self, patch_exec, tmp_path): + proc = _RunningProcess(HAPPY_STREAM) + patch_exec(proc) + recorder = _EventRecorder() + record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOOL_CALL_CAP, stream_callback=recorder) - async def test_a_cap_the_run_stays_under_is_not_exhausted(self, patch_exec, tmp_path): - """The OTHER direction, which decides `FinalStatus`. + assert proc.terminated is True + assert record.crashed is False + assert record.tool_calls_exhausted is True + ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] + assert [e.status for e in ends] == [AgentEndStatus.TOOL_CALLS_EXHAUSTED] - HAPPY_STREAM is exactly 2 steps, so `max_turns=2` is the boundary: an - off-by-one here (`>` becoming `>=`, or counting finished steps instead of - started ones) reports TOOL_CALLS_EXHAUSTED — orchestrator.py turns the flag - straight into `FinalStatus.TOOL_CALLS_EXHAUSTED` — for a run that finished - well inside its budget. A spurious exhaustion also suppresses the non-zero- - exit and zero-telemetry crash guards, which are both conditioned on it, so - the run would score silently instead of failing loudly. - """ - patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path, max_turns=2) + async def test_token_budget_ends_turn_token_budget_exceeded(self, patch_exec, tmp_path): + proc = _RunningProcess(HAPPY_STREAM) + patch_exec(proc) + recorder = _EventRecorder() + record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOKEN_BUDGET, stream_callback=recorder) + assert proc.terminated is True + assert record.crashed is False assert record.tool_calls_exhausted is False - assert record.assistant_turn_count == 2 - # Both steps' telemetry is present — the cap did not truncate the stream. - assert record.token_usage is not None - assert record.token_usage.output_tokens == 57 + ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] + assert [e.status for e in ends] == [AgentEndStatus.TOKEN_BUDGET_EXCEEDED] - async def test_no_cap_is_uncapped(self, patch_exec, tmp_path): + async def test_no_stop_is_uncapped(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + record = await _run(_agent(), tmp_path, should_stop=lambda: None) assert record.tool_calls_exhausted is False assert record.assistant_turn_count == 2 async def test_the_deciding_step_is_kept_whole(self, patch_exec, tmp_path): - """`max_turns=1` cuts at the START of step 2, so step 1 survives complete. + """A stop after step 1's `step_finish` keeps step 1 complete and never opens step 2. Asserting only the flag would let a cut that discards the step that earned - the budget pass — the run would report exhaustion with none of the + the stop pass — the run would report exhaustion with none of the telemetry that reached it. """ - patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path, max_turns=1) + patch_exec(_RunningProcess(HAPPY_STREAM)) + recorder = _EventRecorder() + record = await _run( + _agent(), tmp_path, should_stop=_stop_after(3, StopReason.TOOL_CALL_CAP), stream_callback=recorder + ) assert record.tool_calls_exhausted is True + assert record.assistant_turn_count == 1 + assert len([e for e in recorder.events if isinstance(e, TurnStartEvent)]) == 1 assert len(record.commands) == 1 # step 1's tool call usage = record.token_usage assert usage is not None @@ -1516,16 +1538,12 @@ async def test_the_deciding_step_is_kept_whole(self, patch_exec, tmp_path): assert usage.cache_creation_input_tokens == 5 assert usage.cache_read_input_tokens == 10 - async def test_the_step_past_the_cap_is_never_admitted(self, patch_exec, tmp_path): - """The cap stops at the (N+1)th `step_start`, before it is counted or emitted.""" - patch_exec(_FakeProcess(HAPPY_STREAM)) - recorder = _EventRecorder() - record = await _run(_agent(), tmp_path, max_turns=1, stream_callback=recorder) - + async def test_an_intentional_stop_is_exempt_from_a_non_zero_exit(self, patch_exec, tmp_path): + """Killing the CLI makes it exit non-zero; that must not crash an intentional stop.""" + patch_exec(_RunningProcess(HAPPY_STREAM, returncode=-15, stderr=b"terminated")) + record = await _run(_agent(), tmp_path, should_stop=_stop_after(3, StopReason.TOOL_CALL_CAP)) + assert record.crashed is False assert record.tool_calls_exhausted is True - assert record.assistant_turn_count == 1 - assert len([e for e in recorder.events if isinstance(e, TurnStartEvent)]) == 1 - assert len([e for e in recorder.events if isinstance(e, TurnEndEvent)]) == 1 class _HangingProcess(_FakeProcess): @@ -1730,12 +1748,12 @@ async def test_a_crash_closes_the_open_step(self, patch_exec, tmp_path): assert self._pairs(recorder) == (1, 1) async def test_a_clean_cut_closes_the_open_step(self, patch_exec, tmp_path): - """should_stop and max_turns cut between a step's start and its finish too.""" + """A should_stop cut between a step's start and its finish closes the step too.""" proc = _RunningProcess([_evt("step_start", {"id": "prt_1", "messageID": "msg_1"})]) patch_exec(proc) recorder = _EventRecorder() - await _run(_agent(), tmp_path, should_stop=lambda: True, stream_callback=recorder) + await _run(_agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION, stream_callback=recorder) assert self._pairs(recorder) == (1, 1) end = next(e for e in recorder.events if isinstance(e, TurnEndEvent)) @@ -1849,7 +1867,7 @@ async def test_a_crashed_turn_sweeps_the_group_too(self, patch_exec, tmp_path): async def test_cooperative_stop_sweeps_the_group_too(self, patch_exec, tmp_path): captured = patch_exec(_RunningProcess(HAPPY_STREAM)) - await _run(_agent(), tmp_path, should_stop=lambda: True) + await _run(_agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION) assert (4242, signal.SIGKILL) in captured["killpg"] async def test_kill_sync_signals_pid_and_group(self, patch_exec, monkeypatch, tmp_path): diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index fe823be68..d4600f88b 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -1715,11 +1715,11 @@ def test_batch_run_config_accepts_overrides(): overrides={ "agent.model": "claude-sonnet-4-20250514", "agent.permission_mode": "bypassPermissions", - "run_limits.max_turns": 50, + "run_limits.max_tool_calls": 50, }, ) assert config.overrides["agent.model"] == "claude-sonnet-4-20250514" - assert config.overrides["run_limits.max_turns"] == 50 + assert config.overrides["run_limits.max_tool_calls"] == 50 def test_batch_run_config_overrides_default_empty(): @@ -1757,24 +1757,33 @@ async def test_overrides_apply_permission_mode(tmp_path): @pytest.mark.asyncio -async def test_overrides_apply_max_turns_field_merge(tmp_path): - """run_limits.max_turns override field-merges, preserving other run_limits keys.""" +async def test_overrides_apply_max_tool_calls_field_merge(tmp_path): + """run_limits.max_tool_calls override field-merges, preserving other run_limits keys.""" from coder_eval.orchestration.overrides import apply_overrides task, _ = load_task(Path("tasks/hello_date.yaml")) - # hello_date.yaml ships a baseline run_limits.expected_tool_calls; max_turns is + # hello_date.yaml ships a baseline run_limits.expected_tool_calls; max_tool_calls is # the field this test exercises. The override must field-merge on top. baseline_expected_tool_calls = task.run_limits.expected_tool_calls if task.run_limits else None - assert task.run_limits is None or task.run_limits.max_turns is None + assert task.run_limits is None or task.run_limits.max_tool_calls is None - apply_overrides(task, {"run_limits.max_turns": 42}) + apply_overrides(task, {"run_limits.max_tool_calls": 42}) assert task.run_limits is not None - assert task.run_limits.max_turns == 42 + assert task.run_limits.max_tool_calls == 42 # Field-merge must preserve other run_limits keys from the task YAML. assert task.run_limits.expected_tool_calls == baseline_expected_tool_calls +def test_overrides_reject_removed_run_limits_max_turns(): + """run_limits.max_turns no longer exists; the schema-validated override rejects it.""" + from coder_eval.orchestration.overrides import OverrideError, apply_overrides + + task, _ = load_task(Path("tasks/hello_date.yaml")) + with pytest.raises(OverrideError, match="max_turns"): + apply_overrides(task, {"run_limits.max_turns": 42}) + + # ==================== Duplicate Task ID Validation Tests ==================== @@ -1825,50 +1834,46 @@ def test_resolve_all_tasks_rejects_duplicate_task_ids(tmp_path): ) -# --- Evaluation loop: max_turns exhaustion early-break test --- +# --- Evaluation loop: tool-call cap via the TurnMonitor --- -@pytest.mark.asyncio -async def test_evaluation_loop_breaks_on_tool_calls_exhausted(tmp_path): - """Orchestrator stops iterating when the agent exhausts max_turns without passing criteria.""" - from datetime import datetime - from unittest.mock import AsyncMock, MagicMock, patch - - from coder_eval.models import ( - CriterionResult, - EvaluationResult, - SandboxConfig, - TurnRecord, - ) +def _cap_task(task_id: str, max_tool_calls: int) -> TaskDefinition: + from coder_eval.models import RunLimits agent_cfg = ClaudeCodeAgentConfig.model_construct( type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits", allowed_tools=None, model=None, - max_turns=20, turn_timeout=None, ignore_patterns=[], ) - task = TaskDefinition.model_construct( - task_id="exhaustion_test", - description="Test exhaustion", + return TaskDefinition.model_construct( + task_id=task_id, + description="tool-call cap", initial_prompt="Do something", tags=[], agent=agent_cfg, sandbox=SandboxConfig(driver="tempdir"), success_criteria=[FileExistsCriterion(type="file_exists", path="test.py", description="test.py must exist")], + run_limits=RunLimits(max_tool_calls=max_tool_calls), task_timeout=None, reference=None, ) - run_dir = tmp_path / "run" / "exhaustion_test" - run_dir.mkdir(parents=True) +def _cap_orchestrator(task: TaskDefinition, tmp_path: Path, *, score: float = 0.0) -> Orchestrator: + from datetime import datetime + from unittest.mock import AsyncMock, MagicMock + + from coder_eval.models import CriterionResult, EvaluationResult + + run_dir = tmp_path / "run" / task.task_id + run_dir.mkdir(parents=True) orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant") orchestrator.result = EvaluationResult( - task_id="exhaustion_test", - task_description="Test", + task_id=task.task_id, + task_description=task.description, variant_id="test-variant", agent_type=AgentKind.CLAUDE_CODE, started_at=datetime.now(), @@ -1876,42 +1881,186 @@ async def test_evaluation_loop_breaks_on_tool_calls_exhausted(tmp_path): iteration_count=0, environment_info={}, ) - - # Agent returns a turn record with tool_calls_exhausted=True - exhausted_turn = TurnRecord( - iteration=1, - user_input="test prompt", - agent_output="I ran out of turns", - duration_seconds=5.0, - tool_calls_exhausted=True, - ) - mock_agent = AsyncMock() - mock_agent.communicate = AsyncMock(return_value=exhausted_turn) - orchestrator.agent = mock_agent - - # Mock sandbox mock_sandbox = MagicMock() mock_sandbox.sandbox_dir = tmp_path / "sandbox" mock_sandbox.sandbox_dir.mkdir() orchestrator.sandbox = mock_sandbox - - # Mock success checker that always fails mock_checker = MagicMock() mock_checker.check_all_async = AsyncMock( - return_value=[CriterionResult(criterion_type="file_exists", description="test", score=0.0)] + return_value=[CriterionResult(criterion_type="file_exists", description="test", score=score)] ) orchestrator.success_checker = mock_checker + orchestrator._build_monitor() + return orchestrator + + +class _CooperativeToolAgent: + """Fake agent that emits resolved tool calls and polls ``should_stop`` at each boundary. + + ``plan`` holds one entry per ``communicate`` attempt: the number of tool calls the + attempt intends to make, and whether it then crashes with a partial turn. ``host`` + is the ``AsyncMock`` the orchestrator talks to; its ``communicate`` is this fake's. + """ + + def __init__(self, plan: list[tuple[int, bool]]) -> None: + from unittest.mock import AsyncMock + + self._plan = plan + self.attempt = 0 + self.emitted_per_attempt: list[int] = [] + self.should_stop_callables: list[object] = [] + self._tool_seq = 0 + self.host = AsyncMock() + self.host.communicate = self.communicate + self.host.pending_turn = None + + async def communicate(self, user_input, *, stream_callback=None, timeout=None, should_stop=None): + from datetime import datetime + + from coder_eval.errors import AgentCrashError + from coder_eval.models import CommandTelemetry, TurnRecord + from coder_eval.streaming.events import ( + AgentEndEvent, + AgentStartEvent, + StopReason, + ToolEndEvent, + ToolStartEvent, + end_status_for, + ) + + assert stream_callback is not None + assert should_stop is not None + intended, crash = self._plan[self.attempt] + self.attempt += 1 + self.should_stop_callables.append(should_stop) + stream_callback.on_event(AgentStartEvent(task_id="t", prompt=user_input, iteration=1)) + + commands: list[CommandTelemetry] = [] + reason: StopReason | None = should_stop() + while reason is None and len(commands) < intended: + self._tool_seq += 1 + tool = CommandTelemetry(tool_name="Bash", tool_id=f"tool-{self._tool_seq}", timestamp=datetime.now()) + stream_callback.on_event(ToolStartEvent(task_id="t", tool=tool)) + stream_callback.on_event(ToolEndEvent(task_id="t", tool=tool)) + commands.append(tool) + reason = should_stop() + self.emitted_per_attempt.append(len(commands)) + + if crash: + self.host.pending_turn = TurnRecord( + iteration=1, user_input=user_input, agent_output="", commands=commands, crashed=True + ) + raise AgentCrashError("mid-turn failure") + + status = end_status_for(reason) if reason is not None else None + if status is not None: + stream_callback.on_event(AgentEndEvent(task_id="t", status=status, iteration=1, user_input=user_input)) + return TurnRecord( + iteration=1, + user_input=user_input, + agent_output="stopped", + commands=commands, + tool_calls_exhausted=reason is StopReason.TOOL_CALL_CAP, + ) + + +@pytest.mark.asyncio +async def test_evaluation_loop_breaks_on_tool_call_cap(tmp_path): + """A cooperative agent stops at the cap through ``should_stop``; the result reads the monitor's latch.""" + from unittest.mock import patch + + from coder_eval.streaming.events import StopReason + + orchestrator = _cap_orchestrator(_cap_task("tool_call_cap_test", max_tool_calls=3), tmp_path) + agent = _CooperativeToolAgent([(10, False)]) + orchestrator.agent = agent.host with patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None): success = await orchestrator._evaluation_loop() - # Should NOT succeed assert success is False - # Should have stopped after 1 iteration (not all 5) + assert agent.attempt == 1 + assert agent.emitted_per_attempt == [3] + assert orchestrator._monitor is not None + assert orchestrator._monitor.stop_reason is StopReason.TOOL_CALL_CAP + assert orchestrator._monitor.tool_calls == 3 assert orchestrator.result.iteration_count == 1 - # Agent communicate should have been called only once - assert mock_agent.communicate.call_count == 1 - # tool_calls_exhausted should be propagated to the result + assert orchestrator.result.tool_calls_exhausted is True + + +@pytest.mark.asyncio +async def test_a_latched_cap_the_agent_did_not_stop_on_is_not_labelled_exhausted(tmp_path): + """The label follows the adapter's end status: a cap latched after the agent's last poll is not a capped run.""" + from datetime import datetime + from unittest.mock import AsyncMock, patch + + from coder_eval.models import CommandTelemetry, TurnRecord + from coder_eval.streaming.events import StopReason, ToolEndEvent + + orchestrator = _cap_orchestrator(_cap_task("late_latch_test", max_tool_calls=1), tmp_path) + + async def _communicate(user_input, *, stream_callback=None, timeout=None, should_stop=None): + tool = CommandTelemetry(tool_name="Bash", tool_id="late", timestamp=datetime.now()) + stream_callback.on_event(ToolEndEvent(task_id="t", tool=tool)) + return TurnRecord(iteration=1, user_input=user_input, agent_output="done", commands=[tool]) + + mock_agent = AsyncMock() + mock_agent.communicate = _communicate + orchestrator.agent = mock_agent + + with patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None): + await orchestrator._evaluation_loop() + + assert orchestrator._monitor is not None + assert orchestrator._monitor.stop_reason is StopReason.TOOL_CALL_CAP + assert orchestrator.result.tool_calls_exhausted is False + + +@pytest.mark.asyncio +async def test_tool_call_cap_latched_in_crashed_attempt_stops_the_retry_at_first_poll(tmp_path): + """The monitor keeps a crashed attempt's resolved calls, so the cap is cumulative across retries.""" + from unittest.mock import AsyncMock, patch + + from coder_eval.streaming.events import StopReason + + orchestrator = _cap_orchestrator(_cap_task("cap_retry_test", max_tool_calls=3), tmp_path, score=0.0) + agent = _CooperativeToolAgent([(3, True), (5, False)]) + orchestrator.agent = agent.host + + with ( + patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None), + patch("asyncio.sleep", new_callable=AsyncMock), + ): + success = await orchestrator._evaluation_loop() + + assert success is False + assert agent.attempt == 2 + assert agent.emitted_per_attempt == [3, 0] + first, second = agent.should_stop_callables + assert first.__self__ is second.__self__ is orchestrator._monitor # type: ignore[attr-defined] + assert orchestrator._monitor is not None + assert orchestrator._monitor.stop_reason is StopReason.TOOL_CALL_CAP + assert orchestrator._monitor.tool_calls == 3 + assert [t.crashed for t in orchestrator.result.iterations] == [True, False] + assert orchestrator.result.tool_calls_exhausted is True + + +@pytest.mark.asyncio +async def test_tool_call_cap_counts_a_crashed_attempts_calls_toward_the_retry(tmp_path): + """A crashed attempt under the cap leaves only the remainder of the cap for the retry.""" + from unittest.mock import AsyncMock, patch + + orchestrator = _cap_orchestrator(_cap_task("cap_retry_sum_test", max_tool_calls=3), tmp_path) + agent = _CooperativeToolAgent([(2, True), (5, False)]) + orchestrator.agent = agent.host + + with ( + patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None), + patch("asyncio.sleep", new_callable=AsyncMock), + ): + await orchestrator._evaluation_loop() + + assert agent.emitted_per_attempt == [2, 1] assert orchestrator.result.tool_calls_exhausted is True @@ -1941,7 +2090,6 @@ async def test_evaluation_loop_preserves_partial_on_crash_retry(tmp_path): permission_mode="acceptEdits", allowed_tools=None, model=None, - max_turns=20, turn_timeout=None, ignore_patterns=[], ) @@ -1961,6 +2109,7 @@ async def test_evaluation_loop_preserves_partial_on_crash_retry(tmp_path): run_dir.mkdir(parents=True) orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant") + orchestrator._build_monitor() orchestrator.result = EvaluationResult( task_id=task.task_id, task_description=task.description, @@ -2063,7 +2212,6 @@ async def test_evaluation_loop_stamps_timeout_reason_on_partial(tmp_path): permission_mode="acceptEdits", allowed_tools=None, model=None, - max_turns=20, turn_timeout=None, ignore_patterns=[], ) @@ -2083,6 +2231,7 @@ async def test_evaluation_loop_stamps_timeout_reason_on_partial(tmp_path): run_dir.mkdir(parents=True) orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="v") + orchestrator._build_monitor() orchestrator.result = EvaluationResult( task_id=task.task_id, task_description=task.description, @@ -2161,7 +2310,6 @@ def test_aggregate_token_usage_includes_crashed_partials(tmp_path): permission_mode="acceptEdits", allowed_tools=None, model=None, - max_turns=20, turn_timeout=None, ignore_patterns=[], ) @@ -2367,7 +2515,6 @@ async def test_evaluation_loop_evaluate_only_loads_reference(tmp_path): permission_mode="acceptEdits", allowed_tools=None, model=None, - max_turns=1, turn_timeout=None, ignore_patterns=[], ) diff --git a/tests/test_overrides_engine.py b/tests/test_overrides_engine.py index 5ba6fe344..f33a6e885 100644 --- a/tests/test_overrides_engine.py +++ b/tests/test_overrides_engine.py @@ -100,15 +100,15 @@ def test_sdk_options_merge_preserves_existing(self): def test_run_limits_field_merge_keeps_other_keys(self): task = _make_task(run_limits=RunLimits(task_timeout=600)) - apply_overrides(task, {"run_limits.max_turns": 30}) - assert task.run_limits.max_turns == 30 + apply_overrides(task, {"run_limits.max_tool_calls": 30}) + assert task.run_limits.max_tool_calls == 30 assert task.run_limits.task_timeout == 600 def test_run_limits_none_base(self): task = _make_task(run_limits=None) - apply_overrides(task, {"run_limits.max_turns": 5}) + apply_overrides(task, {"run_limits.max_tool_calls": 5}) assert task.run_limits is not None - assert task.run_limits.max_turns == 5 + assert task.run_limits.max_tool_calls == 5 def test_sandbox_driver(self): task = _make_task() @@ -208,15 +208,15 @@ def test_agent_type_lineage_uses_dash_d_when_explicit(self): def test_preserves_layers_1_4_lineage_for_untouched_fields(self): """Fix #1 guard: layer 5 must leave the layers-1-4 lineage for fields it doesn't touch untouched, adding only a cli entry for the field it sets.""" - task = _make_task(run_limits=RunLimits(max_turns=10)) + task = _make_task(run_limits=RunLimits(max_tool_calls=10)) lineage: dict[str, ConfigLineageEntry] = { "agent.model": ConfigLineageEntry(value="haiku", source="variant"), - "run_limits.max_turns": ConfigLineageEntry(value=10, source="task"), + "run_limits.max_tool_calls": ConfigLineageEntry(value=10, source="task"), } apply_overrides(task, {"run_limits.turn_timeout": 45}, lineage=lineage) # untouched entries unchanged assert lineage["agent.model"].source == "variant" - assert lineage["run_limits.max_turns"].source == "task" + assert lineage["run_limits.max_tool_calls"].source == "task" # only the touched field gained a cli entry assert lineage["run_limits.turn_timeout"].source == "cli" assert lineage["run_limits.turn_timeout"].source_detail == "-D run_limits.turn_timeout" diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 052fa0fd1..da8715f6e 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -32,6 +32,7 @@ AgentEndEvent, AgentEndStatus, AgentStartEvent, + StopReason, ToolEndEvent, ToolEndStatus, ToolStartEvent, @@ -437,46 +438,69 @@ async def test_two_agent_cycles_reduce_to_one_agent_end(self, patch_exec, tmp_pa assert record.assistant_turn_count == 2 +def _stop_after(calls: int, reason: StopReason): + """A ``should_stop`` that returns ``reason`` from its ``calls``-th check on (one check per dispatched line).""" + seen = 0 + + def should_stop() -> StopReason | None: + nonlocal seen + seen += 1 + return reason if seen >= calls else None + + return should_stop + + class TestCooperativeStop: def test_capability_flag_is_declared(self): assert PiAgent.contract.cooperative_stop is True - async def test_should_stop_ends_turn_cleanly(self, patch_exec, tmp_path): + async def test_early_criterion_ends_turn_stopped_early(self, patch_exec, tmp_path): proc = _RunningProcess(HAPPY_STREAM) patch_exec(proc) - record = await _run(_agent(), tmp_path, should_stop=lambda: True) + recorder = _EventRecorder() + record = await _run( + _agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION, stream_callback=recorder + ) assert record.crashed is False assert proc.terminated is True - # should_stop() is True from the first check, which lands after the first - # streamed line (the `session` header) and before any turn completes — so - # the cut is at turn 0, not merely "fewer than the full 3". + # The first check lands after the first streamed line (the `session` + # header), before any turn completes. assert record.assistant_turn_count == 0 + ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] + assert [e.status for e in ends] == [AgentEndStatus.STOPPED_EARLY] - async def test_partial_record_is_returned_not_raised(self, patch_exec, tmp_path): + async def test_tool_call_cap_ends_turn_tool_calls_exhausted(self, patch_exec, tmp_path): proc = _RunningProcess(HAPPY_STREAM) patch_exec(proc) - record = await _run(_agent(), tmp_path, should_stop=lambda: True) - assert isinstance(record.assistant_turn_count, int) - + recorder = _EventRecorder() + record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOOL_CALL_CAP, stream_callback=recorder) -class TestMaxTurns: - async def test_max_turns_marks_exhausted(self, patch_exec, tmp_path): - patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path, max_turns=1) + assert proc.terminated is True + assert record.crashed is False assert record.tool_calls_exhausted is True + ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] + assert [e.status for e in ends] == [AgentEndStatus.TOOL_CALLS_EXHAUSTED] - async def test_a_cap_the_run_stays_under_is_not_exhausted(self, patch_exec, tmp_path): - """The fixture is exactly 3 turns, so max_turns=3 is the boundary.""" - patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path, max_turns=3) + async def test_token_budget_ends_turn_token_budget_exceeded(self, patch_exec, tmp_path): + proc = _RunningProcess(HAPPY_STREAM) + patch_exec(proc) + recorder = _EventRecorder() + record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOKEN_BUDGET, stream_callback=recorder) + + assert proc.terminated is True + assert record.crashed is False assert record.tool_calls_exhausted is False - assert record.assistant_turn_count == 3 + ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] + assert [e.status for e in ends] == [AgentEndStatus.TOKEN_BUDGET_EXCEEDED] async def test_the_deciding_turn_is_kept_whole(self, patch_exec, tmp_path): - """max_turns=1 cuts at the START of turn 2, so turn 1 survives complete.""" - patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path, max_turns=1) + """A stop that lands on turn 2's `turn_start` keeps turn 1 complete.""" + second_turn_start = [i for i, line in enumerate(HAPPY_STREAM) if json.loads(line)["type"] == "turn_start"][1] + patch_exec(_RunningProcess(HAPPY_STREAM)) + record = await _run( + _agent(), tmp_path, should_stop=_stop_after(second_turn_start + 1, StopReason.TOOL_CALL_CAP) + ) assert record.tool_calls_exhausted is True assert len(record.commands) == 1 # turn 1's write @@ -485,22 +509,22 @@ async def test_the_deciding_turn_is_kept_whole(self, patch_exec, tmp_path): assert usage.uncached_input_tokens == 406 # turn 1's input exactly assert usage.output_tokens == 77 # 69 + 8 reasoning - async def test_the_turn_past_the_cap_is_never_admitted(self, patch_exec, tmp_path): - """The cap stops at the (N+1)th `turn_start`, before it is counted or emitted.""" - patch_exec(_FakeProcess(HAPPY_STREAM)) - recorder = _EventRecorder() - record = await _run(_agent(), tmp_path, max_turns=1, stream_callback=recorder) - + async def test_an_intentional_stop_is_exempt_from_a_non_zero_exit(self, patch_exec, tmp_path): + """Killing the CLI makes it exit non-zero; that must not crash an intentional stop.""" + patch_exec(_RunningProcess(HAPPY_STREAM, returncode=-15, stderr=b"terminated")) + record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOOL_CALL_CAP) + assert record.crashed is False assert record.tool_calls_exhausted is True - assert record.assistant_turn_count == 1 - starts = [e for e in recorder.events if isinstance(e, TurnStartEvent)] - ends = [e for e in recorder.events if isinstance(e, TurnEndEvent)] - assert len(starts) == 1 - assert len(ends) == 1 - async def test_no_cap_is_uncapped(self, patch_exec, tmp_path): + async def test_an_intentional_stop_is_exempt_from_no_recognized_events(self, patch_exec, tmp_path): + """A stop can land before the first recognized event; that is not vocabulary drift.""" + patch_exec(_RunningProcess([json.dumps({"type": "not_a_pi_event"}), *HAPPY_STREAM])) + record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOKEN_BUDGET) + assert record.crashed is False + + async def test_no_stop_is_uncapped(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + record = await _run(_agent(), tmp_path, should_stop=lambda: None) assert record.tool_calls_exhausted is False assert record.assistant_turn_count == 3 @@ -848,17 +872,13 @@ async def test_terminal_error_crashes_the_turn(self, patch_exec, tmp_path): assert partial is not None assert partial.crashed is True - async def test_max_turns_cut_after_an_error_turn_finalizes_cleanly(self, patch_exec, tmp_path): - """A max_turns cut landing right after an error turn_end (pi still retrying, - so error_message is set but not yet cleared) must finalize as - tool_calls_exhausted — NOT crash on the stale error. Guards the documented - 'no crash, no retry' contract; without the intentional-cut gate the error - arm would fire on a clean budget exhaustion.""" - # turn 1 errors; turn 2's turn_start trips max_turns=1 before any clean - # turn_end can clear error_message. + async def test_a_stop_after_an_error_turn_finalizes_cleanly(self, patch_exec, tmp_path): + """A stop landing right after an error turn_end (pi still retrying, so + error_message is set but not yet cleared) must finalize with the stop's + status — NOT crash on the stale error.""" stream = [_turn_start(), _turn_end_error("transient 429"), _turn_start(), _turn_end(inp=1, out=1)] - patch_exec(_FakeProcess(stream)) - record = await _run(_agent(), tmp_path, max_turns=1) + patch_exec(_RunningProcess(stream)) + record = await _run(_agent(), tmp_path, should_stop=_stop_after(3, StopReason.TOOL_CALL_CAP)) assert record.tool_calls_exhausted is True assert record.crashed is False diff --git a/tests/test_reference_permissions.py b/tests/test_reference_permissions.py index 733889ee4..e91657683 100644 --- a/tests/test_reference_permissions.py +++ b/tests/test_reference_permissions.py @@ -576,6 +576,7 @@ async def test_agent_cannot_read_reference_mid_turn(self, tmp_path, monkeypatch) variant_id="v", task_file=task_file, ) + orchestrator._build_monitor() await orchestrator._stage_reference() staged = orchestrator._reference_dir assert staged is not None @@ -642,6 +643,7 @@ async def test_reference_is_unshielded_by_the_time_criteria_run(self, tmp_path, task = _reference_task(reference_dir="reference") orchestrator = Orchestrator(task=task, run_dir=tmp_path / "run", variant_id="v", task_file=task_file) + orchestrator._build_monitor() await orchestrator._stage_reference() staged = orchestrator._reference_dir assert staged is not None diff --git a/tests/test_run_limits_models.py b/tests/test_run_limits_models.py index 5c7478a20..ffe1b51ae 100644 --- a/tests/test_run_limits_models.py +++ b/tests/test_run_limits_models.py @@ -31,7 +31,7 @@ class TestRunLimitsValidation: def test_empty_block_is_valid(self): """Empty run_limits constructs (all-None fields).""" rl = RunLimits() - assert rl.max_turns is None + assert rl.max_tool_calls is None assert rl.task_timeout is None assert rl.turn_timeout is None assert rl.max_input_tokens is None @@ -51,8 +51,8 @@ def test_only_max_total_tokens_ok(self): def test_only_max_usd_ok(self): assert RunLimits(max_usd=0.01).max_usd == 0.01 - def test_only_max_turns_ok(self): - assert RunLimits(max_turns=20).max_turns == 20 + def test_only_max_tool_calls_ok(self): + assert RunLimits(max_tool_calls=20).max_tool_calls == 20 def test_only_task_timeout_ok(self): assert RunLimits(task_timeout=600).task_timeout == 600 @@ -60,10 +60,18 @@ def test_only_task_timeout_ok(self): def test_only_turn_timeout_ok(self): assert RunLimits(turn_timeout=120).turn_timeout == 120 - def test_max_turns_validation(self): + def test_max_tool_calls_validation(self): with pytest.raises(ValidationError, match="greater than 0"): - RunLimits(max_turns=0) - RunLimits(max_turns=1) + RunLimits(max_tool_calls=0) + RunLimits(max_tool_calls=1) + + def test_max_turns_under_run_limits_is_rejected(self): + with pytest.raises(ValidationError, match=r"max_turns\n\s+Extra inputs are not permitted"): + RunLimits.model_validate({"max_turns": 5}) + + def test_max_turns_under_task_run_limits_is_rejected(self): + with pytest.raises(ValidationError, match="Extra inputs are not permitted"): + _minimal_task(run_limits={"max_turns": 5}) def test_task_timeout_validation(self): with pytest.raises(ValidationError, match="greater than or equal to 30"): @@ -95,7 +103,7 @@ def test_extra_forbid(self): def test_all_fields_roundtrip(self): rl = RunLimits( - max_turns=20, + max_tool_calls=20, expected_tool_calls=15, task_timeout=600, turn_timeout=120, @@ -120,9 +128,9 @@ def test_expected_tool_calls_lower_bound(self): def test_expected_tool_calls_yaml_coercion(self): assert RunLimits.model_validate({"expected_tool_calls": "10"}).expected_tool_calls == 10 - def test_expected_tool_calls_greater_than_max_turns_allowed(self): - rl = RunLimits(max_turns=5, expected_tool_calls=20) - assert rl.max_turns == 5 + def test_expected_tool_calls_greater_than_max_tool_calls_allowed(self): + rl = RunLimits(max_tool_calls=5, expected_tool_calls=20) + assert rl.max_tool_calls == 5 assert rl.expected_tool_calls == 20 def test_extra_forbid_still_rejects_unknowns(self): @@ -181,7 +189,7 @@ def test_run_limits_round_trip(self): def test_empty_run_limits_on_task_is_valid(self): task = _minimal_task(run_limits={}) assert task.run_limits is not None - assert task.run_limits.max_turns is None + assert task.run_limits.max_tool_calls is None def test_top_level_max_turns_now_dropped_with_unknown_field_warning(self): """The hoist shim is gone: top-level max_turns is now an unknown top-level @@ -203,9 +211,9 @@ def test_top_level_turn_timeout_now_dropped_with_unknown_field_warning(self): def test_top_level_timing_alongside_run_limits_keeps_canonical_block(self): """Top-level timing is dropped (unknown field); the canonical run_limits block stands.""" with pytest.warns(DeprecationWarning, match=r"unknown top-level field 'max_turns'"): - task = _minimal_task(max_turns=20, run_limits={"max_turns": 5}) + task = _minimal_task(max_turns=20, run_limits={"max_tool_calls": 5}) assert task.run_limits is not None - assert task.run_limits.max_turns == 5 + assert task.run_limits.max_tool_calls == 5 def test_max_iterations_dropped_with_warning(self): """max_iterations was removed in PR #191; the soft-launch hook flags it.""" diff --git a/tests/test_run_limits_orchestrator.py b/tests/test_run_limits_orchestrator.py index 266fce88a..c2b8fc9d8 100644 --- a/tests/test_run_limits_orchestrator.py +++ b/tests/test_run_limits_orchestrator.py @@ -104,6 +104,7 @@ def _make_orchestrator(task: TaskDefinition, tmp_path) -> Orchestrator: sandbox.sandbox_dir.mkdir() orchestrator.sandbox = sandbox orchestrator.success_checker = MagicMock() + orchestrator._build_monitor() return orchestrator @@ -432,7 +433,7 @@ def test_noop_when_run_limits_is_none(self, tmp_path, caplog): assert orch._expected_tool_calls_warning_emitted is False def test_noop_when_expected_tool_calls_unset(self, tmp_path, caplog): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_turns=10)), tmp_path) + orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_tool_calls=10)), tmp_path) orch.result.iterations.append(_make_turn(commands=20)) with caplog.at_level(logging.WARNING): orch._check_expected_tool_calls(iteration=1) diff --git a/tests/test_run_limits_resolver.py b/tests/test_run_limits_resolver.py index 93d7828c7..466257483 100644 --- a/tests/test_run_limits_resolver.py +++ b/tests/test_run_limits_resolver.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path + import pytest from coder_eval.models import ( @@ -45,14 +47,48 @@ def test_default_experiment_provides_run_limits(self): assert resolved.run_limits.max_usd == 1.0 assert lineage["run_limits.max_usd"].source == "default" - def test_resolve_max_turns_from_default(self): - default_exp = _default_exp(RunLimits(max_turns=20)) + def test_resolve_max_tool_calls_from_default(self): + default_exp = _default_exp(RunLimits(max_tool_calls=20)) task = _make_task() exp = ExperimentDefinition(experiment_id="e", variants=[ExperimentVariant(variant_id="v")]) resolved, lineage, _ = resolve_task_for_variant(default_exp, task, exp, exp.variants[0]) assert resolved.run_limits is not None - assert resolved.run_limits.max_turns == 20 - assert lineage["run_limits.max_turns"].source == "default" + assert resolved.run_limits.max_tool_calls == 20 + assert lineage["run_limits.max_tool_calls"].source == "default" + + def test_max_tool_calls_merges_through_all_five_layers(self): + from coder_eval.orchestration.config import BatchRunConfig + from coder_eval.orchestration.experiment import _apply_cli_overrides + + def resolve(*, exp_default=None, task_value=None, variant_value=None, cli_value=None): + default_exp = _default_exp(RunLimits(max_tool_calls=100)) + task = _make_task(run_limits={"max_tool_calls": task_value}) if task_value else _make_task() + exp = ExperimentDefinition( + experiment_id="e", + defaults=ExperimentDefaults(run_limits=RunLimits(max_tool_calls=exp_default)) if exp_default else None, + variants=[ + ExperimentVariant( + variant_id="v", + run_limits=RunLimits(max_tool_calls=variant_value) if variant_value else None, + ) + ], + ) + resolved, lineage, _ = resolve_task_for_variant(default_exp, task, exp, exp.variants[0]) + if cli_value: + config = BatchRunConfig(run_dir=Path("runs/test"), overrides={"run_limits.max_tool_calls": cli_value}) + _apply_cli_overrides(resolved, config, lineage=lineage) + assert resolved.run_limits is not None + return resolved.run_limits.max_tool_calls, lineage["run_limits.max_tool_calls"].source + + assert resolve() == (100, "default") + assert resolve(exp_default=50) == (50, "experiment-defaults") + assert resolve(exp_default=50, task_value=20) == (20, "task") + assert resolve(exp_default=50, task_value=20, variant_value=10) == (10, "variant") + assert resolve(exp_default=50, task_value=20, variant_value=10, cli_value=3) == (3, "cli") + + def test_max_turns_under_variant_run_limits_is_rejected(self): + with pytest.raises(ValueError, match="Extra inputs are not permitted"): + ExperimentVariant.model_validate({"variant_id": "v", "run_limits": {"max_turns": 5}}) def test_experiment_defaults_overrides_default(self): default_exp = _default_exp(RunLimits(max_usd=1.0)) @@ -82,12 +118,12 @@ def test_task_overrides_experiment_defaults(self): def test_task_run_limits_overrides_default_per_key(self): """Task setting one key keeps the default for other keys (field merge).""" - default_exp = _default_exp(RunLimits(max_turns=20, task_timeout=600)) - task = _make_task(run_limits={"max_turns": 5}) + default_exp = _default_exp(RunLimits(max_tool_calls=20, task_timeout=600)) + task = _make_task(run_limits={"max_tool_calls": 5}) exp = ExperimentDefinition(experiment_id="e", variants=[ExperimentVariant(variant_id="v")]) resolved, _, _ = resolve_task_for_variant(default_exp, task, exp, exp.variants[0]) assert resolved.run_limits is not None - assert resolved.run_limits.max_turns == 5 + assert resolved.run_limits.max_tool_calls == 5 assert resolved.run_limits.task_timeout == 600 def test_variant_overrides_task(self): @@ -105,17 +141,17 @@ def test_variant_overrides_task(self): def test_variant_run_limits_field_merges_with_task(self): """Variant override of one key leaves other task-set keys intact (field-merge tell-tale).""" default_exp = _default_exp() - task = _make_task(run_limits={"max_turns": 5, "max_usd": 0.5}) + task = _make_task(run_limits={"max_tool_calls": 5, "max_usd": 0.5}) exp = ExperimentDefinition( experiment_id="e", variants=[ExperimentVariant(variant_id="v", run_limits=RunLimits(max_usd=1.0))], ) resolved, lineage, _ = resolve_task_for_variant(default_exp, task, exp, exp.variants[0]) assert resolved.run_limits is not None - # Variant only overrides max_usd; task's max_turns survives. - assert resolved.run_limits.max_turns == 5 + # Variant only overrides max_usd; task's max_tool_calls survives. + assert resolved.run_limits.max_tool_calls == 5 assert resolved.run_limits.max_usd == 1.0 - assert lineage["run_limits.max_turns"].source == "task" + assert lineage["run_limits.max_tool_calls"].source == "task" assert lineage["run_limits.max_usd"].source == "variant" def test_variant_unset_does_not_clear_task_block(self): @@ -158,13 +194,13 @@ def test_no_run_limits_anywhere(self): assert not any(k.startswith("run_limits") for k in lineage) def test_lineage_uses_dotted_keys(self): - """Lineage keys are dotted (run_limits.max_turns), not the bare 'max_turns'.""" + """Lineage keys are dotted (run_limits.max_tool_calls), not the bare 'max_tool_calls'.""" default_exp = _default_exp() - task = _make_task(run_limits={"max_turns": 5}) + task = _make_task(run_limits={"max_tool_calls": 5}) exp = ExperimentDefinition(experiment_id="e", variants=[ExperimentVariant(variant_id="v")]) _, lineage, _ = resolve_task_for_variant(default_exp, task, exp, exp.variants[0]) - assert "run_limits.max_turns" in lineage - assert "max_turns" not in lineage + assert "run_limits.max_tool_calls" in lineage + assert "max_tool_calls" not in lineage def test_lineage_value_is_serializable(self): """lineage.value must be JSON-serializable (scalar from dotted key).""" @@ -186,7 +222,7 @@ def test_unset_non_optional_field_does_not_clobber_lower_layer(self): that didn't mention the field leaves earlier values intact. """ default_exp = _default_exp() - task = _make_task(run_limits={"count_cached_input": True, "max_turns": 10}) + task = _make_task(run_limits={"count_cached_input": True, "max_tool_calls": 10}) # Variant overrides ONLY max_usd — must not touch count_cached_input. exp = ExperimentDefinition( experiment_id="e", @@ -197,7 +233,7 @@ def test_unset_non_optional_field_does_not_clobber_lower_layer(self): assert resolved.run_limits.count_cached_input is True, ( "variant's default count_cached_input=False clobbered the task-level True" ) - assert resolved.run_limits.max_turns == 10 + assert resolved.run_limits.max_tool_calls == 10 assert resolved.run_limits.max_usd == 1.0 # Lineage should only credit fields the variant actually set. assert "run_limits.count_cached_input" not in lineage or ( diff --git a/tests/test_run_record.py b/tests/test_run_record.py index 116c79c7d..f522d3796 100644 --- a/tests/test_run_record.py +++ b/tests/test_run_record.py @@ -14,6 +14,8 @@ from coder_eval.models import ( AgentKind, CommandTelemetry, + EarlyStopInfo, + EarlyStopReason, EvaluationResult, FinalStatus, ResultSummary, @@ -73,11 +75,11 @@ "task_id": "char-task", "task_path": None, "teardown_ms": None, + "tool_calls_remaining_at_stop": None, "tool_ms": None, "total_cost_usd": 0.5, "total_tokens": 1000, "total_turns": 0, - "turns_remaining_at_stop": None, "variant_id": None, "visible_turns": 0, "weighted_score": 0.75, @@ -237,7 +239,7 @@ def test_emits_when_configured(self): def test_none_when_unset(self): result = _make_result( - resolved={"run_limits": {"max_turns": 10}}, + resolved={"run_limits": {"max_tool_calls": 10}}, turns=[_turn_with_expected(5)], ) d = eval_result_to_task_dict(result) @@ -251,10 +253,37 @@ def test_none_when_task_config_none(self): def test_row_carries_the_tool_call_keys_and_none_of_the_historical_ones(self): result = _make_result(resolved={"run_limits": {"expected_turns": 12}}, turns=[_turn_with_expected(5)]) d = eval_result_to_task_dict(result) - assert {"tool_calls_exhausted", "expected_tool_calls", "expected_tool_calls_overage"} <= d.keys() - assert not {"max_turns_exhausted", "expected_turns", "expected_turns_overage"} & d.keys() + assert { + "tool_calls_exhausted", + "tool_calls_remaining_at_stop", + "expected_tool_calls", + "expected_tool_calls_overage", + } <= d.keys() + assert ( + not { + "max_turns_exhausted", + "turns_remaining_at_stop", + "expected_turns", + "expected_turns_overage", + } + & d.keys() + ) assert d["expected_tool_calls"] is None + def test_row_carries_tool_calls_remaining_at_stop_from_early_stop(self): + result = _make_result() + result.early_stop = EarlyStopInfo( + reason=EarlyStopReason.CRITERION_FAILED, + deciding_criterion_type="file_exists", + deciding_criterion_description="x", + sdk_turn_index=2, + tool_call_index=3, + elapsed_seconds=1.0, + tool_calls_remaining_at_stop=7, + ) + d = eval_result_to_task_dict(result) + assert d["tool_calls_remaining_at_stop"] == 7 + def test_none_when_invalid_type(self): result = _make_result( resolved={"run_limits": {"expected_tool_calls": "ten"}}, diff --git a/tests/test_sdk_option_classification.py b/tests/test_sdk_option_classification.py index 0f343f717..4dffb713b 100644 --- a/tests/test_sdk_option_classification.py +++ b/tests/test_sdk_option_classification.py @@ -28,6 +28,7 @@ from claude_agent_sdk import ClaudeAgentOptions +from coder_eval.models import parse_agent_config from coder_eval.models.agent_config import ( _FRAMEWORK_OWNED_SDK_FIELDS, _USER_VISIBLE_SDK_FIELDS, @@ -100,3 +101,14 @@ def test_user_visible_and_framework_owned_are_disjoint() -> None: """The two sets must partition the SDK fields (no key on both lists).""" overlap = set(_USER_VISIBLE_SDK_FIELDS) & _FRAMEWORK_OWNED_SDK_FIELDS assert not overlap, f"Field(s) on both lists: {sorted(overlap)}" + + +def test_max_turns_is_a_user_visible_sdk_option() -> None: + """The run's tool-call cap is `run_limits.max_tool_calls`; the SDK's own turn cap is a pass-through.""" + assert "max_turns" not in _FRAMEWORK_OWNED_SDK_FIELDS + assert "max_turns" in _USER_VISIBLE_SDK_FIELDS + + +def test_sdk_options_max_turns_validates_on_claude_code() -> None: + config = parse_agent_config(type="claude-code", sdk_options={"max_turns": 3}) + assert config.sdk_options == {"max_turns": 3} diff --git a/tests/test_simulation_integration.py b/tests/test_simulation_integration.py index cd8207178..50875e0f3 100644 --- a/tests/test_simulation_integration.py +++ b/tests/test_simulation_integration.py @@ -16,6 +16,7 @@ from coder_eval.models import ( AgentKind, FileExistsCriterion, + RunLimits, SandboxConfig, SimulationConfig, TaskDefinition, @@ -34,6 +35,7 @@ def _build_task( sim_overrides: dict[str, Any] | None = None, *, initial_prompt: str | None = "Please create the file.", + run_limits: RunLimits | None = None, ) -> TaskDefinition: sim_kwargs: dict[str, Any] = { "enabled": True, @@ -54,6 +56,7 @@ def _build_task( sandbox=SandboxConfig(driver="tempdir"), success_criteria=[FileExistsCriterion(path="test.txt", description="file must exist")], simulation=SimulationConfig(**sim_kwargs), + run_limits=run_limits, ) @@ -64,6 +67,41 @@ async def _create(self): monkeypatch.setattr(Orchestrator, "_create_agent", _create) +class _CooperativeToolAgent(MockAgent): + """MockAgent that makes ``calls_per_turn`` resolved tool calls per turn, polling ``should_stop``.""" + + def __init__(self, task: TaskDefinition, calls_per_turn: int) -> None: + super().__init__(task, scenario="failure") + self._calls_per_turn = calls_per_turn + self._tool_seq = 0 + self.emitted_per_turn: list[int] = [] + + async def communicate(self, user_input: str, **kwargs: Any) -> TurnRecord: + from datetime import datetime + + from coder_eval.models import CommandTelemetry + from coder_eval.streaming.events import StopReason, ToolEndEvent, ToolStartEvent + + self._iteration += 1 + stream_callback = kwargs["stream_callback"] + should_stop = kwargs["should_stop"] + commands: list[CommandTelemetry] = [] + while should_stop() is None and len(commands) < self._calls_per_turn: + self._tool_seq += 1 + tool = CommandTelemetry(tool_name="Bash", tool_id=f"tool-{self._tool_seq}", timestamp=datetime.now()) + stream_callback.on_event(ToolStartEvent(task_id=self.task.task_id, tool=tool)) + stream_callback.on_event(ToolEndEvent(task_id=self.task.task_id, tool=tool)) + commands.append(tool) + self.emitted_per_turn.append(len(commands)) + return TurnRecord( + iteration=self._iteration, + user_input=user_input, + agent_output="working", + commands=commands, + tool_calls_exhausted=should_stop() is StopReason.TOOL_CALL_CAP, + ) + + def _install_fake_simulator( monkeypatch: pytest.MonkeyPatch, responses: list[str], @@ -399,3 +437,32 @@ async def _create(self): ] assert openers, "expected a standalone turn holding the pinned opener" assert all(t.duration_seconds == 0.0 for t in openers) + + +@pytest.mark.asyncio +async def test_simulation_tool_call_cap_is_cumulative_across_dialog_turns(tmp_path, monkeypatch): + """Two turns of 2 tool calls under max_tool_calls=3: the cap latches in turn 2 and ends the dialog.""" + agents: list[_CooperativeToolAgent] = [] + + async def _create(self): + agent = _CooperativeToolAgent(self.task, calls_per_turn=2) + agents.append(agent) + return agent + + monkeypatch.setattr(Orchestrator, "_create_agent", _create) + stub = _install_fake_simulator(monkeypatch, responses=["keep going"] * 10) + + task = _build_task({"max_turns": 4}, run_limits=RunLimits(max_tool_calls=3)) + orch = Orchestrator(task=task, run_dir=tmp_path / "run" / "cap", variant_id="default") + result = await orch.run() + + from coder_eval.simulation import DialogStopReason + + (agent,) = agents + assert agent.emitted_per_turn == [2, 1] + assert result.simulation is not None + assert result.simulation.stop_reason == DialogStopReason.TOOL_CALL_CAP.value == "tool_call_cap" + assert result.simulation.total_turns == 2 + assert result.tool_calls_exhausted is True + # The simulator answered turn 1 only; the cap ended the dialog before it was asked again. + assert len(stub.calls) == 1 diff --git a/tests/test_spi.py b/tests/test_spi.py index 0dc6b0346..714683dd7 100644 --- a/tests/test_spi.py +++ b/tests/test_spi.py @@ -18,8 +18,12 @@ ) -def test_spi_version_is_one() -> None: - assert spi.SPI_VERSION == 1 +def test_spi_version_is_two() -> None: + assert spi.SPI_VERSION == 2 + + +def test_the_stop_channel_is_exported() -> None: + assert {"StopReason", "end_status_for"} <= set(spi.__all__) def test_all_is_sorted_and_unique() -> None: diff --git a/tests/test_sub_agent_runner.py b/tests/test_sub_agent_runner.py index 4828323ea..6e58843fb 100644 --- a/tests/test_sub_agent_runner.py +++ b/tests/test_sub_agent_runner.py @@ -79,11 +79,11 @@ async def test_runner_happy_path(sandbox: Sandbox, tmp_path: Path) -> None: ) mock_agent = _make_mock_agent() with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - turn = await runner.run_async("grade this", max_turns=10, turn_timeout=30.0) + turn = await runner.run_async("grade this", turn_timeout=30.0) assert turn.agent_output == '{"score": 1.0, "rationale": "ok"}' mock_agent.start.assert_awaited_once() - mock_agent.communicate.assert_awaited_once_with("grade this", timeout=30.0, max_turns=10) + mock_agent.communicate.assert_awaited_once_with("grade this", timeout=30.0) mock_agent.stop.assert_awaited() @@ -124,7 +124,7 @@ async def capture_files(_msg: str, **_kw: object) -> TurnRecord: return _make_turn() mock_agent.communicate.side_effect = capture_files - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) assert captured["has_reference_dir"] == "True" assert captured["has_main"] == "True" @@ -175,7 +175,7 @@ async def capture_state(_msg: str, **_kw: object) -> TurnRecord: mock_agent.communicate.side_effect = capture_state # Must not raise — this is the regression assertion. - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) # _reference/ contains the REAL reference content, NOT the agent-planted file. assert captured["ref_main_content"] == "" @@ -208,7 +208,7 @@ async def capture_no_ref(_msg: str, **_kw: object) -> TurnRecord: return _make_turn() mock_agent.communicate.side_effect = capture_no_ref - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) assert captured["has_reference_dir"] == "False" @@ -224,7 +224,7 @@ async def test_runner_starts_in_temp_copy_not_original(sandbox: Sandbox, tmp_pat ) mock_agent = _make_mock_agent() with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) start_arg = mock_agent.start.call_args.args[0] assert start_arg != str(sandbox.sandbox_dir) @@ -246,7 +246,7 @@ async def capture_start(path: str, **_kwargs: object) -> None: mock_agent.start.side_effect = capture_start with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) assert captured["path"] assert not Path(captured["path"]).exists() @@ -283,7 +283,7 @@ async def hang_forever(*_args: object, **_kwargs: object) -> TurnRecord: mock_agent.communicate.side_effect = hang_forever with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - task = asyncio.ensure_future(runner.run_async("grade", max_turns=10, turn_timeout=30.0)) + task = asyncio.ensure_future(runner.run_async("grade", turn_timeout=30.0)) await started.wait() task.cancel() with pytest.raises(asyncio.CancelledError): @@ -334,7 +334,7 @@ def capture_mkdtemp(*args: object, **kwargs: object) -> str: patch("coder_eval.evaluation.sub_agent.shutil.copytree", side_effect=slow_copytree), patch("coder_eval.evaluation.sub_agent.tempfile.mkdtemp", side_effect=capture_mkdtemp), ): - task = asyncio.ensure_future(runner.run_async("grade", max_turns=10, turn_timeout=30.0)) + task = asyncio.ensure_future(runner.run_async("grade", turn_timeout=30.0)) while not copy_started.is_set(): await asyncio.sleep(0.01) task.cancel() @@ -365,7 +365,7 @@ async def capture_start(path: str, **_kwargs: object) -> None: patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent), pytest.raises(RuntimeError, match="boom"), ): - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) assert not Path(captured["path"]).exists() mock_agent.kill.assert_awaited() @@ -386,7 +386,7 @@ async def test_runner_cleans_up_on_start_failure(sandbox: Sandbox) -> None: patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent), pytest.raises(RuntimeError, match="claude binary not found"), ): - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) mock_agent.kill.assert_awaited() @@ -411,7 +411,7 @@ async def capture_start(path: str, **_kwargs: object) -> None: patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent), pytest.raises(TurnTimeoutError), ): - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) assert not Path(captured["path"]).exists() @@ -514,7 +514,7 @@ async def capture_start(path: str, **_kwargs: object) -> None: mock_agent.start.side_effect = capture_start with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) assert "real.txt" in captured["entries"] assert "leak" not in captured["entries"] @@ -544,7 +544,7 @@ async def capture_start(path: str, **_kwargs: object) -> None: mock_agent.start.side_effect = capture_start with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) assert "keep.txt" in captured["sub_entries"] assert "nested_leak" not in captured["sub_entries"] @@ -570,7 +570,7 @@ async def capture_start(path: str, **_kwargs: object) -> None: mock_agent.start.side_effect = capture_start with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) assert captured["has_git"] == "False" assert captured["has_main"] == "True" @@ -602,7 +602,7 @@ async def capture_start(path: str, **_kwargs: object) -> None: mock_agent.start.side_effect = capture_start with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) assert ".claude" not in captured["sub_entries"] assert ".mcp.json" not in captured["sub_entries"] @@ -656,7 +656,7 @@ async def capture_state(_msg: str, **_kw: object) -> TurnRecord: return _make_turn() mock_agent.communicate.side_effect = capture_state - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) assert captured["inner_present"] == "True" assert captured["inner_content"] == "CUSTOMER-CONTENT" @@ -695,7 +695,7 @@ async def capture_state(_msg: str, **_kw: object) -> TurnRecord: return _make_turn() mock_agent.communicate.side_effect = capture_state - await runner.run_async("grade", max_turns=10, turn_timeout=30.0) + await runner.run_async("grade", turn_timeout=30.0) assert captured["keep_present"] == "True" assert captured["log_present"] == "False" diff --git a/tests/test_timeout_orchestrator.py b/tests/test_timeout_orchestrator.py index 080113c79..e4931804f 100644 --- a/tests/test_timeout_orchestrator.py +++ b/tests/test_timeout_orchestrator.py @@ -44,7 +44,7 @@ def _make_task(*, turn_timeout: float | None = None, task_timeout: float | None ignore_patterns=[], ) run_limits = RunLimits.model_construct( - max_turns=None, + max_tool_calls=None, turn_timeout=turn_timeout, task_timeout=task_timeout, ) @@ -92,6 +92,7 @@ def _make_initialized_orchestrator(task: TaskDefinition, tmp_path) -> Orchestrat mock_sandbox.sandbox_dir.mkdir() orchestrator.sandbox = mock_sandbox orchestrator.success_checker = MagicMock() + orchestrator._build_monitor() return orchestrator @@ -568,6 +569,7 @@ async def test_turn_timeout_is_per_attempt_not_cycle(tmp_path): run_dir.mkdir(parents=True) orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant") + orchestrator._build_monitor() orchestrator.result = EvaluationResult( task_id="per_attempt_budget", task_description="per-attempt budget", @@ -642,6 +644,7 @@ async def test_wait_for_backstop_calls_discard_pending_turn(tmp_path): run_dir.mkdir(parents=True) orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="v") + orchestrator._build_monitor() orchestrator.result = EvaluationResult( task_id="discard_pending", task_description="discard_pending", diff --git a/tests/test_timing_identity_contract.py b/tests/test_timing_identity_contract.py index 382157324..54ccf2c82 100644 --- a/tests/test_timing_identity_contract.py +++ b/tests/test_timing_identity_contract.py @@ -491,7 +491,6 @@ def _monotonic() -> float: task_id="t", user_input="go", iteration=1, - max_turns=None, log=agent._log, turn_start_time=_monotonic(), deadline=None, @@ -570,7 +569,6 @@ def _monotonic() -> float: task_id="t", user_input="go", iteration=1, - max_turns=None, log=agent._log, turn_start_time=_monotonic(), deadline=None, diff --git a/tests/test_turn_monitor.py b/tests/test_turn_monitor.py new file mode 100644 index 000000000..ef0cc6113 --- /dev/null +++ b/tests/test_turn_monitor.py @@ -0,0 +1,213 @@ +"""``TurnMonitor``: the one answerer of the agent's ``should_stop`` poll.""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import patch + +from coder_eval.models import ( + AgentKind, + CommandTelemetry, + EarlyStopReason, + FileExistsCriterion, + RunLimits, + SandboxConfig, + SkillTriggeredCriterion, + StopEarlyPolicy, + TaskDefinition, + TokenUsage, + parse_agent_config, +) +from coder_eval.orchestration.turn_monitor import TurnMonitor +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentStartEvent, + StopReason, + ToolEndEvent, + ToolEndStatus, + ToolStartEvent, + TurnEndEvent, +) +from tests._fixtures.live_criteria import FROZEN_TS + + +def _task(*, criteria: list[Any] | None = None, max_tool_calls: int | None = None) -> TaskDefinition: + return TaskDefinition( + task_id="monitor-test", + description="monitor test task", + initial_prompt="do the thing", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + sandbox=SandboxConfig(driver="tempdir"), + success_criteria=criteria or [FileExistsCriterion(path="x", description="x exists")], + run_limits=RunLimits(max_tool_calls=max_tool_calls), + ) + + +def _skill_crit(skill: str, *, on_pass: bool) -> SkillTriggeredCriterion: + return SkillTriggeredCriterion( + type="skill_triggered", + description=f"{skill} activation", + skill_name=skill, + expected_skill=skill, + stop_early=StopEarlyPolicy(on_pass="stop" if on_pass else "continue"), + ) + + +def _cmd(tool_id: str, *, tool_name: str = "Bash", parameters: dict[str, Any] | None = None) -> CommandTelemetry: + return CommandTelemetry(tool_name=tool_name, tool_id=tool_id, timestamp=FROZEN_TS, parameters=parameters or {}) + + +def _end(tool_id: str, *, status: ToolEndStatus = ToolEndStatus.OK, **kwargs: Any) -> ToolEndEvent: + return ToolEndEvent(task_id="t", tool=_cmd(tool_id, **kwargs), status=status) + + +def _feed(monitor: TurnMonitor, events: list[Any]) -> None: + for event in events: + monitor.on_event(event) + + +class TestToolCallCap: + def test_the_cap_latches_on_the_resolved_call_that_reaches_it(self) -> None: + monitor = TurnMonitor.for_task(_task(max_tool_calls=3), arm=True) + _feed(monitor, [AgentStartEvent(task_id="t"), _end("a"), _end("b")]) + assert monitor.should_stop() is None + monitor.on_event(_end("c")) + assert monitor.should_stop() is StopReason.TOOL_CALL_CAP + assert monitor.stop_reason is StopReason.TOOL_CALL_CAP + assert monitor.info is None + + def test_an_unresolved_end_is_not_counted(self) -> None: + monitor = TurnMonitor.for_task(_task(max_tool_calls=1), arm=True) + monitor.on_event(_end("orphan", status=ToolEndStatus.UNRESOLVED)) + assert monitor.tool_calls == 0 + assert monitor.should_stop() is None + + def test_a_re_emitted_end_is_not_counted_twice(self) -> None: + monitor = TurnMonitor.for_task(_task(max_tool_calls=2), arm=True) + _feed(monitor, [_end("a"), _end("a")]) + assert monitor.tool_calls == 1 + assert monitor.should_stop() is None + + def test_a_tool_start_alone_does_not_count(self) -> None: + monitor = TurnMonitor.for_task(_task(max_tool_calls=1), arm=True) + monitor.on_event(ToolStartEvent(task_id="t", tool=_cmd("a"))) + assert monitor.should_stop() is None + + def test_the_count_is_cumulative_across_communicate_calls(self) -> None: + monitor = TurnMonitor.for_task(_task(max_tool_calls=3), arm=True) + _feed(monitor, [AgentStartEvent(task_id="t"), _end("a"), _end("b"), AgentEndEvent(task_id="t")]) + assert monitor.should_stop() is None + _feed(monitor, [AgentStartEvent(task_id="t"), _end("c")]) + assert monitor.should_stop() is StopReason.TOOL_CALL_CAP + + def test_a_latched_cap_stops_the_next_attempt_at_its_first_poll(self) -> None: + monitor = TurnMonitor.for_task(_task(max_tool_calls=1), arm=True) + _feed(monitor, [AgentStartEvent(task_id="t"), _end("a"), AgentEndEvent(task_id="t", crashed=True)]) + monitor.on_event(AgentStartEvent(task_id="t")) + assert monitor.should_stop() is StopReason.TOOL_CALL_CAP + + def test_no_cap_never_stops(self) -> None: + monitor = TurnMonitor.for_task(_task(max_tool_calls=None), arm=True) + _feed(monitor, [_end(str(i)) for i in range(50)]) + assert monitor.should_stop() is None + assert monitor.tool_calls == 50 + + def test_an_unarmed_monitor_still_caps(self) -> None: + criteria = [_skill_crit("date-teller", on_pass=True)] + monitor = TurnMonitor.for_task(_task(criteria=criteria, max_tool_calls=1), arm=False) + assert not monitor.armed + monitor.on_event(_end("sk", tool_name="Skill", parameters={"skill": "date-teller"})) + assert monitor.should_stop() is StopReason.TOOL_CALL_CAP + assert monitor.info is None + + +class TestEarlyCriterionAndPrecedence: + def test_arm_false_never_fires_the_criterion(self) -> None: + criteria = [_skill_crit("date-teller", on_pass=True)] + monitor = TurnMonitor.for_task(_task(criteria=criteria), arm=False) + monitor.on_event(_end("sk", tool_name="Skill", parameters={"skill": "date-teller"})) + assert monitor.should_stop() is None + + def test_the_armed_stop_wins_a_tie_with_the_cap(self) -> None: + criteria = [_skill_crit("date-teller", on_pass=True)] + monitor = TurnMonitor.for_task(_task(criteria=criteria, max_tool_calls=1), arm=True) + monitor.on_event(_end("sk", tool_name="Skill", parameters={"skill": "date-teller"})) + assert monitor.should_stop() is StopReason.EARLY_CRITERION + assert monitor.info is not None + assert monitor.info.reason is EarlyStopReason.CRITERION_PASSED + + def test_the_first_latched_reason_is_final(self) -> None: + criteria = [_skill_crit("date-teller", on_pass=True)] + monitor = TurnMonitor.for_task(_task(criteria=criteria, max_tool_calls=1), arm=True) + monitor.on_event(_end("a")) + monitor.on_event(_end("sk", tool_name="Skill", parameters={"skill": "date-teller"})) + assert monitor.should_stop() is StopReason.TOOL_CALL_CAP + assert monitor.info is None + + def test_tool_calls_remaining_at_stop_counts_down_from_the_cap(self) -> None: + criteria = [_skill_crit("date-teller", on_pass=True)] + monitor = TurnMonitor.for_task(_task(criteria=criteria, max_tool_calls=10), arm=True) + _feed( + monitor, + [_end("a"), _end("b"), _end("sk", tool_name="Skill", parameters={"skill": "date-teller"})], + ) + assert monitor.info is not None + assert monitor.info.tool_call_index == 3 + assert monitor.info.tool_calls_remaining_at_stop == 7 + + def test_tool_calls_remaining_at_stop_is_none_without_a_cap(self) -> None: + criteria = [_skill_crit("date-teller", on_pass=True)] + monitor = TurnMonitor.for_task(_task(criteria=criteria), arm=True) + monitor.on_event(_end("sk", tool_name="Skill", parameters={"skill": "date-teller"})) + assert monitor.info is not None + assert monitor.info.tool_calls_remaining_at_stop is None + + def test_a_raising_verdict_disarms_the_criteria_but_not_the_cap(self) -> None: + criteria = [_skill_crit("date-teller", on_pass=True)] + monitor = TurnMonitor.for_task(_task(criteria=criteria, max_tool_calls=2), arm=True) + with patch.object(monitor._armed[0][1], "live_verdict", side_effect=RuntimeError("boom")): + monitor.on_event(_end("a")) + assert monitor.disarmed + assert monitor.should_stop() is None + monitor.on_event(_end("sk", tool_name="Skill", parameters={"skill": "date-teller"})) + assert monitor.should_stop() is StopReason.TOOL_CALL_CAP + + def test_a_verdict_raising_on_the_call_that_reaches_the_cap_still_latches_the_cap(self) -> None: + criteria = [_skill_crit("date-teller", on_pass=True)] + monitor = TurnMonitor.for_task(_task(criteria=criteria, max_tool_calls=1), arm=True) + with patch.object(monitor._armed[0][1], "live_verdict", side_effect=RuntimeError("boom")): + monitor.on_event(_end("a")) + assert monitor.disarmed + assert monitor.should_stop() is StopReason.TOOL_CALL_CAP + + def test_sub_agent_events_are_not_counted(self) -> None: + monitor = TurnMonitor.for_task(_task(max_tool_calls=1), arm=True) + monitor.on_event(ToolEndEvent(task_id="t", tool=_cmd("child"), parent_thread_id="parent")) + assert monitor.tool_calls == 0 + assert monitor.should_stop() is None + + +class TestUsageAccumulators: + def test_in_flight_deltas_are_replaced_by_the_authoritative_end_usage(self) -> None: + monitor = TurnMonitor.for_task(_task(), arm=True) + _feed( + monitor, + [ + AgentStartEvent(task_id="t"), + TurnEndEvent(task_id="t", tokens=TokenUsage(uncached_input_tokens=10, output_tokens=5)), + TurnEndEvent(task_id="t", tokens=None), + TurnEndEvent(task_id="t", tokens=TokenUsage(uncached_input_tokens=20, output_tokens=5)), + ], + ) + assert monitor.usage.uncached_input_tokens == 30 + monitor.on_event(AgentEndEvent(task_id="t", usage=TokenUsage(uncached_input_tokens=28, output_tokens=9))) + assert (monitor.usage.uncached_input_tokens, monitor.usage.output_tokens) == (28, 9) + + def test_committed_usage_accumulates_across_communicate_calls(self) -> None: + monitor = TurnMonitor.for_task(_task(), arm=True) + for _ in range(2): + _feed( + monitor, + [AgentStartEvent(task_id="t"), AgentEndEvent(task_id="t", usage=TokenUsage(output_tokens=4))], + ) + assert monitor.usage.output_tokens == 8 diff --git a/tests/test_user_simulator.py b/tests/test_user_simulator.py index ca1e5aa49..859f25706 100644 --- a/tests/test_user_simulator.py +++ b/tests/test_user_simulator.py @@ -139,6 +139,30 @@ async def test_opener_uses_primer_when_history_empty(self): await sim.stop() +class TestSdkTurnCap: + def test_sdk_max_turns_is_one_via_sdk_options(self): + sim = UserSimulator(config=_sim_cfg(), task_description="T", initial_prompt="start") + assert sim._agent_config.sdk_options == {"max_turns": 1} + + async def test_communicate_receives_no_turn_cap_kwarg(self): + class _KwargRecordingStub(TextStubAgent): + def __init__(self, responses: list[str]) -> None: + super().__init__(responses) + self.kwargs: list[dict[str, object]] = [] + + async def communicate(self, user_input: str, **kwargs: object): + self.kwargs.append(kwargs) + return await super().communicate(user_input, **kwargs) + + stub = _KwargRecordingStub(["hello"]) + sim = await _make_started( + UserSimulator(config=_sim_cfg(), task_description="T", initial_prompt="start", agent_override=stub) + ) + await sim.next_user_message([_pair("start", "reply")]) + assert stub.kwargs == [{}] + await sim.stop() + + class TestStopTokenHandling: async def test_stop_token_triggers_flag_and_strips(self): stub = TextStubAgent(["Looks good. <<>>"]) diff --git a/tests/test_verify_published_workflow.py b/tests/test_verify_published_workflow.py index 286000de9..a4c2a6adc 100644 --- a/tests/test_verify_published_workflow.py +++ b/tests/test_verify_published_workflow.py @@ -296,7 +296,7 @@ def test_inline_consumer_task_declares_run_limits(tmp_path: Path): limits = load_task(path)[0].run_limits assert limits is not None, "the unattended paid task must declare run_limits" - assert limits.max_turns, "an unbounded turn count on a cron-triggered paid run" + assert limits.max_tool_calls, "an unbounded tool-call count on a cron-triggered paid run" assert limits.max_usd, "an unbounded spend on a cron-triggered paid run" assert limits.task_timeout, "no wall-clock cap below the job's timeout-minutes" diff --git a/tests/test_visible_turn_cap.py b/tests/test_visible_turn_cap.py deleted file mode 100644 index cde8e7ae1..000000000 --- a/tests/test_visible_turn_cap.py +++ /dev/null @@ -1,68 +0,0 @@ -"""``run_limits.max_turns`` must mean the same thing on Codex and Antigravity. - -Neither SDK can express the cap natively — each delivers exactly one SDK turn per -``communicate()`` call, so a native counter would clamp at 1 no matter what the task -asked for. Both therefore count VISIBLE turns (resolved tool calls) off one shared -definition, ``EventCollector.visible_turn_count``, rather than two per-agent counters -that happen to agree. See docs/agents/HARNESS_PARITY.md. - -Per-agent enforcement (where the cap fires in the loop, and how the run finalizes) -is covered in test_codex_agent.py and test_antigravity_agent.py. -""" - -from datetime import datetime - -import pytest - -from coder_eval.agents.antigravity_agent import AntigravityAgent -from coder_eval.agents.codex_agent import CodexAgent -from coder_eval.models import CommandTelemetry -from coder_eval.streaming.collector import EventCollector -from coder_eval.streaming.events import ToolEndEvent, ToolEndStatus - - -def _tool_end(collector: EventCollector, tool_id: str) -> None: - collector.on_event( - ToolEndEvent( - task_id="t", - turn_id="turn-1", - tool=CommandTelemetry(tool_name="Bash", tool_id=tool_id, timestamp=datetime.now(), sequence_number=0), - status=ToolEndStatus.OK, - ) - ) - - -def test_collector_visible_turn_count_counts_resolved_tool_calls(): - """The single definition Codex and Antigravity both cap against.""" - collector = EventCollector() - assert collector.visible_turn_count == 0 - - _tool_end(collector, "a") - _tool_end(collector, "b") - - assert collector.visible_turn_count == 2 - - -def test_collector_visible_turn_count_does_not_double_count_a_tool_id(): - """Keyed on tool_id, so a re-emitted end event cannot inflate the count past the cap.""" - collector = EventCollector() - - _tool_end(collector, "a") - _tool_end(collector, "a") - - assert collector.visible_turn_count == 1 - - -def test_collector_visible_turn_count_matches_the_built_record(): - """It is the live view of exactly the list ``TurnRecord.commands`` ends up holding.""" - collector = EventCollector() - for tool_id in ("a", "b", "c"): - _tool_end(collector, tool_id) - - assert collector.visible_turn_count == len(collector.build_turn_record().commands) - - -@pytest.mark.parametrize("agent_cls", [CodexAgent, AntigravityAgent]) -def test_both_capped_agents_declare_cooperative_stop(agent_cls): - """The turn cap reuses the cooperative-stop boundary, so both must support it.""" - assert agent_cls.contract.cooperative_stop is True diff --git a/tests/test_yaml_migration.py b/tests/test_yaml_migration.py index 1a890fa6e..9874d318d 100644 --- a/tests/test_yaml_migration.py +++ b/tests/test_yaml_migration.py @@ -31,20 +31,20 @@ def _experiment_yamls() -> list[Path]: @pytest.mark.parametrize("path", _task_yamls(), ids=lambda p: p.relative_to(ROOT).as_posix()) def test_task_yaml_has_no_stale_top_level_keys(path: Path) -> None: - """No top-level max_turns / task_timeout / turn_timeout on any task YAML.""" + """No top-level max_turns / max_tool_calls / task_timeout / turn_timeout on any task YAML.""" # `encoding="utf-8"` matches CE008/CE011 (which guard the same in src/); # task YAMLs frequently contain UTF-8 (← arrows, ≤, em-dashes, smart quotes) # and the Windows-default cp1252 raises UnicodeDecodeError on those bytes. data = yaml.safe_load(path.read_text(encoding="utf-8")) if not isinstance(data, dict): return - for key in ("max_turns", "task_timeout", "turn_timeout"): + for key in ("max_turns", "max_tool_calls", "task_timeout", "turn_timeout"): assert key not in data, f"{path}: stale top-level {key!r} (should be under run_limits:)" @pytest.mark.parametrize("path", _experiment_yamls(), ids=lambda p: p.name) def test_experiment_yaml_has_no_stale_top_level_keys(path: Path) -> None: - """No top-level max_turns / task_timeout / turn_timeout on experiment defaults / variants.""" + """No top-level max_turns / max_tool_calls / task_timeout / turn_timeout on experiment defaults / variants.""" data = yaml.safe_load(path.read_text(encoding="utf-8")) if not isinstance(data, dict): return @@ -53,7 +53,7 @@ def test_experiment_yaml_has_no_stale_top_level_keys(path: Path) -> None: for blob, label in [(defaults, "defaults"), *((v, f"variant {v.get('variant_id')!r}") for v in variants)]: if not isinstance(blob, dict): continue - for key in ("max_turns", "task_timeout", "turn_timeout"): + for key in ("max_turns", "max_tool_calls", "task_timeout", "turn_timeout"): assert key not in blob, f"{path} {label}: stale top-level {key!r} (should be under run_limits:)" @@ -66,8 +66,8 @@ def test_migrated_smoke_tasks_load() -> None: from coder_eval.orchestration.task_loader import load_task samples = [ - ("tasks/smoke_task_timeout.yaml", {"max_turns": 2, "task_timeout": 30}), - ("tasks/smoke_cost_budget_exceeded.yaml", {"max_turns": 2, "max_usd": 0.0001}), + ("tasks/smoke_task_timeout.yaml", {"max_tool_calls": 2, "task_timeout": 30}), + ("tasks/smoke_cost_budget_exceeded.yaml", {"max_tool_calls": 2, "max_usd": 0.0001}), ] with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) From 9a4ac75ac65a9aa6b7e612cf1d62ffc1c8a299d4 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 13:14:22 -0700 Subject: [PATCH 3/9] =?UTF-8?q?feat(limits):=203/6=20=E2=80=94=20token=20a?= =?UTF-8?q?nd=20USD=20budgets=20are=20live=20TurnMonitor=20stop=20reasons;?= =?UTF-8?q?=20an=20unpriceable=20max=5Fusd=20is=20an=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _check_run_limits, its silent cost skip and cost_data_available are deleted. The monitor latches TOKEN_BUDGET / USD_BUDGET from the stream's usage and raise_if_over_budget raises at the former call sites; a max_usd no turn can price raises BudgetUnenforceableError and finalizes ERROR. RunLimits owns the one bucket rule, every harness contract declares usage_granularity, and validate_resolved_task is the one resolution-check list. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 5 +- docs/EXTENDING.md | 3 +- docs/REPORT_SCHEMA.md | 2 +- docs/TASK_DEFINITION_GUIDE.md | 17 +- docs/agents/HARNESS_PARITY.md | 4 + src/coder_eval/agents/antigravity_agent.py | 2 + src/coder_eval/agents/claude_code_agent.py | 11 +- src/coder_eval/agents/codex_agent.py | 2 + src/coder_eval/agents/noop_agent.py | 11 +- src/coder_eval/agents/opencode_agent.py | 2 + src/coder_eval/agents/pi_agent.py | 2 + src/coder_eval/cli/plan_command.py | 8 +- src/coder_eval/errors/__init__.py | 3 +- src/coder_eval/errors/budget.py | 10 +- src/coder_eval/models/__init__.py | 3 +- src/coder_eval/models/harness_contract.py | 14 + src/coder_eval/models/limits.py | 44 ++- src/coder_eval/orchestration/experiment.py | 7 +- .../orchestration/resolution_checks.py | 22 ++ src/coder_eval/orchestration/turn_monitor.py | 114 +++++++- src/coder_eval/orchestrator.py | 98 +------ src/coder_eval/spi.py | 2 + tests/fixtures/harness_stubs.py | 3 +- tests/test_custom_lint.py | 2 + tests/test_harness_contract.py | 6 + tests/test_run_limits_orchestrator.py | 255 ++++++++---------- tests/test_turn_monitor.py | 196 +++++++++++++- tests/test_ungraded_reporting.py | 2 +- 28 files changed, 565 insertions(+), 285 deletions(-) create mode 100644 src/coder_eval/orchestration/resolution_checks.py diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index f937f35cd..4ac0e76c3 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -328,8 +328,9 @@ The Claude SDK's own `costUSD` is a client-side estimate assuming Anthropic pric is wrong for an open-weight model behind LiteLLM and is repriced from the token buckets at the model's real rate. The buckets are untouched, so the reconciliation invariant holds — only the cost scalar changes. An unpriced model sets the cost to `None` (an honest N/A) -**and warns**, because a silent `None` makes the orchestrator skip the `max_usd` gate with -no diagnostic. +**and warns**. When the task sets `max_usd`, the `TurnMonitor` then raises +`BudgetUnenforceableError` at the turn end, so the row finishes `ERROR` and is never a +silent skip. ## Codex rollout rebuild diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 22df940f1..53c237a66 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -99,7 +99,7 @@ at resolution, so `coder-eval plan` fails before any run. This is a JSONL CLI ag that appends a system prompt and honors `plan` and tool lists natively: ```python -from coder_eval.spi import Agent, Enforcement, HarnessContract, PermissionMode, ToolNameMap +from coder_eval.spi import Agent, Enforcement, HarnessContract, PermissionMode, ToolNameMap, UsageGranularity # native tool name -> canonical (Claude) name; also used for telemetry _TOOL_NAME_MAP = {"bash": "Bash", "read": "Read", "write": "Write", "edit": "Edit", "task": "Agent"} @@ -114,6 +114,7 @@ class MyAgent(Agent[MyAgentConfig]): allowed_tools=Enforcement.ENFORCED, disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, + usage_granularity=UsageGranularity.STEP, ) tool_names = ToolNameMap.from_inverse( _TOOL_NAME_MAP, diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index be83730d8..739ec7383 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -358,7 +358,7 @@ crash, timeout, or budget breach under `execute` reports `ERROR` / `TIMEOUT` / `TOKEN_BUDGET_EXCEEDED` and `COST_BUDGET_EXCEEDED` are produced by the cumulative budget caps under `run_limits:` (`max_input_tokens` / `max_output_tokens` / `max_total_tokens`, and `max_usd` -respectively), checked after each completed agent turn — see +respectively), enforced live by the `TurnMonitor` — see [Task Definition Guide → Run Limits](TASK_DEFINITION_GUIDE.md#run-limits). --- diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 10e7771a5..62b480251 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -297,17 +297,22 @@ model. **Budget-cap semantics:** -- **Checked after each completed agent turn**, and **cumulative** across all of the task's turns. - There is no mid-turn enforcement, so a single runaway turn can overshoot the cap before the - between-turns check sees it. Size caps with headroom for one turn. +- **Enforced live**, and **cumulative** across all of the task's turns. The `TurnMonitor` stops the + agent at its next poll once a cap is crossed. The overshoot is bounded by one usage report plus + any tool calls in flight; how often a harness reports usage is its `usage_granularity` in + [Run-Limit Parity](agents/HARNESS_PARITY.md). A harness that reports usage only once per turn is + checked at the turn end. Size caps with that headroom. - **Subject agent only.** Judge (`llm_judge` / `agent_judge`) and user-simulator token spend are **not** counted against these caps. - A breach aborts the task with `FinalStatus.TOKEN_BUDGET_EXCEEDED` (any of the three token caps) or `FinalStatus.COST_BUDGET_EXCEEDED` (`max_usd`). Both categorize as `failed` — see [Report Schema](REPORT_SCHEMA.md). -- **`max_usd` needs per-turn cost from the SDK.** If no turn reports a cost, the check is **skipped - with a one-shot warning per task**, not failed. A run can therefore blow past `max_usd` silently - on a backend that doesn't report cost — don't rely on it as your only guardrail. +- **`max_usd` is priced from the harness's reported cost**, else from the rate card in + `coder_eval.pricing` for the model the harness reports (then `agent.model`). A turn with no usage + costs nothing. A run that can price a turn neither way finishes **`ERROR`** at that turn's end with + the message "run_limits.max_usd could not be enforced". It is never skipped. Add a rate with + `register_pricing`, pin a priced model, or remove `max_usd`. Mid-turn usage reports rarely carry a + cost, so when the model has no rate the USD cap is checked once the turn's reported cost arrives. - **Cached-read and cache-creation tokens are excluded by default.** `count_cache_creation: true` is what makes an input-token budget meaningful for **Codex**, which buckets its fresh (full-price) prompt slice into `cache_creation`; with the default `false`, a Codex token budget effectively diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index db0fe1380..b94c59da7 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -32,6 +32,7 @@ Generated from each agent class's `contract` by `make parity-table`; CE069 fails | `allowed_tools` | enforced | unsupported | enforced | enforced | enforced | unsupported | | `disallowed_tools` | enforced | unsupported | enforced | enforced | enforced | unsupported | | `cooperative_stop` | yes | yes | yes | yes | yes | no | +| `usage_granularity` | generation | turn | turn | step | step | turn | | `permission_modes` | acceptEdits, bypassPermissions, default, plan | — | bypassPermissions, plan | bypassPermissions, plan | bypassPermissions, plan | — | @@ -39,6 +40,9 @@ A task that sets a field a harness marks `unsupported`, or a `permission_mode` v outside that harness's `permission_modes`, is rejected at resolution and `coder-eval plan` exits non-zero. `system_prompt_semantics` `append` / `replace` mean the system or developer instruction channel of the model request, never the user turn. +`usage_granularity` is how often a harness reports token usage on the stream (per model +generation, per agent-loop step, or once per `communicate()`); a token or USD budget can +overshoot by one such report. ### Tool names diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 24b408042..97f74ce75 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -53,6 +53,7 @@ ToolNameMap, TranscriptMessage, TurnRecord, + UsageGranularity, ) from coder_eval.pricing import calculate_cost from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback @@ -208,6 +209,7 @@ class AntigravityAgent(Agent[AntigravityAgentConfig]): allowed_tools=Enforcement.ENFORCED, disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, + usage_granularity=UsageGranularity.TURN, permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), ) tool_names = _TOOL_NAMES diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index a56eb479e..aed3345ef 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -61,6 +61,7 @@ ToolNameMap, TranscriptMessage, TurnRecord, + UsageGranularity, to_bedrock_inference_profile, ) from coder_eval.models import ( @@ -695,6 +696,7 @@ class ClaudeCodeAgent(Agent[ClaudeCodeAgentConfig]): allowed_tools=Enforcement.ENFORCED, disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, + usage_granularity=UsageGranularity.GENERATION, permission_modes=frozenset(PermissionMode), ) tool_names = ToolNameMap(names={name: (name,) for name in CANONICAL_TOOL_NAMES}, mcp_names=True) @@ -1491,18 +1493,15 @@ def _reprice_for_litellm(usage: TokenUsage, model: str | None) -> None: LiteLLM. The token buckets are left UNTOUCHED, so the reconciliation invariant is unaffected — only the cost scalar changes. - An unpriced model sets the cost to ``None`` **and warns**: a silent - ``None`` makes the orchestrator skip the ``max_usd`` gate with no - diagnostic. + An unpriced model sets the cost to ``None`` **and warns**, so the log names + the model when a ``max_usd`` task then finishes ``ERROR``. Rationale: .claude/notes/agents.md § Cost: the stream versus the rate card """ cost = ClaudeCodeAgent._price_from_buckets(usage, model) usage.total_cost_usd = cost if cost is None: - logger.warning( - "No pricing for litellm model %r; turn cost left unset (max_usd gate will be skipped)", model - ) + logger.warning("No pricing for litellm model %r; turn cost left unset", model) def get_sdk_options(self) -> dict[str, Any] | None: """Get the raw SDK options used for the last agent query. diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 8473ca7b7..09ae21546 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -38,6 +38,7 @@ TokenUsage, TranscriptMessage, TurnRecord, + UsageGranularity, ) from coder_eval.pricing import calculate_cost from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback @@ -750,6 +751,7 @@ class CodexAgent(Agent[CodexAgentConfig]): allowed_tools=Enforcement.UNSUPPORTED, disallowed_tools=Enforcement.UNSUPPORTED, cooperative_stop=True, + usage_granularity=UsageGranularity.TURN, ) def __init__( diff --git a/src/coder_eval/agents/noop_agent.py b/src/coder_eval/agents/noop_agent.py index fb3c0ef4a..c2c902f87 100644 --- a/src/coder_eval/agents/noop_agent.py +++ b/src/coder_eval/agents/noop_agent.py @@ -19,7 +19,15 @@ from coder_eval.agent import Agent, AgentState from coder_eval.agents.registry import AgentRegistry -from coder_eval.models import AgentKind, ApiRoute, Enforcement, HarnessContract, NoneAgentConfig, TurnRecord +from coder_eval.models import ( + AgentKind, + ApiRoute, + Enforcement, + HarnessContract, + NoneAgentConfig, + TurnRecord, + UsageGranularity, +) from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( @@ -52,6 +60,7 @@ class NoOpAgent(Agent[NoneAgentConfig]): allowed_tools=Enforcement.UNSUPPORTED, disallowed_tools=Enforcement.UNSUPPORTED, cooperative_stop=False, + usage_granularity=UsageGranularity.TURN, ) def __init__( diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index 5c0249d3a..c62db7256 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -51,6 +51,7 @@ ToolNameMap, TranscriptMessage, TurnRecord, + UsageGranularity, ) from coder_eval.pricing import calculate_cost from coder_eval.streaming.callbacks import StreamCallback, safe_emit @@ -762,6 +763,7 @@ class OpenCodeAgent(Agent[OpenCodeAgentConfig]): allowed_tools=Enforcement.ENFORCED, disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, + usage_granularity=UsageGranularity.STEP, permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), ) tool_names = _TOOL_NAMES diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 60c956ca6..1fbef6f47 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -59,6 +59,7 @@ ToolNameMap, TranscriptMessage, TurnRecord, + UsageGranularity, ) from coder_eval.pricing import calculate_cost from coder_eval.streaming.callbacks import StreamCallback, safe_emit @@ -684,6 +685,7 @@ class PiAgent(Agent[PiAgentConfig]): allowed_tools=Enforcement.ENFORCED, disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, + usage_granularity=UsageGranularity.STEP, permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), ) tool_names = _TOOL_NAMES diff --git a/src/coder_eval/cli/plan_command.py b/src/coder_eval/cli/plan_command.py index 879bc4d7a..8039f4e45 100644 --- a/src/coder_eval/cli/plan_command.py +++ b/src/coder_eval/cli/plan_command.py @@ -72,9 +72,9 @@ def run_plan(*, task_files: list[Path] | None = None, experiment: Path | None = check_api_keys() # Lazy import to avoid circular dependency at module level - from ..orchestration.early_stop import validate_early_stop from ..orchestration.experiment import DEFAULT_EXPERIMENT_PATH, load_experiment, resolve_task_for_variant - from ..orchestration.harness_contract import TaskResolutionError, validate_harness_contract + from ..orchestration.harness_contract import TaskResolutionError + from ..orchestration.resolution_checks import validate_resolved_task from ..orchestration.run_limits import validate_run_limits # Always load experiment (defaults to experiments/default.yaml) @@ -142,9 +142,7 @@ def run_plan(*, task_files: list[Path] | None = None, experiment: Path | None = for variant in exp_def.variants: try: resolved, _lineage, _ = resolve_task_for_variant(default_exp, task, exp_def, variant) - # Early-stop guardrails (no-op unless a criterion carries a stop_early: block). - validate_early_stop(resolved) - validate_harness_contract(resolved) + validate_resolved_task(resolved) for message in validate_run_limits(resolved): console.print( f" [yellow]⚠[/yellow] [yellow]Variant '{variant.variant_id}': {message}[/yellow]" diff --git a/src/coder_eval/errors/__init__.py b/src/coder_eval/errors/__init__.py index df5f9e429..5ce73afbd 100644 --- a/src/coder_eval/errors/__init__.py +++ b/src/coder_eval/errors/__init__.py @@ -8,7 +8,7 @@ """ from .agent import AgentConfigError, AgentCrashError, format_timeout_reason, truncate_crash_message -from .budget import BudgetExceededError +from .budget import BudgetExceededError, BudgetUnenforceableError from .checker_misuse import CheckerMisuseError from .judge import JudgeInfrastructureError from .reference import ReferenceTamperedError @@ -19,6 +19,7 @@ "AgentConfigError", "AgentCrashError", "BudgetExceededError", + "BudgetUnenforceableError", "CheckerMisuseError", "EvaluationTimeoutError", "JudgeInfrastructureError", diff --git a/src/coder_eval/errors/budget.py b/src/coder_eval/errors/budget.py index b81bb875c..3ae329ff7 100644 --- a/src/coder_eval/errors/budget.py +++ b/src/coder_eval/errors/budget.py @@ -4,7 +4,7 @@ class BudgetExceededError(Exception): - """Raised when a RunLimits budget is exceeded between agent turns. + """Raised when a RunLimits budget is exceeded. Carries which budget tripped and the over-budget value so the orchestrator can record the status reason without re-computing. @@ -26,3 +26,11 @@ def __init__( self.iteration = iteration suffix = f" (iteration {iteration})" if iteration is not None else "" super().__init__(f"{budget_name} budget exceeded: {actual:g} > {limit:g}{suffix}") + + +class BudgetUnenforceableError(Exception): + """Raised when ``run_limits.max_usd`` is set but the run can price no turn it ran. + + The harness reported no cost and ``agent.model`` has no rate card entry, so the + budget could not be enforced. An eval-config error: the run finalizes ``ERROR``. + """ diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index 2df3fbc64..a999658fc 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -94,7 +94,7 @@ ) # Harness contract -from coder_eval.models.harness_contract import Enforcement, HarnessContract, ToolNameMap +from coder_eval.models.harness_contract import Enforcement, HarnessContract, ToolNameMap, UsageGranularity # Judge from coder_eval.models.judge import JudgeVerdict @@ -253,6 +253,7 @@ "Enforcement", "HarnessContract", "ToolNameMap", + "UsageGranularity", # Enums "AgentKind", "AgentState", diff --git a/src/coder_eval/models/harness_contract.py b/src/coder_eval/models/harness_contract.py index 06c56c27b..4d9e9e072 100644 --- a/src/coder_eval/models/harness_contract.py +++ b/src/coder_eval/models/harness_contract.py @@ -19,6 +19,14 @@ class Enforcement(StrEnum): UNSUPPORTED = "unsupported" +class UsageGranularity(StrEnum): + """How often a harness reports token usage on the event stream, which bounds a budget's overshoot.""" + + GENERATION = "generation" + STEP = "step" + TURN = "turn" + + class HarnessContract(BaseModel): """The per-agent declaration of which uniform fields reach the harness. @@ -42,6 +50,12 @@ class HarnessContract(BaseModel): allowed_tools: Enforcement = Field(description="Whether agent.allowed_tools restricts the harness's tools.") disallowed_tools: Enforcement = Field(description="Whether agent.disallowed_tools denies the harness's tools.") cooperative_stop: bool = Field(description="Whether communicate() honors the should_stop poll.") + usage_granularity: UsageGranularity = Field( + description=( + "How often TurnEndEvent.tokens reports usage: per model generation, per agent-loop step, or once " + "per communicate() call. A budget can overshoot by one such report." + ) + ) permission_modes: frozenset[PermissionMode] | None = Field( default=None, description=( diff --git a/src/coder_eval/models/limits.py b/src/coder_eval/models/limits.py index 188e720ce..e893e148b 100644 --- a/src/coder_eval/models/limits.py +++ b/src/coder_eval/models/limits.py @@ -2,11 +2,15 @@ from __future__ import annotations -from typing import Final +from typing import TYPE_CHECKING, Final from pydantic import BaseModel, ConfigDict, Field +if TYPE_CHECKING: + from coder_eval.models.telemetry import TokenUsage + + DEFAULT_STOP_EARLY_GATE_THRESHOLD: Final[float] = 1.0 """Default ``stop_early_gate_threshold``: reproduces strict-AND gating exactly. @@ -14,18 +18,19 @@ fallback, and the orchestrator's finalize fallback can never drift apart. """ +_BUDGET_OVERSHOOT: Final[str] = ( + "Enforced live by the TurnMonitor; overshoot is soft by one usage report (see usage_granularity in " + "docs/agents/HARNESS_PARITY.md) plus any calls in flight. None = unlimited." +) + class RunLimits(BaseModel): """Run-time caps on a task. Unifies structural caps (max_tool_calls, task_timeout, turn_timeout) and - budget caps (tokens, USD). The tool-call cap stops the task at the agent's - next poll boundary and is cumulative across every turn. Budget caps are - checked after each completed agent turn and are cumulative across all - turns of a single task: a single-iteration task finishes and is then - marked over budget, and a dialog stops after the turn that crossed the - budget. Budgets apply to the subject agent only — judge and simulator - token spend are not counted. + budget caps (tokens, USD). Structural caps and budget caps stop the task at + the agent's next poll boundary; both are cumulative across every turn of the + task and apply to the subject agent only. Any subset of fields is valid; an empty block is legal. """ @@ -65,25 +70,27 @@ class RunLimits(BaseModel): max_input_tokens: int | None = Field( default=None, ge=1, - description="Max cumulative input (prompt) tokens. None = unlimited.", + description="Max cumulative input (prompt) tokens across the task. " + _BUDGET_OVERSHOOT, ) max_output_tokens: int | None = Field( default=None, ge=1, - description="Max cumulative output (completion) tokens. None = unlimited.", + description="Max cumulative output (completion) tokens across the task. " + _BUDGET_OVERSHOOT, ) max_total_tokens: int | None = Field( default=None, ge=1, - description="Max cumulative input+output tokens. None = unlimited.", + description="Max cumulative input+output tokens across the task. " + _BUDGET_OVERSHOOT, ) max_usd: float | None = Field( default=None, gt=0.0, description=( - "Max cumulative cost in USD. Requires per-turn cost reporting " - "(SDK-provided cost). Silently skipped if cost " - "is None for every turn." + "Max cumulative cost in USD across the task. Enforced live by the TurnMonitor: priced from the " + "harness's reported cost when it reports one, else from pricing.py for the reported model or " + "agent.model. A run that can do neither finishes ERROR at that turn's end (register_pricing adds a " + "plugin rate). Overshoot is soft by one usage report (see usage_granularity in " + "docs/agents/HARNESS_PARITY.md) plus any calls in flight." ), ) count_cached_input: bool = Field( @@ -148,3 +155,12 @@ class RunLimits(BaseModel): # here. Whether a task is armed lives on the criteria, which RunLimits cannot # see, and post-merge is the only place with enough visibility. # Rationale: .claude/notes/orchestration.md § Why the guardrails are not model validators + + def budgeted_tokens(self, usage: TokenUsage) -> tuple[int, int, int]: + """(input, output, total) as this block counts them: cache buckets join input only when flagged.""" + input_tokens = usage.uncached_input_tokens + if self.count_cache_creation: + input_tokens += usage.cache_creation_input_tokens + if self.count_cached_input: + input_tokens += usage.cache_read_input_tokens + return input_tokens, usage.output_tokens, input_tokens + usage.output_tokens diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index b0a4c287e..5214167e5 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -673,8 +673,8 @@ def resolve_all_tasks( Raises: ValueError: If duplicate task IDs are found after resolution. """ - from .early_stop import validate_early_stop - from .harness_contract import TaskResolutionError, validate_harness_contract + from .harness_contract import TaskResolutionError + from .resolution_checks import validate_resolved_task resolved: list[ResolvedTask] = [] skipped: list[SkippedTask] = [] @@ -744,8 +744,7 @@ def resolve_all_tasks( # Once the task is fully resolved, so the -D kill switch is # already merged. No-op unless armed. - validate_early_stop(resolved_task) - validate_harness_contract(resolved_task) + validate_resolved_task(resolved_task) # Fan-out: simulation n_trials takes precedence over experiment repeats # when simulation is active; otherwise use experiment-level repeats. diff --git a/src/coder_eval/orchestration/resolution_checks.py b/src/coder_eval/orchestration/resolution_checks.py new file mode 100644 index 000000000..3d456d3ef --- /dev/null +++ b/src/coder_eval/orchestration/resolution_checks.py @@ -0,0 +1,22 @@ +"""The one list of resolution-time checks every path runs on a fully merged task.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from coder_eval.orchestration.early_stop import validate_early_stop +from coder_eval.orchestration.harness_contract import validate_harness_contract + + +if TYPE_CHECKING: + from coder_eval.models import TaskDefinition + + +def validate_resolved_task(task: TaskDefinition) -> None: + """Every resolution-time rejection, in raise order: early stop, harness contract. + + Raises: + TaskResolutionError: the task asks for something the run cannot honor. + """ + validate_early_stop(task) + validate_harness_contract(task) diff --git a/src/coder_eval/orchestration/turn_monitor.py b/src/coder_eval/orchestration/turn_monitor.py index 776bc338a..932d59856 100644 --- a/src/coder_eval/orchestration/turn_monitor.py +++ b/src/coder_eval/orchestration/turn_monitor.py @@ -1,4 +1,4 @@ -"""The run's single ``should_stop`` answerer: armed early stop and the tool-call cap. +"""The run's single ``should_stop`` answerer: armed early stop, the tool-call cap and the budgets. ``TurnMonitor`` is a ``StreamCallback`` composed into the agent's callback chain for the whole task. It owns ONE ``EventCollector`` across every retry attempt and @@ -6,7 +6,9 @@ polls ``should_stop`` at its safe boundaries; the first non-None ``StopReason`` is latched and final. -Precedence on one round: ``EARLY_CRITERION`` then ``TOOL_CALL_CAP``. +Precedence on one round: ``EARLY_CRITERION``, ``TOOL_CALL_CAP``, ``TOKEN_BUDGET``, +``USD_BUDGET``. A budget breach seen mid-turn latches its reason and its figures, so a +turn that stopped on a budget always finalizes as that budget's status. FAIL-OPEN covers the armed criteria only: any exception while reducing an event or evaluating them disarms them and the run degrades to a full run. The cap reads @@ -18,9 +20,11 @@ from __future__ import annotations import logging +import math import time from typing import TYPE_CHECKING, Any +from coder_eval.errors import BudgetExceededError, BudgetUnenforceableError from coder_eval.models import ( DEFAULT_STOP_EARLY_GATE_THRESHOLD, EarlyStopInfo, @@ -32,6 +36,7 @@ TokenUsage, ) from coder_eval.orchestration.early_stop import early_stop_active +from coder_eval.pricing import calculate_cost from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( AgentEndEvent, @@ -74,6 +79,7 @@ def __init__( armed: list[_ArmedPair], *, limits: RunLimits | None, + model: str | None = None, gate_threshold: float = DEFAULT_STOP_EARLY_GATE_THRESHOLD, ) -> None: self._task_id = task_id @@ -115,13 +121,18 @@ def __init__( # but on a non-fanned task this is dead config. logger.warning("[%s] all armed stop triggers are inert for this row; run cannot stop early", task_id) self._limits = limits + self._model = model + self._reported_model: str | None = None self._collector = EventCollector() self._resolved_tool_ids: set[str] = set() self._sdk_turn_index = 0 self._tool_call_index = 0 self._started_monotonic: float | None = None self._committed = TokenUsage() + self._committed_cost = 0.0 + self._unpriced_turn = False self._in_flight = TokenUsage() + self._budget_breach: tuple[str, float, float] | None = None # Once an entry leaves "undecided" on a RESOLVED round its checker is # never polled again. `_budget_expired` marks a latched fail as # timeout-driven, reported as DECISION_BUDGET_EXCEEDED. @@ -157,7 +168,8 @@ def for_task(cls, task: TaskDefinition, *, arm: bool) -> TurnMonitor: ] limits = task.run_limits gate_threshold = limits.stop_early_gate_threshold if limits is not None else DEFAULT_STOP_EARLY_GATE_THRESHOLD - return cls(task.task_id, armed, limits=limits, gate_threshold=gate_threshold) + model = task.agent.model if task.agent is not None else None + return cls(task.task_id, armed, limits=limits, model=model, gate_threshold=gate_threshold) def on_event(self, event: StreamEvent) -> None: """Reduce one event; an unexpected exception disarms the criteria and never stops the counters.""" @@ -185,14 +197,18 @@ def _on_event_impl(self, event: StreamEvent) -> None: if self._started_monotonic is None: self._started_monotonic = time.monotonic() self._in_flight = TokenUsage() + self._reported_model = event.model or self._reported_model elif isinstance(event, TurnStartEvent): self._sdk_turn_index += 1 + self._reported_model = event.model or self._reported_model elif isinstance(event, TurnEndEvent): if event.tokens is not None: self._in_flight += event.tokens + self._evaluate_budgets() elif isinstance(event, AgentEndEvent): - self._committed += event.usage - self._in_flight = TokenUsage() + self._reported_model = event.model_used or self._reported_model + self._commit(event.usage) + self._evaluate_budgets() elif isinstance(event, ToolStartEvent): self._evaluate_armed(in_flight=event.tool) return @@ -241,6 +257,94 @@ def usage(self) -> TokenUsage: """Committed usage from every finished ``communicate()`` plus the in-flight deltas.""" return self._committed + self._in_flight + def cost_usd(self) -> float | None: + """Cumulative USD: every finished turn priced on its own, plus the priceable in-flight deltas. + + A turn is priced from its reported cost, else from the rate card for the model + the harness reported, else for ``agent.model``; a turn with no usage costs 0. + ``None`` once any finished turn could be priced none of these ways. + """ + if self._unpriced_turn: + return None + return self._committed_cost + (self._price(self._in_flight) or 0.0) + + def raise_if_over_budget(self, *, iteration: int) -> None: + """Raise when the task is over a budget, or its ``max_usd`` could not be enforced. + + Raises: + BudgetExceededError: a budget reason latched mid-turn (with the figures + from that moment), or the finished turns' totals breach a budget. + BudgetUnenforceableError: ``max_usd`` is set and a finished turn was unpriceable. + """ + breach = self._budget_breach if self._budget_breach is not None else self._breach() + if breach is not None: + name, actual, limit = breach + raise BudgetExceededError(name, actual=actual, limit=limit, task_id=self._task_id, iteration=iteration) + if self._limits is not None and self._limits.max_usd is not None and self._unpriced_turn: + raise BudgetUnenforceableError( + "run_limits.max_usd could not be enforced: the harness reported no cost and " + + f"agent.model {self._model!r} (reported {self._reported_model!r}) has no rate in " + + "coder_eval.pricing (register one with " + + "register_pricing, pin a priced model, or remove max_usd)" + ) + + def _commit(self, usage: TokenUsage) -> None: + self._committed += usage + cost = self._price(usage) + if cost is None: + self._unpriced_turn = True + else: + self._committed_cost += cost + self._in_flight = TokenUsage() + + def _price(self, usage: TokenUsage) -> float | None: + if usage.total_cost_usd is not None: + return usage.total_cost_usd if math.isfinite(usage.total_cost_usd) else None + if usage.is_empty(): + return 0.0 + for model in (self._reported_model, self._model): + if model is None: + continue + cost = calculate_cost( + model, + usage.uncached_input_tokens, + usage.output_tokens, + usage.cache_creation_input_tokens, + usage.cache_read_input_tokens, + ) + if cost is not None: + return cost + return None + + def _breach(self) -> tuple[str, float, float] | None: + """The first budget over its cap as ``(budget name, actual, limit)``: input, output, total, usd.""" + limits = self._limits + if limits is None: + return None + input_tokens, output_tokens, total_tokens = limits.budgeted_tokens(self.usage) + for name, actual, limit in ( + ("input_tokens", input_tokens, limits.max_input_tokens), + ("output_tokens", output_tokens, limits.max_output_tokens), + ("total_tokens", total_tokens, limits.max_total_tokens), + ): + if limit is not None and actual > limit: + return name, actual, limit + cost = self.cost_usd() if limits.max_usd is not None else None + if limits.max_usd is not None and cost is not None and cost > limits.max_usd: + return "usd", cost, limits.max_usd + return None + + def _evaluate_budgets(self) -> None: + if self._stop_reason is not None: + return + breach = self._breach() + if breach is None: + return + self._budget_breach = breach + name, actual, limit = breach + logger.info("[%s] %s budget reached: %g > %g", self._task_id, name, actual, limit) + self._latch(StopReason.USD_BUDGET if name == "usd" else StopReason.TOKEN_BUDGET) + def _latch(self, reason: StopReason) -> None: if self._stop_reason is None: self._stop_reason = reason diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 1f9356ab1..2fcf8739d 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -67,7 +67,7 @@ ) from .orchestration.early_stop import early_stop_active, validate_early_stop from .orchestration.evaluation import resolve_reference_dir, stage_reference_dir -from .orchestration.harness_contract import validate_harness_contract +from .orchestration.resolution_checks import validate_resolved_task from .orchestration.run_limits import validate_run_limits from .orchestration.turn_monitor import TurnMonitor from .path_utils import ( @@ -476,10 +476,6 @@ def __init__( # count it answers the should_stop poll from is cumulative per task. self._monitor: TurnMonitor | None = None - # One-shot flag: emit the "cost budget configured but no cost data" warning - # exactly once per task even if _check_run_limits fires every turn. - self._cost_budget_skipped_logged: bool = False - # One-shot flag: emit the expected_tool_calls rollup warning exactly once per # task run even though _check_expected_tool_calls is called after every turn. self._expected_tool_calls_warning_emitted: bool = False @@ -1079,14 +1075,6 @@ def _finalize_result(self, start_time: float) -> None: # Aggregate token usage self._aggregate_token_usage() - # Record whether per-turn cost data was available when a cost budget was set. - # Lets users audit whether a configured max_usd budget was actually enforceable. - if self.task.run_limits is not None and self.task.run_limits.max_usd is not None: - any_cost_reported = any( - t.token_usage is not None and t.token_usage.total_cost_usd is not None for t in self.result.iterations - ) - self.result.environment_info["cost_data_available"] = any_cost_reported - if self.result.iterations: self.result.total_assistant_turns = sum(t.assistant_turn_count for t in self.result.iterations) @@ -1160,74 +1148,6 @@ def _finalize_result(self, start_time: float) -> None: write_task_html(self.result, self.html_report_path) - def _check_run_limits(self, *, iteration: int) -> None: - """Raise BudgetExceededError if any RunLimits budget is exceeded. - - Called after each completed turn. Aggregates across self.result.iterations. - No-op when self.task.run_limits is None. - """ - assert self.result is not None - limits = self.task.run_limits - if limits is None: - return - - usages = [t.token_usage for t in self.result.iterations if t.token_usage is not None] - if not usages: - return - - input_tokens = sum(u.uncached_input_tokens for u in usages) - if limits.count_cache_creation: - input_tokens += sum(u.cache_creation_input_tokens for u in usages) - if limits.count_cached_input: - input_tokens += sum(u.cache_read_input_tokens for u in usages) - output_tokens = sum(u.output_tokens for u in usages) - total_tokens = input_tokens + output_tokens - - if limits.max_input_tokens is not None and input_tokens > limits.max_input_tokens: - raise BudgetExceededError( - "input_tokens", - actual=input_tokens, - limit=limits.max_input_tokens, - task_id=self.task.task_id, - iteration=iteration, - ) - if limits.max_output_tokens is not None and output_tokens > limits.max_output_tokens: - raise BudgetExceededError( - "output_tokens", - actual=output_tokens, - limit=limits.max_output_tokens, - task_id=self.task.task_id, - iteration=iteration, - ) - if limits.max_total_tokens is not None and total_tokens > limits.max_total_tokens: - raise BudgetExceededError( - "total_tokens", - actual=total_tokens, - limit=limits.max_total_tokens, - task_id=self.task.task_id, - iteration=iteration, - ) - - if limits.max_usd is not None: - costs = [u.total_cost_usd for u in usages if u.total_cost_usd is not None] - if not costs: - if not self._cost_budget_skipped_logged: - logger.warning( - "[%s] max_usd budget configured but no turn reported cost; skipping cost check", - self.task.task_id, - ) - self._cost_budget_skipped_logged = True - return - total_cost = sum(costs) - if total_cost > limits.max_usd: - raise BudgetExceededError( - "usd", - actual=total_cost, - limit=limits.max_usd, - task_id=self.task.task_id, - iteration=iteration, - ) - def _check_expected_tool_calls(self, *, iteration: int) -> None: """Emit a one-shot warning if visible tool calls exceed expected_tool_calls. @@ -1495,7 +1415,7 @@ async def _setup(self) -> None: # After the evaluate-only return: a re-grade builds no agent, so a recorded # config from before the contract existed stays gradable. - validate_harness_contract(self.task) + validate_resolved_task(self.task) # validate_api_keys exempts the no-op agent internally — it makes no API # call, so it needs no agent keys. @@ -2195,8 +2115,9 @@ async def _evaluation_loop(self) -> bool: # Record early-stop info (if the monitor tripped) BEFORE check_all_async, so it # survives even if a checker raises. None on a full run or when unarmed. - assert self._monitor is not None - self.result.early_stop = self._monitor.info + monitor = self._monitor + assert monitor is not None + self.result.early_stop = monitor.info logger.debug(f"Agent response received ({len(turn_record.agent_output)} chars)") @@ -2212,7 +2133,7 @@ async def _evaluation_loop(self) -> bool: self.result.tool_calls_exhausted = True logger.warning( "Agent reached the tool-call cap (%d resolved tool calls).", - self._monitor.tool_calls, + monitor.tool_calls, ) # Soft cumulative-turn check (logs once; never aborts). self._check_expected_tool_calls(iteration=iteration) @@ -2226,7 +2147,7 @@ async def _evaluation_loop(self) -> bool: logger.info("Grading disabled (execute mode): skipping success criteria.") # A run limit, not a verdict: its only reason to sit after the # criteria on the graded path is partial-credit visibility. - self._check_run_limits(iteration=iteration) + monitor.raise_if_over_budget(iteration=iteration) return False # Check success criteria (reference_dir feeds reference_comparison + judges) @@ -2257,7 +2178,7 @@ async def _evaluation_loop(self) -> bool: self._emit_criteria_event(criteria_results) # AFTER the criteria, so partial-credit visibility is preserved. - self._check_run_limits(iteration=iteration) + monitor.raise_if_over_budget(iteration=iteration) return all_passed @@ -2582,8 +2503,9 @@ async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir: # Budget gate: aborts the dialog with a dedicated stop reason and # ensures end-of-dialog criteria still run for partial credit. + assert self._monitor is not None try: - self._check_run_limits(iteration=turns_completed) + self._monitor.raise_if_over_budget(iteration=turns_completed) except BudgetExceededError: stop_reason = DialogStopReason.RUN_LIMIT_EXCEEDED if not criteria_checked_this_turn: diff --git a/src/coder_eval/spi.py b/src/coder_eval/spi.py index c3cdec7ca..6082ac398 100644 --- a/src/coder_eval/spi.py +++ b/src/coder_eval/spi.py @@ -27,6 +27,7 @@ ToolNameMap, TranscriptMessage, TurnRecord, + UsageGranularity, ) from coder_eval.pricing import ModelPricing, register_pricing from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback @@ -89,6 +90,7 @@ "TurnRecord", "TurnStartEvent", "TurnTimeoutError", + "UsageGranularity", "close_window", "end_status_for", "register_pricing", diff --git a/tests/fixtures/harness_stubs.py b/tests/fixtures/harness_stubs.py index 6d4bbe25b..6cd5be52b 100644 --- a/tests/fixtures/harness_stubs.py +++ b/tests/fixtures/harness_stubs.py @@ -6,7 +6,7 @@ from pydantic import create_model -from coder_eval.models import BaseAgentConfig, Enforcement, HarnessContract +from coder_eval.models import BaseAgentConfig, Enforcement, HarnessContract, UsageGranularity def stub_contract(*, cooperative_stop: bool = True) -> HarnessContract: @@ -19,6 +19,7 @@ def stub_contract(*, cooperative_stop: bool = True) -> HarnessContract: allowed_tools=Enforcement.UNSUPPORTED, disallowed_tools=Enforcement.UNSUPPORTED, cooperative_stop=cooperative_stop, + usage_granularity=UsageGranularity.TURN, ) diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 699f3de26..676b08302 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -2191,6 +2191,8 @@ def test_render_pins_the_header_and_a_known_cell(self): assert lines[0] == "| field | claude-code | codex | antigravity | opencode | pi | none |" system_prompt = next(line for line in lines if line.startswith("| `system_prompt` |")) assert system_prompt.split(" | ")[5] == "enforced" + granularity = next(line for line in lines if line.startswith("| `usage_granularity` |")) + assert granularity == "| `usage_granularity` | generation | turn | turn | step | step | turn |" @pytest.mark.lint diff --git a/tests/test_harness_contract.py b/tests/test_harness_contract.py index ba8829cc2..5772e4539 100644 --- a/tests/test_harness_contract.py +++ b/tests/test_harness_contract.py @@ -80,6 +80,12 @@ def test_contract_is_frozen(self) -> None: with pytest.raises(ValidationError): contract.cooperative_stop = False # type: ignore[misc] + def test_usage_granularity_is_required(self) -> None: + fields = stub_contract().model_dump() + del fields["usage_granularity"] + with pytest.raises(ValidationError, match="usage_granularity"): + HarnessContract(**fields) + def test_unknown_field_rejected(self) -> None: with pytest.raises(ValidationError, match="timing_basis"): HarnessContract(**{**stub_contract().model_dump(), "timing_basis": "wall"}) diff --git a/tests/test_run_limits_orchestrator.py b/tests/test_run_limits_orchestrator.py index c2b8fc9d8..18ad333d3 100644 --- a/tests/test_run_limits_orchestrator.py +++ b/tests/test_run_limits_orchestrator.py @@ -9,7 +9,7 @@ import pytest -from coder_eval.errors import BudgetExceededError +from coder_eval.errors import BudgetExceededError, BudgetUnenforceableError from coder_eval.models import ( DEFAULT_SIMULATOR_MODEL, AgentKind, @@ -26,6 +26,7 @@ TurnRecord, ) from coder_eval.orchestrator import Orchestrator +from coder_eval.streaming.events import AgentEndEvent, AgentStartEvent def _make_task(*, run_limits: RunLimits | None = None) -> TaskDefinition: @@ -85,6 +86,24 @@ def _make_turn( ) +def _reporting_agent(*turns: TurnRecord) -> AsyncMock: + """A fake agent whose each ``communicate`` reports its turn's usage on the stream, as real agents do.""" + remaining = list(turns) + + async def communicate(user_input, *, stream_callback=None, timeout=None, should_stop=None): + turn = remaining.pop(0) if len(remaining) > 1 else remaining[0] + assert stream_callback is not None + stream_callback.on_event(AgentStartEvent(task_id="budget_test", prompt=user_input)) + stream_callback.on_event(AgentEndEvent(task_id="budget_test", usage=turn.token_usage or TokenUsage())) + return turn + + agent = AsyncMock() + agent.communicate = communicate + agent.get_sdk_options = MagicMock(return_value={}) + agent.get_environment_info = MagicMock(return_value={}) + return agent + + def _make_orchestrator(task: TaskDefinition, tmp_path) -> Orchestrator: run_dir = tmp_path / "run" / "budget_test" run_dir.mkdir(parents=True) @@ -108,91 +127,6 @@ def _make_orchestrator(task: TaskDefinition, tmp_path) -> Orchestrator: return orchestrator -class TestCheckRunLimitsUnit: - """Direct tests of the _check_run_limits helper.""" - - def test_noop_when_no_limits(self, tmp_path): - orch = _make_orchestrator(_make_task(), tmp_path) - orch.result.iterations.append(_make_turn(input_tokens=100)) - # Should not raise. - orch._check_run_limits(iteration=1) - - def test_noop_when_no_turns(self, tmp_path): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_total_tokens=1)), tmp_path) - # No turns recorded yet — no usage to check. - orch._check_run_limits(iteration=0) - - def test_input_token_trip(self, tmp_path): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_input_tokens=1000)), tmp_path) - orch.result.iterations.append(_make_turn(input_tokens=2000)) - with pytest.raises(BudgetExceededError) as exc: - orch._check_run_limits(iteration=1) - assert exc.value.budget_name == "input_tokens" - assert exc.value.actual == 2000 - assert exc.value.limit == 1000 - - def test_output_token_trip(self, tmp_path): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_output_tokens=1000)), tmp_path) - orch.result.iterations.append(_make_turn(output_tokens=2000)) - with pytest.raises(BudgetExceededError) as exc: - orch._check_run_limits(iteration=1) - assert exc.value.budget_name == "output_tokens" - - def test_total_token_trip(self, tmp_path): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_total_tokens=2500)), tmp_path) - orch.result.iterations.append(_make_turn(input_tokens=1500, output_tokens=1500)) - with pytest.raises(BudgetExceededError) as exc: - orch._check_run_limits(iteration=1) - assert exc.value.budget_name == "total_tokens" - - def test_cost_trip(self, tmp_path): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_usd=0.10)), tmp_path) - orch.result.iterations.append(_make_turn(input_tokens=10, total_cost_usd=0.20)) - with pytest.raises(BudgetExceededError) as exc: - orch._check_run_limits(iteration=1) - assert exc.value.budget_name == "usd" - assert exc.value.actual == pytest.approx(0.20) - - def test_cost_skipped_when_no_cost_reported(self, tmp_path, caplog): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_usd=0.10)), tmp_path) - # turn has token_usage but no total_cost_usd - orch.result.iterations.append(_make_turn(input_tokens=10, total_cost_usd=None)) - with caplog.at_level(logging.WARNING, logger="coder_eval.orchestrator"): - orch._check_run_limits(iteration=1) - assert any("max_usd budget configured but no turn reported cost" in m for m in caplog.messages) - - def test_cost_warning_only_once(self, tmp_path, caplog): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_usd=0.10)), tmp_path) - orch.result.iterations.append(_make_turn(input_tokens=10, total_cost_usd=None)) - with caplog.at_level(logging.WARNING, logger="coder_eval.orchestrator"): - orch._check_run_limits(iteration=1) - orch._check_run_limits(iteration=2) - warns = [m for m in caplog.messages if "max_usd budget configured" in m] - assert len(warns) == 1 - - def test_count_cached_input_false_default(self, tmp_path): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_input_tokens=1000)), tmp_path) - orch.result.iterations.append(_make_turn(input_tokens=500, cache_read_input_tokens=600)) - # 500 < 1000 — cache reads don't count by default. - orch._check_run_limits(iteration=1) - - def test_count_cached_input_true(self, tmp_path): - orch = _make_orchestrator( - _make_task(run_limits=RunLimits(max_input_tokens=1000, count_cached_input=True)), tmp_path - ) - orch.result.iterations.append(_make_turn(input_tokens=500, cache_read_input_tokens=600)) - with pytest.raises(BudgetExceededError) as exc: - orch._check_run_limits(iteration=1) - assert exc.value.actual == 1100 - - def test_cumulative_across_turns(self, tmp_path): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_input_tokens=1000)), tmp_path) - orch.result.iterations.append(_make_turn(iteration=1, input_tokens=600)) - orch.result.iterations.append(_make_turn(iteration=2, input_tokens=500)) - with pytest.raises(BudgetExceededError): - orch._check_run_limits(iteration=2) - - async def _run_orchestrator( task: TaskDefinition, tmp_path, *, raising_error: BudgetExceededError | None = None ) -> EvaluationResult: @@ -222,9 +156,7 @@ class TestSingleShotEnforcement: async def _run_eval_loop_with_turn(self, task: TaskDefinition, tmp_path, turn: TurnRecord) -> EvaluationResult: orch = _make_orchestrator(task, tmp_path) - mock_agent = AsyncMock() - mock_agent.communicate = AsyncMock(return_value=turn) - orch.agent = mock_agent + orch.agent = _reporting_agent(turn) mock_checker = MagicMock() mock_checker.check_all_async = AsyncMock( @@ -248,10 +180,68 @@ async def test_under_budget_passes(self, tmp_path): async def test_input_budget_trip_records_criteria(self, tmp_path): task = _make_task(run_limits=RunLimits(max_input_tokens=10)) - turn = _make_turn(input_tokens=200) - result = await self._run_eval_loop_with_turn(task, tmp_path, turn) - # Criteria still ran before budget check (single-shot order). - assert len(result.success_criteria_results) == 1 + orch = _make_orchestrator(task, tmp_path) + orch.agent = _reporting_agent(_make_turn(input_tokens=200)) + orch.success_checker.check_all_async = AsyncMock( + return_value=[CriterionResult(criterion_type="file_exists", description="x", score=1.0)] + ) + with ( + patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None), + pytest.raises(BudgetExceededError) as exc, + ): + await orch._evaluation_loop() + assert (exc.value.budget_name, exc.value.actual, exc.value.limit) == ("input_tokens", 200, 10) + # Criteria still ran before the budget check (single-shot order). + assert len(orch.result.success_criteria_results) == 1 + + @pytest.mark.parametrize( + ("limits", "turn", "budget_name"), + [ + (RunLimits(max_output_tokens=10), {"output_tokens": 20}, "output_tokens"), + (RunLimits(max_total_tokens=25), {"input_tokens": 15, "output_tokens": 15}, "total_tokens"), + (RunLimits(max_usd=0.10), {"input_tokens": 10, "total_cost_usd": 0.20}, "usd"), + ], + ) + async def test_each_budget_trips_through_the_monitor(self, tmp_path, limits, turn, budget_name): + orch = _make_orchestrator(_make_task(run_limits=limits), tmp_path) + orch.agent = _reporting_agent(_make_turn(**turn)) + orch.success_checker.check_all_async = AsyncMock( + return_value=[CriterionResult(criterion_type="file_exists", description="x", score=1.0)] + ) + with ( + patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None), + pytest.raises(BudgetExceededError) as exc, + ): + await orch._evaluation_loop() + assert exc.value.budget_name == budget_name + + async def test_an_unpriceable_max_usd_finalizes_error_after_the_first_turn(self, tmp_path): + task = _make_task(run_limits=RunLimits(max_usd=0.10)) + run_dir = tmp_path / "run" / "unpriceable" + run_dir.mkdir(parents=True) + orch = Orchestrator(task=task, run_dir=run_dir, variant_id="v") + agent = _reporting_agent(_make_turn(input_tokens=10, total_cost_usd=None)) + + async def setup() -> None: + orch._build_monitor() + orch.sandbox = MagicMock() + orch.sandbox.sandbox_dir = tmp_path / "sandbox" + orch.sandbox.sandbox_dir.mkdir() + orch.agent = agent + orch.success_checker = MagicMock() + orch.success_checker.check_all_async = AsyncMock( + return_value=[CriterionResult(criterion_type="file_exists", description="x", score=1.0)] + ) + + orch._setup = setup # type: ignore[method-assign] + orch._cleanup = AsyncMock() # type: ignore[method-assign] + with patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None): + result = await orch.run() + + assert result.final_status == FinalStatus.ERROR + assert "run_limits.max_usd could not be enforced" in (result.error_message or "") + assert "agent.model None" in (result.error_message or "") + assert result.iteration_count == 1 async def test_complete_canonical_results_skip_post_failure_regrade(self, tmp_path): task = _make_task(run_limits=RunLimits(max_input_tokens=10)) @@ -312,53 +302,6 @@ async def test_run_arm_maps_budget_to_status( assert mock_ctx.call_args.kwargs["component"] == expected_component -class TestCostDataAvailableFlag: - """The cost_data_available flag is set on result.environment_info in _finalize_result.""" - - @staticmethod - def _invoke_finalize(orch: Orchestrator) -> None: - """Run _finalize_result with side-effecting persistence (report writes) patched out. - - Both ``write_task_html`` and ``spill_judge_transcripts`` are imported lazily - inside ``_finalize_result``, so they must be patched on their defining - modules rather than on ``coder_eval.orchestrator``. - """ - import time as _time - from unittest.mock import patch as _patch - - # The report_path lives under tmp_path so the write_text call lands - # in a real (test-scoped) file and we don't need to mock pathlib. - with ( - _patch("coder_eval.reports.write_task_html", return_value=None), - _patch("coder_eval.evaluation.judge_persistence.spill_judge_transcripts", return_value=None), - ): - orch._finalize_result(_time.time()) - - def test_flag_true_when_costs_reported(self, tmp_path): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_usd=0.5)), tmp_path) - orch.result.iterations.append(_make_turn(input_tokens=10, total_cost_usd=0.001)) - self._invoke_finalize(orch) - assert orch.result.environment_info["cost_data_available"] is True - - def test_flag_false_when_no_cost_reported(self, tmp_path): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_usd=0.5)), tmp_path) - orch.result.iterations.append(_make_turn(input_tokens=10, total_cost_usd=None)) - self._invoke_finalize(orch) - assert orch.result.environment_info["cost_data_available"] is False - - def test_flag_absent_when_no_max_usd_budget(self, tmp_path): - orch = _make_orchestrator(_make_task(run_limits=RunLimits(max_total_tokens=1000)), tmp_path) - orch.result.iterations.append(_make_turn(input_tokens=10, total_cost_usd=0.001)) - self._invoke_finalize(orch) - assert "cost_data_available" not in orch.result.environment_info - - def test_flag_absent_when_no_run_limits(self, tmp_path): - orch = _make_orchestrator(_make_task(), tmp_path) - orch.result.iterations.append(_make_turn(input_tokens=10, total_cost_usd=0.001)) - self._invoke_finalize(orch) - assert "cost_data_available" not in orch.result.environment_info - - @pytest.mark.asyncio class TestSimulationBudgetAbort: """The simulation arm raises BudgetExceededError mid-dialog and records telemetry.""" @@ -383,10 +326,7 @@ async def test_dialog_aborts_with_run_limit_stop_reason(self, tmp_path): orch = _make_orchestrator(task, tmp_path) # The agent's first turn reports tokens above the budget. - turn = _make_turn(input_tokens=200, output_tokens=10) - mock_agent = AsyncMock() - mock_agent.communicate = AsyncMock(return_value=turn) - orch.agent = mock_agent + orch.agent = _reporting_agent(_make_turn(input_tokens=200, output_tokens=10)) mock_checker = MagicMock() mock_checker.check_all_async = AsyncMock( @@ -420,6 +360,33 @@ async def test_dialog_aborts_with_run_limit_stop_reason(self, tmp_path): # Simulator must not have been asked for another message after the budget trip. mock_simulator.next_user_message.assert_not_called() + async def test_an_unpriceable_max_usd_ends_the_dialog_after_its_first_turn(self, tmp_path): + from coder_eval.models import SimulationConfig + + sim = SimulationConfig(enabled=True, persona="user", goal="g", max_turns=5, check_criteria="end_of_dialog") + task = _make_task(run_limits=RunLimits(max_usd=0.10)).model_copy( + update={"simulation": sim, "initial_prompt": "first message"} + ) + orch = _make_orchestrator(task, tmp_path) + orch.agent = _reporting_agent(_make_turn(input_tokens=200, output_tokens=10)) + orch.success_checker.check_all_async = AsyncMock(return_value=[]) + mock_simulator = MagicMock() + mock_simulator.model = DEFAULT_SIMULATOR_MODEL + mock_simulator.start = AsyncMock() + mock_simulator.stop = AsyncMock() + mock_simulator.next_user_message = AsyncMock() + + with ( + patch("coder_eval.orchestrator.UserSimulator", return_value=mock_simulator), + patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None), + pytest.raises(BudgetUnenforceableError, match="could not be enforced"), + ): + await orch._simulation_dialog_loop("first message", tmp_path / "sandbox") + + assert orch.result.simulation is not None + assert orch.result.simulation.total_turns == 1 + mock_simulator.next_user_message.assert_not_called() + class TestCheckExpectedTurnsUnit: """Direct unit tests of Orchestrator._check_expected_tool_calls.""" diff --git a/tests/test_turn_monitor.py b/tests/test_turn_monitor.py index ef0cc6113..648d73a12 100644 --- a/tests/test_turn_monitor.py +++ b/tests/test_turn_monitor.py @@ -5,6 +5,9 @@ from typing import Any from unittest.mock import patch +import pytest + +from coder_eval.errors import BudgetExceededError, BudgetUnenforceableError from coder_eval.models import ( AgentKind, CommandTelemetry, @@ -31,18 +34,34 @@ from tests._fixtures.live_criteria import FROZEN_TS -def _task(*, criteria: list[Any] | None = None, max_tool_calls: int | None = None) -> TaskDefinition: +def _task( + *, + criteria: list[Any] | None = None, + max_tool_calls: int | None = None, + limits: RunLimits | None = None, + model: str | None = None, +) -> TaskDefinition: return TaskDefinition( task_id="monitor-test", description="monitor test task", initial_prompt="do the thing", - agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE, model=model), sandbox=SandboxConfig(driver="tempdir"), success_criteria=criteria or [FileExistsCriterion(path="x", description="x exists")], - run_limits=RunLimits(max_tool_calls=max_tool_calls), + run_limits=limits if limits is not None else RunLimits(max_tool_calls=max_tool_calls), ) +def _turn(usage: TokenUsage) -> list[Any]: + return [AgentStartEvent(task_id="t"), AgentEndEvent(task_id="t", usage=usage)] + + +def _raised(monitor: TurnMonitor) -> BudgetExceededError: + with pytest.raises(BudgetExceededError) as exc: + monitor.raise_if_over_budget(iteration=1) + return exc.value + + def _skill_crit(skill: str, *, on_pass: bool) -> SkillTriggeredCriterion: return SkillTriggeredCriterion( type="skill_triggered", @@ -211,3 +230,174 @@ def test_committed_usage_accumulates_across_communicate_calls(self) -> None: [AgentStartEvent(task_id="t"), AgentEndEvent(task_id="t", usage=TokenUsage(output_tokens=4))], ) assert monitor.usage.output_tokens == 8 + + +class TestTokenBudgets: + @pytest.mark.parametrize( + ("limits", "usage", "name", "actual"), + [ + (RunLimits(max_input_tokens=100), TokenUsage(uncached_input_tokens=101), "input_tokens", 101), + (RunLimits(max_output_tokens=10), TokenUsage(output_tokens=11), "output_tokens", 11), + ( + RunLimits(max_total_tokens=100), + TokenUsage(uncached_input_tokens=60, output_tokens=41), + "total_tokens", + 101, + ), + ( + RunLimits(max_input_tokens=100, count_cached_input=True), + TokenUsage(uncached_input_tokens=50, cache_read_input_tokens=51), + "input_tokens", + 101, + ), + ( + RunLimits(max_input_tokens=100, count_cache_creation=True), + TokenUsage(uncached_input_tokens=50, cache_creation_input_tokens=51), + "input_tokens", + 101, + ), + ], + ) + def test_each_bucket_rule_breaches(self, limits: RunLimits, usage: TokenUsage, name: str, actual: int) -> None: + monitor = TurnMonitor.for_task(_task(limits=limits), arm=True) + _feed(monitor, _turn(usage)) + assert monitor.should_stop() is StopReason.TOKEN_BUDGET + error = _raised(monitor) + assert (error.budget_name, error.actual) == (name, actual) + + def test_cache_buckets_do_not_count_unless_flagged(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_input_tokens=100)), arm=True) + _feed(monitor, _turn(TokenUsage(uncached_input_tokens=50, cache_read_input_tokens=500))) + assert monitor.should_stop() is None + monitor.raise_if_over_budget(iteration=1) + + def test_the_cap_is_exclusive(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_output_tokens=10)), arm=True) + _feed(monitor, _turn(TokenUsage(output_tokens=10))) + assert monitor.should_stop() is None + + def test_an_in_flight_delta_latches_mid_turn(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_output_tokens=10)), arm=True) + _feed(monitor, [AgentStartEvent(task_id="t"), TurnEndEvent(task_id="t", tokens=TokenUsage(output_tokens=11))]) + assert monitor.should_stop() is StopReason.TOKEN_BUDGET + + def test_a_latched_breach_is_final_even_when_the_end_usage_lands_under_the_cap(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_output_tokens=10)), arm=True) + _feed( + monitor, + [ + AgentStartEvent(task_id="t"), + TurnEndEvent(task_id="t", tokens=TokenUsage(output_tokens=12)), + AgentEndEvent(task_id="t", usage=TokenUsage(output_tokens=9)), + ], + ) + error = _raised(monitor) + assert (error.budget_name, error.actual, error.limit) == ("output_tokens", 12, 10) + + def test_a_breach_seen_only_at_turn_end_is_raised_after_the_turn(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_output_tokens=10)), arm=True) + _feed(monitor, _turn(TokenUsage(output_tokens=6)) + _turn(TokenUsage(output_tokens=6))) + assert _raised(monitor).actual == 12 + + def test_the_cap_outranks_a_later_budget_breach_but_the_budget_still_raises(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_tool_calls=1, max_output_tokens=10)), arm=True) + _feed( + monitor, + [AgentStartEvent(task_id="t"), _end("a"), AgentEndEvent(task_id="t", usage=TokenUsage(output_tokens=11))], + ) + assert monitor.should_stop() is StopReason.TOOL_CALL_CAP + assert _raised(monitor).budget_name == "output_tokens" + + def test_a_token_breach_outranks_a_usd_breach(self) -> None: + limits = RunLimits(max_output_tokens=10, max_usd=0.01) + monitor = TurnMonitor.for_task(_task(limits=limits), arm=True) + _feed(monitor, _turn(TokenUsage(output_tokens=11, total_cost_usd=1.0))) + assert monitor.should_stop() is StopReason.TOKEN_BUDGET + assert _raised(monitor).budget_name == "output_tokens" + + +class TestUsdBudget: + def test_the_reported_cost_prices_the_turn(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_usd=0.10)), arm=True) + _feed(monitor, _turn(TokenUsage(output_tokens=1, total_cost_usd=0.20))) + assert monitor.should_stop() is StopReason.USD_BUDGET + error = _raised(monitor) + assert error.budget_name == "usd" + assert error.actual == pytest.approx(0.20) + + def test_the_rate_card_prices_a_turn_with_no_reported_cost(self) -> None: + task = _task(limits=RunLimits(max_usd=0.50), model="claude-haiku-4-5") + monitor = TurnMonitor.for_task(task, arm=True) + _feed(monitor, _turn(TokenUsage(uncached_input_tokens=1_000_000))) + assert monitor.cost_usd() == pytest.approx(1.0) + assert monitor.should_stop() is StopReason.USD_BUDGET + + def test_a_reported_cost_is_enough_when_the_model_is_not_on_the_card(self) -> None: + task = _task(limits=RunLimits(max_usd=0.10), model="openrouter/anthropic/claude-haiku-4.5") + monitor = TurnMonitor.for_task(task, arm=True) + _feed(monitor, _turn(TokenUsage(output_tokens=100, total_cost_usd=0.02))) + assert monitor.cost_usd() == pytest.approx(0.02) + monitor.raise_if_over_budget(iteration=1) + + def test_turns_are_priced_on_their_own_and_summed(self) -> None: + task = _task(limits=RunLimits(max_usd=10.0), model="claude-haiku-4-5") + monitor = TurnMonitor.for_task(task, arm=True) + _feed(monitor, _turn(TokenUsage(output_tokens=1, total_cost_usd=0.25))) + _feed(monitor, _turn(TokenUsage(uncached_input_tokens=1_000_000))) + assert monitor.cost_usd() == pytest.approx(1.25) + + @pytest.mark.parametrize("model", [None, "openrouter/anthropic/claude-haiku-4.5"]) + def test_an_unpriceable_turn_makes_max_usd_unenforceable(self, model: str | None) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_usd=0.10), model=model), arm=True) + _feed(monitor, _turn(TokenUsage(output_tokens=100))) + assert monitor.cost_usd() is None + assert monitor.should_stop() is None + with pytest.raises(BudgetUnenforceableError, match=r"run_limits\.max_usd could not be enforced"): + monitor.raise_if_over_budget(iteration=1) + + def test_an_empty_crashed_attempt_costs_nothing_and_does_not_poison_the_retry(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_usd=0.10)), arm=True) + _feed(monitor, [AgentStartEvent(task_id="t"), AgentEndEvent(task_id="t", crashed=True)]) + _feed(monitor, _turn(TokenUsage(output_tokens=10, total_cost_usd=0.01))) + assert monitor.cost_usd() == pytest.approx(0.01) + monitor.raise_if_over_budget(iteration=1) + + def test_the_model_the_harness_reports_prices_a_turn_when_agent_model_is_unset(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_usd=0.50)), arm=True) + _feed( + monitor, + [ + AgentStartEvent(task_id="t", model="claude-haiku-4-5"), + AgentEndEvent(task_id="t", usage=TokenUsage(uncached_input_tokens=1_000_000)), + ], + ) + assert monitor.cost_usd() == pytest.approx(1.0) + assert monitor.should_stop() is StopReason.USD_BUDGET + + def test_the_reported_model_prices_in_flight_deltas(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_usd=0.50)), arm=True) + _feed( + monitor, + [ + AgentStartEvent(task_id="t", model="claude-haiku-4-5"), + TurnEndEvent(task_id="t", tokens=TokenUsage(uncached_input_tokens=1_000_000)), + ], + ) + assert monitor.should_stop() is StopReason.USD_BUDGET + + def test_a_non_finite_reported_cost_is_unpriceable(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_usd=0.10)), arm=True) + _feed(monitor, _turn(TokenUsage(output_tokens=10, total_cost_usd=float("nan")))) + with pytest.raises(BudgetUnenforceableError): + monitor.raise_if_over_budget(iteration=1) + + def test_an_unpriceable_turn_without_max_usd_is_fine(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_output_tokens=1000)), arm=True) + _feed(monitor, _turn(TokenUsage(output_tokens=100))) + monitor.raise_if_over_budget(iteration=1) + + def test_an_unpriced_in_flight_delta_contributes_nothing(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_usd=0.10)), arm=True) + _feed(monitor, [AgentStartEvent(task_id="t"), TurnEndEvent(task_id="t", tokens=TokenUsage(output_tokens=5))]) + assert monitor.cost_usd() == 0.0 + assert monitor.should_stop() is None diff --git a/tests/test_ungraded_reporting.py b/tests/test_ungraded_reporting.py index 93450ca9c..9f00b5eca 100644 --- a/tests/test_ungraded_reporting.py +++ b/tests/test_ungraded_reporting.py @@ -86,7 +86,7 @@ def test_variant_aggregate_has_no_pass_rate_when_the_only_non_ungraded_row_is_a_ """The bug the shared helper exists for. TIMEOUT and the two budget stops are category ``failed`` and reachable under - ``execute`` (``_check_run_limits`` still runs on the ungraded branch), so a + ``execute`` (the budget check still runs on the ungraded branch), so a bucket-count test read one timed-out row in a 100-task ungraded night as proof the run was measured and published ``pass_rate 0.0`` — a real 0% point on the evalboard trend for a run that graded nothing. From 0f64347ea1b735996a4a946e5f51e7c67cd6b998 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 13:45:01 -0700 Subject: [PATCH 4/9] =?UTF-8?q?feat(orchestration):=204/6=20=E2=80=94=20st?= =?UTF-8?q?age=20plugins=20into=20one=20canonical=20root;=20skill=5Ftrigge?= =?UTF-8?q?red=20escalates=20a=20skill=20that=20was=20never=20offered?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stage_plugins turns both authored layouts into /plugin_root, records environment_info.skills_offered, and every harness receives the root natively. A plugin path with no skill fails resolution, and a skill_triggered target outside skills_offered raises CheckerMisuseError. The per-harness scanners, utils.process_plugins and CE045 are deleted. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 81 +++--- .claude/notes/contracts.md | 7 + .claude/shared/run-layout.md | 1 + docs/AB_EXPERIMENTS.md | 14 +- docs/PLUGIN.md | 14 +- docs/REPORT_SCHEMA.md | 2 + docs/TASK_DEFINITION_GUIDE.md | 2 + docs/agents/ANTIGRAVITY.md | 22 +- docs/agents/CLAUDE_CODE.md | 20 +- docs/agents/CODEX.md | 15 +- docs/agents/HARNESS_PARITY.md | 82 ++---- docs/agents/OPENCODE.md | 34 +-- docs/agents/PI.md | 21 +- docs/tutorials/07-plugin-in-claude-code.md | 10 +- experiments/default.yaml | 4 +- experiments/plugin-comparison.yaml | 5 +- plugins/coder-eval/reference/run-layout.md | 1 + .../reference/templates/activation.yaml | 27 +- .../coder-eval/skills/check-skill/SKILL.md | 40 +-- plugins/coder-eval/skills/ci/SKILL.md | 31 +-- pyproject.toml | 1 - src/coder_eval/agent.py | 4 + src/coder_eval/agents/_skills.py | 102 -------- src/coder_eval/agents/antigravity_agent.py | 80 ++---- src/coder_eval/agents/claude_code_agent.py | 15 +- src/coder_eval/agents/codex_agent.py | 107 ++------ src/coder_eval/agents/noop_agent.py | 2 + src/coder_eval/agents/opencode_agent.py | 23 +- src/coder_eval/agents/pi_agent.py | 27 +- src/coder_eval/criteria/base.py | 2 + src/coder_eval/criteria/skill_triggered.py | 17 +- src/coder_eval/evaluation/checker.py | 16 +- src/coder_eval/isolation/docker_runner.py | 30 ++- src/coder_eval/models/agent_config.py | 10 +- .../orchestration/plugin_staging.py | 153 +++++++++++ .../orchestration/resolution_checks.py | 4 +- src/coder_eval/orchestrator.py | 20 ++ src/coder_eval/path_utils.py | 3 + src/coder_eval/utils.py | 62 ----- .../probe_plugin/skills/probe-skill/SKILL.md | 6 + tasks/skills/skill_not_offered.yaml | 24 ++ tasks/skills/skill_offered.yaml | 24 ++ tests/fixtures/mock_agent.py | 1 + tests/fixtures/text_stub_agent.py | 1 + tests/test_agent_telemetry.py | 17 +- tests/test_antigravity_agent.py | 104 +++----- tests/test_codex_agent.py | 68 +++++ tests/test_custom_lint.py | 238 ------------------ tests/test_docker_runner_mounts.py | 100 ++++++++ tests/test_experiment_resolver.py | 13 + tests/test_harness_conformance.py | 79 +++--- tests/test_opencode_agent.py | 169 ++----------- tests/test_orchestrator.py | 189 +++++++++++++- tests/test_pi_agent.py | 45 +--- tests/test_plan_command.py | 10 + tests/test_plugin_processing.py | 157 ------------ tests/test_plugin_staging.py | 169 +++++++++++++ tests/test_simulation_integration.py | 2 + tests/test_skill_triggered.py | 55 ++++ 59 files changed, 1245 insertions(+), 1337 deletions(-) delete mode 100644 src/coder_eval/agents/_skills.py create mode 100644 src/coder_eval/orchestration/plugin_staging.py create mode 100644 tasks/skills/probe_plugin/skills/probe-skill/SKILL.md create mode 100644 tasks/skills/skill_not_offered.yaml create mode 100644 tasks/skills/skill_offered.yaml delete mode 100644 tests/test_plugin_processing.py create mode 100644 tests/test_plugin_staging.py diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 4ac0e76c3..15ff2a35c 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -66,13 +66,9 @@ intentionally brief and out of scope; trimming for DISPLAY belongs in the render finalizes cleanly as `tool_calls_exhausted` (no crash, no retry). The **known unfixed divergences** — which config fields each harness does and does not - enforce, and the per-harness `agent.plugins[].path` depth (claude-code REQUIRES a - plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, - which is the costly direction: no error, every positive row of an activation suite - scores 0, and the suite reports recall 0.0, reading exactly like a skill that never - triggers; held to the plugin-root shape for `SKILL_SOURCE_PATH` by CE045) — are the - table's to state, not this file's. Full table + rationale: - docs/agents/HARNESS_PARITY.md. + enforce — are the table's to state, not this file's. Full table + rationale: + docs/agents/HARNESS_PARITY.md. The per-harness `agent.plugins[].path` depth is no longer + a divergence: staging hands every harness one layout (§ Skills, per harness). The agent-field half of parity is now the `HarnessContract` each agent class declares: a field, `permission_mode` value or tool name a harness cannot honor is a resolution @@ -632,8 +628,7 @@ tempdir is still reclaimed. `_TERM_GRACE_SECONDS` is re-declared at the same value in both nd-JSON harnesses rather than shared: the CLI-driver hoist that would unify their teardown constants and reducers is -a tracked follow-up. The shared plugin→skills resolver already lives in `agents/_skills.py`, -and `STDOUT_LINE_LIMIT_BYTES`, which IS canonical, is imported. +a tracked follow-up. `STDOUT_LINE_LIMIT_BYTES`, which IS canonical, is imported. ## The system_prompt_semantics marker @@ -657,36 +652,50 @@ cannot disagree with what was sent. ## Skills, per harness -A `plugins:` entry is a Claude-plugin root, and only the SKILLS half of it is honored -anywhere — a plugin's agents, hooks, commands and MCP servers have no equivalent outside -claude-code and are dropped. The manifest's `skills` field is read rather than `skills/` -being hardcoded, so a plugin that relocates its skills keeps working. - -- **OpenCode** maps each root to `skills.paths` via `OPENCODE_CONFIG_CONTENT`, which the - CLI merges as a final local-scope layer. That was chosen over writing - `/.opencode/skills/` because it writes nothing into the sandbox that is later - preserved as a run artifact and inspected by file criteria, and does not depend on how - the CLI resolves a project root from `--dir`. Verified orthogonal to `--pure`, which - skips external *plugins*, not configured skill paths. An inherited value is appended to - rather than clobbered, since the host may legitimately configure OpenCode the same way. -- **Pi** passes each as `--skill `. -- **Codex** symlinks (or copies, on Windows) each skill dir into `.agents/skills/`, which - the CLI auto-discovers from the working directory upward. -- **Antigravity** takes search paths natively via `skills_paths` — but those only drive - DISCOVERY. The file-tool allowlist is `workspaces` alone, so the skill roots must appear +`orchestration/plugin_staging.py` stages every `plugins:` entry into one canonical root, +`/plugin_root`, before `Agent.start`. Each harness then receives the SAME layout: +`.claude-plugin/plugin.json` and `skills/` links. The staging exists because each +adapter used to scan the authored path its own way. claude-code loaded nothing from a bare +skills directory, with no error, so an activation suite scored recall 0.0 and read exactly +like a skill that never triggers. + +- **Both authored layouts are accepted.** For each root, the manifest-declared skill dirs + that exist are scanned (default `skills/`). If none exists, the root is a bare skills + directory. The manifest's `skills` field is read, not hardcoded, so a plugin that + relocates its skills keeps working. +- **Only skills are staged.** A plugin's agents, hooks, commands and MCP servers are + dropped on every harness, claude-code included. That also removes a confound: a project + subagent beside `skills/` can no longer answer the request the skill should answer. +- **The staged manifest is `{"name": "coder-eval-plugins"}` and nothing else.** A spike + with `claude -p --plugin-dir` showed that a manifest declaring `"skills": ["skills"]` + loaded no skill, symlinked or copied. A name-only manifest loads the `skills/` default. +- **Refusal is at resolution.** `validate_plugins` runs in `validate_resolved_task`, so a + path with no skill, an unresolvable path, or one skill name from two sources fails + `plan`. A run can no longer measure the model WITHOUT the skill under test and look normal. +- **`skills_offered` is recorded** in `environment_info` and passed to the checker. + `skill_triggered` raises `CheckerMisuseError` when its `skill_name` is not offered: the + positive control cannot run, so the row escalates instead of scoring 0.0. + +Delivery, per harness: + +- **Claude Code** takes the root as an SDK `{"type": "local", "path": plugin_root}` plugin. +- **OpenCode** appends `/skills` to `skills.paths` via + `OPENCODE_CONFIG_CONTENT`, which the CLI merges as a final local-scope layer. That was + chosen over writing `/.opencode/skills/` because it writes nothing into the + sandbox that is later preserved as a run artifact and inspected by file criteria, and + does not depend on how the CLI resolves a project root from `--dir`. Verified orthogonal + to `--pure`, which skips external *plugins*, not configured skill paths. An inherited + value is appended to rather than clobbered. The staged `skills/` holds only skill links, + so the recursive scan no longer walks a repo root's self-referential symlinks. +- **Pi** passes `--skill /skills`. +- **Codex** links each `/skills/` into `.agents/skills/` with + `link_or_copy`, which the CLI auto-discovers from the working directory upward. +- **Antigravity** takes `/skills` in `skills_paths` — but those only drive + DISCOVERY. The file-tool allowlist is `workspaces` alone, so the same path must appear there too, or the agent discovers a skill and every read of its `SKILL.md` is denied as out-of-workspace. -A bare skills directory is used as-is only when the root declares no `skills/` subdir. -That is deliberately not a fallback for a root that HAS one: `skills.paths` is scanned -recursively and a repo root can contain self-referential symlinks (`UiPath/skills` has -`plugins/uipath -> ..`), which resolves skills through an arbitrary path and silently drops -duplicate names. - -Every way this can come up empty is logged loudly — an unresolved env var, a missing dir, -a root with no `/SKILL.md` under it. A plugin whose skills never reach the agent -still *looks* like a normal run, which is precisely the failure the logging closes: the -run measures the model WITHOUT the skill under test. +`plugin_tools_dir` is not a skills source on any harness. ## Why the registry rejects a re-registration diff --git a/.claude/notes/contracts.md b/.claude/notes/contracts.md index e337dc7f3..f8cfc8ce2 100644 --- a/.claude/notes/contracts.md +++ b/.claude/notes/contracts.md @@ -190,6 +190,13 @@ is genuinely a gating 0.0. `reference_file` is confined to the reference directo judge's author-written `files:` entry, because it names one file of the solution being compared against and traversal out of the staged copy is always a mistake. +`skill_triggered` escalates the same way when its `skill_name` is not among the skills +`agent.plugins` offered (`CheckContext.skills_offered`). The agent was never offered the +skill, so the positive control cannot run. Scored as 0.0, every positive row of an +activation suite would read as a skill that never triggers. The gate applies only when the +task sets plugins: with `skills_offered` `None` the criterion scores as before, so a skill +the harness finds by other means still counts. + Grading time is accumulated at the checker, not at the four orchestrator call sites, so a fifth site cannot be added without it — the same reason the tool subtraction lives at one collector seam. It is monotonic, booked in a `finally` so a grade that raises still records diff --git a/.claude/shared/run-layout.md b/.claude/shared/run-layout.md index c00a82c69..c006e9352 100644 --- a/.claude/shared/run-layout.md +++ b/.claude/shared/run-layout.md @@ -16,6 +16,7 @@ runs/////{task.json, task.log, artifacts/} - `task.json.graded` — present only after `coder-eval execute --driver docker` refused a container's verdict: the runtime image predated `execute` and graded anyway, so the runner quarantines the graded record here rather than leaving it readable as `task.json`, where a later `--resume` / `aggregate` would fold in exactly the row it declined to publish. Diagnostic-only; `rglob("task.json")` consumers do not match it. - `grade.log` — present only after a DETACHED grade over this directory (`coder-eval run --resume`). The grading pass's own log. It is a separate file because the log handler truncates whatever file it opens, so writing to `task.log` would destroy the agent trajectory log the run already paid for. - `task.log` — the human-readable task log; `artifacts/` — files the agent produced. +- `plugin_root/` — the staged plugin root the agent was handed; symlinks into the authored plugin; present only when the task sets `agent.plugins`. **Scope-marker files** (used to detect what a given path represents): diff --git a/docs/AB_EXPERIMENTS.md b/docs/AB_EXPERIMENTS.md index 44aeaac27..ade3cef5b 100644 --- a/docs/AB_EXPERIMENTS.md +++ b/docs/AB_EXPERIMENTS.md @@ -205,7 +205,7 @@ variants: agent: plugins: - type: "local" - path: ".." # PLUGIN ROOT holding skills/ — see note below + path: ".." # plugin root or bare skills directory — see note below ``` Notes: @@ -217,14 +217,12 @@ Notes: it. Pair the experiment with a [`skill_triggered`](TASK_DEFINITION_GUIDE.md#skill_triggered) criterion to measure _whether it fired_ alongside your real success criteria that measure _whether outcomes improved_. -- **`path` must be a plugin ROOT — a directory holding `skills/`** — so the skill - resolves at `/skills//SKILL.md`. Point one level deeper, at the - directory of skill directories, and claude-code loads **nothing**: the `with-skill` - arm then silently matches the baseline and the A/B compares two identical arms. - Codex and Antigravity accept either depth, so this fails on claude-code alone — - see [Harness parity](agents/HARNESS_PARITY.md#agentpluginspath-accepts-different-depths-per-harness). +- `path` names a plugin root (`/skills//SKILL.md`) or a bare skills + directory (`//SKILL.md`). Both are staged. A path with no skill fails + `plan`, so the `with-skill` arm cannot silently match the baseline. See + [Plugin staging](agents/HARNESS_PARITY.md#plugin-staging). - Plugin paths are environment-dependent. The shipped example expects a - `$PLUGIN_PATH` env var pointing at your plugin **root**. See + `$PLUGIN_PATH` env var pointing at your plugin root or skills directory. See `experiments/plugin-comparison.yaml`. Run it: diff --git a/docs/PLUGIN.md b/docs/PLUGIN.md index 3c3536129..97901ce5e 100644 --- a/docs/PLUGIN.md +++ b/docs/PLUGIN.md @@ -106,18 +106,18 @@ It then: One prerequisite the suite cannot infer: the evaluated agent runs in a fresh sandbox holding none of your files, so it is offered no skills unless the task says where they live. The template reads that location from an environment -variable — point it at a **plugin root**: a directory holding a `skills/` -subdirectory, so the skill sits at `/skills//SKILL.md`. For -`.claude/skills/pdf-forms/SKILL.md` that root is `.claude`, not `.claude/skills`: +variable. Point it at a plugin root (`/skills//SKILL.md`) or at a +bare skills directory (`//SKILL.md`). Both are staged. For +`.claude/skills/pdf-forms/SKILL.md`, `.claude` works and `.claude/skills` works too: ```bash export SKILL_SOURCE_PATH="$(pwd)/.claude" ``` -Leave it unset and the skill is simply absent, every positive row scores 0, and -the result is indistinguishable from a skill that never fires. It stays an -environment variable rather than a path baked into the YAML so the suite is -portable — it is committed and re-run on other machines, and in CI. +Leave it unset, or point it at a directory with no skill, and `coder-eval plan` +fails with a config error. It stays an environment variable rather than a path +baked into the YAML so the suite is portable — it is committed and re-run on other +machines, and in CI. ### A low-recall result has three causes, not one diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index 739ec7383..57aedd042 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -174,6 +174,8 @@ each of `system_prompt`, `plugin_skills`, `permission_mode`, `allowed_tools` and `disallowed_tools`, `"enforced"` or `"unsupported"`, plus `system_prompt_semantics` (the class default), `cooperative_stop`, and `permission_modes` (the sorted `permission_mode` values the harness honors, or `null`). +`environment_info.skills_offered` is the list of skill names the staged plugin root +offered to the agent. It is absent when the task sets no `agent.plugins`. `sdk_options.system_prompt` is a `SystemPromptPreset` dict (`{type: "preset", preset: "claude_code", exclude_dynamic_sections: true, append?: str}`) on append-mode Claude Code runs and a plain string only in replace mode — it is diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 62b480251..efea94ea4 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -1358,6 +1358,8 @@ Observed label is `"yes"` when either signal is found, else `"no"`. Expected lab **Requires agent telemetry.** This criterion reads `turn_records`, so it only works against a real agent run (not a static check). With no turn records it reports `score=0.0` and an `error`. +**The skill must be offered.** When the task sets `agent.plugins`, coder-eval stages the skills those paths offer (a plugin root or a bare skills directory) and records their names in `environment_info.skills_offered`. A `skill_name` that is not among them makes the criterion finish `ERROR`, not `0.0`: the positive control cannot run. This applies even when the skill reaches the agent another way (for example a template's `.claude/skills/`): with `agent.plugins` set, put the skill under test in a plugin path. A plugin path that offers no skill fails `coder-eval plan`. See [Plugin staging](agents/HARNESS_PARITY.md#plugin-staging). + **Classification metrics.** `skill_triggered` returns a `ClassificationCriterionResult`, so on a [dataset-backed task](#dataset) the suite aggregator computes accuracy / precision / recall / F1 / confusion matrix across all rows. Gate the suite with `suite_thresholds` using any of: `accuracy`, `macro_f1`, `weighted_f1`, `micro_f1`, or per-label `precision.