From 84168042780689a82e3fe407bce7e17855fd89b8 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 16:45:45 -0700 Subject: [PATCH 01/25] =?UTF-8?q?feat(pricing):=201/10=20=E2=80=94=20price?= =?UTF-8?q?=5Fturn=20is=20the=20one=20cost=20rule=20for=20adapters=20and?= =?UTF-8?q?=20the=20turn=20monitor;=20CE071?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every adapter and TurnMonitor._price call pricing.price_turn. A reported $0 on a priced model is repriced from the rate card everywhere (the monitor and Claude included); an empty turn keeps its reported cost, so it reads token_usage null on every harness. CE071 keeps calculate_cost out of agents/ and the monitor. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 26 ++- pyproject.toml | 1 + src/coder_eval/agents/antigravity_agent.py | 8 +- src/coder_eval/agents/claude_code_agent.py | 36 +--- src/coder_eval/agents/codex_agent.py | 36 ++-- src/coder_eval/agents/opencode_agent.py | 45 +--- src/coder_eval/agents/pi_agent.py | 40 +--- src/coder_eval/orchestration/turn_monitor.py | 27 +-- src/coder_eval/pricing.py | 48 ++++- tests/lint/rules/ce071_price_turn_only.py | 53 +++++ tests/lint/runner.py | 7 +- tests/test_custom_lint.py | 45 ++++ tests/test_opencode_agent.py | 4 +- tests/test_pi_agent.py | 4 +- tests/test_price_turn.py | 204 +++++++++++++++++++ tests/test_turn_monitor.py | 16 +- 16 files changed, 433 insertions(+), 167 deletions(-) create mode 100644 tests/lint/rules/ce071_price_turn_only.py create mode 100644 tests/test_price_turn.py diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 75f36ed3..14ceb223 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -297,6 +297,10 @@ them a CLI upgrade silently zeroes the run's tokens and cost and blinds the budg ## Cost: the stream versus the rate card +`pricing.price_turn(usage, models)` is the one rule for a turn's cost. Every adapter and +`TurnMonitor` call it, and CE071 keeps `calculate_cost` out of both, because five copies +of the rule once let the `max_usd` stop and the persisted cost disagree on one turn. + A non-zero cost the CLI reported always wins — it is the provider's own accounting, and on OpenRouter per-request routing makes it strictly better than a static headline rate. The rate card fills two gaps that would otherwise book tokens with no money: @@ -309,13 +313,25 @@ rate card fills two gaps that would otherwise book tokens with no money: and understating cost silently defeats `max_usd`, which is the worse failure. A genuinely free model has an all-zero rate entry (or none), so it still resolves to 0. +Empty usage returns the reported cost unchanged, `None` included: `EventCollector` +publishes `token_usage=None` only for empty usage with no cost, so pricing an empty turn +at `0.0` would publish a zero-cost usage row for a turn that spent nothing. So an empty +turn reads `token_usage: null` on every harness (Codex, Antigravity and the LiteLLM route +used to price it at `0.0` on a priced model). The monitor adds its own "an empty turn +costs 0" in front, because it sums. A non-finite reported cost counts as unreported. + +The ORDER of `models` is the caller's decision. Adapters pass their one model. The monitor +passes `agent.model`, then the model the agent resolved at start, then the last model a +message reported: the configured model wins so that a sub-agent's model on the stream +cannot reprice the run. + The Claude SDK's own `costUSD` is a client-side estimate assuming Anthropic pricing, so it is wrong for an open-weight model behind LiteLLM and is repriced from the token buckets at -the model's real rate. The buckets are untouched, so the reconciliation invariant holds — -only the cost scalar changes. An unpriced model sets the cost to `None` (an honest N/A) -**and warns**. When the task sets `max_usd`, the `TurnMonitor` then raises -`BudgetUnenforceableError` at the turn end, so the row finishes `ERROR` and is never a -silent skip. +the model's real rate (`price_turn` with the report cleared). The buckets are untouched, so +the reconciliation invariant holds — only the cost scalar changes. An unpriced model sets +the cost to `None` (an honest N/A) **and warns**. When the task sets `max_usd`, the +`TurnMonitor` then raises `BudgetUnenforceableError` at the turn end, so the row finishes +`ERROR` and is never a silent skip. ## Codex rollout rebuild diff --git a/pyproject.toml b/pyproject.toml index 53e42040..e421b64f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -304,6 +304,7 @@ external = [ "CE068", "CE069", "CE070", + "CE071", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 7e82f417..a15ec830 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -55,7 +55,7 @@ TurnRecord, UsageGranularity, ) -from coder_eval.pricing import calculate_cost +from coder_eval.pricing import price_turn from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( @@ -182,14 +182,14 @@ def _to_token_usage(usage: Any, model: str | None) -> TokenUsage: thoughts = getattr(usage, "thoughts_token_count", 0) or 0 uncached_input = max(prompt - cached, 0) output = candidates + thoughts - cost = calculate_cost(model, uncached_input, output, 0, cached) if model else None - return TokenUsage( + tokens = TokenUsage( uncached_input_tokens=uncached_input, output_tokens=output, cache_creation_input_tokens=0, cache_read_input_tokens=cached, - total_cost_usd=cost, ) + tokens.total_cost_usd = price_turn(tokens, (model,)) + return tokens @AgentRegistry.register(AgentKind.ANTIGRAVITY, AntigravityAgentConfig) diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 6f381312..5312c7be 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -67,7 +67,7 @@ from coder_eval.models import ( AssistantMessage as AssistantMessageTelemetry, ) -from coder_eval.pricing import calculate_cost +from coder_eval.pricing import price_turn from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( @@ -1474,36 +1474,16 @@ def _build_token_usage( model, ) - @staticmethod - def _price_from_buckets(usage: TokenUsage, model: str | None) -> float | None: - """Price the four token buckets at ``model``'s list rate. - - ``None`` when ``model`` is unset or absent from the rate card. - """ - if not model: - return None - return calculate_cost( - model, - uncached_input_tokens=usage.uncached_input_tokens, - output_tokens=usage.output_tokens, - cache_creation_tokens=usage.cache_creation_input_tokens, - cache_read_tokens=usage.cache_read_input_tokens, - ) - @staticmethod def _backfill_cost(usage: TokenUsage, model: str | None) -> TokenUsage: - """Price the token buckets when the SDK gave no cost (timeout / kill). + """Price the turn in place with ``pricing.price_turn`` and return it. A timed-out or killed turn has no terminal ``ResultMessage``, so the cost - is absent even though the tokens are fully captured. A no-op when the cost - is already set or the model is unpriced. + is absent even though the tokens are fully captured; the rate card fills it. """ - if usage.total_cost_usd is not None or not model: - return usage - cost = ClaudeCodeAgent._price_from_buckets(usage, model) - if cost is not None: - usage.total_cost_usd = cost - else: + reported = usage.total_cost_usd + usage.total_cost_usd = price_turn(usage, (model,)) + if usage.total_cost_usd is None and reported is None and model and not usage.is_empty(): # Not in the rate card, so the turn reverts to a null cost. Surface # it, or a stale pricing table silently reads as "Cost = —". logger.warning("No pricing for model %r; timeout/kill turn cost left unset", model) @@ -1522,9 +1502,9 @@ def _reprice_for_litellm(usage: TokenUsage, model: str | None) -> None: Rationale: .claude/notes/agents.md § Cost: the stream versus the rate card """ - cost = ClaudeCodeAgent._price_from_buckets(usage, model) + cost = price_turn(usage.model_copy(update={"total_cost_usd": None}), (model,)) usage.total_cost_usd = cost - if cost is None: + if cost is None and not usage.is_empty(): logger.warning("No pricing for litellm model %r; turn cost left unset", model) def get_sdk_options(self) -> dict[str, Any] | None: diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 5e041b41..c8ec85dd 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -41,7 +41,7 @@ UsageGranularity, ) from coder_eval.orchestration.plugin_staging import link_or_copy -from coder_eval.pricing import calculate_cost +from coder_eval.pricing import price_turn from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( @@ -2080,18 +2080,13 @@ def _token_usage_from_sdk(self, sdk_token_usage: Any) -> TokenUsage | None: self._thread_usage_baseline = cumulative # Fresh slice = full prompt minus the cached prefix. uncached = _fresh_input_tokens(turn.input, turn.cached) - cost = calculate_cost( - self._effective_model() or "", - uncached_input_tokens=uncached, - output_tokens=turn.output, - cache_read_tokens=turn.cached, - ) - return TokenUsage( + usage = TokenUsage( uncached_input_tokens=uncached, output_tokens=turn.output, cache_read_input_tokens=turn.cached, - total_cost_usd=cost, ) + usage.total_cost_usd = price_turn(usage, (self._effective_model(),)) + return usage def _advance_usage_baseline(self, usage: TokenUsage | None) -> None: """Move the thread baseline past a turn whose SDK total never arrived. @@ -2135,11 +2130,13 @@ def _fold_subagent_tokens(self, parent: TokenUsage | None, messages: list[Transc # Each child generation on its own model, then sum. The total is unpriced # when any priced-from-tokens part is: a partial sum would read as the bill. child_costs = [ - calculate_cost( - m.model or self._effective_model() or "", - uncached_input_tokens=_message_uncached_input(m), - output_tokens=m.output_tokens, - cache_read_tokens=m.cache_read_tokens, + price_turn( + TokenUsage( + uncached_input_tokens=_message_uncached_input(m), + output_tokens=m.output_tokens, + cache_read_input_tokens=m.cache_read_tokens, + ), + (m.model or self._effective_model(),), ) for m in children ] @@ -2171,18 +2168,13 @@ def _token_usage_from_messages(self, messages: list[TranscriptMessage]) -> Token cache_read = sum(m.cache_read_tokens for m in assistant) if not (uncached or output or cache_read): return None - cost = calculate_cost( - self._effective_model() or "", - uncached_input_tokens=uncached, - output_tokens=output, - cache_read_tokens=cache_read, - ) - return TokenUsage( + usage = TokenUsage( uncached_input_tokens=uncached, output_tokens=output, cache_read_input_tokens=cache_read, - total_cost_usd=cost, ) + usage.total_cost_usd = price_turn(usage, (self._effective_model(),)) + return usage @staticmethod async def _run_async(func: Any, *args: Any, **kwargs: Any) -> Any: diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index e799cf2a..bda5c757 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -53,7 +53,7 @@ TurnRecord, UsageGranularity, ) -from coder_eval.pricing import calculate_cost +from coder_eval.pricing import price_turn from coder_eval.streaming.callbacks import StreamCallback, safe_emit from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( @@ -442,43 +442,6 @@ def _close_tool( ) ) - def _rate_card_cost(self) -> float | None: - """Price the captured buckets from the static rate card. - - ``None`` when the model is unpinned or unpriced. - """ - if not self.model or self.usage.is_empty(): - return None - return calculate_cost( - self.model, - uncached_input_tokens=self.usage.uncached_input_tokens, - output_tokens=self.usage.output_tokens, - cache_creation_tokens=self.usage.cache_creation_input_tokens, - cache_read_tokens=self.usage.cache_read_input_tokens, - ) - - def _resolve_cost(self) -> float | None: - """Decide the turn's cost: the stream's own accounting vs the rate card. - - A non-zero cost the CLI reported always wins. The rate card fills two gaps - that would otherwise book tokens with no money: no ``cost`` field at all, - and ``cost: 0`` for tokens the rate card prices above zero. - - Rationale: .claude/notes/agents.md § Cost: the stream versus the rate card - """ - rate = self._rate_card_cost() - if not self.saw_cost: - return rate - if self.cost_usd == 0.0 and rate: - logger.warning( - "opencode: the stream reported $0 for a turn the rate card prices at $%.6f " - + "(model unpriced in OpenCode's registry, or subscription auth); using the rate card " - + "so the run total is not understated.", - rate, - ) - return rate - return self.cost_usd - def _warn_token_shape(self, message: str, *args: Any) -> None: """Report a token-bucket surprise ONCE per turn (a broken stream repeats it).""" if self.warned_token_shape: @@ -703,10 +666,8 @@ def finalize( return self.finalized = True self.close_open_tools() - usage = self.usage - cost = self._resolve_cost() - if cost is not None: - usage = usage.model_copy(update={"total_cost_usd": cost}) + reported = self.usage.model_copy(update={"total_cost_usd": self.cost_usd if self.saw_cost else None}) + usage = self.usage.model_copy(update={"total_cost_usd": price_turn(reported, (self.model,))}) # A step still open never received its `step_finish`; close it or the # one-pair-per-inner-turn contract breaks. Completed steps already closed # themselves, so this fires ONLY for the straggler. diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 6cf26b79..0e25943b 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -61,7 +61,7 @@ TurnRecord, UsageGranularity, ) -from coder_eval.pricing import calculate_cost +from coder_eval.pricing import price_turn from coder_eval.streaming.callbacks import StreamCallback, safe_emit from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( @@ -575,38 +575,6 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: ) ) - def _rate_card_cost(self) -> float | None: - if not self.model or self.usage.is_empty(): - return None - return calculate_cost( - self.model, - uncached_input_tokens=self.usage.uncached_input_tokens, - output_tokens=self.usage.output_tokens, - cache_creation_tokens=self.usage.cache_creation_input_tokens, - cache_read_tokens=self.usage.cache_read_input_tokens, - ) - - def _resolve_cost(self) -> float | None: - """Decide the turn's cost: the stream's own accounting vs the rate card. - - Pi reports a real per-call ``cost.total``, which wins for any nonzero - total. It falls back to the rate card when the stream reported no cost at - all, or reported exactly ``$0`` on a model the rate card DOES price. - - Rationale: .claude/notes/agents.md § Cost: the stream versus the rate card - """ - rate = self._rate_card_cost() - if not self.saw_cost: - return rate - if self.cost_usd == 0.0 and rate: - logger.debug( - "pi: the stream reported $0 for a turn the rate card prices at $%.6f; using the rate card " - + "so the run total is not understated.", - rate, - ) - return rate - return self.cost_usd - def close_open_tools(self) -> None: """Force-close every tool still awaiting a result (crash/timeout orphans).""" for call_id in list(self.open_tools): @@ -624,10 +592,8 @@ def finalize( return self.finalized = True self.close_open_tools() - usage = self.usage - cost = self._resolve_cost() - if cost is not None: - usage = usage.model_copy(update={"total_cost_usd": cost}) + reported = self.usage.model_copy(update={"total_cost_usd": self.cost_usd if self.saw_cost else None}) + usage = self.usage.model_copy(update={"total_cost_usd": price_turn(reported, (self.model,))}) # A turn still open never received its `turn_end`; close it or the # one-pair-per-inner-turn contract breaks. if self.turn_open: diff --git a/src/coder_eval/orchestration/turn_monitor.py b/src/coder_eval/orchestration/turn_monitor.py index 0e30ac7a..39d38a6c 100644 --- a/src/coder_eval/orchestration/turn_monitor.py +++ b/src/coder_eval/orchestration/turn_monitor.py @@ -20,7 +20,6 @@ from __future__ import annotations import logging -import math import time from typing import TYPE_CHECKING, Any @@ -36,7 +35,7 @@ TokenUsage, ) from coder_eval.orchestration.early_stop import early_stop_active -from coder_eval.pricing import calculate_cost +from coder_eval.pricing import price_turn from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( AgentEndEvent, @@ -261,9 +260,9 @@ def usage(self) -> TokenUsage: def cost_usd(self) -> float | None: """Cumulative USD: every finished turn priced on its own, plus the priceable in-flight deltas. - A turn is priced from its reported cost, else from the rate card for the first - priced model of ``agent.model``, the model the agent resolved at start, and the - last model a message reported; a turn with no usage costs 0. + A turn is priced by ``pricing.price_turn`` with the models ``agent.model``, the + model the agent resolved at start, and the last model a message reported, in + that order; a turn with no usage and no reported cost costs 0. ``None`` once any finished turn could be priced none of these ways. """ if self._unpriced_turn: @@ -300,23 +299,9 @@ def _commit(self, usage: TokenUsage) -> None: self._in_flight = TokenUsage() def _price(self, usage: TokenUsage) -> float | None: - if usage.total_cost_usd is not None: - return usage.total_cost_usd if math.isfinite(usage.total_cost_usd) else None - if usage.is_empty(): + if usage.is_empty() and usage.total_cost_usd is None: return 0.0 - for model in (self._model, self._start_model, self._reported_model): - if model is None: - continue - cost = calculate_cost( - model, - usage.uncached_input_tokens, - usage.output_tokens, - usage.cache_creation_input_tokens, - usage.cache_read_input_tokens, - ) - if cost is not None: - return cost - return None + return price_turn(usage, (self._model, self._start_model, self._reported_model)) def _breach(self) -> tuple[str, float, float] | None: """The first budget over its cap as ``(budget name, actual, limit)``: input, output, total, usd.""" diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py index 0e4d2fc1..eb5748a0 100644 --- a/src/coder_eval/pricing.py +++ b/src/coder_eval/pricing.py @@ -13,9 +13,19 @@ generated file. """ -from collections.abc import Iterable, Mapping +import logging +import math +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from coder_eval.models import TokenUsage + + +logger = logging.getLogger(__name__) @dataclass(frozen=True) @@ -244,3 +254,39 @@ def calculate_cost( + cache_creation_tokens * pricing.cache_write_per_mtok + cache_read_tokens * pricing.cache_read_per_mtok ) / 1_000_000 + + +def price_turn(usage: "TokenUsage", models: Sequence[str | None]) -> float | None: + """The cost of one turn: ``usage.total_cost_usd`` is what the harness reported. + + In order: a finite, non-zero reported cost wins; empty usage returns the reported + cost unchanged (``None`` when nothing finite was reported); else the rate card for + the first model in ``models`` that it prices (falsy entries skipped); else a + reported ``0.0``; else ``None``. A non-finite reported cost counts as unreported. + + Rationale: .claude/notes/agents.md § Cost: the stream versus the rate card + """ + reported = usage.total_cost_usd + if reported is not None and not math.isfinite(reported): + reported = None + if reported: + return reported + if usage.is_empty(): + return reported + for model in models: + if not model: + continue + cost = calculate_cost( + model, + usage.uncached_input_tokens, + usage.output_tokens, + usage.cache_creation_input_tokens, + usage.cache_read_input_tokens, + ) + if cost is not None: + if reported == 0.0 and cost: + logger.debug( + "a turn reported $0 that the rate card prices at $%.6f for %r; using the rate card", cost, model + ) + return cost + return reported diff --git a/tests/lint/rules/ce071_price_turn_only.py b/tests/lint/rules/ce071_price_turn_only.py new file mode 100644 index 00000000..ecc28c45 --- /dev/null +++ b/tests/lint/rules/ce071_price_turn_only.py @@ -0,0 +1,53 @@ +"""CE071: agent adapters and the turn monitor price a turn only through ``pricing.price_turn``. + +The defect: five adapters and ``TurnMonitor`` each carried their own copy of the cost +rule on top of ``calculate_cost``. Pi and OpenCode priced a reported ``$0`` from the rate +card, the monitor took it as free, Antigravity and Codex never looked at a report, and +Claude kept a third variant for LiteLLM. So the ``max_usd`` stop and the persisted turn +cost could disagree on the same turn. ``price_turn`` is now the one rule, and an adapter +or the monitor that calls ``calculate_cost`` is re-growing a copy. + +Fires, in files under ``src/coder_eval/agents/`` and in +``src/coder_eval/orchestration/turn_monitor.py``, on any name, attribute or +``from``-import alias spelled ``calculate_cost``. Pricing outside a subject turn (the +simulator in ``models/results.py``, ``evaluation/judge_usage.py``) is out of scope. + +Blind spot: a copy of the rate arithmetic under another name, e.g. reading +``ModelPricing`` fields directly. +""" + +import ast +import re + +from tests.lint.rules._model_ctor import AGENTS_ROOT +from tests.lint.rules.base import BaseRule + + +_TURN_MONITOR = re.compile(r"(?:^|[/\\])orchestration[/\\]turn_monitor\.py$") +_BANNED = "calculate_cost" +_FIX = "price a turn with coder_eval.pricing.price_turn, the one cost rule shared with the max_usd monitor" + + +class PriceTurnOnly(BaseRule): + id = "CE071" + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(AGENTS_ROOT.search(filepath) or _TURN_MONITOR.search(filepath)) + + def _flag(self, node: ast.AST, name: str) -> None: + if self._in_scope and name == _BANNED: + self.violation(node, f"architectural violation: '{_BANNED}' used to price a turn — {_FIX}") + + def visit_Name(self, node: ast.Name) -> None: + self._flag(node, node.id) + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute) -> None: + self._flag(node, node.attr) + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + for alias in node.names: + self._flag(node, alias.name) + self.generic_visit(node) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index 0b83a950..ddfb62ca 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -44,6 +44,7 @@ from tests.lint.rules.ce066_no_report_imports_in_core import NoReportImportsInCore from tests.lint.rules.ce068_no_kind_names_in_kernel import NoKindNamesInKernel from tests.lint.rules.ce070_no_cap_or_skill_scan_in_adapters import NoCapOrSkillScanInAdapters +from tests.lint.rules.ce071_price_turn_only import PriceTurnOnly from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -66,8 +67,9 @@ # comment carrying 062 in an older branch, review or commit message must never # start meaning something new. # -# Claim 071 next (069 is TestCE069HarnessParityTable, 070 is NoCapOrSkillScanInAdapters). NOTE 065 IS TAKEN and is -# not in ALL_RULES: doc-surface and +# Claim 074 next (069 is TestCE069HarnessParityTable, 070 is NoCapOrSkillScanInAdapters, 071 is +# PriceTurnOnly; 072 and 073 are reserved for the emitter sole-writer and subprocess-stdin rules). +# NOTE 065 IS TAKEN and is not in ALL_RULES: doc-surface and # whole-tree rules are `@pytest.mark.lint` classes in tests/test_custom_lint.py # rather than BaseRules, so the `_rule_ids` uniqueness assert below cannot see # them. Enumerating them here is how this note fell behind CE044, so grep @@ -126,6 +128,7 @@ NoReportImportsInCore, NoKindNamesInKernel, NoCapOrSkillScanInAdapters, + PriceTurnOnly, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 84cd2067..add96a63 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4054,6 +4054,51 @@ def _write_pair(root: Path, entry_extra: dict | None = None) -> None: (market_dir / "marketplace.json").write_text(json.dumps({"name": "demo", "plugins": [entry]}), encoding="utf-8") +class TestCE071PriceTurnOnly: + """CE071 — adapters and the turn monitor price a turn only through ``price_turn``.""" + + @staticmethod + def _violations(source: str, filepath: str) -> list: + import ast + + from tests.lint.rules.ce071_price_turn_only import PriceTurnOnly + + return list(PriceTurnOnly(filepath).check(ast.parse(source))) + + @pytest.mark.parametrize( + "source", + [ + "from coder_eval.pricing import calculate_cost", + "cost = calculate_cost(model, 1, 2)", + "cost = pricing.calculate_cost(model, 1, 2)", + ], + ) + @pytest.mark.parametrize( + "filepath", + ["/repo/src/coder_eval/agents/x_agent.py", "/repo/src/coder_eval/orchestration/turn_monitor.py"], + ) + def test_calculate_cost_in_an_adapter_or_the_monitor_violates(self, source: str, filepath: str): + found = self._violations(source, filepath) + assert found + assert "price_turn" in found[0].message + + @pytest.mark.parametrize( + "filepath", + [ + "/repo/src/coder_eval/pricing.py", + "/repo/src/coder_eval/evaluation/judge_usage.py", + "/repo/src/coder_eval/orchestration/early_stop.py", + ], + ) + def test_the_same_code_elsewhere_is_allowed(self, filepath: str): + source = "from coder_eval.pricing import calculate_cost\ncost = calculate_cost(model, 1, 2)" + assert not self._violations(source, filepath) + + def test_price_turn_is_allowed_in_an_adapter(self): + source = "from coder_eval.pricing import price_turn\ncost = price_turn(usage, (model,))" + assert not self._violations(source, "/repo/src/coder_eval/agents/x_agent.py") + + class TestCE054EnvInfoKeyRoundTrip: """CE054 fires when an environment_info key is read with no writer anywhere. diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 5483bf6a..346753bc 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -316,14 +316,14 @@ async def test_zero_reported_cost_on_a_priced_model_uses_the_rate_card(self, pat ), ] patch_exec(_FakeProcess(stream)) - with caplog.at_level("WARNING"): + with caplog.at_level("DEBUG", logger="coder_eval.pricing"): record = await _run(_agent(), tmp_path) expected = calculate_cost("deepseek/deepseek-v4-pro", uncached_input_tokens=1000, output_tokens=500) assert expected is not None and expected > 0 assert record.token_usage is not None assert record.token_usage.total_cost_usd == pytest.approx(expected) - assert "not understated" in caplog.text + assert "using the rate card" in caplog.text async def test_zero_reported_cost_on_an_unpriced_model_stays_zero(self, patch_exec, tmp_path): """With no rate to fall back to, the stream's 0 is the best information we have.""" diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 004c349f..989e2f09 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -1050,7 +1050,7 @@ async def test_cancel_parks_partial_and_reraises(self, patch_exec, tmp_path): def _turn_end_no_cost(*, inp: int, out: int) -> str: """A `turn_end` whose usage object omits the `cost` key (provider/auth mode that - reports no cost) — so `_resolve_cost` must fall back to the rate card.""" + reports no cost) — so `price_turn` must fall back to the rate card.""" return json.dumps( { "type": "turn_end", @@ -1104,7 +1104,7 @@ async def test_zero_reported_cost_on_a_priced_model_uses_the_rate_card(self, pat assert expected is not None and expected > 0 assert record.token_usage is not None assert record.token_usage.total_cost_usd == pytest.approx(expected) - assert "not understated" in caplog.text + assert "using the rate card" in caplog.text async def test_zero_reported_cost_on_an_unpriced_model_stays_zero(self, patch_exec, tmp_path): """With no rate to fall back to, the stream's 0 is the best information we have.""" diff --git a/tests/test_price_turn.py b/tests/test_price_turn.py new file mode 100644 index 00000000..cbae99ac --- /dev/null +++ b/tests/test_price_turn.py @@ -0,0 +1,204 @@ +"""``pricing.price_turn``: the one rule for a turn's cost, shared by every adapter and the monitor.""" + +from __future__ import annotations + +import asyncio +import json +import os +from types import SimpleNamespace +from typing import Any +from unittest.mock import patch + +import pytest + +from coder_eval.models import RunLimits, TokenUsage +from coder_eval.orchestration.turn_monitor import TurnMonitor +from coder_eval.pricing import calculate_cost, price_turn +from coder_eval.streaming.events import AgentEndEvent, AgentStartEvent, StreamEvent + + +_HAIKU = "claude-haiku-4-5" +_USAGE = TokenUsage( + uncached_input_tokens=1000, output_tokens=500, cache_creation_input_tokens=10, cache_read_input_tokens=20 +) + + +def _rate(model: str, usage: TokenUsage = _USAGE) -> float: + cost = calculate_cost( + model, + usage.uncached_input_tokens, + usage.output_tokens, + usage.cache_creation_input_tokens, + usage.cache_read_input_tokens, + ) + assert cost is not None + return cost + + +def _reported(cost: float | None, usage: TokenUsage = _USAGE) -> TokenUsage: + return usage.model_copy(update={"total_cost_usd": cost}) + + +class TestTheRule: + def test_a_finite_non_zero_reported_cost_wins(self) -> None: + assert price_turn(_reported(0.42), (_HAIKU,)) == 0.42 + + @pytest.mark.parametrize("reported", [None, 0.0, 1.5]) + def test_empty_usage_returns_the_reported_cost_unchanged(self, reported: float | None) -> None: + assert price_turn(TokenUsage(total_cost_usd=reported), (_HAIKU,)) == reported + + def test_the_rate_card_prices_an_unreported_cost(self) -> None: + assert price_turn(_USAGE, (_HAIKU,)) == pytest.approx(_rate(_HAIKU)) + + def test_a_reported_zero_on_a_priced_model_uses_the_rate_card(self, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level("DEBUG", logger="coder_eval.pricing"): + assert price_turn(_reported(0.0), (_HAIKU,)) == pytest.approx(_rate(_HAIKU)) + assert "using the rate card" in caplog.text + + def test_a_reported_zero_that_no_model_prices_stays_zero(self) -> None: + assert price_turn(_reported(0.0), ("nowhere/not-a-model", None)) == 0.0 + + def test_nothing_reported_and_nothing_priced_is_none(self) -> None: + assert price_turn(_USAGE, ("nowhere/not-a-model", None)) is None + assert price_turn(_USAGE, ()) is None + + @pytest.mark.parametrize("reported", [float("nan"), float("inf"), float("-inf")]) + def test_a_non_finite_reported_cost_counts_as_unreported(self, reported: float) -> None: + assert price_turn(_reported(reported), (_HAIKU,)) == pytest.approx(_rate(_HAIKU)) + assert price_turn(_reported(reported), ("nowhere/not-a-model",)) is None + assert price_turn(TokenUsage(total_cost_usd=reported), (_HAIKU,)) is None + + def test_the_first_priced_model_wins_and_none_is_skipped(self) -> None: + models = (None, "", "nowhere/not-a-model", "claude-sonnet-4-6", _HAIKU) + assert price_turn(_USAGE, models) == pytest.approx(_rate("claude-sonnet-4-6")) + + def test_a_bedrock_prefixed_id_prices_like_the_bare_id(self) -> None: + assert price_turn(_USAGE, ("eu.anthropic.claude-haiku-4-5",)) == pytest.approx(_rate(_HAIKU)) + + +class _Recorder: + def __init__(self) -> None: + self.events: list[StreamEvent] = [] + + def on_event(self, event: StreamEvent) -> None: + self.events.append(event) + + +def _monitor_cost(events: list[StreamEvent], model: str, reported: float | None) -> float | None: + """The monitor's price for the same turn, fed the harness's RAW report instead of the adapter's price.""" + monitor = TurnMonitor("t", [], limits=RunLimits(max_usd=1000.0), model=model) + for event in events: + if isinstance(event, AgentEndEvent): + event = event.model_copy(update={"usage": _reported(reported, event.usage)}) + monitor.on_event(event) + return monitor.cost_usd() + + +def _adapter_cost(events: list[StreamEvent]) -> float | None: + ends = [e for e in events if isinstance(e, AgentEndEvent)] + assert len(ends) == 1 + return ends[0].usage.total_cost_usd + + +async def _run_cli(agent: Any, cli: str, lines: list[str], working_dir: str) -> list[StreamEvent]: + from tests._fixtures.golden_streams.pi_fixtures import _FakeProcess + + proc = _FakeProcess(lines) + + async def fake_exec(*_argv: str, **_kwargs: Any) -> _FakeProcess: + proc.stderr = proc # type: ignore[assignment] + return proc + + recorder = _Recorder() + with ( + patch.object(asyncio, "create_subprocess_exec", fake_exec), + patch("shutil.which", lambda _name: f"/usr/local/bin/{cli}"), + patch.object(os, "killpg", lambda _pgid, _sig: None, create=True), + ): + await agent.start(working_dir) + await agent.communicate("do it", stream_callback=recorder) + return recorder.events + + +def _pi_lines(cost: float) -> list[str]: + from tests._fixtures.golden_streams.pi_fixtures import _turn_end, _turn_start + + return [_turn_start(), _turn_end(inp=1000, out=500, cost=cost)] + + +def _opencode_lines(cost: float) -> list[str]: + tokens = {"total": 1500, "input": 1000, "output": 500, "reasoning": 0, "cache": {"write": 0, "read": 0}} + part = {"sessionID": "ses_1", "id": "prt_2", "messageID": "msg_1", "reason": "stop", "cost": cost, "tokens": tokens} + return [ + json.dumps({"type": "step_start", "sessionID": "ses_1", "part": {"sessionID": "ses_1", "id": "prt_1"}}), + json.dumps({"type": "step_finish", "sessionID": "ses_1", "part": part}), + ] + + +class TestAdapterAndMonitorAgree: + """Each harness's published turn cost equals what the ``max_usd`` monitor sums for the same turn.""" + + @pytest.mark.parametrize("cost", [0.25, 0.0]) + async def test_pi(self, cost: float, tmp_path: Any) -> None: + from coder_eval.agents.pi_agent import PiAgent + from coder_eval.models import PiAgentConfig + + model = "openrouter/moonshotai/kimi-k3" + agent = PiAgent(PiAgentConfig(type="pi", model=model), task_id="t") + events = await _run_cli(agent, "pi", _pi_lines(cost), str(tmp_path)) + expected = cost or _rate(model, TokenUsage(uncached_input_tokens=1000, output_tokens=500)) + assert _adapter_cost(events) == pytest.approx(expected) + assert _monitor_cost(events, model, cost) == pytest.approx(expected) + + @pytest.mark.parametrize("cost", [0.25, 0.0]) + async def test_opencode(self, cost: float, tmp_path: Any) -> None: + from coder_eval.agents.opencode_agent import OpenCodeAgent + from coder_eval.models import OpenCodeAgentConfig + + model = "deepseek/deepseek-v4-pro" + agent = OpenCodeAgent(OpenCodeAgentConfig(type="opencode", model=model), task_id="t") + events = await _run_cli(agent, "opencode", _opencode_lines(cost), str(tmp_path)) + expected = cost or _rate(model, TokenUsage(uncached_input_tokens=1000, output_tokens=500)) + assert _adapter_cost(events) == pytest.approx(expected) + assert _monitor_cost(events, model, cost) == pytest.approx(expected) + + async def test_antigravity(self, tmp_path: Any) -> None: + from coder_eval.agents import antigravity_agent + from tests._fixtures.golden_streams.antigravity_fixtures import _agent_with_steps, _no_sleep, _step, _usage + + steps = [_step("TEXT_RESPONSE", "DONE", content="ok", complete=True, usage=_usage(1000, 200, 300, 50))] + agent = _agent_with_steps(steps) + agent.working_directory = tmp_path + recorder = _Recorder() + with patch.object(antigravity_agent.asyncio, "sleep", _no_sleep): + await agent.communicate("do it", stream_callback=recorder) + expected = _rate( + "gemini-3.5-flash", TokenUsage(uncached_input_tokens=800, output_tokens=350, cache_read_input_tokens=200) + ) + assert _adapter_cost(recorder.events) == pytest.approx(expected) + assert _monitor_cost(recorder.events, "gemini-3.5-flash", None) == pytest.approx(expected) + + def test_codex(self) -> None: + from coder_eval.agents.codex_agent import CodexAgent + from coder_eval.models import CodexAgentConfig + + model = "gpt-5.6-terra" + agent = CodexAgent(CodexAgentConfig(type="codex", model=model)) + sdk = SimpleNamespace(total=SimpleNamespace(input_tokens=1000, output_tokens=500, cached_input_tokens=400)) + usage = agent._token_usage_from_sdk(sdk) + expected = _rate(model, TokenUsage(uncached_input_tokens=600, output_tokens=500, cache_read_input_tokens=400)) + assert usage is not None and usage.total_cost_usd == pytest.approx(expected) + events: list[StreamEvent] = [AgentStartEvent(task_id="t"), AgentEndEvent(task_id="t", usage=usage)] + assert _monitor_cost(events, model, None) == pytest.approx(expected) + + @pytest.mark.parametrize("sdk_cost", [None, 0.0, 0.33]) + def test_claude(self, sdk_cost: float | None) -> None: + from coder_eval.agents.claude_code_agent import ClaudeCodeAgent + + sdk_usage = {"input_tokens": 1000, "output_tokens": 500, "cache_read_input_tokens": 20} + usage = ClaudeCodeAgent._build_token_usage([], sdk_usage, sdk_cost, None, _HAIKU) + assert usage is not None and usage.total_cost_usd is not None + expected = sdk_cost or _rate(_HAIKU, usage) + assert usage.total_cost_usd == pytest.approx(expected) + events: list[StreamEvent] = [AgentStartEvent(task_id="t"), AgentEndEvent(task_id="t", usage=usage)] + assert _monitor_cost(events, _HAIKU, sdk_cost) == pytest.approx(expected) diff --git a/tests/test_turn_monitor.py b/tests/test_turn_monitor.py index dfcd86db..0b1bc2f7 100644 --- a/tests/test_turn_monitor.py +++ b/tests/test_turn_monitor.py @@ -403,7 +403,21 @@ def test_a_sub_agent_model_on_the_stream_does_not_reprice_the_configured_model(s assert monitor.cost_usd() == pytest.approx(calculate_cost("claude-sonnet-4-6", 1_000_000, 0)) - def test_a_non_finite_reported_cost_is_unpriceable(self) -> None: + def test_a_reported_zero_on_a_priced_model_is_priced_from_the_rate_card(self) -> None: + task = _task(limits=RunLimits(max_usd=0.50), model="claude-haiku-4-5") + monitor = TurnMonitor.for_task(task, arm=True) + _feed(monitor, _turn(TokenUsage(uncached_input_tokens=1_000_000, total_cost_usd=0.0))) + assert monitor.cost_usd() == pytest.approx(1.0) + assert monitor.should_stop() is StopReason.USD_BUDGET + + def test_a_reported_zero_on_an_unpriced_model_is_an_enforceable_zero(self) -> None: + task = _task(limits=RunLimits(max_usd=0.10), model="openrouter/free/not-on-the-card") + monitor = TurnMonitor.for_task(task, arm=True) + _feed(monitor, _turn(TokenUsage(output_tokens=100, total_cost_usd=0.0))) + assert monitor.cost_usd() == 0.0 + monitor.raise_if_over_budget(iteration=1) + + def test_a_non_finite_reported_cost_on_an_unpriced_model_is_unpriceable(self) -> None: monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_usd=0.10)), arm=True) _feed(monitor, _turn(TokenUsage(output_tokens=10, total_cost_usd=float("nan")))) with pytest.raises(BudgetUnenforceableError): From 644e26e0e9633e35808b6f7099c0cc3fe94da10a Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 17:02:13 -0700 Subject: [PATCH 02/25] =?UTF-8?q?feat(streaming):=202/10=20=E2=80=94=20Tur?= =?UTF-8?q?nEmitter,=20Window,=20TimingBasis=20and=20coder=5Feval.testing;?= =?UTF-8?q?=20SPI=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernel the ports build on. close_window returns a Window; every contract declares timing_basis; TurnEmitter is the one writer of the event protocol; coder_eval.testing gives replay, identity, balance and conformance sensors. EventCollector records nested tool ends and derives assistant_turn_index from the messages; TurnMonitor counts main-thread tool calls only. Every golden stream is checked with assert_stream_balanced (pi_f is a strict xfail). Reviewed golden diff (assistant_turn_index only): pi_b 1,2 -> 0,1; pi_c, pi_d, opencode_b, opencode_c, opencode_d 1 -> 0; codex_b, codex_d, codex_e null -> 0; codex_f null,null -> 0,0; antigravity_b, antigravity_c null -> 0. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/orchestration.md | 12 + docs/EXTENDING.md | 16 +- docs/REPORT_SCHEMA.md | 3 +- docs/agents/HARNESS_PARITY.md | 3 +- src/coder_eval/agents/antigravity_agent.py | 10 +- src/coder_eval/agents/claude_code_agent.py | 12 +- src/coder_eval/agents/codex_agent.py | 12 +- src/coder_eval/agents/noop_agent.py | 2 + src/coder_eval/agents/opencode_agent.py | 10 +- src/coder_eval/agents/pi_agent.py | 10 +- src/coder_eval/models/__init__.py | 9 +- src/coder_eval/models/harness_contract.py | 15 + src/coder_eval/models/limits.py | 2 +- src/coder_eval/models/results.py | 28 +- src/coder_eval/orchestration/turn_monitor.py | 12 + src/coder_eval/pricing.py | 16 +- src/coder_eval/spi.py | 11 +- src/coder_eval/streaming/collector.py | 35 +- src/coder_eval/streaming/emitter.py | 551 +++++++++++++++++ src/coder_eval/testing.py | 318 ++++++++++ src/coder_eval/timing.py | 24 +- tests/_fixtures/golden_streams/_recorder.py | 11 + .../golden_streams/antigravity_fixtures.py | 11 +- .../golden_streams/claude_fixtures.py | 13 +- .../golden_streams/codex_fixtures.py | 11 +- .../antigravity_b_tool_call_resolved.json | 2 +- ...y_c_thinking_and_tool_same_generation.json | 2 +- .../expected/codex_b_command_execution.json | 2 +- .../codex_d_cross_flush_is_error.json | 2 +- .../expected/codex_e_orphan_tool.json | 2 +- .../expected/codex_f_collab_fallback.json | 4 +- .../opencode_b_tool_call_resolved.json | 2 +- .../opencode_c_multi_step_tiling.json | 2 +- .../expected/opencode_d_orphaned_tool.json | 2 +- .../expected/pi_b_tool_call_resolved.json | 4 +- .../expected/pi_c_multi_turn_tiling.json | 2 +- .../expected/pi_d_orphaned_tool.json | 2 +- .../golden_streams/opencode_fixtures.py | 13 +- tests/_fixtures/golden_streams/pi_fixtures.py | 11 +- tests/fixtures/byoa_demo_plugin/byoa_demo.py | 2 +- tests/fixtures/harness_stubs.py | 3 +- tests/lint/harness_parity.py | 4 +- tests/test_agent_golden_master.py | 70 ++- tests/test_event_collector.py | 74 ++- tests/test_harness_conformance.py | 86 +-- tests/test_harness_contract.py | 12 +- tests/test_spi.py | 9 +- tests/test_testing_harness.py | 227 +++++++ tests/test_timing_close_window.py | 31 +- tests/test_timing_identity_contract.py | 68 +-- tests/test_turn_emitter.py | 567 ++++++++++++++++++ tests/test_turn_monitor.py | 49 ++ 52 files changed, 2151 insertions(+), 260 deletions(-) create mode 100644 src/coder_eval/streaming/emitter.py create mode 100644 src/coder_eval/testing.py create mode 100644 tests/_fixtures/golden_streams/_recorder.py create mode 100644 tests/test_testing_harness.py create mode 100644 tests/test_turn_emitter.py diff --git a/.claude/notes/orchestration.md b/.claude/notes/orchestration.md index 121099ee..cfd893e2 100644 --- a/.claude/notes/orchestration.md +++ b/.claude/notes/orchestration.md @@ -475,6 +475,18 @@ while the cap and budgets read counters and must keep running on a run that has criteria. `result.tool_calls_exhausted` still comes from the turn's end status, not the latch, because a cap latched after the agent's last poll stopped nothing. +### The monitor scopes to the main thread + +A sub-agent's events arrive tagged with `parent_thread_id`. The cap and the armed criteria +count and evaluate MAIN-THREAD resolved tool calls only, and a nested `TurnStartEvent` +never becomes the reported model. The cap once tripped on a Claude sub-agent's `Bash` +call before the main thread wrote its answer (decision 2026-09-16): the author capped the +agent they configured, and a sub-agent's calls are already covered by the spawning `Agent` +call. A nested `ToolEndEvent` still reaches the monitor's collector, so its command set +equals the authoritative one, and `TurnRecord.commands` keeps sub-agent calls. Budgets +still add a nested `TurnEndEvent`'s tokens: money a sub-agent spends is spent. +`expected_tool_calls` is a post-run warning over `TurnRecord.commands` and is unchanged. + ### Inert triggers are by design, and the watcher fails open A trigger whose polarity an instance can never decide is INERT, not an error — one diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 995f6d85..5ba95a17 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -48,7 +48,7 @@ signature. from coder_eval.spi import SPI_VERSION, AgentRegistry def register(registry: type[AgentRegistry]) -> None: - assert SPI_VERSION == 2, f"my-agent supports coder_eval SPI 2, not {SPI_VERSION}" + assert SPI_VERSION == 3, f"my-agent supports coder_eval SPI 3, not {SPI_VERSION}" # Bind type string → config class → agent class. registry.register("my-agent", MyAgentConfig)(MyAgent) # Optionally contribute pricing here too (see §3): @@ -99,7 +99,15 @@ at resolution, so `coder-eval plan` fails before any run. This is a JSONL CLI ag that appends a system prompt and honors `plan` and tool lists natively: ```python -from coder_eval.spi import Agent, Enforcement, HarnessContract, PermissionMode, ToolNameMap, UsageGranularity +from coder_eval.spi import ( + Agent, + Enforcement, + HarnessContract, + PermissionMode, + TimingBasis, + ToolNameMap, + UsageGranularity, +) # native tool name -> canonical (Claude) name; also used for telemetry _TOOL_NAME_MAP = {"bash": "Bash", "read": "Read", "write": "Write", "edit": "Edit", "task": "Agent"} @@ -115,6 +123,7 @@ class MyAgent(Agent[MyAgentConfig]): disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, usage_granularity=UsageGranularity.STEP, + timing_basis=TimingBasis.TURN_CLOCK, ) tool_names = ToolNameMap.from_inverse( _TOOL_NAME_MAP, @@ -129,6 +138,9 @@ class MyAgent(Agent[MyAgentConfig]): meaning (`plan` is read-only, `bypassPermissions` runs every permitted tool). - `tool_names` is required exactly when a tool-list row is `ENFORCED`. It must map every canonical name; list a name your harness has no tool for in `no_equivalent`. +- `timing_basis` says who stamps the turn. `TURN_CLOCK`: the `TurnEmitter` stamps every + tool and the turn bracket from one clock. `CLI_EPOCH_MS`: your harness reports its own + stamps, and you pass them for every main-thread tool and window. - Set `cooperative_stop=True` only if your `communicate()` honors `should_stop` (needed for criterion-level `stop_early:` arming and for `run_limits.max_tool_calls` to cut a turn). `False` means early stop is rejected at resolution for your agent. diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index 57aedd04..7f74a5e2 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -230,7 +230,8 @@ canonical score remains 0.0. cache buckets, captured proxy-side on the LiteLLM open-weight backend and rendered by the evalboard as a per-call table; empty on every other backend), `num_turns`, `tool_calls_exhausted`, -`result_summary` (`{is_error, subtype, stop_reason, result}`), `crashed`, +`result_summary` (`{is_error, subtype, stop_reason, result}`: how a clean turn ended, `result` being +the agent's final reply; `null` on a crashed or timed-out turn), `crashed`, `crash_reason`. > **Token invariant.** Summing the four token buckets across `messages` diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 84b193e5..e9703f8c 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -17,7 +17,7 @@ CE069 fails the build on drift. | limit | claude-code | codex | antigravity | opencode | pi | none | | --- | --- | --- | --- | --- | --- | --- | -| `max_tool_calls` | TurnMonitor at the should_stop poll, resolved tool calls | TurnMonitor at the should_stop poll, resolved tool calls | TurnMonitor at the should_stop poll, resolved tool calls | TurnMonitor at the should_stop poll, resolved tool calls | TurnMonitor at the should_stop poll, resolved tool calls | not polled (never fires) | +| `max_tool_calls` | TurnMonitor at the should_stop poll, main-thread resolved tool calls | TurnMonitor at the should_stop poll, main-thread resolved tool calls | TurnMonitor at the should_stop poll, main-thread resolved tool calls | TurnMonitor at the should_stop poll, main-thread resolved tool calls | TurnMonitor at the should_stop poll, main-thread resolved tool calls | not polled (never fires) | | `expected_tool_calls` | orchestrator, cumulative visible tool calls, warns only | orchestrator, cumulative visible tool calls, warns only | orchestrator, cumulative visible tool calls, warns only | orchestrator, cumulative visible tool calls, warns only | orchestrator, cumulative visible tool calls, warns only | orchestrator, cumulative visible tool calls, warns only | | `task_timeout` | orchestrator, agent-agnostic | orchestrator, agent-agnostic | orchestrator, agent-agnostic | orchestrator, agent-agnostic | orchestrator, agent-agnostic | orchestrator, agent-agnostic | | `turn_timeout` | agent watchdog (see Timeouts) | agent watchdog (see Timeouts) | agent watchdog (see Timeouts) | agent watchdog (see Timeouts) | agent watchdog (see Timeouts) | agent watchdog (see Timeouts) | @@ -55,6 +55,7 @@ Generated from each agent class's `contract` by `make parity-table`; CE069 fails | `disallowed_tools` | enforced | unsupported | enforced | enforced | enforced | unsupported | | `cooperative_stop` | yes | yes | yes | yes | yes | no | | `usage_granularity` | generation | turn | turn | step | step | turn | +| `timing_basis` | turn_clock | cli_epoch_ms | turn_clock | mixed | turn_clock | turn_clock | | `permission_modes` | acceptEdits, bypassPermissions, default, plan | — | bypassPermissions, plan | bypassPermissions, plan | bypassPermissions, plan | — | diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index a15ec830..576006c0 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -49,6 +49,7 @@ Enforcement, HarnessContract, PermissionMode, + TimingBasis, TokenUsage, ToolNameMap, TranscriptMessage, @@ -209,6 +210,7 @@ class AntigravityAgent(Agent[AntigravityAgentConfig]): disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, usage_granularity=UsageGranularity.TURN, + timing_basis=TimingBasis.TURN_CLOCK, permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), ) tool_names = _TOOL_NAMES @@ -879,14 +881,14 @@ def _flush_generation(self, gen: TokenUsage, reasoning_tokens: int) -> None: # the collector clips the tool union out of it. Resetting instead drops # the model time around a fast tool. # Rationale: .claude/notes/agents.md § Per-harness generation marks - _, generation_ms = close_window(mark=self._gen_mark_wall, now=now_wall) + window = close_window(mark=self._gen_mark_wall, now=now_wall) for i, block in enumerate(self._blocks): block.sequence = i self.messages.append( AssistantMessage( - started_at=self._gen_mark_wall, - completed_at=now_wall, - generation_duration_ms=generation_ms, + started_at=window.started_at, + completed_at=window.completed_at, + generation_duration_ms=window.duration_ms, content_blocks=list(self._blocks), tool_use_ids=[b.tool_use_id for b in self._blocks if b.block_type == "tool_use" and b.tool_use_id], input_tokens=gen.uncached_input_tokens, diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 5312c7be..d5f3e720 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -57,6 +57,7 @@ PermissionMode, ResultSummary, SystemPromptSemantics, + TimingBasis, TokenUsage, ToolNameMap, TranscriptMessage, @@ -451,14 +452,14 @@ def on_assistant_message(self, message: Message) -> None: out_tok = int(msg_usage.get("output_tokens", 0) or 0) self.pending_delta_output_tokens = None - # The RAW window. `started` is the mark, since this stream carries no + # The RAW window, opened at the mark, since this stream carries no # per-emission item start to pull the window open to. # Rationale: .claude/notes/agents.md § Per-harness generation marks - started, raw_generation_ms = close_window(mark=generation_started_wall, now=message_arrival_wall) + window = close_window(mark=generation_started_wall, now=message_arrival_wall) assistant_telemetry = AssistantMessageTelemetry( - started_at=started, - completed_at=message_arrival_wall, - generation_duration_ms=raw_generation_ms, + started_at=window.started_at, + completed_at=window.completed_at, + generation_duration_ms=window.duration_ms, content_blocks=turn_content_blocks, tool_use_ids=turn_tool_use_ids, input_tokens=in_tok, @@ -716,6 +717,7 @@ class ClaudeCodeAgent(Agent[ClaudeCodeAgentConfig]): disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, usage_granularity=UsageGranularity.GENERATION, + timing_basis=TimingBasis.TURN_CLOCK, permission_modes=frozenset(PermissionMode), ) tool_names = ToolNameMap(names={name: (name,) for name in CANONICAL_TOOL_NAMES}, mcp_names=True) diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index c8ec85dd..952a834e 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -35,6 +35,7 @@ DirectRoute, Enforcement, HarnessContract, + TimingBasis, TokenUsage, TranscriptMessage, TurnRecord, @@ -420,7 +421,7 @@ def _flush_message(self, last: Any) -> None: # the collector takes it back out. That is also what makes the sub-message # split safe: the two specs SHARE these bounds, so the overlap is # subtracted once rather than once per part. - started, gen_ms = close_window( + window = close_window( mark=mark, now=completed, item_start=_ms_to_dt(self.open_start_ms) if self.open_start_ms is not None else None, @@ -451,9 +452,9 @@ def _flush_message(self, last: Any) -> None: assigned = 0.0 for idx, (_, out_tok, _) in enumerate(specs): if idx == len(specs) - 1: - gen_parts.append(gen_ms - assigned) + gen_parts.append(window.duration_ms - assigned) else: - share = round(gen_ms * (out_tok / out_total if out_total > 0 else 1 / len(specs)), 6) + share = round(window.duration_ms * (out_tok / out_total if out_total > 0 else 1 / len(specs)), 6) gen_parts.append(share) assigned += share @@ -463,8 +464,8 @@ def _flush_message(self, last: Any) -> None: first = idx == 0 self.messages.append( AssistantMessage( - started_at=started, - completed_at=completed, + started_at=window.started_at, + completed_at=window.completed_at, generation_duration_ms=gen_parts[idx], content_blocks=blocks, tool_use_ids=[b.tool_use_id for b in blocks if b.block_type == "tool_use" and b.tool_use_id], @@ -752,6 +753,7 @@ class CodexAgent(Agent[CodexAgentConfig]): disallowed_tools=Enforcement.UNSUPPORTED, cooperative_stop=True, usage_granularity=UsageGranularity.TURN, + timing_basis=TimingBasis.CLI_EPOCH_MS, ) def __init__( diff --git a/src/coder_eval/agents/noop_agent.py b/src/coder_eval/agents/noop_agent.py index e49860fe..88bf05f7 100644 --- a/src/coder_eval/agents/noop_agent.py +++ b/src/coder_eval/agents/noop_agent.py @@ -26,6 +26,7 @@ Enforcement, HarnessContract, NoneAgentConfig, + TimingBasis, TurnRecord, UsageGranularity, ) @@ -62,6 +63,7 @@ class NoOpAgent(Agent[NoneAgentConfig]): disallowed_tools=Enforcement.UNSUPPORTED, cooperative_stop=False, usage_granularity=UsageGranularity.TURN, + timing_basis=TimingBasis.TURN_CLOCK, ) def __init__( diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index bda5c757..8fdf8f3d 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -47,6 +47,7 @@ OpenCodeAgentConfig, PermissionMode, ResultSummary, + TimingBasis, TokenUsage, ToolNameMap, TranscriptMessage, @@ -581,16 +582,16 @@ def on_step_finish(self, part: dict[str, Any]) -> None: blocks.append(ContentBlock(block_type="tool_use", sequence=i, tool_use_id=tool_id)) # Tile from the previous step's finish. The RAW window only. - started, generation_ms = close_window( + window = close_window( mark=self.gen_mark if self.gen_mark is not None else step_start, now=completed, item_start=step_start, ) self.messages.append( AssistantMessage( - started_at=started, - completed_at=completed, - generation_duration_ms=generation_ms, + started_at=window.started_at, + completed_at=window.completed_at, + generation_duration_ms=window.duration_ms, content_blocks=blocks, tool_use_ids=list(self.step_tool_ids), input_tokens=step_in, @@ -724,6 +725,7 @@ class OpenCodeAgent(Agent[OpenCodeAgentConfig]): disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, usage_granularity=UsageGranularity.STEP, + timing_basis=TimingBasis.MIXED, permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), ) tool_names = _TOOL_NAMES diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 0e25943b..d681a652 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -55,6 +55,7 @@ PermissionMode, PiAgentConfig, ResultSummary, + TimingBasis, TokenUsage, ToolNameMap, TranscriptMessage, @@ -527,16 +528,16 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: # Tile from the previous turn's end. The RAW window only. turn_start = self.turn_started_at if self.turn_started_at is not None else completed - started, generation_ms = close_window( + window = close_window( mark=self.gen_mark if self.gen_mark is not None else turn_start, now=completed, item_start=turn_start, ) self.messages.append( AssistantMessage( - started_at=started, - completed_at=completed, - generation_duration_ms=generation_ms, + started_at=window.started_at, + completed_at=window.completed_at, + generation_duration_ms=window.duration_ms, content_blocks=blocks, tool_use_ids=list(self.turn_tool_ids), input_tokens=step_in, @@ -652,6 +653,7 @@ class PiAgent(Agent[PiAgentConfig]): disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, usage_granularity=UsageGranularity.STEP, + timing_basis=TimingBasis.TURN_CLOCK, permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), ) tool_names = _TOOL_NAMES diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index a999658f..1c9bbab1 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -94,7 +94,13 @@ ) # Harness contract -from coder_eval.models.harness_contract import Enforcement, HarnessContract, ToolNameMap, UsageGranularity +from coder_eval.models.harness_contract import ( + Enforcement, + HarnessContract, + TimingBasis, + ToolNameMap, + UsageGranularity, +) # Judge from coder_eval.models.judge import JudgeVerdict @@ -253,6 +259,7 @@ "Enforcement", "HarnessContract", "ToolNameMap", + "TimingBasis", "UsageGranularity", # Enums "AgentKind", diff --git a/src/coder_eval/models/harness_contract.py b/src/coder_eval/models/harness_contract.py index 4d9e9e07..ff3f83cf 100644 --- a/src/coder_eval/models/harness_contract.py +++ b/src/coder_eval/models/harness_contract.py @@ -27,6 +27,14 @@ class UsageGranularity(StrEnum): TURN = "turn" +class TimingBasis(StrEnum): + """Where a harness's recorded stamps come from, which decides who stamps a tool and a window.""" + + TURN_CLOCK = "turn_clock" + CLI_EPOCH_MS = "cli_epoch_ms" + MIXED = "mixed" + + class HarnessContract(BaseModel): """The per-agent declaration of which uniform fields reach the harness. @@ -56,6 +64,13 @@ class HarnessContract(BaseModel): "per communicate() call. A budget can overshoot by one such report." ) ) + timing_basis: TimingBasis = Field( + description=( + "Where recorded stamps come from: turn_clock (the TurnEmitter stamps the turn bracket, every tool " + "and every window from one TurnClock) or cli_epoch_ms (the adapter passes the CLI's own stamps for " + "windows and main-thread tools). mixed is OpenCode's interim value." + ) + ) permission_modes: frozenset[PermissionMode] | None = Field( default=None, description=( diff --git a/src/coder_eval/models/limits.py b/src/coder_eval/models/limits.py index e893e148..c3066f15 100644 --- a/src/coder_eval/models/limits.py +++ b/src/coder_eval/models/limits.py @@ -41,7 +41,7 @@ class RunLimits(BaseModel): default=None, gt=0, description=( - "Hard cap on resolved tool calls across the whole task (every retry attempt and every " + "Hard cap on main-thread resolved tool calls across the whole task (every retry attempt and every " "dialog turn). Enforced by the TurnMonitor at the agent's next poll boundary on every " "harness: the round that reaches the cap is processed whole, so tool calls already in " "flight can still land after it. The run finalizes cleanly as tool_calls_exhausted; " diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 464d643e..7bb0fd1f 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -281,24 +281,20 @@ def _criterion_result_discriminator(v: Any) -> str: class ResultSummary(BaseModel): - """Diagnostic fields lifted from the SDK's final ResultMessage. - - Powers the agent's debug log and the error-path formatter that - surfaces a useful detail string when the CLI crashes. Persisted on - ``TurnRecord`` for clean turns only — on a crash the agent raises - before the TurnRecord is constructed, so post-mortem persistence on - error turns is out of scope here. - - Mirrors the diagnostic-bearing subset of - ``claude_agent_sdk.ResultMessage``; pure accounting fields - (``num_turns``, ``duration_ms``) live on ``TurnRecord`` / - ``TokenUsage``. + """How a clean turn ended, on every harness. + + Persisted on ``TurnRecord`` for clean turns only; a crashed or timed-out turn + carries none, and its failure is in ``crash_reason``. ``result`` is the agent's + final reply: the text of the last main-thread assistant message when that + message calls no tool. A harness with its own final summary may pass that + instead. Accounting fields (``num_turns``, durations) live on + ``TurnRecord`` / ``TokenUsage``. """ - is_error: bool = Field(description="Whether the SDK reported the turn as errored") - subtype: str = Field(description="Coarse classification (e.g. 'success', 'error_during_execution')") + is_error: bool = Field(description="Whether the harness reported the finished turn as errored") + subtype: str = Field(description="Coarse classification: the end status, or the harness's own subtype") stop_reason: str | None = Field(default=None, description="Why the model stopped, if reported") - result: str | None = Field(default=None, description="Free-form result/error text from the SDK") + result: str | None = Field(default=None, description="The agent's final reply text, or the harness's result text") class TurnRecord(BaseModel): @@ -404,7 +400,7 @@ class TurnRecord(BaseModel): ) result_summary: ResultSummary | None = Field( default=None, - description="SDK ResultMessage summary, when one was emitted (clean turns or partials that got one).", + description="How a clean turn ended, including the agent's final reply; None on a crashed or timed-out turn.", ) provider_call_costs: list[ProviderCallCost] = Field( default_factory=list, diff --git a/src/coder_eval/orchestration/turn_monitor.py b/src/coder_eval/orchestration/turn_monitor.py index 39d38a6c..acd621df 100644 --- a/src/coder_eval/orchestration/turn_monitor.py +++ b/src/coder_eval/orchestration/turn_monitor.py @@ -189,9 +189,14 @@ def _on_event_impl(self, event: StreamEvent) -> None: monitor would reduce a strictly smaller command set than the authoritative check. The cap counts distinct resolved tool ids. + A nested (sub-agent) event is main-thread-scoped out: its tool end is recorded + but never counted or evaluated, its turn start sets no model, and only its + turn-end tokens count, toward the budgets. + Rationale: .claude/notes/orchestration.md § Verdicts latch, and the decision happens on the CALL """ if event.parent_thread_id is not None: + self._on_nested_event(event) return if isinstance(event, AgentStartEvent): if self._started_monotonic is None: @@ -223,6 +228,13 @@ def _on_event_impl(self, event: StreamEvent) -> None: return self._collector.on_event(event) + def _on_nested_event(self, event: StreamEvent) -> None: + if isinstance(event, ToolEndEvent): + self._collector.on_event(event) + elif isinstance(event, TurnEndEvent) and event.tokens is not None: + self._in_flight += event.tokens + self._evaluate_budgets() + def should_stop(self) -> StopReason | None: """The cooperative poll the agent calls at each safe boundary.""" return self._stop_reason diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py index eb5748a0..1665ca4a 100644 --- a/src/coder_eval/pricing.py +++ b/src/coder_eval/pricing.py @@ -18,11 +18,19 @@ from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING +from typing import Protocol -if TYPE_CHECKING: - from coder_eval.models import TokenUsage +class _TurnUsage(Protocol): + """The ``TokenUsage`` shape ``price_turn`` reads; a protocol, because models import this module.""" + + uncached_input_tokens: int + output_tokens: int + cache_creation_input_tokens: int + cache_read_input_tokens: int + total_cost_usd: float | None + + def is_empty(self) -> bool: ... logger = logging.getLogger(__name__) @@ -256,7 +264,7 @@ def calculate_cost( ) / 1_000_000 -def price_turn(usage: "TokenUsage", models: Sequence[str | None]) -> float | None: +def price_turn(usage: _TurnUsage, models: Sequence[str | None]) -> float | None: """The cost of one turn: ``usage.total_cost_usd`` is what the harness reported. In order: a finite, non-zero reported cost wins; empty usage returns the reported diff --git a/src/coder_eval/spi.py b/src/coder_eval/spi.py index 6082ac39..33e7e058 100644 --- a/src/coder_eval/spi.py +++ b/src/coder_eval/spi.py @@ -23,6 +23,7 @@ PermissionMode, ResultSummary, SystemPromptMode, + TimingBasis, TokenUsage, ToolNameMap, TranscriptMessage, @@ -32,6 +33,7 @@ from coder_eval.pricing import ModelPricing, register_pricing from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.emitter import Generation, TurnEmitter, TurnOutcome from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -46,10 +48,10 @@ TurnStartEvent, end_status_for, ) -from coder_eval.timing import TurnClock, close_window +from coder_eval.timing import TurnClock, Window, close_window -SPI_VERSION: Final[int] = 2 +SPI_VERSION: Final[int] = 3 __all__ = [ # noqa: RUF022 - plain sort, pinned by tests/test_spi.py "Agent", @@ -67,6 +69,7 @@ "CompositeStreamCallback", "Enforcement", "EventCollector", + "Generation", "HarnessContract", "LocalPluginConfig", "ModelPricing", @@ -78,6 +81,7 @@ "StreamCallback", "SystemPromptMode", "TextChunkEvent", + "TimingBasis", "TokenUsage", "ToolEndEvent", "ToolEndStatus", @@ -85,12 +89,15 @@ "ToolStartEvent", "TranscriptMessage", "TurnClock", + "TurnEmitter", "TurnEndEvent", "TurnEndStatus", + "TurnOutcome", "TurnRecord", "TurnStartEvent", "TurnTimeoutError", "UsageGranularity", + "Window", "close_window", "end_status_for", "register_pricing", diff --git a/src/coder_eval/streaming/collector.py b/src/coder_eval/streaming/collector.py index 27971d63..c9bb0603 100644 --- a/src/coder_eval/streaming/collector.py +++ b/src/coder_eval/streaming/collector.py @@ -42,8 +42,9 @@ class EventCollector: Tolerant by construction: ``build_turn_record()`` can be called at any point (including mid-stream after a crash) and returns the best record derivable from the events seen so far. Sub-agent activity is captured as - ``parent_tool_use_id``-tagged messages in the transcript; per-sub-agent - attribution is derived by grouping those messages, not from a separate field. + ``parent_tool_use_id``-tagged messages in the transcript. A nested event + (``parent_thread_id`` set) contributes only its ``ToolEndEvent`` to ``commands``; + it sets no model and counts no turn. """ def __init__(self) -> None: @@ -57,10 +58,15 @@ def __init__(self) -> None: self._commands: dict[str, CommandTelemetry] = {} self._agent_end: AgentEndEvent | None = None + @property + def ended(self) -> bool: + """True once the current attempt's ``AgentEndEvent`` has been seen.""" + return self._agent_end is not None + def on_event(self, event: StreamEvent) -> None: - # Only the main agent's own events shape its TurnRecord. Forward-looking: - # no agent emits nested sub-agent events yet, so this never fires today. if event.parent_thread_id is not None: + if isinstance(event, ToolEndEvent): + self._commands[event.tool.tool_id] = event.tool return if isinstance(event, AgentStartEvent): @@ -82,8 +88,20 @@ def on_event(self, event: StreamEvent) -> None: elif isinstance(event, AgentEndEvent): self._agent_end = event - def _ordered_commands(self) -> list[CommandTelemetry]: - return sorted(self._commands.values(), key=lambda c: c.sequence_number) + def _ordered_commands(self, messages: list[TranscriptMessage]) -> list[CommandTelemetry]: + """Commands by ``sequence_number``, each a copy carrying its derived ``assistant_turn_index``. + + The index is the position, among the ``AssistantMessage`` entries, of the first + message whose ``tool_use_ids`` names the command; ``None`` when none does. + """ + owner: dict[str, int] = {} + for index, message in enumerate(m for m in messages if isinstance(m, AssistantMessage)): + for tool_id in message.tool_use_ids: + owner.setdefault(tool_id, index) + return [ + command.model_copy(update={"assistant_turn_index": owner.get(command.tool_id)}) + for command in sorted(self._commands.values(), key=lambda c: c.sequence_number) + ] def _overhead_ms( self, messages: list[TranscriptMessage], tool_spans: list[tuple[datetime, datetime]] @@ -175,7 +193,6 @@ def _reconciled_messages(messages: list[TranscriptMessage], usage: TokenUsage) - def build_turn_record(self) -> TurnRecord: """Assemble the ``TurnRecord`` from the events observed so far.""" end = self._agent_end - commands = self._ordered_commands() if end is None: # No terminal event yet (mid-stream snapshot): minimal record. @@ -183,7 +200,7 @@ def build_turn_record(self) -> TurnRecord: iteration=self._iteration, user_input=self._user_input, agent_output="", - commands=commands, + commands=self._ordered_commands([]), token_usage=None, model_used=self._model, assistant_turn_count=self._turn_starts, @@ -216,7 +233,7 @@ def build_turn_record(self) -> TurnRecord: iteration=end.iteration or self._iteration, user_input=end.user_input or self._user_input, agent_output=end.agent_output, - commands=commands, + commands=self._ordered_commands(messages), duration_seconds=end.duration_seconds, token_usage=token_usage, model_used=end.model_used or self._model, diff --git a/src/coder_eval/streaming/emitter.py b/src/coder_eval/streaming/emitter.py new file mode 100644 index 00000000..a3cf65f2 --- /dev/null +++ b/src/coder_eval/streaming/emitter.py @@ -0,0 +1,551 @@ +"""TurnEmitter: the one writer of the event protocol for one ``communicate()`` turn. + +An adapter opens one per turn (``Agent._open_emitter``), calls ``begin``, reports what +its harness did through the write methods, and returns ``finalize(...)`` or +``fail(...)``. The emitter owns every per-turn value: open tools, sequence numbers, +the transcript, reported usage, text output, the open inner turn and the end. + +Rationale: .claude/notes/agents.md § Shared turn lifecycle +""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Literal, Protocol + +from coder_eval.errors import AgentCrashError, TurnTimeoutError +from coder_eval.errors.agent import truncate_crash_message +from coder_eval.models import ( + AssistantMessage, + CommandTelemetry, + ContentBlock, + ResultSummary, + TimingBasis, + TokenUsage, + TurnRecord, +) +from coder_eval.streaming.callbacks import StreamCallback, safe_emit +from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentEndStatus, + AgentStartEvent, + StreamEvent, + TextChunkEvent, + ToolEndEvent, + ToolEndStatus, + ToolStartEvent, + TurnEndEvent, + TurnEndStatus, + TurnStartEvent, +) +from coder_eval.timing import Window + + +logger = logging.getLogger(__name__) + +_UNSET: Any = object() +_FAILED = (AgentEndStatus.CRASHED, AgentEndStatus.TIMEOUT) +_BUCKETS = ("uncached_input_tokens", "output_tokens", "cache_creation_input_tokens", "cache_read_input_tokens") +_RESULT_STATUS: dict[ToolEndStatus, Literal["success", "error", "unknown"]] = { + ToolEndStatus.OK: "success", + ToolEndStatus.ERROR: "error", + ToolEndStatus.PERMISSION_DENIED: "error", + ToolEndStatus.UNRESOLVED: "unknown", +} + + +class Clock(Protocol): + """The source of every stamp the emitter writes.""" + + def now(self) -> datetime: ... + + +@dataclass(frozen=True) +class Generation: + """One sub-message of one model generation; ``tokens`` is this part's own delta.""" + + blocks: list[ContentBlock] + tokens: TokenUsage + reasoning_tokens: int = 0 + stop_reason: str | None = None + + +@dataclass(frozen=True) +class TurnOutcome: + """What a ``communicate()`` turn produced. + + ``error`` is the untruncated failure message, set only for ``CRASHED`` and ``TIMEOUT``. + """ + + record: TurnRecord + status: AgentEndStatus + error: str | None + + def record_or_raise( + self, *, timeout_seconds: float | None = None, task_id: str | None = None, iteration: int | None = None + ) -> TurnRecord: + """The record, or the exception the orchestrator's retry policy classifies. + + Raises: + AgentCrashError: the status is ``CRASHED``. + TurnTimeoutError: the status is ``TIMEOUT``. + """ + if self.status is AgentEndStatus.CRASHED: + raise AgentCrashError(self.error or "agent turn crashed") + if self.status is AgentEndStatus.TIMEOUT: + raise TurnTimeoutError(timeout_seconds or 0.0, task_id=task_id, iteration=iteration) + return self.record + + +@dataclass +class _OpenTool: + telemetry: CommandTelemetry + turn_id: str + parent_tool_id: str | None + + +class TurnEmitter: + """The sole writer of the event protocol for one turn. + + Every event goes to an internal ``EventCollector`` first, then to each sink through + ``safe_emit``, stamped ``clock.now()``. After ``finalize`` or ``fail`` every write + is dropped. ``RuntimeError`` and ``TypeError`` from a write mean a harness bug. + + Rationale: .claude/notes/agents.md § Shared turn lifecycle + """ + + def __init__( + self, + *, + task_id: str, + iteration: int, + prompt: str, + model: str | None, + basis: TimingBasis, + clock: Clock, + sinks: Sequence[StreamCallback], + ) -> None: + self._task_id = task_id + self._iteration = iteration + self._prompt = prompt + self._model = model + self._basis = basis + self._clock = clock + self._sinks = list(sinks) + self._collector = EventCollector() + self._began = False + self._began_monotonic: float | None = None + self._outcome: TurnOutcome | None = None + self._ending = False + self._dropped_logged = False + self._open_tools: dict[str, _OpenTool] = {} + self._sequence = 0 + self._messages: list[AssistantMessage] = [] + self._text: list[str] = [] + self._turn_id: str | None = None + self._turn_parent: str | None = None + self._main_turns = 0 + self._reported = TokenUsage() + + @property + def iteration(self) -> int: + return self._iteration + + @property + def inner_turn_open(self) -> bool: + return self._turn_id is not None + + def now(self) -> datetime: + return self._clock.now() + + def begin(self) -> None: + """Emit the ``AgentStartEvent``; once per emitter.""" + if self._began: + raise RuntimeError("TurnEmitter.begin() called twice") + self._began = True + self._began_monotonic = time.monotonic() + self._emit( + AgentStartEvent(task_id=self._task_id, prompt=self._prompt, iteration=self._iteration, model=self._model) + ) + + def begin_inner_turn(self, turn_id: str, model: str | None = None, *, parent_tool_id: str | None = None) -> None: + """Open one inner turn; raises ``RuntimeError`` while another is open.""" + if self._ended(): + return + if self._turn_id is not None: + raise RuntimeError(f"inner turn {turn_id!r} begun while {self._turn_id!r} is still open") + self._turn_id = turn_id + self._turn_parent = parent_tool_id + if parent_tool_id is None: + self._main_turns += 1 + model = model or self._model + self._emit(TurnStartEvent(task_id=self._task_id, turn_id=turn_id, model=model), parent_tool_id) + + def end_inner_turn( + self, status: TurnEndStatus = TurnEndStatus.COMPLETED, *, tokens: TokenUsage | None = None + ) -> None: + """Close the open inner turn, adding ``tokens`` (a delta) to the reported usage.""" + if self._ended(): + return + if self._turn_id is None: + raise RuntimeError("end_inner_turn() with no inner turn open") + if tokens is not None: + self._reported += tokens + turn_id, parent = self._turn_id, self._turn_parent + self._turn_id = self._turn_parent = None + self._emit(TurnEndEvent(task_id=self._task_id, turn_id=turn_id, status=status, tokens=tokens), parent) + + def text(self, chunk: str, *, parent_tool_id: str | None = None) -> None: + """Stream visible assistant text; main-thread chunks form the default ``agent_output``.""" + if self._ended(): + return + if parent_tool_id is None: + self._text.append(chunk) + self._emit(TextChunkEvent(task_id=self._task_id, turn_id=self._turn_id or "", text=chunk), parent_tool_id) + + def open_tool( + self, + tool_id: str, + name: str, + params: dict[str, Any], + *, + parent_tool_id: str | None = None, + started_at: datetime | None = _UNSET, + generation_completed: bool = False, + ) -> None: + """Record a tool call's start. + + Raises: + TypeError: ``started_at`` passed under ``TURN_CLOCK``, or omitted on a + main-thread tool under ``CLI_EPOCH_MS``. + """ + if self._ended(): + return + now = self.now() + telemetry = CommandTelemetry( + tool_name=name, + tool_id=tool_id, + timestamp=now, + parameters=params, + sequence_number=self._next_sequence(), + execution_started_at=self._stamp("started_at", started_at, now, parent_tool_id), + generation_completed_at=now if generation_completed else None, + ) + turn_id = self._turn_id or "" + self._open_tools[tool_id] = _OpenTool(telemetry, turn_id, parent_tool_id) + self._emit(ToolStartEvent(task_id=self._task_id, turn_id=turn_id, tool=telemetry), parent_tool_id) + + def close_tool( + self, + tool_id: str, + *, + status: ToolEndStatus, + summary: str | None = None, + error: str | None = None, + result_data: dict[str, Any] | list[Any] | None = None, + parameters: dict[str, Any] | None = None, + completed_at: datetime | None = _UNSET, + ) -> None: + """Record a tool call's end; an unknown id synthesizes a ``tool_name="unknown"`` call. + + Only a resolved call is timed: ``UNRESOLVED`` keeps ``execution_started_at`` + and sets no completion stamp and no duration. + + Raises: + TypeError: the same basis rule as ``open_tool``, for ``completed_at``. + """ + if self._ended(): + return + opened = self._open_tools.get(tool_id) + stamp = self._stamp("completed_at", completed_at, self.now(), opened.parent_tool_id if opened else None) + self._close(tool_id, status, summary, error, result_data, parameters, stamp) + + def add_generation( + self, + *, + message_id: str | None, + window: Window, + parts: Sequence[Generation], + model: str | None = None, + parent_tool_id: str | None = None, + ) -> list[AssistantMessage]: + """Add one measured generation, one ``AssistantMessage`` per part, sharing ``window``. + + ``window.duration_ms`` is apportioned by each part's output tokens (evenly when + none has output), each share but the last rounded to 1e-6 ms. The returned + messages and the passed blocks are the live objects in the transcript; after + the turn ended they are detached and change nothing. + + Raises: + ValueError: ``parts`` is empty. + """ + if not parts: + raise ValueError("add_generation() needs at least one part") + total_ms = window.duration_ms + output = sum(part.tokens.output_tokens for part in parts) + messages: list[AssistantMessage] = [] + assigned = 0.0 + for index, part in enumerate(parts): + if index == len(parts) - 1: + share = total_ms - assigned + else: + share = round(total_ms * (part.tokens.output_tokens / output if output > 0 else 1 / len(parts)), 6) + assigned += share + messages.append( + self._message(part, window.started_at, window.completed_at, share, message_id, model, parent_tool_id) + ) + if not self._ended(): + self._messages.extend(messages) + return messages + + def add_unmeasured_generation( + self, + *, + message_id: str | None, + part: Generation, + model: str | None = None, + parent_tool_id: str | None = None, + ) -> AssistantMessage: + """Add a generation with no measurable window: equal bounds at ``now()``, no duration.""" + now = self.now() + message = self._message(part, now, now, None, message_id, model, parent_tool_id) + if not self._ended(): + self._messages.append(message) + return message + + def finalize( + self, + status: AgentEndStatus, + *, + usage: TokenUsage | None = None, + stop_reason: str | None = None, + agent_output: str | None = None, + model_used: str | None = None, + assistant_turn_count: int | None = None, + num_turns: int | None = None, + result_summary: ResultSummary | None = _UNSET, + ) -> TurnOutcome: + """End a clean turn; a second call returns the first outcome and emits nothing. + + ``usage`` defaults to the sum of ``end_inner_turn`` tokens. ``result_summary`` + defaults to the final reply: the text of the last main-thread message when it + calls no tool. + + Raises: + ValueError: ``status`` is ``CRASHED`` or ``TIMEOUT`` (use ``fail``). + """ + if status in _FAILED: + raise ValueError(f"finalize({status.value}): a failed turn ends with fail()") + if self._outcome is not None: + return self._outcome + if self._ending: + raise RuntimeError("the turn already ended, but its record could not be built") + if result_summary is _UNSET: + result_summary = ResultSummary( + is_error=False, subtype=status.value, stop_reason=stop_reason, result=self._final_reply() + ) + return self._end( + status, + reason=None, + usage=usage, + agent_output=agent_output, + model_used=model_used, + assistant_turn_count=assistant_turn_count, + num_turns=num_turns, + result_summary=result_summary, + ) + + def fail( + self, + status: Literal[AgentEndStatus.CRASHED, AgentEndStatus.TIMEOUT], + reason: str, + *, + usage: TokenUsage | None = None, + ) -> TurnOutcome: + """End a failed turn with the full ``reason``; a second call returns the first outcome. + + Raises: + ValueError: ``status`` is not ``CRASHED`` or ``TIMEOUT``. + """ + if status not in _FAILED: + raise ValueError(f"fail({status.value}): a clean turn ends with finalize()") + if self._outcome is not None: + return self._outcome + if self._ending: + raise RuntimeError("the turn already ended, but its record could not be built") + return self._end( + status, + reason=reason, + usage=usage, + agent_output=None, + model_used=None, + assistant_turn_count=None, + num_turns=None, + result_summary=None, + ) + + def _ended(self) -> bool: + if self._outcome is None and not self._ending: + return False + if not self._dropped_logged: + self._dropped_logged = True + logger.debug("[%s] a write after the turn ended was dropped", self._task_id) + return True + + def _emit(self, event: StreamEvent, parent_tool_id: str | None = None) -> None: + event.timestamp = self.now() + event.thread_id = event.parent_thread_id = parent_tool_id + self._collector.on_event(event) + for sink in self._sinks: + safe_emit(sink, event) + + def _next_sequence(self) -> int: + sequence = self._sequence + self._sequence += 1 + return sequence + + def _stamp( + self, keyword: str, value: datetime | None, now: datetime, parent_tool_id: str | None + ) -> datetime | None: + if parent_tool_id is not None: + if value is not _UNSET: + return value + return now if self._basis is TimingBasis.TURN_CLOCK else None + if self._basis is TimingBasis.TURN_CLOCK: + if value is not _UNSET: + raise TypeError( + f"{keyword}= is not accepted under TimingBasis.TURN_CLOCK: the emitter stamps the clock" + ) + return now + if value is _UNSET: + raise TypeError(f"{keyword}= is required under TimingBasis.{self._basis.name} (None means no CLI stamp)") + return value + + def _close( + self, + tool_id: str, + status: ToolEndStatus, + summary: str | None, + error: str | None, + result_data: dict[str, Any] | list[Any] | None, + parameters: dict[str, Any] | None, + stamp: datetime | None, + ) -> None: + now = self.now() + opened = self._open_tools.pop(tool_id, None) + if opened is None: + opened = _OpenTool( + CommandTelemetry( + tool_name="unknown", tool_id=tool_id, timestamp=now, sequence_number=self._next_sequence() + ), + self._turn_id or "", + None, + ) + telemetry = opened.telemetry + if status is not ToolEndStatus.UNRESOLVED and stamp is not None: + telemetry.execution_completed_at = stamp + if telemetry.execution_started_at is not None: + telemetry.duration_ms = max(0.0, (stamp - telemetry.execution_started_at).total_seconds() * 1000) + telemetry.result_status = _RESULT_STATUS[status] + telemetry.result_summary = summary + telemetry.error_message = error + telemetry.result_data = result_data + if parameters is not None: + telemetry.parameters = parameters + self._emit( + ToolEndEvent(task_id=self._task_id, turn_id=opened.turn_id, tool=telemetry, status=status), + opened.parent_tool_id, + ) + + def _message( + self, + part: Generation, + started_at: datetime, + completed_at: datetime, + duration_ms: float | None, + message_id: str | None, + model: str | None, + parent_tool_id: str | None, + ) -> AssistantMessage: + tokens = part.tokens + return AssistantMessage( + started_at=started_at, + completed_at=completed_at, + generation_duration_ms=duration_ms, + content_blocks=part.blocks, + tool_use_ids=[b.tool_use_id for b in part.blocks if b.block_type == "tool_use" and b.tool_use_id], + input_tokens=tokens.uncached_input_tokens, + output_tokens=tokens.output_tokens, + cache_creation_tokens=tokens.cache_creation_input_tokens, + cache_read_tokens=tokens.cache_read_input_tokens, + reasoning_tokens=part.reasoning_tokens, + stop_reason=part.stop_reason, + model=model or self._model, + message_id=message_id, + parent_tool_use_id=parent_tool_id, + ) + + def _final_reply(self) -> str | None: + main = [m for m in self._messages if m.parent_tool_use_id is None] + if not main or any(b.block_type == "tool_use" for b in main[-1].content_blocks): + return None + return "".join(b.text or "" for b in main[-1].content_blocks if b.block_type == "text") or None + + def _end( + self, + status: AgentEndStatus, + *, + reason: str | None, + usage: TokenUsage | None, + agent_output: str | None, + model_used: str | None, + assistant_turn_count: int | None, + num_turns: int | None, + result_summary: ResultSummary | None, + ) -> TurnOutcome: + for tool_id in list(self._open_tools): + self._close(tool_id, ToolEndStatus.UNRESOLVED, None, None, None, None, None) + if self._turn_id is not None: + self.end_inner_turn(TurnEndStatus(status.value)) + published = usage if usage is not None else self._reported + self._warn_on_delta_overshoot(published) + crashed = status in _FAILED + self._emit( + AgentEndEvent( + task_id=self._task_id, + status=status, + usage=published, + iteration=self._iteration, + user_input=self._prompt, + agent_output=agent_output if agent_output is not None else "".join(self._text), + model_used=model_used if model_used is not None else self._model, + assistant_turn_count=assistant_turn_count if assistant_turn_count is not None else self._main_turns, + messages=list(self._messages), + num_turns=num_turns if num_turns is not None else self._main_turns, + result_summary=result_summary, + crashed=crashed, + crash_reason=truncate_crash_message(reason) if reason is not None else None, + duration_seconds=time.monotonic() - self._began_monotonic if self._began_monotonic is not None else 0.0, + ) + ) + self._ending = True + self._outcome = TurnOutcome(record=self._collector.build_turn_record(), status=status, error=reason) + return self._outcome + + def _warn_on_delta_overshoot(self, published: TokenUsage) -> None: + over = [ + f"{bucket} {getattr(self._reported, bucket)} > {getattr(published, bucket)}" + for bucket in _BUCKETS + if getattr(self._reported, bucket) > getattr(published, bucket) + ] + if over: + logger.warning( + "[%s] the inner-turn token deltas exceed the published turn usage (%s); a harness double-counts", + self._task_id, + "; ".join(over), + ) diff --git a/src/coder_eval/testing.py b/src/coder_eval/testing.py new file mode 100644 index 00000000..31367b54 --- /dev/null +++ b/src/coder_eval/testing.py @@ -0,0 +1,318 @@ +"""The test harness a harness adapter shares with the in-tree suites: replay, identity, balance, conformance. + +No ``pytest`` import: every check raises ``AssertionError``, and callers parametrize. +A plugin calls these from its own tests exactly as ``tests/`` does. +""" + +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import TYPE_CHECKING, Any + +from coder_eval.models import AssistantMessage, Enforcement, HarnessContract, PermissionMode, TimingBasis, TurnRecord +from coder_eval.streaming.emitter import TurnEmitter, TurnOutcome +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentEndStatus, + AgentStartEvent, + StreamEvent, + ToolEndEvent, + ToolStartEvent, + TurnEndEvent, + TurnStartEvent, +) +from coder_eval.timing import main_thread_tool_spans, union_ms + + +if TYPE_CHECKING: + from coder_eval.models import TaskDefinition + + +@dataclass(frozen=True) +class Tick: + """A stream element that moves the replay's ``ScriptedClock`` to ``at_ms`` after its origin.""" + + at_ms: float + + +class ScriptedClock: + """A clock that reads ``origin + at_ms`` and moves only on a ``Tick``.""" + + def __init__(self, origin: datetime) -> None: + self._origin = origin + self._at_ms = 0.0 + + def now(self) -> datetime: + return self._origin + timedelta(milliseconds=self._at_ms) + + def _move_to(self, at_ms: float) -> None: + self._at_ms = at_ms + + +@dataclass(frozen=True) +class Replay: + """What a replayed turn produced; ``started_at`` / ``ended_at`` are its bracket stamps.""" + + record: TurnRecord + events: list[StreamEvent] + outcome: TurnOutcome + started_at: datetime + ended_at: datetime + + +class _Recorder: + def __init__(self) -> None: + self.events: list[StreamEvent] = [] + + def on_event(self, event: StreamEvent) -> None: + self.events.append(event) + + +def replay[D: Callable[[Any], None]]( + stream: Iterable[Any], + make_decoder: Callable[[TurnEmitter], D], + *, + clock: ScriptedClock, + basis: TimingBasis = TimingBasis.TURN_CLOCK, + model: str | None = "m", + end: Callable[[D], TurnOutcome] | None = None, +) -> Replay: + """Drive a decoder over ``stream`` through a real ``TurnEmitter`` on ``clock``. + + A ``Tick`` moves the clock; every other element is passed to the decoder. The turn + ends with ``end(decoder)`` when given, else ``emitter.finalize(COMPLETED)``. An + exception from the decoder propagates. + """ + recorder = _Recorder() + emitter = TurnEmitter( + task_id="replay", iteration=1, prompt="go", model=model, basis=basis, clock=clock, sinks=[recorder] + ) + emitter.begin() + decoder = make_decoder(emitter) + for element in stream: + if isinstance(element, Tick): + clock._move_to(element.at_ms) # pyright: ignore[reportPrivateUsage] + else: + decoder(element) + outcome = end(decoder) if end is not None else emitter.finalize(AgentEndStatus.COMPLETED) + starts = [e for e in recorder.events if isinstance(e, AgentStartEvent)] + ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] + return Replay( + record=outcome.record, + events=recorder.events, + outcome=outcome, + started_at=starts[0].timestamp, + ended_at=ends[-1].timestamp, + ) + + +def assert_identity_closes(record: TurnRecord, *, started_at: datetime, ended_at: datetime) -> None: + """Assert head + Σ generation + UNION(tool) + tail equals the turn's span, to float precision. + + Main thread only on both sides, through production's own span selector. Also + asserts the stored ``tool_union_ms`` equals the union computed here. + + Raises: + AssertionError: a bucket is missing, the stored union disagrees, or the buckets + do not tile the span. + """ + span_ms = (ended_at - started_at).total_seconds() * 1000.0 + generation_ms = sum( + m.generation_duration_ms + for m in record.messages + if isinstance(m, AssistantMessage) and m.parent_tool_use_id is None and m.generation_duration_ms is not None + ) + tool_ms = union_ms(main_thread_tool_spans(record.messages, record.commands)) + head, tail = record.harness_startup_ms, record.harness_teardown_ms + if head is None or tail is None: + raise AssertionError(f"a turn that generated has a measured head and tail (head={head}, tail={tail})") + stored = record.tool_union_ms + if (stored is None and tool_ms > 0) or (stored is not None and not math.isclose(stored, tool_ms, abs_tol=1e-6)): + raise AssertionError( + f"TurnRecord.tool_union_ms is {record.tool_union_ms}, but the main-thread command spans union to " + + f"{tool_ms:.4f} ms: the stored value and the selection rule have come apart" + ) + bucket_sum = head + generation_ms + tool_ms + tail + if not math.isclose(bucket_sum, span_ms, abs_tol=1e-6): + raise AssertionError( + f"the four buckets sum to {bucket_sum:.4f} ms against a {span_ms:.4f} ms turn " + + f"(off by {bucket_sum - span_ms:+.4f} ms): head={head:.4f}, generation={generation_ms:.4f}, " + + f"tool_union={tool_ms:.4f}, tail={tail:.4f}. A sum UNDER the turn means some interval is " + + "booked nowhere; a sum OVER it means one is booked twice." + ) + + +_BUCKETS = ("uncached_input_tokens", "output_tokens", "cache_creation_input_tokens", "cache_read_input_tokens") + + +def assert_stream_balanced(events: Sequence[StreamEvent]) -> None: + """Assert one turn's event stream is well formed. + + Exactly one ``AgentStartEvent``, first, and one ``AgentEndEvent``, last; every + ``TurnStartEvent`` closed by a ``TurnEndEvent`` of the same ``turn_id`` before the + next start and before the end, and no end without its start; every tool id started + once is ended exactly once, and none ends unstarted; per token bucket, the sum of + ``TurnEndEvent.tokens`` is at most ``AgentEndEvent.usage``. + + Raises: + AssertionError: naming every violation found. + """ + problems: list[str] = [] + if not events: + raise AssertionError("the stream is empty") + starts = [e for e in events if isinstance(e, AgentStartEvent)] + ends = [e for e in events if isinstance(e, AgentEndEvent)] + if len(starts) != 1 or not isinstance(events[0], AgentStartEvent): + problems.append(f"{len(starts)} AgentStartEvent(s), first event is {type(events[0]).__name__}") + if len(ends) != 1 or not isinstance(events[-1], AgentEndEvent): + problems.append(f"{len(ends)} AgentEndEvent(s), last event is {type(events[-1]).__name__}") + open_turn: str | None = None + for event in events: + if isinstance(event, TurnStartEvent): + if open_turn is not None: + problems.append(f"turn {event.turn_id!r} started while {open_turn!r} is open") + open_turn = event.turn_id + elif isinstance(event, TurnEndEvent): + if open_turn != event.turn_id: + problems.append(f"turn {event.turn_id!r} ended while the open turn is {open_turn!r}") + open_turn = None + elif isinstance(event, AgentEndEvent) and open_turn is not None: + problems.append(f"turn {open_turn!r} is still open at the AgentEndEvent") + open_turn = None + tool_starts = Counter(e.tool.tool_id for e in events if isinstance(e, ToolStartEvent)) + tool_ends = Counter(e.tool.tool_id for e in events if isinstance(e, ToolEndEvent)) + problems += [f"tool {tid!r} started {n} times" for tid, n in tool_starts.items() if n != 1] + problems += [f"tool {tid!r} ended {tool_ends[tid]} times" for tid in tool_starts if tool_ends[tid] != 1] + problems += [f"tool {tid!r} ended without a start" for tid in tool_ends if tid not in tool_starts] + if ends: + usage = ends[-1].usage + for bucket in _BUCKETS: + reported = sum(getattr(e.tokens, bucket) for e in events if isinstance(e, TurnEndEvent) and e.tokens) + if reported > getattr(usage, bucket): + problems.append(f"TurnEndEvent {bucket} sum {reported} > AgentEndEvent.usage {getattr(usage, bucket)}") + if problems: + raise AssertionError("unbalanced event stream: " + "; ".join(problems)) + + +_FIELDS = ("system_prompt", "plugin_skills", "permission_mode", "allowed_tools", "disallowed_tools") +_CONFIG_FIELD = {"plugin_skills": "plugins"} +_GATED_VALUES: dict[str, Any] = { + "system_prompt": "CONFORMANCE-MARKER-7f3a", + "plugins": [{"type": "local", "path": "/plugins/p"}], + "permission_mode": "plan", + "allowed_tools": ["Bash"], + "disallowed_tools": ["Bash"], +} + + +def enforced_cells(contract: HarnessContract, kind: str) -> set[tuple[str, str]]: + """``(kind, cell)`` for every ENFORCED field; ``permission_mode`` gives one ``permission_mode=`` per mode.""" + cells: set[tuple[str, str]] = set() + for field in _FIELDS: + if getattr(contract, field) is not Enforcement.ENFORCED: + continue + if field == "permission_mode": + cells |= {(kind, f"permission_mode={mode.value}") for mode in contract.permission_modes or ()} + else: + cells.add((kind, field)) + return cells + + +def _task(kind: str, **agent: Any) -> TaskDefinition: + from coder_eval.models import AgentKind, FileExistsCriterion, SandboxConfig, TaskDefinition, parse_agent_config + + return TaskDefinition( + task_id="t", + description="d", + initial_prompt=None if kind == AgentKind.NONE.value else "do the task", + agent=parse_agent_config(type=kind, **agent), + sandbox=SandboxConfig(driver="tempdir"), + success_criteria=[FileExistsCriterion(description="c", path="out.txt")], + ) + + +def _expect_rejected(task: Callable[[], TaskDefinition], match: str) -> None: + import re + + from coder_eval.orchestration.harness_contract import HarnessContractError, validate_harness_contract + + try: + validate_harness_contract(task()) + except HarnessContractError as error: + if not re.search(match, str(error)): + raise AssertionError(f"rejected, but {str(error)!r} does not match {match!r}") from error + return + raise AssertionError(f"expected a HarnessContractError matching {match!r}; the task resolved") + + +def rejections(kind: str) -> list[tuple[str, Callable[[], None]]]: + """The resolution-time rejections ``kind``'s contract implies, as named checks that raise ``AssertionError``.""" + from coder_eval.agents.registry import AgentRegistry + from coder_eval.plugins import ensure_plugins_loaded + + ensure_plugins_loaded() + registration = AgentRegistry.get(kind) + if registration is None: + raise AssertionError(f"agent kind {kind!r} is not registered") + contract = registration.agent_class.contract + checks: list[tuple[str, Callable[[], None]]] = [] + for field in _FIELDS: + if getattr(contract, field) is Enforcement.UNSUPPORTED: + config_field = _CONFIG_FIELD.get(field, field) + value = _GATED_VALUES[config_field] + checks.append( + ( + f"unsupported {config_field}", + lambda c=config_field, v=value: _expect_rejected( + lambda: _task(kind, **{c: v}), rf"agent\.{c}.*{kind!r}" + ), + ) + ) + if contract.permission_mode is Enforcement.ENFORCED: + for mode in PermissionMode: + if mode not in (contract.permission_modes or frozenset()): + checks.append( + ( + f"undeclared permission_mode={mode.value}", + lambda m=mode: _expect_rejected( + lambda: _task(kind, permission_mode=m), "has no documented meaning" + ), + ) + ) + if Enforcement.ENFORCED in (contract.allowed_tools, contract.disallowed_tools): + checks.append( + ( + "misspelled tool name", + lambda: _expect_rejected(lambda: _task(kind, allowed_tools=["Bassh"]), "did you mean 'Bash'"), + ) + ) + return checks + + +async def conformance(kind: str, probes: Mapping[tuple[str, str], Callable[[], Awaitable[None]]]) -> None: + """Assert ``kind`` rejects what its contract marks unsupported and honors every enforced cell. + + Runs every check from ``rejections(kind)``, asserts ``probes`` covers exactly + ``enforced_cells(contract, kind)``, then awaits every probe. + + Raises: + AssertionError: a rejection is missing, a probe is missing or extra, or a probe fails. + """ + from coder_eval.agents.registry import AgentRegistry + + for _name, check in rejections(kind): + check() + registration = AgentRegistry.get(kind) + assert registration is not None + expected = enforced_cells(registration.agent_class.contract, kind) + if set(probes) != expected: + raise AssertionError( + f"probes for {kind!r} do not match its enforced cells: missing {sorted(expected - set(probes))}, " + + f"extra {sorted(set(probes) - expected)}" + ) + for cell in sorted(probes): + await probes[cell]() diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index 25c7c77f..f092e079 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -21,6 +21,7 @@ import math import time from collections.abc import Iterable +from dataclasses import dataclass from datetime import datetime, timedelta from coder_eval.models import AssistantMessage, CommandTelemetry, TranscriptMessage @@ -127,8 +128,21 @@ def union_ms(spans: list[tuple[datetime, datetime]]) -> float: return busy_ms(spans, min(s for s, _ in spans), max(e for _, e in spans)) -def close_window(*, mark: datetime, now: datetime, item_start: datetime | None = None) -> tuple[datetime, float]: - """Open one generation window at ``mark`` and close it at ``now``: its ``(started, span_ms)``. +@dataclass(frozen=True, slots=True) +class Window: + """One generation window: its two bounds, and nothing a caller could set apart from them.""" + + started_at: datetime + completed_at: datetime + + @property + def duration_ms(self) -> float: + """``completed_at - started_at`` in ms, clamped at ``0.0``: an inverted window is a measured zero.""" + return max(0.0, (self.completed_at - self.started_at).total_seconds() * 1000.0) + + +def close_window(*, mark: datetime, now: datetime, item_start: datetime | None = None) -> Window: + """Open one generation window at ``mark`` and close it at ``now``. The shape all five reducers share, and what it returns is the RAW window — tool execution comes back out centrally, in ``subtract_tool_time``. @@ -141,14 +155,12 @@ def close_window(*, mark: datetime, now: datetime, item_start: datetime | None = ``item_start`` is this emission's own first stamp, when the harness has one; the ``min()`` against ``mark`` stops a backwards stamp inverting the span. - The span is clamped at ``0.0``: an inverted window is a measured zero, not a - negative generation. ``completed`` is deliberately not returned — it is - always ``now``, which the caller already has. + An inverted window keeps its bounds and its duration clamps to ``0.0``. Rationale: .claude/notes/timing.md § close_window """ started = min(mark, item_start) if item_start is not None else mark - return started, max(0.0, (now - started).total_seconds() * 1000.0) + return Window(started_at=started, completed_at=now) def decompose_turn( diff --git a/tests/_fixtures/golden_streams/_recorder.py b/tests/_fixtures/golden_streams/_recorder.py new file mode 100644 index 00000000..0ef18d14 --- /dev/null +++ b/tests/_fixtures/golden_streams/_recorder.py @@ -0,0 +1,11 @@ +"""A stream callback that keeps every event a golden replay emitted.""" + +from coder_eval.streaming.events import StreamEvent + + +class EventRecorder: + def __init__(self) -> None: + self.events: list[StreamEvent] = [] + + def on_event(self, event: StreamEvent) -> None: + self.events.append(event) diff --git a/tests/_fixtures/golden_streams/antigravity_fixtures.py b/tests/_fixtures/golden_streams/antigravity_fixtures.py index 17f4ccf8..1cc8f7a3 100644 --- a/tests/_fixtures/golden_streams/antigravity_fixtures.py +++ b/tests/_fixtures/golden_streams/antigravity_fixtures.py @@ -22,6 +22,8 @@ from coder_eval.agents import antigravity_agent from coder_eval.agents.antigravity_agent import AntigravityAgent from coder_eval.models import parse_agent_config +from coder_eval.streaming.events import StreamEvent +from tests._fixtures.golden_streams._recorder import EventRecorder def _usage(prompt: int, cached: int, candidates: int, thoughts: int) -> SimpleNamespace: @@ -135,8 +137,11 @@ class AntigravityScenario: steps: list[Any] -async def run_antigravity_scenario(scenario: AntigravityScenario, working_dir: str) -> dict[str, Any]: +async def run_antigravity_scenario( + scenario: AntigravityScenario, working_dir: str +) -> tuple[dict[str, Any], list[StreamEvent]]: """Replay one scenario and return the resulting record as a plain dump.""" + recorder = EventRecorder() agent = _agent_with_steps(scenario.steps) agent.working_directory = pathlib.Path(working_dir) # Neutralize the orphan poll loop's real 5s sleeps. `d_orphaned_tool` @@ -144,8 +149,8 @@ async def run_antigravity_scenario(scenario: AntigravityScenario, working_dir: s # up to 120 cycles, i.e. ten minutes of wall clock in a unit test. The # loop's LOGIC is what the scenario records; the waiting is not. with patch.object(antigravity_agent.asyncio, "sleep", _no_sleep): - record = await agent.communicate("do it") - return record.model_dump(mode="json") + record = await agent.communicate("do it", stream_callback=recorder) + return record.model_dump(mode="json"), recorder.events def _build_catalogue() -> list[AntigravityScenario]: diff --git a/tests/_fixtures/golden_streams/claude_fixtures.py b/tests/_fixtures/golden_streams/claude_fixtures.py index 2340bc27..9e5b89d8 100644 --- a/tests/_fixtures/golden_streams/claude_fixtures.py +++ b/tests/_fixtures/golden_streams/claude_fixtures.py @@ -19,6 +19,8 @@ import coder_eval.agents.claude_code_agent as claude_module from coder_eval.models import AgentKind, parse_agent_config +from coder_eval.streaming import events as protocol +from tests._fixtures.golden_streams._recorder import EventRecorder ClaudeCodeAgent = claude_module.ClaudeCodeAgent @@ -432,13 +434,16 @@ def _patches(scenario: ClaudeScenario) -> Iterator[Any]: yield patch.object(claude_module.time, "monotonic", scenario.monotonic) -async def run_claude_scenario(scenario: ClaudeScenario, working_dir: str) -> dict[str, Any]: +async def run_claude_scenario( + scenario: ClaudeScenario, working_dir: str +) -> tuple[dict[str, Any], list[protocol.StreamEvent]]: """Run ``scenario`` and return the ``TurnRecord``/``pending_turn`` model_dump. Raises ``AssertionError`` if a crash/timeout scenario fails to raise its expected exception (so a refactor that silently swallows the failure is caught). """ + recorder = EventRecorder() import pytest config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") @@ -450,10 +455,10 @@ async def run_claude_scenario(scenario: ClaudeScenario, working_dir: str) -> dic stack.enter_context(ctx) if scenario.expects is not None: with pytest.raises(scenario.expects): - await agent.communicate(scenario.prompt, timeout=scenario.timeout) + await agent.communicate(scenario.prompt, timeout=scenario.timeout, stream_callback=recorder) record = agent.pending_turn assert record is not None, f"{scenario.name}: pending_turn was not set on the failure path" else: - record = await agent.communicate(scenario.prompt, timeout=scenario.timeout) + record = await agent.communicate(scenario.prompt, timeout=scenario.timeout, stream_callback=recorder) - return record.model_dump(mode="json") + return record.model_dump(mode="json"), recorder.events diff --git a/tests/_fixtures/golden_streams/codex_fixtures.py b/tests/_fixtures/golden_streams/codex_fixtures.py index e0a157fd..98d72615 100644 --- a/tests/_fixtures/golden_streams/codex_fixtures.py +++ b/tests/_fixtures/golden_streams/codex_fixtures.py @@ -23,6 +23,8 @@ from coder_eval.agents.codex_agent import CodexAgent from coder_eval.models import AgentKind, parse_agent_config +from coder_eval.streaming.events import StreamEvent +from tests._fixtures.golden_streams._recorder import EventRecorder CODEX_MODEL = "gpt-5-codex" @@ -378,8 +380,9 @@ def _rebase_notifications(notifications: list[Any]) -> list[Any]: return rebased -async def run_codex_scenario(scenario: CodexScenario, working_dir: str) -> dict[str, Any]: +async def run_codex_scenario(scenario: CodexScenario, working_dir: str) -> tuple[dict[str, Any], list[StreamEvent]]: """Run ``scenario`` with fakes and return the TurnRecord/pending_turn dump.""" + recorder = EventRecorder() import pytest config = parse_agent_config(type=AgentKind.CODEX, model=CODEX_MODEL) @@ -393,10 +396,10 @@ async def run_codex_scenario(scenario: CodexScenario, working_dir: str) -> dict[ with patch.dict(os.environ, {"CODEX_HOME": working_dir}): if scenario.expects is not None: with pytest.raises(scenario.expects): - await agent.communicate("do it") + await agent.communicate("do it", stream_callback=recorder) record = agent.pending_turn assert record is not None, f"{scenario.name}: pending_turn was not set on the failure path" else: - record = await agent.communicate("do it") + record = await agent.communicate("do it", stream_callback=recorder) - return record.model_dump(mode="json") + return record.model_dump(mode="json"), recorder.events diff --git a/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json index 9b697ba3..c040a91e 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json @@ -3,7 +3,7 @@ "assistant_turn_count": 1, "commands": [ { - "assistant_turn_index": null, + "assistant_turn_index": 0, "duration_ms": "", "error_message": null, "execution_completed_at": "", diff --git a/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json b/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json index 0ddd04e5..e15d3bee 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json @@ -3,7 +3,7 @@ "assistant_turn_count": 1, "commands": [ { - "assistant_turn_index": null, + "assistant_turn_index": 0, "duration_ms": "", "error_message": null, "execution_completed_at": "", diff --git a/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json b/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json index 1d172aff..310386e8 100644 --- a/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json +++ b/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json @@ -3,7 +3,7 @@ "assistant_turn_count": 1, "commands": [ { - "assistant_turn_index": null, + "assistant_turn_index": 0, "duration_ms": "", "error_message": null, "execution_completed_at": "", diff --git a/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json b/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json index 8371a9a8..efa7c12f 100644 --- a/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json +++ b/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json @@ -3,7 +3,7 @@ "assistant_turn_count": 1, "commands": [ { - "assistant_turn_index": null, + "assistant_turn_index": 0, "duration_ms": "", "error_message": "hi\n", "execution_completed_at": "", diff --git a/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json b/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json index 663e1e36..96288488 100644 --- a/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json +++ b/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json @@ -3,7 +3,7 @@ "assistant_turn_count": 1, "commands": [ { - "assistant_turn_index": null, + "assistant_turn_index": 0, "duration_ms": null, "error_message": null, "execution_completed_at": null, diff --git a/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json b/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json index 7ca68b57..8741173b 100644 --- a/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json +++ b/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json @@ -3,7 +3,7 @@ "assistant_turn_count": 1, "commands": [ { - "assistant_turn_index": null, + "assistant_turn_index": 0, "duration_ms": "", "error_message": null, "execution_completed_at": "", @@ -24,7 +24,7 @@ "tool_name": "Agent" }, { - "assistant_turn_index": null, + "assistant_turn_index": 0, "duration_ms": "", "error_message": null, "execution_completed_at": "", diff --git a/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json index 0b4faf28..d5aa832d 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json @@ -3,7 +3,7 @@ "assistant_turn_count": 2, "commands": [ { - "assistant_turn_index": 1, + "assistant_turn_index": 0, "duration_ms": "", "error_message": null, "execution_completed_at": "", diff --git a/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json b/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json index c486f259..4b496314 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json +++ b/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json @@ -3,7 +3,7 @@ "assistant_turn_count": 2, "commands": [ { - "assistant_turn_index": 1, + "assistant_turn_index": 0, "duration_ms": "", "error_message": null, "execution_completed_at": "", diff --git a/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json index 9a2d261f..0b8d6284 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json @@ -3,7 +3,7 @@ "assistant_turn_count": 1, "commands": [ { - "assistant_turn_index": 1, + "assistant_turn_index": 0, "duration_ms": null, "error_message": "no result observed", "execution_completed_at": "", diff --git a/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json index da349e69..45645524 100644 --- a/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json @@ -3,7 +3,7 @@ "assistant_turn_count": 3, "commands": [ { - "assistant_turn_index": 1, + "assistant_turn_index": 0, "duration_ms": "", "error_message": null, "execution_completed_at": "", @@ -23,7 +23,7 @@ "tool_name": "Write" }, { - "assistant_turn_index": 2, + "assistant_turn_index": 1, "duration_ms": "", "error_message": null, "execution_completed_at": "", diff --git a/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json b/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json index d70291ca..864106fe 100644 --- a/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json +++ b/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json @@ -3,7 +3,7 @@ "assistant_turn_count": 2, "commands": [ { - "assistant_turn_index": 1, + "assistant_turn_index": 0, "duration_ms": "", "error_message": null, "execution_completed_at": "", diff --git a/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json index 04b07999..94e455c3 100644 --- a/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json @@ -3,7 +3,7 @@ "assistant_turn_count": 1, "commands": [ { - "assistant_turn_index": 1, + "assistant_turn_index": 0, "duration_ms": null, "error_message": "no result observed", "execution_completed_at": null, diff --git a/tests/_fixtures/golden_streams/opencode_fixtures.py b/tests/_fixtures/golden_streams/opencode_fixtures.py index 5f138147..88a9b031 100644 --- a/tests/_fixtures/golden_streams/opencode_fixtures.py +++ b/tests/_fixtures/golden_streams/opencode_fixtures.py @@ -32,6 +32,8 @@ from coder_eval.agents.opencode_agent import OpenCodeAgent from coder_eval.errors import AgentCrashError from coder_eval.models import OpenCodeAgentConfig +from coder_eval.streaming.events import StreamEvent +from tests._fixtures.golden_streams._recorder import EventRecorder SESSION = "ses_test123" @@ -214,8 +216,11 @@ class OpenCodeScenario: expects: type[BaseException] | None = None -async def run_opencode_scenario(scenario: OpenCodeScenario, working_dir: str) -> dict[str, Any]: +async def run_opencode_scenario( + scenario: OpenCodeScenario, working_dir: str +) -> tuple[dict[str, Any], list[StreamEvent]]: """Replay one scenario and return the resulting record as a plain dump.""" + recorder = EventRecorder() import pytest proc = _FakeProcess(_rebase_lines(scenario.lines)) @@ -233,12 +238,12 @@ async def fake_exec(*_argv: str, **_kwargs: Any) -> _FakeProcess: await agent.start(working_dir) if scenario.expects is not None: with pytest.raises(scenario.expects): - await agent.communicate("do it") + await agent.communicate("do it", stream_callback=recorder) record = agent.pending_turn assert record is not None, f"{scenario.name}: pending_turn was not set on the failure path" else: - record = await agent.communicate("do it") - return record.model_dump(mode="json") + record = await agent.communicate("do it", stream_callback=recorder) + return record.model_dump(mode="json"), recorder.events def _build_catalogue() -> list[OpenCodeScenario]: diff --git a/tests/_fixtures/golden_streams/pi_fixtures.py b/tests/_fixtures/golden_streams/pi_fixtures.py index 2d26d7d6..817f247c 100644 --- a/tests/_fixtures/golden_streams/pi_fixtures.py +++ b/tests/_fixtures/golden_streams/pi_fixtures.py @@ -26,6 +26,8 @@ from coder_eval.agents.pi_agent import PiAgent from coder_eval.errors import AgentCrashError from coder_eval.models import PiAgentConfig +from coder_eval.streaming.events import StreamEvent +from tests._fixtures.golden_streams._recorder import EventRecorder # tests/_fixtures/golden_streams/ -> tests/fixtures/ (this module moved two @@ -203,8 +205,9 @@ class PiScenario: expects: type[BaseException] | None = None -async def run_pi_scenario(scenario: PiScenario, working_dir: str) -> dict[str, Any]: +async def run_pi_scenario(scenario: PiScenario, working_dir: str) -> tuple[dict[str, Any], list[StreamEvent]]: """Replay one scenario and return the resulting record as a plain dump.""" + recorder = EventRecorder() import pytest proc = _FakeProcess(scenario.lines) @@ -222,12 +225,12 @@ async def fake_exec(*_argv: str, **_kwargs: Any) -> _FakeProcess: await agent.start(working_dir) if scenario.expects is not None: with pytest.raises(scenario.expects): - await agent.communicate("do it") + await agent.communicate("do it", stream_callback=recorder) record = agent.pending_turn assert record is not None, f"{scenario.name}: pending_turn was not set on the failure path" else: - record = await agent.communicate("do it") - return record.model_dump(mode="json") + record = await agent.communicate("do it", stream_callback=recorder) + return record.model_dump(mode="json"), recorder.events def _build_catalogue() -> list[PiScenario]: diff --git a/tests/fixtures/byoa_demo_plugin/byoa_demo.py b/tests/fixtures/byoa_demo_plugin/byoa_demo.py index d5ff8ca4..dba7691c 100644 --- a/tests/fixtures/byoa_demo_plugin/byoa_demo.py +++ b/tests/fixtures/byoa_demo_plugin/byoa_demo.py @@ -44,5 +44,5 @@ def register(registry: type[AgentRegistry]) -> None: ``registry`` is the ``AgentRegistry`` class (not an instance). """ - assert SPI_VERSION == 2, f"byoa_demo supports coder_eval SPI 2, not {SPI_VERSION}" + assert SPI_VERSION == 3, f"byoa_demo supports coder_eval SPI 3, not {SPI_VERSION}" registry.register(DEMO_KIND, DemoAgentConfig)(DemoAgent) diff --git a/tests/fixtures/harness_stubs.py b/tests/fixtures/harness_stubs.py index 6cd5be52..fdeca425 100644 --- a/tests/fixtures/harness_stubs.py +++ b/tests/fixtures/harness_stubs.py @@ -6,7 +6,7 @@ from pydantic import create_model -from coder_eval.models import BaseAgentConfig, Enforcement, HarnessContract, UsageGranularity +from coder_eval.models import BaseAgentConfig, Enforcement, HarnessContract, TimingBasis, UsageGranularity def stub_contract(*, cooperative_stop: bool = True) -> HarnessContract: @@ -20,6 +20,7 @@ def stub_contract(*, cooperative_stop: bool = True) -> HarnessContract: disallowed_tools=Enforcement.UNSUPPORTED, cooperative_stop=cooperative_stop, usage_granularity=UsageGranularity.TURN, + timing_basis=TimingBasis.TURN_CLOCK, ) diff --git a/tests/lint/harness_parity.py b/tests/lint/harness_parity.py index 07bdf0a1..a2b5158b 100644 --- a/tests/lint/harness_parity.py +++ b/tests/lint/harness_parity.py @@ -87,7 +87,9 @@ def _budget_cell(contract: HarnessContract) -> str: _RUN_LIMIT_CELLS: dict[str, Callable[[HarnessContract], str]] = { "max_tool_calls": lambda c: ( - "TurnMonitor at the should_stop poll, resolved tool calls" if c.cooperative_stop else "not polled (never fires)" + "TurnMonitor at the should_stop poll, main-thread resolved tool calls" + if c.cooperative_stop + else "not polled (never fires)" ), "expected_tool_calls": lambda _c: "orchestrator, cumulative visible tool calls, warns only", "task_timeout": lambda _c: "orchestrator, agent-agnostic", diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index 216bbe80..069b133d 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -24,6 +24,7 @@ import pytest from coder_eval.models import AgentKind +from coder_eval.testing import assert_stream_balanced from tests._fixtures.golden_streams import assert_reconciliation, assert_timing_captured, scrub from tests._fixtures.golden_streams.antigravity_fixtures import ANTIGRAVITY_SCENARIOS, run_antigravity_scenario from tests._fixtures.golden_streams.claude_fixtures import CLAUDE_SCENARIOS, run_claude_scenario @@ -161,7 +162,7 @@ def _compare_or_regen(name: str, actual_scrubbed: dict[str, Any]) -> None: @pytest.mark.asyncio @pytest.mark.parametrize("scenario", CLAUDE_SCENARIOS, ids=lambda s: s.name) async def test_claude_golden(scenario, tmp_path): - raw = await run_claude_scenario(scenario, str(tmp_path)) + raw, _ = await run_claude_scenario(scenario, str(tmp_path)) # Reconciliation is asserted on the UNscrubbed dump (token buckets are never # scrubbed, but cost/timestamps are — assert before masking to be explicit). assert_reconciliation(raw) @@ -177,7 +178,7 @@ async def test_claude_golden(scenario, tmp_path): @pytest.mark.asyncio @pytest.mark.parametrize("scenario", CODEX_SCENARIOS, ids=lambda s: s.name) async def test_codex_golden(scenario, tmp_path): - raw = await run_codex_scenario(scenario, str(tmp_path)) + raw, _ = await run_codex_scenario(scenario, str(tmp_path)) assert_reconciliation(raw) assert_timing_captured( raw, @@ -191,7 +192,7 @@ async def test_codex_golden(scenario, tmp_path): @pytest.mark.parametrize("scenario", CLAUDE_SCENARIOS, ids=lambda s: s.name) async def test_claude_reconciliation_invariant(scenario, tmp_path): """The per-bucket reconciliation invariant holds for every Claude snapshot.""" - raw = await run_claude_scenario(scenario, str(tmp_path)) + raw, _ = await run_claude_scenario(scenario, str(tmp_path)) assert_reconciliation(raw) @@ -200,14 +201,14 @@ async def test_claude_reconciliation_invariant(scenario, tmp_path): @pytest.mark.parametrize("scenario", CODEX_SCENARIOS, ids=lambda s: s.name) async def test_codex_reconciliation_invariant(scenario, tmp_path): """The per-bucket reconciliation invariant holds for every Codex snapshot.""" - raw = await run_codex_scenario(scenario, str(tmp_path)) + raw, _ = await run_codex_scenario(scenario, str(tmp_path)) assert_reconciliation(raw) @pytest.mark.asyncio @pytest.mark.parametrize("scenario", ANTIGRAVITY_SCENARIOS, ids=lambda s: s.name) async def test_antigravity_golden(scenario, tmp_path): - raw = await run_antigravity_scenario(scenario, str(tmp_path)) + raw, _ = await run_antigravity_scenario(scenario, str(tmp_path)) assert_reconciliation(raw) assert_timing_captured( raw, @@ -221,13 +222,13 @@ async def test_antigravity_golden(scenario, tmp_path): @pytest.mark.parametrize("scenario", ANTIGRAVITY_SCENARIOS, ids=lambda s: s.name) async def test_antigravity_reconciliation_invariant(scenario, tmp_path): """The per-bucket reconciliation invariant holds for every Antigravity snapshot.""" - assert_reconciliation(await run_antigravity_scenario(scenario, str(tmp_path))) + assert_reconciliation((await run_antigravity_scenario(scenario, str(tmp_path)))[0]) @pytest.mark.asyncio @pytest.mark.parametrize("scenario", OPENCODE_SCENARIOS, ids=lambda s: s.name) async def test_opencode_golden(scenario, tmp_path): - raw = await run_opencode_scenario(scenario, str(tmp_path)) + raw, _ = await run_opencode_scenario(scenario, str(tmp_path)) assert_reconciliation(raw) assert_timing_captured( raw, @@ -241,13 +242,13 @@ async def test_opencode_golden(scenario, tmp_path): @pytest.mark.parametrize("scenario", OPENCODE_SCENARIOS, ids=lambda s: s.name) async def test_opencode_reconciliation_invariant(scenario, tmp_path): """The per-bucket reconciliation invariant holds for every OpenCode snapshot.""" - assert_reconciliation(await run_opencode_scenario(scenario, str(tmp_path))) + assert_reconciliation((await run_opencode_scenario(scenario, str(tmp_path)))[0]) @pytest.mark.asyncio @pytest.mark.parametrize("scenario", PI_SCENARIOS, ids=lambda s: s.name) async def test_pi_golden(scenario, tmp_path): - raw = await run_pi_scenario(scenario, str(tmp_path)) + raw, _ = await run_pi_scenario(scenario, str(tmp_path)) assert_reconciliation(raw) assert_timing_captured( raw, @@ -261,7 +262,56 @@ async def test_pi_golden(scenario, tmp_path): @pytest.mark.parametrize("scenario", PI_SCENARIOS, ids=lambda s: s.name) async def test_pi_reconciliation_invariant(scenario, tmp_path): """The per-bucket reconciliation invariant holds for every Pi snapshot.""" - assert_reconciliation(await run_pi_scenario(scenario, str(tmp_path))) + assert_reconciliation((await run_pi_scenario(scenario, str(tmp_path)))[0]) + + +def _balance_params(harness: str, scenarios: list[Any]) -> list[Any]: + """Every scenario, with the one known-unbalanced stream a strict xfail until its port fixes it.""" + return [ + pytest.param( + s, + id=s.name, + marks=pytest.mark.xfail( + strict=True, + reason="duplicate turn_end emits a TurnEndEvent with no open turn; fixed by the Pi port", + ), + ) + if f"{harness}_{s.name}" == "pi_f_duplicate_turn_end" + else pytest.param(s, id=s.name) + for s in scenarios + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scenario", _balance_params("claude", CLAUDE_SCENARIOS)) +async def test_claude_stream_balanced(scenario, tmp_path): + assert_stream_balanced((await run_claude_scenario(scenario, str(tmp_path)))[1]) + + +@pytest.mark.skipif(not _HAS_CODEX, reason="openai_codex extra not installed") +@pytest.mark.asyncio +@pytest.mark.parametrize("scenario", _balance_params("codex", CODEX_SCENARIOS)) +async def test_codex_stream_balanced(scenario, tmp_path): + assert run_codex_scenario is not None + assert_stream_balanced((await run_codex_scenario(scenario, str(tmp_path)))[1]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scenario", _balance_params("antigravity", ANTIGRAVITY_SCENARIOS)) +async def test_antigravity_stream_balanced(scenario, tmp_path): + assert_stream_balanced((await run_antigravity_scenario(scenario, str(tmp_path)))[1]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scenario", _balance_params("opencode", OPENCODE_SCENARIOS)) +async def test_opencode_stream_balanced(scenario, tmp_path): + assert_stream_balanced((await run_opencode_scenario(scenario, str(tmp_path)))[1]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scenario", _balance_params("pi", PI_SCENARIOS)) +async def test_pi_stream_balanced(scenario, tmp_path): + assert_stream_balanced((await run_pi_scenario(scenario, str(tmp_path)))[1]) # The ONE place a harness is listed for golden coverage. Derived from AgentKind diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index b51609e4..4161e9b8 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -102,47 +102,101 @@ def test_zero_tokens_with_cost_is_kept(self): assert record.token_usage.total_cost_usd == 0.0 -class TestSubAgentEventFiltering: - """Events with parent_thread_id set are ignored (collector.py ~61).""" +class TestSubAgentEvents: + """A nested event (parent_thread_id set) records its tool call and shapes nothing else.""" - def test_sub_agent_events_do_not_affect_record(self): + def test_nested_start_and_end_do_not_affect_record_but_the_nested_tool_is_a_command(self): collector = EventCollector() _feed( collector, [ AgentStartEvent(task_id=TASK_ID, prompt="main prompt", iteration=2), - # A nested sub-agent's events (parent_thread_id set) must be skipped. AgentStartEvent( task_id=TASK_ID, prompt="child prompt", iteration=99, thread_id="tool_x", - parent_thread_id="main", + parent_thread_id="tool_x", ), ToolEndEvent( task_id=TASK_ID, tool=_tool("child_tool", 0), thread_id="tool_x", - parent_thread_id="main", + parent_thread_id="tool_x", ), + AgentEndEvent(task_id=TASK_ID, iteration=99, thread_id="tool_x", parent_thread_id="tool_x"), AgentEndEvent( task_id=TASK_ID, iteration=2, user_input="main prompt", agent_output="main out", usage=TokenUsage(output_tokens=10), - parent_thread_id=None, ), ], ) record = collector.build_turn_record() - # The child AgentStart did not overwrite iteration/user_input. assert record.iteration == 2 assert record.user_input == "main prompt" assert record.agent_output == "main out" - # The child ToolEnd contributed no command. - assert record.commands == [] + assert [c.tool_id for c in record.commands] == ["child_tool"] + + def test_a_nested_turn_start_sets_no_model_and_counts_no_turn(self): + collector = EventCollector() + _feed( + collector, + [ + AgentStartEvent(task_id=TASK_ID, model="main-model"), + TurnStartEvent(task_id=TASK_ID, model="sub-model", thread_id="t1", parent_thread_id="t1"), + ], + ) + record = collector.build_turn_record() + assert record.model_used == "main-model" + assert record.assistant_turn_count == 0 + + def test_ended_tracks_the_current_attempt(self): + collector = EventCollector() + assert not collector.ended + _feed(collector, [AgentStartEvent(task_id=TASK_ID), AgentEndEvent(task_id=TASK_ID, parent_thread_id="x")]) + assert not collector.ended + collector.on_event(AgentEndEvent(task_id=TASK_ID)) + assert collector.ended + collector.on_event(AgentStartEvent(task_id=TASK_ID)) + assert not collector.ended + + +class TestAssistantTurnIndexIsDerived: + """``assistant_turn_index`` is the owning AssistantMessage's position, computed by the collector.""" + + @staticmethod + def _message(*tool_ids: str) -> AssistantMessage: + now = datetime(2026, 1, 1) + return AssistantMessage(started_at=now, completed_at=now, tool_use_ids=list(tool_ids)) + + def test_the_index_counts_assistant_messages_only(self): + from coder_eval.models import UserMessage + + collector = EventCollector() + tool_a, tool_b, orphan = _tool("a", 0), _tool("b", 1), _tool("orphan", 2) + tool_a.assistant_turn_index = 7 + messages = [ + self._message("a"), + UserMessage(text="hi"), + ReconciliationMessage(), + self._message(), + self._message("b"), + ] + _feed( + collector, + [ + AgentStartEvent(task_id=TASK_ID), + *(ToolEndEvent(task_id=TASK_ID, tool=t) for t in (tool_a, tool_b, orphan)), + AgentEndEvent(task_id=TASK_ID, messages=messages), + ], + ) + record = collector.build_turn_record() + assert [(c.tool_id, c.assistant_turn_index) for c in record.commands] == [("a", 0), ("b", 2), ("orphan", None)] + assert tool_a.assistant_turn_index == 7, "the event's own telemetry is never mutated" class TestToolReduction: diff --git a/tests/test_harness_conformance.py b/tests/test_harness_conformance.py index a513c95a..5a150bfe 100644 --- a/tests/test_harness_conformance.py +++ b/tests/test_harness_conformance.py @@ -27,25 +27,19 @@ from coder_eval.agents.registry import AgentRegistry from coder_eval.models import ( AgentKind, - Enforcement, - FileExistsCriterion, HarnessContract, PermissionMode, - SandboxConfig, - TaskDefinition, parse_agent_config, ) -from coder_eval.orchestration.harness_contract import HarnessContractError, validate_harness_contract from coder_eval.orchestration.plugin_staging import stage_plugins from coder_eval.plugins import ensure_plugins_loaded from coder_eval.streaming.events import AgentEndEvent, StopReason, ToolEndEvent, end_status_for +from coder_eval.testing import conformance, enforced_cells, rejections from tests.test_antigravity_agent import _install_fake_sdk MARKER = "CONFORMANCE-MARKER-7f3a" USER_TURN = "do the task" -_FIELDS = ("system_prompt", "plugin_skills", "permission_mode", "allowed_tools", "disallowed_tools") -_CONFIG_FIELD = {"plugin_skills": "plugins"} _KINDS = [kind for kind in AgentKind if kind is not AgentKind.UNKNOWN] type Probe = Callable[[Path, pytest.MonkeyPatch], Awaitable[None]] @@ -58,17 +52,6 @@ def _contract(kind: AgentKind) -> HarnessContract: return registration.agent_class.contract -def _task(kind: AgentKind, **agent: Any) -> TaskDefinition: - return TaskDefinition( - task_id="t", - description="d", - initial_prompt=None if kind is AgentKind.NONE else USER_TURN, - agent=parse_agent_config(type=kind, **agent), - sandbox=SandboxConfig(driver="tempdir"), - success_criteria=[FileExistsCriterion(description="c", path="out.txt")], - ) - - def _plugin_root(tmp_path: Path) -> Path: """A root staged by ``stage_plugins`` over one authored ``probe-skill``.""" skill = tmp_path / "plugin" / "skills" / "probe-skill" @@ -77,49 +60,20 @@ def _plugin_root(tmp_path: Path) -> Path: return stage_plugins([{"type": "local", "path": str(tmp_path / "plugin")}], tmp_path / "plugin_root").root -_GATED_VALUES: dict[str, Any] = { - "system_prompt": MARKER, - "plugins": [{"type": "local", "path": "/plugins/p"}], - "permission_mode": "plan", - "allowed_tools": ["Bash"], - "disallowed_tools": ["Bash"], -} - - # --- rejections, derived from the contracts ---------------------------------------- @pytest.mark.parametrize( - ("kind", "field"), - [(k, f) for k in _KINDS for f in _FIELDS if getattr(_contract(k), f) is Enforcement.UNSUPPORTED], -) -def test_unsupported_field_is_rejected(kind: AgentKind, field: str) -> None: - config_field = _CONFIG_FIELD.get(field, field) - with pytest.raises(HarnessContractError, match=rf"agent\.{config_field}.*{kind.value!r}"): - validate_harness_contract(_task(kind, **{config_field: _GATED_VALUES[config_field]})) - - -@pytest.mark.parametrize( - ("kind", "mode"), - [ - (k, m) - for k in _KINDS - if _contract(k).permission_mode is Enforcement.ENFORCED - for m in PermissionMode - if m not in (_contract(k).permission_modes or frozenset()) - ], + ("kind", "check"), + [pytest.param(k, check, id=f"{k.value}-{name}") for k in _KINDS for name, check in rejections(k.value)], ) -def test_undeclared_permission_value_is_rejected(kind: AgentKind, mode: PermissionMode) -> None: - with pytest.raises(HarnessContractError, match="has no documented meaning"): - validate_harness_contract(_task(kind, permission_mode=mode)) +def test_contract_rejection(kind: AgentKind, check: Callable[[], None]) -> None: + check() -@pytest.mark.parametrize( - "kind", [k for k in _KINDS if AgentRegistry.get(k) and AgentRegistry.get(k).agent_class.tool_names] -) -def test_unknown_tool_name_is_rejected(kind: AgentKind) -> None: - with pytest.raises(HarnessContractError, match="did you mean 'Bash'"): - validate_harness_contract(_task(kind, allowed_tools=["Bassh"])) +def test_every_kind_rejects_what_its_contract_does_not_honor() -> None: + names = {name for k in _KINDS for name, _ in rejections(k.value)} + assert {"unsupported system_prompt", "undeclared permission_mode=default", "misspelled tool name"} <= names # --- probes: the value reaches the native call -------------------------------------- @@ -418,17 +372,7 @@ async def _probe_pi_disallowed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) def _enforced_cells() -> set[tuple[str, str]]: - cells: set[tuple[str, str]] = set() - for kind in _KINDS: - contract = _contract(kind) - for field in _FIELDS: - if getattr(contract, field) is not Enforcement.ENFORCED: - continue - if field == "permission_mode": - cells |= {(kind.value, f"permission_mode={m.value}") for m in contract.permission_modes or ()} - else: - cells.add((kind.value, field)) - return cells + return {cell for kind in _KINDS for cell in enforced_cells(_contract(kind), kind.value)} def test_every_enforced_cell_has_exactly_one_probe() -> None: @@ -440,6 +384,18 @@ async def test_probe(cell: tuple[str, str], tmp_path: Path, monkeypatch: pytest. await _PROBES[cell](tmp_path, monkeypatch) +@pytest.mark.parametrize("kind", _KINDS, ids=lambda k: k.value) +async def test_conformance_per_kind(kind: AgentKind, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def bound(cell: tuple[str, str], probe: Probe) -> Callable[[], Awaitable[None]]: + directory = tmp_path / cell[1].replace("=", "-") + directory.mkdir() + return lambda: probe(directory, monkeypatch) + + await conformance( + kind.value, {cell: bound(cell, probe) for cell, probe in _PROBES.items() if cell[0] == kind.value} + ) + + # --- cooperative stop: every StopReason ends the turn at the boundary --------------- diff --git a/tests/test_harness_contract.py b/tests/test_harness_contract.py index be36a089..d95b3455 100644 --- a/tests/test_harness_contract.py +++ b/tests/test_harness_contract.py @@ -86,10 +86,20 @@ def test_usage_granularity_is_required(self) -> None: with pytest.raises(ValidationError, match="usage_granularity"): HarnessContract(**fields) - def test_unknown_field_rejected(self) -> None: + def test_timing_basis_is_required(self) -> None: + fields = stub_contract().model_dump() + del fields["timing_basis"] + with pytest.raises(ValidationError, match="timing_basis"): + HarnessContract(**fields) + + def test_unknown_timing_basis_rejected(self) -> None: with pytest.raises(ValidationError, match="timing_basis"): HarnessContract(**{**stub_contract().model_dump(), "timing_basis": "wall"}) + def test_unknown_field_rejected(self) -> None: + with pytest.raises(ValidationError, match="clock_basis"): + HarnessContract(**{**stub_contract().model_dump(), "clock_basis": "wall"}) + class TestPermissionModes: def _contract(self, **fields: Any) -> HarnessContract: diff --git a/tests/test_spi.py b/tests/test_spi.py index 714683dd..4fdca890 100644 --- a/tests/test_spi.py +++ b/tests/test_spi.py @@ -13,13 +13,18 @@ "coder_eval.pricing", "coder_eval.streaming.callbacks", "coder_eval.streaming.collector", + "coder_eval.streaming.emitter", "coder_eval.streaming.events", "coder_eval.timing", ) -def test_spi_version_is_two() -> None: - assert spi.SPI_VERSION == 2 +def test_spi_version_is_three() -> None: + assert spi.SPI_VERSION == 3 + + +def test_the_emitter_surface_is_exported() -> None: + assert {"TurnEmitter", "TurnOutcome", "Generation", "Window", "TimingBasis"} <= set(spi.__all__) def test_the_stop_channel_is_exported() -> None: diff --git a/tests/test_testing_harness.py b/tests/test_testing_harness.py new file mode 100644 index 00000000..365de8e3 --- /dev/null +++ b/tests/test_testing_harness.py @@ -0,0 +1,227 @@ +"""``coder_eval.testing``: the sensors a plugin shares with the in-tree harness suites.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from datetime import datetime, timedelta +from typing import Any + +import pytest + +from coder_eval.models import AgentKind, CommandTelemetry, TokenUsage +from coder_eval.plugins import ensure_plugins_loaded +from coder_eval.streaming.emitter import Generation, TurnEmitter +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentEndStatus, + AgentStartEvent, + StreamEvent, + ToolEndEvent, + ToolStartEvent, + TurnEndEvent, + TurnStartEvent, +) +from coder_eval.testing import ( + ScriptedClock, + Tick, + assert_identity_closes, + assert_stream_balanced, + conformance, + enforced_cells, + replay, +) +from coder_eval.timing import close_window + + +ORIGIN = datetime(2026, 9, 16, 9, 0, 0) + + +def at(ms: float) -> datetime: + return ORIGIN + timedelta(milliseconds=ms) + + +class _Decoder: + """A minimal reducer: a window per ``end``, tiled from the previous end unless ``untiled``.""" + + def __init__(self, emitter: TurnEmitter, *, untiled: bool = False) -> None: + self.emitter = emitter + self.untiled = untiled + self.mark: datetime | None = None + self.turn_start: datetime | None = None + + def __call__(self, event: dict[str, Any]) -> None: + kind = event["type"] + if kind == "start": + self.turn_start = self.emitter.now() + self.emitter.begin_inner_turn(event["id"]) + elif kind == "tool_start": + self.emitter.open_tool(event["id"], "Bash", {}) + elif kind == "tool_end": + from coder_eval.streaming.events import ToolEndStatus + + self.emitter.close_tool(event["id"], status=ToolEndStatus.OK) + elif kind == "end": + now = self.emitter.now() + assert self.turn_start is not None + mark = self.turn_start if self.untiled or self.mark is None else self.mark + tokens = TokenUsage(output_tokens=1) + self.emitter.add_generation( + message_id=None, + window=close_window(mark=mark, now=now, item_start=self.turn_start), + parts=[Generation(blocks=[], tokens=tokens)], + ) + self.mark = now + self.emitter.end_inner_turn(tokens=tokens) + + +_STREAM: list[Any] = [ + Tick(500), + {"type": "start", "id": "t1"}, + Tick(700), + {"type": "tool_start", "id": "c1"}, + Tick(1200), + {"type": "tool_end", "id": "c1"}, + Tick(2000), + {"type": "end"}, + Tick(2600), + {"type": "start", "id": "t2"}, + Tick(3000), + {"type": "end"}, + Tick(3500), +] + + +class TestReplay: + def test_ticks_script_the_bracket_and_the_decoder_sees_the_rest(self) -> None: + result = replay(_STREAM, _Decoder, clock=ScriptedClock(ORIGIN)) + assert (result.started_at, result.ended_at) == (at(0), at(3500)) + assert result.outcome.status is AgentEndStatus.COMPLETED + assert result.record is result.outcome.record + assert [c.tool_id for c in result.record.commands] == ["c1"] + assert isinstance(result.events[0], AgentStartEvent) and isinstance(result.events[-1], AgentEndEvent) + + def test_a_scripted_clock_before_any_tick_reads_its_origin(self) -> None: + assert ScriptedClock(ORIGIN).now() == ORIGIN + + def test_end_decides_how_the_turn_ends(self) -> None: + result = replay([{"type": "start", "id": "t"}], _Decoder, clock=ScriptedClock(ORIGIN), end=_fail) + assert result.outcome.status is AgentEndStatus.CRASHED + + def test_a_decoder_exception_propagates(self) -> None: + with pytest.raises(KeyError): + replay([{"no": "type"}], _Decoder, clock=ScriptedClock(ORIGIN)) + + +def _fail(decoder: _Decoder) -> Any: + return decoder.emitter.fail(AgentEndStatus.CRASHED, "stopped") + + +class TestAssertIdentityCloses: + def test_a_tiled_replay_closes(self) -> None: + result = replay(_STREAM, _Decoder, clock=ScriptedClock(ORIGIN)) + assert_identity_closes(result.record, started_at=result.started_at, ended_at=result.ended_at) + + def test_an_untiled_replay_is_caught(self) -> None: + result = replay(_STREAM, lambda e: _Decoder(e, untiled=True), clock=ScriptedClock(ORIGIN)) + with pytest.raises(AssertionError, match="booked nowhere"): + assert_identity_closes(result.record, started_at=result.started_at, ended_at=result.ended_at) + + def test_a_record_without_a_head_fails(self) -> None: + result = replay([], _Decoder, clock=ScriptedClock(ORIGIN)) + with pytest.raises(AssertionError, match="head and tail"): + assert_identity_closes(result.record, started_at=result.started_at, ended_at=result.ended_at) + + +def _tool(tool_id: str) -> CommandTelemetry: + return CommandTelemetry(tool_name="Bash", tool_id=tool_id, timestamp=ORIGIN) + + +def _clean() -> list[StreamEvent]: + return [ + AgentStartEvent(task_id="t"), + TurnStartEvent(task_id="t", turn_id="a"), + ToolStartEvent(task_id="t", turn_id="a", tool=_tool("c1")), + ToolEndEvent(task_id="t", turn_id="a", tool=_tool("c1")), + TurnEndEvent(task_id="t", turn_id="a", tokens=TokenUsage(output_tokens=3)), + AgentEndEvent(task_id="t", usage=TokenUsage(output_tokens=3)), + ] + + +class TestAssertStreamBalanced: + def test_a_clean_stream_passes(self) -> None: + assert_stream_balanced(_clean()) + + @pytest.mark.parametrize( + ("mutate", "match"), + [ + (lambda ev: ev[:-1], "0 AgentEndEvent"), + (lambda ev: [ev[0], *ev], "2 AgentStartEvent"), + (lambda ev: ev[1:], "first event is TurnStartEvent"), + (lambda ev: [*ev[:4], ev[5]], "still open"), + (lambda ev: [ev[0], ev[1], ev[1], *ev[2:]], "started while"), + (lambda ev: [ev[0], ev[4], ev[5]], "ended while the open turn is None"), + (lambda ev: [*ev[:3], *ev[4:]], "ended 0 times"), + (lambda ev: [*ev[:4], ev[3], *ev[4:]], "ended 2 times"), + (lambda ev: [ev[0], ev[1], ev[3], *ev[4:]], "ended without a start"), + ( + lambda ev: [*ev[:5], AgentEndEvent(task_id="t", usage=TokenUsage(output_tokens=2))], + "output_tokens sum 3 > AgentEndEvent.usage 2", + ), + ], + ) + def test_each_violation_is_named( + self, mutate: Callable[[list[StreamEvent]], list[StreamEvent]], match: str + ) -> None: + with pytest.raises(AssertionError, match=match): + assert_stream_balanced(mutate(_clean())) + + def test_an_empty_stream_fails(self) -> None: + with pytest.raises(AssertionError, match="empty"): + assert_stream_balanced([]) + + +def _contract(kind: AgentKind) -> Any: + from coder_eval.agents.registry import AgentRegistry + + ensure_plugins_loaded() + registration = AgentRegistry.get(kind) + assert registration is not None + return registration.agent_class.contract + + +class TestEnforcedCells: + def test_claude_code_has_one_cell_per_permission_mode(self) -> None: + cells = enforced_cells(_contract(AgentKind.CLAUDE_CODE), "claude-code") + assert ("claude-code", "system_prompt") in cells + assert {c for c in cells if c[1].startswith("permission_mode=")} == { + ("claude-code", f"permission_mode={m.value}") for m in _contract(AgentKind.CLAUDE_CODE).permission_modes + } + + def test_the_noop_harness_enforces_nothing(self) -> None: + assert enforced_cells(_contract(AgentKind.NONE), "none") == set() + + @pytest.mark.parametrize("kind", [k for k in AgentKind if k not in (AgentKind.UNKNOWN, AgentKind.NONE)]) + def test_every_in_tree_harness_enforces_its_system_prompt_cell(self, kind: AgentKind) -> None: + assert (kind.value, "system_prompt") in enforced_cells(_contract(kind), kind.value) + + +async def _noop() -> None: + return None + + +class TestConformance: + async def test_the_noop_harness_conforms_with_no_probes(self) -> None: + await conformance("none", {}) + + async def test_a_missing_probe_fails(self) -> None: + with pytest.raises(AssertionError, match="missing"): + await conformance("pi", {}) + + async def test_an_extra_probe_fails(self) -> None: + probes: dict[tuple[str, str], Callable[[], Awaitable[None]]] = {("none", "system_prompt"): _noop} + with pytest.raises(AssertionError, match="extra"): + await conformance("none", probes) + + async def test_an_unregistered_kind_fails(self) -> None: + with pytest.raises(AssertionError, match="not registered"): + await conformance("no-such-harness", {}) diff --git a/tests/test_timing_close_window.py b/tests/test_timing_close_window.py index e69c11ba..f64e00da 100644 --- a/tests/test_timing_close_window.py +++ b/tests/test_timing_close_window.py @@ -13,7 +13,7 @@ import pytest -from coder_eval.timing import busy_ms, close_window, decompose_turn, main_thread_tool_spans, union_ms +from coder_eval.timing import Window, busy_ms, close_window, decompose_turn, main_thread_tool_spans, union_ms MARK = datetime(2026, 9, 11, 12, 0, 0) @@ -45,28 +45,33 @@ class TestCloseWindow: """ def test_the_window_is_the_whole_span_from_the_mark(self): - started, span_ms = close_window(mark=MARK, now=_at(1000)) - assert started == MARK - assert span_ms == pytest.approx(1000.0) + window = close_window(mark=MARK, now=_at(1000)) + assert window == Window(started_at=MARK, completed_at=_at(1000)) + assert window.duration_ms == pytest.approx(1000.0) def test_item_start_before_the_mark_wins(self): # A stamp that went backwards: the window must cover the item, so the # min() moves the start back rather than inverting the span. - started, span_ms = close_window(mark=MARK, now=_at(1000), item_start=_at(-200)) - assert started == _at(-200) - assert span_ms == pytest.approx(1200.0) + window = close_window(mark=MARK, now=_at(1000), item_start=_at(-200)) + assert window.started_at == _at(-200) + assert window.duration_ms == pytest.approx(1200.0) def test_item_start_after_the_mark_keeps_the_mark(self): # The normal tiling case: the gap between the previous close and this # item's first stamp IS model time and belongs inside the window. - started, span_ms = close_window(mark=MARK, now=_at(1000), item_start=_at(400)) - assert started == MARK - assert span_ms == pytest.approx(1000.0) + window = close_window(mark=MARK, now=_at(1000), item_start=_at(400)) + assert window.started_at == MARK + assert window.duration_ms == pytest.approx(1000.0) def test_an_inverted_window_clamps_to_zero_rather_than_going_negative(self): - started, span_ms = close_window(mark=_at(1000), now=MARK) - assert started == _at(1000) - assert span_ms == 0.0 + window = close_window(mark=_at(1000), now=MARK) + assert (window.started_at, window.completed_at) == (_at(1000), MARK) + assert window.duration_ms == 0.0 + + def test_a_window_is_frozen(self): + window = close_window(mark=MARK, now=_at(1000)) + with pytest.raises(AttributeError): + window.started_at = _at(5) # type: ignore[misc] def test_mark_is_keyword_only_and_has_no_default(self): # A reducer cannot open a window without STATING what it tiles from. diff --git a/tests/test_timing_identity_contract.py b/tests/test_timing_identity_contract.py index 54ccf2c8..f8bab325 100644 --- a/tests/test_timing_identity_contract.py +++ b/tests/test_timing_identity_contract.py @@ -69,7 +69,7 @@ ToolEndEvent, ToolEndStatus, ) -from coder_eval.timing import main_thread_tool_spans, union_ms +from coder_eval.testing import assert_identity_closes # The two CLI harnesses (opencode, codex) report their stamps as epoch @@ -127,56 +127,8 @@ def _record(turn: Turn) -> TurnRecord: return collector.build_turn_record() -def assert_identity_closes(turn: Turn) -> None: - """head + Σ generation + UNION(tool) + tail == the scripted span, EXACTLY. - - ``pytest.approx`` rather than an order-of-magnitude bound: every input is - scripted, so the only slack is float representation. A bound wide enough to - absorb a real defect is the sensor this module exists to replace. - - MAIN THREAD ONLY on both sides, and both through production's own helpers: - a sub-agent's generations bubble into the same stream, and the spawning - Agent call's own interval already spans them and their tools. - """ - record = _record(turn) - span_ms = turn.ended_ms - turn.started_ms - - generation_ms = sum( - m.generation_duration_ms or 0.0 - for m in record.messages - if isinstance(m, AssistantMessage) and m.parent_tool_use_id is None - ) - # The PRODUCTION selector, not a re-derivation of it. Unioning every command - # would assert a different identity than the collector computes: production, - # the golden sensor, the live residual gate and the HTML report all exclude - # a sub-agent's own tools (the spawning Agent call's interval already spans - # them). No case here has a child command yet, so a local copy stayed green - # while quietly testing something else — and the first sub-agent case added - # would have reported a false regression. - tool_ms = union_ms(main_thread_tool_spans(record.messages, record.commands)) - assert record.harness_startup_ms is not None, "a turn that generated has a measured head" - assert record.harness_teardown_ms is not None, "a turn that generated has a measured tail" - # The STORED bucket must equal the one just computed independently. Without - # this the ms-exact sensor would cover three of the four buckets and read - # the fourth from a re-derivation, leaving the published field unchecked on - # every harness — which is how a stored value and its consumers drift. - # `None` only when no bounded span exists, in which case the union is 0.0. - stored_tool_ms = record.tool_union_ms if record.tool_union_ms is not None else 0.0 - assert stored_tool_ms == pytest.approx(tool_ms), ( - f"TurnRecord.tool_union_ms is {record.tool_union_ms}, but this turn's main-thread " - f"command spans union to {tool_ms:.4f} ms. The collector writes the field from the same " - "span set it measures the head and the tail against, so a disagreement means the stored " - "value and the selection rule have come apart." - ) - bucket_sum = record.harness_startup_ms + generation_ms + tool_ms + record.harness_teardown_ms - - assert bucket_sum == pytest.approx(span_ms), ( - f"the four buckets sum to {bucket_sum:.4f} ms against a {span_ms:.4f} ms turn " - f"(off by {bucket_sum - span_ms:+.4f} ms): head={record.harness_startup_ms:.4f}, " - f"generation={generation_ms:.4f}, tool_union={tool_ms:.4f}, tail={record.harness_teardown_ms:.4f}. " - "They tile the turn, so a sum UNDER it means some interval is booked nowhere — the " - "defect class the golden corpus cannot see — and a sum OVER it means one is booked twice." - ) +def _assert_closes(turn: Turn) -> None: + assert_identity_closes(_record(turn), started_at=at(turn.started_ms), ended_at=at(turn.ended_ms)) # -------------------------------------------------------------------------- @@ -619,23 +571,23 @@ def _monotonic() -> float: def test_pi_buckets_tile_the_turn(): - assert_identity_closes(_pi_turn()) + _assert_closes(_pi_turn()) def test_opencode_buckets_tile_the_turn(monkeypatch: pytest.MonkeyPatch): - assert_identity_closes(_opencode_turn(monkeypatch)) + _assert_closes(_opencode_turn(monkeypatch)) def test_antigravity_buckets_tile_the_turn(): - assert_identity_closes(_antigravity_turn()) + _assert_closes(_antigravity_turn()) def test_codex_buckets_tile_the_turn(): - assert_identity_closes(_codex_turn()) + _assert_closes(_codex_turn()) def test_claude_code_buckets_tile_the_turn(monkeypatch: pytest.MonkeyPatch): - assert_identity_closes(_claude_turn(monkeypatch)) + _assert_closes(_claude_turn(monkeypatch)) def test_a_slow_tool_result_round_trip_is_not_lost(monkeypatch: pytest.MonkeyPatch): @@ -645,7 +597,7 @@ def test_a_slow_tool_result_round_trip_is_not_lost(monkeypatch: pytest.MonkeyPat `on_user_message`) fails THIS and leaves every other case in the file green, which is exactly what happened in production. """ - assert_identity_closes(_claude_slow_result_turn(monkeypatch)) + _assert_closes(_claude_slow_result_turn(monkeypatch)) def test_every_built_in_harness_has_a_case(): @@ -689,4 +641,4 @@ def _generation_ms(turn: Turn) -> float: assert _generation_ms(healthy) - _generation_ms(mutated) == pytest.approx(600.0) with pytest.raises(AssertionError, match="booked nowhere"): - assert_identity_closes(mutated) + _assert_closes(mutated) diff --git a/tests/test_turn_emitter.py b/tests/test_turn_emitter.py new file mode 100644 index 00000000..aa03c7fb --- /dev/null +++ b/tests/test_turn_emitter.py @@ -0,0 +1,567 @@ +"""``TurnEmitter``: the one writer of the event protocol, and the rules it enforces at runtime.""" + +from __future__ import annotations + +import math +from datetime import datetime, timedelta +from typing import Any + +import pytest + +from coder_eval.errors import AgentCrashError, TurnTimeoutError +from coder_eval.errors.agent import CRASH_REASON_MAX_CHARS +from coder_eval.models import ContentBlock, ResultSummary, TimingBasis, TokenUsage +from coder_eval.streaming.emitter import Generation, TurnEmitter, TurnOutcome +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentEndStatus, + AgentStartEvent, + StreamEvent, + TextChunkEvent, + ToolEndEvent, + ToolEndStatus, + ToolStartEvent, + TurnEndEvent, + TurnEndStatus, + TurnStartEvent, +) +from coder_eval.timing import Window, close_window + + +BASE = datetime(2026, 9, 16, 12, 0, 0) + + +def at(ms: float) -> datetime: + return BASE + timedelta(milliseconds=ms) + + +class _Clock: + def __init__(self) -> None: + self.ms = 0.0 + + def now(self) -> datetime: + return at(self.ms) + + +class _Sink: + def __init__(self) -> None: + self.events: list[StreamEvent] = [] + + def on_event(self, event: StreamEvent) -> None: + self.events.append(event) + + def of(self, kind: type[Any]) -> list[Any]: + return [e for e in self.events if isinstance(e, kind)] + + +def _emitter( + basis: TimingBasis = TimingBasis.TURN_CLOCK, *, sinks: list[Any] | None = None +) -> tuple[TurnEmitter, _Clock, _Sink]: + clock, sink = _Clock(), _Sink() + emitter = TurnEmitter( + task_id="t", + iteration=3, + prompt="go", + model="m", + basis=basis, + clock=clock, + sinks=[sink] if sinks is None else sinks, + ) + emitter.begin() + return emitter, clock, sink + + +def _text(text: str) -> ContentBlock: + return ContentBlock(block_type="text", sequence=0, text=text) + + +def _tool_use(tool_id: str) -> ContentBlock: + return ContentBlock(block_type="tool_use", sequence=0, tool_use_id=tool_id) + + +def _part(*blocks: ContentBlock, output: int = 0) -> Generation: + return Generation(blocks=list(blocks), tokens=TokenUsage(output_tokens=output)) + + +class TestBracketAndSinks: + def test_the_bracket_is_stamped_from_the_clock(self) -> None: + clock, sink = _Clock(), _Sink() + clock.ms = 100 + emitter = TurnEmitter( + task_id="t", iteration=1, prompt="go", model="m", basis=TimingBasis.TURN_CLOCK, clock=clock, sinks=[sink] + ) + emitter.begin() + clock.ms = 900 + emitter.finalize(AgentEndStatus.COMPLETED) + assert sink.of(AgentStartEvent)[0].timestamp == at(100) + assert sink.of(AgentEndEvent)[0].timestamp == at(900) + + def test_begin_twice_raises(self) -> None: + emitter, _, _ = _emitter() + with pytest.raises(RuntimeError, match="twice"): + emitter.begin() + + def test_a_raising_sink_does_not_break_the_emitter(self) -> None: + class _Broken: + def on_event(self, event: StreamEvent) -> None: + raise ValueError("boom") + + good = _Sink() + emitter, _, _ = _emitter(sinks=[_Broken(), good]) + emitter.text("hi") + outcome = emitter.finalize(AgentEndStatus.COMPLETED) + assert outcome.record.agent_output == "hi" + assert len(good.of(AgentEndEvent)) == 1 + + def test_no_sinks_is_valid(self) -> None: + emitter, _, _ = _emitter(sinks=[]) + assert emitter.finalize(AgentEndStatus.COMPLETED).status is AgentEndStatus.COMPLETED + + +class TestToolBasis: + @pytest.mark.parametrize("keyword", ["started_at", "completed_at"]) + @pytest.mark.parametrize("value", [None, BASE]) + def test_turn_clock_rejects_an_explicit_stamp(self, keyword: str, value: datetime | None) -> None: + emitter, _, _ = _emitter() + with pytest.raises(TypeError, match="TURN_CLOCK"): + if keyword == "started_at": + emitter.open_tool("c1", "Bash", {}, started_at=value) + else: + emitter.open_tool("c1", "Bash", {}) + emitter.close_tool("c1", status=ToolEndStatus.OK, completed_at=value) + + def test_cli_epoch_requires_both_stamps_on_a_main_thread_tool(self) -> None: + emitter, _, _ = _emitter(TimingBasis.CLI_EPOCH_MS) + with pytest.raises(TypeError, match="started_at"): + emitter.open_tool("c1", "Bash", {}) + emitter.open_tool("c1", "Bash", {}, started_at=None) + with pytest.raises(TypeError, match="completed_at"): + emitter.close_tool("c1", status=ToolEndStatus.OK) + + def test_cli_epoch_uses_the_stamps_passed(self) -> None: + emitter, _, _ = _emitter(TimingBasis.CLI_EPOCH_MS) + emitter.open_tool("c1", "Bash", {}, started_at=at(700)) + emitter.close_tool("c1", status=ToolEndStatus.OK, completed_at=at(1200)) + command = emitter.finalize(AgentEndStatus.COMPLETED).record.commands[0] + assert (command.execution_started_at, command.execution_completed_at) == (at(700), at(1200)) + assert command.duration_ms == pytest.approx(500.0) + + @pytest.mark.parametrize("basis", [TimingBasis.TURN_CLOCK, TimingBasis.CLI_EPOCH_MS]) + def test_a_nested_tool_is_exempt_from_both_checks(self, basis: TimingBasis) -> None: + emitter, _, _ = _emitter(basis) + emitter.open_tool("a", "Bash", {}, parent_tool_id="agent_1") + emitter.close_tool("a", status=ToolEndStatus.OK) + emitter.open_tool("b", "Bash", {}, parent_tool_id="agent_1", started_at=at(5)) + emitter.close_tool("b", status=ToolEndStatus.OK, completed_at=at(9)) + commands = {c.tool_id: c for c in emitter.finalize(AgentEndStatus.COMPLETED).record.commands} + assert commands["b"].duration_ms == pytest.approx(4.0) + stamped = basis is TimingBasis.TURN_CLOCK + assert (commands["a"].execution_started_at is not None) is stamped + assert (commands["a"].execution_completed_at is not None) is stamped + + def test_a_cli_tool_with_no_stamps_is_untimed(self) -> None: + emitter, _, _ = _emitter(TimingBasis.CLI_EPOCH_MS) + emitter.open_tool("c1", "Bash", {}, started_at=None) + emitter.close_tool("c1", status=ToolEndStatus.OK, completed_at=None) + command = emitter.finalize(AgentEndStatus.COMPLETED).record.commands[0] + assert (command.execution_started_at, command.execution_completed_at, command.duration_ms) == (None, None, None) + assert command.result_status == "success" + + def test_a_completion_before_the_start_is_a_zero_duration(self) -> None: + emitter, _, _ = _emitter(TimingBasis.CLI_EPOCH_MS) + emitter.open_tool("c1", "Bash", {}, started_at=at(5000)) + emitter.close_tool("c1", status=ToolEndStatus.OK, completed_at=at(4000)) + assert emitter.finalize(AgentEndStatus.COMPLETED).record.commands[0].duration_ms == 0.0 + + def test_an_unknown_id_close_under_cli_epoch_still_needs_its_stamp(self) -> None: + emitter, _, _ = _emitter(TimingBasis.CLI_EPOCH_MS) + with pytest.raises(TypeError, match="completed_at"): + emitter.close_tool("ghost", status=ToolEndStatus.OK) + + def test_turn_clock_stamps_tools_from_the_clock(self) -> None: + emitter, clock, _ = _emitter() + clock.ms = 700 + emitter.open_tool("c1", "Bash", {"command": "ls"}, generation_completed=True) + clock.ms = 1200 + emitter.close_tool("c1", status=ToolEndStatus.OK, summary="out") + command = emitter.finalize(AgentEndStatus.COMPLETED).record.commands[0] + assert command.timestamp == command.execution_started_at == command.generation_completed_at == at(700) + assert command.execution_completed_at == at(1200) + assert command.duration_ms == pytest.approx(500.0) + assert (command.result_status, command.result_summary) == ("success", "out") + + +class TestTools: + def test_sequence_numbers_start_at_zero_in_open_order(self) -> None: + emitter, _, _ = _emitter() + for tool_id in ("a", "b", "c"): + emitter.open_tool(tool_id, "Bash", {}) + emitter.close_tool("c", status=ToolEndStatus.OK) + emitter.close_tool("a", status=ToolEndStatus.OK) + emitter.close_tool("b", status=ToolEndStatus.OK) + record = emitter.finalize(AgentEndStatus.COMPLETED).record + assert [(c.tool_id, c.sequence_number) for c in record.commands] == [("a", 0), ("b", 1), ("c", 2)] + + def test_an_unresolved_tool_keeps_its_start_and_gets_no_completion(self) -> None: + emitter, clock, _ = _emitter() + clock.ms = 10 + emitter.open_tool("c1", "Bash", {}) + clock.ms = 20 + emitter.close_tool("c1", status=ToolEndStatus.UNRESOLVED) + command = emitter.finalize(AgentEndStatus.COMPLETED).record.commands[0] + assert command.execution_started_at == at(10) + assert command.execution_completed_at is None and command.duration_ms is None + assert command.result_status == "unknown" + + def test_the_orphan_sweep_closes_an_open_tool_unresolved(self) -> None: + emitter, _, sink = _emitter() + emitter.open_tool("c1", "Bash", {}) + record = emitter.fail(AgentEndStatus.CRASHED, "died").record + ends = sink.of(ToolEndEvent) + assert [e.status for e in ends] == [ToolEndStatus.UNRESOLVED] + command = record.commands[0] + assert command.result_status == "unknown" + assert command.execution_started_at is not None + assert (command.execution_completed_at, command.duration_ms, command.error_message) == (None, None, None) + + def test_a_close_for_an_unknown_id_synthesizes_a_call(self) -> None: + emitter, _, _ = _emitter() + emitter.open_tool("known", "Bash", {}) + emitter.close_tool("ghost", status=ToolEndStatus.ERROR, error="no start") + commands = {c.tool_id: c for c in emitter.fail(AgentEndStatus.CRASHED, "x").record.commands} + assert commands["ghost"].tool_name == "unknown" + assert commands["ghost"].sequence_number == 1 + assert commands["ghost"].result_status == "error" + + def test_close_refreshes_parameters_and_result_data(self) -> None: + emitter, _, _ = _emitter() + emitter.open_tool("c1", "Bash", {"a": 1}) + emitter.close_tool("c1", status=ToolEndStatus.PERMISSION_DENIED, parameters={"b": 2}, result_data=[1]) + command = emitter.finalize(AgentEndStatus.COMPLETED).record.commands[0] + assert (command.parameters, command.result_data, command.result_status) == ({"b": 2}, [1], "error") + + def test_a_tool_end_carries_its_opening_turn_id(self) -> None: + emitter, _, sink = _emitter() + emitter.begin_inner_turn("turn_1") + emitter.open_tool("c1", "Bash", {}) + emitter.end_inner_turn() + emitter.begin_inner_turn("turn_2") + emitter.close_tool("c1", status=ToolEndStatus.OK) + assert sink.of(ToolStartEvent)[0].turn_id == sink.of(ToolEndEvent)[0].turn_id == "turn_1" + + +class TestInnerTurns: + def test_begin_while_open_raises(self) -> None: + emitter, _, _ = _emitter() + emitter.begin_inner_turn("a") + with pytest.raises(RuntimeError, match="still open"): + emitter.begin_inner_turn("b") + + def test_end_with_none_open_raises(self) -> None: + emitter, _, _ = _emitter() + with pytest.raises(RuntimeError, match="no inner turn"): + emitter.end_inner_turn() + + @pytest.mark.parametrize( + ("end", "expected"), + [("finalize", TurnEndStatus.TOOL_CALLS_EXHAUSTED), ("fail", TurnEndStatus.TIMEOUT)], + ) + def test_the_end_closes_an_open_inner_turn_with_the_mapped_status(self, end: str, expected: TurnEndStatus) -> None: + emitter, _, sink = _emitter() + emitter.begin_inner_turn("a") + assert emitter.inner_turn_open + if end == "finalize": + emitter.finalize(AgentEndStatus.TOOL_CALLS_EXHAUSTED) + else: + emitter.fail(AgentEndStatus.TIMEOUT, "late") + turn_end = sink.of(TurnEndEvent)[0] + assert (turn_end.turn_id, turn_end.status, turn_end.tokens) == ("a", expected, None) + assert isinstance(sink.events[-1], AgentEndEvent) + + def test_turns_are_counted_on_the_main_thread_only(self) -> None: + emitter, _, sink = _emitter() + emitter.begin_inner_turn("a") + emitter.end_inner_turn() + emitter.begin_inner_turn("sub", "sub-model", parent_tool_id="agent_1") + emitter.end_inner_turn() + end = emitter.finalize(AgentEndStatus.COMPLETED) + assert end.record.assistant_turn_count == 1 + assert end.record.num_turns == 1 + assert [e.model for e in sink.of(TurnStartEvent)] == ["m", "sub-model"] + + +class TestGenerations: + def test_two_parts_share_bounds_and_apportion_by_output(self) -> None: + emitter, _, _ = _emitter() + window = close_window(mark=at(0), now=at(1000)) + messages = emitter.add_generation( + message_id="g1", window=window, parts=[_part(_text("a"), output=1), _part(_text("b"), output=2)] + ) + assert {(m.started_at, m.completed_at, m.message_id) for m in messages} == {(at(0), at(1000), "g1")} + assert messages[0].generation_duration_ms == round(1000 / 3, 6) + assert math.isclose(sum(m.generation_duration_ms or 0 for m in messages), 1000.0, abs_tol=1e-6) + + def test_parts_with_no_output_split_evenly(self) -> None: + emitter, _, _ = _emitter() + window = Window(started_at=at(0), completed_at=at(90)) + messages = emitter.add_generation(message_id=None, window=window, parts=[_part(), _part(), _part()]) + assert [m.generation_duration_ms for m in messages] == pytest.approx([30.0, 30.0, 30.0]) + + def test_empty_parts_raise(self) -> None: + emitter, _, _ = _emitter() + with pytest.raises(ValueError, match="at least one part"): + emitter.add_generation(message_id=None, window=Window(at(0), at(1)), parts=[]) + + def test_message_id_is_a_required_keyword(self) -> None: + emitter, _, _ = _emitter() + with pytest.raises(TypeError): + emitter.add_generation(window=Window(at(0), at(1)), parts=[_part()]) # type: ignore[call-arg] + + def test_the_message_carries_its_part_and_the_live_blocks(self) -> None: + emitter, _, _ = _emitter() + block = _tool_use("c1") + part = Generation( + blocks=[_text("x"), block], + tokens=TokenUsage( + uncached_input_tokens=1, output_tokens=2, cache_creation_input_tokens=3, cache_read_input_tokens=4 + ), + reasoning_tokens=1, + stop_reason="tool_use", + ) + (message,) = emitter.add_generation( + message_id="g", window=Window(at(0), at(5)), parts=[part], parent_tool_id="agent_1", model="sub" + ) + assert message.tool_use_ids == ["c1"] + assert message.content_blocks[1] is block + assert ( + message.input_tokens, + message.output_tokens, + message.cache_creation_tokens, + message.cache_read_tokens, + ) == ( + 1, + 2, + 3, + 4, + ) + assert (message.reasoning_tokens, message.stop_reason, message.model) == (1, "tool_use", "sub") + assert message.parent_tool_use_id == "agent_1" + block.is_error = True + end = emitter.finalize(AgentEndStatus.COMPLETED) + assert end.record.messages[0].content_blocks[1].is_error is True # type: ignore[union-attr] + + def test_an_unmeasured_generation_has_equal_bounds_and_no_duration(self) -> None: + emitter, clock, _ = _emitter() + clock.ms = 42 + message = emitter.add_unmeasured_generation(message_id="u", part=_part(_text("late"))) + assert message.started_at == message.completed_at == at(42) + assert message.generation_duration_ms is None + assert message.model == "m" + + +class TestThreads: + def test_nested_events_carry_the_parent_tool_as_thread_and_parent(self) -> None: + emitter, _, sink = _emitter() + emitter.begin_inner_turn("sub", parent_tool_id="agent_1") + emitter.text("child says", parent_tool_id="agent_1") + emitter.open_tool("c1", "Bash", {}, parent_tool_id="agent_1") + emitter.close_tool("c1", status=ToolEndStatus.OK) + emitter.end_inner_turn() + nested = sink.events[1:5] + assert {(e.thread_id, e.parent_thread_id) for e in nested} == {("agent_1", "agent_1")} + assert sink.of(TurnEndEvent)[0].parent_thread_id == "agent_1" + end = emitter.finalize(AgentEndStatus.COMPLETED) + assert end.record.agent_output == "" + + def test_main_thread_events_have_no_thread_id(self) -> None: + emitter, _, sink = _emitter() + emitter.begin_inner_turn("a") + emitter.text("hi") + emitter.open_tool("c1", "Bash", {}) + emitter.close_tool("c1", status=ToolEndStatus.OK) + emitter.finalize(AgentEndStatus.COMPLETED) + assert {(e.thread_id, e.parent_thread_id) for e in sink.events} == {(None, None)} + + +class TestTokens: + def test_the_end_publishes_the_sum_of_turn_deltas_by_default(self) -> None: + emitter, _, sink = _emitter() + for n in (1, 2): + emitter.begin_inner_turn(f"t{n}") + emitter.end_inner_turn(tokens=TokenUsage(output_tokens=n)) + record = emitter.finalize(AgentEndStatus.COMPLETED).record + assert record.token_usage == TokenUsage(output_tokens=3) + assert sink.of(AgentEndEvent)[0].usage.output_tokens == 3 + + def test_no_turn_and_no_usage_publishes_no_token_usage(self) -> None: + emitter, _, _ = _emitter() + assert emitter.finalize(AgentEndStatus.COMPLETED).record.token_usage is None + + def test_deltas_over_the_published_usage_warn_once_and_do_not_raise(self, caplog: pytest.LogCaptureFixture) -> None: + emitter, _, _ = _emitter() + emitter.begin_inner_turn("a") + emitter.end_inner_turn(tokens=TokenUsage(output_tokens=10, cache_read_input_tokens=5)) + with caplog.at_level("WARNING", logger="coder_eval.streaming.emitter"): + outcome = emitter.finalize(AgentEndStatus.COMPLETED, usage=TokenUsage(output_tokens=4, total_cost_usd=9.0)) + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "output_tokens 10 > 4" in warnings[0].getMessage() + assert "cache_read_input_tokens 5 > 0" in warnings[0].getMessage() + assert "[t]" in warnings[0].getMessage() + assert outcome.status is AgentEndStatus.COMPLETED + + def test_a_cost_difference_is_not_a_bucket(self, caplog: pytest.LogCaptureFixture) -> None: + emitter, _, _ = _emitter() + emitter.begin_inner_turn("a") + emitter.end_inner_turn(tokens=TokenUsage(output_tokens=1, total_cost_usd=5.0)) + with caplog.at_level("WARNING", logger="coder_eval.streaming.emitter"): + emitter.finalize(AgentEndStatus.COMPLETED, usage=TokenUsage(output_tokens=1, total_cost_usd=1.0)) + assert not caplog.records + + +class TestTheEnd: + def test_fail_timeout_is_a_crashed_record_with_the_untruncated_error(self) -> None: + emitter, _, _ = _emitter() + reason = "x" * (CRASH_REASON_MAX_CHARS + 50) + outcome = emitter.fail(AgentEndStatus.TIMEOUT, reason) + assert outcome.status is AgentEndStatus.TIMEOUT + assert outcome.error == reason + assert outcome.record.crashed is True + assert outcome.record.crash_reason is not None + assert len(outcome.record.crash_reason) == CRASH_REASON_MAX_CHARS + 1 + assert outcome.record.result_summary is None + assert outcome.record.iteration == 3 + + def test_finalize_summary_is_the_final_text_reply(self) -> None: + emitter, _, _ = _emitter() + emitter.add_generation(message_id=None, window=Window(at(0), at(1)), parts=[_part(_tool_use("c1"))]) + emitter.add_generation( + message_id=None, window=Window(at(1), at(2)), parts=[_part(_text("Done "), _text("now."))] + ) + emitter.add_unmeasured_generation(message_id=None, part=_part(_text("child")), parent_tool_id="agent_1") + summary = emitter.finalize(AgentEndStatus.COMPLETED, stop_reason="end_turn").record.result_summary + assert summary == ResultSummary(is_error=False, subtype="completed", stop_reason="end_turn", result="Done now.") + + def test_a_last_message_that_calls_a_tool_is_no_final_reply(self) -> None: + emitter, _, _ = _emitter() + emitter.add_generation( + message_id=None, window=Window(at(0), at(1)), parts=[_part(_text("calling"), _tool_use("c1"))] + ) + summary = emitter.finalize(AgentEndStatus.TOOL_CALLS_EXHAUSTED).record.result_summary + assert summary is not None and summary.result is None and summary.subtype == "tool_calls_exhausted" + + @pytest.mark.parametrize("given", [None, ResultSummary(is_error=True, subtype="sdk", result="detail")]) + def test_an_explicit_result_summary_wins(self, given: ResultSummary | None) -> None: + emitter, _, _ = _emitter() + emitter.add_generation(message_id=None, window=Window(at(0), at(1)), parts=[_part(_text("reply"))]) + assert emitter.finalize(AgentEndStatus.COMPLETED, result_summary=given).record.result_summary == given + + @pytest.mark.parametrize("status", [AgentEndStatus.CRASHED, AgentEndStatus.TIMEOUT]) + def test_finalize_refuses_a_failed_status(self, status: AgentEndStatus) -> None: + emitter, _, _ = _emitter() + with pytest.raises(ValueError, match="fail"): + emitter.finalize(status) + + def test_fail_refuses_a_clean_status(self) -> None: + emitter, _, _ = _emitter() + with pytest.raises(ValueError, match="finalize"): + emitter.fail(AgentEndStatus.COMPLETED, "x") # type: ignore[arg-type] + + def test_the_end_is_idempotent(self) -> None: + emitter, _, sink = _emitter() + first = emitter.finalize(AgentEndStatus.COMPLETED) + assert emitter.finalize(AgentEndStatus.STOPPED_EARLY) is first + assert emitter.fail(AgentEndStatus.CRASHED, "late") is first + assert len(sink.of(AgentEndEvent)) == 1 + + def test_writes_after_the_end_are_dropped(self, caplog: pytest.LogCaptureFixture) -> None: + emitter, _, sink = _emitter() + emitter.begin_inner_turn("a") + emitter.fail(AgentEndStatus.CRASHED, "gone") + count = len(sink.events) + with caplog.at_level("DEBUG", logger="coder_eval.streaming.emitter"): + emitter.text("late") + emitter.open_tool("c1", "Bash", {}) + emitter.close_tool("c1", status=ToolEndStatus.OK) + emitter.begin_inner_turn("b") + emitter.end_inner_turn() + late = emitter.add_generation(message_id=None, window=Window(at(0), at(1)), parts=[_part(_text("late"))]) + emitter.add_unmeasured_generation(message_id=None, part=_part()) + assert len(sink.events) == count + assert late[0] not in sink.of(AgentEndEvent)[0].messages + assert sink.of(AgentEndEvent)[0].messages == [] + assert len([r for r in caplog.records if "dropped" in r.getMessage()]) == 1 + + def test_default_agent_output_is_the_joined_main_thread_text(self) -> None: + emitter, _, sink = _emitter() + emitter.text("Hello, ") + emitter.text("ignored", parent_tool_id="agent_1") + emitter.text("world") + outcome = emitter.finalize(AgentEndStatus.COMPLETED) + assert outcome.record.agent_output == "Hello, world" + assert [e.text for e in sink.of(TextChunkEvent)] == ["Hello, ", "ignored", "world"] + + def test_explicit_payload_overrides(self) -> None: + emitter, _, _ = _emitter() + emitter.text("streamed") + record = emitter.finalize( + AgentEndStatus.COMPLETED, agent_output="final", model_used="m2", assistant_turn_count=4, num_turns=5 + ).record + assert (record.agent_output, record.model_used, record.assistant_turn_count, record.num_turns) == ( + "final", + "m2", + 4, + 5, + ) + + +def _outcome(status: AgentEndStatus, error: str | None = None) -> TurnOutcome: + emitter, _, _ = _emitter() + return emitter.fail(status, error or "") if error is not None else emitter.finalize(status) # type: ignore[arg-type] + + +class TestRecordOrRaise: + @pytest.mark.parametrize( + "status", + [s for s in AgentEndStatus if s not in (AgentEndStatus.CRASHED, AgentEndStatus.TIMEOUT)], + ) + def test_a_clean_status_returns_the_record(self, status: AgentEndStatus) -> None: + outcome = _outcome(status) + assert outcome.record_or_raise() is outcome.record + + def test_crashed_raises_agent_crash_error_with_the_full_error(self) -> None: + with pytest.raises(AgentCrashError, match="provider exploded"): + _outcome(AgentEndStatus.CRASHED, "provider exploded").record_or_raise() + + def test_timeout_raises_turn_timeout_error(self) -> None: + with pytest.raises(TurnTimeoutError) as raised: + _outcome(AgentEndStatus.TIMEOUT, "late").record_or_raise(timeout_seconds=30, task_id="t9", iteration=2) + assert raised.value.timeout_seconds == 30 + assert raised.value.iteration == 2 + + +class TestEndRobustness: + def test_duration_seconds_is_measured_from_begin(self, monkeypatch: pytest.MonkeyPatch) -> None: + from coder_eval.streaming import emitter as emitter_module + + ticks = iter([100.0, 102.5]) + monkeypatch.setattr(emitter_module.time, "monotonic", lambda: next(ticks)) + emitter, _, _ = _emitter() + assert emitter.finalize(AgentEndStatus.COMPLETED).record.duration_seconds == pytest.approx(2.5) + + def test_a_record_that_cannot_be_built_ends_the_turn_once(self, monkeypatch: pytest.MonkeyPatch) -> None: + from coder_eval.streaming.collector import EventCollector + + emitter, _, sink = _emitter() + + def explode(_self: Any) -> Any: + raise ValueError("bad record") + + monkeypatch.setattr(EventCollector, "build_turn_record", explode) + with pytest.raises(ValueError, match="bad record"): + emitter.finalize(AgentEndStatus.COMPLETED) + with pytest.raises(RuntimeError, match="could not be built"): + emitter.fail(AgentEndStatus.CRASHED, "again") + emitter.text("late") + assert len(sink.of(AgentEndEvent)) == 1 + assert not sink.of(TextChunkEvent) diff --git a/tests/test_turn_monitor.py b/tests/test_turn_monitor.py index 0b1bc2f7..116746b5 100644 --- a/tests/test_turn_monitor.py +++ b/tests/test_turn_monitor.py @@ -433,3 +433,52 @@ def test_an_unpriced_in_flight_delta_contributes_nothing(self) -> None: _feed(monitor, [AgentStartEvent(task_id="t"), TurnEndEvent(task_id="t", tokens=TokenUsage(output_tokens=5))]) assert monitor.cost_usd() == 0.0 assert monitor.should_stop() is None + + +class TestSubAgentScope: + """A nested (sub-agent) event reaches the collector and the budgets, never the cap, criteria or model.""" + + @staticmethod + def _nested(event: Any) -> Any: + return event.model_copy(update={"thread_id": "agent_1", "parent_thread_id": "agent_1"}) + + def test_nested_tool_calls_do_not_reach_the_cap_but_reach_the_collector(self) -> None: + monitor = TurnMonitor.for_task(_task(max_tool_calls=1), arm=True) + _feed(monitor, [AgentStartEvent(task_id="t"), self._nested(_end("child"))]) + assert monitor.tool_calls == 0 + assert monitor.should_stop() is None + assert [c.tool_id for c in monitor._collector.build_turn_record().commands] == ["child"] + monitor.on_event(_end("main")) + assert monitor.should_stop() is StopReason.TOOL_CALL_CAP + + def test_nested_tool_calls_do_not_reach_armed_criteria(self) -> None: + criteria = [_skill_crit("date-teller", on_pass=True)] + monitor = TurnMonitor.for_task(_task(criteria=criteria), arm=True) + skill = _end("sk", tool_name="Skill", parameters={"skill": "date-teller"}) + with patch.object(TurnMonitor, "_evaluate_impl") as evaluate: + _feed(monitor, [self._nested(ToolStartEvent(task_id="t", tool=skill.tool)), self._nested(skill)]) + evaluate.assert_not_called() + assert monitor.should_stop() is None + + def test_a_nested_model_never_becomes_the_reported_model(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_usd=100.0)), arm=True) + _feed( + monitor, + [ + AgentStartEvent(task_id="t"), + self._nested(TurnStartEvent(task_id="t", model="claude-haiku-4-5")), + AgentEndEvent(task_id="t", usage=TokenUsage(uncached_input_tokens=1_000_000)), + ], + ) + assert monitor.cost_usd() is None + + def test_nested_tokens_count_toward_budgets(self) -> None: + monitor = TurnMonitor.for_task(_task(limits=RunLimits(max_output_tokens=10)), arm=True) + _feed( + monitor, + [ + AgentStartEvent(task_id="t"), + self._nested(TurnEndEvent(task_id="t", tokens=TokenUsage(output_tokens=11))), + ], + ) + assert monitor.should_stop() is StopReason.TOKEN_BUDGET From ac69e40611ac3af253ccb1270ab0e5e1c2eec9c6 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 18:08:11 -0700 Subject: [PATCH 03/25] =?UTF-8?q?feat(agents):=203/10=20=E2=80=94=20commun?= =?UTF-8?q?icate=20returns=20a=20TurnOutcome;=20the=20orchestrator=20owns?= =?UTF-8?q?=20the=20iteration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit communicate(..., iteration=) returns a TurnOutcome: a crash or timeout is an outcome with a crashed record, not an exception. The orchestrator appends a CRASHED/TIMEOUT record and raises it through record_or_raise (retry policy unchanged), and recovers a cancelled turn from a per-attempt EventCollector. run_with_watchdog runs a turn body as a child task so a watchdog timeout never cancels the caller. NoOpAgent writes through TurnEmitter; the five unported adapters keep their bodies behind Agent._legacy_outcome. on_attempt_error, _drain_pending_turn and _on_attempt_failure are deleted. Goldens unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- .../commands/coder-eval-code-review-full.md | 6 +- .claude/commands/coder-eval-create-plan.md | 2 +- .claude/commands/coder-eval-implement-plan.md | 4 +- .claude/notes/agents.md | 50 +- .claude/notes/reporting.md | 10 +- .claude/shared/review-rubric.md | 2 +- CLAUDE.md | 6 +- docs/EXTENDING.md | 51 +- docs/agents/CODEX.md | 14 +- docs/agents/HARNESS_PARITY.md | 4 +- src/coder_eval/agent.py | 140 ++++-- src/coder_eval/agents/antigravity_agent.py | 20 + src/coder_eval/agents/claude_code_agent.py | 20 + src/coder_eval/agents/codex_agent.py | 20 + src/coder_eval/agents/noop_agent.py | 72 +-- src/coder_eval/agents/opencode_agent.py | 20 + src/coder_eval/agents/pi_agent.py | 20 + src/coder_eval/agents/watchdog.py | 46 +- src/coder_eval/errors/executor.py | 21 +- src/coder_eval/evaluation/sub_agent.py | 3 +- src/coder_eval/orchestrator.py | 199 ++++---- src/coder_eval/simulation/user_simulator.py | 4 +- src/coder_eval/spi.py | 3 + tests/_fixtures/golden_streams/__init__.py | 2 +- .../golden_streams/antigravity_fixtures.py | 2 +- .../golden_streams/claude_fixtures.py | 32 +- .../golden_streams/codex_fixtures.py | 21 +- .../golden_streams/opencode_fixtures.py | 25 +- tests/_fixtures/golden_streams/pi_fixtures.py | 25 +- tests/fixtures/mock_agent.py | 15 +- tests/fixtures/text_stub_agent.py | 12 +- tests/test_agent.py | 462 ++++++++++++------ tests/test_agent_golden_master.py | 2 +- tests/test_agent_judge_criterion.py | 22 +- tests/test_agent_telemetry.py | 50 +- tests/test_agent_telemetry_advanced.py | 10 +- tests/test_agent_timeout.py | 34 +- tests/test_agentless.py | 8 +- tests/test_antigravity_agent.py | 132 +++-- tests/test_byoa_plugin_live.py | 11 +- .../test_claude_settings_enforcement_live.py | 2 +- tests/test_codex_agent.py | 164 +++---- tests/test_codex_agent_live.py | 53 +- tests/test_early_stop.py | 180 +++---- tests/test_harness_conformance.py | 14 +- tests/test_opencode_agent.py | 105 ++-- tests/test_orchestrator.py | 67 +-- tests/test_pi_agent.py | 184 ++++--- tests/test_plugins.py | 3 +- tests/test_price_turn.py | 4 +- tests/test_reference_permissions.py | 12 +- tests/test_retry_logic_comprehensive.py | 107 +--- tests/test_run_limits_orchestrator.py | 19 +- tests/test_simulation_integration.py | 21 +- tests/test_spi.py | 2 + tests/test_streaming_agent_integration.py | 22 +- tests/test_sub_agent_runner.py | 65 ++- tests/test_timeout_orchestrator.py | 384 ++++++++++++--- tests/test_token_usage.py | 24 +- tests/test_user_simulator.py | 76 ++- tests/test_watchdog.py | 71 ++- 61 files changed, 1920 insertions(+), 1261 deletions(-) diff --git a/.claude/commands/coder-eval-code-review-full.md b/.claude/commands/coder-eval-code-review-full.md index bb5538b3..afae25ca 100644 --- a/.claude/commands/coder-eval-code-review-full.md +++ b/.claude/commands/coder-eval-code-review-full.md @@ -347,7 +347,7 @@ in a value that doesn't match the formula. coder_eval has several pairs of structures that *must* stay in sync. When one is changed, check the other: - Models with the same field across types (e.g. `RunSummary` and `VariantAggregate`, `TaskDefinition` and `ResolvedTask`, `EvaluationResult` and the per-row `CriterionResult`): verify type, default, validator, and field description match. - Parallel orchestration code paths: `orchestration/batch.py` ↔ `orchestration/experiment.py`. A bug fixed in one routinely needs to be fixed in the other (precedent in this codebase: dataset fan-out, run_limits merging, lineage tracking). - - Parallel agent paths: `Orchestrator` ↔ any new driver (e.g. `isolation/docker_runner.py`) — does the driver preserve the `pending_turn` / `crashed=True TurnRecord` contract documented in CLAUDE.md? + - Parallel agent paths: `Orchestrator` ↔ any new driver (e.g. `isolation/docker_runner.py`) — does the driver preserve the `TurnOutcome` / `crashed=True TurnRecord` contract (a failed turn is an outcome; a cancelled turn is ended before it propagates)? - Parallel renderers: `reports/markdown.py` ↔ `reports/experiment.py` ↔ `reports/html.py` ↔ `reports/helpers.py` — if a new field is added to `EvaluationResult`, do all four render it (and if not, is that deliberate)? Flag any divergence as a finding even if the unchanged side is technically still correct in isolation — the divergence itself is the bug, and silent drift between parallel paths is one of the most expensive defects to debug later. @@ -368,11 +368,11 @@ in a value that doesn't match the formula. 6. **Verify conformance to extension-point contracts (agents, criteria, backends, drivers, renderers).** coder_eval is a plugin-based, agnostic, multi-agent core (Claude / Codex / NoOp agents via the BYOA SPI; auto-discovered criteria; Bedrock / Anthropic backends; in-process / docker drivers). For every registered member of one of these extension points, confirm it honors the documented contract — a member that *registers* but silently *violates* the contract is a high-severity defect that a "read the code" pass misses because the code looks locally fine. - - **Agents** (every `Agent` subclass in `agents/`): `communicate()` calls `self._begin_turn()` at the top and `self._end_turn_ok()` on the success path; `stop()` calls `self._mark_stopped()`; it does NOT override `discard_pending_turn()` / `get_state()`. It emits one `AgentStartEvent` at the top and a matching `AgentEndEvent` on EVERY exit path (success / crash / timeout — from a `finally`), with `TurnStart`/`TurnEnd` per turn and `ToolStart`/`ToolEnd` per tool (orphaned tools closed `status=unresolved`). Before any mid-turn `raise AgentCrashError` / `TurnTimeoutError`, `self.pending_turn` is set to a `crashed=True` `TurnRecord`. The returned `TurnRecord` is built ONLY by the internal `EventCollector` — flag any `TurnRecord(` hand-assembled outside the synthetic-crash path. If the agent shells out / holds OS resources, `stop()` / `kill()` / `kill_sync()` are real, and `kill_sync()` is synchronous (no `await` — it runs on the watchdog's non-asyncio thread). It registers via `registry.register("kind", Config)(Agent)` in a `register(registry)` hook on a `coder_eval.plugins` entry point with its own `type: Literal["kind"]` config — and it does NOT wire itself in by editing the `AgentKind` enum or `Orchestrator._create_agent` (which delegates to the registry's `create_agent()` factory); registration is via the SPI hook only. + - **Agents** (every `Agent` subclass in `agents/`): `communicate(..., iteration=)` opens one `TurnEmitter` via `self._open_emitter(...)`, writes the whole turn through it, and returns `finalize(...)` / `fail(...)` — a crash or timeout is a `TurnOutcome` with a `crashed=True` record, never a raised `AgentCrashError` / `TurnTimeoutError`; on `CancelledError` it calls `fail(CRASHED, "turn cancelled")` before re-raising. `stop()` calls `self._mark_stopped()`; it does NOT override `get_state()`. Flag any event, `AssistantMessage`, `EventCollector` or `TurnRecord(` built in an adapter. If the agent shells out / holds OS resources, `stop()` / `kill()` / `kill_sync()` are real, and `kill_sync()` is synchronous (no `await` — it runs on the watchdog's non-asyncio thread). It registers via `registry.register("kind", Config)(Agent)` in a `register(registry)` hook on a `coder_eval.plugins` entry point with its own `type: Literal["kind"]` config — and it does NOT wire itself in by editing the `AgentKind` enum or `Orchestrator._create_agent` (which delegates to the registry's `create_agent()` factory); registration is via the SPI hook only. - **Per-agent coverage when a new agent is added:** `Settings.validate_api_keys` has a branch for it (don't let it fall through silently — a recurring gap); it supports the run's backends (Bedrock / Anthropic / Azure-OpenAI) or fails with a clear error; it surfaces per-turn `total_cost_usd` so the `max_usd` budget gate can fire; and the token-bucket reconciliation invariant (Σ buckets across `TurnRecord.messages` == `token_usage`) holds, with a test. Agnostic-core litmus: `grep -ri src/coder_eval/` outside the agent's own package + the registry should be ~zero. - **Criteria** (every file in `criteria/`): carries `@register_criterion`, implements `_check_impl`, exposes `aggregate()`, is a member of the `SuccessCriterion` union, AND is re-exported from `coder_eval.models`. - **Backends / drivers / renderers:** every `ApiBackend` is handled in judge routing + pricing + `validate_api_keys`; every sandbox driver / preservation mode preserves the stale-artifact-clear, synthetic-`task.json`-on-death, and env-scrub contracts; every `reports*.py` renderer covers each `EvaluationResult` field / `FinalStatus`. - Several of these are statically enforceable — when you find a violation whose shape is grep-/AST-detectable (an `Agent` subclass missing `_begin_turn`, a bare `raise AgentCrashError` with no preceding `self.pending_turn =`, an `async def kill_sync`, a `TurnRecord(` built outside `EventCollector`, a criterion missing from the `SuccessCriterion` union), propose it as a `CEnnn` lint rule in the Harness & Lint pass. + Several of these are statically enforceable — when you find a violation whose shape is grep-/AST-detectable (an `Agent` subclass that never calls `_open_emitter`, a `raise AgentCrashError` inside `communicate`, an `async def kill_sync`, a `TurnRecord(` built outside `EventCollector`, a criterion missing from the `SuccessCriterion` union), propose it as a `CEnnn` lint rule in the Harness & Lint pass. Apply these techniques while reading. Findings produced this way go into the same output as ordinary findings, tagged with the appropriate axis and severity. ``` diff --git a/.claude/commands/coder-eval-create-plan.md b/.claude/commands/coder-eval-create-plan.md index 6061b29d..9b994538 100644 --- a/.claude/commands/coder-eval-create-plan.md +++ b/.claude/commands/coder-eval-create-plan.md @@ -47,7 +47,7 @@ Follow these steps: - Does this change touch the evaluation flow? (CLI → ExperimentRunner → run_batch → Orchestrator → Sandbox + Agent + SuccessChecker) - Does this affect the 5-layer config merge? (default.yaml → experiment defaults → task YAML → variant → CLI flags). Each list/dict field must declare its `MergeField` strategy (lint rule CE014). - If adding a new criterion: does it fit `BaseCriterion` / `@register_criterion` / the `SuccessCriterion` discriminated union? Does it need a custom `aggregate()` for suite thresholds? - - If adding a new agent: does it follow the plugin SPI (a `BaseAgentConfig` subclass + `Agent` ABC + a `register(registry)` hook exposed via the `coder_eval.plugins` entry-point group)? Does it use the shared turn lifecycle (`_begin_turn`/`_end_turn_ok`/`_mark_stopped`) and emit the standardized event protocol? + - If adding a new agent: does it follow the plugin SPI (a `BaseAgentConfig` subclass + `Agent` ABC + a `register(registry)` hook exposed via the `coder_eval.plugins` entry-point group)? Does `communicate(..., iteration=)` write the turn through one `TurnEmitter` (`_open_emitter`) and return a `TurnOutcome`, and does `stop()` call `_mark_stopped`? - Does this change the task YAML schema? If so, what happens to existing task files in `tasks/`? - Are there edge cases in sandbox isolation, agent lifecycle, retry/crash recovery, or token accounting? - Does this introduce new dependencies? Prefer what's already in the project (pydantic, typer, rich, anyio, anthropic). diff --git a/.claude/commands/coder-eval-implement-plan.md b/.claude/commands/coder-eval-implement-plan.md index 8ae4f326..21d30095 100644 --- a/.claude/commands/coder-eval-implement-plan.md +++ b/.claude/commands/coder-eval-implement-plan.md @@ -57,10 +57,10 @@ The plan's Master Acceptance Checklist and the **Review Criteria** below are the - **All models import from `coder_eval.models`** — never from submodules (lint-guarded). New models are exported from `models/__init__.py`. - **New criterion → two edits.** The `@register_criterion` checker in `criteria/` **and** the `SuccessCriterion` discriminated union in `models/criteria.py`. Discriminated unions use `Field(discriminator="type")` — a bare `A | B` union silently coerces. -- **New agent → plugin SPI, not enum dispatch.** Register via a `register(registry)` hook exposed through the `coder_eval.plugins` entry-point group; do **not** edit `Orchestrator._create_agent` (it already delegates to the registry's `create_agent()` factory) or the `AgentKind` enum (known built-in kinds only). Use the shared turn lifecycle (`_begin_turn`/`_end_turn_ok`/`_mark_stopped`) and emit the standardized event protocol through `EventCollector`. +- **New agent → plugin SPI, not enum dispatch.** Register via a `register(registry)` hook exposed through the `coder_eval.plugins` entry-point group; do **not** edit `Orchestrator._create_agent` (it already delegates to the registry's `create_agent()` factory) or the `AgentKind` enum (known built-in kinds only). Write the turn through one `TurnEmitter` (`_open_emitter`) and return its `TurnOutcome`; `stop()` calls `_mark_stopped`. - **Ripple completeness.** Adding/removing/renaming a model field, config key, or CLI flag means tracing every reference — task YAMLs in `tasks/`, experiment YAMLs in `experiments/`, `experiments/default.yaml`, `.claude/commands/`, docs, and `models/__init__.py`. - **Config merge.** New list/dict fields declare their `MergeField` strategy (CE014). New `ResolvedTask`/`AgentConfig` fields need coverage across all 5 layers and a matching `-D` override path. -- **Crash/retry hygiene.** On `AgentCrashError` / `TurnTimeoutError`, set the partial `crashed=True` TurnRecord on `pending_turn`, then raise bare; reset `_session_id`, `pending_turn`, watchdog refs, streaming `ContextVar`s, and iteration counters before the next attempt. +- **Crash/retry hygiene.** A failed turn returns an outcome with `record.crashed=True`; cancellation ends the turn with `fail(CRASHED, ...)` before it propagates; no cross-attempt state lives on the agent (reset `_session_id`, watchdog refs and streaming `ContextVar`s before the next attempt). - **`extra="forbid"`** on config models that consume YAML/CLI; **Haiku/Sonnet, never Opus** in tests (cost). ## Reference blocks diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 14ceb223..be3eccbf 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -67,15 +67,31 @@ TurnMonitor); CE070 keeps adapters from counting one again. ## Shared turn lifecycle -Every adapter drives the same skeleton, on the base class: `_begin_turn()` resets the -pending slot and bumps the iteration counter, `_end_turn_ok()` marks the turn clean, and -`_mark_stopped()` closes the agent. Before raising on a mid-turn failure an adapter sets -`pending_turn` to a `crashed=True` `TurnRecord` and raises bare, which is what lets the -orchestrator drain the partial record and un-bump the iteration. - -The record is BUILT before `_end_turn_ok()` on every harness: a failure inside the -reduction is a failed turn, and `_end_turn_ok` would already have cleared the rollback -flag `discard_pending_turn` needs. +`communicate(..., iteration=...)` returns a `TurnOutcome` (Appendix C of the harness +target design). A crash or a timeout is an outcome with a `crashed=True` record, not an +exception, and the CALLER owns the iteration: a retry of the same turn passes the same +number. A side channel on the agent (a parked partial record, plus an iteration counter +rolled back once per failed turn) is cross-attempt state that every harness would have to +set correctly on every failure branch. + +The orchestrator maps the status in one place: the clean statuses (an explicit allowlist) +return the record; `CRASHED` / `TIMEOUT` append the record to the result and then raise +through `TurnOutcome.record_or_raise`, so the retry categorisation (a crash retries, a +timeout does not) is unchanged; anything else raises `RuntimeError`. + +Cancellation cannot return a value. A `CancelledError` from the task watchdog, the +orchestrator's `wait_for` backstop or the task timeout must keep propagating, or +`task_timeout` stops working. So an adapter ends the turn FIRST +(`fail(CRASHED, "turn cancelled")`) and re-raises, and the orchestrator reads the partial +record from a per-attempt `EventCollector` it attaches to the callback chain itself +(`_attempt_collector`). The agent-side and orchestrator-side records are the same events +through the same reducer, so they cannot differ. The attempt clears that collector on +every exit except a cancel, so a task timeout that fires later (during grading, between +retries) never appends a finished attempt twice. + +Until an adapter is ported onto `TurnEmitter` it keeps its old body as +`_communicate_legacy`, and `Agent._legacy_outcome` maps its record or raised exception +to an outcome. Three exit paths converge on `finalize`, and it is idempotent on all of them, because the protocol allows EXACTLY ONE `AgentEndEvent` per `communicate()`: the clean return, the @@ -727,6 +743,22 @@ one-way — the plugin loader and the models layer import the registry, never th `create_agent` deliberately does not import `coder_eval.plugins` itself for the same reason; callers reach a config through `parse_agent_config`, which loads them. +## Why the watchdog cancels a child task + +An adapter that returns a `TIMEOUT` outcome when its OWN watchdog cancels the turn cannot +cancel the turn's own task. Measured on Python 3.13.11 (2026-09-16): a handler that catches +that cancel and returns leaves the task's `cancelling()` at 1, so an enclosing +`asyncio.timeout` later raises `CancelledError` instead of `TimeoutError`, and a cancel +that lands just after the body finished hits caller code. `uncancel()` would fix the count +but can also erase a real task-timeout cancel that arrived in the same iteration. + +`run_with_watchdog` runs the body as a CHILD task and arms the watchdog on the child. The +caller's count stays 0, an external cancel of the caller still propagates (and cancels the +child), and a late cancel lands on a finished child, where it does nothing. The watchdog +is unchanged. Verified with plain asyncio, and live with the Claude SDK (anyio) and the +Codex SDK (a threaded iterator). A `ContextVar` set inside the body is not visible to the +caller afterwards; the only one in `src/` is the logging task id, which the child inherits. + ## The threaded watchdog `asyncio.wait_for` is not enough for these harnesses: the Claude SDK wraps its subprocess diff --git a/.claude/notes/reporting.md b/.claude/notes/reporting.md index 823d940a..88be9aa1 100644 --- a/.claude/notes/reporting.md +++ b/.claude/notes/reporting.md @@ -75,12 +75,10 @@ authoring walkthrough is [docs/EXTENDING.md](../../docs/EXTENDING.md) and the fi numbered lifecycle requirements are in CLAUDE.md § Adding a New Agent; what follows is why the seams are shaped the way they are. -The turn-lifecycle bookkeeping lives on the BASE class as class-level defaults, so a -subclass gets the behaviour without re-declaring it. `_iteration_was_incremented` is set -right after the counter bump at the top of `communicate()` and consumed by -`discard_pending_turn()`, which rolls the counter back exactly once per failed turn — even -when partial-record assembly leaves `pending_turn` at None. That is why rollback is the -caller's move, not the agent's: only the caller knows a turn failed. +`communicate` returns a `TurnOutcome` and takes the `iteration` from its caller, so no +turn bookkeeping lives on the agent between attempts: only the caller knows a turn failed +and whether it retries it. A cancelled turn ends through `fail(CRASHED, "turn cancelled")` +before the cancel propagates. Why: [agents.md](agents.md) § Shared turn lifecycle. Capabilities are declared rather than probed, on the agent's `HarnessContract`. `contract.cooperative_stop` gates arming early-stop, so arming it on an agent that ignores diff --git a/.claude/shared/review-rubric.md b/.claude/shared/review-rubric.md index e6e0ba00..68305c70 100644 --- a/.claude/shared/review-rubric.md +++ b/.claude/shared/review-rubric.md @@ -52,7 +52,7 @@ The coder_eval-specific quality checklist. Check every item: 10. **Layer-merge coverage**: new fields on `ResolvedTask` / `AgentConfig` / `BatchRunConfig` have explicit coverage in `test_experiment_resolver.py` exercising all 5 merge layers (default → exp defaults → task → variant → CLI), and a matching `-D` override path. New list/dict fields declare a `MergeField` strategy (CE014). 11. **Pydantic round-trip integrity**: changes to layered configs or polymorphic `CriterionResult` subclasses preserve `model_fields_set` and the discriminator across `model_dump(exclude_unset=True)` → `model_validate()`. Round-trip tests exist for new variants. 12. **Discriminated unions**: new or modified Pydantic unions use `Annotated[..., Field(discriminator="type")]`. Bare `A | B | C` unions silently coerce to the first variant on a missing or typo'd `type`. -13. **Cross-retry state hygiene**: after `AgentCrashError` / `TurnTimeoutError` / `is_error=True` SDK message, the agent resets `_session_id`, `pending_turn`, watchdog references, streaming-event `ContextVar`s, and iteration counters before the next attempt. Test covers a crashing turn followed by a successful turn in the same `Orchestrator` instance. +13. **Cross-retry state hygiene**: a failed turn returns an outcome with `record.crashed=True`; cancellation ends the turn with `fail(CRASHED, ...)` before it propagates; no cross-attempt state lives on the agent (after a crash or an `is_error=True` SDK message it resets `_session_id`, watchdog references and streaming-event `ContextVar`s). Test covers a crashing turn followed by a successful turn in the same `Orchestrator` instance. 14. **Untrusted text in evaluator prompts**: strings derived from agent output (tool-call args, stdout, file contents, dialog history) injected into a judge / simulator / reviewer prompt are wrapped in a fenced block with explicit untrusted-data framing; the system prompt instructs the model to treat that block as adversarial. 15. **NaN / non-finite guards**: score and threshold clamps via `max(lo, min(hi, x))` are preceded by `math.isfinite(x)`. Bad parses fail explicitly instead of silently returning the upper bound (`max(0.0, min(1.0, nan)) == 1.0`). 16. **Registry over hardcoded dispatch**: new agent / criterion / template / route variants go through the existing registry (`@register_criterion`, the `coder_eval.plugins` SPI, etc.). `if x.type == ...` / `isinstance(...)` ladders in `orchestrator.py`, `simulation/`, or `evaluation/` are rejected. diff --git a/CLAUDE.md b/CLAUDE.md index aebcd9e0..1b8d8280 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -136,9 +136,9 @@ CLI → ExperimentRunner (task × variant, 5-layer merge) → run_batch → Orch Per-task (single iteration; simulation mode runs a multi-turn dialog): 1. Orchestrator._communicate_with_retry(prompt, iteration) → TurnRecord - (wraps agent.communicate with retry, per-attempt turn_timeout, the task's - TurnMonitor as the should_stop poll, and on_attempt_error → preserves - crashed=True partial TurnRecords) + (wraps agent.communicate(..., iteration=) → TurnOutcome with retry, per-attempt + turn_timeout, the task's TurnMonitor as the should_stop poll; a CRASHED/TIMEOUT + outcome's crashed=True record is appended before it is raised) 2. SuccessChecker.check_all_async() → List[CriterionResult] Cleanup: stop agent, save EvaluationResult, generate reports. diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 5ba95a17..efa50300 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -154,7 +154,7 @@ it on every LiteLLM route. Implement these three abstract methods: - [ ] `async def start(self, working_directory, *, env_path_prepend=None, plugin_tools_dir=None, plugin_root: Path | None = None) -> None` -- [ ] `async def communicate(self, user_input, *, stream_callback=None, timeout=None, should_stop: Callable[[], StopReason | None] | None = None) -> TurnRecord` +- [ ] `async def communicate(self, user_input, *, iteration: int, stream_callback=None, timeout=None, should_stop: Callable[[], StopReason | None] | None = None) -> TurnOutcome` - [ ] `async def stop(self) -> None` `plugin_root` is the staged plugin root (`/skills//SKILL.md`), or `None` when @@ -170,28 +170,41 @@ often you report them as `usage_granularity`. With `cooperative_stop=True`: - [ ] Call `should_stop()` at each safe boundary (for example, after each resolved tool call, before you pull the next unit of work). - [ ] When it returns a `StopReason`, stop pulling work and remember the reason. -- [ ] Finalize the turn with `AgentEndStatus` `end_status_for(reason)` (both names - come from `coder_eval.spi`), with `crashed=False`. Do not raise. +- [ ] End the turn with `emitter.finalize(end_status_for(reason))` (both names come + from `coder_eval.spi`). Do not raise. Optional overrides (sensible defaults exist): `kill()`, `kill_sync()` (called from a -non-asyncio watchdog thread — must **not** await), `discard_pending_turn()`. +non-asyncio watchdog thread — must **not** await). + +Write the turn through one `TurnEmitter` (do **not** build events, messages or a +`TurnRecord` yourself): + +- [ ] Open it with `emitter = self._open_emitter(prompt=user_input, iteration=iteration, + model=..., task_id=..., stream_callback=stream_callback)` and call `emitter.begin()`. +- [ ] Report what the harness did: `begin_inner_turn` / `end_inner_turn(tokens=delta)`, + `text`, `open_tool` / `close_tool`, and `add_generation(message_id=..., window=close_window(...), parts=[Generation(...)])`. +- [ ] Return `emitter.finalize(status, ...)` for a clean end, or + `emitter.fail(AgentEndStatus.CRASHED | TIMEOUT, reason)` for a failed one. A crash or + timeout is an outcome, not an exception; an exception out of `communicate` is a bug. +- [ ] On `asyncio.CancelledError`, call `emitter.fail(AgentEndStatus.CRASHED, "turn cancelled")`, + then re-raise: the orchestrator recovers the record from its own collector. +- [ ] Run an SDK turn body under `run_with_watchdog(...)`, and return + `emitter.fail(AgentEndStatus.TIMEOUT, format_timeout_reason(timeout))` on `WatchdogFired`. +- [ ] Call `self._mark_stopped()` in `stop()` after your own teardown. -Follow the shared turn lifecycle (do **not** hand-assemble a `TurnRecord`): +The emitter owns the event protocol: one `AgentStartEvent`, one `AgentEndEvent` on every +exit, balanced inner turns and tool calls (orphans closed `unresolved`), and the record. -- [ ] Call `self._begin_turn()` at the top of `communicate()`. -- [ ] Call `self._end_turn_ok()` on the success path. -- [ ] Call `self._mark_stopped()` in `stop()` after your own teardown. -- [ ] Before raising on a mid-turn failure, set `self.pending_turn` to a - `crashed=True` `TurnRecord` (built from an `EventCollector`), then raise - `AgentCrashError` / `TurnTimeoutError` (bare — no payload). The orchestrator - drains it and calls `discard_pending_turn()`. - -Emit the standardized event protocol (you are the **sole emitter**): one -`AgentStartEvent` at the top of `communicate()` and one matching `AgentEndEvent` on -**every** exit path (emit from `finally`), a `TurnStart`/`TurnEnd` pair per inner -turn, and `ToolStart`/`ToolEnd` per tool call (close orphaned tools with -`status=unresolved`). Fan events through an internal `EventCollector` — it builds the -returned `TurnRecord`, the single agent-agnostic capture path. +### The sixth-harness checklist + +- [ ] A `HarnessContract` (every field, `timing_basis` included). +- [ ] A config class and its registration. +- [ ] A translation from config to the harness's native call that delivers the staged + `plugin_root`. +- [ ] A decoder: one object per turn that takes the harness's events and calls the emitter. +- [ ] The four `coder_eval.testing` sensors in your own tests: `replay` your decoder over a + recorded stream, `assert_identity_closes` on the replay, `assert_stream_balanced` on + its events, and `conformance(kind, probes)` for the contract. ### Worked example diff --git a/docs/agents/CODEX.md b/docs/agents/CODEX.md index e00b8818..7da001cf 100644 --- a/docs/agents/CODEX.md +++ b/docs/agents/CODEX.md @@ -133,14 +133,13 @@ Agent (ABC) ### Key Methods - **`start(working_directory)`** - Initialize Codex client and set working directory -- **`communicate(user_input, timeout, stream_callback)`** - Execute one turn with Codex +- **`communicate(user_input, iteration, timeout, stream_callback)`** - Execute one turn with Codex and return its `TurnOutcome` - **`stop()`** - Clean up resources - **`get_state()`** - Return current agent state -- **`discard_pending_turn()`** - Rollback on failure ### TurnRecord Format -Each turn returns a `TurnRecord` with: +Each turn's outcome carries a `TurnRecord` with: - `iteration` - Turn number - `user_input` - The prompt sent - `agent_output` - assembled from the streamed `agentMessage` deltas @@ -154,14 +153,13 @@ Each turn returns a `TurnRecord` with: ### Timeout Handling -The agent uses a `ThreadedWatchdog` to enforce wall-clock timeouts. If a turn exceeds the deadline, a `TurnTimeoutError` is raised with a partial `TurnRecord` preserved in `pending_turn`. +The agent uses a `ThreadedWatchdog` to enforce wall-clock timeouts. If a turn exceeds the deadline, `communicate` returns a `TIMEOUT` outcome whose record is the `crashed=True` partial turn. ### Error Recovery -On failure, the agent: -1. Sets `pending_turn` to a `crashed=True` TurnRecord with captured telemetry -2. Raises `AgentCrashError` or `TurnTimeoutError` -3. The orchestrator reads `pending_turn` and calls `discard_pending_turn()` to roll back state +On failure, the agent returns a `CRASHED` or `TIMEOUT` outcome whose record is a +`crashed=True` `TurnRecord` with the captured telemetry. The orchestrator appends that +record to the result, then retries a crash and ends the task on a timeout. ### Permission and Tool Mapping diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index e9703f8c..0cfa2935 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -624,8 +624,8 @@ How each harness enforces `run_limits.turn_timeout` (the meaning is the same eve ### What a timeout looks like On Claude Code and Codex a `turn_timeout` breach is a *failure*: the watchdog fires -at the deadline, the partial turn is preserved on `pending_turn`, and the turn is -marked `crashed`. +at the deadline, the agent returns a `TIMEOUT` outcome, and its partial turn is kept as a +`crashed` record. Antigravity stops earlier and more gently, for the reason in the next section. diff --git a/src/coder_eval/agent.py b/src/coder_eval/agent.py index 60bdd494..a1e95bc6 100644 --- a/src/coder_eval/agent.py +++ b/src/coder_eval/agent.py @@ -5,17 +5,20 @@ import logging from abc import ABC, abstractmethod -from collections.abc import Callable +from collections.abc import Awaitable, Callable +from datetime import datetime from pathlib import Path from typing import Any, ClassVar, NoReturn, Protocol from .errors import AgentCrashError, TurnTimeoutError from .errors.agent import format_timeout_reason, truncate_crash_message from .models import AgentState as AgentState -from .models import ApiRoute, BaseAgentConfig, HarnessContract, ToolNameMap, TurnRecord -from .streaming.callbacks import StreamCallback +from .models import ApiRoute, BaseAgentConfig, HarnessContract, TimingBasis, ToolNameMap, TurnRecord +from .streaming.callbacks import CompositeStreamCallback, StreamCallback from .streaming.collector import EventCollector -from .streaming.events import AgentEndStatus, StopReason +from .streaming.emitter import Clock, TurnEmitter, TurnOutcome +from .streaming.events import AgentEndEvent, AgentEndStatus, StopReason, StreamEvent +from .timing import TurnClock logger = logging.getLogger(__name__) @@ -54,19 +57,11 @@ def __init__(self, config: ClaudeCodeAgentConfig, ...): """ pending_turn: TurnRecord | None = None - """Side-channel for partial turn records from failed ``communicate()`` calls. - - Implementations must set this to a ``crashed=True`` TurnRecord before - raising any mid-turn exception that carries captured telemetry. Callers - must read this slot after every failed ``communicate()`` call, then call - ``discard_pending_turn()`` to clear it. Outside ``communicate()``, this - slot is always None. + """A not-yet-ported adapter parks its crashed partial record here before raising; + ``_legacy_outcome`` reads and clears it. Always None once ``communicate()`` returns. """ - # Class-level defaults so a subclass gets the behaviour without re-declaring it. - # `_iteration_was_incremented` is consumed by `discard_pending_turn()`, which - # rolls the counter back exactly once per failed turn. - # Rationale: .claude/notes/reporting.md § The Agent ABC contract + # Class-level defaults for the not-yet-ported adapters' turn bookkeeping. _state: AgentState = AgentState.WORKING _iteration: int = 0 _iteration_was_incremented: bool = False @@ -216,51 +211,101 @@ async def communicate( self, user_input: str, *, + iteration: int, stream_callback: StreamCallback | None = None, timeout: float | None = None, should_stop: Callable[[], StopReason | None] | None = None, - ) -> TurnRecord: - """Send a message to the agent and receive its response. + ) -> TurnOutcome: + """Run one turn and return its outcome; a crash or timeout is an outcome, not an exception. Args: user_input: The message/prompt to send to the agent + iteration: The caller's turn number, stamped on the record; a retry of + the same turn passes the same number. stream_callback: Optional callback for real-time event streaming - timeout: Hard wall-clock deadline in seconds. When exceeded the - agent must force-terminate any in-flight subprocess and raise - TurnTimeoutError. Do not rely solely on asyncio cancellation -- - some SDKs swallow it. + timeout: Hard wall-clock deadline in seconds. When exceeded the agent + force-terminates any in-flight subprocess and returns a ``TIMEOUT`` + outcome. Do not rely solely on asyncio cancellation -- some SDKs + swallow it. should_stop: The run's single stop poll. An implementation with ``contract.cooperative_stop`` calls it at each safe boundary; a - non-None reason means stop pulling work, remember the reason, and - finalize with ``end_status_for(reason)`` (``crashed=False``, no - raise). Agents that do not support it accept and ignore it. + non-None reason means stop pulling work and finalize with + ``end_status_for(reason)``. Agents that do not support it ignore it. Returns: - TurnRecord containing the complete interaction + The ``TurnOutcome`` from the turn's ``TurnEmitter``: ``finalize(...)`` for a + clean status, ``fail(...)`` for ``CRASHED`` / ``TIMEOUT`` (its record is + ``crashed=True``). Raises: - RuntimeError: If agent is not started or communication fails. - TurnTimeoutError: Timeout elapsed; implementations must set - ``self.pending_turn`` to a ``crashed=True`` partial TurnRecord - before raising if telemetry was captured. - AgentCrashError: Agent failed mid-turn; same ``pending_turn`` contract. - - On success ``pending_turn`` must be None. On failure it holds the partial - record, and only ``discard_pending_turn`` — which the caller invokes after - every failed call — rolls back per-turn bookkeeping. - - The agent is the SOLE emitter of the event protocol. Emit exactly one - ``AgentStartEvent`` at entry and one matching ``AgentEndEvent`` from - ``finally`` on every exit path, one ``TurnStartEvent`` / ``TurnEndEvent`` - pair per inner turn, and a ``ToolStartEvent`` closed by a ``ToolEndEvent`` - for every tool call (``status=unresolved`` when a crash orphans one). Fan - every event through an internal ``EventCollector``, which builds the - returned ``TurnRecord``, and through the caller's ``stream_callback``. + asyncio.CancelledError: the turn was cancelled from outside. The agent + ends the turn first with ``fail(CRASHED, "turn cancelled")``, then + re-raises. Any other exception is a harness bug. + + Open one ``TurnEmitter`` per turn with ``_open_emitter``; it is the sole writer + of the event protocol. Rationale: .claude/notes/agents.md § Shared turn lifecycle """ pass + def _open_emitter( + self, + *, + prompt: str, + iteration: int, + model: str | None, + task_id: str, + stream_callback: StreamCallback | None, + ) -> TurnEmitter: + """The turn's emitter, on a fresh ``TurnClock`` or the wall clock per ``contract.timing_basis``.""" + clock: Clock = TurnClock() if self.contract.timing_basis is TimingBasis.TURN_CLOCK else datetime + return TurnEmitter( + task_id=task_id, + iteration=iteration, + prompt=prompt, + model=model, + basis=self.contract.timing_basis, + clock=clock, + sinks=[stream_callback] if stream_callback is not None else [], + ) + + async def _legacy_outcome( + self, + body: Callable[..., Awaitable[TurnRecord]], + user_input: str, + *, + iteration: int, + stream_callback: StreamCallback | None, + timeout: float | None, + should_stop: Callable[[], StopReason | None] | None, + ) -> TurnOutcome: + """Adapt a not-yet-ported raise-and-park ``communicate`` body to the outcome contract. + + ``CancelledError`` propagates untouched: the body already finalized the turn. + """ + self._iteration = iteration - 1 + last = _LastEndStatus() + callback = CompositeStreamCallback([last, stream_callback] if stream_callback is not None else [last]) + try: + record = await body(user_input, stream_callback=callback, timeout=timeout, should_stop=should_stop) + except (AgentCrashError, TurnTimeoutError) as err: + fallback = AgentEndStatus.TIMEOUT if isinstance(err, TurnTimeoutError) else AgentEndStatus.CRASHED + partial = self.pending_turn or TurnRecord( + iteration=iteration, + user_input=user_input, + agent_output="", + crashed=True, + crash_reason=truncate_crash_message(str(err)), + ) + self.pending_turn = None + self._iteration_was_incremented = False + failed = fallback + if last.status is AgentEndStatus.CRASHED or last.status is AgentEndStatus.TIMEOUT: + failed = last.status + return TurnOutcome(record=partial, status=failed, error=str(err)) + return TurnOutcome(record=record, status=last.status or AgentEndStatus.COMPLETED, error=None) + @abstractmethod async def stop(self) -> None: """Stop the agent and clean up resources.""" @@ -341,3 +386,14 @@ def get_environment_info(self) -> dict[str, Any]: "system_prompt_semantics": self.contract.system_prompt_semantics or "unknown", "harness_contract": self.contract.model_dump(mode="json"), } + + +class _LastEndStatus: + """Remembers the last ``AgentEndEvent.status`` a legacy turn body emitted.""" + + def __init__(self) -> None: + self.status: AgentEndStatus | None = None + + def on_event(self, event: StreamEvent) -> None: + if isinstance(event, AgentEndEvent): + self.status = event.status diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 576006c0..b2624d69 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -59,6 +59,7 @@ from coder_eval.pricing import price_turn from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.emitter import TurnOutcome from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -413,6 +414,25 @@ async def _drain( await asyncio.sleep(0) async def communicate( + self, + user_input: str, + *, + iteration: int, + stream_callback: StreamCallback | None = None, + timeout: float | None = None, + should_stop: Callable[[], StopReason | None] | None = None, + ) -> TurnOutcome: + """Run one turn; see ``Agent.communicate``.""" + return await self._legacy_outcome( + self._communicate_legacy, + user_input, + iteration=iteration, + stream_callback=stream_callback, + timeout=timeout, + should_stop=should_stop, + ) + + async def _communicate_legacy( self, user_input: str, *, diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index d5f3e720..0edece27 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -71,6 +71,7 @@ from coder_eval.pricing import price_turn from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.emitter import TurnOutcome from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -936,6 +937,25 @@ def _resolve_effective_model( return config_model or route_model async def communicate( + self, + user_input: str, + *, + iteration: int, + stream_callback: StreamCallback | None = None, + timeout: float | None = None, + should_stop: Callable[[], StopReason | None] | None = None, + ) -> TurnOutcome: + """Run one turn; see ``Agent.communicate``.""" + return await self._legacy_outcome( + self._communicate_legacy, + user_input, + iteration=iteration, + stream_callback=stream_callback, + timeout=timeout, + should_stop=should_stop, + ) + + async def _communicate_legacy( self, user_input: str, *, diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 952a834e..1e7d8c19 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -45,6 +45,7 @@ from coder_eval.pricing import price_turn from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.emitter import TurnOutcome from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -846,6 +847,25 @@ async def start( raise RuntimeError(f"Failed to initialize Codex client: {e}") from e async def communicate( + self, + user_input: str, + *, + iteration: int, + stream_callback: StreamCallback | None = None, + timeout: float | None = None, + should_stop: Callable[[], StopReason | None] | None = None, + ) -> TurnOutcome: + """Run one turn; see ``Agent.communicate``.""" + return await self._legacy_outcome( + self._communicate_legacy, + user_input, + iteration=iteration, + stream_callback=stream_callback, + timeout=timeout, + should_stop=should_stop, + ) + + async def _communicate_legacy( self, user_input: str, *, diff --git a/src/coder_eval/agents/noop_agent.py b/src/coder_eval/agents/noop_agent.py index 88bf05f7..baa8821f 100644 --- a/src/coder_eval/agents/noop_agent.py +++ b/src/coder_eval/agents/noop_agent.py @@ -4,8 +4,8 @@ for system / canary checks that reuse the eval infrastructure (sandbox, ``pre_run``, reports, evalboard, ADX) without running a coding agent. Its ``start`` / ``communicate`` / ``stop`` are no-ops and it makes no model API -call; ``communicate`` emits the standardized event protocol for a single empty -turn and returns the ``EventCollector``'s reduction (an empty +call; ``communicate`` writes a single empty turn through its ``TurnEmitter`` and +returns its outcome (an empty :class:`~coder_eval.models.results.TurnRecord`), so the orchestrator's normal lifecycle runs unmodified and then checks the success criteria directly against the sandbox. @@ -27,20 +27,11 @@ HarnessContract, NoneAgentConfig, TimingBasis, - TurnRecord, UsageGranularity, ) -from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback -from coder_eval.streaming.collector import EventCollector -from coder_eval.streaming.events import ( - AgentEndEvent, - AgentEndStatus, - AgentStartEvent, - StopReason, - TurnEndEvent, - TurnEndStatus, - TurnStartEvent, -) +from coder_eval.streaming.callbacks import StreamCallback +from coder_eval.streaming.emitter import TurnOutcome +from coder_eval.streaming.events import AgentEndStatus, StopReason @AgentRegistry.register(AgentKind.NONE, NoneAgentConfig) @@ -49,10 +40,9 @@ class NoOpAgent(Agent[NoneAgentConfig]): Created and driven by the orchestrator exactly like any other agent, so no ``agentless`` branching is needed: the single signal is ``agent.type == - AgentKind.NONE``. ``communicate`` is the SOLE emitter of one clean, balanced - event tree (``AgentStart`` -> ``TurnStart`` -> ``TurnEnd`` -> ``AgentEnd``, - all ``COMPLETED``) and returns the empty turn the ``EventCollector`` reduces - from it. + AgentKind.NONE``. ``communicate`` writes one clean, balanced event tree + (``AgentStart`` -> ``TurnStart`` -> ``TurnEnd`` -> ``AgentEnd``, all + ``COMPLETED``) and returns its outcome. """ contract = HarnessContract( @@ -86,43 +76,27 @@ async def communicate( self, user_input: str, *, + iteration: int, stream_callback: StreamCallback | None = None, timeout: float | None = None, should_stop: Callable[[], StopReason | None] | None = None, - ) -> TurnRecord: - """Return an empty turn without contacting any model. - - ``should_stop`` is accepted for ``Agent.communicate`` override - compatibility and ignored — a no-op turn has nothing to interrupt. + ) -> TurnOutcome: + """Return one empty, completed turn without contacting any model. - Honors the streaming contract — sole emitter of a balanced event tree - (``AgentStart`` -> ``TurnStart`` -> ``TurnEnd`` -> ``AgentEnd``) — so the - task-log handler and renderers see a clean turn boundary. The returned - ``TurnRecord`` is the ``EventCollector``'s reduction of those events. + ``timeout`` and ``should_stop`` are accepted and ignored: a no-op turn has + nothing to interrupt. """ - self._begin_turn() - - task_id = str(self.config.type) # str() so a plugin subclass with a non-enum kind also works - turn_id = f"none-{self._iteration}" - collector = EventCollector() - emit = CompositeStreamCallback([c for c in (collector, stream_callback) if c is not None]) - - emit.on_event(AgentStartEvent(task_id=task_id, prompt=user_input, iteration=self._iteration)) - emit.on_event(TurnStartEvent(task_id=task_id, turn_id=turn_id)) - emit.on_event(TurnEndEvent(task_id=task_id, turn_id=turn_id, status=TurnEndStatus.COMPLETED)) - emit.on_event( - AgentEndEvent( - task_id=task_id, - status=AgentEndStatus.COMPLETED, - iteration=self._iteration, - user_input=user_input, - agent_output="", - assistant_turn_count=0, - ) + emitter = self._open_emitter( + prompt=user_input, + iteration=iteration, + model=None, + task_id=str(self.config.type), # str() so a plugin subclass with a non-enum kind also works + stream_callback=stream_callback, ) - - self._end_turn_ok() - return collector.build_turn_record() + emitter.begin() + emitter.begin_inner_turn(f"none-{iteration}") + emitter.end_inner_turn() + return emitter.finalize(AgentEndStatus.COMPLETED, assistant_turn_count=0, num_turns=None, result_summary=None) async def stop(self) -> None: """No-op: nothing to tear down.""" diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index 8fdf8f3d..ab35dc59 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -57,6 +57,7 @@ from coder_eval.pricing import price_turn from coder_eval.streaming.callbacks import StreamCallback, safe_emit from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.emitter import TurnOutcome from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -968,6 +969,25 @@ def _inject_config_content(self, env: dict[str, str]) -> None: # --- the turn ---------------------------------------------------------- async def communicate( + self, + user_input: str, + *, + iteration: int, + stream_callback: StreamCallback | None = None, + timeout: float | None = None, + should_stop: Callable[[], StopReason | None] | None = None, + ) -> TurnOutcome: + """Run one turn; see ``Agent.communicate``.""" + return await self._legacy_outcome( + self._communicate_legacy, + user_input, + iteration=iteration, + stream_callback=stream_callback, + timeout=timeout, + should_stop=should_stop, + ) + + async def _communicate_legacy( self, user_input: str, *, diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index d681a652..80ecb2e6 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -65,6 +65,7 @@ from coder_eval.pricing import price_turn from coder_eval.streaming.callbacks import StreamCallback, safe_emit from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.emitter import TurnOutcome from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -848,6 +849,25 @@ def _build_env(self) -> dict[str, str]: # --- the turn ---------------------------------------------------------- async def communicate( + self, + user_input: str, + *, + iteration: int, + stream_callback: StreamCallback | None = None, + timeout: float | None = None, + should_stop: Callable[[], StopReason | None] | None = None, + ) -> TurnOutcome: + """Run one turn; see ``Agent.communicate``.""" + return await self._legacy_outcome( + self._communicate_legacy, + user_input, + iteration=iteration, + stream_callback=stream_callback, + timeout=timeout, + should_stop=should_stop, + ) + + async def _communicate_legacy( self, user_input: str, *, diff --git a/src/coder_eval/agents/watchdog.py b/src/coder_eval/agents/watchdog.py index 7e3c1038..012eb1a1 100644 --- a/src/coder_eval/agents/watchdog.py +++ b/src/coder_eval/agents/watchdog.py @@ -13,8 +13,8 @@ import contextlib import logging import threading -from collections.abc import Callable -from typing import Self +from collections.abc import Callable, Coroutine +from typing import Any, Self logger = logging.getLogger(__name__) @@ -45,7 +45,7 @@ def __init__( *, timeout_seconds: float | None, on_timeout: Callable[[], None], - asyncio_task_to_cancel: asyncio.Task[object] | None = None, + asyncio_task_to_cancel: asyncio.Task[Any] | None = None, label: str = "watchdog", ) -> None: self._timeout = timeout_seconds @@ -95,3 +95,43 @@ def __exit__(self, exc_type: object, exc: object, tb: object) -> None: if self._timer is not None: self._timer.cancel() self._timer = None + + +class WatchdogFired(Exception): # noqa: N818 - a signal, not an error: the plan-named SPI export + """The watchdog cancelled the guarded body at its deadline.""" + + +async def run_with_watchdog[T]( + body: Coroutine[Any, Any, T], + *, + timeout_seconds: float | None, + on_timeout: Callable[[], None], + label: str, +) -> T: + """Run ``body`` as a child task that a ``ThreadedWatchdog`` cancels at ``timeout_seconds``. + + Returns the body's value; any exception the body raises propagates unchanged. + + Raises: + WatchdogFired: the watchdog fired and cancelled the body while the caller + itself was not being cancelled. The caller's cancel count is untouched. + asyncio.CancelledError: the caller was cancelled; the body is cancelled too. + + Rationale: .claude/notes/agents.md § Why the watchdog cancels a child task + """ + child = asyncio.create_task(body) + watchdog = ThreadedWatchdog( + timeout_seconds=timeout_seconds, on_timeout=on_timeout, asyncio_task_to_cancel=child, label=label + ) + try: + with watchdog: + return await child + except asyncio.CancelledError: + caller = asyncio.current_task() + if watchdog.fired and child.cancelled() and (caller is None or caller.cancelling() == 0): + raise WatchdogFired(label) from None + child.cancel() + raise + except BaseException: + child.cancel() + raise diff --git a/src/coder_eval/errors/executor.py b/src/coder_eval/errors/executor.py index 80f3cce9..d0db6312 100644 --- a/src/coder_eval/errors/executor.py +++ b/src/coder_eval/errors/executor.py @@ -18,7 +18,6 @@ async def execute_with_retry( operation_name: str, context: dict[str, Any], max_attempts: int | None = None, - on_attempt_error: Callable[[Exception, int], Awaitable[None]] | None = None, ) -> Any: """Execute an operation with automatic retry on transient errors. @@ -32,11 +31,6 @@ async def execute_with_retry( operation_name: Human-readable name, for logging only. context: Requires ``task_id``; ``component`` and ``agent_name`` are optional. max_attempts: Overrides the safety limit of 10. - on_attempt_error: Async ``(exception, zero_indexed_attempt) -> None`` invoked - after every failed attempt, including the final non-retryable one, and - before the backoff. Its own exceptions are logged and swallowed so they - cannot mask the original. The orchestrator uses it to drain - ``agent.pending_turn`` and call ``agent.discard_pending_turn()``. Returns: Whatever ``operation`` returned. @@ -46,7 +40,7 @@ async def execute_with_retry( Example: >>> async def flaky_api_call(): - ... return await agent.communicate(prompt) + ... return (await agent.communicate(prompt, iteration=1)).record_or_raise() >>> >>> result = await execute_with_retry( ... operation=flaky_api_call, @@ -73,19 +67,6 @@ async def execute_with_retry( except Exception as e: last_error = e - # Fire callback before the retry decision so partial telemetry - # is captured even on the final non-retryable attempt. - if on_attempt_error is not None: - try: - await on_attempt_error(e, attempt) - except Exception: - logger.exception( - "[%s] on_attempt_error callback raised for %s (attempt %d); ignoring", - task_id, - operation_name, - attempt + 1, - ) - # Categorize error category = categorize_error(e, context) config = RETRY_CONFIG.get(category, RetryConfig()) diff --git a/src/coder_eval/evaluation/sub_agent.py b/src/coder_eval/evaluation/sub_agent.py index e1940a60..5433887c 100644 --- a/src/coder_eval/evaluation/sub_agent.py +++ b/src/coder_eval/evaluation/sub_agent.py @@ -213,7 +213,8 @@ async def _run_agent( """ try: await agent.start(str(judge_dir), plugin_tools_dir=plugin_tools_dir) - return await agent.communicate(user_msg, timeout=turn_timeout) + outcome = await agent.communicate(user_msg, iteration=1, timeout=turn_timeout) + return outcome.record_or_raise(timeout_seconds=turn_timeout) except BaseException: with contextlib.suppress(Exception): await agent.kill() diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 6506289a..d683dad8 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -84,7 +84,8 @@ from .sandbox import Sandbox from .simulation import DialogStopReason, SimulatorResult, UserSimulator, evaluate_stop from .streaming.callbacks import CompositeStreamCallback, StreamCallback, TaskScopedCallback, safe_emit -from .streaming.events import CriteriaCheckEvent, CriterionSummary +from .streaming.collector import EventCollector +from .streaming.events import AgentEndStatus, CriteriaCheckEvent, CriterionSummary from .telemetry import Scalar, hash_identifier from .utils import get_version_info, looks_like_version, runtime_uip_versions @@ -97,6 +98,18 @@ # wins the race against the asyncio cancel path (which doesn't). _WAIT_FOR_GRACE_SECONDS = 2.0 +# The clean end statuses a turn's outcome is returned for; CRASHED and TIMEOUT raise. +# An allowlist, so a new AgentEndStatus member fails loudly until it is placed. +_RETURNED_END_STATUSES = frozenset( + { + AgentEndStatus.COMPLETED, + AgentEndStatus.STOPPED_EARLY, + AgentEndStatus.TOOL_CALLS_EXHAUSTED, + AgentEndStatus.TOKEN_BUDGET_EXCEEDED, + AgentEndStatus.COST_BUDGET_EXCEEDED, + } +) + def _close_subprocess_transport(proc: asyncio.subprocess.Process | None) -> None: """Release a finished subprocess's pipe transport deterministically. @@ -478,6 +491,11 @@ def __init__( # count it answers the should_stop poll from is cumulative per task. self._monitor: TurnMonitor | None = None + # The in-flight communicate attempt's own collector, so a task timeout can + # recover the turn the agent ended before the cancel propagated. Cleared on + # every other exit, so a finished attempt is never appended twice. + self._attempt_collector: EventCollector | None = None + # The skill names the staged plugin root offered; None when the task sets # no plugins. Read back from the prior result on an evaluate-only grade. self._skills_offered: tuple[str, ...] | None = None @@ -648,8 +666,7 @@ def _kill_agent_subprocess_sync() -> None: logger.error(f"Task timed out: {e}") # Nothing else on this path recovers the in-flight turn: the - # cancel arrives as a BaseException, so it never reaches the retry - # executor's per-attempt hook. + # cancel arrives as a BaseException, so no outcome ever returns. await self._drain_killed_turn() except BudgetExceededError as e: # Map token-budget breaches and cost-budget breaches to distinct @@ -931,27 +948,20 @@ async def _evaluate_post_failure_criteria(self) -> None: self.result.post_failure_criteria_results = recovered async def _drain_killed_turn(self) -> None: - """Move a hard-killed turn's partial record from the agent onto the result. - - The only reader of ``pending_turn`` on the task-timeout path. Ordering - matters both ways: it must run before ``_cleanup`` (whose ``agent.stop()`` - clears the slot) and before ``_finalize_result``, so the recovered turn - feeds token aggregation and command stats like any other. + """Move a hard-killed turn's record from the in-flight attempt's collector onto the result. - Best-effort: a task killed before its first turn has nothing parked, and - this runs on the way to a saved row, so it must not raise. + Runs before ``_cleanup`` and ``_finalize_result``, so the recovered turn feeds + token aggregation and command stats like any other. Best-effort: a task + killed before its turn ended has nothing to recover, and this must not raise. """ - if self.agent is None or self.result is None: + if self.result is None: return try: - partial = self.agent.pending_turn - # `pending_turn` is a slot any agent implementation fills, so a non-record - # here would fail validation during teardown and take the row down with it. - if not isinstance(partial, TurnRecord): + collector = self._attempt_collector + partial = self._append_attempt_record(collector) if collector is not None else None + if partial is None: logger.debug("[%s] Hard-killed task preserved no partial turn", self.task.task_id) return - self.result.iterations.append(partial) - await self.agent.discard_pending_turn() usage = partial.token_usage logger.info( "[%s] Recovered the hard-killed turn: %d tokens, %s", @@ -964,6 +974,17 @@ async def _drain_killed_turn(self) -> None: except Exception: logger.warning("[%s] Could not recover the hard-killed turn", self.task.task_id, exc_info=True) + def _append_attempt_record(self, collector: EventCollector) -> TurnRecord | None: + """Append the attempt's record when its turn ended, once; the collector is then spent.""" + assert self.result is not None + self._attempt_collector = None + if not collector.ended: + logger.debug("[%s] The killed attempt never ended its turn; nothing appended", self.task.task_id) + return None + record = collector.build_turn_record() + self.result.iterations.append(record) + return record + def _finalize_weighted_score(self) -> None: """Write ``weighted_score``, or ``None`` when this run was not graded. @@ -1812,13 +1833,12 @@ async def _communicate_with_retry( ) -> TurnRecord: """Run ``agent.communicate`` with retry, partial-preservation, and a per-attempt timeout. - Shared by the criteria-feedback and simulation loops. Crashed partials - from ``AgentCrashError`` / ``TurnTimeoutError`` are appended to - ``self.result.iterations`` via the ``on_attempt_error`` hook (terminal - failures included) so observational criteria still see them. Each - attempt gets a fresh ``turn_timeout``; ``TurnTimeoutError`` is - ``AGENT_TIMEOUT`` (``max_retries=0``) so it still terminates after - one attempt. + Shared by the criteria-feedback and simulation loops. A ``CRASHED`` or + ``TIMEOUT`` outcome's record is appended to ``self.result.iterations`` + before it is raised as ``AgentCrashError`` / ``TurnTimeoutError`` (terminal + failures included), so observational criteria still see it. Each attempt + gets a fresh ``turn_timeout``; ``TurnTimeoutError`` is ``AGENT_TIMEOUT`` + (``max_retries=0``) so it still terminates after one attempt. """ assert self.agent is not None assert self.task.agent is not None @@ -1832,82 +1852,59 @@ async def _communicate_with_retry( monitor = self._monitor assert monitor is not None, "TurnMonitor not built" - # The sole callback when --stream is off, else alongside the - # TaskScopedCallback. The same instance persists across retry attempts and - # dialog turns, so its counters and wall-clock origin accumulate. - agent_callback: StreamCallback = monitor - if self.stream_callback is not None: - agent_callback = CompositeStreamCallback( - [monitor, TaskScopedCallback(self.stream_callback, self._log_task_id)] - ) - - def _drain_pending_turn(*, attempt: int) -> None: - """Read agent.pending_turn and, if set, append it to result.iterations.""" - partial = agent.pending_turn - if partial is not None: - result.iterations.append(partial) - logger.debug( - "[%s] Drained partial turn record (attempt %d, iteration %d): %d commands", - self.task.task_id, - attempt + 1, - iteration, - len(partial.commands), - ) - else: - logger.debug( - "[%s] No pending_turn to drain on attempt %d (iteration %d)", - self.task.task_id, - attempt + 1, - iteration, - ) - - async def _on_attempt_failure( - err: Exception, - attempt: int, - ) -> None: - if not isinstance(err, (AgentCrashError, TurnTimeoutError)): - return - _drain_pending_turn(attempt=attempt) - try: - await agent.discard_pending_turn() - except Exception: - logger.warning( - "[%s] discard_pending_turn raised on attempt %d", - self.task.task_id, - attempt + 1, - exc_info=True, - ) + unhandled: list[AgentEndStatus] = [] async def _communicate_attempt() -> TurnRecord: - coro = agent.communicate( - prompt, - stream_callback=agent_callback, - timeout=turn_timeout, - should_stop=monitor.should_stop, - ) - if turn_timeout is None: - return await coro - # Grace buffer: agent's in-band watchdog (sets pending_turn) must beat - # wait_for cancel so the slot is populated before we give up. - outer_timeout = turn_timeout + _WAIT_FOR_GRACE_SECONDS + # A fresh collector per attempt is how a cancelled turn is recovered: + # the agent ends the turn before the cancel propagates, and + # `_drain_killed_turn` reads the record from here. + attempt_collector = EventCollector() + self._attempt_collector = attempt_collector + callbacks: list[StreamCallback] = [monitor, attempt_collector] + if self.stream_callback is not None: + callbacks.append(TaskScopedCallback(self.stream_callback, self._log_task_id)) + cancelled = False try: - return await asyncio.wait_for(coro, timeout=outer_timeout) - except TimeoutError: - # Watchdog wedged or too slow. Kill only — drain + discard happen - # in _on_attempt_failure when this TurnTimeoutError propagates up. - try: - await agent.kill() - except Exception: - logger.warning( - "[%s] agent.kill() raised on wait_for backstop path", - self.task.task_id, - exc_info=True, - ) - raise TurnTimeoutError( - turn_timeout, - task_id=self.task.task_id, + coro = agent.communicate( + prompt, iteration=iteration, - ) from None + stream_callback=CompositeStreamCallback(callbacks), + timeout=turn_timeout, + should_stop=monitor.should_stop, + ) + if turn_timeout is None: + outcome = await coro + else: + try: + # Grace buffer: the agent's own watchdog must beat this backstop. + outcome = await asyncio.wait_for(coro, timeout=turn_timeout + _WAIT_FOR_GRACE_SECONDS) + except TimeoutError: + try: + await agent.kill() + except Exception: + logger.warning( + "[%s] agent.kill() raised on wait_for backstop path", + self.task.task_id, + exc_info=True, + ) + self._append_attempt_record(attempt_collector) + raise TurnTimeoutError(turn_timeout, task_id=self.task.task_id, iteration=iteration) from None + if outcome.status in _RETURNED_END_STATUSES: + return outcome.record + if outcome.status in (AgentEndStatus.CRASHED, AgentEndStatus.TIMEOUT): + result.iterations.append(outcome.record) + return outcome.record_or_raise( + timeout_seconds=turn_timeout, task_id=self.task.task_id, iteration=iteration + ) + # Raised after the retry executor, which would retry a RuntimeError. + unhandled.append(outcome.status) + return outcome.record + except asyncio.CancelledError: + cancelled = True + raise + finally: + if not cancelled: + self._attempt_collector = None # ANTI-CHEAT WINDOW. Both the reference and the task dir sit at mode 000 # for the whole of every communicate attempt — retries included, since this @@ -1927,9 +1924,10 @@ async def _communicate_attempt() -> TurnRecord: "component": "agent", "agent_name": self._agent_name, }, - on_attempt_error=_on_attempt_failure, ) assert turn_record is not None # execute_with_retry returns the turn or raises + if unhandled: + raise RuntimeError(f"unhandled end status {unhandled[0]}") return turn_record @staticmethod @@ -2468,9 +2466,8 @@ async def _simulation_dialog_loop(self, initial_prompt: str | None, sandbox_dir: # Keyed by (position, criterion_type) — a stable criterion identity. judge_usage_accum: dict[tuple[int, str], TokenUsage] = {} - # In lockstep with the agent's _iteration — one - # _communicate_with_retry per sim turn — so a partial turn and its - # successful retry share an iteration number. + # One _communicate_with_retry per sim turn, passed this turn's number, + # so a partial turn and its successful retry share an iteration number. while True: turns_completed += 1 self.result.iteration_count = turns_completed diff --git a/src/coder_eval/simulation/user_simulator.py b/src/coder_eval/simulation/user_simulator.py index c119c759..9a42b5cc 100644 --- a/src/coder_eval/simulation/user_simulator.py +++ b/src/coder_eval/simulation/user_simulator.py @@ -186,6 +186,7 @@ def __init__( self._route: ApiRoute | None = route self._agent_override: Agent[Any] | None = agent_override self._agent: Agent[Any] | None = None + self._turns = 0 self._scratch_dir: Path | None = None # PINNED from config, not inherited from the route: leaving it None let @@ -332,7 +333,8 @@ async def next_user_message(self, dialog_pairs: list[tuple[str, str]]) -> Simula assert self._agent is not None, "UserSimulator.start() must be called before next_user_message()" prompt = dialog_pairs[-1][1] if dialog_pairs else _OPENER_NUDGE - turn = await self._agent.communicate(prompt) + self._turns += 1 + turn = (await self._agent.communicate(prompt, iteration=self._turns)).record_or_raise() raw = turn.agent_output or "" usage = turn.token_usage input_tokens = usage.uncached_input_tokens if usage is not None else None diff --git a/src/coder_eval/spi.py b/src/coder_eval/spi.py index 33e7e058..c18a57e9 100644 --- a/src/coder_eval/spi.py +++ b/src/coder_eval/spi.py @@ -9,6 +9,7 @@ from coder_eval.agent import Agent from coder_eval.agents.registry import AgentRegistry +from coder_eval.agents.watchdog import WatchdogFired, run_with_watchdog from coder_eval.errors import AgentConfigError, AgentCrashError, TurnTimeoutError from coder_eval.models import ( CANONICAL_TOOL_NAMES, @@ -97,8 +98,10 @@ "TurnStartEvent", "TurnTimeoutError", "UsageGranularity", + "WatchdogFired", "Window", "close_window", "end_status_for", "register_pricing", + "run_with_watchdog", ] diff --git a/tests/_fixtures/golden_streams/__init__.py b/tests/_fixtures/golden_streams/__init__.py index dbec503a..d7363b77 100644 --- a/tests/_fixtures/golden_streams/__init__.py +++ b/tests/_fixtures/golden_streams/__init__.py @@ -3,7 +3,7 @@ A safety net for the ``ClaudeCodeAgent.communicate`` / ``CodexAgent`` turn-loop decomposition: each scenario replays a recorded SDK event stream through ``communicate()`` and snapshots the resulting ``TurnRecord`` (or, on a -crash/timeout, the ``pending_turn`` partial) as canonical JSON. The decomposition +crash/timeout, the outcome's crashed partial) as canonical JSON. The decomposition must keep these snapshots byte-identical post-scrub. The scrubber masks only the inherently per-run fields (timestamps, durations, diff --git a/tests/_fixtures/golden_streams/antigravity_fixtures.py b/tests/_fixtures/golden_streams/antigravity_fixtures.py index 1cc8f7a3..eed13afb 100644 --- a/tests/_fixtures/golden_streams/antigravity_fixtures.py +++ b/tests/_fixtures/golden_streams/antigravity_fixtures.py @@ -149,7 +149,7 @@ async def run_antigravity_scenario( # up to 120 cycles, i.e. ten minutes of wall clock in a unit test. The # loop's LOGIC is what the scenario records; the waiting is not. with patch.object(antigravity_agent.asyncio, "sleep", _no_sleep): - record = await agent.communicate("do it", stream_callback=recorder) + record = (await agent.communicate("do it", iteration=1, stream_callback=recorder)).record return record.model_dump(mode="json"), recorder.events diff --git a/tests/_fixtures/golden_streams/claude_fixtures.py b/tests/_fixtures/golden_streams/claude_fixtures.py index 9e5b89d8..d1ed88bd 100644 --- a/tests/_fixtures/golden_streams/claude_fixtures.py +++ b/tests/_fixtures/golden_streams/claude_fixtures.py @@ -20,6 +20,7 @@ import coder_eval.agents.claude_code_agent as claude_module from coder_eval.models import AgentKind, parse_agent_config from coder_eval.streaming import events as protocol +from coder_eval.streaming.events import AgentEndStatus from tests._fixtures.golden_streams._recorder import EventRecorder @@ -180,7 +181,7 @@ class ClaudeScenario: name: str build_query: Callable[[], Callable[..., Any]] timeout: float | None = None - expects: type[BaseException] | None = None + expects: AgentEndStatus | None = None # When set, ``ClaudeCodeAgent._timed_out`` is patched to this constant so the # timeout-vs-crash classification is deterministic without real wall-clock. timed_out: bool | None = None @@ -364,7 +365,7 @@ def _scenario_g() -> ClaudeScenario: return ClaudeScenario( name="g_crash_format_placeholder", build_query=lambda: _raising_query(events, RuntimeError("crash after poison")), - expects=None, # set below to AgentCrashError + expects=None, # set below to CRASHED ) @@ -379,7 +380,7 @@ def _scenario_h1() -> ClaudeScenario: def _scenario_h2() -> ClaudeScenario: - """Non-timeout ProcessError -> AgentCrashError.""" + """Non-timeout ProcessError -> a CRASHED outcome.""" return ClaudeScenario( name="h2_process_error_crash", build_query=lambda: _raising_query([], ProcessError("boom", exit_code=1, stderr="bad config")), @@ -387,7 +388,6 @@ def _scenario_h2() -> ClaudeScenario: def _build_catalogue() -> list[ClaudeScenario]: - from coder_eval.errors import AgentCrashError, TurnTimeoutError scenarios = [ _scenario_a(), @@ -399,19 +399,19 @@ def _build_catalogue() -> list[ClaudeScenario]: ] g = _scenario_g() - g.expects = AgentCrashError + g.expects = AgentEndStatus.CRASHED scenarios.append(g) h1 = _scenario_h1() - h1.expects = TurnTimeoutError + h1.expects = AgentEndStatus.TIMEOUT scenarios.append(h1) h2 = _scenario_h2() - h2.expects = AgentCrashError + h2.expects = AgentEndStatus.CRASHED scenarios.append(h2) i = _build_deadline_break_scenario() - i.expects = TurnTimeoutError + i.expects = AgentEndStatus.TIMEOUT scenarios.append(i) return scenarios @@ -437,14 +437,13 @@ def _patches(scenario: ClaudeScenario) -> Iterator[Any]: async def run_claude_scenario( scenario: ClaudeScenario, working_dir: str ) -> tuple[dict[str, Any], list[protocol.StreamEvent]]: - """Run ``scenario`` and return the ``TurnRecord``/``pending_turn`` model_dump. + """Run ``scenario`` and return its outcome record's model_dump and the events. Raises ``AssertionError`` if a crash/timeout scenario fails to raise its expected exception (so a refactor that silently swallows the failure is caught). """ recorder = EventRecorder() - import pytest config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") agent = ClaudeCodeAgent(config) @@ -453,12 +452,11 @@ async def run_claude_scenario( with contextlib.ExitStack() as stack: for ctx in _patches(scenario): stack.enter_context(ctx) - if scenario.expects is not None: - with pytest.raises(scenario.expects): - await agent.communicate(scenario.prompt, timeout=scenario.timeout, stream_callback=recorder) - record = agent.pending_turn - assert record is not None, f"{scenario.name}: pending_turn was not set on the failure path" - else: - record = await agent.communicate(scenario.prompt, timeout=scenario.timeout, stream_callback=recorder) + outcome = await agent.communicate( + scenario.prompt, iteration=1, timeout=scenario.timeout, stream_callback=recorder + ) + expected = scenario.expects or AgentEndStatus.COMPLETED + assert outcome.status is expected, f"{scenario.name}: ended {outcome.status}, expected {expected}" + record = outcome.record return record.model_dump(mode="json"), recorder.events diff --git a/tests/_fixtures/golden_streams/codex_fixtures.py b/tests/_fixtures/golden_streams/codex_fixtures.py index 98d72615..0613d862 100644 --- a/tests/_fixtures/golden_streams/codex_fixtures.py +++ b/tests/_fixtures/golden_streams/codex_fixtures.py @@ -23,7 +23,7 @@ from coder_eval.agents.codex_agent import CodexAgent from coder_eval.models import AgentKind, parse_agent_config -from coder_eval.streaming.events import StreamEvent +from coder_eval.streaming.events import AgentEndStatus, StreamEvent from tests._fixtures.golden_streams._recorder import EventRecorder @@ -172,11 +172,10 @@ def _collab( class CodexScenario: name: str notifications: list[Any] - expects: type[BaseException] | None = None + expects: AgentEndStatus | None = None def _build_catalogue() -> list[CodexScenario]: - from coder_eval.errors import AgentCrashError scenarios: list[CodexScenario] = [] @@ -317,7 +316,7 @@ def _build_catalogue() -> list[CodexScenario]: ), _token_usage(inp=100, out=40, cached=8), ], - expects=AgentCrashError, + expects=AgentEndStatus.CRASHED, ) ) @@ -381,9 +380,8 @@ def _rebase_notifications(notifications: list[Any]) -> list[Any]: async def run_codex_scenario(scenario: CodexScenario, working_dir: str) -> tuple[dict[str, Any], list[StreamEvent]]: - """Run ``scenario`` with fakes and return the TurnRecord/pending_turn dump.""" + """Run ``scenario`` with fakes and return the outcome record's dump and the events.""" recorder = EventRecorder() - import pytest config = parse_agent_config(type=AgentKind.CODEX, model=CODEX_MODEL) agent = CodexAgent(config) @@ -394,12 +392,9 @@ async def run_codex_scenario(scenario: CodexScenario, working_dir: str) -> tuple # Point CODEX_HOME at a sessions-less dir so sub-agent rollout recovery # short-circuits instead of polling the real ~/.codex. with patch.dict(os.environ, {"CODEX_HOME": working_dir}): - if scenario.expects is not None: - with pytest.raises(scenario.expects): - await agent.communicate("do it", stream_callback=recorder) - record = agent.pending_turn - assert record is not None, f"{scenario.name}: pending_turn was not set on the failure path" - else: - record = await agent.communicate("do it", stream_callback=recorder) + outcome = await agent.communicate("do it", iteration=1, stream_callback=recorder) + expected = scenario.expects or AgentEndStatus.COMPLETED + assert outcome.status is expected, f"{scenario.name}: ended {outcome.status}, expected {expected}" + record = outcome.record return record.model_dump(mode="json"), recorder.events diff --git a/tests/_fixtures/golden_streams/opencode_fixtures.py b/tests/_fixtures/golden_streams/opencode_fixtures.py index 88a9b031..6e784da8 100644 --- a/tests/_fixtures/golden_streams/opencode_fixtures.py +++ b/tests/_fixtures/golden_streams/opencode_fixtures.py @@ -30,9 +30,8 @@ from unittest.mock import patch from coder_eval.agents.opencode_agent import OpenCodeAgent -from coder_eval.errors import AgentCrashError from coder_eval.models import OpenCodeAgentConfig -from coder_eval.streaming.events import StreamEvent +from coder_eval.streaming.events import AgentEndStatus, StreamEvent from tests._fixtures.golden_streams._recorder import EventRecorder @@ -204,8 +203,8 @@ def _agent() -> OpenCodeAgent: class OpenCodeScenario: """One recorded CLI event stream. - ``expects`` names the exception a scenario is supposed to raise, and the - runner then snapshots ``pending_turn`` instead of the returned record — + ``expects`` names the failed end status a scenario is supposed to reach, and the + runner asserts that end status and snapshots the crashed record — the same knob ``ClaudeScenario`` carries, for the same reason: the partial a crash preserves is a real capture path, and one nobody was comparing against a snapshot on this harness. @@ -213,7 +212,7 @@ class OpenCodeScenario: name: str lines: list[str] - expects: type[BaseException] | None = None + expects: AgentEndStatus | None = None async def run_opencode_scenario( @@ -221,7 +220,6 @@ async def run_opencode_scenario( ) -> tuple[dict[str, Any], list[StreamEvent]]: """Replay one scenario and return the resulting record as a plain dump.""" recorder = EventRecorder() - import pytest proc = _FakeProcess(_rebase_lines(scenario.lines)) @@ -236,13 +234,10 @@ async def fake_exec(*_argv: str, **_kwargs: Any) -> _FakeProcess: patch.object(os, "killpg", lambda _pgid, _sig: None, create=True), ): await agent.start(working_dir) - if scenario.expects is not None: - with pytest.raises(scenario.expects): - await agent.communicate("do it", stream_callback=recorder) - record = agent.pending_turn - assert record is not None, f"{scenario.name}: pending_turn was not set on the failure path" - else: - record = await agent.communicate("do it", stream_callback=recorder) + outcome = await agent.communicate("do it", iteration=1, stream_callback=recorder) + expected = scenario.expects or AgentEndStatus.COMPLETED + assert outcome.status is expected, f"{scenario.name}: ended {outcome.status}, expected {expected}" + record = outcome.record return record.model_dump(mode="json"), recorder.events @@ -349,7 +344,7 @@ def _build_catalogue() -> list[OpenCodeScenario]: ) # (e) the CLI's own structured error AFTER a complete generation. `_settle_turn` - # crashes on it, and the partial `pending_turn` must still carry that + # crashes on it, and the crashed record must still carry that # generation and its head/tail — a crash does not un-measure what was # measured before it. scenarios.append( @@ -370,7 +365,7 @@ def _build_catalogue() -> list[OpenCodeScenario]: } ), ], - expects=AgentCrashError, + expects=AgentEndStatus.CRASHED, ) ) diff --git a/tests/_fixtures/golden_streams/pi_fixtures.py b/tests/_fixtures/golden_streams/pi_fixtures.py index 817f247c..1906cb8e 100644 --- a/tests/_fixtures/golden_streams/pi_fixtures.py +++ b/tests/_fixtures/golden_streams/pi_fixtures.py @@ -24,9 +24,8 @@ from unittest.mock import patch from coder_eval.agents.pi_agent import PiAgent -from coder_eval.errors import AgentCrashError from coder_eval.models import PiAgentConfig -from coder_eval.streaming.events import StreamEvent +from coder_eval.streaming.events import AgentEndStatus, StreamEvent from tests._fixtures.golden_streams._recorder import EventRecorder @@ -193,8 +192,8 @@ def _agent() -> PiAgent: class PiScenario: """One recorded CLI event stream. - ``expects`` names the exception a scenario is supposed to raise, and the - runner then snapshots ``pending_turn`` instead of the returned record — + ``expects`` names the failed end status a scenario is supposed to reach, and the + runner asserts that end status and snapshots the crashed record — the same knob ``ClaudeScenario`` carries, for the same reason: the partial a crash preserves is a real capture path, and one nobody was comparing against a snapshot on this harness. @@ -202,13 +201,12 @@ class PiScenario: name: str lines: list[str] - expects: type[BaseException] | None = None + expects: AgentEndStatus | None = None async def run_pi_scenario(scenario: PiScenario, working_dir: str) -> tuple[dict[str, Any], list[StreamEvent]]: """Replay one scenario and return the resulting record as a plain dump.""" recorder = EventRecorder() - import pytest proc = _FakeProcess(scenario.lines) @@ -223,13 +221,10 @@ async def fake_exec(*_argv: str, **_kwargs: Any) -> _FakeProcess: patch.object(os, "killpg", lambda _pgid, _sig: None, create=True), ): await agent.start(working_dir) - if scenario.expects is not None: - with pytest.raises(scenario.expects): - await agent.communicate("do it", stream_callback=recorder) - record = agent.pending_turn - assert record is not None, f"{scenario.name}: pending_turn was not set on the failure path" - else: - record = await agent.communicate("do it", stream_callback=recorder) + outcome = await agent.communicate("do it", iteration=1, stream_callback=recorder) + expected = scenario.expects or AgentEndStatus.COMPLETED + assert outcome.status is expected, f"{scenario.name}: ended {outcome.status}, expected {expected}" + record = outcome.record return record.model_dump(mode="json"), recorder.events @@ -299,7 +294,7 @@ def _build_catalogue() -> list[PiScenario]: # (e) the provider error pi's internal retries could not clear, AFTER a # complete generation. The CLI still exits 0, so `_settle_turn` crashes on - # `stopReason=error` alone — and the partial `pending_turn` must still carry + # `stopReason=error` alone — and the crashed record must still carry # that generation and its head/tail. A crash does not un-measure what was # measured before it. scenarios.append( @@ -312,7 +307,7 @@ def _build_catalogue() -> list[PiScenario]: _turn_start(), _turn_end_error("provider returned 529 after 5 retries"), ], - expects=AgentCrashError, + expects=AgentEndStatus.CRASHED, ) ) diff --git a/tests/fixtures/mock_agent.py b/tests/fixtures/mock_agent.py index 0966cba9..d42b1cbd 100644 --- a/tests/fixtures/mock_agent.py +++ b/tests/fixtures/mock_agent.py @@ -9,6 +9,8 @@ from coder_eval.agent import Agent, AgentState from coder_eval.models import TaskDefinition, TurnRecord +from coder_eval.streaming.emitter import TurnOutcome +from coder_eval.streaming.events import AgentEndStatus from tests.fixtures.harness_stubs import stub_contract @@ -70,28 +72,29 @@ def get_state(self) -> AgentState: """ return self.state - async def communicate(self, user_input: str, **kwargs) -> TurnRecord: + async def communicate(self, user_input: str, *, iteration: int, **kwargs) -> TurnOutcome: """Simulate agent turn based on configured scenario. Args: user_input: Prompt from orchestrator Returns: - TurnRecord with simulated agent response and file changes + A completed outcome carrying the simulated agent response and file changes Raises: ValueError: If scenario is unknown """ - self._iteration += 1 # Increment iteration count + self._iteration = iteration if self.scenario == "success": - return self._success_turn(user_input) + record = self._success_turn(user_input) elif self.scenario == "failure": - return self._failure_turn(user_input) + record = self._failure_turn(user_input) elif self.scenario == "partial": - return self._partial_turn(user_input) + record = self._partial_turn(user_input) else: raise ValueError(f"Unknown scenario: {self.scenario}") + return TurnOutcome(record=record, status=AgentEndStatus.COMPLETED, error=None) def _success_turn(self, user_input: str) -> TurnRecord: """Simulate successful task completion. diff --git a/tests/fixtures/text_stub_agent.py b/tests/fixtures/text_stub_agent.py index 90f8e310..fa294588 100644 --- a/tests/fixtures/text_stub_agent.py +++ b/tests/fixtures/text_stub_agent.py @@ -11,6 +11,8 @@ from coder_eval.agent import Agent, AgentState from coder_eval.models import TurnRecord +from coder_eval.streaming.emitter import TurnOutcome +from coder_eval.streaming.events import AgentEndStatus from tests.fixtures.harness_stubs import stub_contract @@ -47,12 +49,8 @@ async def stop(self) -> None: def get_state(self) -> AgentState: return self._state - async def communicate(self, user_input: str, **kwargs: object) -> TurnRecord: - self._iteration += 1 + async def communicate(self, user_input: str, *, iteration: int, **kwargs: object) -> TurnOutcome: self.calls.append(user_input) text = self._responses.pop(0) if self._responses else "" - return TurnRecord( - iteration=self._iteration, - user_input=user_input, - agent_output=text, - ) + record = TurnRecord(iteration=iteration, user_input=user_input, agent_output=text) + return TurnOutcome(record=record, status=AgentEndStatus.COMPLETED, error=None) diff --git a/tests/test_agent.py b/tests/test_agent.py index 7aa151a6..d634816a 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -38,77 +38,6 @@ def test_pending_turn_defaults_to_none(): assert agent.pending_turn is None -@pytest.mark.asyncio -async def test_discard_pending_turn_clears_slot_and_decrements(): - """discard_pending_turn clears the slot and rolls back _iteration once.""" - from coder_eval.models import TurnRecord - - config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") - agent = ClaudeCodeAgent(config) - - partial = TurnRecord(iteration=1, user_input="p", agent_output="", crashed=True) - agent._iteration = 1 - agent.pending_turn = partial - - await agent.discard_pending_turn() - - assert agent.pending_turn is None - assert agent._iteration == 0 - - -@pytest.mark.asyncio -async def test_discard_pending_turn_idempotent(): - """discard_pending_turn is a no-op when pending_turn is already None.""" - config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") - agent = ClaudeCodeAgent(config) - - assert agent.pending_turn is None - assert agent._iteration == 0 - - # First call: nothing to discard — counter must not go negative. - await agent.discard_pending_turn() - assert agent.pending_turn is None - assert agent._iteration == 0 - - # Second call after a real discard: still a no-op. - from coder_eval.models import TurnRecord - - partial = TurnRecord(iteration=2, user_input="p", agent_output="", crashed=True) - agent._iteration = 2 - agent.pending_turn = partial - await agent.discard_pending_turn() # real discard - await agent.discard_pending_turn() # idempotent second call - assert agent.pending_turn is None - assert agent._iteration == 1 # decremented once, not twice - - -@pytest.mark.asyncio -async def test_discard_pending_turn_rolls_back_when_partial_build_failed(): - """If _set_pending swallowed an exception and left pending_turn=None, discard - must still roll back the iteration counter. - - Regression: previously the rollback gated on (pending_turn is not None), so - a swallowed partial-build exception caused _iteration to drift permanently - higher on every double-failure. - """ - config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") - agent = ClaudeCodeAgent(config) - - # Simulate communicate() incrementing the counter and then crashing before - # _set_pending could finish (partial-build exception swallowed → pending_turn None). - agent._iteration = 5 - agent._iteration_was_incremented = True - agent.pending_turn = None - - await agent.discard_pending_turn() - assert agent._iteration == 4, "rollback must fire even when pending_turn is None" - assert agent._iteration_was_incremented is False - - # Second call is idempotent — neither signal fires. - await agent.discard_pending_turn() - assert agent._iteration == 4 - - @pytest.mark.asyncio async def test_stop_clears_pending_turn(): """stop() clears pending_turn so stale partials don't leak between runs.""" @@ -210,7 +139,7 @@ async def mock_query(prompt, options): with tempfile.TemporaryDirectory() as tmpdir: await agent.start(tmpdir, env_path_prepend=env_path_prepend) with patch("coder_eval.agents.claude_code_agent.query", mock_query): - await agent.communicate("hello") + await agent.communicate("hello", iteration=1) return captured_options @@ -969,7 +898,9 @@ def test_format_messages_system_message_subclasses_are_filtered(): @pytest.mark.asyncio async def test_claude_agent_process_error_includes_stderr(): - """Test that ProcessError is caught and its stderr is included in RuntimeError.""" + """Test that ProcessError is caught and its stderr is included in the CRASHED outcome's error.""" + import re + config = parse_agent_config( type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits", @@ -986,18 +917,20 @@ async def mock_query(*args, **kwargs): raise ProcessError("process failed", exit_code=1, stderr="Error: invalid config") yield # makes this an async generator - with ( - patch("coder_eval.agents.claude_code_agent.query", mock_query), - pytest.raises(RuntimeError, match=r"CLI process failed \(exit code 1\): Error: invalid config"), - ): - await agent.communicate("do something") + with patch("coder_eval.agents.claude_code_agent.query", mock_query): + outcome = await agent.communicate("do something", iteration=1) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None + assert re.search(r"CLI process failed \(exit code 1\): Error: invalid config", outcome.error) assert agent.get_state() == AgentState.ERROR @pytest.mark.asyncio async def test_claude_agent_process_error_no_stderr_at_all(): """Test that ProcessError with no stderr and no stderr_lines shows sentinel message.""" + import re + config = parse_agent_config( type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits", @@ -1013,11 +946,12 @@ async def mock_query(*args, **kwargs): raise ProcessError("process failed", exit_code=None, stderr=None) yield # makes this an async generator - with ( - patch("coder_eval.agents.claude_code_agent.query", mock_query), - pytest.raises(RuntimeError, match=r"CLI process failed \(exit code None\): No stderr captured"), - ): - await agent.communicate("do something") + with patch("coder_eval.agents.claude_code_agent.query", mock_query): + outcome = await agent.communicate("do something", iteration=1) + + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None + assert re.search(r"CLI process failed \(exit code None\): No stderr captured", outcome.error) @pytest.mark.asyncio @@ -1058,12 +992,12 @@ async def mock_query(prompt, options): with patch("coder_eval.agents.claude_code_agent.query", mock_query): # First call: no session_id yet - await agent.communicate("first prompt") + await agent.communicate("first prompt", iteration=1) assert captured_options[0].resume is None assert agent._session_id == "test-session-abc" # Second call: should pass session_id as resume - await agent.communicate("second prompt") + await agent.communicate("second prompt", iteration=2) assert captured_options[1].resume == "test-session-abc" @@ -1102,7 +1036,7 @@ async def mock_ok(prompt, options): yield ResultMessage(session_id="good-session", is_error=False) with patch("coder_eval.agents.claude_code_agent.query", mock_ok): - await agent.communicate("clean turn") + await agent.communicate("clean turn", iteration=1) assert agent._session_id == "good-session" # Second: an errored turn arriving with a NEW session_id must NOT @@ -1114,7 +1048,7 @@ async def mock_err(prompt, options): yield ResultMessage(session_id="poisoned-session", is_error=True) with patch("coder_eval.agents.claude_code_agent.query", mock_err): - await agent.communicate("errored turn") + await agent.communicate("errored turn", iteration=2) assert agent._session_id == "good-session" @@ -1154,11 +1088,11 @@ async def mock_query(prompt, options): yield ResultMessage(session_id=None) with patch("coder_eval.agents.claude_code_agent.query", mock_query): - await agent.communicate("first prompt") + await agent.communicate("first prompt", iteration=1) assert agent._session_id is None # Second call: resume should be None (fresh session) - await agent.communicate("second prompt") + await agent.communicate("second prompt", iteration=2) assert captured_options[1].resume is None @@ -1202,15 +1136,15 @@ async def mock_query(prompt, options): yield ResultMessage(session_id=f"session-{call_count}") with patch("coder_eval.agents.claude_code_agent.query", mock_query): - await agent.communicate("first prompt") + await agent.communicate("first prompt", iteration=1) assert agent._session_id == "session-1" - await agent.communicate("second prompt") + await agent.communicate("second prompt", iteration=2) assert captured_options[1].resume == "session-1" assert agent._session_id == "session-2" # Third call should use the rotated session_id - await agent.communicate("third prompt") + await agent.communicate("third prompt", iteration=3) assert captured_options[2].resume == "session-2" @@ -1248,7 +1182,7 @@ async def mock_query_ok(prompt, options): yield ResultMessage(session_id="good-session") with patch("coder_eval.agents.claude_code_agent.query", mock_query_ok): - await agent.communicate("first prompt") + await agent.communicate("first prompt", iteration=1) assert agent._session_id == "good-session" # Second call raises an error mid-stream @@ -1256,11 +1190,12 @@ async def mock_query_error(prompt, options): raise RuntimeError("SDK connection lost") yield - with ( - patch("coder_eval.agents.claude_code_agent.query", mock_query_error), - pytest.raises(RuntimeError, match="SDK connection lost"), - ): - await agent.communicate("second prompt") + with patch("coder_eval.agents.claude_code_agent.query", mock_query_error): + outcome = await agent.communicate("second prompt", iteration=2) + + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None + assert "SDK connection lost" in outcome.error # session_id should still be the value from the successful call assert agent._session_id == "good-session" @@ -1385,8 +1320,10 @@ async def mock_query(prompt, options): ) with patch("coder_eval.agents.claude_code_agent.query", mock_query): - turn = await agent.communicate("hello") + outcome = await agent.communicate("hello", iteration=1) + assert outcome.status is AgentEndStatus.COMPLETED + turn = outcome.record assert turn.result_summary is not None assert turn.result_summary.is_error is False assert turn.result_summary.subtype == "success" @@ -1398,12 +1335,12 @@ async def mock_query(prompt, options): @pytest.mark.asyncio async def test_claude_agent_crash_preserves_partial_turn_record(): - """When communicate() fails mid-turn, agent.pending_turn carries a partial + """When communicate() fails mid-turn, the CRASHED outcome carries a partial TurnRecord populated with tool calls captured before the crash. - This is the whole point of the pending_turn slot + on_attempt_error - plumbing: typed criteria like skill_triggered must still be able to - observe a Skill invocation that happened before the crash. + This is the whole point of the pending_turn slot + outcome plumbing: typed + criteria like skill_triggered must still be able to observe a Skill + invocation that happened before the crash. """ config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") agent = ClaudeCodeAgent(config) @@ -1427,14 +1364,11 @@ async def mock_query(prompt, options, transport=None): with tempfile.TemporaryDirectory() as tmpdir: await agent.start(tmpdir) - with ( - patch("coder_eval.agents.claude_code_agent.query", mock_query), - pytest.raises(AgentCrashError), - ): - await agent.communicate("do the thing") + with patch("coder_eval.agents.claude_code_agent.query", mock_query): + outcome = await agent.communicate("do the thing", iteration=1) - # Slot is populated before the raise; not yet cleared (caller must drain). - partial = agent.pending_turn + assert outcome.status is AgentEndStatus.CRASHED + partial = outcome.record assert partial is not None assert partial.crashed is True assert partial.tool_calls_exhausted is False @@ -1444,14 +1378,11 @@ async def mock_query(prompt, options, transport=None): assert len(partial.commands) == 1 assert partial.commands[0].tool_name == "Skill" assert partial.commands[0].parameters == {"skill": "my_skill"} - # Iteration contract: partial carries the bumped iteration number; the - # counter is NOT rolled back until discard_pending_turn() is called. + # Iteration contract: partial carries the bumped iteration number. assert partial.iteration == 1 assert agent._iteration == 1 - - await agent.discard_pending_turn() + # The pending_turn side-channel is always cleared once communicate() returns. assert agent.pending_turn is None - assert agent._iteration == 0 @pytest.mark.asyncio @@ -1473,13 +1404,11 @@ async def mock_query(prompt, options, transport=None): with tempfile.TemporaryDirectory() as tmpdir: await agent.start(tmpdir) - with ( - patch("coder_eval.agents.claude_code_agent.query", mock_query), - pytest.raises(AgentCrashError), - ): - await agent.communicate("go") + with patch("coder_eval.agents.claude_code_agent.query", mock_query): + outcome = await agent.communicate("go", iteration=1) - partial = agent.pending_turn + assert outcome.status is AgentEndStatus.CRASHED + partial = outcome.record assert partial is not None assert partial.crash_reason is not None # The crash message is truncated at 200 chars, but a short message @@ -1504,11 +1433,11 @@ async def mock_query(prompt, options, transport=None): with ( patch("coder_eval.agents.claude_code_agent.query", mock_query), patch.object(ClaudeCodeAgent, "_timed_out", staticmethod(lambda *a, **k: True)), - pytest.raises(TurnTimeoutError), ): - await agent.communicate("go", timeout=42.0) + outcome = await agent.communicate("go", iteration=1, timeout=42.0) - partial = agent.pending_turn + assert outcome.status is AgentEndStatus.TIMEOUT + partial = outcome.record assert partial is not None # Normalised reason: integer-second formatting matches the # orchestrator's defensive fallback so report rendering is consistent. @@ -1517,13 +1446,14 @@ async def mock_query(prompt, options, transport=None): @pytest.mark.asyncio async def test_claude_agent_repeated_crashes_keep_iteration_stable(): - """Consecutive crashes in one orchestrator iteration all carry the same iteration number. - - discard_pending_turn() rolls back _iteration after each crash (simulating - what the orchestrator does), so repeated failures in a single logical - orchestrator iteration all stamp the same iteration on their partial records. - A subsequent clean call then advances the counter by one. This is what the - orchestrator's multiple-partials-per-iteration contract relies on. + """Consecutive crashes for the same caller-supplied iteration all carry that + same iteration number. + + Retries of one logical turn pass the same ``iteration`` on every attempt; + ``_legacy_outcome`` resets ``_iteration`` to ``iteration - 1`` on each call, so + repeated failures for that iteration all stamp the same number on their + partial records. A subsequent clean call at ``iteration + 1`` then advances + the counter by one. """ config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") agent = ClaudeCodeAgent(config) @@ -1556,24 +1486,20 @@ async def clean_query(prompt, options, transport=None): partials: list = [] for _ in range(3): - with ( - patch("coder_eval.agents.claude_code_agent.query", crashing_query), - pytest.raises(AgentCrashError), - ): - await agent.communicate("go") - partials.append(agent.pending_turn) - # Simulate the orchestrator draining and discarding after a failed attempt. - await agent.discard_pending_turn() - assert agent._iteration == 0 + with patch("coder_eval.agents.claude_code_agent.query", crashing_query): + outcome = await agent.communicate("go", iteration=1) + assert outcome.status is AgentEndStatus.CRASHED + partials.append(outcome.record) assert all(p is not None and p.iteration == 1 and p.crashed for p in partials) - # The clean retry advances the counter and produces iteration=1 again, - # so all four records for this logical orchestrator iteration share 1. + # The clean retry passes the SAME iteration number as its failed + # predecessors — a retry of one logical turn, not a new one. with patch("coder_eval.agents.claude_code_agent.query", clean_query): - turn_record = await agent.communicate("go") + outcome = await agent.communicate("go", iteration=1) assert clean_finished + turn_record = outcome.record assert turn_record.iteration == 1 assert turn_record.crashed is False assert agent._iteration == 1 @@ -1581,8 +1507,7 @@ async def clean_query(prompt, options, transport=None): @pytest.mark.asyncio async def test_claude_agent_timeout_preserves_partial_turn_record(): - """agent.pending_turn carries a partial TurnRecord with pre-kill tool calls - after a TurnTimeoutError. + """A TIMEOUT outcome carries a partial TurnRecord with pre-kill tool calls. Watchdog-killed turns are exactly where observational telemetry is most valuable (an agent that looped on tool calls and ran the wall @@ -1617,20 +1542,19 @@ async def mock_query(prompt, options, transport=None): with ( patch("coder_eval.agents.claude_code_agent.query", mock_query), patch.object(ClaudeCodeAgent, "_timed_out", staticmethod(lambda *a, **k: True)), - pytest.raises(TurnTimeoutError), ): - await agent.communicate("start", timeout=0.01) + outcome = await agent.communicate("start", iteration=1, timeout=0.01) - partial = agent.pending_turn + assert outcome.status is AgentEndStatus.TIMEOUT + partial = outcome.record assert partial is not None assert partial.crashed is True assert len(partial.commands) == 1 assert partial.commands[0].tool_name == "Bash" - # Slot carries the bumped iteration; counter rolls back after discard. + # The outcome carries the bumped iteration. assert partial.iteration == 1 assert agent._iteration == 1 - await agent.discard_pending_turn() - assert agent._iteration == 0 + assert agent.pending_turn is None @pytest.mark.asyncio @@ -1681,9 +1605,11 @@ async def mock_query(prompt, options, transport=None): await agent.start(tmpdir) with patch("coder_eval.agents.claude_code_agent.query", mock_query): - # Must NOT raise: error_max_turns is a clean completion path. - turn_record = await agent.communicate("solve something hard", stream_callback=recorder) + # Must NOT crash: error_max_turns is a clean completion path. + outcome = await agent.communicate("solve something hard", iteration=1, stream_callback=recorder) + assert outcome.status is AgentEndStatus.COMPLETED + turn_record = outcome.record assert turn_record.crashed is False assert turn_record.tool_calls_exhausted is False assert [e.status for e in recorder.events if isinstance(e, AgentEndEvent)] == [AgentEndStatus.COMPLETED] @@ -1738,8 +1664,10 @@ async def mock_query(prompt, options, transport=None): await agent.start(tmpdir) with patch("coder_eval.agents.claude_code_agent.query", mock_query): - turn_record = await agent.communicate("solve something hard", stream_callback=recorder) + outcome = await agent.communicate("solve something hard", iteration=1, stream_callback=recorder) + assert outcome.status is AgentEndStatus.COMPLETED + turn_record = outcome.record assert turn_record.crashed is False assert turn_record.tool_calls_exhausted is False assert [e.status for e in recorder.events if isinstance(e, AgentEndEvent)] == [AgentEndStatus.COMPLETED] @@ -1785,8 +1713,10 @@ async def mock_query(prompt, options, transport=None): with tempfile.TemporaryDirectory() as tmpdir: await agent.start(tmpdir) with patch("coder_eval.agents.claude_code_agent.query", mock_query): - turn_record = await agent.communicate("go", stream_callback=recorder, should_stop=lambda: reason) + outcome = await agent.communicate("go", iteration=1, stream_callback=recorder, should_stop=lambda: reason) + assert outcome.status is status + turn_record = outcome.record assert dispatched == ["first"] assert turn_record.crashed is False assert turn_record.tool_calls_exhausted is exhausted @@ -2034,8 +1964,8 @@ async def mock_query(prompt, options, transport=None): ), patch("coder_eval.agents.claude_code_agent.query", mock_query), ): - await agent.communicate("turn 1", timeout=30.0) - await agent.communicate("turn 2", timeout=30.0) + await agent.communicate("turn 1", iteration=1, timeout=30.0) + await agent.communicate("turn 2", iteration=2, timeout=30.0) assert len(captured_callbacks) == 2 # Both turns finished, so self._active_transport is None. Fire turn 1's @@ -2044,3 +1974,221 @@ async def mock_query(prompt, options, transport=None): captured_callbacks[0]() transport_a._process.kill.assert_called_once() transport_b._process.kill.assert_not_called() + + +class TestLegacyOutcome: + """Unit tests for ``Agent._legacy_outcome``, the adapter that wraps a + not-yet-ported raise-and-park ``communicate`` body onto the ``TurnOutcome`` + contract. Drives it with a tiny fake body, not a real harness. + """ + + @staticmethod + def _agent(): + return ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits")) + + @pytest.mark.asyncio + async def test_sets_iteration_so_the_body_sees_the_caller_iteration(self): + from coder_eval.models import TurnRecord + + agent = self._agent() + agent._iteration = 99 + seen: list[int] = [] + + async def body(user_input, *, stream_callback, timeout, should_stop): + agent._begin_turn() + seen.append(agent._iteration) + return TurnRecord(iteration=agent._iteration, user_input=user_input, agent_output="ok") + + outcome = await agent._legacy_outcome( + body, "hi", iteration=5, stream_callback=None, timeout=None, should_stop=None + ) + + assert seen == [5] + assert outcome.record.iteration == 5 + + @pytest.mark.asyncio + async def test_returned_record_maps_to_the_recorded_end_status(self): + from coder_eval.models import TurnRecord + + agent = self._agent() + record = TurnRecord(iteration=1, user_input="hi", agent_output="done") + + async def body(user_input, *, stream_callback, timeout, should_stop): + stream_callback.on_event(AgentEndEvent(task_id="t", status=AgentEndStatus.STOPPED_EARLY)) + return record + + outcome = await agent._legacy_outcome( + body, "hi", iteration=1, stream_callback=None, timeout=None, should_stop=None + ) + + assert outcome.status is AgentEndStatus.STOPPED_EARLY + assert outcome.record is record + assert outcome.error is None + + @pytest.mark.asyncio + async def test_returned_record_falls_back_to_completed_with_no_recorded_end_event(self): + from coder_eval.models import TurnRecord + + agent = self._agent() + record = TurnRecord(iteration=1, user_input="hi", agent_output="done") + + async def body(user_input, *, stream_callback, timeout, should_stop): + return record + + outcome = await agent._legacy_outcome( + body, "hi", iteration=1, stream_callback=None, timeout=None, should_stop=None + ) + + assert outcome.status is AgentEndStatus.COMPLETED + assert outcome.record is record + assert outcome.error is None + + @pytest.mark.asyncio + async def test_crash_uses_pending_turn_when_present(self): + from coder_eval.models import TurnRecord + + agent = self._agent() + pending = TurnRecord(iteration=3, user_input="hi", agent_output="", crashed=True, crash_reason="boom") + agent.pending_turn = pending + agent._iteration_was_incremented = True + + async def body(user_input, *, stream_callback, timeout, should_stop): + raise AgentCrashError("boom") + + outcome = await agent._legacy_outcome( + body, "hi", iteration=3, stream_callback=None, timeout=None, should_stop=None + ) + + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.record is pending + assert outcome.error == "boom" + assert agent.pending_turn is None + assert agent._iteration_was_incremented is False + + @pytest.mark.asyncio + async def test_crash_without_pending_turn_builds_an_empty_crashed_record(self): + from coder_eval.errors.agent import truncate_crash_message + + agent = self._agent() + + async def body(user_input, *, stream_callback, timeout, should_stop): + raise AgentCrashError("kaboom") + + outcome = await agent._legacy_outcome( + body, "hi there", iteration=2, stream_callback=None, timeout=None, should_stop=None + ) + + assert outcome.status is AgentEndStatus.CRASHED + record = outcome.record + assert record.iteration == 2 + assert record.user_input == "hi there" + assert record.agent_output == "" + assert record.crashed is True + assert record.crash_reason == truncate_crash_message("kaboom") + assert outcome.error == "kaboom" + assert agent.pending_turn is None + + @pytest.mark.asyncio + async def test_timeout_without_pending_turn_falls_back_to_timeout_status(self): + agent = self._agent() + + async def body(user_input, *, stream_callback, timeout, should_stop): + raise TurnTimeoutError(30.0) + + outcome = await agent._legacy_outcome( + body, "hi", iteration=1, stream_callback=None, timeout=30.0, should_stop=None + ) + + assert outcome.status is AgentEndStatus.TIMEOUT + assert outcome.record.crashed is True + assert outcome.error == "Agent turn timed out after 30s" + + @pytest.mark.asyncio + async def test_a_recorded_failed_status_wins_over_the_exception_type_fallback(self): + agent = self._agent() + + async def body(user_input, *, stream_callback, timeout, should_stop): + stream_callback.on_event(AgentEndEvent(task_id="t", status=AgentEndStatus.TIMEOUT)) + raise AgentCrashError("late failure") + + outcome = await agent._legacy_outcome( + body, "hi", iteration=1, stream_callback=None, timeout=None, should_stop=None + ) + + assert outcome.status is AgentEndStatus.TIMEOUT + assert outcome.error == "late failure" + + @pytest.mark.asyncio + async def test_a_recorded_clean_status_never_labels_a_raised_crash(self): + agent = self._agent() + + async def body(user_input, *, stream_callback, timeout, should_stop): + stream_callback.on_event(AgentEndEvent(task_id="t", status=AgentEndStatus.COMPLETED)) + raise AgentCrashError("record build failed after a clean finalize") + + outcome = await agent._legacy_outcome( + body, "hi", iteration=1, stream_callback=None, timeout=None, should_stop=None + ) + + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.record.crashed is True + + @pytest.mark.asyncio + async def test_forwards_events_to_the_given_callback(self): + from coder_eval.models import TurnRecord + + agent = self._agent() + received: list = [] + + class _Recorder: + def on_event(self, event): + received.append(event) + + async def body(user_input, *, stream_callback, timeout, should_stop): + stream_callback.on_event(AgentEndEvent(task_id="t", status=AgentEndStatus.COMPLETED)) + return TurnRecord(iteration=1, user_input=user_input, agent_output="ok") + + outcome = await agent._legacy_outcome( + body, "hi", iteration=1, stream_callback=_Recorder(), timeout=None, should_stop=None + ) + + assert outcome.status is AgentEndStatus.COMPLETED + assert len(received) == 1 + assert received[0].status is AgentEndStatus.COMPLETED + + @pytest.mark.asyncio + async def test_works_with_stream_callback_none(self): + from coder_eval.models import TurnRecord + + agent = self._agent() + + async def body(user_input, *, stream_callback, timeout, should_stop): + # No external sink was given; the internal status tracker is still fed. + stream_callback.on_event(AgentEndEvent(task_id="t", status=AgentEndStatus.COMPLETED)) + return TurnRecord(iteration=1, user_input=user_input, agent_output="ok") + + outcome = await agent._legacy_outcome( + body, "hi", iteration=1, stream_callback=None, timeout=None, should_stop=None + ) + + assert outcome.status is AgentEndStatus.COMPLETED + + @pytest.mark.asyncio + async def test_cancelled_error_propagates_untouched(self): + import asyncio + + from coder_eval.models import TurnRecord + + agent = self._agent() + pending = TurnRecord(iteration=1, user_input="hi", agent_output="", crashed=True) + agent.pending_turn = pending + + async def body(user_input, *, stream_callback, timeout, should_stop): + raise asyncio.CancelledError() + + with pytest.raises(asyncio.CancelledError): + await agent._legacy_outcome(body, "hi", iteration=1, stream_callback=None, timeout=None, should_stop=None) + + # The body already finalized the turn on cancellation; _legacy_outcome + # does not touch pending_turn on this path. + assert agent.pending_turn is pending diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index 069b133d..d7bc42e4 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -3,7 +3,7 @@ This is the safety net for decomposing ``ClaudeCodeAgent.communicate`` and ``CodexAgent._run_turn_with_streaming``: each scenario replays a recorded SDK event stream through ``communicate()`` and asserts the resulting -``TurnRecord`` / ``pending_turn`` is byte-identical (post-scrub) to a committed +``TurnRecord`` (crashed or not) is byte-identical (post-scrub) to a committed JSON snapshot. The decomposition must not change any snapshot. Regenerate the snapshots after an INTENTIONAL behavior change with:: diff --git a/tests/test_agent_judge_criterion.py b/tests/test_agent_judge_criterion.py index fa73ea6e..0b386c41 100644 --- a/tests/test_agent_judge_criterion.py +++ b/tests/test_agent_judge_criterion.py @@ -36,6 +36,8 @@ ) from coder_eval.models.routing import DirectRoute from coder_eval.sandbox import Sandbox +from coder_eval.streaming.emitter import TurnOutcome +from coder_eval.streaming.events import AgentEndStatus # ClaudeCodeAgent is now imported inside SubAgentRunner; tests patch the runner's binding. @@ -62,10 +64,14 @@ def _make_turn(agent_output: str, duration: float = 1.5) -> TurnRecord: ) +def _make_outcome(record: TurnRecord) -> TurnOutcome: + return TurnOutcome(record=record, status=AgentEndStatus.COMPLETED, error=None) + + def _make_mock_agent(agent_output: str) -> MagicMock: agent = MagicMock() agent.start = AsyncMock(return_value=None) - agent.communicate = AsyncMock(return_value=_make_turn(agent_output)) + agent.communicate = AsyncMock(return_value=_make_outcome(_make_turn(agent_output))) agent.stop = AsyncMock(return_value=None) agent.kill = AsyncMock(return_value=None) return agent @@ -215,11 +221,11 @@ def test_agent_judge_no_verdict_surfaces_untrusted_output(sandbox: Sandbox, dire def test_agent_judge_turn_timeout_maps_to_zero(sandbox: Sandbox, direct_route: DirectRoute) -> None: - from coder_eval.errors.timeout import TurnTimeoutError - criterion = AgentJudgeCriterion(description="x", prompt="grade", turn_timeout=30) mock_agent = _make_mock_agent("irrelevant") - mock_agent.communicate.side_effect = TurnTimeoutError(30.0, task_id="t", iteration=1) + mock_agent.communicate.return_value = TurnOutcome( + record=_make_turn("irrelevant"), status=AgentEndStatus.TIMEOUT, error="timed out" + ) with patch(_AGENT_PATCH_PATH, return_value=mock_agent): result = SuccessChecker(sandbox, init_registry=False, route=direct_route).check(criterion) @@ -831,7 +837,7 @@ def test_agent_judge_transcript_captures_tool_calls(sandbox: Sandbox, direct_rou mock_agent = MagicMock() mock_agent.start = AsyncMock(return_value=None) mock_agent.communicate = AsyncMock( - return_value=_make_turn_with_commands('{"score": 0.9, "rationale": "ok"}', [cmd1, cmd2]) + return_value=_make_outcome(_make_turn_with_commands('{"score": 0.9, "rationale": "ok"}', [cmd1, cmd2])) ) mock_agent.stop = AsyncMock(return_value=None) mock_agent.kill = AsyncMock(return_value=None) @@ -943,11 +949,11 @@ def test_agent_judge_round_trips_through_evaluation_result(sandbox: Sandbox, dir def test_agent_judge_timeout_uses_base_criterion_result(sandbox: Sandbox, direct_route: DirectRoute) -> None: """Timeout path returns a base CriterionResult (no transcript / verdict fields) because no turn was produced — there's nothing to capture.""" - from coder_eval.errors.timeout import TurnTimeoutError - criterion = AgentJudgeCriterion(description="x", prompt="grade", turn_timeout=30) mock_agent = _make_mock_agent("irrelevant") - mock_agent.communicate.side_effect = TurnTimeoutError(30.0, task_id="t", iteration=1) + mock_agent.communicate.return_value = TurnOutcome( + record=_make_turn("irrelevant"), status=AgentEndStatus.TIMEOUT, error="timed out" + ) with patch(_AGENT_PATCH_PATH, return_value=mock_agent): result = SuccessChecker(sandbox, init_registry=False, route=direct_route).check(criterion) diff --git a/tests/test_agent_telemetry.py b/tests/test_agent_telemetry.py index 6bc6c501..d6fc06ef 100644 --- a/tests/test_agent_telemetry.py +++ b/tests/test_agent_telemetry.py @@ -105,7 +105,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("List files") + turn = (await agent.communicate("List files", iteration=1)).record # Verify command telemetry assert len(turn.commands) == 1 @@ -150,7 +150,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("Read file") + turn = (await agent.communicate("Read file", iteration=1)).record assert len(turn.commands) == 1 cmd = turn.commands[0] @@ -189,7 +189,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("Write file") + turn = (await agent.communicate("Write file", iteration=1)).record assert len(turn.commands) == 1 cmd = turn.commands[0] @@ -239,7 +239,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("Multiple commands") + turn = (await agent.communicate("Multiple commands", iteration=1)).record assert len(turn.commands) == 3 @@ -290,7 +290,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("Timed command") + turn = (await agent.communicate("Timed command", iteration=1)).record assert len(turn.commands) == 1 cmd = turn.commands[0] @@ -328,7 +328,7 @@ async def mock_query(prompt, options): await agent.start(str(tmp_path)) with caplog.at_level(logging.WARNING): - turn = await agent.communicate("Orphaned result") + turn = (await agent.communicate("Orphaned result", iteration=1)).record assert len(turn.commands) == 0 assert any("Unhandled SDK message type" in record.message for record in caplog.records), ( @@ -372,7 +372,7 @@ async def mock_query(prompt, options): await agent.start(str(tmp_path)) with caplog.at_level(logging.DEBUG): - turn = await agent.communicate("Duplicate results") + turn = (await agent.communicate("Duplicate results", iteration=1)).record assert len(turn.commands) == 1 cmd = turn.commands[0] @@ -416,7 +416,7 @@ async def mock_query(prompt, options): # Capture both INFO and WARNING levels with caplog.at_level(logging.INFO): - turn = await agent.communicate("Missing result") + turn = (await agent.communicate("Missing result", iteration=1)).record assert len(turn.commands) == 1 assert turn.commands[0].result_status == "unknown" @@ -465,7 +465,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("Non-dict input") + turn = (await agent.communicate("Non-dict input", iteration=1)).record # Should capture the command without crashing assert len(turn.commands) == 1 @@ -512,7 +512,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - await agent.communicate("Non-dict stream", stream_callback=CollectingCallback()) + await agent.communicate("Non-dict stream", iteration=1, stream_callback=CollectingCallback()) from coder_eval.streaming.events import ToolStartEvent @@ -555,7 +555,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("Read a file") + turn = (await agent.communicate("Read a file", iteration=1)).record # Verify assistant_turns list is populated assert len(turn.messages) == 1 @@ -607,7 +607,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("Test ordering") + turn = (await agent.communicate("Test ordering", iteration=1)).record assert len(turn.messages) == 1 aturn = turn.messages[0] @@ -654,7 +654,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("Test thinking") + turn = (await agent.communicate("Test thinking", iteration=1)).record assert len(turn.messages) == 1 aturn = turn.messages[0] @@ -708,7 +708,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("Multiple turns") + turn = (await agent.communicate("Multiple turns", iteration=1)).record # Verify both turns captured (a trailing ReconciliationMessage may # follow when the authoritative total exceeds the per-message sum). @@ -762,7 +762,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("Test execution timing") + turn = (await agent.communicate("Test execution timing", iteration=1)).record assert len(turn.commands) == 1 cmd = turn.commands[0] @@ -812,7 +812,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("Test command indexing") + turn = (await agent.communicate("Test command indexing", iteration=1)).record # Both commands should reference assistant turn index 0 assert len(turn.commands) == 2 @@ -927,7 +927,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("hi") + turn = (await agent.communicate("hi", iteration=1)).record assert len(turn.messages) == 1 aturn = turn.messages[0] assert isinstance(aturn, AssistantMessage) @@ -999,7 +999,7 @@ async def mock_query(prompt, options): agent_module.query = mock_query try: await agent.start(str(tmp_path)) - turn = await agent.communicate("hi") + turn = (await agent.communicate("hi", iteration=1)).record assistants = [m for m in turn.messages if isinstance(m, AssistantMessage)] assert len(assistants) == 3 # A-text, A-tool, B-text a_text, a_tool, b_text = assistants @@ -1061,7 +1061,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("hi") + turn = (await agent.communicate("hi", iteration=1)).record assert len(turn.messages) == 2 first, second = turn.messages assert isinstance(first, AssistantMessage) @@ -1123,7 +1123,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("hi") + turn = (await agent.communicate("hi", iteration=1)).record aturn = turn.messages[0] assert isinstance(aturn, AssistantMessage) # delta wins over partial 0; ResultMessage fallback is suppressed. @@ -1166,7 +1166,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("hi") + turn = (await agent.communicate("hi", iteration=1)).record aturn = turn.messages[0] assert isinstance(aturn, AssistantMessage) # Backfilled from ResultMessage. @@ -1216,7 +1216,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("hi") + turn = (await agent.communicate("hi", iteration=1)).record assert len(turn.messages) == 2 a1, a2 = turn.messages assert isinstance(a1, AssistantMessage) @@ -1264,7 +1264,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn = await agent.communicate("hi") + turn = (await agent.communicate("hi", iteration=1)).record # A trailing ReconciliationMessage may follow the two assistant # emissions (here the snapshot total differs from the per-message sum). assistant_msgs = [m for m in turn.messages if isinstance(m, AssistantMessage)] @@ -1580,7 +1580,7 @@ async def mock_query(prompt, options): agent = agent_module.ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE)) await agent.start(str(tmp_path)) seen: list[Any] = [] - await agent.communicate("go", stream_callback=SimpleNamespace(on_event=seen.append)) + await agent.communicate("go", iteration=1, stream_callback=SimpleNamespace(on_event=seen.append)) assert_bracket_on_the_clock(seen) @@ -1601,6 +1601,6 @@ async def mock_query(prompt, options): agent = agent_module.ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE)) await agent.start(str(tmp_path)) - record = await agent.communicate("go") + record = (await agent.communicate("go", iteration=1)).record assert_overhead_is_measured(record) diff --git a/tests/test_agent_telemetry_advanced.py b/tests/test_agent_telemetry_advanced.py index 2345485f..b45ffeea 100644 --- a/tests/test_agent_telemetry_advanced.py +++ b/tests/test_agent_telemetry_advanced.py @@ -73,7 +73,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn_record = await agent.communicate("test prompt") + turn_record = (await agent.communicate("test prompt", iteration=1)).record # Verify command has 'unknown' status assert len(turn_record.commands) == 1 @@ -125,7 +125,7 @@ async def mock_query(prompt, options): await agent.start(str(tmp_path)) with caplog.at_level(logging.DEBUG): - turn_record = await agent.communicate("test prompt") + turn_record = (await agent.communicate("test prompt", iteration=1)).record # Verify last result wins assert len(turn_record.commands) == 1 @@ -177,7 +177,7 @@ async def mock_query(prompt, options): await agent.start(str(tmp_path)) with caplog.at_level(logging.WARNING): - turn_record = await agent.communicate("test prompt") + turn_record = (await agent.communicate("test prompt", iteration=1)).record # Verify all three commands recorded assert len(turn_record.commands) == 3 @@ -231,7 +231,7 @@ async def mock_query(prompt, options): await agent.start(str(tmp_path)) with caplog.at_level(logging.WARNING): - turn_record = await agent.communicate("test prompt") + turn_record = (await agent.communicate("test prompt", iteration=1)).record # The orphaned result is handled gracefully: a single synthesized # 'unknown' command captures it (rather than being silently dropped). @@ -283,7 +283,7 @@ async def mock_query(prompt, options): try: await agent.start(str(tmp_path)) - turn_record = await agent.communicate("test prompt") + turn_record = (await agent.communicate("test prompt", iteration=1)).record assert len(turn_record.commands) == 3 diff --git a/tests/test_agent_timeout.py b/tests/test_agent_timeout.py index 5777dea0..160285f8 100644 --- a/tests/test_agent_timeout.py +++ b/tests/test_agent_timeout.py @@ -16,6 +16,7 @@ from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.errors.timeout import TurnTimeoutError from coder_eval.models import AgentKind, parse_agent_config +from coder_eval.streaming.events import AgentEndStatus def _make_agent() -> ClaudeCodeAgent: @@ -91,11 +92,13 @@ async def mock_query(prompt, options, transport=None): ): mock_transport_cls.return_value = MagicMock() - with pytest.raises(TurnTimeoutError) as exc: - # 100ms watchdog — bypasses AgentConfig's ge=10 validator - # because we pass it directly as a call arg, not via config. - await agent.communicate("prompt", timeout=0.1) + # 100ms watchdog — bypasses AgentConfig's ge=10 validator + # because we pass it directly as a call arg, not via config. + outcome = await agent.communicate("prompt", iteration=1, timeout=0.1) + assert outcome.status is AgentEndStatus.TIMEOUT + with pytest.raises(TurnTimeoutError) as exc: + outcome.record_or_raise(timeout_seconds=0.1) assert exc.value.timeout_seconds == 0.1 assert exc.value.layer == "turn" mock_kill_transport.assert_called() @@ -127,8 +130,9 @@ async def mock_query(prompt, options, transport=None): ): mock_transport_cls.return_value = MagicMock() - with pytest.raises(TurnTimeoutError): - await agent.communicate("prompt", timeout=0.1) + outcome = await agent.communicate("prompt", iteration=1, timeout=0.1) + + assert outcome.status is AgentEndStatus.TIMEOUT @pytest.mark.asyncio @@ -181,10 +185,11 @@ async def swap_active_transport_before_watchdog() -> None: patch("coder_eval.agents.claude_code_agent.query", mock_query), ): swapper = asyncio.create_task(swap_active_transport_before_watchdog()) - with pytest.raises(TurnTimeoutError): - await agent.communicate("A", timeout=0.1) + outcome = await agent.communicate("A", iteration=1, timeout=0.1) await swapper + assert outcome.status is AgentEndStatus.TIMEOUT + # The watchdog must have killed transport A (its captured target), # not transport B (which happened to be in self._active_transport # when the watchdog fired). @@ -230,10 +235,11 @@ def fake_monotonic() -> float: patch("coder_eval.agents.claude_code_agent.time.monotonic", fake_monotonic), ): mock_transport_cls.return_value = MagicMock() - # Must return a TurnRecord, not raise TurnTimeoutError, even - # though wall-clock is way past deadline. - result = await agent.communicate("prompt", timeout=1.0) - assert result is not None + # Must complete cleanly, not TIMEOUT, even though wall-clock is + # way past deadline. + outcome = await agent.communicate("prompt", iteration=1, timeout=1.0) + assert outcome.status is AgentEndStatus.COMPLETED + assert outcome.record is not None @pytest.mark.asyncio @@ -256,7 +262,7 @@ async def mock_query(prompt, options): patch("coder_eval.agents.claude_code_agent.SubprocessCLITransport") as mock_transport_cls, patch("coder_eval.agents.claude_code_agent.query", mock_query), ): - await agent.communicate("prompt") # no timeout + await agent.communicate("prompt", iteration=1) # no timeout mock_transport_cls.assert_not_called() @@ -299,7 +305,7 @@ async def mock_query(prompt, options): await asyncio.sleep(30) with patch("coder_eval.agents.claude_code_agent.query", mock_query): - turn = asyncio.create_task(agent.communicate("prompt")) + turn = asyncio.create_task(agent.communicate("prompt", iteration=1)) await streaming.wait() turn.cancel() with pytest.raises(asyncio.CancelledError): diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 5db1f558..6cb3ca3c 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -47,6 +47,7 @@ from coder_eval.orchestration.experiment import _apply_cli_overrides, resolve_task_for_variant from coder_eval.orchestration.task_loader import resolve_initial_prompt_file from coder_eval.orchestrator import Orchestrator +from coder_eval.streaming.events import AgentEndStatus def _none_task(criteria=None, **overrides) -> TaskDefinition: @@ -67,11 +68,14 @@ def _none_task(criteria=None, **overrides) -> TaskDefinition: @pytest.mark.asyncio class TestNoOpAgent: async def test_lifecycle_returns_empty_turn(self) -> None: - """start/communicate/stop are no-ops; communicate returns an empty TurnRecord.""" + """start/communicate/stop are no-ops; communicate returns a COMPLETED outcome + wrapping an empty TurnRecord.""" agent = NoOpAgent(NoneAgentConfig(type=AgentKind.NONE)) await agent.start("/tmp/whatever") - turn = await agent.communicate("this prompt is ignored") + outcome = await agent.communicate("this prompt is ignored", iteration=1) + assert outcome.status is AgentEndStatus.COMPLETED + turn = outcome.record assert turn.agent_output == "" assert turn.iteration == 1 assert turn.commands == [] diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index 20d18c76..09b7e18e 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -278,7 +278,7 @@ async def test_communicate_maps_steps_to_turn_record(): ), ] agent = _agent_with_steps(steps) - tr = await agent.communicate("make hello.py") + tr = (await agent.communicate("make hello.py", iteration=1)).record assert tr.crashed is False assert tr.agent_output == "All done." @@ -314,7 +314,6 @@ async def test_communicate_maps_steps_to_turn_record(): # distinctness (a uuid would pass) does not. ids = [m.message_id for m in tr.messages if isinstance(m, AssistantMessage)] assert ids == ["antigravity-1-msg-0", "antigravity-1-msg-1", "antigravity-1-msg-2"] - assert agent.pending_turn is None # success path leaves no partial async def test_communicate_normalizes_arg_keys_and_strips_done_only_results(): @@ -344,7 +343,7 @@ async def test_communicate_normalizes_arg_keys_and_strips_done_only_results(): ), _step("TEXT_RESPONSE", "DONE", content="done", complete=True, usage=_usage(10, 0, 1, 0)), ] - tr = await _agent_with_steps(steps).communicate("x") + tr = (await _agent_with_steps(steps).communicate("x", iteration=1)).record ls = next(c for c in tr.commands if c.tool_name == "LS") assert ls.parameters == {"path": "/work"} # renamed, results stripped web = next(c for c in tr.commands if c.tool_name == "WebSearch") @@ -364,14 +363,13 @@ async def test_communicate_records_tool_error_from_nonzero_exit(): ), _step("TEXT_RESPONSE", "DONE", content="done", complete=True, usage=_usage(510, 0, 3, 0)), ] - tr = await _agent_with_steps(steps).communicate("run it") + tr = (await _agent_with_steps(steps).communicate("run it", iteration=1)).record bash = next(c for c in tr.commands if c.tool_name == "Bash") assert bash.result_status == "error" async def test_communicate_crash_sets_pending_partial_turn(): - """A mid-stream SDK error raises AgentCrashError and leaves a crashed partial.""" - from coder_eval.errors import AgentCrashError + """A mid-stream SDK error crashes the turn and leaves a crashed partial record.""" class _Boom: last_response = "" @@ -389,17 +387,14 @@ async def receive_steps(self): agent.working_directory = Path("/tmp") agent._sdk_agent = SimpleNamespace(conversation=_Boom(), is_started=True) - with pytest.raises(AgentCrashError): - await agent.communicate("x") - assert agent.pending_turn is not None - assert agent.pending_turn.crashed is True + outcome = await agent.communicate("x", iteration=1) - await agent.discard_pending_turn() - assert agent.pending_turn is None + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.record.crashed is True async def test_communicate_timeout_sets_pending_partial_turn(monkeypatch): - """A turn timeout raises TurnTimeoutError and leaves a crashed partial turn. + """A turn timeout ends the turn TIMEOUT and leaves a crashed partial record. Drives the timeout branch deterministically: a fake watchdog fires its ``on_timeout`` callback synchronously on entry (setting ``state.timeout_hit``, @@ -408,8 +403,6 @@ async def test_communicate_timeout_sets_pending_partial_turn(monkeypatch): """ import asyncio - from coder_eval.errors import TurnTimeoutError - monkeypatch.setattr("coder_eval.agents.antigravity_agent.ThreadedWatchdog", _FiringWatchdog) class _Cancelled: @@ -428,19 +421,16 @@ async def receive_steps(self): agent.working_directory = Path("/tmp") agent._sdk_agent = SimpleNamespace(conversation=_Cancelled(), is_started=True) - with pytest.raises(TurnTimeoutError): - await agent.communicate("x", timeout=30.0) - assert agent.pending_turn is not None - assert agent.pending_turn.crashed is True + outcome = await agent.communicate("x", iteration=1, timeout=30.0) - await agent.discard_pending_turn() - assert agent.pending_turn is None + assert outcome.status is AgentEndStatus.TIMEOUT + assert outcome.record.crashed is True async def test_communicate_requires_started_agent(): agent = AntigravityAgent(parse_agent_config(type="antigravity")) with pytest.raises(RuntimeError, match="not started"): - await agent.communicate("x") + await agent.communicate("x", iteration=1) def _install_fake_sdk(monkeypatch, sdk_agent_cls) -> None: @@ -534,7 +524,7 @@ async def _sleep_should_not_be_called(_seconds: float) -> None: _step("TEXT_RESPONSE", "DONE", content="done", complete=True, usage=_usage(10, 0, 1, 0)), ] agent = _agent_with_steps(steps) - tr = await agent.communicate("run it") + tr = (await agent.communicate("run it", iteration=1)).record conv = agent._sdk_agent.conversation assert conv.receive_steps_call_count == 1 @@ -565,7 +555,7 @@ async def _sleep_should_not_be_called(_seconds: float) -> None: _step("TEXT_RESPONSE", "DONE", content="waiting on you", complete=True, usage=_usage(10, 0, 1, 0)), ] agent = _agent_with_steps(steps) - tr = await agent.communicate("do it") + tr = (await agent.communicate("do it", iteration=1)).record conv = agent._sdk_agent.conversation assert conv.receive_steps_call_count == 1 # poll loop never entered @@ -627,7 +617,7 @@ async def _record_sleep(seconds: float) -> None: ), ] agent = _agent_with_steps([batch1, batch2]) - tr = await agent.communicate("do it") + tr = (await agent.communicate("do it", iteration=1)).record assert sleep_calls == [antigravity_agent._BACKGROUND_POLL_INTERVAL_SECONDS] bash = next(c for c in tr.commands if c.tool_name == "Bash") @@ -671,7 +661,7 @@ async def test_communicate_resolves_backgrounded_tool_call_with_no_id(monkeypatc _step("TEXT_RESPONSE", "DONE", content="All finished.", complete=True, usage=_usage(5, 0, 1, 0)), ] agent = _agent_with_steps([batch1, batch2]) - tr = await agent.communicate("do it") + tr = (await agent.communicate("do it", iteration=1)).record assert agent._sdk_agent.conversation.receive_steps_call_count == 2 # closed on the first poll, not the cap bash = next(c for c in tr.commands if c.tool_name == "Bash") @@ -707,7 +697,7 @@ async def test_id_less_tool_calls_in_different_trajectories_do_not_collide(): _step("TEXT_RESPONSE", "DONE", content="done", complete=True, usage=_usage(10, 0, 1, 0)), ] agent = _agent_with_steps(steps) - tr = await agent.communicate("do two things") + tr = (await agent.communicate("do two things", iteration=1)).record bash_calls = [c for c in tr.commands if c.tool_name == "Bash"] assert len(bash_calls) == 2 # distinct cids, not collapsed into one @@ -765,7 +755,7 @@ async def _record_sleep(seconds: float) -> None: _step("TEXT_RESPONSE", "DONE", content="all done", complete=True, usage=_usage(10, 0, 1, 0)), ] agent = _agent_with_steps([batch1, batch2, batch3]) - tr = await agent.communicate("do two things") + tr = (await agent.communicate("do two things", iteration=1)).record assert len(sleep_calls) == 2 # exactly two poll cycles, one per backgrounded job bash_calls = [c for c in tr.commands if c.tool_name == "Bash"] @@ -800,7 +790,7 @@ async def _record_sleep(seconds: float) -> None: # empty batch (see _FakeConversation's docstring, matching the real SDK) -- # the orphan is never closed, simulating a job whose state never changes. agent = _agent_with_steps([never_closing]) - tr = await agent.communicate("do it forever") + tr = (await agent.communicate("do it forever", iteration=1)).record assert len(sleep_calls) == 3 # exactly _MAX_BACKGROUND_POLLS, not infinite bash = next(c for c in tr.commands if c.tool_name == "Bash") @@ -845,7 +835,7 @@ async def test_communicate_finalizes_gracefully_under_a_realistic_turn_timeout(m ] agent = _agent_with_steps([never_closing]) - tr = await agent.communicate("do it forever", timeout=300.0) # the real default turn_timeout + tr = (await agent.communicate("do it forever", iteration=1, timeout=300.0)).record # the real default turn_timeout # Finalized and graded -- no TurnTimeoutError, no crash. assert tr is not None @@ -905,20 +895,18 @@ async def _fire_watchdog_on_second_sleep(seconds: float) -> None: ), _step("TEXT_RESPONSE", "DONE", content="waiting...", complete=True, usage=_usage(10, 0, 1, 0)), ] - from coder_eval.errors import TurnTimeoutError agent = _agent_with_steps([never_closing]) - with pytest.raises(TurnTimeoutError): - await agent.communicate("do it forever", timeout=30.0) + outcome = await agent.communicate("do it forever", iteration=1, timeout=30.0) + assert outcome.status is AgentEndStatus.TIMEOUT # Stopped right after the sleep that flipped timeout_hit -- NOT the (patched) cap of 50. assert len(sleep_calls) == 2 # 1 initial drain + 1 poll re-drain (after sleep #1) -- the mid-loop # `if state.timeout_hit: break` skips the re-drain that would otherwise # follow sleep #2, so no 3rd receive_steps() call happens. assert agent._sdk_agent.conversation.receive_steps_call_count == 2 - assert agent.pending_turn is not None - bash = next(c for c in agent.pending_turn.commands if c.tool_name == "Bash") + bash = next(c for c in outcome.record.commands if c.tool_name == "Bash") assert bash.result_status == "unknown" @@ -959,7 +947,7 @@ def should_stop() -> StopReason | None: # None for batch1's 2 steps; a reason on the post-sleep check return StopReason.EARLY_CRITERION if call_count > 2 else None - await agent.communicate("do it", should_stop=should_stop) + await agent.communicate("do it", iteration=1, should_stop=should_stop) assert conv.receive_steps_call_count == 1 # the poll's re-drain never happened assert conv.cancel_call_count == 1 @@ -1048,13 +1036,14 @@ async def test_communicate_recovers_from_transient_reentrancy_after_cooperative_ agent.working_directory = Path("/tmp") agent._sdk_agent = SimpleNamespace(conversation=conversation, is_started=True) - await agent.communicate("do it", should_stop=lambda: StopReason.EARLY_CRITERION) # breaks after the first step + # breaks after the first step + await agent.communicate("do it", iteration=1, should_stop=lambda: StopReason.EARLY_CRITERION) - # Without the retry, this second call raises AgentCrashError wrapping the - # fake's RuntimeError (verified live before the fix landed). With it, the + # Without the retry, this second call crashes wrapping the fake's + # RuntimeError (verified live before the fix landed). With it, the # transient window clears within a couple of asyncio.sleep(0) yields and # the second turn's real content is delivered, not silently dropped. - tr = await agent.communicate("do it again") + tr = (await agent.communicate("do it again", iteration=2)).record assert tr.agent_output == "second turn" @@ -1090,8 +1079,6 @@ async def test_a_runtime_error_after_a_step_is_not_retried_as_reentrancy(monkeyp """Only an error raised before the first step is the re-entrancy window. A RuntimeError while processing a pulled step is a real failure: retrying it would re-pull the stream and emit the same steps again.""" - from coder_eval.errors import AgentCrashError - conversation_pulls = 0 class _Conversation: @@ -1116,8 +1103,10 @@ def _boom(self, step): agent.working_directory = __import__("pathlib").Path("/tmp") agent._sdk_agent = SimpleNamespace(conversation=_Conversation(), is_started=True) - with pytest.raises(AgentCrashError, match="reducer bug"): - await agent.communicate("do it") + outcome = await agent.communicate("do it", iteration=1) + + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "reducer bug" in outcome.error assert conversation_pulls == 1 @@ -1133,7 +1122,6 @@ async def test_communicate_poll_budget_exhausted_finalizes_via_existing_timeout_ callback and the ``CancelledError`` it triggers are the same causal event, not two independently-timed ones.""" from coder_eval.agents import antigravity_agent - from coder_eval.errors import TurnTimeoutError monkeypatch.setattr(antigravity_agent.asyncio, "sleep", _no_sleep) monkeypatch.setattr("coder_eval.agents.antigravity_agent.ThreadedWatchdog", _WatchdogFiresLater) @@ -1173,14 +1161,11 @@ async def cancel(self): conversation = _FiresWatchdogThenCancelsOnSecondDrain() agent._sdk_agent = SimpleNamespace(conversation=conversation, is_started=True) - with pytest.raises(TurnTimeoutError): - await agent.communicate("x", timeout=30.0) - assert conversation.call_count == 2 # the re-drain genuinely ran, not skipped - assert agent.pending_turn is not None - assert agent.pending_turn.crashed is True + outcome = await agent.communicate("x", iteration=1, timeout=30.0) - await agent.discard_pending_turn() - assert agent.pending_turn is None + assert outcome.status is AgentEndStatus.TIMEOUT + assert conversation.call_count == 2 # the re-drain genuinely ran, not skipped + assert outcome.record.crashed is True # --- env_path_prepend / mock-CLI PATH shadowing ----------------------------------- @@ -1507,7 +1492,7 @@ async def test_should_stop_reason_ends_the_turn_with_its_status(reason, status, agent = _agent_with_steps(_tool_steps(5)) capture = _EndCapture() - record = await agent.communicate("go", stream_callback=capture, should_stop=lambda: reason) + record = (await agent.communicate("go", iteration=1, stream_callback=capture, should_stop=lambda: reason)).record assert capture.end is not None assert capture.end.status is status @@ -1527,7 +1512,7 @@ def should_stop() -> StopReason | None: polls += 1 return StopReason.TOOL_CALL_CAP if polls >= 2 else None - record = await agent.communicate("go", should_stop=should_stop) + record = (await agent.communicate("go", iteration=1, should_stop=should_stop)).record assert len(record.commands) == 1 assert record.commands[0].result_status == "success" @@ -1537,7 +1522,7 @@ def should_stop() -> StopReason | None: async def test_no_reason_consumes_every_step(): agent = _agent_with_steps(_tool_steps(4)) - record = await agent.communicate("go", should_stop=lambda: None) + record = (await agent.communicate("go", iteration=1, should_stop=lambda: None)).record assert len(record.commands) == 4 assert record.tool_calls_exhausted is False @@ -1582,7 +1567,8 @@ async def test_cap_reached_on_a_poll_redrain_stops_polling(monkeypatch): conv = agent._sdk_agent.conversation monitor = TurnMonitor("t", [], limits=RunLimits(max_tool_calls=2)) - record = await agent.communicate("go", stream_callback=monitor, should_stop=monitor.should_stop) + outcome = await agent.communicate("go", iteration=1, stream_callback=monitor, should_stop=monitor.should_stop) + record = outcome.record assert monitor.stop_reason is StopReason.TOOL_CALL_CAP assert record.tool_calls_exhausted is True @@ -1743,7 +1729,7 @@ async def test_concurrent_tools_do_not_over_subtract(monkeypatch): closes, _step("THINKING", "DONE", thinking="second", usage=_usage(100, 0, 5, 5)), ] - record = await _agent_with_steps(steps).communicate("go") + record = (await _agent_with_steps(steps).communicate("go", iteration=1)).record second = _assistant(record)[1] tools = [c for c in record.commands if c.tool_id.startswith("t")] @@ -1770,7 +1756,7 @@ async def test_generation_window_is_measured_not_zero(): _step("THINKING", "DONE", thinking="first", usage=_usage(100, 0, 5, 5)), _step("THINKING", "DONE", thinking="second", usage=_usage(120, 0, 6, 4)), ] - record = await _agent_with_steps(steps).communicate("go") + record = (await _agent_with_steps(steps).communicate("go", iteration=1)).record messages = _assistant(record) assert len(messages) == 2 @@ -1787,7 +1773,7 @@ async def test_consecutive_windows_chain_end_to_start(): _step("THINKING", "DONE", thinking="b", usage=_usage(100, 0, 5, 5)), _step("TEXT_RESPONSE", "DONE", content="c", content_delta="c", complete=True, usage=_usage(100, 0, 5, 0)), ] - record = await _agent_with_steps(steps).communicate("go") + record = (await _agent_with_steps(steps).communicate("go", iteration=1)).record messages = _assistant(record) assert len(messages) == 3 @@ -1823,7 +1809,7 @@ async def test_tool_execution_is_subtracted_from_the_window(monkeypatch): "TEXT_RESPONSE", "DONE", content="done", content_delta="done", complete=True, usage=_usage(200, 0, 10, 0) ), ] - record = await _agent_with_steps(steps).communicate("go") + record = (await _agent_with_steps(steps).communicate("go", iteration=1)).record messages = _assistant(record) assert len(messages) == 2 @@ -1889,7 +1875,7 @@ async def test_a_straddling_tool_is_charged_only_for_its_in_window_part(monkeypa ), _step("THINKING", "DONE", thinking="second", usage=_usage(100, 0, 5, 5)), ] - record = await _agent_with_steps(steps).communicate("go") + record = (await _agent_with_steps(steps).communicate("go", iteration=1)).record second = _assistant(record)[1] slow = next(c for c in record.commands if c.tool_id == "t1") @@ -1932,7 +1918,7 @@ async def test_a_tool_still_open_at_the_flush_is_not_generation_time(monkeypatch ), _step("THINKING", "DONE", thinking="second", usage=_usage(100, 0, 5, 5)), ] - record = await _agent_with_steps(steps).communicate("go") + record = (await _agent_with_steps(steps).communicate("go", iteration=1)).record first = _assistant(record)[0] slow = next(c for c in record.commands if c.tool_id == "t1") @@ -1964,10 +1950,10 @@ async def test_a_no_op_flush_does_not_move_the_mark(monkeypatch): # clock read, so it must leave the window — and therefore the real # generation's recorded bounds — byte-identical. _install_clock(monkeypatch, _Clock()) - without = _assistant(await _agent_with_steps([real]).communicate("go")) + without = _assistant((await _agent_with_steps([real]).communicate("go", iteration=1)).record) _install_clock(monkeypatch, _Clock()) - with_empty = _assistant(await _agent_with_steps([empty, real]).communicate("go")) + with_empty = _assistant((await _agent_with_steps([empty, real]).communicate("go", iteration=1)).record) assert len(with_empty) == 1, "the empty generation must not produce a message" assert with_empty[0].started_at == without[0].started_at @@ -2005,7 +1991,7 @@ async def test_generation_and_tool_time_account_for_the_turn(): "TEXT_RESPONSE", "DONE", content="done", content_delta="done", complete=True, usage=_usage(200, 0, 10, 0) ), ] - record = await _agent_with_steps(steps).communicate("go") + record = (await _agent_with_steps(steps).communicate("go", iteration=1)).record gen_ms = sum(m.generation_duration_ms or 0.0 for m in _assistant(record)) tool_ms = sum(c.duration_ms or 0.0 for c in record.commands) @@ -2058,7 +2044,7 @@ async def test_timing_change_moves_no_token_bucket(): "TEXT_RESPONSE", "DONE", content="done", content_delta="done", complete=True, usage=_usage(1300, 0, 30, 0) ), ] - record = await _agent_with_steps(steps).communicate("go") + record = (await _agent_with_steps(steps).communicate("go", iteration=1)).record assert record.token_usage is not None assert record.token_usage.output_tokens == (10 + 20) + (15 + 5) + (30 + 0) @@ -2098,7 +2084,7 @@ async def test_the_published_window_reconciles_to_its_own_bounds(monkeypatch): ), _step("THINKING", "DONE", thinking="second", usage=_usage(100, 0, 5, 5)), ] - record = await _agent_with_steps(steps).communicate("go") + record = (await _agent_with_steps(steps).communicate("go", iteration=1)).record second = _assistant(record)[1] spans = [ @@ -2138,7 +2124,7 @@ async def test_the_window_is_measured_without_relying_on_the_negative_clamp(monk ), _step("THINKING", "DONE", thinking="second", usage=_usage(100, 0, 5, 5)), ] - record = await _agent_with_steps(steps).communicate("go") + record = (await _agent_with_steps(steps).communicate("go", iteration=1)).record second = _assistant(record)[1] assert second.generation_duration_ms > 0.0 @@ -2158,13 +2144,13 @@ async def test_each_turn_gets_a_fresh_clock(): """ step = _step("THINKING", "DONE", thinking="a", usage=_usage(100, 0, 5, 5)) agent = _agent_with_steps([step]) - first = _assistant(await agent.communicate("go")) + first = _assistant((await agent.communicate("go", iteration=1)).record) # The fake conversation yields one batch and is then spent, so borrow a # fresh one. The agent INSTANCE is deliberately the same: what is under # test is that its second turn builds its own clock rather than inheriting # the first turn's origin. agent._sdk_agent = _agent_with_steps([step])._sdk_agent - second = _assistant(await agent.communicate("again")) + second = _assistant((await agent.communicate("again", iteration=2)).record) assert first and second # Re-anchored: the later turn's window opens after the earlier one closed. @@ -2327,7 +2313,9 @@ def _steps(): async def test_both_brackets_are_stamped_from_the_injected_clock(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) seen: list[Any] = [] - await _agent_with_steps(self._steps()).communicate("go", stream_callback=SimpleNamespace(on_event=seen.append)) + await _agent_with_steps(self._steps()).communicate( + "go", iteration=1, stream_callback=SimpleNamespace(on_event=seen.append) + ) assert_bracket_on_the_clock(seen) @@ -2341,6 +2329,6 @@ async def test_the_tail_is_a_measurement_rather_than_a_clamped_zero(self, monkey holds its process across turns and so has the shortest real tail. """ monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) - record = await _agent_with_steps(self._steps()).communicate("go") + record = (await _agent_with_steps(self._steps()).communicate("go", iteration=1)).record assert_overhead_is_measured(record) diff --git a/tests/test_byoa_plugin_live.py b/tests/test_byoa_plugin_live.py index 44bd8718..a7abf3c3 100644 --- a/tests/test_byoa_plugin_live.py +++ b/tests/test_byoa_plugin_live.py @@ -81,10 +81,13 @@ async def test_byoa_plugin_agent_runs_real_turn(demo_plugin_registered, tmp_path await agent.start(str(tmp_path)) try: - record = await agent.communicate( - "Reply with exactly the word PONG and nothing else.", - timeout=120, - ) + record = ( + await agent.communicate( + "Reply with exactly the word PONG and nothing else.", + iteration=1, + timeout=120, + ) + ).record finally: await agent.stop() diff --git a/tests/test_claude_settings_enforcement_live.py b/tests/test_claude_settings_enforcement_live.py index 618b9b12..415597bf 100644 --- a/tests/test_claude_settings_enforcement_live.py +++ b/tests/test_claude_settings_enforcement_live.py @@ -102,7 +102,7 @@ async def _run_single_turn(sandbox_dir: Path, prompt: str, claude_settings: dict agent = ClaudeCodeAgent(config, route=_route_from_env()) await agent.start(str(sandbox_dir)) try: - turn = await agent.communicate(prompt, timeout=60.0) + turn = (await agent.communicate(prompt, iteration=1, timeout=60.0)).record finally: await agent.stop() return agent, turn diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index 9a96898b..297880d4 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -37,7 +37,6 @@ def test_codex_agent_initialization(self): assert agent.config == config assert agent.codex_client is None assert agent.get_state() == AgentState.WORKING - assert agent.pending_turn is None def test_codex_agent_with_disallowed_tools(self): """Test initialization with disallowed_tools.""" @@ -505,29 +504,6 @@ def test_build_thread_options_with_permission_and_tools(self): assert options["approval_mode"] == ApprovalMode.deny_all -@pytest.mark.asyncio -async def test_discard_pending_turn(): - """Test discard_pending_turn clears pending_turn and decrements iteration.""" - from coder_eval.models import TurnRecord - - config = parse_agent_config(type=AgentKind.CODEX) - agent = CodexAgent(config) - - partial = TurnRecord( - iteration=1, - user_input="test", - agent_output="", - crashed=True, - ) - agent._iteration = 1 - agent.pending_turn = partial - - await agent.discard_pending_turn() - - assert agent.pending_turn is None - assert agent._iteration == 0 - - def test_get_state_returns_current_state(): """Test get_state returns the agent's current state.""" config = parse_agent_config(type=AgentKind.CODEX) @@ -556,7 +532,6 @@ def test_get_state_returns_current_state(): from openai_codex.generated.v2_all import Turn, TurnCompletedNotification # noqa: E402 -from coder_eval.errors import AgentCrashError, TurnTimeoutError # noqa: E402 from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus, StopReason # noqa: E402 @@ -677,7 +652,7 @@ async def test_happy_path_collects_output_commands_and_tokens(self): ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("do it") + record = (await agent.communicate("do it", iteration=1)).record assert record.agent_output == "Hello world" # One shell command + one file change both recorded as telemetry. @@ -695,7 +670,6 @@ async def test_happy_path_collects_output_commands_and_tokens(self): assert record.token_usage.cache_read_input_tokens == 8 assert record.token_usage.input_tokens == 100 assert agent.get_state() == AgentState.WORKING - assert agent.pending_turn is None assert agent._active_turn_handle is None async def test_state_resets_to_working_after_a_prior_error(self): @@ -703,7 +677,7 @@ async def test_state_resets_to_working_after_a_prior_error(self): agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) agent._state = AgentState.ERROR - await agent.communicate("retry") + await agent.communicate("retry", iteration=1) assert agent.get_state() == AgentState.WORKING @@ -797,7 +771,7 @@ async def test_per_message_uncached_input_on_first_submessage_only(self): _turn_completed(), ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("go") + record = (await agent.communicate("go", iteration=1)).record assistant_msgs = [m for m in record.messages if hasattr(m, "cache_creation_tokens")] assert assistant_msgs, "expected at least one AssistantMessage" @@ -834,7 +808,7 @@ def _gen(item_id: str, text: str, inp: int, out: int, cached: int, tot_in: int, _turn_completed(), ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("go") + record = (await agent.communicate("go", iteration=1)).record msgs = [m for m in record.messages if hasattr(m, "cache_creation_tokens")] # Gen 1 (cold): all 1000 fresh is uncached input, no cache. @@ -859,17 +833,12 @@ async def test_missing_turn_completed_raises_agent_crash_with_pending(self): notifications = [_delta("partial")] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - with pytest.raises(AgentCrashError): - await agent.communicate("do it") + outcome = await agent.communicate("do it", iteration=1) - assert agent.pending_turn is not None - assert agent.pending_turn.crashed is True + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.record.crashed is True assert agent.get_state() == AgentState.ERROR - # discard rolls back the iteration bump (flag-only branch still works). - await agent.discard_pending_turn() - assert agent._iteration == 0 - async def test_thread_start_failure_funnels_through_crash(self): agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX)) agent.working_directory = __import__("pathlib").Path(".") @@ -881,12 +850,9 @@ def _boom(**_kwargs): agent.codex_client.thread_start = _boom - with pytest.raises(AgentCrashError): - await agent.communicate("do it") + outcome = await agent.communicate("do it", iteration=1) - assert agent.pending_turn is not None - await agent.discard_pending_turn() - assert agent._iteration == 0 + assert outcome.status is AgentEndStatus.CRASHED class _RaisingStream: @@ -945,17 +911,17 @@ async def test_crash_emits_crashed_end_with_token_fallback(self): def _cb(event): captured.append(event) - with pytest.raises(AgentCrashError): - await agent.communicate("do it", stream_callback=SimpleNamespace(on_event=_cb)) + outcome = await agent.communicate("do it", iteration=1, stream_callback=SimpleNamespace(on_event=_cb)) + + assert outcome.status is AgentEndStatus.CRASHED # A CRASHED AgentEndEvent closes the event tree. end_events = [e for e in captured if isinstance(e, AgentEndEvent)] assert end_events and end_events[-1].status == AgentEndStatus.CRASHED - # The pending turn carries the tokens captured before the crash (fallback). - assert agent.pending_turn is not None - assert agent.pending_turn.crashed is True - tu = agent.pending_turn.token_usage + # The turn's record carries the tokens captured before the crash (fallback). + assert outcome.record.crashed is True + tu = outcome.record.token_usage assert tu is not None # Fresh slice 100 - 8 = 92 -> uncached_input; cached 8 -> cache_read; out 40; no cache-write. assert tu.uncached_input_tokens == 92 @@ -1115,7 +1081,7 @@ async def test_reasoning_lands_on_thinking_submessage_with_placeholder(self): _turn_completed(), ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("think then answer") + record = (await agent.communicate("think then answer", iteration=1)).record assistant = [m for m in record.messages if isinstance(m, AssistantMessage)] assert assistant @@ -1203,7 +1169,7 @@ async def test_spawn_nests_subagent_message_and_records_tool_calls(self, monkeyp ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("delegate it") + record = (await agent.communicate("delegate it", iteration=1)).record # Both collab calls surface as Agent tool calls in the transcript. agent_calls = [c for c in record.commands if c.tool_name == "Agent"] @@ -1230,7 +1196,7 @@ async def test_wait_only_records_no_subagent(self, monkeypatch, tmp_path): ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("wait") + record = (await agent.communicate("wait", iteration=1)).record # No spawn → no nested sub-agent generations. assert not any(getattr(m, "parent_tool_use_id", None) for m in record.messages) @@ -1251,7 +1217,7 @@ async def test_orphan_tool_started_without_completed_is_closed_unresolved(self, ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("go") + record = (await agent.communicate("go", iteration=1)).record # The orphan survives as a named command with 'unknown' status (not dropped, # not "unknown" tool name). @@ -1338,7 +1304,7 @@ async def test_inner_shell_command_recovered_and_nested(self, monkeypatch, tmp_p ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("delegate it") + record = (await agent.communicate("delegate it", iteration=1)).record # The sub-agent's inner Bash command is recovered as telemetry... bash = [c for c in record.commands if c.tool_name == "Bash"] @@ -1368,7 +1334,7 @@ async def test_missing_rollout_is_silently_skipped(self, monkeypatch, tmp_path): ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("delegate it") + record = (await agent.communicate("delegate it", iteration=1)).record assert not [c for c in record.commands if c.tool_name == "Bash"] assert any(getattr(m, "parent_tool_use_id", None) == "call_spawn" for m in record.messages) @@ -1411,7 +1377,7 @@ async def test_generations_carry_per_generation_tokens_in_order(self, monkeypatc ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("delegate it") + record = (await agent.communicate("delegate it", iteration=1)).record nested = [m for m in record.messages if getattr(m, "parent_tool_use_id", None) == "call_spawn"] assert len(nested) == 2 @@ -1469,7 +1435,7 @@ async def test_fold_does_not_double_count_parent_plus_child(self, monkeypatch, t ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("delegate it") + record = (await agent.communicate("delegate it", iteration=1)).record tu = record.token_usage assert tu is not None @@ -1508,7 +1474,7 @@ async def test_mcp_and_websearch_items_become_tool_calls(self): ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("use tools") + record = (await agent.communicate("use tools", iteration=1)).record names = sorted(c.tool_name for c in record.commands) assert names == ["Mcp", "WebSearch"] @@ -1534,7 +1500,7 @@ async def test_failed_mcp_call_records_error(self): ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("use tools") + record = (await agent.communicate("use tools", iteration=1)).record tel = next(c for c in record.commands if c.tool_name == "Mcp") assert tel.result_status == "error" @@ -1550,7 +1516,7 @@ async def test_unknown_tool_kind_falls_back_to_raw_type_name(self): ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("use tools") + record = (await agent.communicate("use tools", iteration=1)).record assert any(c.tool_name == "someNewTool" for c in record.commands) @@ -1573,7 +1539,7 @@ async def test_failed_file_change_records_error_telemetry_without_crashing(self) ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("write out.txt") + record = (await agent.communicate("write out.txt", iteration=1)).record # Turn completes normally (no crash / no retry), but the Write telemetry # honestly reflects the failure. @@ -1606,11 +1572,10 @@ async def test_timeout_raises_turn_timeout_with_pending(self): agent.codex_client = SimpleNamespace(close=lambda: None) agent.thread = SimpleNamespace(turn=lambda _u: handle) - with pytest.raises(TurnTimeoutError): - await agent.communicate("do it", timeout=0.2) + outcome = await agent.communicate("do it", iteration=1, timeout=0.2) - assert agent.pending_turn is not None - assert agent.pending_turn.crashed is True + assert outcome.status is AgentEndStatus.TIMEOUT + assert outcome.record.crashed is True assert agent.get_state() == AgentState.ERROR @@ -1646,26 +1611,12 @@ async def test_post_watchdog_timeout_sets_error_state_and_partial(self, monkeypa # normally and the post-watchdog `if state.timeout_hit:` block fires. monkeypatch.setattr("coder_eval.agents.codex_agent.ThreadedWatchdog", _ImmediateTimeoutWatchdog) - with pytest.raises(TurnTimeoutError): - await agent.communicate("do it", timeout=30.0) + outcome = await agent.communicate("do it", iteration=1, timeout=30.0) + assert outcome.status is AgentEndStatus.TIMEOUT # The fix: this race path now ends in ERROR (would be WORKING before). assert agent.get_state() == AgentState.ERROR - assert agent.pending_turn is not None - assert agent.pending_turn.crashed is True - - -class TestDiscardIdempotency: - async def test_double_discard_only_rolls_back_once(self): - agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX)) - agent._iteration = 3 - agent._iteration_was_incremented = True - - await agent.discard_pending_turn() - assert agent._iteration == 2 - - await agent.discard_pending_turn() - assert agent._iteration == 2 # idempotent + assert outcome.record.crashed is True class TestTeardown: @@ -2192,9 +2143,11 @@ async def test_tool_call_cap_ends_tool_calls_exhausted(self): agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(5)) capture = _EndCapture() - record = await agent.communicate( - "go", stream_callback=capture, should_stop=_stop_on_call(1, StopReason.TOOL_CALL_CAP) - ) + record = ( + await agent.communicate( + "go", iteration=1, stream_callback=capture, should_stop=_stop_on_call(1, StopReason.TOOL_CALL_CAP) + ) + ).record assert capture.end is not None assert capture.end.status is AgentEndStatus.TOOL_CALLS_EXHAUSTED @@ -2206,9 +2159,11 @@ async def test_token_budget_ends_token_budget_exceeded(self): agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(5)) capture = _EndCapture() - record = await agent.communicate( - "go", stream_callback=capture, should_stop=_stop_on_call(1, StopReason.TOKEN_BUDGET) - ) + record = ( + await agent.communicate( + "go", iteration=1, stream_callback=capture, should_stop=_stop_on_call(1, StopReason.TOKEN_BUDGET) + ) + ).record assert capture.end is not None assert capture.end.status is AgentEndStatus.TOKEN_BUDGET_EXCEEDED @@ -2219,7 +2174,8 @@ async def test_stop_keeps_the_deciding_call_complete(self): """A stop polled after a call's completion keeps that call's result.""" agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(3)) - record = await agent.communicate("go", should_stop=_stop_on_call(2, StopReason.TOOL_CALL_CAP)) + outcome = await agent.communicate("go", iteration=1, should_stop=_stop_on_call(2, StopReason.TOOL_CALL_CAP)) + record = outcome.record assert len(record.commands) == 1 assert record.commands[0].result_status == "success" @@ -2228,14 +2184,14 @@ async def test_stop_interrupts_the_in_flight_turn(self): """Best-effort server-side interrupt, so the stop actually ends spend.""" agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(5)) - await agent.communicate("go", should_stop=_stop_on_call(2, StopReason.TOOL_CALL_CAP)) + await agent.communicate("go", iteration=1, should_stop=_stop_on_call(2, StopReason.TOOL_CALL_CAP)) assert agent.thread.last_handle.interrupted is True async def test_no_reason_consumes_the_whole_stream(self): agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._cmd_notifications(4)) - record = await agent.communicate("go", should_stop=lambda: None) + record = (await agent.communicate("go", iteration=1, should_stop=lambda: None)).record assert len(record.commands) == 4 assert record.tool_calls_exhausted is False @@ -2272,7 +2228,9 @@ async def test_cap_stop_still_folds_sub_agent_tokens(self, monkeypatch, tmp_path child = "019e0000-eeee-7000-8000-000000000005" agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._delegation(tmp_path, child)) - record = await agent.communicate("delegate it", should_stop=_stop_on_call(4, StopReason.TOOL_CALL_CAP)) + record = ( + await agent.communicate("delegate it", iteration=1, should_stop=_stop_on_call(4, StopReason.TOOL_CALL_CAP)) + ).record assert record.tool_calls_exhausted is True assert [c for c in record.commands if c.tool_name == "Bash"] @@ -2288,7 +2246,11 @@ async def test_early_criterion_stop_skips_sub_agent_recovery(self, monkeypatch, child = "019e0000-ffff-7000-8000-000000000006" agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), self._delegation(tmp_path, child)) - record = await agent.communicate("delegate it", should_stop=_stop_on_call(4, StopReason.EARLY_CRITERION)) + record = ( + await agent.communicate( + "delegate it", iteration=1, should_stop=_stop_on_call(4, StopReason.EARLY_CRITERION) + ) + ).record assert record.tool_calls_exhausted is False assert not [c for c in record.commands if c.tool_name == "Bash"] @@ -2328,7 +2290,7 @@ async def test_stamps_reach_the_record_with_the_right_values(self): _turn_completed(), ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("go") + record = (await agent.communicate("go", iteration=1)).record cmd = next(c for c in record.commands if c.tool_id == "cmd_1") assert cmd.execution_started_at == datetime.fromtimestamp(_BOUNDS_EPOCH_MS / 1000) @@ -2346,7 +2308,7 @@ async def test_an_orphaned_tool_keeps_its_known_start_but_no_end(self): _turn_completed(), ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("go") + record = (await agent.communicate("go", iteration=1)).record orphan = next(c for c in record.commands if c.tool_id == "cmd_orphan") assert orphan.result_status == "unknown" @@ -2379,7 +2341,7 @@ async def test_a_tool_only_emission_reports_no_generation_time(self): _turn_completed(), ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("go") + record = (await agent.communicate("go", iteration=1)).record assistant = [m for m in record.messages if m.role == "assistant"] cmd = next(c for c in record.commands if c.tool_id == "cmd_1") @@ -2401,7 +2363,7 @@ async def test_generation_plus_tool_exec_does_not_exceed_the_window(self): _turn_completed(), ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("go") + record = (await agent.communicate("go", iteration=1)).record assistant = [m for m in record.messages if m.role == "assistant"] gen_ms = sum(m.generation_duration_ms or 0.0 for m in assistant) @@ -2440,7 +2402,7 @@ async def test_the_published_window_reconciles_to_its_own_bounds(self): _turn_completed(), ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("go") + record = (await agent.communicate("go", iteration=1)).record assistant = [m for m in record.messages if m.role == "assistant"] spans = [ @@ -2485,7 +2447,7 @@ async def test_the_gap_before_an_emission_is_its_generation_time(self): _turn_completed(), ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("go") + record = (await agent.communicate("go", iteration=1)).record assistant = [m for m in record.messages if m.role == "assistant"] assert len(assistant) == 2 @@ -2509,7 +2471,7 @@ async def test_tool_time_is_still_excluded_from_a_tiled_window(self): _turn_completed(), ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("go") + record = (await agent.communicate("go", iteration=1)).record gen_ms = sum(m.generation_duration_ms or 0.0 for m in record.messages if m.role == "assistant") tool_ms = sum(c.duration_ms or 0.0 for c in record.commands) @@ -2721,7 +2683,7 @@ async def test_the_split_survives_end_to_end_through_communicate(self): _turn_completed(), ] agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) - record = await agent.communicate("think then answer") + record = (await agent.communicate("think then answer", iteration=1)).record assistant = [m for m in record.messages if m.role == "assistant"] assert len(assistant) == 2, "expected a thinking and an action sub-message" diff --git a/tests/test_codex_agent_live.py b/tests/test_codex_agent_live.py index 8e7f2c34..b31b5f74 100644 --- a/tests/test_codex_agent_live.py +++ b/tests/test_codex_agent_live.py @@ -51,10 +51,13 @@ async def test_codex_live_produces_text(tmp_path): agent = _make_agent() await agent.start(str(tmp_path)) try: - record = await agent.communicate( - "Reply with exactly the word PONG and nothing else.", - timeout=120, - ) + record = ( + await agent.communicate( + "Reply with exactly the word PONG and nothing else.", + iteration=1, + timeout=120, + ) + ).record finally: await agent.stop() @@ -69,10 +72,13 @@ async def test_codex_live_runs_shell_command_captured_as_telemetry(tmp_path): agent = _make_agent() await agent.start(str(tmp_path)) try: - record = await agent.communicate( - "Run the shell command `echo coder-eval-live` and report its output.", - timeout=120, - ) + record = ( + await agent.communicate( + "Run the shell command `echo coder-eval-live` and report its output.", + iteration=1, + timeout=120, + ) + ).record finally: await agent.stop() @@ -93,10 +99,13 @@ async def test_codex_live_edits_file_and_records_telemetry(tmp_path): agent = _make_agent() await agent.start(str(tmp_path)) try: - record = await agent.communicate( - "Create a file named hello.txt in the current directory containing the text 'hi'.", - timeout=120, - ) + record = ( + await agent.communicate( + "Create a file named hello.txt in the current directory containing the text 'hi'.", + iteration=1, + timeout=120, + ) + ).record finally: await agent.stop() @@ -127,19 +136,21 @@ def on_event(self, event): agent = _make_agent() await agent.start(str(tmp_path)) try: - record = await agent.communicate( - "Run `echo one`, then `echo two`, then `echo three`, each as a separate shell command, " - "then create three files a.txt, b.txt and c.txt.", - timeout=180, - stream_callback=sink, - should_stop=lambda: StopReason.EARLY_CRITERION if sink.tool_started else None, - ) + record = ( + await agent.communicate( + "Run `echo one`, then `echo two`, then `echo three`, each as a separate shell command, " + "then create three files a.txt, b.txt and c.txt.", + iteration=1, + timeout=180, + stream_callback=sink, + should_stop=lambda: StopReason.EARLY_CRITERION if sink.tool_started else None, + ) + ).record finally: await agent.stop() # Clean cooperative stop: no crash, no pending partial, STOPPED_EARLY status. assert record.crashed is False - assert agent.pending_turn is None assert sink.ends, "expected an AgentEndEvent" assert sink.ends[-1].status == AgentEndStatus.STOPPED_EARLY @@ -150,7 +161,7 @@ async def test_codex_live_token_usage_populated(tmp_path): agent = _make_agent() await agent.start(str(tmp_path)) try: - record = await agent.communicate("Say hello.", timeout=120) + record = (await agent.communicate("Say hello.", iteration=1, timeout=120)).record finally: await agent.stop() diff --git a/tests/test_early_stop.py b/tests/test_early_stop.py index 869c3585..837f3bbb 100644 --- a/tests/test_early_stop.py +++ b/tests/test_early_stop.py @@ -43,7 +43,6 @@ from coder_eval.criteria import CriterionRegistry, init_criteria from coder_eval.criteria.command_executed import CommandExecutedChecker from coder_eval.criteria.skill_triggered import SkillTriggeredChecker, _engaged_skill_names -from coder_eval.errors import AgentCrashError, TurnTimeoutError from coder_eval.models import ( AgentKind, ApiBackend, @@ -81,6 +80,7 @@ from coder_eval.reports import ReportGenerator from coder_eval.reports.html import _render_criteria, _render_header from coder_eval.run_record import eval_result_to_task_dict +from coder_eval.streaming.emitter import TurnOutcome from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -91,6 +91,7 @@ ToolStartEvent, TurnEndStatus, TurnStartEvent, + end_status_for, ) from tests._fixtures.live_criteria import FROZEN_TS, make_command, make_turn from tests.fixtures.harness_stubs import config_for_kind, stub_contract @@ -1132,14 +1133,14 @@ def on_event(self, event: Any) -> None: async def _run_claude_communicate( *, stop_after: int | None = None, never: bool = False, n_messages: int = 3 -) -> tuple[ClaudeCodeAgent, TurnRecord, _EventSink, int]: +) -> tuple[ClaudeCodeAgent, TurnOutcome, _EventSink, int]: """Drive ``ClaudeCodeAgent.communicate`` over a mocked ``query`` yielding ``n_messages`` dummy messages. ``stop_after``: build a should_stop that returns ``EARLY_CRITERION`` once that many messages have been pulled (checked after each dispatch). ``never``: pass an explicit always-None should_stop. Neither: pass ``should_stop=None``. Returns - ``(agent, record, sink, pulled_count)``. + ``(agent, outcome, sink, pulled_count)``. """ config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") agent = ClaudeCodeAgent(config) @@ -1169,8 +1170,8 @@ async def mock_query(prompt: Any, options: Any, transport: Any = None) -> Any: sink = _EventSink() with patch("coder_eval.agents.claude_code_agent.query", mock_query): - record = await agent.communicate("prompt", stream_callback=sink, should_stop=should_stop) - return agent, record, sink, pulled["n"] + outcome = await agent.communicate("prompt", iteration=1, stream_callback=sink, should_stop=should_stop) + return agent, outcome, sink, pulled["n"] def _agent_end_events(sink: _EventSink) -> list[AgentEndEvent]: @@ -1190,14 +1191,13 @@ def __exit__(self, *args: Any) -> bool: return False -async def _run_claude_communicate_timeout() -> tuple[ClaudeCodeAgent, _EventSink, BaseException | None]: +async def _run_claude_communicate_timeout() -> tuple[ClaudeCodeAgent, _EventSink, TurnOutcome]: """Drive ``communicate`` with a slow query (50ms) against a 10ms deadline AND a should_stop returning ``EARLY_CRITERION`` — the deadline guard must win. Returns - ``(agent, sink, raised_exception)``.""" + ``(agent, sink, outcome)``.""" config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") agent = ClaudeCodeAgent(config) sink = _EventSink() - raised: BaseException | None = None with tempfile.TemporaryDirectory() as tmpdir: await agent.start(tmpdir) @@ -1209,13 +1209,10 @@ async def slow_query(prompt: Any, options: Any, transport: Any = None) -> Any: patch("coder_eval.agents.claude_code_agent.query", slow_query), patch("coder_eval.agents.claude_code_agent.ThreadedWatchdog", _NoopWatchdog), ): - try: - await agent.communicate( - "p", stream_callback=sink, timeout=0.01, should_stop=lambda: StopReason.EARLY_CRITERION - ) - except TurnTimeoutError as exc: - raised = exc - return agent, sink, raised + outcome = await agent.communicate( + "p", iteration=1, stream_callback=sink, timeout=0.01, should_stop=lambda: StopReason.EARLY_CRITERION + ) + return agent, sink, outcome class TestNewFixtureTasksResolve: @@ -1252,31 +1249,31 @@ def test_turnendstatus_conversion_from_agentendstatus(self) -> None: assert TurnEndStatus(AgentEndStatus.STOPPED_EARLY.value) == TurnEndStatus.STOPPED_EARLY async def test_stop_after_first_dispatched_message(self) -> None: - _agent, record, sink, pulled = await _run_claude_communicate(stop_after=1, n_messages=3) + _agent, outcome, sink, pulled = await _run_claude_communicate(stop_after=1, n_messages=3) # The deciding message is kept; the next is never pulled. assert pulled == 1 - assert record.crashed is False + assert outcome.record.crashed is False ends = _agent_end_events(sink) assert len(ends) == 1 assert ends[0].status == AgentEndStatus.STOPPED_EARLY assert ends[0].crashed is False async def test_early_stop_is_clean_not_crashed(self) -> None: - agent, record, _sink, _pulled = await _run_claude_communicate(stop_after=1) - # A clean stop: no partial pending_turn, no ERROR state, no raise (we got here). - assert agent.pending_turn is None + agent, outcome, _sink, _pulled = await _run_claude_communicate(stop_after=1) + # A clean stop: no CRASHED/TIMEOUT status, no ERROR state. + assert outcome.status is AgentEndStatus.STOPPED_EARLY assert agent.get_state().value != "error" - assert record.crashed is False + assert outcome.record.crashed is False async def test_should_stop_none_consumes_full_stream(self) -> None: - _agent, _record, sink, pulled = await _run_claude_communicate(stop_after=None, n_messages=3) + _agent, _outcome, sink, pulled = await _run_claude_communicate(stop_after=None, n_messages=3) assert pulled == 3 ends = _agent_end_events(sink) assert len(ends) == 1 assert ends[0].status == AgentEndStatus.COMPLETED async def test_should_stop_returning_none_consumes_full_stream(self) -> None: - _agent, _record, sink, pulled = await _run_claude_communicate(never=True, n_messages=3) + _agent, _outcome, sink, pulled = await _run_claude_communicate(never=True, n_messages=3) assert pulled == 3 assert _agent_end_events(sink)[0].status == AgentEndStatus.COMPLETED @@ -1284,14 +1281,14 @@ async def test_timeout_beats_stop_precedence(self) -> None: # Both signals live in one turn: a deadline breach AND should_stop=True. # The top-of-loop deadline guard returns BEFORE dispatch, so the stop # check is never reached — TIMEOUT wins over the pending stop. - agent, sink, raised = await _run_claude_communicate_timeout() - assert isinstance(raised, TurnTimeoutError) + _agent, sink, outcome = await _run_claude_communicate_timeout() + assert outcome.status is AgentEndStatus.TIMEOUT ends = _agent_end_events(sink) assert len(ends) == 1 assert ends[0].status == AgentEndStatus.TIMEOUT assert ends[0].crashed is True # The crashed partial is preserved for the orchestrator to drain. - assert agent.pending_turn is not None and agent.pending_turn.crashed is True + assert outcome.record.crashed is True # STOPPED_EARLY must NOT appear — the stop lost the race. assert AgentEndStatus.STOPPED_EARLY not in {e.status for e in ends} @@ -2276,12 +2273,12 @@ def test_decision_budget_accumulates_across_retry_attempts(self) -> None: class _ScriptedAgent: """Duck-typed agent: replays scripted events through the callback, polling ``should_stop`` after each and breaking on a reason (mirrors the real - message-boundary cut). Returns a fixed ``TurnRecord``.""" + message-boundary cut). Returns a fixed ``TurnRecord`` wrapped in a + ``TurnOutcome`` whose status reflects whether ``should_stop`` fired.""" def __init__(self, events: list[Any], turn: TurnRecord) -> None: self._events = events self._turn = turn - self.pending_turn: TurnRecord | None = None self.delivered = 0 def get_sdk_options(self) -> dict[str, Any] | None: @@ -2291,17 +2288,22 @@ async def communicate( self, prompt: str, *, + iteration: int, stream_callback: Any = None, timeout: float | None = None, should_stop: Callable[[], StopReason | None] | None = None, - ) -> TurnRecord: + ) -> TurnOutcome: + reason: StopReason | None = None for event in self._events: if stream_callback is not None: stream_callback.on_event(event) self.delivered += 1 - if should_stop is not None and should_stop(): - break - return self._turn + if should_stop is not None: + reason = should_stop() + if reason is not None: + break + status = end_status_for(reason) if reason is not None else AgentEndStatus.COMPLETED + return TurnOutcome(record=self._turn, status=status, error=None) async def _run_wiring( @@ -2912,7 +2914,7 @@ async def _run_codex_communicate( stop_after: int | None = None, never: bool = False, timeout: float | None = None, -) -> tuple[CodexAgent, TurnRecord, _EventSink, _FakeCodexStream, _FakeCodexTurnHandle]: +) -> tuple[CodexAgent, TurnOutcome, _EventSink, _FakeCodexStream, _FakeCodexTurnHandle]: """Drive ``CodexAgent.communicate`` over a fake notification stream. ``stop_after``: should_stop returns ``EARLY_CRITERION`` once that many @@ -2934,38 +2936,40 @@ async def _run_codex_communicate( sink = _EventSink() with patch.object(_CodexTurnState, "on_turn_completed", _stub_on_turn_completed): - record = await agent.communicate("prompt", stream_callback=sink, timeout=timeout, should_stop=should_stop) - return agent, record, sink, stream, handle + outcome = await agent.communicate( + "prompt", iteration=1, stream_callback=sink, timeout=timeout, should_stop=should_stop + ) + return agent, outcome, sink, stream, handle class TestCodexCooperativeStopSeam: async def test_stop_after_first_dispatched_notification(self) -> None: notifications = [_codex_delta(0), _codex_delta(1), _codex_delta(2), _codex_completed()] - agent, record, sink, stream, handle = await _run_codex_communicate(notifications=notifications, stop_after=1) + agent, outcome, sink, stream, handle = await _run_codex_communicate(notifications=notifications, stop_after=1) # The deciding notification is kept; the next is never pulled. assert stream.iter.pulled == 1 # The in-flight turn was interrupted exactly once (server-side spend cut). assert handle.interrupts == 1 - assert record.crashed is False + assert outcome.record.crashed is False ends = _agent_end_events(sink) assert len(ends) == 1 assert ends[0].status == AgentEndStatus.STOPPED_EARLY assert ends[0].crashed is False - # A clean stop: no partial pending_turn, no ERROR state, no raise. - assert agent.pending_turn is None + # A clean stop: no CRASHED/TIMEOUT status, no ERROR state. + assert outcome.status is AgentEndStatus.STOPPED_EARLY assert agent.get_state().value != "error" async def test_should_stop_none_consumes_full_stream(self) -> None: notifications = [_codex_delta(0), _codex_delta(1), _codex_completed()] - _agent, record, sink, stream, handle = await _run_codex_communicate(notifications=notifications) + _agent, outcome, sink, stream, handle = await _run_codex_communicate(notifications=notifications) assert stream.iter.pulled == 3 assert handle.interrupts == 0 - assert record.crashed is False + assert outcome.record.crashed is False assert _agent_end_events(sink)[0].status == AgentEndStatus.COMPLETED async def test_should_stop_returning_none_consumes_full_stream(self) -> None: notifications = [_codex_delta(0), _codex_delta(1), _codex_completed()] - _agent, _record, sink, stream, _handle = await _run_codex_communicate(notifications=notifications, never=True) + _agent, _outcome, sink, stream, _handle = await _run_codex_communicate(notifications=notifications, never=True) assert stream.iter.pulled == 3 assert _agent_end_events(sink)[0].status == AgentEndStatus.COMPLETED @@ -2973,21 +2977,22 @@ async def test_stop_before_turn_completed_does_not_raise(self) -> None: # The stream is cut before any turn/completed: turn_result is None, but the # stop makes the "turn never completed" raise conditional — no crash. notifications = [_codex_delta(0), _codex_delta(1), _codex_delta(2)] - _agent, record, sink, _stream, _handle = await _run_codex_communicate(notifications=notifications, stop_after=1) - assert record.crashed is False + _agent, outcome, sink, _stream, _handle = await _run_codex_communicate( + notifications=notifications, stop_after=1 + ) + assert outcome.record.crashed is False assert _agent_end_events(sink)[0].status == AgentEndStatus.STOPPED_EARLY - async def test_stream_dying_without_stop_still_raises(self) -> None: + async def test_stream_dying_without_stop_still_crashes(self) -> None: # Regression guard: a stream that ends with NO turn/completed and NO stop # is still a crash (the RuntimeError survives for genuine stream deaths). agent = _codex_agent() stream = _FakeCodexStream([_codex_delta(0)]) agent.thread = SimpleNamespace(turn=lambda _prompt: _FakeCodexTurnHandle(stream)) - with pytest.raises(AgentCrashError, match="did not complete"): - await agent.communicate("prompt", stream_callback=_EventSink(), should_stop=None) - assert agent.pending_turn is not None - assert agent.pending_turn.crashed is True - await agent.discard_pending_turn() + outcome = await agent.communicate("prompt", iteration=1, stream_callback=_EventSink(), should_stop=None) + assert outcome.status is AgentEndStatus.CRASHED + assert "did not complete" in (outcome.error or "") + assert outcome.record.crashed is True async def test_timeout_beats_stop_precedence(self, monkeypatch: pytest.MonkeyPatch) -> None: # Both signals in one turn: the watchdog fires (timeout_hit) AND should_stop @@ -3008,18 +3013,17 @@ def __exit__(self, *_exc: Any) -> bool: stream = _FakeCodexStream([_codex_delta(0), _codex_delta(1)]) agent.thread = SimpleNamespace(turn=lambda _prompt: _FakeCodexTurnHandle(stream)) sink = _EventSink() - with pytest.raises(TurnTimeoutError): - await agent.communicate( - "prompt", stream_callback=sink, timeout=30.0, should_stop=lambda: StopReason.EARLY_CRITERION - ) + outcome = await agent.communicate( + "prompt", iteration=1, stream_callback=sink, timeout=30.0, should_stop=lambda: StopReason.EARLY_CRITERION + ) + assert outcome.status is AgentEndStatus.TIMEOUT ends = _agent_end_events(sink) assert len(ends) == 1 assert ends[0].status == AgentEndStatus.TIMEOUT assert ends[0].crashed is True - assert agent.pending_turn is not None and agent.pending_turn.crashed is True + assert outcome.record.crashed is True # STOPPED_EARLY must NOT appear — the stop lost the race. assert AgentEndStatus.STOPPED_EARLY not in {e.status for e in ends} - await agent.discard_pending_turn() async def test_post_stop_exception_stays_clean(self, monkeypatch: pytest.MonkeyPatch) -> None: # The retry-poisoning gap: an exception AFTER the cooperative break (here: @@ -3031,14 +3035,16 @@ def _boom(self: Any) -> None: monkeypatch.setattr(_CodexTurnState, "close_open_tools", _boom) notifications = [_codex_delta(0), _codex_delta(1)] - agent, record, sink, _stream, _handle = await _run_codex_communicate(notifications=notifications, stop_after=1) - # No AgentCrashError raised (we got a record back), clean STOPPED_EARLY. - assert record.crashed is False + _agent, outcome, sink, _stream, _handle = await _run_codex_communicate( + notifications=notifications, stop_after=1 + ) + # No crash outcome (we got a clean record back), clean STOPPED_EARLY. + assert outcome.record.crashed is False ends = _agent_end_events(sink) assert len(ends) == 1 assert ends[0].status == AgentEndStatus.STOPPED_EARLY assert ends[0].crashed is False - assert agent.pending_turn is None + assert outcome.status is AgentEndStatus.STOPPED_EARLY async def test_post_stop_cleanup_exception_without_stop_still_crashes( self, monkeypatch: pytest.MonkeyPatch @@ -3052,9 +3058,9 @@ def _boom(self: Any) -> None: agent = _codex_agent() stream = _FakeCodexStream([_codex_delta(0)]) agent.thread = SimpleNamespace(turn=lambda _prompt: _FakeCodexTurnHandle(stream)) - with pytest.raises(AgentCrashError): - await agent.communicate("prompt", stream_callback=_EventSink(), should_stop=None) - await agent.discard_pending_turn() + outcome = await agent.communicate("prompt", iteration=1, stream_callback=_EventSink(), should_stop=None) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.record.crashed is True async def test_stopped_turn_skips_subagent_recovery(self) -> None: # A stopped turn must not attempt rollout recovery: children may have no @@ -3078,6 +3084,7 @@ def _capturing_init(self: Any, *args: Any, **kwargs: Any) -> None: ): await agent.communicate( "prompt", + iteration=1, stream_callback=_EventSink(), should_stop=lambda: StopReason.EARLY_CRITERION if stream.iter.pulled >= 1 else None, ) @@ -3145,7 +3152,7 @@ async def _run_antigravity_communicate( stop_after: int | None = None, never: bool = False, cancel_raises: bool = False, -) -> tuple[AntigravityAgent, TurnRecord, _EventSink, _CountingConversation]: +) -> tuple[AntigravityAgent, TurnOutcome, _EventSink, _CountingConversation]: """Drive ``AntigravityAgent.communicate`` over a fake step stream (same stop_after / never / None semantics as the Claude and Codex drivers).""" conversation = _CountingConversation([_ag_step(i) for i in range(n_steps)], cancel_raises=cancel_raises) @@ -3160,45 +3167,45 @@ async def _run_antigravity_communicate( should_stop = None sink = _EventSink() - record = await agent.communicate("prompt", stream_callback=sink, should_stop=should_stop) - return agent, record, sink, conversation + outcome = await agent.communicate("prompt", iteration=1, stream_callback=sink, should_stop=should_stop) + return agent, outcome, sink, conversation class TestAntigravityCooperativeStopSeam: async def test_stop_after_first_processed_step(self) -> None: - agent, record, sink, conversation = await _run_antigravity_communicate(stop_after=1, n_steps=3) + agent, outcome, sink, conversation = await _run_antigravity_communicate(stop_after=1, n_steps=3) # The deciding step is kept; the next is never pulled. assert conversation.yielded == 1 # The conversation was cancelled once (best-effort server-side cut). assert conversation.cancels == 1 - assert record.crashed is False + assert outcome.record.crashed is False ends = _agent_end_events(sink) assert len(ends) == 1 assert ends[0].status == AgentEndStatus.STOPPED_EARLY assert ends[0].crashed is False - assert agent.pending_turn is None + assert outcome.status is AgentEndStatus.STOPPED_EARLY assert agent.get_state().value != "error" async def test_should_stop_none_consumes_full_stream(self) -> None: - _agent, record, sink, conversation = await _run_antigravity_communicate(n_steps=3) + _agent, outcome, sink, conversation = await _run_antigravity_communicate(n_steps=3) assert conversation.yielded == 3 assert conversation.cancels == 0 - assert record.crashed is False + assert outcome.record.crashed is False assert _agent_end_events(sink)[0].status == AgentEndStatus.COMPLETED async def test_should_stop_returning_none_consumes_full_stream(self) -> None: - _agent, _record, sink, conversation = await _run_antigravity_communicate(never=True, n_steps=3) + _agent, _outcome, sink, conversation = await _run_antigravity_communicate(never=True, n_steps=3) assert conversation.yielded == 3 assert _agent_end_events(sink)[0].status == AgentEndStatus.COMPLETED async def test_raising_cancel_still_stops_clean(self) -> None: # conversation.cancel() is best-effort: a raising cancel must not escalate # a stopped turn to a crash. - agent, record, sink, conversation = await _run_antigravity_communicate(stop_after=1, cancel_raises=True) + _agent, outcome, sink, conversation = await _run_antigravity_communicate(stop_after=1, cancel_raises=True) assert conversation.cancels == 1 - assert record.crashed is False + assert outcome.record.crashed is False assert _agent_end_events(sink)[0].status == AgentEndStatus.STOPPED_EARLY - assert agent.pending_turn is None + assert outcome.status is AgentEndStatus.STOPPED_EARLY async def test_timeout_beats_stop_precedence(self, monkeypatch: pytest.MonkeyPatch) -> None: class _FiringWatchdog: @@ -3216,17 +3223,16 @@ def __exit__(self, *_exc: Any) -> bool: conversation = _CountingConversation([_ag_step(0), _ag_step(1)]) agent = _antigravity_agent(conversation) sink = _EventSink() - with pytest.raises(TurnTimeoutError): - await agent.communicate( - "prompt", stream_callback=sink, timeout=30.0, should_stop=lambda: StopReason.EARLY_CRITERION - ) + outcome = await agent.communicate( + "prompt", iteration=1, stream_callback=sink, timeout=30.0, should_stop=lambda: StopReason.EARLY_CRITERION + ) + assert outcome.status is AgentEndStatus.TIMEOUT ends = _agent_end_events(sink) assert len(ends) == 1 assert ends[0].status == AgentEndStatus.TIMEOUT assert ends[0].crashed is True - assert agent.pending_turn is not None and agent.pending_turn.crashed is True + assert outcome.record.crashed is True assert AgentEndStatus.STOPPED_EARLY not in {e.status for e in ends} - await agent.discard_pending_turn() async def test_post_stop_exception_stays_clean(self, monkeypatch: pytest.MonkeyPatch) -> None: # The retry-poisoning gap, antigravity flavor: an exception raised by @@ -3246,13 +3252,13 @@ def __exit__(self, exc_type: Any, *_exc: Any) -> bool: return False monkeypatch.setattr("coder_eval.agents.antigravity_agent.ThreadedWatchdog", _ExplodingExitWatchdog) - agent, record, sink, _conversation = await _run_antigravity_communicate(stop_after=1) - assert record.crashed is False + _agent, outcome, sink, _conversation = await _run_antigravity_communicate(stop_after=1) + assert outcome.record.crashed is False ends = _agent_end_events(sink) assert len(ends) == 1 assert ends[0].status == AgentEndStatus.STOPPED_EARLY assert ends[0].crashed is False - assert agent.pending_turn is None + assert outcome.status is AgentEndStatus.STOPPED_EARLY async def test_post_stop_cleanup_exception_without_stop_still_crashes( self, monkeypatch: pytest.MonkeyPatch @@ -3274,9 +3280,9 @@ def __exit__(self, exc_type: Any, *_exc: Any) -> bool: monkeypatch.setattr("coder_eval.agents.antigravity_agent.ThreadedWatchdog", _ExplodingExitWatchdog) conversation = _CountingConversation([_ag_step(0)]) agent = _antigravity_agent(conversation) - with pytest.raises(AgentCrashError): - await agent.communicate("prompt", stream_callback=_EventSink(), should_stop=None) - await agent.discard_pending_turn() + outcome = await agent.communicate("prompt", iteration=1, stream_callback=_EventSink(), should_stop=None) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.record.crashed is True # --------------------------------------------------------------------------- # diff --git a/tests/test_harness_conformance.py b/tests/test_harness_conformance.py index 5a150bfe..64a10ef4 100644 --- a/tests/test_harness_conformance.py +++ b/tests/test_harness_conformance.py @@ -93,7 +93,7 @@ async def fake_query(prompt: str, options: Any): claude = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE, **agent)) await claude.start(str(tmp_path), plugin_root=plugin_root) with patch("coder_eval.agents.claude_code_agent.query", fake_query): - await claude.communicate(USER_TURN) + await claude.communicate(USER_TURN, iteration=1) return captured["options"], captured["prompt"] @@ -145,7 +145,7 @@ def turn(self, user_input: str): # type: ignore[override] codex = _started_agent(parse_agent_config(type=AgentKind.CODEX, system_prompt=MARKER), [_turn_completed()]) options = codex._build_thread_options() codex.thread = _RecordingThread([_turn_completed()]) - await codex.communicate(USER_TURN) + await codex.communicate(USER_TURN, iteration=1) assert options["developer_instructions"] == MARKER assert turn_inputs == [USER_TURN] @@ -205,7 +205,7 @@ async def send(self, prompt: str, **kwargs: Any) -> None: agent = AntigravityAgent(parse_agent_config(type=AgentKind.ANTIGRAVITY, system_prompt=MARKER)) agent.working_directory = tmp_path agent._sdk_agent = SimpleNamespace(conversation=_RecordingConversation([done]), is_started=True) - await agent.communicate(USER_TURN) + await agent.communicate(USER_TURN, iteration=1) assert sent == [USER_TURN] @@ -451,7 +451,7 @@ async def fake_query(prompt: Any, options: Any, transport: Any = None): claude = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE)) await claude.start(str(tmp_path)) with patch("coder_eval.agents.claude_code_agent.query", fake_query): - await claude.communicate(USER_TURN, stream_callback=stop, should_stop=stop) + await claude.communicate(USER_TURN, iteration=1, stream_callback=stop, should_stop=stop) return [getattr(e.content[0], "id", None) for e in pulled if hasattr(e, "content")] @@ -482,7 +482,7 @@ def turn(self, _user_input: str): # type: ignore[override] codex = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) codex.thread = _RecordingThread(notifications) - await codex.communicate(USER_TURN, stream_callback=stop, should_stop=stop) + await codex.communicate(USER_TURN, iteration=1, stream_callback=stop, should_stop=stop) return [n.payload.item.root.id for n in pulled] @@ -514,7 +514,7 @@ async def receive_steps(self): agent = AntigravityAgent(parse_agent_config(type=AgentKind.ANTIGRAVITY)) agent.working_directory = tmp_path agent._sdk_agent = SimpleNamespace(conversation=_RecordingConversation([]), is_started=True) - await agent.communicate(USER_TURN, stream_callback=stop, should_stop=stop) + await agent.communicate(USER_TURN, iteration=1, stream_callback=stop, should_stop=stop) return [s.tool_calls[0].id for s in pulled] @@ -542,7 +542,7 @@ async def fake_exec(*_argv: str, **_kwargs: Any) -> _RecordingProcess: monkeypatch.setattr("os.killpg", lambda _pgid, _sig: None, raising=False) cli = await _cli_agent(cls, kind, tmp_path, monkeypatch) try: - await cli.communicate(USER_TURN, stream_callback=stop, should_stop=stop) + await cli.communicate(USER_TURN, iteration=1, stream_callback=stop, should_stop=stop) finally: await cli.stop() return [tool_id for tool_id in ("first", _SECOND) if any(tool_id in json.dumps(p) for p in pulled)] diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 346753bc..5eb26e71 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -85,13 +85,21 @@ async def fake_exec(*argv: str, **kwargs: Any) -> _FakeProcess: return _install -async def _run( +async def _run_outcome( agent: OpenCodeAgent, tmp_path: Any, prompt: str = "do the thing", *, plugin_root: Path | None = None, **kwargs: Any ): await agent.start(str(tmp_path), plugin_root=plugin_root) + kwargs.setdefault("iteration", 1) return await agent.communicate(prompt, **kwargs) +async def _run( + agent: OpenCodeAgent, tmp_path: Any, prompt: str = "do the thing", *, plugin_root: Path | None = None, **kwargs: Any +): + outcome = await _run_outcome(agent, tmp_path, prompt, plugin_root=plugin_root, **kwargs) + return outcome.record_or_raise() + + def _agent(**overrides: Any) -> OpenCodeAgent: config = OpenCodeAgentConfig(type="opencode", **{"model": "deepseek/deepseek-v4-pro", **overrides}) return OpenCodeAgent(config, task_id="t1") @@ -607,7 +615,7 @@ async def test_prepends_mock_dirs_ahead_of_the_inherited_path(self, patch_exec, captured = patch_exec(_FakeProcess(HAPPY_STREAM)) agent = _agent() await agent.start(str(tmp_path), env_path_prepend=["/sandbox/mocks", "/sandbox/bins"]) - await agent.communicate("do the thing") + await agent.communicate("do the thing", iteration=1) expected = os.pathsep.join(["/sandbox/mocks", "/sandbox/bins", "/parent/bin"]) assert captured["kwargs"]["env"]["PATH"] == expected @@ -617,7 +625,7 @@ async def test_plugin_tools_dir_is_exported(self, patch_exec, tmp_path, monkeypa captured = patch_exec(_FakeProcess(HAPPY_STREAM)) agent = _agent() await agent.start(str(tmp_path), plugin_tools_dir="/sandbox/tools") - await agent.communicate("do the thing") + await agent.communicate("do the thing", iteration=1) assert captured["kwargs"]["env"]["PLUGIN_TOOLS_DIR"] == "/sandbox/tools" @@ -627,7 +635,7 @@ async def test_inherited_plugin_tools_dir_wins(self, patch_exec, tmp_path, monke captured = patch_exec(_FakeProcess(HAPPY_STREAM)) agent = _agent() await agent.start(str(tmp_path), plugin_tools_dir="/sandbox/tools") - await agent.communicate("do the thing") + await agent.communicate("do the thing", iteration=1) assert captured["kwargs"]["env"]["PLUGIN_TOOLS_DIR"] == "/host/tools" @@ -874,7 +882,7 @@ async def test_second_turn_resumes_session(self, patch_exec, tmp_path): assert agent._session_id == SESSION captured2 = patch_exec(_FakeProcess(HAPPY_STREAM)) - await agent.communicate("follow up") + await agent.communicate("follow up", iteration=2) argv = captured2["argv"] assert argv[argv.index("--session") + 1] == SESSION @@ -1024,11 +1032,11 @@ async def test_error_event_raises_and_parks_partial(self, patch_exec, tmp_path): patch_exec(_FakeProcess(stream)) agent = _agent() - with pytest.raises(AgentCrashError, match="provider exploded"): - await _run(agent, tmp_path) + outcome = await _run_outcome(agent, tmp_path) - partial = agent.pending_turn - assert partial is not None + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "provider exploded" in outcome.error + partial = outcome.record assert partial.crashed is True # The in-flight tool was force-closed rather than dropped. assert [c.result_status for c in partial.commands] == ["unknown"] @@ -1080,12 +1088,14 @@ async def test_unrecognized_vocabulary_crashes_and_names_the_types(self, patch_e patch_exec(_FakeProcess(stream)) agent = _agent() - with pytest.raises(AgentCrashError, match="no recognized events") as exc: - await _run(agent, tmp_path) + outcome = await _run_outcome(agent, tmp_path) + + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "no recognized events" in outcome.error # The crash names what it DID see, for diagnosis. - assert "session.next.step.ended" in str(exc.value) + assert "session.next.step.ended" in outcome.error - partial = agent.pending_turn + partial = outcome.record assert partial is not None assert partial.crashed is True @@ -1126,10 +1136,13 @@ async def test_finished_step_without_tokens_crashes(self, patch_exec, tmp_path): patch_exec(_FakeProcess(self._stream_without_tokens())) agent = _agent() - with pytest.raises(AgentCrashError, match="zero token telemetry") as exc: - await _run(agent, tmp_path) - assert "1 finished step(s)" in str(exc.value) - assert agent.pending_turn is not None # telemetry captured so far still parked + outcome = await _run_outcome(agent, tmp_path) + + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None + assert "zero token telemetry" in outcome.error + assert "1 finished step(s)" in outcome.error + assert outcome.record is not None # telemetry captured so far still parked async def test_cost_without_tokens_still_crashes(self, patch_exec, tmp_path): """Reported cost does not excuse missing tokens: the USD gate might trip, @@ -1195,12 +1208,11 @@ def on_event(self, event: Any) -> None: class TestUnexpectedErrorContract: - """An unanticipated exception must still honor the pending-turn contract. + """An unanticipated exception must still end the turn as a crashed outcome. - Escaping raw would break it three ways: no terminal ``AgentEndEvent`` (an - unbalanced event tree for every renderer), captured telemetry dropped instead - of parked on ``pending_turn``, and ``_iteration`` left incremented because the - orchestrator never reaches ``discard_pending_turn``. + Escaping raw would break it two ways: no terminal ``AgentEndEvent`` (an + unbalanced event tree for every renderer), and captured telemetry dropped + instead of kept on the crashed record. """ async def test_stream_error_becomes_a_crash_with_partial_parked(self, patch_exec, tmp_path): @@ -1221,11 +1233,11 @@ async def test_stream_error_becomes_a_crash_with_partial_parked(self, patch_exec patch_exec(_ExplodingProcess(stream)) agent = _agent() - with pytest.raises(AgentCrashError, match="OpenCode turn failed"): - await _run(agent, tmp_path) + outcome = await _run_outcome(agent, tmp_path) - partial = agent.pending_turn - assert partial is not None + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "OpenCode turn failed" in outcome.error + partial = outcome.record assert partial.crashed is True # Telemetry captured before the failure survives, orphan tool force-closed. assert [c.result_status for c in partial.commands] == ["unknown"] @@ -1257,18 +1269,6 @@ async def test_terminal_event_is_emitted_exactly_once(self, patch_exec, tmp_path assert ends[0].crashed is True assert ends[0].status is AgentEndStatus.CRASHED - async def test_iteration_rolls_back_after_the_crash(self, patch_exec, tmp_path): - """`discard_pending_turn` must find the bump it needs to undo.""" - patch_exec(_ExplodingProcess([])) - agent = _agent() - - with pytest.raises(AgentCrashError): - await _run(agent, tmp_path) - assert agent._iteration == 1 - await agent.discard_pending_turn() - assert agent._iteration == 0 - assert agent.pending_turn is None - class _LeakyPipeProcess(_FakeProcess): """Replays events, then never signals EOF — the real CLI's behavior. @@ -1492,17 +1492,17 @@ async def readline(self) -> bytes: class TestTimeoutContract: async def test_deadline_raises_turn_timeout_with_partial_parked(self, patch_exec, tmp_path): - """A wedged CLI must yield TurnTimeoutError + a crashed partial record, + """A wedged CLI must yield a TIMEOUT outcome with a crashed partial record, with exactly one terminal AgentEndEvent (status TIMEOUT) emitted.""" proc = _HangingProcess([_evt("step_start", {"id": "prt_1", "messageID": "msg_1"})]) patch_exec(proc) agent = _agent() recorder = _EventRecorder() - with pytest.raises(TurnTimeoutError): - await _run(agent, tmp_path, timeout=0.2, stream_callback=recorder) + outcome = await _run_outcome(agent, tmp_path, timeout=0.2, stream_callback=recorder) - partial = agent.pending_turn + assert outcome.status is AgentEndStatus.TIMEOUT + partial = outcome.record assert partial is not None assert partial.crashed is True assert proc.terminated is True # the CLI was torn down, not abandoned @@ -1510,9 +1510,6 @@ async def test_deadline_raises_turn_timeout_with_partial_parked(self, patch_exec assert len(ends) == 1 assert ends[0].status is AgentEndStatus.TIMEOUT - await agent.discard_pending_turn() - assert agent._iteration == 0 # the failed turn's bump was rolled back - async def test_eof_without_exit_hits_the_deadline(self, patch_exec, tmp_path): """Stream closed, process wedged: the post-EOF reap must be bounded by the turn deadline instead of waiting for an exit that never comes.""" @@ -1520,11 +1517,11 @@ async def test_eof_without_exit_hits_the_deadline(self, patch_exec, tmp_path): patch_exec(proc) agent = _agent() - with pytest.raises(TurnTimeoutError): - await asyncio.wait_for(_run(agent, tmp_path, timeout=0.3), timeout=10) + outcome = await asyncio.wait_for(_run_outcome(agent, tmp_path, timeout=0.3), timeout=10) + assert outcome.status is AgentEndStatus.TIMEOUT # Everything parsed before the wedge survives on the partial record. - partial = agent.pending_turn + partial = outcome.record assert partial is not None assert partial.crashed is True assert partial.token_usage is not None @@ -1552,7 +1549,7 @@ async def test_cancel_parks_partial_and_reraises(self, patch_exec, tmp_path): await agent.start(str(tmp_path)) recorder = _EventRecorder() - task = asyncio.ensure_future(agent.communicate("do the thing", stream_callback=recorder)) + task = asyncio.ensure_future(agent.communicate("do the thing", iteration=1, stream_callback=recorder)) await asyncio.sleep(0.05) # let it spawn and read the first event task.cancel() with pytest.raises(asyncio.CancelledError): @@ -1609,7 +1606,7 @@ async def test_a_cancel_closes_the_open_step(self, patch_exec, tmp_path): await agent.start(str(tmp_path)) recorder = _EventRecorder() - task = asyncio.ensure_future(agent.communicate("do the thing", stream_callback=recorder)) + task = asyncio.ensure_future(agent.communicate("do the thing", iteration=1, stream_callback=recorder)) await asyncio.sleep(0.05) task.cancel() with pytest.raises(asyncio.CancelledError): @@ -1656,9 +1653,9 @@ async def test_a_completed_step_is_never_closed_twice(self, patch_exec, tmp_path class TestTurnAlwaysReapsTheCli: """No exit from `communicate()` may leave the CLI running. - `AgentCrashError` is categorized AGENT_CRASH (max_retries=2) and the - orchestrator's attempt-failure hook only drains `pending_turn` — it never - kills the agent. An abandoned CLI therefore means attempt 2 spawns a SECOND + A crash is categorized AGENT_CRASH (max_retries=2), and the orchestrator + only appends the crashed record — it never kills the agent. An abandoned CLI + therefore means attempt 2 spawns a SECOND `opencode --dir --session ` while attempt 1 is still editing the very files the criteria are about to score, and whichever writer wins decides the task's result. @@ -1685,7 +1682,7 @@ async def test_external_cancel_kills_the_cli(self, patch_exec, tmp_path): agent = _agent() await agent.start(str(tmp_path)) - task = asyncio.ensure_future(agent.communicate("do the thing")) + task = asyncio.ensure_future(agent.communicate("do the thing", iteration=1)) await asyncio.sleep(0.05) # let it spawn and read the first event task.cancel() with pytest.raises(asyncio.CancelledError): diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 416bd4ad..7f4bf7d7 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -1913,15 +1913,15 @@ def __init__(self, plan: list[tuple[int, bool]]) -> None: self._tool_seq = 0 self.host = AsyncMock() self.host.communicate = self.communicate - self.host.pending_turn = None - async def communicate(self, user_input, *, stream_callback=None, timeout=None, should_stop=None): + async def communicate(self, user_input, *, iteration=1, stream_callback=None, timeout=None, should_stop=None): from datetime import datetime - from coder_eval.errors import AgentCrashError from coder_eval.models import CommandTelemetry, TurnRecord + from coder_eval.streaming.emitter import TurnOutcome from coder_eval.streaming.events import ( AgentEndEvent, + AgentEndStatus, AgentStartEvent, StopReason, ToolEndEvent, @@ -1934,7 +1934,7 @@ async def communicate(self, user_input, *, stream_callback=None, timeout=None, s intended, crash = self._plan[self.attempt] self.attempt += 1 self.should_stop_callables.append(should_stop) - stream_callback.on_event(AgentStartEvent(task_id="t", prompt=user_input, iteration=1)) + stream_callback.on_event(AgentStartEvent(task_id="t", prompt=user_input, iteration=iteration)) commands: list[CommandTelemetry] = [] reason: StopReason | None = should_stop() @@ -1948,21 +1948,24 @@ async def communicate(self, user_input, *, stream_callback=None, timeout=None, s self.emitted_per_attempt.append(len(commands)) if crash: - self.host.pending_turn = TurnRecord( - iteration=1, user_input=user_input, agent_output="", commands=commands, crashed=True + partial = TurnRecord( + iteration=iteration, user_input=user_input, agent_output="", commands=commands, crashed=True ) - raise AgentCrashError("mid-turn failure") + return TurnOutcome(record=partial, status=AgentEndStatus.CRASHED, error="mid-turn failure") status = end_status_for(reason) if reason is not None else None if status is not None: - stream_callback.on_event(AgentEndEvent(task_id="t", status=status, iteration=1, user_input=user_input)) - return TurnRecord( - iteration=1, + stream_callback.on_event( + AgentEndEvent(task_id="t", status=status, iteration=iteration, user_input=user_input) + ) + record = TurnRecord( + iteration=iteration, user_input=user_input, agent_output="stopped", commands=commands, tool_calls_exhausted=reason is StopReason.TOOL_CALL_CAP, ) + return TurnOutcome(record=record, status=status or AgentEndStatus.COMPLETED, error=None) @pytest.mark.asyncio @@ -1996,14 +1999,16 @@ async def test_a_latched_cap_the_agent_did_not_stop_on_is_not_labelled_exhausted from unittest.mock import AsyncMock, patch from coder_eval.models import CommandTelemetry, TurnRecord - from coder_eval.streaming.events import StopReason, ToolEndEvent + from coder_eval.streaming.emitter import TurnOutcome + from coder_eval.streaming.events import AgentEndStatus, StopReason, ToolEndEvent orchestrator = _cap_orchestrator(_cap_task("late_latch_test", max_tool_calls=1), tmp_path) - async def _communicate(user_input, *, stream_callback=None, timeout=None, should_stop=None): + async def _communicate(user_input, *, iteration=1, stream_callback=None, timeout=None, should_stop=None): tool = CommandTelemetry(tool_name="Bash", tool_id="late", timestamp=datetime.now()) stream_callback.on_event(ToolEndEvent(task_id="t", tool=tool)) - return TurnRecord(iteration=1, user_input=user_input, agent_output="done", commands=[tool]) + record = TurnRecord(iteration=iteration, user_input=user_input, agent_output="done", commands=[tool]) + return TurnOutcome(record=record, status=AgentEndStatus.COMPLETED, error=None) mock_agent = AsyncMock() mock_agent.communicate = _communicate @@ -2067,17 +2072,16 @@ async def test_tool_call_cap_counts_a_crashed_attempts_calls_toward_the_retry(tm @pytest.mark.asyncio async def test_evaluation_loop_preserves_partial_on_crash_retry(tmp_path): - """First agent.communicate raises AgentCrashError with a partial; retry succeeds. + """First agent.communicate outcome is CRASHED with a partial; retry succeeds. - Locks the orchestrator wiring between `execute_with_retry` and the - `_preserve_partial_on_failure` callback: the partial record reaches - `result.iterations` before the successful retry's record, and both share the - same iteration number (per the agent-side rollback contract). + Locks the orchestrator wiring in `_communicate_with_retry`: a CRASHED + outcome's record reaches `result.iterations` before it is raised (and + retried), so the partial lands before the successful retry's record, and + both share the same iteration number (per the agent-side rollback contract). """ from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch - from coder_eval.errors import AgentCrashError from coder_eval.models import ( CommandTelemetry, CriterionResult, @@ -2085,6 +2089,8 @@ async def test_evaluation_loop_preserves_partial_on_crash_retry(tmp_path): SandboxConfig, TurnRecord, ) + from coder_eval.streaming.emitter import TurnOutcome + from coder_eval.streaming.events import AgentEndStatus agent_cfg = ClaudeCodeAgentConfig.model_construct( type=AgentKind.CLAUDE_CODE, @@ -2151,9 +2157,8 @@ async def test_evaluation_loop_preserves_partial_on_crash_retry(tmp_path): async def crash_then_succeed_impl(_prompt, **kwargs): call_index[0] += 1 if call_index[0] == 1: - mock_agent.pending_turn = partial_record - raise AgentCrashError("mid-turn failure") - return success_record + return TurnOutcome(record=partial_record, status=AgentEndStatus.CRASHED, error="mid-turn failure") + return TurnOutcome(record=success_record, status=AgentEndStatus.COMPLETED, error=None) mock_agent.communicate.side_effect = crash_then_succeed_impl orchestrator.agent = mock_agent @@ -2207,6 +2212,8 @@ async def test_evaluation_loop_stamps_timeout_reason_on_partial(tmp_path): SandboxConfig, TurnRecord, ) + from coder_eval.streaming.emitter import TurnOutcome + from coder_eval.streaming.events import AgentEndStatus agent_cfg = ClaudeCodeAgentConfig.model_construct( type=AgentKind.CLAUDE_CODE, @@ -2255,8 +2262,9 @@ async def test_evaluation_loop_stamps_timeout_reason_on_partial(tmp_path): mock_agent = AsyncMock() async def timeout_impl(_prompt, **kwargs): - mock_agent.pending_turn = partial_record - raise TurnTimeoutError(600.0, iteration=1) + return TurnOutcome( + record=partial_record, status=AgentEndStatus.TIMEOUT, error="Agent turn timed out after 600s" + ) mock_agent.communicate.side_effect = timeout_impl orchestrator.agent = mock_agent @@ -2275,9 +2283,9 @@ async def timeout_impl(_prompt, **kwargs): with ( patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None), patch("asyncio.sleep", new_callable=AsyncMock), - # TurnTimeoutError is non-retryable, so the loop re-raises after the - # on_attempt_error callback has already stamped + appended the partial. - # We only care about the side-effect, so suppress the re-raise. + # TurnTimeoutError is non-retryable, so the loop re-raises after + # `_communicate_with_retry` has already appended the TIMEOUT outcome's + # partial. We only care about the side-effect, so suppress the re-raise. pytest.raises(TurnTimeoutError), ): await orchestrator._evaluation_loop() @@ -2691,12 +2699,15 @@ async def communicate(self, user_input: str, **kwargs): from datetime import datetime from coder_eval.models import CommandTelemetry, TurnRecord + from coder_eval.streaming.emitter import TurnOutcome + from coder_eval.streaming.events import AgentEndStatus self._iteration += 1 skill = CommandTelemetry( tool_name="Skill", tool_id="s1", timestamp=datetime.now(), parameters={"skill": "probe-skill"} ) - return TurnRecord(iteration=self._iteration, user_input=user_input, agent_output="done", commands=[skill]) + record = TurnRecord(iteration=self._iteration, user_input=user_input, agent_output="done", commands=[skill]) + return TurnOutcome(record=record, status=AgentEndStatus.COMPLETED, error=None) def _patch_routes(monkeypatch) -> None: diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 989e2f09..860499dc 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -24,7 +24,6 @@ import pytest from coder_eval.agents.pi_agent import PiAgent, _PiTurnState, _result_text -from coder_eval.errors import AgentCrashError, TurnTimeoutError from coder_eval.models import AgentKind, AssistantMessage, CommandTelemetry, PiAgentConfig, TokenUsage from coder_eval.orchestration.plugin_staging import stage_plugins from coder_eval.pricing import calculate_cost @@ -79,9 +78,9 @@ async def fake_exec(*argv: str, **kwargs: Any) -> _FakeProcess: return _install -async def _run(agent: PiAgent, tmp_path: Any, prompt: str = "do the thing", **kwargs: Any): +async def _run(agent: PiAgent, tmp_path: Any, prompt: str = "do the thing", *, iteration: int = 1, **kwargs: Any): await agent.start(str(tmp_path)) - return await agent.communicate(prompt, **kwargs) + return await agent.communicate(prompt, iteration=iteration, **kwargs) def _agent(**overrides: Any) -> PiAgent: @@ -100,7 +99,8 @@ def on_event(self, event: Any) -> None: class TestHappyPath: async def test_builds_turn_record(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assert record.crashed is False # 3 turn_start steps in the fixture (write, read, summarize). @@ -110,7 +110,8 @@ async def test_builds_turn_record(self, patch_exec, tmp_path): async def test_token_buckets_sum_across_turns(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record usage = record.token_usage assert usage is not None @@ -123,7 +124,8 @@ async def test_token_buckets_sum_across_turns(self, patch_exec, tmp_path): async def test_reconciliation_invariant(self, patch_exec, tmp_path): """Summing the four buckets across messages must equal token_usage exactly.""" patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record usage = record.token_usage assert usage is not None @@ -133,7 +135,8 @@ async def test_reconciliation_invariant(self, patch_exec, tmp_path): async def test_tool_calls_captured(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record # write + read, normalized to the canonical vocabulary. assert [c.tool_name for c in record.commands] == ["Write", "Read"] @@ -146,7 +149,8 @@ async def test_tool_calls_captured(self, patch_exec, tmp_path): async def test_messages_attributed_to_turns(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assistants = [m for m in record.messages if isinstance(m, AssistantMessage)] assert len(assistants) == 3 @@ -191,7 +195,8 @@ async def test_bash_maps_to_canonical(self, patch_exec, tmp_path): json.dumps({"type": "agent_settled"}), ] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assert record.commands[0].tool_name == "Bash" assert record.commands[0].parameters == {"command": "pytest -q"} @@ -207,7 +212,8 @@ async def test_find_maps_to_glob(self, patch_exec, tmp_path): json.dumps({"type": "agent_settled"}), ] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assert record.commands[0].tool_name == "Glob" async def test_unknown_tool_passes_through(self, patch_exec, tmp_path): @@ -218,7 +224,8 @@ async def test_unknown_tool_passes_through(self, patch_exec, tmp_path): _turn_end(inp=10, out=5), ] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assert record.commands[0].tool_name == "some_new_tool" assert record.commands[0].parameters == {"whatever": 1} @@ -230,7 +237,8 @@ async def test_edit_arg_keys_map_to_canonical(self, patch_exec, tmp_path): _turn_end(inp=10, out=5), ] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assert record.commands[0].parameters == { "file_path": "a.py", "old_string": "a", @@ -323,7 +331,7 @@ async def test_successive_calls_reuse_the_same_session(self, patch_exec, tmp_pat sdir1 = argv1[argv1.index("--session-dir") + 1] captured2 = patch_exec(_FakeProcess(HAPPY_STREAM)) - await agent.communicate("follow up") + await agent.communicate("follow up", iteration=2) argv2 = captured2["argv"] assert argv2[argv2.index("--session-id") + 1] == sid1 assert argv2[argv2.index("--session-dir") + 1] == sdir1 @@ -388,7 +396,7 @@ async def test_prepends_mock_dirs_ahead_of_the_inherited_path(self, patch_exec, captured = patch_exec(_FakeProcess(HAPPY_STREAM)) agent = _agent() await agent.start(str(tmp_path), env_path_prepend=["/sandbox/mocks", "/sandbox/bins"]) - await agent.communicate("do the thing") + await agent.communicate("do the thing", iteration=1) assert captured["kwargs"]["env"]["PATH"] == os.pathsep.join(["/sandbox/mocks", "/sandbox/bins", "/parent/bin"]) @@ -418,7 +426,8 @@ async def test_two_agent_cycles_reduce_to_one_agent_end(self, patch_exec, tmp_pa ] patch_exec(_FakeProcess(stream)) recorder = _EventRecorder() - record = await _run(_agent(), tmp_path, stream_callback=recorder) + outcome = await _run(_agent(), tmp_path, stream_callback=recorder) + record = outcome.record assert record.crashed is False assert len([e for e in recorder.events if isinstance(e, AgentEndEvent)]) == 1 @@ -449,9 +458,10 @@ async def test_early_criterion_ends_turn_stopped_early(self, patch_exec, tmp_pat proc = _RunningProcess(HAPPY_STREAM) patch_exec(proc) recorder = _EventRecorder() - record = await _run( + outcome = await _run( _agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION, stream_callback=recorder ) + record = outcome.record assert record.crashed is False assert proc.terminated is True @@ -465,7 +475,8 @@ async def test_tool_call_cap_ends_turn_tool_calls_exhausted(self, patch_exec, tm proc = _RunningProcess(HAPPY_STREAM) patch_exec(proc) recorder = _EventRecorder() - record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOOL_CALL_CAP, stream_callback=recorder) + outcome = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOOL_CALL_CAP, stream_callback=recorder) + record = outcome.record assert proc.terminated is True assert record.crashed is False @@ -477,7 +488,8 @@ async def test_token_budget_ends_turn_token_budget_exceeded(self, patch_exec, tm proc = _RunningProcess(HAPPY_STREAM) patch_exec(proc) recorder = _EventRecorder() - record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOKEN_BUDGET, stream_callback=recorder) + outcome = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOKEN_BUDGET, stream_callback=recorder) + record = outcome.record assert proc.terminated is True assert record.crashed is False @@ -489,9 +501,10 @@ async def test_the_deciding_turn_is_kept_whole(self, patch_exec, tmp_path): """A stop that lands on turn 2's `turn_start` keeps turn 1 complete.""" second_turn_start = [i for i, line in enumerate(HAPPY_STREAM) if json.loads(line)["type"] == "turn_start"][1] patch_exec(_RunningProcess(HAPPY_STREAM)) - record = await _run( + outcome = await _run( _agent(), tmp_path, should_stop=_stop_after(second_turn_start + 1, StopReason.TOOL_CALL_CAP) ) + record = outcome.record assert record.tool_calls_exhausted is True assert len(record.commands) == 1 # turn 1's write @@ -503,19 +516,22 @@ async def test_the_deciding_turn_is_kept_whole(self, patch_exec, tmp_path): async def test_an_intentional_stop_is_exempt_from_a_non_zero_exit(self, patch_exec, tmp_path): """Killing the CLI makes it exit non-zero; that must not crash an intentional stop.""" patch_exec(_RunningProcess(HAPPY_STREAM, returncode=-15, stderr=b"terminated")) - record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOOL_CALL_CAP) + outcome = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOOL_CALL_CAP) + record = outcome.record assert record.crashed is False assert record.tool_calls_exhausted is True async def test_an_intentional_stop_is_exempt_from_no_recognized_events(self, patch_exec, tmp_path): """A stop can land before the first recognized event; that is not vocabulary drift.""" patch_exec(_RunningProcess([json.dumps({"type": "not_a_pi_event"}), *HAPPY_STREAM])) - record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOKEN_BUDGET) + outcome = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOKEN_BUDGET) + record = outcome.record assert record.crashed is False async def test_no_stop_is_uncapped(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path, should_stop=lambda: None) + outcome = await _run(_agent(), tmp_path, should_stop=lambda: None) + record = outcome.record assert record.tool_calls_exhausted is False assert record.assistant_turn_count == 3 @@ -556,10 +572,10 @@ async def test_deadline_raises_turn_timeout_with_partial_parked(self, patch_exec agent = _agent() recorder = _EventRecorder() - with pytest.raises(TurnTimeoutError): - await _run(agent, tmp_path, timeout=0.2, stream_callback=recorder) + outcome = await _run(agent, tmp_path, timeout=0.2, stream_callback=recorder) - partial = agent.pending_turn + assert outcome.status is AgentEndStatus.TIMEOUT + partial = outcome.record assert partial is not None assert partial.crashed is True assert proc.terminated is True @@ -567,9 +583,6 @@ async def test_deadline_raises_turn_timeout_with_partial_parked(self, patch_exec assert len(ends) == 1 assert ends[0].status is AgentEndStatus.TIMEOUT - await agent.discard_pending_turn() - assert agent._iteration == 0 - class _ExplodingProcess(_FakeProcess): async def readline(self) -> bytes: @@ -581,13 +594,15 @@ async def readline(self) -> bytes: class TestFailurePaths: async def test_nonzero_exit_crashes(self, patch_exec, tmp_path): patch_exec(_FakeProcess([], returncode=1, stderr=b"boom: bad model")) - with pytest.raises(AgentCrashError, match="boom: bad model"): - await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "boom: bad model" in outcome.error async def test_empty_clean_exit_crashes_on_no_recognized_events(self, patch_exec, tmp_path): patch_exec(_FakeProcess([], returncode=0)) - with pytest.raises(AgentCrashError, match="no recognized events"): - await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "no recognized events" in outcome.error async def test_drift_crash_names_the_unrecognized_types(self, patch_exec, tmp_path): """A clean exit whose events are all unrecognized (schema drift) crashes and @@ -597,19 +612,22 @@ async def test_drift_crash_names_the_unrecognized_types(self, patch_exec, tmp_pa json.dumps({"type": "another.unknown", "bar": 2}), ] patch_exec(_FakeProcess(stream)) - with pytest.raises(AgentCrashError, match=r"another\.unknown, some\.new\.event") as exc: - await _run(_agent(), tmp_path) - assert "no recognized events" in str(exc.value) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None + assert "another.unknown, some.new.event" in outcome.error + assert "no recognized events" in outcome.error async def test_stream_error_becomes_a_crash_with_partial_parked(self, patch_exec, tmp_path): stream = [_turn_start(), _tool_start("w:0", "write", {"path": "a.txt", "content": "x"})] patch_exec(_ExplodingProcess(stream)) agent = _agent() - with pytest.raises(AgentCrashError, match="Pi turn failed"): - await _run(agent, tmp_path) + outcome = await _run(agent, tmp_path) - partial = agent.pending_turn + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "Pi turn failed" in outcome.error + partial = outcome.record assert partial is not None assert partial.crashed is True # The in-flight tool was force-closed rather than dropped. @@ -622,13 +640,15 @@ async def boom(*_argv: str, **_kwargs: Any): monkeypatch.setattr(asyncio, "create_subprocess_exec", boom) monkeypatch.setattr("shutil.which", lambda _name: "/usr/local/bin/pi") - with pytest.raises(AgentCrashError, match="no fork for you"): - await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "no fork for you" in outcome.error async def test_malformed_line_is_skipped(self, patch_exec, tmp_path): stream = ["not json at all", *HAPPY_STREAM] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assert record.crashed is False assert record.assistant_turn_count == 3 @@ -643,25 +663,14 @@ async def test_terminal_event_emitted_exactly_once_on_crash(self, patch_exec, tm patch_exec(_ExplodingProcess([_turn_start()])) recorder = _EventRecorder() - with pytest.raises(AgentCrashError): - await _run(_agent(), tmp_path, stream_callback=recorder) + outcome = await _run(_agent(), tmp_path, stream_callback=recorder) + assert outcome.status is AgentEndStatus.CRASHED ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] assert len(ends) == 1 assert ends[0].crashed is True assert ends[0].status is AgentEndStatus.CRASHED - async def test_iteration_rolls_back_after_the_crash(self, patch_exec, tmp_path): - patch_exec(_ExplodingProcess([])) - agent = _agent() - - with pytest.raises(AgentCrashError): - await _run(agent, tmp_path) - assert agent._iteration == 1 - await agent.discard_pending_turn() - assert agent._iteration == 0 - assert agent.pending_turn is None - class TestTurnEventsAreBalanced: @staticmethod @@ -681,8 +690,8 @@ async def test_a_timeout_closes_the_open_turn(self, patch_exec, tmp_path): patch_exec(proc) recorder = _EventRecorder() - with pytest.raises(TurnTimeoutError): - await _run(_agent(), tmp_path, timeout=0.2, stream_callback=recorder) + outcome = await _run(_agent(), tmp_path, timeout=0.2, stream_callback=recorder) + assert outcome.status is AgentEndStatus.TIMEOUT assert self._pairs(recorder) == (1, 1) end = next(e for e in recorder.events if isinstance(e, TurnEndEvent)) @@ -763,7 +772,8 @@ async def test_tool_error_is_captured(self, patch_exec, tmp_path): ] patch_exec(_FakeProcess(stream)) recorder = _EventRecorder() - record = await _run(_agent(), tmp_path, stream_callback=recorder) + outcome = await _run(_agent(), tmp_path, stream_callback=recorder) + record = outcome.record [cmd] = record.commands assert cmd.result_status == "error" @@ -807,7 +817,8 @@ async def test_zero_usage_turn_is_scored_not_crashed(self, patch_exec, tmp_path) json.dumps({"type": "agent_settled"}), ] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assert record.crashed is False @@ -857,9 +868,10 @@ async def test_terminal_error_crashes_the_turn(self, patch_exec, tmp_path): stream = [_turn_start(), _turn_end_error("404: blocked by guardrail"), json.dumps({"type": "agent_settled"})] patch_exec(_FakeProcess(stream)) agent = _agent() - with pytest.raises(AgentCrashError, match="blocked by guardrail"): - await _run(agent, tmp_path) - partial = agent.pending_turn + outcome = await _run(agent, tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "blocked by guardrail" in outcome.error + partial = outcome.record assert partial is not None assert partial.crashed is True @@ -869,7 +881,8 @@ async def test_a_stop_after_an_error_turn_finalizes_cleanly(self, patch_exec, tm status — NOT crash on the stale error.""" stream = [_turn_start(), _turn_end_error("transient 429"), _turn_start(), _turn_end(inp=1, out=1)] patch_exec(_RunningProcess(stream)) - record = await _run(_agent(), tmp_path, should_stop=_stop_after(3, StopReason.TOOL_CALL_CAP)) + outcome = await _run(_agent(), tmp_path, should_stop=_stop_after(3, StopReason.TOOL_CALL_CAP)) + record = outcome.record assert record.tool_calls_exhausted is True assert record.crashed is False @@ -930,7 +943,8 @@ async def test_all_zero_usage_object_warns(self, patch_exec, tmp_path, caplog): stream = [_turn_start(), zero, json.dumps({"type": "agent_settled"})] patch_exec(_FakeProcess(stream)) with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assert record.crashed is False # score, don't crash (documented Pi policy) assert any("all-zero token buckets" in r.getMessage() for r in caplog.records) @@ -953,7 +967,8 @@ async def test_total_tokens_mismatch_warns(self, patch_exec, tmp_path, caplog): stream = [_turn_start(), bad, json.dumps({"type": "agent_settled"})] patch_exec(_FakeProcess(stream)) with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assert record.crashed is False assert any("does not reconcile" in r.getMessage() for r in caplog.records) @@ -972,7 +987,7 @@ async def test_staged_root_emits_skill_arg(self, patch_exec, tmp_path): captured = patch_exec(_FakeProcess(HAPPY_STREAM)) agent = _agent() await agent.start(str(tmp_path), plugin_root=root) - await agent.communicate("do the thing") + await agent.communicate("do the thing", iteration=1) argv = captured["argv"] assert argv[argv.index("--skill") + 1] == str(root / "skills") assert "pi_skill_paths" not in agent.get_environment_info() @@ -984,9 +999,9 @@ async def test_no_plugin_root_means_no_skill_arg(self, patch_exec, tmp_path): class TestTurnAlwaysReapsTheCli: - """No exit from ``communicate()`` may leave the CLI running. ``AgentCrashError`` - is categorized AGENT_CRASH (max_retries=2) and the orchestrator's attempt-failure - hook only drains ``pending_turn`` — it never kills the agent. An abandoned CLI + """No exit from ``communicate()`` may leave the CLI running. A crash is + categorized AGENT_CRASH (max_retries=2), and the orchestrator only appends the + crashed record — it never kills the agent. An abandoned CLI therefore means attempt 2 spawns a SECOND ``pi`` editing the very files the criteria are about to score. The graceful ``kill()`` covers the intentional cuts and the timeout; these pin the two paths that reach ``finally`` with a live child. @@ -996,8 +1011,9 @@ async def test_read_loop_crash_kills_the_cli(self, patch_exec, tmp_path): """``_crash_turn`` is synchronous and raises — nothing below it reaps.""" proc = _ExplodingRunningProcess([_turn_start()]) patch_exec(proc) - with pytest.raises(AgentCrashError, match="Pi turn failed"): - await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "Pi turn failed" in outcome.error assert proc.killed is True async def test_external_cancel_kills_the_cli(self, patch_exec, tmp_path): @@ -1007,7 +1023,7 @@ async def test_external_cancel_kills_the_cli(self, patch_exec, tmp_path): patch_exec(proc) agent = _agent() await agent.start(str(tmp_path)) - task = asyncio.ensure_future(agent.communicate("do the thing")) + task = asyncio.ensure_future(agent.communicate("do the thing", iteration=1)) await asyncio.sleep(0.05) # let it spawn and read the first event task.cancel() with pytest.raises(asyncio.CancelledError): @@ -1033,7 +1049,7 @@ async def test_cancel_parks_partial_and_reraises(self, patch_exec, tmp_path): agent = _agent() await agent.start(str(tmp_path)) recorder = _EventRecorder() - task = asyncio.ensure_future(agent.communicate("do the thing", stream_callback=recorder)) + task = asyncio.ensure_future(agent.communicate("do the thing", iteration=1, stream_callback=recorder)) await asyncio.sleep(0.05) task.cancel() with pytest.raises(asyncio.CancelledError): @@ -1071,7 +1087,8 @@ async def test_stream_cost_wins_when_reported(self, patch_exec, tmp_path): """The provider's own accounting beats a static headline rate.""" stream = [_turn_start(), _turn_end(inp=1000, out=500, cost=0.5), self._SETTLED] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assert record.token_usage is not None assert record.token_usage.total_cost_usd == pytest.approx(0.5) @@ -1079,7 +1096,8 @@ async def test_missing_cost_is_priced_from_the_rate_card(self, patch_exec, tmp_p """No `cost` key at all — without the fallback the turn books tokens with no money.""" stream = [_turn_start(), _turn_end_no_cost(inp=1000, out=500), self._SETTLED] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record expected = calculate_cost("openrouter/moonshotai/kimi-k3", uncached_input_tokens=1000, output_tokens=500) assert expected is not None and expected > 0 assert record.token_usage is not None @@ -1089,7 +1107,8 @@ async def test_unpriced_model_reports_no_cost(self, patch_exec, tmp_path): """`None` (not 0.0) so "unpriceable" stays distinct from "ran for free".""" stream = [_turn_start(), _turn_end_no_cost(inp=10, out=5), self._SETTLED] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(model="nowhere/not-a-real-model"), tmp_path) + outcome = await _run(_agent(model="nowhere/not-a-real-model"), tmp_path) + record = outcome.record assert record.token_usage is not None assert record.token_usage.total_cost_usd is None @@ -1099,7 +1118,8 @@ async def test_zero_reported_cost_on_a_priced_model_uses_the_rate_card(self, pat stream = [_turn_start(), _turn_end(inp=1000, out=500, cost=0.0), self._SETTLED] patch_exec(_FakeProcess(stream)) with caplog.at_level("DEBUG"): - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record expected = calculate_cost("openrouter/moonshotai/kimi-k3", uncached_input_tokens=1000, output_tokens=500) assert expected is not None and expected > 0 assert record.token_usage is not None @@ -1110,7 +1130,8 @@ async def test_zero_reported_cost_on_an_unpriced_model_stays_zero(self, patch_ex """With no rate to fall back to, the stream's 0 is the best information we have.""" stream = [_turn_start(), _turn_end(inp=10, out=5, cost=0.0), self._SETTLED] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(model="nowhere/not-a-real-model"), tmp_path) + outcome = await _run(_agent(model="nowhere/not-a-real-model"), tmp_path) + record = outcome.record assert record.token_usage is not None assert record.token_usage.total_cost_usd == 0.0 @@ -1549,12 +1570,12 @@ class TestClockIsFreshPerTurn: async def test_a_turn_after_a_crash_is_anchored_to_a_fresh_clock(self, patch_exec, tmp_path): agent = _agent() patch_exec(_FakeProcess([], returncode=1, stderr=b"boom: bad model")) - with pytest.raises(AgentCrashError): - await _run(agent, tmp_path) + outcome = await _run(agent, tmp_path) + assert outcome.status is AgentEndStatus.CRASHED crashed_clock = agent # the state is gone; only the agent survives a crash patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await crashed_clock.communicate("try again") + record = (await crashed_clock.communicate("try again", iteration=2)).record # The recovered turn measured a real window of its own, rather than one # anchored before the crash — which a stale clock would have produced @@ -1615,6 +1636,7 @@ async def test_the_head_and_tail_are_measured_within_one_basis( monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record assert_overhead_is_measured(record) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 54a3f907..7a71735c 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -222,9 +222,10 @@ class _PluginNoneConfig(NoneAgentConfig): agent = NoOpAgent(_PluginNoneConfig()) await agent.start(str(tmp_path)) # Would raise AttributeError if communicate used `.type.value` instead of str(...). - record = await agent.communicate("hello") + outcome = await agent.communicate("hello", iteration=1) await agent.stop() + record = outcome.record assert record.crashed is False assert record.iteration == 1 diff --git a/tests/test_price_turn.py b/tests/test_price_turn.py index cbae99ac..fa9b48ad 100644 --- a/tests/test_price_turn.py +++ b/tests/test_price_turn.py @@ -116,7 +116,7 @@ async def fake_exec(*_argv: str, **_kwargs: Any) -> _FakeProcess: patch.object(os, "killpg", lambda _pgid, _sig: None, create=True), ): await agent.start(working_dir) - await agent.communicate("do it", stream_callback=recorder) + await agent.communicate("do it", iteration=1, stream_callback=recorder) return recorder.events @@ -171,7 +171,7 @@ async def test_antigravity(self, tmp_path: Any) -> None: agent.working_directory = tmp_path recorder = _Recorder() with patch.object(antigravity_agent.asyncio, "sleep", _no_sleep): - await agent.communicate("do it", stream_callback=recorder) + await agent.communicate("do it", iteration=1, stream_callback=recorder) expected = _rate( "gemini-3.5-flash", TokenUsage(uncached_input_tokens=800, output_tokens=350, cache_read_input_tokens=200) ) diff --git a/tests/test_reference_permissions.py b/tests/test_reference_permissions.py index e9165768..ceb0e9d8 100644 --- a/tests/test_reference_permissions.py +++ b/tests/test_reference_permissions.py @@ -561,6 +561,8 @@ async def test_agent_cannot_read_reference_mid_turn(self, tmp_path, monkeypatch) from coder_eval.models import EvaluationResult, FinalStatus, TurnRecord from coder_eval.orchestrator import Orchestrator + from coder_eval.streaming.emitter import TurnOutcome + from coder_eval.streaming.events import AgentEndStatus task_dir = tmp_path / "task" reference = task_dir / "reference" @@ -598,11 +600,11 @@ async def test_agent_cannot_read_reference_mid_turn(self, tmp_path, monkeypatch) async def _cheating_communicate(prompt, **kwargs): observed["reference"] = _try_read(staged / "solution.py") observed["task_dir"] = _try_read(task_file) - return TurnRecord(iteration=1, prompt=prompt, user_input=prompt, agent_output="done") + record = TurnRecord(iteration=1, prompt=prompt, user_input=prompt, agent_output="done") + return TurnOutcome(record=record, status=AgentEndStatus.COMPLETED, error=None) agent = MagicMock() agent.communicate = _cheating_communicate - agent.pending_turn = None orchestrator.agent = agent await orchestrator._communicate_with_retry(prompt="go", iteration=1, operation_label="test") @@ -633,6 +635,8 @@ async def test_reference_is_unshielded_by_the_time_criteria_run(self, tmp_path, from coder_eval.models import CriterionResult, EvaluationResult, FinalStatus, TurnRecord from coder_eval.orchestrator import Orchestrator + from coder_eval.streaming.emitter import TurnOutcome + from coder_eval.streaming.events import AgentEndStatus task_dir = tmp_path / "task" reference = task_dir / "reference" @@ -664,11 +668,11 @@ async def test_reference_is_unshielded_by_the_time_criteria_run(self, tmp_path, async def _communicate(prompt, **kwargs): seen["during_turn"] = _mode(staged) - return TurnRecord(iteration=1, prompt=prompt, user_input=prompt, agent_output="done") + record = TurnRecord(iteration=1, prompt=prompt, user_input=prompt, agent_output="done") + return TurnOutcome(record=record, status=AgentEndStatus.COMPLETED, error=None) agent = MagicMock() agent.communicate = _communicate - agent.pending_turn = None orchestrator.agent = agent async def _check_all_async(*args, **kwargs): diff --git a/tests/test_retry_logic_comprehensive.py b/tests/test_retry_logic_comprehensive.py index 0bf0dfb3..709b6d3d 100644 --- a/tests/test_retry_logic_comprehensive.py +++ b/tests/test_retry_logic_comprehensive.py @@ -9,7 +9,7 @@ import pytest -from coder_eval.errors.categories import RETRY_CONFIG, ErrorCategory +from coder_eval.errors.categories import ErrorCategory from coder_eval.errors.categorization import categorize_error from coder_eval.errors.executor import execute_with_retry from coder_eval.errors.retry import get_retry_delay, should_retry @@ -229,111 +229,6 @@ async def flaky_operation(): assert attempts == 3 # Failed twice, succeeded third time -@pytest.mark.asyncio -async def test_on_attempt_error_fires_for_every_failure(): - """Callback fires on every failed attempt, including retried and terminal ones. - - The orchestrator uses this hook to drain partial telemetry from - AgentCrashError before the retry decision is made. - """ - # Lock the assumption: AGENT_RATE_LIMIT must be retryable for this test to - # exercise the "fires across multiple retries" path. If the policy ever - # flips to non-retryable, the test would silently degenerate into the - # terminal-failure case. - assert RETRY_CONFIG[ErrorCategory.AGENT_RATE_LIMIT].max_retries >= 2 - - attempts = 0 - calls: list[tuple[str, int]] = [] - - async def flaky_operation(): - nonlocal attempts - attempts += 1 - if attempts < 3: - raise Exception("Rate limit exceeded") # Retryable (AGENT_RATE_LIMIT) - return "success" - - async def callback(err: Exception, attempt: int) -> None: - calls.append((str(err), attempt)) - - context = {"task_id": "test-task", "component": "agent"} - - with patch("asyncio.sleep", new_callable=AsyncMock): - result = await execute_with_retry( - flaky_operation, "test_op", context, max_attempts=5, on_attempt_error=callback - ) - - assert result == "success" - # Two failures before success → callback fires twice, with the - # zero-indexed attempt number. - assert calls == [("Rate limit exceeded", 0), ("Rate limit exceeded", 1)] - - -@pytest.mark.asyncio -async def test_on_attempt_error_fires_on_terminal_failure(): - """Callback also fires on the final, non-retried attempt. - - This is the hook's whole point on terminal failures: preserve - telemetry before the exception propagates up and the run is abandoned. - """ - # Lock the assumption this test is built on: AGENT_AUTH_ERROR is - # non-retryable. If that policy ever flips, the "calls == [0]" - # assertion below would still pass for the wrong reason, so fail - # loudly here instead. The default RetryConfig has max_retries=0, - # and AGENT_AUTH_ERROR is not overridden in RETRY_CONFIG — so either - # a missing entry (treated as default) or an explicit max_retries=0 - # passes. - from coder_eval.errors.categories import RetryConfig - - assert RETRY_CONFIG.get(ErrorCategory.AGENT_AUTH_ERROR, RetryConfig()).max_retries == 0 - - calls: list[int] = [] - - async def always_fails(): - raise Exception("Invalid API Key") # Non-retryable - - async def callback(err: Exception, attempt: int) -> None: - calls.append(attempt) - - context = {"task_id": "test-task", "component": "agent"} - - with pytest.raises(Exception, match="Invalid API Key"): - await execute_with_retry(always_fails, "test_op", context, max_attempts=5, on_attempt_error=callback) - - assert calls == [0] # Fired once, before the error propagates. - - -@pytest.mark.asyncio -async def test_on_attempt_error_exceptions_are_swallowed(): - """A raising callback must not mask the original error. - - The comment in executor.py explicitly calls this out as a contract: - telemetry-draining code should never take down the retry loop. - """ - # Pin the categorization + retry policy this test relies on so a - # change to the categorizer's string patterns can't silently turn - # this into a no-retry scenario that passes for the wrong reason. - sentinel = Exception("Connection failed") - assert categorize_error(sentinel, {"component": "agent"}) == ErrorCategory.AGENT_API_ERROR - assert RETRY_CONFIG[ErrorCategory.AGENT_API_ERROR].max_retries >= 1 - - callback_called = False - - async def flaky_operation(): - raise Exception("Connection failed") # Retryable (AGENT_API_ERROR) - - async def bad_callback(err: Exception, attempt: int) -> None: - nonlocal callback_called - callback_called = True - raise RuntimeError("callback blew up") - - context = {"task_id": "test-task", "component": "agent"} - - with patch("asyncio.sleep", new_callable=AsyncMock), pytest.raises(Exception, match="Connection failed"): - await execute_with_retry(flaky_operation, "test_op", context, max_attempts=2, on_attempt_error=bad_callback) - - assert callback_called # Callback was invoked despite raising. - - def test_should_retry_respects_config(): """Test that should_retry correctly uses RetryConfig. diff --git a/tests/test_run_limits_orchestrator.py b/tests/test_run_limits_orchestrator.py index 18ad333d..9af3ff25 100644 --- a/tests/test_run_limits_orchestrator.py +++ b/tests/test_run_limits_orchestrator.py @@ -26,7 +26,8 @@ TurnRecord, ) from coder_eval.orchestrator import Orchestrator -from coder_eval.streaming.events import AgentEndEvent, AgentStartEvent +from coder_eval.streaming.emitter import TurnOutcome +from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus, AgentStartEvent def _make_task(*, run_limits: RunLimits | None = None) -> TaskDefinition: @@ -90,12 +91,12 @@ def _reporting_agent(*turns: TurnRecord) -> AsyncMock: """A fake agent whose each ``communicate`` reports its turn's usage on the stream, as real agents do.""" remaining = list(turns) - async def communicate(user_input, *, stream_callback=None, timeout=None, should_stop=None): + async def communicate(user_input, *, iteration, stream_callback=None, timeout=None, should_stop=None): turn = remaining.pop(0) if len(remaining) > 1 else remaining[0] assert stream_callback is not None stream_callback.on_event(AgentStartEvent(task_id="budget_test", prompt=user_input)) stream_callback.on_event(AgentEndEvent(task_id="budget_test", usage=turn.token_usage or TokenUsage())) - return turn + return TurnOutcome(record=turn, status=AgentEndStatus.COMPLETED, error=None) agent = AsyncMock() agent.communicate = communicate @@ -470,7 +471,9 @@ async def test_warning_does_not_abort_run(self, tmp_path, caplog): orch = _make_orchestrator(task, tmp_path) mock_agent = AsyncMock() - mock_agent.communicate = AsyncMock(return_value=turn) + mock_agent.communicate = AsyncMock( + return_value=TurnOutcome(record=turn, status=AgentEndStatus.COMPLETED, error=None) + ) orch.agent = mock_agent mock_checker = MagicMock() mock_checker.check_all_async = AsyncMock( @@ -515,7 +518,9 @@ async def test_warning_fires_in_simulation_and_does_not_abort(self, tmp_path, ca # Each agent turn = 1 tool call → cumulative still under 3 after one turn. turn = _make_turn(commands=1) mock_agent = AsyncMock() - mock_agent.communicate = AsyncMock(return_value=turn) + mock_agent.communicate = AsyncMock( + return_value=TurnOutcome(record=turn, status=AgentEndStatus.COMPLETED, error=None) + ) orch.agent = mock_agent mock_checker = MagicMock() @@ -571,7 +576,9 @@ async def test_warning_fires_when_single_simulation_turn_exceeds(self, tmp_path, # 4 tools + reply = 5 visible turns, exceeds 2. turn = _make_turn(commands=4, reply="done") mock_agent = AsyncMock() - mock_agent.communicate = AsyncMock(return_value=turn) + mock_agent.communicate = AsyncMock( + return_value=TurnOutcome(record=turn, status=AgentEndStatus.COMPLETED, error=None) + ) orch.agent = mock_agent mock_checker = MagicMock() diff --git a/tests/test_simulation_integration.py b/tests/test_simulation_integration.py index 1935c199..3d8cac84 100644 --- a/tests/test_simulation_integration.py +++ b/tests/test_simulation_integration.py @@ -27,6 +27,7 @@ ) from coder_eval.orchestrator import Orchestrator from coder_eval.simulation.user_simulator import UserSimulator +from coder_eval.streaming.emitter import TurnOutcome from tests.fixtures.harness_stubs import stub_contract from tests.fixtures.mock_agent import MockAgent from tests.fixtures.text_stub_agent import TextStubAgent @@ -77,11 +78,11 @@ def __init__(self, task: TaskDefinition, calls_per_turn: int) -> None: self._tool_seq = 0 self.emitted_per_turn: list[int] = [] - async def communicate(self, user_input: str, **kwargs: Any) -> TurnRecord: + async def communicate(self, user_input: str, **kwargs: Any) -> TurnOutcome: from datetime import datetime from coder_eval.models import CommandTelemetry - from coder_eval.streaming.events import StopReason, ToolEndEvent, ToolStartEvent + from coder_eval.streaming.events import AgentEndStatus, StopReason, ToolEndEvent, ToolStartEvent self._iteration += 1 stream_callback = kwargs["stream_callback"] @@ -94,13 +95,14 @@ async def communicate(self, user_input: str, **kwargs: Any) -> TurnRecord: stream_callback.on_event(ToolEndEvent(task_id=self.task.task_id, tool=tool)) commands.append(tool) self.emitted_per_turn.append(len(commands)) - return TurnRecord( + record = TurnRecord( iteration=self._iteration, user_input=user_input, agent_output="working", commands=commands, tool_calls_exhausted=should_stop() is StopReason.TOOL_CALL_CAP, ) + return TurnOutcome(record=record, status=AgentEndStatus.COMPLETED, error=None) def _install_fake_simulator( @@ -131,15 +133,16 @@ def __init__(self, responses: list[str], *, tokens: tuple[int, int] = (5, 7)) -> super().__init__(responses) self._tokens = tokens - async def communicate(self, user_input: str, **kwargs: object) -> TurnRecord: - turn = await super().communicate(user_input, **kwargs) + async def communicate(self, user_input: str, **kwargs: object) -> TurnOutcome: + outcome = await super().communicate(user_input, **kwargs) in_tok, out_tok = self._tokens - return TurnRecord( - iteration=turn.iteration, - user_input=turn.user_input, - agent_output=turn.agent_output, + record = TurnRecord( + iteration=outcome.record.iteration, + user_input=outcome.record.user_input, + agent_output=outcome.record.agent_output, token_usage=TokenUsage(uncached_input_tokens=in_tok, output_tokens=out_tok), ) + return TurnOutcome(record=record, status=outcome.status, error=outcome.error) class _ExplodingAgent(Agent): diff --git a/tests/test_spi.py b/tests/test_spi.py index 4fdca890..bfcf8f95 100644 --- a/tests/test_spi.py +++ b/tests/test_spi.py @@ -8,6 +8,7 @@ _ORIGINS = ( "coder_eval.agent", "coder_eval.agents.registry", + "coder_eval.agents.watchdog", "coder_eval.errors", "coder_eval.models", "coder_eval.pricing", @@ -25,6 +26,7 @@ def test_spi_version_is_three() -> None: def test_the_emitter_surface_is_exported() -> None: assert {"TurnEmitter", "TurnOutcome", "Generation", "Window", "TimingBasis"} <= set(spi.__all__) + assert {"run_with_watchdog", "WatchdogFired"} <= set(spi.__all__) def test_the_stop_channel_is_exported() -> None: diff --git a/tests/test_streaming_agent_integration.py b/tests/test_streaming_agent_integration.py index 4e05a753..6df400ef 100644 --- a/tests/test_streaming_agent_integration.py +++ b/tests/test_streaming_agent_integration.py @@ -156,7 +156,7 @@ async def fake_query(**kwargs): yield msg with patch("coder_eval.agents.claude_code_agent.query", side_effect=fake_query): - await agent.communicate("test prompt", stream_callback=callback) + await agent.communicate("test prompt", iteration=1, stream_callback=callback) _assert_well_formed_tree(callback.events, expected_agent_status=AgentEndStatus.COMPLETED) @@ -191,7 +191,7 @@ async def fake_query(**kwargs): yield msg with patch("coder_eval.agents.claude_code_agent.query", side_effect=fake_query): - await agent.communicate("test prompt", stream_callback=callback) + await agent.communicate("test prompt", iteration=1, stream_callback=callback) _assert_well_formed_tree(callback.events, expected_agent_status=AgentEndStatus.COMPLETED) @@ -226,7 +226,7 @@ async def fake_query(**kwargs): yield msg with patch("coder_eval.agents.claude_code_agent.query", side_effect=fake_query): - await agent.communicate("test prompt", stream_callback=callback) + await agent.communicate("test prompt", iteration=1, stream_callback=callback) end_events = [e for e in callback.events if isinstance(e, ToolEndEvent)] assert len(end_events) == 1 @@ -260,7 +260,7 @@ async def fake_query(**kwargs): yield msg with patch("coder_eval.agents.claude_code_agent.query", side_effect=fake_query): - await agent.communicate("test prompt", stream_callback=callback) + await agent.communicate("test prompt", iteration=1, stream_callback=callback) _assert_well_formed_tree(callback.events, expected_agent_status=AgentEndStatus.COMPLETED) @@ -292,7 +292,7 @@ async def fake_query(**kwargs): yield msg with patch("coder_eval.agents.claude_code_agent.query", side_effect=fake_query): - await agent.communicate("hello world", stream_callback=callback) + await agent.communicate("hello world", iteration=1, stream_callback=callback) start = callback.events[0] assert isinstance(start, AgentStartEvent) @@ -323,7 +323,7 @@ async def fake_query(**kwargs): yield msg with patch("coder_eval.agents.claude_code_agent.query", side_effect=fake_query): - returned = await agent.communicate("test prompt", stream_callback=callback) + outcome = await agent.communicate("test prompt", iteration=1, stream_callback=callback) # Replay the captured stream through a fresh collector and confirm it # reduces to a record consistent with what communicate() returned. @@ -335,7 +335,7 @@ async def fake_query(**kwargs): assert rebuilt.user_input == "test prompt" assert len(rebuilt.commands) == 1 assert rebuilt.commands[0].tool_id == "tc" - assert len(returned.commands) == len(rebuilt.commands) + assert len(outcome.record.commands) == len(rebuilt.commands) @pytest.mark.asyncio @@ -359,10 +359,10 @@ async def fake_query(**kwargs): yield msg with patch("coder_eval.agents.claude_code_agent.query", side_effect=fake_query): - turn = await agent.communicate("test prompt") # No callback + outcome = await agent.communicate("test prompt", iteration=1) # No callback - assert turn is not None - assert len(turn.commands) == 1 + assert outcome.record is not None + assert len(outcome.record.commands) == 1 @pytest.mark.asyncio @@ -389,7 +389,7 @@ async def fake_query(**kwargs): yield msg with patch("coder_eval.agents.claude_code_agent.query", side_effect=fake_query): - await agent.communicate("test prompt", stream_callback=scoped_callback) + await agent.communicate("test prompt", iteration=1, stream_callback=scoped_callback) # All events (AgentStart -> ... -> AgentEnd) should now carry the real task ID. assert len(inner_callback.events) > 0 diff --git a/tests/test_sub_agent_runner.py b/tests/test_sub_agent_runner.py index 6e58843f..4d5f439c 100644 --- a/tests/test_sub_agent_runner.py +++ b/tests/test_sub_agent_runner.py @@ -9,6 +9,7 @@ import pytest +from coder_eval.errors.agent import AgentCrashError from coder_eval.errors.timeout import TurnTimeoutError from coder_eval.evaluation.sub_agent import ( SubAgentRunner, @@ -17,6 +18,8 @@ from coder_eval.models import AgentKind, ClaudeCodeAgentConfig, TurnRecord, parse_agent_config from coder_eval.models.routing import DirectRoute from coder_eval.sandbox import Sandbox +from coder_eval.streaming.emitter import TurnOutcome +from coder_eval.streaming.events import AgentEndStatus # Symlink creation on Windows requires either admin privileges or Developer @@ -56,10 +59,14 @@ def _make_turn() -> TurnRecord: return TurnRecord(iteration=1, user_input="x", agent_output='{"score": 1.0, "rationale": "ok"}') +def _make_outcome(record: TurnRecord | None = None) -> TurnOutcome: + return TurnOutcome(record=record or _make_turn(), status=AgentEndStatus.COMPLETED, error=None) + + def _make_mock_agent() -> MagicMock: agent = MagicMock() agent.start = AsyncMock(return_value=None) - agent.communicate = AsyncMock(return_value=_make_turn()) + agent.communicate = AsyncMock(return_value=_make_outcome()) agent.stop = AsyncMock(return_value=None) agent.kill = AsyncMock(return_value=None) return agent @@ -83,7 +90,7 @@ async def test_runner_happy_path(sandbox: Sandbox, tmp_path: Path) -> None: assert turn.agent_output == '{"score": 1.0, "rationale": "ok"}' mock_agent.start.assert_awaited_once() - mock_agent.communicate.assert_awaited_once_with("grade this", timeout=30.0) + mock_agent.communicate.assert_awaited_once_with("grade this", iteration=1, timeout=30.0) mock_agent.stop.assert_awaited() @@ -115,13 +122,13 @@ async def capture_start(workdir: str, **_kwargs: object) -> None: mock_agent.start.side_effect = capture_start with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): # Capture the workdir before it's torn down by the finally block. - async def capture_files(_msg: str, **_kw: object) -> TurnRecord: + async def capture_files(_msg: str, **_kw: object) -> TurnOutcome: workdir = Path(captured["workdir"]) captured["has_reference_dir"] = str((workdir / "_reference").is_dir()) captured["has_main"] = str((workdir / "_reference" / "Main.xaml").is_file()) captured["main_content"] = (workdir / "_reference" / "Main.xaml").read_text() captured["has_subdir"] = str((workdir / "_reference" / "subdir" / "Helper.xaml").is_file()) - return _make_turn() + return _make_outcome() mock_agent.communicate.side_effect = capture_files await runner.run_async("grade", turn_timeout=30.0) @@ -166,12 +173,12 @@ async def capture_start(workdir: str, **_kwargs: object) -> None: mock_agent.start.side_effect = capture_start with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - async def capture_state(_msg: str, **_kw: object) -> TurnRecord: + async def capture_state(_msg: str, **_kw: object) -> TurnOutcome: workdir = Path(captured["workdir"]) ref_main = workdir / "_reference" / "Main.xaml" captured["ref_main_content"] = ref_main.read_text() captured["agent_planted_present"] = str((workdir / "_reference" / "agent_planted.txt").exists()) - return _make_turn() + return _make_outcome() mock_agent.communicate.side_effect = capture_state # Must not raise — this is the regression assertion. @@ -202,10 +209,10 @@ async def capture_start(workdir: str, **_kwargs: object) -> None: mock_agent.start.side_effect = capture_start with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - async def capture_no_ref(_msg: str, **_kw: object) -> TurnRecord: + async def capture_no_ref(_msg: str, **_kw: object) -> TurnOutcome: workdir = Path(captured["workdir"]) captured["has_reference_dir"] = str((workdir / "_reference").exists()) - return _make_turn() + return _make_outcome() mock_agent.communicate.side_effect = capture_no_ref await runner.run_async("grade", turn_timeout=30.0) @@ -274,7 +281,7 @@ async def test_runner_cleans_up_when_cancelled_mid_communicate(sandbox: Sandbox) async def capture_start(path: str, **_kwargs: object) -> None: captured["path"] = path - async def hang_forever(*_args: object, **_kwargs: object) -> TurnRecord: + async def hang_forever(*_args: object, **_kwargs: object) -> TurnOutcome: started.set() await asyncio.sleep(3600) raise AssertionError("should have been cancelled before waking up") @@ -399,7 +406,9 @@ async def test_runner_propagates_turn_timeout(sandbox: Sandbox) -> None: route=DirectRoute(), ) mock_agent = _make_mock_agent() - mock_agent.communicate.side_effect = TurnTimeoutError(30.0, task_id="t", iteration=1) + mock_agent.communicate.return_value = TurnOutcome( + record=_make_turn(), status=AgentEndStatus.TIMEOUT, error="timed out" + ) captured: dict[str, str] = {} async def capture_start(path: str, **_kwargs: object) -> None: @@ -416,6 +425,34 @@ async def capture_start(path: str, **_kwargs: object) -> None: assert not Path(captured["path"]).exists() +async def test_runner_propagates_agent_crash_error(sandbox: Sandbox) -> None: + runner = SubAgentRunner( + sandbox=sandbox, + agent_config=_make_agent_config(), + ignore_patterns=[], + route=DirectRoute(), + ) + mock_agent = _make_mock_agent() + crashed_record = TurnRecord(iteration=1, user_input="x", agent_output="", crashed=True, crash_reason="boom") + mock_agent.communicate.return_value = TurnOutcome( + record=crashed_record, status=AgentEndStatus.CRASHED, error="boom" + ) + captured: dict[str, str] = {} + + async def capture_start(path: str, **_kwargs: object) -> None: + captured["path"] = path + + mock_agent.start.side_effect = capture_start + + with ( + patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent), + pytest.raises(AgentCrashError, match="boom"), + ): + await runner.run_async("grade", turn_timeout=30.0) + + assert not Path(captured["path"]).exists() + + # --- security contract --- @@ -648,12 +685,12 @@ async def capture_start(workdir: str, **_kwargs: object) -> None: mock_agent.start.side_effect = capture_start with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - async def capture_state(_msg: str, **_kw: object) -> TurnRecord: + async def capture_state(_msg: str, **_kw: object) -> TurnOutcome: workdir = Path(captured["workdir"]) inner = workdir / "_reference" / "nested" / "_reference" / "inner.txt" captured["inner_present"] = str(inner.exists()) captured["inner_content"] = inner.read_text(encoding="utf-8") if inner.exists() else "" - return _make_turn() + return _make_outcome() mock_agent.communicate.side_effect = capture_state await runner.run_async("grade", turn_timeout=30.0) @@ -688,11 +725,11 @@ async def capture_start(workdir: str, **_kwargs: object) -> None: mock_agent.start.side_effect = capture_start with patch("coder_eval.evaluation.sub_agent.ClaudeCodeAgent", return_value=mock_agent): - async def capture_state(_msg: str, **_kw: object) -> TurnRecord: + async def capture_state(_msg: str, **_kw: object) -> TurnOutcome: workdir = Path(captured["workdir"]) captured["keep_present"] = str((workdir / "_reference" / "keep.txt").exists()) captured["log_present"] = str((workdir / "_reference" / "drop.log").exists()) - return _make_turn() + return _make_outcome() mock_agent.communicate.side_effect = capture_state await runner.run_async("grade", turn_timeout=30.0) diff --git a/tests/test_timeout_orchestrator.py b/tests/test_timeout_orchestrator.py index e4931804..a866bbba 100644 --- a/tests/test_timeout_orchestrator.py +++ b/tests/test_timeout_orchestrator.py @@ -26,6 +26,8 @@ ) from coder_eval.orchestrator import Orchestrator from coder_eval.sandbox import Sandbox +from coder_eval.streaming.emitter import TurnOutcome +from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus, AgentStartEvent def _make_task(*, turn_timeout: float | None = None, task_timeout: float | None = None): @@ -72,6 +74,18 @@ def _make_turn_record(iteration: int = 1) -> TurnRecord: ) +def _completed_outcome(record: TurnRecord) -> TurnOutcome: + return TurnOutcome(record=record, status=AgentEndStatus.COMPLETED, error=None) + + +def _crashed_outcome(record: TurnRecord, error: str) -> TurnOutcome: + return TurnOutcome(record=record, status=AgentEndStatus.CRASHED, error=error) + + +def _timeout_outcome(record: TurnRecord, error: str) -> TurnOutcome: + return TurnOutcome(record=record, status=AgentEndStatus.TIMEOUT, error=error) + + def _make_initialized_orchestrator(task: TaskDefinition, tmp_path) -> Orchestrator: """Build an Orchestrator with a pre-initialized EvaluationResult and mock sandbox/checker.""" run_dir = tmp_path / "run" / "timeout_test" @@ -116,7 +130,8 @@ async def test_turn_timeout_propagates_from_agent(tmp_path) -> None: async def timeout_communicate(_prompt, **kwargs): await asyncio.sleep(0.01) - raise TurnTimeoutError(turn_timeout, iteration=1) + record = TurnRecord(iteration=kwargs["iteration"], user_input=_prompt, agent_output="", crashed=True) + return _timeout_outcome(record, "agent turn timed out") mock_agent.communicate = timeout_communicate orchestrator.agent = mock_agent @@ -196,7 +211,7 @@ async def test_no_timeout_when_none(tmp_path) -> None: orchestrator = _make_initialized_orchestrator(task, tmp_path) mock_agent = AsyncMock() - mock_agent.communicate = AsyncMock(return_value=_make_turn_record()) + mock_agent.communicate = AsyncMock(return_value=_completed_outcome(_make_turn_record())) orchestrator.agent = mock_agent orchestrator.success_checker.check_all_async = AsyncMock( # type: ignore[union-attr] @@ -241,10 +256,11 @@ async def slow_loop(): async def test_task_timeout_recovers_the_killed_turn(tmp_path) -> None: """A hard-killed task's spend lands on the result instead of vanishing. - The agent parks the interrupted turn on ``pending_turn`` when it is cancelled, - and the task-timeout handler is the only reader of that slot: the cancel is a - BaseException, so it never reaches the retry executor's per-attempt hook that - drains it on a turn-level timeout. Without the drain the row reports no turns + The in-flight attempt's ``EventCollector`` is parked on + ``orchestrator._attempt_collector`` for the duration of the attempt, and + ``_drain_killed_turn`` is the only reader of that slot: the cancel is a + BaseException, so no outcome ever returns to the retry wrapper that would + append it. Without the drain the row reports no turns and no cost for a task that spent real money. """ task = _make_task(task_timeout=0.1) @@ -263,13 +279,16 @@ async def test_task_timeout_recovers_the_killed_turn(tmp_path) -> None: token_usage=TokenUsage(uncached_input_tokens=40_000, output_tokens=2_000, total_cost_usd=0.15), ) + collector = MagicMock() + collector.ended = True + collector.build_turn_record.return_value = partial + mock_agent = MagicMock() - mock_agent.pending_turn = partial - mock_agent.discard_pending_turn = AsyncMock() mock_agent.get_sdk_options = MagicMock(return_value=None) orchestrator.agent = mock_agent async def slow_loop(): + orchestrator._attempt_collector = collector await asyncio.sleep(10) return False @@ -282,8 +301,6 @@ async def slow_loop(): assert result.total_token_usage is not None assert result.total_token_usage.output_tokens == 2_000 assert result.total_token_usage.total_cost_usd == pytest.approx(0.15) - # Drained through the documented contract, so the slot is left clean. - mock_agent.discard_pending_turn.assert_awaited_once() @pytest.mark.asyncio @@ -302,7 +319,6 @@ async def test_task_timeout_with_nothing_to_recover_still_lands(tmp_path) -> Non orchestrator._cleanup = AsyncMock() # type: ignore[method-assign] mock_agent = MagicMock() - mock_agent.pending_turn = None mock_agent.get_sdk_options = MagicMock(return_value=None) orchestrator.agent = mock_agent @@ -331,7 +347,8 @@ async def test_turn_timeout_not_rewrapped_as_task_timeout(tmp_path) -> None: async def turn_out_communicate(_prompt, **kwargs): await asyncio.sleep(0.01) - raise TurnTimeoutError(turn_timeout, iteration=1) + record = TurnRecord(iteration=kwargs["iteration"], user_input=_prompt, agent_output="", crashed=True) + return _timeout_outcome(record, "agent turn timed out") mock_agent.communicate = turn_out_communicate orchestrator.agent = mock_agent @@ -562,8 +579,6 @@ async def test_turn_timeout_is_per_attempt_not_cycle(tmp_path): same ``timeout=turn_timeout`` kwarg. A shared retry-cycle budget would decrement (or omit) the second-attempt timeout. """ - from coder_eval.errors import AgentCrashError - task = _make_task(turn_timeout=1.0) run_dir = tmp_path / "run" / "per_attempt_budget" run_dir.mkdir(parents=True) @@ -589,9 +604,8 @@ async def test_turn_timeout_is_per_attempt_not_cycle(tmp_path): async def flaky_communicate(_prompt, **kwargs): timeouts_seen.append(kwargs.get("timeout")) if len(timeouts_seen) == 1: - mock_agent.pending_turn = partial_record - raise AgentCrashError("mid-turn failure") - return success_record + return _crashed_outcome(partial_record, "mid-turn failure") + return _completed_outcome(success_record) mock_agent = AsyncMock() mock_agent.communicate = flaky_communicate @@ -620,64 +634,139 @@ async def fast_retry_sleep(delay: float) -> None: assert success is True assert timeouts_seen == [1.0, 1.0], "every attempt must receive turn_timeout fresh" - # Result.turns: partial (from on_attempt_error) + success (from main flow). + # Result.iterations: the CRASHED outcome's partial (appended by + # `_communicate_with_retry`) + the retry's success record (appended by the + # main flow). assert len(orchestrator.result.iterations) == 2 assert orchestrator.result.iterations[0].crashed is True assert orchestrator.result.iterations[1].crashed is False @pytest.mark.asyncio -async def test_wait_for_backstop_calls_discard_pending_turn(tmp_path): - """When the outer ``asyncio.wait_for`` fires, the orchestrator must call - ``agent.discard_pending_turn()`` after ``agent.kill()``. - - The wait_for cancels ``communicate()`` via ``CancelledError`` - (a ``BaseException``), which bypasses the agent's ``except Exception`` - handlers — so the per-turn iteration counter that ``communicate()`` bumped - at entry never gets rolled back by the agent's normal failure path. The - orchestrator must invoke ``discard_pending_turn()`` so any future change - to the AGENT_TIMEOUT retry policy doesn't silently break the - "partials and the retry share an iteration number" contract. +async def test_crash_then_success_appends_both_records_same_iteration(tmp_path) -> None: + """A crashed attempt's partial and the retry's record both land on + ``result.iterations``, in order, sharing the iteration number — a retry + resumes the same turn rather than starting a new one. """ - task = _make_task(turn_timeout=0.05) - run_dir = tmp_path / "run" / "discard_pending" - run_dir.mkdir(parents=True) + task = _make_task() + orchestrator = _make_initialized_orchestrator(task, tmp_path) - orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="v") - orchestrator._build_monitor() - orchestrator.result = EvaluationResult( - task_id="discard_pending", - task_description="discard_pending", - variant_id="v", - agent_type=AgentKind.CLAUDE_CODE, - started_at=datetime.now(), - final_status="FAILURE", - iteration_count=0, - environment_info={}, + partial = TurnRecord(iteration=1, user_input="p", agent_output="", crashed=True) + success = _make_turn_record(iteration=1) + calls: list[int] = [] + + async def flaky_communicate(_prompt, **kwargs): + calls.append(kwargs["iteration"]) + if len(calls) == 1: + return _crashed_outcome(partial, "mid-turn failure") + return _completed_outcome(success) + + mock_agent = AsyncMock() + mock_agent.communicate = flaky_communicate + orchestrator.agent = mock_agent + orchestrator.success_checker.check_all_async = AsyncMock( # type: ignore[union-attr] + return_value=[CriterionResult(criterion_type="file_exists", description="test", score=1.0)] ) - # An Event().wait() coroutine never completes on its own — wait_for must - # cancel it. Plain asyncio.sleep would be vulnerable to a global sleep - # patch elsewhere; Event.wait isolates this test from that. + async def fast_retry_sleep(delay: float) -> None: + return None + + with ( + patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None), + patch("asyncio.sleep", side_effect=fast_retry_sleep), + ): + success_result = await orchestrator._evaluation_loop() + + assert success_result is True + assert calls == [1, 1] + assert orchestrator.result.iterations == [partial, success] + + +@pytest.mark.asyncio +async def test_timeout_outcome_is_not_retried(tmp_path) -> None: + """A TIMEOUT outcome is terminal: ``communicate`` is called exactly once, + and its record is appended to ``result.iterations`` before the + ``TurnTimeoutError`` propagates. + """ + task = _make_task(turn_timeout=5.0) + orchestrator = _make_initialized_orchestrator(task, tmp_path) + + record = TurnRecord(iteration=1, user_input="p", agent_output="", crashed=True) + calls = 0 + + async def timeout_once(_prompt, **kwargs): + nonlocal calls + calls += 1 + return _timeout_outcome(record, "timed out") + + mock_agent = AsyncMock() + mock_agent.communicate = timeout_once + orchestrator.agent = mock_agent + + with pytest.raises(TurnTimeoutError): + await orchestrator._evaluation_loop() + + assert calls == 1 + assert orchestrator.result.iterations == [record] + + +@pytest.mark.asyncio +async def test_wait_for_backstop_appends_record_when_agent_ended_its_turn(tmp_path, monkeypatch) -> None: + """The ``wait_for`` backstop kills a hung agent and, when the attempt's + ``EventCollector`` already saw an ``AgentEndEvent`` before the hang, appends + that record to ``result.iterations``. + """ + monkeypatch.setattr("coder_eval.orchestrator._WAIT_FOR_GRACE_SECONDS", 0.05) + task = _make_task(turn_timeout=0.05) + orchestrator = _make_initialized_orchestrator(task, tmp_path) + never_set = asyncio.Event() - async def hanging_communicate(_prompt, **kwargs): + async def hanging_but_ended(_prompt, *, stream_callback, **kwargs): + stream_callback.on_event(AgentStartEvent(task_id=task.task_id, iteration=kwargs["iteration"], prompt=_prompt)) + stream_callback.on_event( + AgentEndEvent( + task_id=task.task_id, iteration=kwargs["iteration"], crashed=True, status=AgentEndStatus.CRASHED + ) + ) await never_set.wait() raise AssertionError("unreachable: wait_for should have cancelled this") mock_agent = AsyncMock() - mock_agent.communicate = hanging_communicate + mock_agent.communicate = hanging_but_ended mock_agent.kill = AsyncMock() - mock_agent.discard_pending_turn = AsyncMock() orchestrator.agent = mock_agent - mock_sandbox = MagicMock() - mock_sandbox.sandbox_dir = tmp_path / "sandbox" - mock_sandbox.sandbox_dir.mkdir() - orchestrator.sandbox = mock_sandbox + with ( + patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None), + pytest.raises(TurnTimeoutError), + ): + await orchestrator._evaluation_loop() + + assert mock_agent.kill.await_count == 1 + assert len(orchestrator.result.iterations) == 1 + assert orchestrator.result.iterations[0].crashed is True - mock_checker = MagicMock() - orchestrator.success_checker = mock_checker + +@pytest.mark.asyncio +async def test_wait_for_backstop_appends_nothing_when_agent_never_ended(tmp_path, monkeypatch) -> None: + """When the hung agent never got as far as an ``AgentEndEvent``, the + backstop's kill has nothing to recover and appends nothing. + """ + monkeypatch.setattr("coder_eval.orchestrator._WAIT_FOR_GRACE_SECONDS", 0.05) + task = _make_task(turn_timeout=0.05) + orchestrator = _make_initialized_orchestrator(task, tmp_path) + + never_set = asyncio.Event() + + async def hanging_never_started(_prompt, **kwargs): + await never_set.wait() + raise AssertionError("unreachable: wait_for should have cancelled this") + + mock_agent = AsyncMock() + mock_agent.communicate = hanging_never_started + mock_agent.kill = AsyncMock() + orchestrator.agent = mock_agent with ( patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None), @@ -685,32 +774,169 @@ async def hanging_communicate(_prompt, **kwargs): ): await orchestrator._evaluation_loop() - # kill() and discard_pending_turn() must both have run. assert mock_agent.kill.await_count == 1 - assert mock_agent.discard_pending_turn.await_count == 1 + assert orchestrator.result.iterations == [] + + +@pytest.mark.asyncio +async def test_task_timeout_recovers_in_flight_turn_via_attempt_collector(tmp_path) -> None: + """A real task-timeout cancellation, hitting the agent mid-``communicate``, + is recovered through ``orchestrator._attempt_collector`` (not + ``agent.pending_turn``) by ``_drain_killed_turn``. + """ + task = _make_task(task_timeout=0.1) + run_dir = tmp_path / "run" / "drain_killed_turn" + run_dir.mkdir(parents=True) + + orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant") + orchestrator._build_monitor() + orchestrator._setup = AsyncMock() # type: ignore[method-assign] + orchestrator._cleanup = AsyncMock() # type: ignore[method-assign] + + mock_sandbox = MagicMock() + mock_sandbox.sandbox_dir = tmp_path / "sandbox" + mock_sandbox.sandbox_dir.mkdir() + orchestrator.sandbox = mock_sandbox + orchestrator.success_checker = MagicMock() + + async def hang_after_ending(_prompt, *, stream_callback, **kwargs): + stream_callback.on_event(AgentStartEvent(task_id=task.task_id, iteration=kwargs["iteration"], prompt=_prompt)) + stream_callback.on_event( + AgentEndEvent( + task_id=task.task_id, iteration=kwargs["iteration"], crashed=True, status=AgentEndStatus.CRASHED + ) + ) + await asyncio.Event().wait() + + mock_agent = AsyncMock() + mock_agent.communicate = hang_after_ending + mock_agent.kill_sync = MagicMock() + mock_agent.get_sdk_options = MagicMock(return_value=None) + orchestrator.agent = mock_agent + + result = await orchestrator.run() + + assert result.final_status == "TIMEOUT" + assert len(result.iterations) == 1 + assert result.iterations[0].crashed is True @pytest.mark.asyncio -async def test_claude_agent_discard_pending_turn_rolls_back_iteration(): - """ClaudeCodeAgent.discard_pending_turn is slot-gated: it decrements _iteration - only when pending_turn is set, and is idempotent when the slot is empty. +async def test_task_timeout_after_finished_attempt_appends_nothing_extra(tmp_path) -> None: + """A task timeout that fires AFTER the agent's attempt already finished + (e.g. during a slow criteria check) finds ``_attempt_collector`` already + cleared to ``None`` and drains nothing extra. """ - from coder_eval.agents.claude_code_agent import ClaudeCodeAgent - from coder_eval.models import AgentKind, TurnRecord, parse_agent_config - - config = parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits") - agent = ClaudeCodeAgent(config) - - # Idle agent (no pending turn): discard is a no-op. Negative values would - # break the "partials and retry share an iteration" contract. - assert agent._iteration == 0 - await agent.discard_pending_turn() - assert agent._iteration == 0 - - # With pending_turn set: discard clears the slot AND decrements _iteration. - partial = TurnRecord(iteration=3, user_input="p", agent_output="", crashed=True) - agent._iteration = 3 - agent.pending_turn = partial - await agent.discard_pending_turn() - assert agent.pending_turn is None - assert agent._iteration == 2 + task = _make_task(task_timeout=0.15) + run_dir = tmp_path / "run" / "drain_after_finished_attempt" + run_dir.mkdir(parents=True) + + orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant") + orchestrator._build_monitor() + orchestrator._setup = AsyncMock() # type: ignore[method-assign] + orchestrator._cleanup = AsyncMock() # type: ignore[method-assign] + + mock_sandbox = MagicMock() + mock_sandbox.sandbox_dir = tmp_path / "sandbox" + mock_sandbox.sandbox_dir.mkdir() + orchestrator.sandbox = mock_sandbox + + record = _make_turn_record() + + async def quick_communicate(_prompt, **kwargs): + callback = kwargs["stream_callback"] + callback.on_event(AgentStartEvent(task_id="t", iteration=kwargs["iteration"])) + callback.on_event(AgentEndEvent(task_id="t", iteration=kwargs["iteration"])) + return _completed_outcome(record) + + mock_agent = AsyncMock() + mock_agent.communicate = quick_communicate + mock_agent.kill_sync = MagicMock() + mock_agent.get_sdk_options = MagicMock(return_value=None) + orchestrator.agent = mock_agent + + mock_checker = MagicMock() + + async def slow_check(*_args, **_kwargs): + await asyncio.sleep(10) + return [] + + mock_checker.check_all_async = slow_check + orchestrator.success_checker = mock_checker + + result = await orchestrator.run() + + assert orchestrator._attempt_collector is None + assert result.final_status == "TIMEOUT" + assert result.iterations == [record] + + +@pytest.mark.asyncio +async def test_unhandled_end_status_raises_runtime_error_and_ends_error(tmp_path, monkeypatch) -> None: + """An outcome status outside the returned/crash/timeout allowlist is a + harness bug: it raises ``RuntimeError`` and the task ends ``ERROR``. + """ + monkeypatch.setattr("coder_eval.orchestrator._RETURNED_END_STATUSES", frozenset()) + task = _make_task() + run_dir = tmp_path / "run" / "unhandled_status" + run_dir.mkdir(parents=True) + + orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant") + orchestrator._build_monitor() + orchestrator._setup = AsyncMock() # type: ignore[method-assign] + orchestrator._cleanup = AsyncMock() # type: ignore[method-assign] + + mock_sandbox = MagicMock() + mock_sandbox.sandbox_dir = tmp_path / "sandbox" + mock_sandbox.sandbox_dir.mkdir() + orchestrator.sandbox = mock_sandbox + orchestrator.success_checker = MagicMock() + + record = _make_turn_record() + calls: list[int] = [] + + async def completed_communicate(_prompt, **kwargs): + calls.append(kwargs["iteration"]) + return _completed_outcome(record) + + mock_agent = AsyncMock() + mock_agent.communicate = completed_communicate + mock_agent.get_sdk_options = MagicMock(return_value=None) + orchestrator.agent = mock_agent + + result = await orchestrator.run() + + assert result.final_status == "ERROR" + assert "unhandled end status" in (result.error_message or "") + assert calls == [1], "an unhandled status is a harness bug, never a retried turn" + + +@pytest.mark.asyncio +async def test_every_attempt_crashing_appends_every_partial_before_the_error(tmp_path) -> None: + """CRASHED on the final retry still appends its record before ``AgentCrashError`` escapes.""" + task = _make_task() + orchestrator = _make_initialized_orchestrator(task, tmp_path) + partials = [TurnRecord(iteration=1, user_input="p", agent_output=f"", crashed=True) for n in range(3)] + calls: list[int] = [] + + async def crashing_communicate(_prompt, **kwargs): + calls.append(kwargs["iteration"]) + return _crashed_outcome(partials[len(calls) - 1], "provider exploded") + + mock_agent = AsyncMock() + mock_agent.communicate = crashing_communicate + orchestrator.agent = mock_agent + + async def fast_retry_sleep(delay: float) -> None: + return None + + with ( + patch("coder_eval.orchestrator.resolve_reference_dir", return_value=None), + patch("asyncio.sleep", side_effect=fast_retry_sleep), + pytest.raises(AgentCrashError, match="provider exploded"), + ): + await orchestrator._evaluation_loop() + + assert calls == [1, 1, 1] + assert orchestrator.result.iterations == partials + assert orchestrator._attempt_collector is None diff --git a/tests/test_token_usage.py b/tests/test_token_usage.py index ed78af8b..7397b917 100644 --- a/tests/test_token_usage.py +++ b/tests/test_token_usage.py @@ -12,7 +12,6 @@ _is_sdk_result_message, _is_task_notification, ) -from coder_eval.errors import AgentCrashError from coder_eval.models import ( AgentKind, AssistantMessage, @@ -26,6 +25,7 @@ ) from coder_eval.pricing import calculate_cost from coder_eval.reports import ReportGenerator +from coder_eval.streaming.events import AgentEndStatus def _assistant( @@ -523,7 +523,7 @@ async def mock_query(*args, **kwargs): yield sdk_result with patch("coder_eval.agents.claude_code_agent.query", side_effect=mock_query): - record = await agent.communicate("test prompt") + record = (await agent.communicate("test prompt", iteration=1)).record assert record.token_usage is not None assert record.token_usage.uncached_input_tokens == 1000 @@ -559,7 +559,7 @@ async def mock_query(*args, **kwargs): yield assistant_msg with patch("coder_eval.agents.claude_code_agent.query", side_effect=mock_query): - record = await agent.communicate("test prompt") + record = (await agent.communicate("test prompt", iteration=1)).record assert record.token_usage is None @@ -568,7 +568,7 @@ async def test_crashed_turn_backfills_cost_end_to_end(self): """End-to-end wiring (issue #386): on a crash there is no ResultMessage, so the SDK supplies no cost. The agent must thread its resolved ``effective_model`` through ``communicate() → _finalize → - _build_token_usage`` so the partial ``pending_turn`` records a + _build_token_usage`` so the crashed partial record carries a rate-card cost instead of None. The unit tests for ``_build_token_usage`` pass an explicit model; this proves the closure is actually wired.""" config = parse_agent_config( @@ -602,14 +602,14 @@ async def mock_query(*args, **kwargs): with ( patch("coder_eval.agents.claude_code_agent.SubprocessCLITransport", return_value=MagicMock()), patch("coder_eval.agents.claude_code_agent.query", side_effect=mock_query), - pytest.raises(AgentCrashError), ): - await agent.communicate("test prompt") + outcome = await agent.communicate("test prompt", iteration=1) - # The crashed partial turn is parked on pending_turn with cost backfilled. - assert agent.pending_turn is not None - assert agent.pending_turn.crashed is True - usage = agent.pending_turn.token_usage + # The crashed partial turn is returned on the outcome with cost backfilled. + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.record is not None + assert outcome.record.crashed is True + usage = outcome.record.token_usage assert usage is not None expected = calculate_cost( "claude-opus-4-8", @@ -653,7 +653,7 @@ async def mock_query(*args, **kwargs): yield sdk_result with patch("coder_eval.agents.claude_code_agent.query", side_effect=mock_query): - record = await agent.communicate("test prompt") + record = (await agent.communicate("test prompt", iteration=1)).record assert record.token_usage is not None assert record.token_usage.cache_creation_input_tokens == 0 @@ -716,7 +716,7 @@ async def mock_query(*args, **kwargs): yield result with patch("coder_eval.agents.claude_code_agent.query", side_effect=mock_query): - record = await agent.communicate("delegate it") + record = (await agent.communicate("delegate it", iteration=1)).record # Turn total is the model_usage figure — UNCHANGED by the synthetic message. assert record.token_usage is not None diff --git a/tests/test_user_simulator.py b/tests/test_user_simulator.py index 859f2570..7034aec8 100644 --- a/tests/test_user_simulator.py +++ b/tests/test_user_simulator.py @@ -14,8 +14,13 @@ import pytest -from coder_eval.models import SimulationConfig +from coder_eval.agent import Agent, AgentState +from coder_eval.errors import AgentCrashError +from coder_eval.models import SimulationConfig, TurnRecord from coder_eval.simulation.user_simulator import UserSimulator +from coder_eval.streaming.emitter import TurnOutcome +from coder_eval.streaming.events import AgentEndStatus +from tests.fixtures.harness_stubs import stub_contract from tests.fixtures.text_stub_agent import TextStubAgent @@ -159,7 +164,74 @@ async def communicate(self, user_input: str, **kwargs: object): UserSimulator(config=_sim_cfg(), task_description="T", initial_prompt="start", agent_override=stub) ) await sim.next_user_message([_pair("start", "reply")]) - assert stub.kwargs == [{}] + assert stub.kwargs == [{"iteration": 1}] + await sim.stop() + + +class _OutcomeStubAgent(Agent): + """Minimal Agent fake returning a caller-supplied ``TurnOutcome`` per call. + + Unlike ``TextStubAgent`` (which only ever produces a clean turn), this lets + a test drive an arbitrary ``TurnOutcome`` (including CRASHED/TIMEOUT), and + records the ``iteration`` each ``communicate`` call was given. + """ + + contract = stub_contract() + + def __init__(self, outcomes: list[TurnOutcome]) -> None: + self._outcomes = list(outcomes) + self.iterations_seen: list[int] = [] + self._state = AgentState.WORKING + self.working_directory: Path | None = None + + async def start( + self, + working_directory: str, + *, + env_path_prepend: list[str] | None = None, + plugin_tools_dir: str | None = None, + plugin_root: Path | None = None, + ) -> None: + self.working_directory = Path(working_directory) + self._state = AgentState.WORKING + + async def stop(self) -> None: + self._state = AgentState.FINISHED + + def get_state(self) -> AgentState: + return self._state + + async def communicate(self, user_input: str, *, iteration: int, **kwargs: object) -> TurnOutcome: + self.iterations_seen.append(iteration) + return self._outcomes.pop(0) if len(self._outcomes) > 1 else self._outcomes[0] + + +class TestCommunicateOutcome: + """``next_user_message`` unwraps the underlying agent's ``TurnOutcome`` via ``record_or_raise()``.""" + + async def test_crashed_outcome_raises_agent_crash_error(self): + crashed_record = TurnRecord(iteration=1, user_input="start", agent_output="", crashed=True) + stub = _OutcomeStubAgent([TurnOutcome(record=crashed_record, status=AgentEndStatus.CRASHED, error="boom")]) + sim = await _make_started( + UserSimulator(config=_sim_cfg(), task_description="T", initial_prompt="start", agent_override=stub) + ) + with pytest.raises(AgentCrashError, match="boom"): + await sim.next_user_message([_pair("start", "reply")]) + await sim.stop() + + async def test_iteration_increments_per_simulator_turn(self): + def _completed(n: int) -> TurnOutcome: + record = TurnRecord(iteration=n, user_input="x", agent_output=f"reply {n}") + return TurnOutcome(record=record, status=AgentEndStatus.COMPLETED, error=None) + + stub = _OutcomeStubAgent([_completed(1), _completed(2), _completed(3)]) + sim = await _make_started( + UserSimulator(config=_sim_cfg(), task_description="T", initial_prompt="start", agent_override=stub) + ) + await sim.next_user_message([]) + await sim.next_user_message([_pair("start", "reply 1")]) + await sim.next_user_message([_pair("reply 1", "reply 2")]) + assert stub.iterations_seen == [1, 2, 3] await sim.stop() diff --git a/tests/test_watchdog.py b/tests/test_watchdog.py index 85301e2f..f3b912ba 100644 --- a/tests/test_watchdog.py +++ b/tests/test_watchdog.py @@ -3,13 +3,14 @@ from __future__ import annotations import asyncio +import contextlib import logging import threading import time import pytest -from coder_eval.agents.watchdog import ThreadedWatchdog +from coder_eval.agents.watchdog import ThreadedWatchdog, WatchdogFired, run_with_watchdog def test_watchdog_fires_and_invokes_callback() -> None: @@ -116,3 +117,71 @@ def test_watchdog_logger_emits_warning(caplog: pytest.LogCaptureFixture) -> None assert wd.fired is True assert any("my-label" in rec.message and rec.levelname == "WARNING" for rec in caplog.records) + + +async def _sleep(seconds: float) -> str: + await asyncio.sleep(seconds) + return "done" + + +class TestRunWithWatchdog: + """The turn body runs as a child task, so a watchdog timeout never lands a cancel on the caller.""" + + async def test_a_fired_watchdog_raises_and_leaves_the_caller_uncancelled(self) -> None: + async def caller() -> str: + fired: list[bool] = [] + try: + async with asyncio.timeout(0.6): + with pytest.raises(WatchdogFired): + await run_with_watchdog( + _sleep(5), timeout_seconds=0.1, on_timeout=lambda: fired.append(True), label="t" + ) + task = asyncio.current_task() + assert task is not None and task.cancelling() == 0 + assert fired == [True] + await asyncio.sleep(5) + except TimeoutError: + return "enclosing timeout raised TimeoutError" + return "no timeout" + + assert await asyncio.ensure_future(caller()) == "enclosing timeout raised TimeoutError" + + async def test_cancelling_the_caller_propagates_and_cancels_the_body(self) -> None: + started = asyncio.Event() + bodies: list[asyncio.Task[object]] = [] + + async def body() -> str: + task = asyncio.current_task() + assert task is not None + bodies.append(task) + started.set() + await asyncio.sleep(5) + return "done" + + outer = asyncio.ensure_future(run_with_watchdog(body(), timeout_seconds=10, on_timeout=lambda: None, label="t")) + await started.wait() + outer.cancel() + with pytest.raises(asyncio.CancelledError): + await outer + await asyncio.sleep(0) + assert bodies[0].cancelled() + + async def test_a_body_finishing_at_the_deadline_leaks_no_late_cancel(self) -> None: + async def caller() -> str: + value = await run_with_watchdog(_sleep(0.19), timeout_seconds=0.2, on_timeout=lambda: None, label="t") + await asyncio.sleep(0.3) + return value + + for _ in range(20): + with contextlib.suppress(WatchdogFired): + assert await asyncio.ensure_future(caller()) == "done" + + async def test_a_body_exception_propagates_unchanged(self) -> None: + async def body() -> str: + raise ValueError("boom") + + with pytest.raises(ValueError, match="boom"): + await run_with_watchdog(body(), timeout_seconds=5, on_timeout=lambda: None, label="t") + + async def test_no_timeout_still_runs_the_body(self) -> None: + assert await run_with_watchdog(_sleep(0), timeout_seconds=None, on_timeout=lambda: None, label="t") == "done" From 5b6e12aff07aee4cc8e66da7b64694c21cf59d7b Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 18:19:32 -0700 Subject: [PATCH 04/25] =?UTF-8?q?feat(agents):=204/10=20=E2=80=94=20port?= =?UTF-8?q?=20Pi=20onto=20TurnEmitter;=20the=20CLI=20never=20inherits=20st?= =?UTF-8?q?din?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _PiTurnState becomes _PiDecoder, a per-turn reducer that calls the emitter; PiAgent.communicate returns outcomes directly. The spawn passes stdin=DEVNULL: pi reads a non-TTY stdin to EOF before it emits anything, so an inherited open stdin stalled every turn to its deadline. The identity case runs through coder_eval.testing.replay; pi_f_duplicate_turn_end now balances. Reviewed golden diffs: result_summary: pi_a/b/c result null -> the final reply; pi_e summary -> null. sequence_number 1-based -> 0-based: pi_b 1,2 -> 0,1; pi_c, pi_d 1 -> 0. pi_d orphan error_message "no result observed" -> null. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 18 + docs/agents/PI.md | 4 + src/coder_eval/agents/pi_agent.py | 648 +++++------------- src/coder_eval/streaming/emitter.py | 4 + .../expected/pi_a_single_text_turn.json | 2 +- .../expected/pi_b_tool_call_resolved.json | 6 +- .../expected/pi_c_multi_turn_tiling.json | 4 +- .../expected/pi_d_orphaned_tool.json | 4 +- .../expected/pi_e_error_after_generation.json | 7 +- tests/test_agent_golden_master.py | 27 +- tests/test_pi_agent.py | 574 +++++++--------- tests/test_timing_identity_contract.py | 72 +- 12 files changed, 511 insertions(+), 859 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index be3eccbf..e4d97786 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -613,6 +613,24 @@ profiles and loses the prepend again. Nested zsh keeps it, because `ZDOTDIR` sta exported. No-op on Windows, where Codex shells through PowerShell (`-NoProfile`) or `cmd /c`, neither of which re-sources a profile chain. +## Why a CLI never inherits stdin + +`pi` (`readPipedStdin()`) and `opencode` (`process.stdin.isTTY ? void 0 : await +Bun.stdin.text()`) both read stdin TO EOF when it is not a TTY, before they emit anything. +A CLI spawned without `stdin=` inherits the parent's stdin, so when `coder-eval` itself runs +with stdin on a pipe that stays open (a backgrounded or tool-spawned batch), every CLI +blocks with zero events until the 300 s `turn_timeout`. Measured on 2026-09-16: + +| command | result | +|---|---| +| `(sleep 25) \| timeout 15 pi -p --mode json … "Reply PONG"` | 0 lines, killed at 25 s | +| `pi -p --mode json … "Reply PONG" < /dev/null` | 24 lines, exit 0 in 1 s | +| `(sleep 25) \| timeout 15 opencode run --format json … "Reply PONG"` | 0 lines, killed at 25 s | +| `coder-eval run tasks/pi_smoke_test.yaml -D run_limits.turn_timeout=40 < <(sleep 170)` | `ERROR` after 40 s, 0 commands | +| the same with `< /dev/null` | `SUCCESS` in 10 s | + +So every CLI spawn passes `stdin=asyncio.subprocess.DEVNULL`, which gives an immediate EOF. + ## Reaping the CLI harnesses `opencode run` leaves a local server child alive after the CLI exits, and it INHERITS the diff --git a/docs/agents/PI.md b/docs/agents/PI.md index 0fc1bf01..823625f9 100644 --- a/docs/agents/PI.md +++ b/docs/agents/PI.md @@ -242,6 +242,10 @@ docker` whenever the task prompt or workspace is not fully trusted. - **No sub-agent attribution.** Pi's CLI stream does not expose nested agent generations, so per-sub-agent token grouping (available for Claude and Codex) is not derivable. +- **The CLI never inherits stdin.** `pi -p` reads a non-TTY stdin to EOF before it + emits anything, so a CLI that inherited an open stdin (a backgrounded or + tool-spawned `coder-eval run`) would stall with zero events until `turn_timeout`. + The adapter spawns it with stdin on `/dev/null`. - **Cooperative stop is at event granularity.** `should_stop` is polled between events and honored by terminating the CLI, so `stop_early` works, but the cut lands on an event boundary rather than mid-tool. Pi streams incrementally, so diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 80ecb2e6..fd044037 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -1,16 +1,13 @@ """Pi agent implementation (the ``pi`` Node coding agent — https://pi.dev/). Drives the ``pi`` CLI in JSON print mode, which streams newline-delimited JSON -events on stdout, and reduces that stream into the standardized coder_eval event -protocol so :class:`EventCollector` builds the ``TurnRecord``. The design mirrors -:mod:`coder_eval.agents.opencode_agent`. +events on stdout, and reduces that stream through one ``TurnEmitter`` per turn. Three grammar facts that are not obvious from the event names (``pi`` 0.84.4): - ``agent_start`` can appear MORE THAN ONCE per invocation — Pi auto-retries a transient provider error internally — and ``agent_end`` is therefore NOT - terminal. ``agent_settled`` (or EOF) is; the single ``AgentEndEvent`` is - emitted there. + terminal. ``agent_settled`` (or EOF) is; the turn ends there. - ``turn_start`` is one per agent-loop step (``num_turns`` on the record). - ``message_end`` is ignored for token accounting: ``turn_end`` echoes the same assistant usage once per step, so reading both would double-count. @@ -34,54 +31,35 @@ import tempfile import time from collections.abc import Callable +from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from typing import Any, Literal, NoReturn +from typing import Any from uuid import uuid4 from coder_eval.agent import Agent -from coder_eval.errors import AgentCrashError, TurnTimeoutError +from coder_eval.errors.agent import format_timeout_reason from coder_eval.isolation.docker_runner import STDOUT_LINE_LIMIT_BYTES from coder_eval.models import ( READ_ONLY_DENIED_TOOLS, AgentKind, AgentState, ApiRoute, - AssistantMessage, - CommandTelemetry, ContentBlock, Enforcement, HarnessContract, PermissionMode, PiAgentConfig, - ResultSummary, TimingBasis, TokenUsage, ToolNameMap, - TranscriptMessage, - TurnRecord, UsageGranularity, ) from coder_eval.pricing import price_turn -from coder_eval.streaming.callbacks import StreamCallback, safe_emit -from coder_eval.streaming.collector import EventCollector -from coder_eval.streaming.emitter import TurnOutcome -from coder_eval.streaming.events import ( - AgentEndEvent, - AgentEndStatus, - AgentStartEvent, - StopReason, - StreamEvent, - TextChunkEvent, - ToolEndEvent, - ToolEndStatus, - ToolStartEvent, - TurnEndEvent, - TurnEndStatus, - TurnStartEvent, - end_status_for, -) -from coder_eval.timing import TurnClock, close_window +from coder_eval.streaming.callbacks import StreamCallback +from coder_eval.streaming.emitter import Generation, TurnEmitter, TurnOutcome +from coder_eval.streaming.events import AgentEndStatus, StopReason, ToolEndStatus, TurnEndStatus, end_status_for +from coder_eval.timing import close_window from .registry import AgentRegistry @@ -167,14 +145,6 @@ } ) -# ToolEndStatus -> CommandTelemetry.result_status (the persisted tri-state). -_RESULT_STATUS: dict[ToolEndStatus, Literal["success", "error", "unknown"]] = { - ToolEndStatus.OK: "success", - ToolEndStatus.ERROR: "error", - ToolEndStatus.PERMISSION_DENIED: "error", - ToolEndStatus.UNRESOLVED: "unknown", -} - def _canonical_params(tool_name: str, params: dict[str, Any]) -> dict[str, Any]: """Rename a tool call's argument keys to the canonical cross-agent vocabulary. @@ -206,50 +176,24 @@ def _result_text(result: Any) -> str | None: return str(result) -class _PiTurnState: - """Per-``communicate()`` accumulator: events in, finalization payload out. +class _PiDecoder: + """One turn's reducer: Pi's nd-JSON events in, ``TurnEmitter`` calls out. - Owns everything the terminal ``AgentEndEvent`` must carry (transcript - messages, summed usage, text output) plus the open-tool bookkeeping needed - to force-close orphans when a turn dies mid-flight. + Holds only what the emitter cannot know: where the next generation window + opens, the current step's text and tool ids, the usage and reported-cost sums, + the last stop reason, and a terminal provider ``error``. """ - def __init__( - self, - *, - task_id: str, - iteration: int, - user_input: str, - model: str | None, - clock: TurnClock | None = None, - ) -> None: - self.task_id = task_id - self.iteration = iteration - self.user_input = user_input - self.model = model - - # ONE clock per turn, so the tool spans and the window bounds they are - # subtracted from share a basis. Injectable so a test supplies a fake - # rather than monkeypatching this module's `datetime` global, which a - # derived stamp would silently escape. - self.clock = clock or TurnClock() - self.started_at = time.monotonic() - self.thread_id: str | None = None - - # Cumulative turn totals (summed across every inner step). + def __init__(self, emitter: TurnEmitter) -> None: + self.emitter = emitter self.usage = TokenUsage() self.cost_usd: float = 0.0 self.saw_cost = False - - self.messages: list[TranscriptMessage] = [] - self.text_parts: list[str] = [] - - # Pi `turn_start` events counted. + self.stop_reason: str | None = None + self.error: str | None = None self.turn_count = 0 - self.turn_id: str = "" - # True between a step's `turn_start` and its `turn_end`. `finalize` needs - # it to close a TurnStartEvent the stream never got to close. - self.turn_open = False + self.tool_count = 0 + self.open_tool_ids: set[str] = set() self.turn_started_at: datetime | None = None self.turn_text_parts: list[str] = [] self.turn_tool_ids: list[str] = [] @@ -258,164 +202,80 @@ def __init__( # before the first `turn_start` is CLI process spawn, not model time. # Rationale: .claude/notes/agents.md § Per-harness generation marks self.gen_mark: datetime | None = None - - # toolCallId -> telemetry for tools awaiting a result. - self.open_tools: dict[str, CommandTelemetry] = {} - self.sequence = 0 - self.stop_reason: str | None = None - self.error_message: str | None = None - # Guards the one-terminal-event rule; see finalize(). - self.finalized = False - # Count of events matched against the recognized Pi vocabulary (drift check), - # plus a bounded sample of the types that did NOT match — so the drift crash - # message can name what it actually saw. - self.recognized_events = 0 - self.unrecognized_types: set[str] = set() - # Warn-once guard for token-accounting drift: the event-vocabulary check - # cannot see inside `usage`. # Rationale: .claude/notes/agents.md § Why token-shape drift warns instead of raising self.warned_token_shape = False - self._emit: Callable[[StreamEvent], None] = lambda _e: None - - def bind(self, emit: Callable[[StreamEvent], None]) -> None: - self._emit = emit - - def emit(self, event: StreamEvent) -> None: - self._emit(event) - - @property - def agent_output(self) -> str: - return "".join(self.text_parts) + def __call__(self, event: dict[str, Any]) -> None: + event_type = event.get("type") + if event_type == "turn_start": + self.on_turn_start() + elif event_type == "message_update": + self.on_message_update(event) + elif event_type == "tool_execution_start": + self.on_tool_execution_start(event) + elif event_type == "tool_execution_end": + self.on_tool_execution_end(event) + elif event_type == "turn_end": + self.on_turn_end(event) + else: + logger.debug("pi: unhandled event type %r", event_type) - # --- event handlers ---------------------------------------------------- + def end(self, status: AgentEndStatus, *, reason: str | None = None) -> TurnOutcome: + """End the turn: ``fail`` for CRASHED / TIMEOUT (with ``reason``), else ``finalize``.""" + reported = self.usage.model_copy(update={"total_cost_usd": self.cost_usd if self.saw_cost else None}) + usage = self.usage.model_copy(update={"total_cost_usd": price_turn(reported, (self.emitter.model,))}) + if status is AgentEndStatus.CRASHED or status is AgentEndStatus.TIMEOUT: + return self.emitter.fail(status, reason or status.value, usage=usage) + return self.emitter.finalize(status, usage=usage, stop_reason=self.stop_reason) def on_turn_start(self) -> None: # A prior step's `turn_start` with no `turn_end` — a generation aborted - # mid-turn (the willRetry case). Close its dangling TurnStartEvent, or the - # stream carries N starts and N-1 ends and breaks the one-pair-per-inner-turn - # contract. `finalize` closes only the LAST open turn, so it cannot cover this. - if self.turn_open: - self.turn_open = False - self.emit( - TurnEndEvent( - task_id=self.task_id, - thread_id=self.thread_id, - turn_id=self.turn_id, - status=TurnEndStatus.CRASHED, - tokens=None, - ) - ) + # mid-turn (the willRetry case). Close its dangling inner turn first. + if self.emitter.inner_turn_open: + self.emitter.end_inner_turn(TurnEndStatus.CRASHED) self.turn_count += 1 - self.turn_open = True - self.turn_id = f"turn_{self.turn_count}" - self.turn_started_at = self.clock.now() + self.emitter.begin_inner_turn(f"turn_{self.turn_count}") + self.turn_started_at = self.emitter.now() self.turn_text_parts = [] self.turn_tool_ids = [] - # No per-turn span list to reset here any more: the collector subtracts - # from final bounds with every span known. - self.emit( - TurnStartEvent( - task_id=self.task_id, - thread_id=self.thread_id, - turn_id=self.turn_id, - model=self.model, - ) - ) - def on_message_update(self, obj: dict[str, Any]) -> None: - """Stream a ``text_delta`` as a ``TextChunkEvent`` (thinking/toolcall ignored).""" - event = obj.get("assistantMessageEvent") - if not isinstance(event, dict) or event.get("type") != "text_delta": + def on_message_update(self, event: dict[str, Any]) -> None: + """Stream a ``text_delta`` (thinking/toolcall ignored).""" + update = event.get("assistantMessageEvent") + if not isinstance(update, dict) or update.get("type") != "text_delta": return - delta = event.get("delta") + delta = update.get("delta") if not isinstance(delta, str) or not delta: return - self.text_parts.append(delta) self.turn_text_parts.append(delta) - self.emit(TextChunkEvent(task_id=self.task_id, thread_id=self.thread_id, turn_id=self.turn_id, text=delta)) + self.emitter.text(delta) - def on_tool_execution_start(self, obj: dict[str, Any]) -> None: - call_id = str(obj.get("toolCallId") or f"call_{self.sequence + 1}") - if call_id in self.open_tools: + def on_tool_execution_start(self, event: dict[str, Any]) -> None: + call_id = str(event.get("toolCallId") or f"call_{self.tool_count + 1}") + if call_id in self.open_tool_ids: return - self.sequence += 1 - raw_tool = str(obj.get("toolName") or "unknown") + self.tool_count += 1 + self.open_tool_ids.add(call_id) + raw_tool = str(event.get("toolName") or "unknown") tool_name = _TOOL_NAME_MAP.get(raw_tool.lower(), raw_tool) - args = obj.get("args") - params = args if isinstance(args, dict) else {} - started = self.clock.now() - telemetry = CommandTelemetry( - tool_name=tool_name, - tool_id=call_id, - assistant_turn_index=self.turn_count, - timestamp=started, - execution_started_at=started, - parameters=_canonical_params(tool_name, params), - sequence_number=self.sequence, - ) - self.open_tools[call_id] = telemetry + args = event.get("args") + self.emitter.open_tool(call_id, tool_name, _canonical_params(tool_name, args if isinstance(args, dict) else {})) self.turn_tool_ids.append(call_id) - self.emit(ToolStartEvent(task_id=self.task_id, thread_id=self.thread_id, turn_id=self.turn_id, tool=telemetry)) - def on_tool_execution_end(self, obj: dict[str, Any]) -> None: - call_id = str(obj.get("toolCallId") or "") - summary = _result_text(obj.get("result")) - is_error = bool(obj.get("isError")) - if is_error: + def on_tool_execution_end(self, event: dict[str, Any]) -> None: + call_id = str(event.get("toolCallId") or "") + summary = _result_text(event.get("result")) + if event.get("isError"): message = summary or "tool failed" # Best-effort: Pi does not tag permission denials, so infer from the - # text. The persisted tri-state folds both to "error", so a - # misclassification is cosmetic. + # text. The persisted tri-state folds both to "error". denied = "permission" in message.lower() or "denied" in message.lower() status = ToolEndStatus.PERMISSION_DENIED if denied else ToolEndStatus.ERROR else: message = None status = ToolEndStatus.OK - self._close_tool(call_id, status=status, summary=summary, error=message) - - def _close_tool( - self, - call_id: str, - *, - status: ToolEndStatus, - summary: str | None, - error: str | None, - ) -> None: - telemetry = self.open_tools.pop(call_id, None) - if telemetry is None: - # A result with no matching call (shouldn't happen, but never drop it). - self.sequence += 1 - telemetry = CommandTelemetry( - tool_name="unknown", - tool_id=call_id, - assistant_turn_index=self.turn_count, - timestamp=self.clock.now(), - sequence_number=self.sequence, - ) - # Only a RESOLVED tool is timed: an orphan was never observed finishing, - # so stamping it would manufacture a span the central subtraction then - # takes out of a window it never occupied. `execution_started_at` IS - # kept — one bound alone forms no span (CE058). - # Rationale: .claude/notes/agents.md § Why only a RESOLVED tool is timed - if status is not ToolEndStatus.UNRESOLVED: - completed = self.clock.now() - telemetry.execution_completed_at = completed - if telemetry.execution_started_at is not None: - telemetry.duration_ms = (completed - telemetry.execution_started_at).total_seconds() * 1000 - telemetry.result_status = _RESULT_STATUS[status] - # Stored untruncated by design (sub-agent returns must survive whole). - telemetry.result_summary = summary - telemetry.error_message = error - self.emit( - ToolEndEvent( - task_id=self.task_id, - thread_id=self.thread_id, - turn_id=self.turn_id, - tool=telemetry, - status=status, - ) - ) + self.open_tool_ids.discard(call_id) + self.emitter.close_tool(call_id, status=status, summary=summary, error=message) def _warn_token_shape(self, message: str, *args: Any) -> None: """Log a token-accounting anomaly at most once per turn (not once per bucket/step).""" @@ -427,10 +287,8 @@ def _warn_token_shape(self, message: str, *args: Any) -> None: def _as_int(self, value: Any) -> int: """Coerce one stream-supplied token count; count a non-number as 0. - A bool is never a token count (``int(True) == 1``). - - ``None`` is a legitimately-absent bucket (silent). Any OTHER unparseable - value is schema drift and warns once. + A bool is never a token count (``int(True) == 1``). ``None`` is a + legitimately-absent bucket (silent); any OTHER unparseable value warns once. Rationale: .claude/notes/agents.md § Why token-shape drift warns instead of raising """ @@ -445,20 +303,16 @@ def _as_int(self, value: Any) -> int: self._warn_token_shape("token count %r was not parseable as an int; counted as 0", value) return 0 - def on_turn_end(self, obj: dict[str, Any]) -> None: - """Accumulate this step's usage and append its assistant message. + def on_turn_end(self, event: dict[str, Any]) -> None: + """Book this step's usage and its generation. Usage is read from ``turn_end`` ONCE per step (not from every - ``message_end``, which echoes the same numbers) so the turn total is the - sum of the per-generation slices. + ``message_end``, which echoes the same numbers). """ - self.turn_open = False - message = obj.get("message") + message = event.get("message") message = message if isinstance(message, dict) else {} raw_usage = message.get("usage") if not isinstance(raw_usage, dict) or not raw_usage: - # A completed step that booked no usage object at all: its tokens and - # cost silently resolve to 0, so say so once. self._warn_token_shape("turn_end carried no usage object; this step's tokens/cost counted as 0") usage = raw_usage if isinstance(raw_usage, dict) else {} @@ -467,28 +321,22 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: step_reasoning = self._as_int(usage.get("reasoning")) step_cw = self._as_int(usage.get("cacheWrite")) step_cr = self._as_int(usage.get("cacheRead")) - # Reasoning bills at the output rate but is reported apart from `output`; - # fold it into the turn total (the per-message record keeps it separately). + # Reasoning bills at the output rate but is reported apart from `output`. step_out = raw_out + step_reasoning - # A usage object whose every bucket resolves to 0 is the drift shape the - # whole-object check cannot see. Warn once; score, don't crash. if raw_usage and step_in == raw_out == step_reasoning == step_cw == step_cr == 0: self._warn_token_shape("turn_end usage object had all-zero token buckets; this step booked 0 tokens/cost") - self.usage = TokenUsage( - uncached_input_tokens=self.usage.uncached_input_tokens + step_in, - output_tokens=self.usage.output_tokens + step_out, - cache_creation_input_tokens=self.usage.cache_creation_input_tokens + step_cw, - cache_read_input_tokens=self.usage.cache_read_input_tokens + step_cr, + tokens = TokenUsage( + uncached_input_tokens=step_in, + output_tokens=step_out, + cache_creation_input_tokens=step_cw, + cache_read_input_tokens=step_cr, ) - # Cross-check the stream's OWN `totalTokens` against the summed buckets. - # Pi's invariant is totalTokens == input + output + cacheRead + cacheWrite - # — reasoning bills at the output rate but is EXCLUDED from this field, so - # compare against raw_out, not step_out. Only when the field is present. + self.usage += tokens + # Pi's invariant is totalTokens == input + output + cacheRead + cacheWrite; + # reasoning is EXCLUDED from it, so compare against raw_out. reported_total = usage.get("totalTokens") - # int OR float: a `123.0`-shaped total is itself a plausible drift, and - # the compare below is exact for whole values. if isinstance(reported_total, int | float) and not isinstance(reported_total, bool): expected_total = step_in + raw_out + step_cw + step_cr if reported_total != expected_total: @@ -509,17 +357,15 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: finish = message.get("stopReason") if isinstance(finish, str) and finish: self.stop_reason = finish - # Capture a terminal provider error so a `pi -p` that exits 0 after - # exhausting retries still surfaces WHY (finalize reads error_message into - # result_summary.result). Reset on a non-error turn so an intermediate - # retry error that a later cycle recovered from never leaks into the result. + # A terminal provider error: `pi -p` exits 0 after exhausting retries. + # Reset on a non-error turn so a recovered retry error never leaks. if finish == "error": err = message.get("errorMessage") - self.error_message = err if isinstance(err, str) and err else "pi reported stopReason=error" + self.error = err if isinstance(err, str) and err else "pi reported stopReason=error" else: - self.error_message = None + self.error = None - completed = self.clock.now() + completed = self.emitter.now() blocks: list[ContentBlock] = [] turn_text = "".join(self.turn_text_parts) if turn_text: @@ -529,113 +375,31 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: # Tile from the previous turn's end. The RAW window only. turn_start = self.turn_started_at if self.turn_started_at is not None else completed - window = close_window( - mark=self.gen_mark if self.gen_mark is not None else turn_start, - now=completed, - item_start=turn_start, - ) - self.messages.append( - AssistantMessage( - started_at=window.started_at, - completed_at=window.completed_at, - generation_duration_ms=window.duration_ms, - content_blocks=blocks, - tool_use_ids=list(self.turn_tool_ids), - input_tokens=step_in, - output_tokens=step_out, - cache_creation_tokens=step_cw, - cache_read_tokens=step_cr, - reasoning_tokens=step_reasoning, - stop_reason=finish if isinstance(finish, str) else None, - model=self.model, - message_id=str(message.get("responseId") or "") or None, - ) + self.emitter.add_generation( + message_id=str(message.get("responseId") or "") or None, + window=close_window( + mark=self.gen_mark if self.gen_mark is not None else turn_start, now=completed, item_start=turn_start + ), + parts=[ + Generation( + blocks=blocks, + tokens=tokens, + reasoning_tokens=step_reasoning, + stop_reason=finish if isinstance(finish, str) else None, + ) + ], ) - # A message was appended, so the next window starts where this one ended. - # Only a FINISHED turn advances the mark. self.gen_mark = completed - # SPENT state, reset HERE and not only in `on_turn_start`: a second - # `turn_end` with no intervening start — a duplicate or replayed line, - # which this reducer promises to survive — would otherwise republish this - # turn's span, text and tool ids as the next turn's. + # SPENT state, reset HERE and not only in `on_turn_start`: a duplicate + # `turn_end` with no intervening start would otherwise republish this + # turn's span, text and tool ids as the next turn's. It still books its + # own generation and tokens, but closes no inner turn. # Rationale: .claude/notes/agents.md § Per-harness generation marks self.turn_started_at = None self.turn_text_parts = [] self.turn_tool_ids = [] - self.emit( - TurnEndEvent( - task_id=self.task_id, - thread_id=self.thread_id, - turn_id=self.turn_id, - status=TurnEndStatus.COMPLETED, - tokens=TokenUsage( - uncached_input_tokens=step_in, - output_tokens=step_out, - cache_creation_input_tokens=step_cw, - cache_read_input_tokens=step_cr, - ), - ) - ) - - def close_open_tools(self) -> None: - """Force-close every tool still awaiting a result (crash/timeout orphans).""" - for call_id in list(self.open_tools): - self._close_tool(call_id, status=ToolEndStatus.UNRESOLVED, summary=None, error="no result observed") - - def finalize( - self, - status: AgentEndStatus, - *, - crashed: bool = False, - crash_reason: str | None = None, - ) -> None: - """Close orphaned tools and emit the terminal ``AgentEndEvent`` (idempotent).""" - if self.finalized: - return - self.finalized = True - self.close_open_tools() - reported = self.usage.model_copy(update={"total_cost_usd": self.cost_usd if self.saw_cost else None}) - usage = self.usage.model_copy(update={"total_cost_usd": price_turn(reported, (self.model,))}) - # A turn still open never received its `turn_end`; close it or the - # one-pair-per-inner-turn contract breaks. - if self.turn_open: - self.turn_open = False - self.emit( - TurnEndEvent( - task_id=self.task_id, - thread_id=self.thread_id, - turn_id=self.turn_id, - status=TurnEndStatus(status.value), - tokens=None, - ) - ) - self.emit( - AgentEndEvent( - task_id=self.task_id, - thread_id=self.thread_id, - status=status, - usage=usage, - iteration=self.iteration, - user_input=self.user_input, - agent_output=self.agent_output, - model_used=self.model, - assistant_turn_count=self.turn_count, - messages=list(self.messages), - num_turns=self.turn_count, - result_summary=ResultSummary( - is_error=crashed, - subtype=status.value, - stop_reason=self.stop_reason, - result=crash_reason or self.error_message, - ), - crashed=crashed, - crash_reason=crash_reason, - duration_seconds=time.monotonic() - self.started_at, - # One basis with the window bounds — see the AgentStartEvent - # site in `communicate`. - timestamp=self.clock.now(), - ) - ) + if self.emitter.inner_turn_open: + self.emitter.end_inner_turn(TurnEndStatus.COMPLETED, tokens=tokens) @AgentRegistry.register(AgentKind.PI, PiAgentConfig) @@ -857,55 +621,20 @@ async def communicate( timeout: float | None = None, should_stop: Callable[[], StopReason | None] | None = None, ) -> TurnOutcome: - """Run one turn; see ``Agent.communicate``.""" - return await self._legacy_outcome( - self._communicate_legacy, - user_input, - iteration=iteration, - stream_callback=stream_callback, - timeout=timeout, - should_stop=should_stop, - ) - - async def _communicate_legacy( - self, - user_input: str, - *, - stream_callback: StreamCallback | None = None, - timeout: float | None = None, - should_stop: Callable[[], StopReason | None] | None = None, - ) -> TurnRecord: + """Run one ``pi -p`` invocation as one turn; see ``Agent.communicate``.""" if self.working_directory is None: raise RuntimeError("PiAgent.start() must be called before communicate()") - self._begin_turn() - collector = EventCollector() - - def emit(event: StreamEvent) -> None: - collector.on_event(event) - if stream_callback is not None: - safe_emit(stream_callback, event) - - state = _PiTurnState( - task_id=self.task_id, - iteration=self._iteration, - user_input=user_input, + emitter = self._open_emitter( + prompt=user_input, + iteration=iteration, model=self.config.model, + task_id=self.task_id, + stream_callback=stream_callback, ) - state.bind(emit) - - emit( - AgentStartEvent( - task_id=self.task_id, - prompt=user_input, - iteration=self._iteration, - model=self.config.model, - # One basis with the window bounds this is subtracted against; - # the model's raw `datetime.now()` default put two clocks inside - # one subtraction (CE058). - timestamp=state.clock.now(), - ) - ) + emitter.begin() + decoder = _PiDecoder(emitter) + vocabulary = _Vocabulary() # Deadlines stay on `time.monotonic()`, deliberately NOT the turn clock: # a deadline must not move when the wall clock steps. @@ -918,6 +647,10 @@ def emit(event: StreamEvent) -> None: try: proc = await asyncio.create_subprocess_exec( *self._build_argv(user_input), + # Pi reads a non-TTY stdin to EOF before it emits anything; an + # inherited, still-open stdin stalls the turn to its deadline. + # Rationale: .claude/notes/agents.md § Why a CLI never inherits stdin + stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=self.working_directory, @@ -948,7 +681,7 @@ def emit(event: StreamEvent) -> None: while True: remaining = None if deadline is None else deadline - time.monotonic() if remaining is not None and remaining <= 0: - await self._timeout_turn(state, collector, timeout or 0.0) + return await self._time_out(decoder, timeout or 0.0) if read_task is None: read_task = asyncio.ensure_future(proc.stdout.readline()) @@ -958,7 +691,7 @@ def emit(event: StreamEvent) -> None: return_when=asyncio.FIRST_COMPLETED, ) if not done: - await self._timeout_turn(state, collector, timeout or 0.0) + return await self._time_out(decoder, timeout or 0.0) if not read_task.done(): try: await asyncio.wait_for(asyncio.shield(read_task), _DRAIN_SECONDS) @@ -969,7 +702,7 @@ def emit(event: StreamEvent) -> None: if not line: break - self._handle_line(line, state) + self._handle_line(line, decoder, vocabulary) requested_stop = should_stop() if should_stop is not None else None if requested_stop is not None: @@ -980,34 +713,24 @@ def emit(event: StreamEvent) -> None: read_task.cancel() exit_waiter.cancel() - status = await self._settle_turn( + return await self._settle_turn( proc, - state, - collector, + decoder, + vocabulary, stderr_drain, requested_stop=requested_stop, deadline=deadline, timeout=timeout, ) - state.finalize(status) - # Build BEFORE marking the turn clean: a failure in the reduction is a - # failed turn, and `_end_turn_ok` clears the rollback flag. - record = collector.build_turn_record() - self._end_turn_ok() - return record - - except (AgentCrashError, TurnTimeoutError): - # Already funneled through finalize by _crash_turn / _timeout_turn. - raise + except asyncio.CancelledError: - self._finalize_external_cancel(state.finalize) - self._capture_partial_turn(collector) + self._state = AgentState.ERROR + decoder.end(AgentEndStatus.CRASHED, reason="turn cancelled") raise except Exception as e: - # A spawn failure, a StreamReader ValueError past `limit`, a malformed - # payload, a pydantic error. Funnel to the pending-turn contract. - self._crash_turn(state, collector, f"Pi turn failed: {e!s}", cause=e) - raise # unreachable (_crash_turn is NoReturn) — makes the no-fall-through explicit + # A spawn failure, a StreamReader ValueError past `limit`, a malformed payload. + logger.warning("pi: turn failed", exc_info=True) + return self._crash(decoder, f"Pi turn failed: {e!s}") finally: if stderr_drain is not None: stderr_drain.cancel() @@ -1031,31 +754,29 @@ def _reap_orphaned_cli(self, proc: asyncio.subprocess.Process | None) -> None: async def _settle_turn( self, proc: asyncio.subprocess.Process, - state: _PiTurnState, - collector: EventCollector, + decoder: _PiDecoder, + vocabulary: _Vocabulary, stderr_drain: asyncio.Future[bytes] | None, *, requested_stop: StopReason | None, deadline: float | None, timeout: float | None, - ) -> AgentEndStatus: - """Reap the CLI once the read loop is done and decide the turn's end status. + ) -> TurnOutcome: + """Reap the CLI once the read loop is done and end the turn. - Raises ``AgentCrashError`` (via :meth:`_crash_turn`) when the process died - with neither an intentional stop nor a recognized event stream. Raises - ``TurnTimeoutError`` when the deadline elapses while waiting for the exit. + CRASHED when the process died with neither an intentional stop nor a + recognized event stream, TIMEOUT when the deadline elapses while waiting + for the exit. """ remaining = None if deadline is None else max(0.0, deadline - time.monotonic()) try: await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_SECONDS if remaining is None else remaining) except TimeoutError: if remaining is not None: - await self._timeout_turn(state, collector, timeout or 0.0) + return await self._time_out(decoder, timeout or 0.0) await self.kill() - self._crash_turn( - state, - collector, - f"Pi closed its event stream but did not exit within {_TERM_GRACE_SECONDS:.0f}s", + return self._crash( + decoder, f"Pi closed its event stream but did not exit within {_TERM_GRACE_SECONDS:.0f}s" ) stderr_bytes = b"" if stderr_drain is not None: @@ -1064,62 +785,40 @@ async def _settle_turn( # A terminal provider error is infrastructure failure, not an agent # failure, and `pi -p` exits 0 after exhausting retries. GATED on - # intentional cuts: a cut can fire before the clearing `turn_end` arrives, - # leaving a stale error from a turn pi was still retrying. + # intentional cuts: a cut can fire before the clearing `turn_end` arrives. # Rationale: .claude/notes/agents.md § Why a clean exit can still be a crash - if state.error_message is not None and requested_stop is None: - self._crash_turn(state, collector, f"Pi error: {state.error_message}") + if decoder.error is not None and requested_stop is None: + return self._crash(decoder, f"Pi error: {decoder.error}") - # A non-zero exit with no intentional cut means the turn died. if proc.returncode not in (0, None) and requested_stop is None: detail = stderr_bytes.decode("utf-8", "replace").strip() or f"exit code {proc.returncode}" - self._crash_turn(state, collector, f"Pi exited non-zero: {detail}") + return self._crash(decoder, f"Pi exited non-zero: {detail}") # A clean exit that recognized NO events is vocabulary drift. Intentional # cuts are exempt: a stop can land before the first event. - if requested_stop is None and state.recognized_events == 0: - seen = ", ".join(sorted(state.unrecognized_types)) or "none (stdout carried no JSON events)" - self._crash_turn( - state, - collector, + if requested_stop is None and vocabulary.recognized == 0: + seen = ", ".join(sorted(vocabulary.unrecognized)) or "none (stdout carried no JSON events)" + return self._crash( + decoder, "Pi exited cleanly but the turn captured no recognized events. Unrecognized event types seen: " + f"{seen}. The CLI's event schema may have changed — see docs/agents/PI.md before trusting any " + "run from this CLI version.", ) - return end_status_for(requested_stop) if requested_stop is not None else AgentEndStatus.COMPLETED + return decoder.end(end_status_for(requested_stop) if requested_stop is not None else AgentEndStatus.COMPLETED) - def _crash_turn( - self, - state: _PiTurnState, - collector: EventCollector, - message: str, - *, - cause: BaseException | None = None, - ) -> NoReturn: - """Park the crashed partial record and raise ``AgentCrashError``.""" - state.close_open_tools() - try: - self._finalize_and_raise_crash(state.finalize, message, cause=cause) - finally: - self._capture_partial_turn(collector) + def _crash(self, decoder: _PiDecoder, message: str) -> TurnOutcome: + self._state = AgentState.ERROR + return decoder.end(AgentEndStatus.CRASHED, reason=message) - async def _timeout_turn( - self, - state: _PiTurnState, - collector: EventCollector, - timeout: float, - ) -> NoReturn: - """Kill the CLI, park the crashed partial record, raise ``TurnTimeoutError``.""" + async def _time_out(self, decoder: _PiDecoder, timeout: float) -> TurnOutcome: + """Kill the CLI and end the turn as a timeout.""" await self.kill() - state.close_open_tools() - try: - self._finalize_and_raise_timeout(state.finalize, timeout) - finally: - self._capture_partial_turn(collector) + self._state = AgentState.ERROR + return decoder.end(AgentEndStatus.TIMEOUT, reason=format_timeout_reason(timeout)) - def _handle_line(self, line: bytes, state: _PiTurnState) -> None: - """Parse one nd-JSON line and dispatch it. Never raises on bad input. + def _handle_line(self, line: bytes, decoder: _PiDecoder, vocabulary: _Vocabulary) -> None: + """Parse one nd-JSON line and hand it to the decoder. Never raises on bad input. ``agent_end`` is NOT terminal — only ``agent_settled`` / stdout EOF is — so it is recognized, ignored, and the read loop keeps going. @@ -1134,24 +833,17 @@ def _handle_line(self, line: bytes, state: _PiTurnState) -> None: return if not isinstance(obj, dict): return - event_type = str(obj.get("type") or "") - # `session`, `message_start`, `message_end`, `agent_end` and - # `agent_settled` carry no state we accumulate, but are all recognized. if event_type in _RECOGNIZED_EVENTS: - state.recognized_events += 1 - elif len(state.unrecognized_types) < _MAX_UNRECOGNIZED_TYPES: - state.unrecognized_types.add(event_type or "") + vocabulary.recognized += 1 + elif len(vocabulary.unrecognized) < _MAX_UNRECOGNIZED_TYPES: + vocabulary.unrecognized.add(event_type or "") + decoder(obj) - if event_type == "turn_start": - state.on_turn_start() - elif event_type == "message_update": - state.on_message_update(obj) - elif event_type == "tool_execution_start": - state.on_tool_execution_start(obj) - elif event_type == "tool_execution_end": - state.on_tool_execution_end(obj) - elif event_type == "turn_end": - state.on_turn_end(obj) - else: - logger.debug("pi: unhandled event type %r", event_type) + +@dataclass +class _Vocabulary: + """The drift check's evidence: how many events matched Pi's vocabulary, and a sample of what did not.""" + + recognized: int = 0 + unrecognized: set[str] = field(default_factory=set) diff --git a/src/coder_eval/streaming/emitter.py b/src/coder_eval/streaming/emitter.py index a3cf65f2..0524de1d 100644 --- a/src/coder_eval/streaming/emitter.py +++ b/src/coder_eval/streaming/emitter.py @@ -156,6 +156,10 @@ def __init__( def iteration(self) -> int: return self._iteration + @property + def model(self) -> str | None: + return self._model + @property def inner_turn_open(self) -> bool: return self._turn_id is not None diff --git a/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json index 94ef4899..af1cce36 100644 --- a/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json @@ -41,7 +41,7 @@ "num_turns": 1, "result_summary": { "is_error": false, - "result": null, + "result": "All done.", "stop_reason": "stop", "subtype": "completed" }, diff --git a/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json index 45645524..9d13e456 100644 --- a/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json @@ -17,7 +17,7 @@ "result_status": "success", "result_summary": "Successfully wrote 2 bytes to hello.txt", "result_tokens": 10, - "sequence_number": 1, + "sequence_number": 0, "timestamp": "", "tool_id": "write:0", "tool_name": "Write" @@ -36,7 +36,7 @@ "result_status": "success", "result_summary": "hi", "result_tokens": 1, - "sequence_number": 2, + "sequence_number": 1, "timestamp": "", "tool_id": "read:1", "tool_name": "Read" @@ -139,7 +139,7 @@ "num_turns": 3, "result_summary": { "is_error": false, - "result": null, + "result": "Created `hello.txt` and read it back. Contents: **hi**", "stop_reason": "stop", "subtype": "completed" }, diff --git a/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json b/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json index 864106fe..21db5cb1 100644 --- a/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json +++ b/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json @@ -16,7 +16,7 @@ "result_status": "success", "result_summary": "main.py", "result_tokens": 2, - "sequence_number": 1, + "sequence_number": 0, "timestamp": "", "tool_id": "call_1", "tool_name": "Bash" @@ -90,7 +90,7 @@ "num_turns": 2, "result_summary": { "is_error": false, - "result": null, + "result": "Listed it.", "stop_reason": "stop", "subtype": "completed" }, diff --git a/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json index 94e455c3..2fe522b7 100644 --- a/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json @@ -5,7 +5,7 @@ { "assistant_turn_index": 0, "duration_ms": null, - "error_message": "no result observed", + "error_message": null, "execution_completed_at": null, "execution_started_at": "", "generation_completed_at": null, @@ -16,7 +16,7 @@ "result_status": "unknown", "result_summary": null, "result_tokens": 0, - "sequence_number": 1, + "sequence_number": 0, "timestamp": "", "tool_id": "call_1", "tool_name": "Bash" diff --git a/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json b/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json index 4bec8274..a7edec74 100644 --- a/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json +++ b/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json @@ -56,12 +56,7 @@ ], "model_used": "openrouter/moonshotai/kimi-k3", "num_turns": 2, - "result_summary": { - "is_error": true, - "result": "Pi error: provider returned 529 after 5 retries", - "stop_reason": "error", - "subtype": "crashed" - }, + "result_summary": null, "timestamp": "", "token_usage": { "cache_creation_input_tokens": 0, diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index d7bc42e4..e6f3faad 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -265,51 +265,34 @@ async def test_pi_reconciliation_invariant(scenario, tmp_path): assert_reconciliation((await run_pi_scenario(scenario, str(tmp_path)))[0]) -def _balance_params(harness: str, scenarios: list[Any]) -> list[Any]: - """Every scenario, with the one known-unbalanced stream a strict xfail until its port fixes it.""" - return [ - pytest.param( - s, - id=s.name, - marks=pytest.mark.xfail( - strict=True, - reason="duplicate turn_end emits a TurnEndEvent with no open turn; fixed by the Pi port", - ), - ) - if f"{harness}_{s.name}" == "pi_f_duplicate_turn_end" - else pytest.param(s, id=s.name) - for s in scenarios - ] - - @pytest.mark.asyncio -@pytest.mark.parametrize("scenario", _balance_params("claude", CLAUDE_SCENARIOS)) +@pytest.mark.parametrize("scenario", CLAUDE_SCENARIOS, ids=lambda s: s.name) async def test_claude_stream_balanced(scenario, tmp_path): assert_stream_balanced((await run_claude_scenario(scenario, str(tmp_path)))[1]) @pytest.mark.skipif(not _HAS_CODEX, reason="openai_codex extra not installed") @pytest.mark.asyncio -@pytest.mark.parametrize("scenario", _balance_params("codex", CODEX_SCENARIOS)) +@pytest.mark.parametrize("scenario", CODEX_SCENARIOS, ids=lambda s: s.name) async def test_codex_stream_balanced(scenario, tmp_path): assert run_codex_scenario is not None assert_stream_balanced((await run_codex_scenario(scenario, str(tmp_path)))[1]) @pytest.mark.asyncio -@pytest.mark.parametrize("scenario", _balance_params("antigravity", ANTIGRAVITY_SCENARIOS)) +@pytest.mark.parametrize("scenario", ANTIGRAVITY_SCENARIOS, ids=lambda s: s.name) async def test_antigravity_stream_balanced(scenario, tmp_path): assert_stream_balanced((await run_antigravity_scenario(scenario, str(tmp_path)))[1]) @pytest.mark.asyncio -@pytest.mark.parametrize("scenario", _balance_params("opencode", OPENCODE_SCENARIOS)) +@pytest.mark.parametrize("scenario", OPENCODE_SCENARIOS, ids=lambda s: s.name) async def test_opencode_stream_balanced(scenario, tmp_path): assert_stream_balanced((await run_opencode_scenario(scenario, str(tmp_path)))[1]) @pytest.mark.asyncio -@pytest.mark.parametrize("scenario", _balance_params("pi", PI_SCENARIOS)) +@pytest.mark.parametrize("scenario", PI_SCENARIOS, ids=lambda s: s.name) async def test_pi_stream_balanced(scenario, tmp_path): assert_stream_balanced((await run_pi_scenario(scenario, str(tmp_path)))[1]) diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 860499dc..d0bac98c 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -23,11 +23,12 @@ import pytest -from coder_eval.agents.pi_agent import PiAgent, _PiTurnState, _result_text -from coder_eval.models import AgentKind, AssistantMessage, CommandTelemetry, PiAgentConfig, TokenUsage +from coder_eval.agents.pi_agent import PiAgent, _PiDecoder, _result_text +from coder_eval.errors.agent import format_timeout_reason +from coder_eval.models import AgentKind, AssistantMessage, PiAgentConfig from coder_eval.orchestration.plugin_staging import stage_plugins from coder_eval.pricing import calculate_cost -from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.emitter import TurnEmitter, TurnOutcome from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -40,6 +41,7 @@ TurnEndStatus, TurnStartEvent, ) +from coder_eval.testing import Replay, ScriptedClock, Tick, assert_identity_closes, assert_stream_balanced, replay from coder_eval.timing import TurnClock from tests._bracket_clock import AnchoredClock, assert_bracket_on_the_clock, assert_overhead_is_measured from tests._fixtures.golden_streams.pi_fixtures import ( @@ -96,6 +98,53 @@ def on_event(self, event: Any) -> None: self.events.append(event) +_SPAN_BASE = datetime(2026, 3, 1, 9, 0, 0) + + +def _ms(at_ms: float) -> datetime: + return _SPAN_BASE + timedelta(milliseconds=at_ms) + + +def _event(line: str) -> dict[str, Any]: + return json.loads(line) + + +def _start() -> dict[str, Any]: + return {"type": "turn_start"} + + +def _end(*, inp: int = 10, out: int = 5) -> dict[str, Any]: + return { + "type": "turn_end", + "message": {"role": "assistant", "usage": {"input": inp, "output": out}, "stopReason": "stop"}, + } + + +def _open(call_id: str) -> dict[str, Any]: + return {"type": "tool_execution_start", "toolCallId": call_id, "toolName": "bash", "args": {}} + + +def _close(call_id: str) -> dict[str, Any]: + return {"type": "tool_execution_end", "toolCallId": call_id, "result": "ok"} + + +def _replay( + stream: list[Any], *, status: AgentEndStatus = AgentEndStatus.COMPLETED, reason: str | None = None +) -> tuple[Replay, _PiDecoder]: + """Drive a `_PiDecoder` through `coder_eval.testing.replay` from `_SPAN_BASE`; return the decoder too.""" + decoders: list[_PiDecoder] = [] + + def end(decoder: _PiDecoder) -> TurnOutcome: + decoders.append(decoder) + return decoder.end(status, reason=reason) + + return replay(stream, _PiDecoder, clock=ScriptedClock(_SPAN_BASE), end=end), decoders[0] + + +def _assistants(result: Replay) -> list[AssistantMessage]: + return [m for m in result.record.messages if isinstance(m, AssistantMessage)] + + class TestHappyPath: async def test_builds_turn_record(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) @@ -282,6 +331,12 @@ async def test_explicit_line_limit_is_passed(self, patch_exec, tmp_path): await _run(_agent(), tmp_path) assert captured["kwargs"]["limit"] > 64 * 1024 + async def test_the_cli_never_inherits_stdin(self, patch_exec, tmp_path): + """Pi reads a non-TTY stdin to EOF before it emits; an inherited open stdin stalls the turn.""" + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(_agent(), tmp_path) + assert captured["kwargs"]["stdin"] is asyncio.subprocess.DEVNULL + class TestToolFlags: def test_allowlist_maps_claude_names_to_pi_tools(self): @@ -566,24 +621,67 @@ def kill(self) -> None: class TestTimeoutContract: - async def test_deadline_raises_turn_timeout_with_partial_parked(self, patch_exec, tmp_path): + async def test_deadline_returns_a_timeout_outcome_with_the_partial(self, patch_exec, tmp_path): proc = _HangingProcess([_turn_start()]) patch_exec(proc) agent = _agent() recorder = _EventRecorder() + ends_seen_at_kill: list[int] = [] + real_kill = agent.kill + + async def spy_kill() -> None: + ends_seen_at_kill.append(len([e for e in recorder.events if isinstance(e, AgentEndEvent)])) + await real_kill() + + agent.kill = spy_kill # type: ignore[method-assign] outcome = await _run(agent, tmp_path, timeout=0.2, stream_callback=recorder) assert outcome.status is AgentEndStatus.TIMEOUT + assert outcome.error == format_timeout_reason(0.2) partial = outcome.record assert partial is not None assert partial.crashed is True assert proc.terminated is True + assert ends_seen_at_kill[:1] == [0], "the CLI is killed BEFORE the turn ends" ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] assert len(ends) == 1 assert ends[0].status is AgentEndStatus.TIMEOUT +class _EofButAliveProcess(_HangingProcess): + """Stdout reaches EOF, but the process never exits until it is signalled.""" + + async def readline(self) -> bytes: + if self._lines: + return self._lines.pop(0) + return b"" + + +class TestSettleWaitsForTheExit: + async def test_no_exit_before_the_deadline_is_a_timeout(self, patch_exec, tmp_path): + proc = _EofButAliveProcess([_turn_start()]) + patch_exec(proc) + recorder = _EventRecorder() + outcome = await _run(_agent(), tmp_path, timeout=0.3, stream_callback=recorder) + assert outcome.status is AgentEndStatus.TIMEOUT + assert proc.terminated is True + assert [e.status for e in recorder.events if isinstance(e, AgentEndEvent)] == [AgentEndStatus.TIMEOUT] + + async def test_no_exit_without_a_deadline_is_a_crash(self, patch_exec, tmp_path, monkeypatch): + from coder_eval.agents import pi_agent + + monkeypatch.setattr(pi_agent, "_TERM_GRACE_SECONDS", 0.1) + proc = _EofButAliveProcess([_turn_start()]) + patch_exec(proc) + recorder = _EventRecorder() + outcome = await _run(_agent(), tmp_path, stream_callback=recorder) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "did not exit within" in outcome.error + assert proc.terminated is True + assert [e.status for e in recorder.events if isinstance(e, AgentEndEvent)] == [AgentEndStatus.CRASHED] + + class _ExplodingProcess(_FakeProcess): async def readline(self) -> bytes: if self._lines: @@ -618,7 +716,7 @@ async def test_drift_crash_names_the_unrecognized_types(self, patch_exec, tmp_pa assert "another.unknown, some.new.event" in outcome.error assert "no recognized events" in outcome.error - async def test_stream_error_becomes_a_crash_with_partial_parked(self, patch_exec, tmp_path): + async def test_stream_error_becomes_a_crash_with_the_crashed_partial(self, patch_exec, tmp_path): stream = [_turn_start(), _tool_start("w:0", "write", {"path": "a.txt", "content": "x"})] patch_exec(_ExplodingProcess(stream)) agent = _agent() @@ -795,15 +893,12 @@ async def test_permission_denial_gets_its_own_status(self, patch_exec, tmp_path) assert end.status is ToolEndStatus.PERMISSION_DENIED def test_orphan_result_is_never_dropped(self): - state = _PiTurnState(task_id="t", iteration=1, user_input="x", model=None) - events: list[Any] = [] - state.bind(events.append) - state._close_tool("ghost", status=ToolEndStatus.UNRESOLVED, summary=None, error="no result observed") + result, _ = _replay([_close("ghost")]) - [event] = events - assert isinstance(event, ToolEndEvent) + [event] = [e for e in result.events if isinstance(e, ToolEndEvent)] + assert event.tool.tool_id == "ghost" assert event.tool.tool_name == "unknown" - assert event.tool.result_status == "unknown" + assert [c.tool_id for c in result.record.commands] == ["ghost"] class TestZeroUsageTurn: @@ -888,11 +983,56 @@ async def test_a_stop_after_an_error_turn_finalizes_cleanly(self, patch_exec, tm def test_error_message_resets_on_a_recovered_turn(self): """#3: an intermediate error a later cycle recovers from must not leak into the result.""" - state = _PiTurnState(task_id="t", iteration=1, user_input="x", model="m") - state.on_turn_end(json.loads(_turn_end_error("transient"))) - assert state.error_message == "transient" - state.on_turn_end(json.loads(_turn_end(inp=5, out=2))) # a later, successful turn - assert state.error_message is None + _, errored = _replay([_start(), _event(_turn_end_error("transient"))]) + assert errored.error == "transient" + _, recovered = _replay([_start(), _event(_turn_end_error("transient")), _start(), _end()]) + assert recovered.error is None + + def test_dangling_turn_is_closed_crashed_at_the_next_turn_start(self): + result, _ = _replay([_start(), _start(), _end()]) + + turn_ends = [e for e in result.events if isinstance(e, TurnEndEvent)] + assert [(e.turn_id, e.status) for e in turn_ends] == [ + ("turn_1", TurnEndStatus.CRASHED), + ("turn_2", TurnEndStatus.COMPLETED), + ] + assert_stream_balanced(result.events) + + def test_a_duplicate_turn_end_publishes_no_second_turn_end_but_books_its_tokens(self): + result, _ = _replay([_start(), _end(inp=10, out=5), _end(inp=7, out=3)]) + + assert len([e for e in result.events if isinstance(e, TurnEndEvent)]) == 1 + assert len(_assistants(result)) == 2 + [agent_end] = [e for e in result.events if isinstance(e, AgentEndEvent)] + assert agent_end.usage.uncached_input_tokens == 17 + assert agent_end.usage.output_tokens == 8 + assert_stream_balanced(result.events) + + def test_token_shape_warns_once_per_turn(self, caplog): + no_usage = {"type": "turn_end", "message": {"role": "assistant", "stopReason": "stop"}} + bad_total = { + "type": "turn_end", + "message": {"role": "assistant", "usage": {"input": 1, "output": 1, "totalTokens": 99}}, + } + + def warnings() -> int: + return len([r for r in caplog.records if "unexpected token accounting" in r.getMessage()]) + + with caplog.at_level("WARNING"): + _replay([_start(), no_usage, _start(), bad_total]) + assert warnings() == 1 + _replay([_start(), bad_total]) + assert warnings() == 2 # the flag is per turn, not per agent + + def test_total_tokens_mismatch_warns_at_the_decoder(self, caplog): + bad_total = { + "type": "turn_end", + "message": {"role": "assistant", "usage": {"input": 10, "output": 5, "totalTokens": 999}}, + } + with caplog.at_level("WARNING"): + _, decoder = _replay([_start(), bad_total]) + assert decoder.warned_token_shape is True + assert any("does not reconcile" in r.getMessage() for r in caplog.records) async def test_bad_token_bucket_warns_once(self, patch_exec, tmp_path, caplog): """#1: a bucket whose type drifted (here a dict) coerces to 0 but warns, once.""" @@ -1008,7 +1148,7 @@ class TestTurnAlwaysReapsTheCli: """ async def test_read_loop_crash_kills_the_cli(self, patch_exec, tmp_path): - """``_crash_turn`` is synchronous and raises — nothing below it reaps.""" + """A read-loop crash ends the turn as an outcome, and ``finally`` still reaps the live CLI.""" proc = _ExplodingRunningProcess([_turn_start()]) patch_exec(proc) outcome = await _run(_agent(), tmp_path) @@ -1040,9 +1180,9 @@ async def test_a_clean_turn_kills_nothing(self, patch_exec, tmp_path): class TestExternalCancel: - async def test_cancel_parks_partial_and_reraises(self, patch_exec, tmp_path): + async def test_cancel_ends_the_turn_and_reraises(self, patch_exec, tmp_path): """The watchdog's CancelledError must not swallow captured telemetry: the - partial is parked, the terminal event says CRASHED, and the cancellation + turn is ended, the terminal event says CRASHED, and the cancellation still propagates.""" proc = _HangingProcess([_turn_start()]) patch_exec(proc) @@ -1054,12 +1194,10 @@ async def test_cancel_parks_partial_and_reraises(self, patch_exec, tmp_path): task.cancel() with pytest.raises(asyncio.CancelledError): _ = await task # the await re-raises the cancellation; no value ever exists - partial = agent.pending_turn - assert partial is not None - assert partial.crashed is True ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] assert len(ends) == 1 assert ends[0].status is AgentEndStatus.CRASHED + assert ends[0].crashed is True assert ends[0].crash_reason == "turn cancelled" assert proc.killed is True # not abandoned mid-stream — see TestTurnAlwaysReapsTheCli @@ -1136,89 +1274,47 @@ async def test_zero_reported_cost_on_an_unpriced_model_stays_zero(self, patch_ex assert record.token_usage.total_cost_usd == 0.0 -class _FixedClock: - """A `TurnClock` stand-in frozen at one instant, injected into the state.""" - - def __init__(self, at: datetime) -> None: - self.at = at - - def now(self) -> datetime: - return self.at - - class TestGenerationWindowExcludesToolExecution: """A tool running inside a turn is not model time — asserted where it is now DECIDED. - The reducer no longer subtracts anything. It publishes the RAW window, and + The decoder no longer subtracts anything. It publishes the RAW window, and `timing.subtract_tool_time` takes the tool union back out of it - once, for all five harnesses. So these cases drive the reducer and then a - real collector, and assert the PUBLISHED number — the one that reaches - `task.json` — rather than an intermediate the reducer used to own. + once, for all five harnesses. So these cases replay the decoder through a + real emitter and assert the PUBLISHED number — the one that reaches + `task.json` — rather than an intermediate the decoder used to own. They are not duplicates of `tests/test_event_collector.py::TestSubtractToolTime`: those pin the - arithmetic, these pin that THIS reducer hands the collector a window and a + arithmetic, these pin that THIS decoder hands the collector a window and a span set the arithmetic can be right about. """ - WINDOW_START = datetime(2026, 1, 1, 12, 0, 0) - WINDOW_END = datetime(2026, 1, 1, 12, 0, 1) # a 1000ms turn - - def _finish_turn(self, spans, open_starts=()): - """Drive the reducer, then publish through a real collector. + def _finish_turn(self, spans: list[tuple[float, float]], open_starts: tuple[float, ...] = ()) -> AssistantMessage: + """Replay one 0 -> 1000 ms turn with tool calls at the given ms offsets. `spans` are RESOLVED calls (both bounds); `open_starts` are calls that - never returned. An unresolved call now contributes NO span — it has no + never returned. An unresolved call contributes NO span — it has no `execution_completed_at`, and inventing one is what `None` exists to - prevent — where the reducer used to bound it at the window's end. That - is a real change and a better one: the collector sees every span at - once, so a call straddling a boundary is clipped to each window it - actually overlapped instead of approximated at the boundary. + prevent. The collector sees every span at once, so a call straddling a + boundary is clipped to each window it actually overlapped. """ - state = _PiTurnState(task_id="t", iteration=1, user_input="x", model="m", clock=_FixedClock(self.WINDOW_END)) - state.turn_started_at = self.WINDOW_START - commands = [ - CommandTelemetry( - tool_name="bash", - tool_id=f"closed-{i}", - timestamp=started, - execution_started_at=started, - execution_completed_at=completed, - result_status="success", - ) - for i, (started, completed) in enumerate(spans) - ] - commands += [ - CommandTelemetry(tool_name="bash", tool_id=f"open-{i}", timestamp=s, execution_started_at=s) - for i, s in enumerate(open_starts) - ] - state.on_turn_end( - {"message": {"role": "assistant", "usage": {"input": 100, "output": 20}, "stopReason": "stop"}} - ) - - collector = EventCollector() - collector.on_event(AgentStartEvent(task_id="t", prompt="x", iteration=1, timestamp=self.WINDOW_START)) - for command in commands: - collector.on_event(ToolEndEvent(task_id="t", turn_id="t1", tool=command)) - collector.on_event( - AgentEndEvent( - task_id="t", - status=AgentEndStatus.COMPLETED, - messages=list(state.messages), - usage=TokenUsage(), - timestamp=self.WINDOW_END, - ) - ) - published = [m for m in collector.build_turn_record().messages if m.role == "assistant"] + timeline: list[tuple[float, dict[str, Any]]] = [(0.0, _start()), (1000.0, _end(inp=100, out=20))] + for i, (started, completed) in enumerate(spans): + timeline += [(started, _open(f"closed-{i}")), (completed, _close(f"closed-{i}"))] + timeline += [(started, _open(f"open-{i}")) for i, started in enumerate(open_starts)] + stream: list[Any] = [] + for at_ms, event in sorted(timeline, key=lambda item: item[0]): + stream += [Tick(at_ms), event] + + result, _ = _replay(stream) + published = _assistants(result) assert len(published) == 1 return published[0] def test_tool_time_inside_the_turn_is_subtracted(self): - message = self._finish_turn( - [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))], - ) + message = self._finish_turn([(200, 700)]) span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 - assert span_ms == pytest.approx(1000.0), "the reducer still publishes the whole window as its bounds" + assert span_ms == pytest.approx(1000.0), "the decoder still publishes the whole window as its bounds" assert message.generation_duration_ms == pytest.approx(500.0) def test_a_turn_with_no_tools_keeps_its_whole_window(self): @@ -1227,36 +1323,24 @@ def test_a_turn_with_no_tools_keeps_its_whole_window(self): def test_concurrent_tools_are_subtracted_once(self): # Two overlapping 500ms tools occupy 600ms, not 1000ms. Summing them # would leave 0 generation for a turn that generated 400. - message = self._finish_turn( - [ - (self.WINDOW_START + timedelta(milliseconds=100), self.WINDOW_START + timedelta(milliseconds=600)), - (self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700)), - ], - ) + message = self._finish_turn([(100, 600), (200, 700)]) assert message.generation_duration_ms == pytest.approx(400.0) def test_the_window_never_goes_negative(self): - message = self._finish_turn( - [(self.WINDOW_START - timedelta(seconds=30), self.WINDOW_END + timedelta(seconds=30))], - ) + message = self._finish_turn([(-30_000, 31_000)]) assert message.generation_duration_ms == 0.0 def test_a_tool_still_open_at_the_boundary_contributes_no_span(self): - """The behaviour that CHANGED with the move, stated rather than implied. + """A call with no `execution_completed_at` was never timed. - The reducer used to bound a still-open call at the window's end and - subtract that slice. The collector cannot: a call with no - `execution_completed_at` was never timed. Its time is subtracted when it - RESOLVES, from whichever windows its real interval overlaps. + Its time is subtracted when it RESOLVES, from whichever windows its real + interval overlaps — never bounded at the window's end. """ - message = self._finish_turn([], open_starts=[self.WINDOW_START + timedelta(milliseconds=600)]) + message = self._finish_turn([], open_starts=(600,)) assert message.generation_duration_ms == pytest.approx(1000.0) def test_a_resolved_tool_overlapping_an_unresolved_one_counts_only_the_resolved(self): - message = self._finish_turn( - [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))], - open_starts=[self.WINDOW_START + timedelta(milliseconds=500)], - ) + message = self._finish_turn([(200, 700)], open_starts=(500,)) assert message.generation_duration_ms == pytest.approx(500.0) def test_the_published_window_reconciles_to_its_own_bounds(self): @@ -1270,44 +1354,18 @@ def test_the_published_window_reconciles_to_its_own_bounds(self): """ from coder_eval.timing import busy_ms - closed = [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))] - message = self._finish_turn(closed) + message = self._finish_turn([(200, 700)]) span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 - expected = span_ms - busy_ms(closed, message.started_at, message.completed_at) + expected = span_ms - busy_ms([(_ms(200), _ms(700))], message.started_at, message.completed_at) assert message.generation_duration_ms == pytest.approx(expected) -_SPAN_BASE = datetime(2026, 3, 1, 9, 0, 0) - - -class _SteppedClock: - """A `TurnClock` stand-in the test moves by hand, in ms from `_SPAN_BASE`. - - INJECTED, never monkeypatched onto the module. Pi derives every wall stamp - from its turn clock now, so patching `agent_module.datetime` would no - longer reach it: the tests would quietly start measuring the real clock and - pass by accident instead of failing. Injection also puts the "one clock per - turn" lifetime in the constructor signature where it can be read. - """ - - def __init__(self, at_ms: float = 0.0) -> None: - self.at_ms = at_ms - - def now(self) -> datetime: - return _SPAN_BASE + timedelta(milliseconds=self.at_ms) - - -def _turn_end_payload(): - return {"message": {"role": "assistant", "usage": {"input": 10, "output": 5}, "stopReason": "stop"}} - - class TestGenerationWindowsTileTheTurn: """Each window runs from the PREVIOUS `turn_end`, not from its own `turn_start`. - Pi was the only harness measuring from its own turn start, so the wall - clock between one `turn_end` and the next `turn_start` — the model time - that PRODUCED the next turn — fell into no bucket at all. The four-bucket - identity is asserted only as an upper bound, so nothing failed. + Measured from its own turn start, the wall clock between one `turn_end` and + the next `turn_start` — the model time that PRODUCED the next turn — fell + into no bucket at all. The gap is small in practice (measured across 25 real window pairs: median 0.25 ms, max 0.75 ms). The value here is that it closes, and that the tool @@ -1315,17 +1373,9 @@ class TestGenerationWindowsTileTheTurn: which is the half that carries the weight. """ - def _two_turns(self): - clock = _SteppedClock() - state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) - state.on_turn_start() - clock.at_ms = 1000 - state.on_turn_end(_turn_end_payload()) - clock.at_ms = 1600 - state.on_turn_start() - clock.at_ms = 2000 - state.on_turn_end(_turn_end_payload()) - return [m for m in state.messages if m.role == "assistant"] + def _two_turns(self) -> list[AssistantMessage]: + result, _ = _replay([_start(), Tick(1000), _end(), Tick(1600), _start(), Tick(2000), _end()]) + return _assistants(result) def test_the_second_window_abuts_the_first(self): messages = self._two_turns() @@ -1342,64 +1392,38 @@ def test_the_inter_turn_gap_is_inside_a_window_rather_than_unaccounted(self): class TestToolSpansSurviveTheTurnBoundary: """A tool that closes BETWEEN two turns still belongs to the next window. - This used to be a bookkeeping problem: a per-turn span list, cleared at - `turn_start` — after the window it feeds had already opened at the mark — - so a call closing in the gap had its span wiped before the flush could - subtract it. That list is gone. `timing.subtract_tool_time` sees - every span at once and clips each to the windows it overlaps, so the - property now holds by construction rather than by a reset rule. - - Kept, and re-pointed at the collector, because the property itself is what - matters and a future reducer change could still break it — by moving a - mark, or by failing to emit the ToolEnd the collector reduces. + `timing.subtract_tool_time` sees every span at once and clips each to the + windows it overlaps, so the property holds by construction rather than by a + reset rule. Kept because the property itself is what matters and a future + decoder change could still break it — by moving a mark, or by failing to + close the tool the collector reduces. """ - def _run(self): - clock = _SteppedClock() - state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) - # The resolved telemetry leaves the state via ToolEnd; the identity - # case below reconciles against what was RECORDED, not against the - # clock the test scripted. - resolved: list[Any] = [] - state.bind(lambda e: resolved.append(e.tool) if isinstance(e, ToolEndEvent) else None) - state.on_turn_start() - clock.at_ms = 100 - state.on_tool_execution_start({"toolCallId": "c1", "toolName": "bash", "args": {}}) - clock.at_ms = 1000 - state.on_turn_end(_turn_end_payload()) - clock.at_ms = 1500 - state.on_tool_execution_end({"toolCallId": "c1", "result": "ok"}) # closes in the GAP - clock.at_ms = 1600 - state.on_turn_start() - clock.at_ms = 2000 - state.on_turn_end(_turn_end_payload()) - - # Published through the real collector: the reducer hands over raw - # windows, and the tool subtraction happens once, there. - collector = EventCollector() - collector.on_event(AgentStartEvent(task_id="t", prompt="go", iteration=1, timestamp=_SPAN_BASE)) - for command in resolved: - collector.on_event(ToolEndEvent(task_id="t", turn_id="t1", tool=command)) - collector.on_event( - AgentEndEvent( - task_id="t", - status=AgentEndStatus.COMPLETED, - messages=list(state.messages), - usage=TokenUsage(), - timestamp=_SPAN_BASE + timedelta(milliseconds=2000), - ) - ) - published = [m for m in collector.build_turn_record().messages if m.role == "assistant"] - return resolved, published + def _run(self) -> Replay: + return _replay( + [ + _start(), + Tick(100), + _open("c1"), + Tick(1000), + _end(), + Tick(1500), + _close("c1"), # closes in the GAP + Tick(1600), + _start(), + Tick(2000), + _end(), + ] + )[0] def test_the_gap_slice_of_a_straddling_call_is_not_published_as_generation(self): - _, messages = self._run() + messages = _assistants(self._run()) # Window 2 tiles 1000 -> 2000. c1 ran for 1000 -> 1500 of it, so 500ms # is model time. With the reset left at `turn_start` this reads 1000.0. assert messages[1].generation_duration_ms == pytest.approx(500.0) def test_the_call_is_subtracted_from_exactly_one_window(self): - _, messages = self._run() + messages = _assistants(self._run()) # Window 1 bounded c1 at its own close (100 -> 1000); window 2 takes # only the remainder. assert messages[0].generation_duration_ms == pytest.approx(100.0) @@ -1411,71 +1435,49 @@ def test_the_four_bucket_identity_closes_exactly_across_the_boundary(self): This is the assertion the golden corpus CANNOT make: `_scrub.py` masks `generation_duration_ms` and both bounds to a placeholder, so a snapshot records that a window was measured and never what it measured. - Its identity check (`_scrub.py`) is an upper bound besides, so - under-accounting — the defect this phase fixes — passes it silently. - `scripts/timing/decompose_run.py --max-residual-pct` is the two-sided - check on live runs; this is the committed one. """ from coder_eval.timing import busy_ms - resolved, messages = self._run() + result = self._run() + messages = _assistants(result) lo, hi = messages[0].started_at, messages[1].completed_at generation_ms = sum(m.generation_duration_ms or 0.0 for m in messages) - command = next(c for c in resolved if c.tool_id == "c1") + command = next(c for c in result.record.commands if c.tool_id == "c1") + assert command.execution_started_at is not None and command.execution_completed_at is not None tool_ms = busy_ms([(command.execution_started_at, command.execution_completed_at)], lo, hi) assert generation_ms + tool_ms == pytest.approx((hi - lo).total_seconds() * 1000.0) + assert_identity_closes(result.record, started_at=result.started_at, ended_at=result.ended_at) def test_a_duplicate_turn_end_does_not_republish_the_previous_window(self): """A spent `turn_started_at` must not seed the next window. `close_window`'s `min(mark, item_start)` pulls the window open to cover - the item's own start. That is the backwards-clock defence, but a start - stamp left in place after its turn was published is not a backwards - clock — it is a stale value BEFORE the mark, so the guard reopens the - next window at the previous turn's start and publishes that whole span - again. Reproduced before the fix: 3000 ms of generation for a 2000 ms - turn. This reducer promises to survive a malformed stream, and Pi's CLI - retries internally, so a duplicate or replayed `turn_end` is a transport - hiccup rather than a hypothetical. + the item's own start. A start stamp left in place after its turn was + published is a stale value BEFORE the mark, so the guard would reopen the + next window at the previous turn's start and publish that whole span + again (3000 ms of generation for a 2000 ms turn). Pi's CLI retries + internally, so a duplicate `turn_end` is a transport hiccup rather than a + hypothetical. """ - clock = _SteppedClock() - state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) - state.on_turn_start() - clock.at_ms = 1000 - state.on_turn_end(_turn_end_payload()) - clock.at_ms = 2000 - state.on_turn_end(_turn_end_payload()) # no intervening `turn_start` - - messages = [m for m in state.messages if m.role == "assistant"] + result, _ = _replay([_start(), Tick(1000), _end(), Tick(2000), _end()]) # no intervening `turn_start` + + messages = _assistants(result) assert len(messages) == 2 assert messages[1].started_at == messages[0].completed_at assert sum(m.generation_duration_ms or 0.0 for m in messages) == pytest.approx(2000.0) def test_a_duplicate_turn_end_does_not_republish_the_previous_content(self): - """The CONTENT half of the same reset, and the same argument. - - `turn_text_parts` / `turn_tool_ids` were cleared in `on_turn_start` - only, so the replayed line re-emitted the first turn's text as its own - assistant message and re-listed the same `tool_use_ids` — one tool call - appearing to belong to two generations, and the text counted twice by - anything that reads the transcript. The sibling above pinned the timing - half while this one silently stayed broken, which is why it is asserted - separately rather than folded in. + """The CONTENT half of the same reset. + + Without it the replayed line re-emits the first turn's text as its own + assistant message and re-lists the same `tool_use_ids` — one tool call + appearing to belong to two generations. """ - clock = _SteppedClock() - state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) - state.on_turn_start() - state.on_message_update( - {"assistantMessageEvent": {"type": "text_delta", "delta": "First."}}, - ) - state.on_tool_execution_start({"toolCallId": "c1", "toolName": "bash", "args": {}}) - clock.at_ms = 1000 - state.on_turn_end(_turn_end_payload()) - clock.at_ms = 2000 - state.on_turn_end(_turn_end_payload()) # no intervening `turn_start` + text = {"type": "message_update", "assistantMessageEvent": {"type": "text_delta", "delta": "First."}} + result, _ = _replay([_start(), text, _open("c1"), Tick(1000), _end(), Tick(2000), _end()]) - messages = [m for m in state.messages if m.role == "assistant"] + messages = _assistants(result) assert len(messages) == 2 assert [b.text for b in messages[0].content_blocks if b.block_type == "text"] == ["First."] assert messages[0].tool_use_ids == ["c1"] @@ -1485,76 +1487,31 @@ def test_a_duplicate_turn_end_does_not_republish_the_previous_content(self): def test_an_unresolved_orphan_is_not_given_a_completion_or_a_duration(self): """Force-closing is not observing a completion. - The orphan sweep runs at finalization; stamping its instant as - `execution_completed_at` manufactures a bound, and the `duration_ms` - derived from it is the distance to whenever the sweep happened to run. - The pair then reads as a measured span that - `timing.subtract_tool_time` takes back out of a generation - window the tool never occupied. `execution_started_at` IS kept: the CLI - really did emit that start, and one bound alone forms no span. Same - rule as claude-code's `_finalize_commands` — unknown status and unknown - duration are one fact (CE058). + Stamping the sweep's instant as `execution_completed_at` manufactures a + bound that `timing.subtract_tool_time` then takes back out of a + generation window the tool never occupied. `execution_started_at` IS + kept: the CLI really did emit that start, and one bound alone forms no + span (CE058). """ - clock = _SteppedClock() - state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) - state.on_turn_start() - clock.at_ms = 500 - state.on_tool_execution_start({"toolCallId": "c1", "toolName": "bash", "args": {}}) - clock.at_ms = 4000 - closed: list[CommandTelemetry] = [] - state.bind(lambda e: closed.append(e.tool) if isinstance(e, ToolEndEvent) else None) - state.close_open_tools() + result, _ = _replay([_start(), Tick(500), _open("c1"), Tick(4000)]) + closed = [e.tool for e in result.events if isinstance(e, ToolEndEvent)] assert len(closed) == 1 assert closed[0].result_status == "unknown" - assert closed[0].execution_started_at == _SPAN_BASE + timedelta(milliseconds=500) + assert closed[0].error_message is None + assert closed[0].execution_started_at == _ms(500) assert closed[0].execution_completed_at is None assert closed[0].duration_ms is None def test_a_resolved_tool_still_gets_both_bounds_and_a_duration(self): - """The guard narrows the UNRESOLVED case only. - - Without this, deleting the whole stamping block would leave the sibling - above green while every real tool call lost its timing. - """ - clock = _SteppedClock() - state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) - state.on_turn_start() - clock.at_ms = 500 - state.on_tool_execution_start({"toolCallId": "c1", "toolName": "bash", "args": {}}) - clock.at_ms = 1200 - closed: list[CommandTelemetry] = [] - state.bind(lambda e: closed.append(e.tool) if isinstance(e, ToolEndEvent) else None) - state.on_tool_execution_end({"toolCallId": "c1", "result": "ok"}) + """The guard narrows the UNRESOLVED case only.""" + result, _ = _replay([_start(), Tick(500), _open("c1"), Tick(1200), _close("c1")]) + closed = [e.tool for e in result.events if isinstance(e, ToolEndEvent)] assert len(closed) == 1 - assert closed[0].execution_completed_at == _SPAN_BASE + timedelta(milliseconds=1200) + assert closed[0].execution_completed_at == _ms(1200) assert closed[0].duration_ms == pytest.approx(700.0) - def test_a_turn_that_never_finishes_does_not_advance_the_mark(self): - """The half of this that is still the reducer's job. - - There is no span list to preserve any more — the collector reduces the - ToolEnd stream itself. What the reducer still owns is the MARK: a turn - that published nothing must not advance it, or its time is handed to - whichever turn finishes next. - """ - clock = _SteppedClock() - state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) - state.on_turn_start() - clock.at_ms = 1000 - state.on_turn_end(_turn_end_payload()) - mark_after_flush = state.gen_mark - - clock.at_ms = 1600 - state.on_turn_start() - clock.at_ms = 1700 - state.on_tool_execution_start({"toolCallId": "c2", "toolName": "bash", "args": {}}) - clock.at_ms = 1900 - state.close_open_tools() # crash/timeout orphan sweep — no message appended - - assert state.gen_mark == mark_after_flush - class TestClockIsFreshPerTurn: """A retried turn must not inherit the crashed turn's clock. @@ -1562,8 +1519,8 @@ class TestClockIsFreshPerTurn: `TurnClock` anchors once and derives every later stamp from that anchor, so one surviving a retry would stamp the new turn against the old turn's wall origin — and over a long run accumulate drift against real wall time. The - lifetime is structural (the clock is built with the turn state, and the - state is built per `communicate()`), which is exactly the kind of property + lifetime is structural (the clock is built with the turn's emitter, and the + emitter is built per `communicate()`), which is exactly the kind of property that stays true only while someone is checking. """ @@ -1572,10 +1529,9 @@ async def test_a_turn_after_a_crash_is_anchored_to_a_fresh_clock(self, patch_exe patch_exec(_FakeProcess([], returncode=1, stderr=b"boom: bad model")) outcome = await _run(agent, tmp_path) assert outcome.status is AgentEndStatus.CRASHED - crashed_clock = agent # the state is gone; only the agent survives a crash patch_exec(_FakeProcess(HAPPY_STREAM)) - record = (await crashed_clock.communicate("try again", iteration=2)).record + record = (await agent.communicate("try again", iteration=2)).record # The recovered turn measured a real window of its own, rather than one # anchored before the crash — which a stale clock would have produced @@ -1589,8 +1545,8 @@ async def test_a_turn_after_a_crash_is_anchored_to_a_fresh_clock(self, patch_exe async def test_the_agent_retains_no_clock_between_turns(self, patch_exec, tmp_path): """Nothing to reset, because nothing survives — the structural half. - The clock is reachable only through the turn state, and the turn state - is a local of `communicate()`. If either were ever hoisted onto the + The clock is reachable only through the turn's emitter and decoder, which + are locals of `communicate()`. If either were ever hoisted onto the agent (a plausible refactor — several other fields are), the next turn would silently inherit the previous turn's anchor and no assertion about a single turn's numbers would notice. @@ -1599,7 +1555,9 @@ async def test_the_agent_retains_no_clock_between_turns(self, patch_exec, tmp_pa patch_exec(_FakeProcess(HAPPY_STREAM)) await _run(agent, tmp_path) - leaked = [name for name, value in vars(agent).items() if isinstance(value, _PiTurnState | TurnClock)] + leaked = [ + name for name, value in vars(agent).items() if isinstance(value, _PiDecoder | TurnEmitter | TurnClock) + ] assert not leaked, f"a turn's clock outlived its turn via {leaked}" @@ -1615,7 +1573,7 @@ class TestTheTurnBracketComesFromTheTurnClock: async def test_both_brackets_are_stamped_from_the_injected_clock( self, patch_exec, tmp_path, monkeypatch: pytest.MonkeyPatch ): - from coder_eval.agents import pi_agent as agent_module + import coder_eval.agent as agent_module monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) patch_exec(_FakeProcess(HAPPY_STREAM)) @@ -1632,7 +1590,7 @@ async def test_the_head_and_tail_are_measured_within_one_basis( A mixed pair is off by the anchor offset, not by a millisecond, so the bound here is what the assertion rests on rather than the sign. """ - from coder_eval.agents import pi_agent as agent_module + import coder_eval.agent as agent_module monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) patch_exec(_FakeProcess(HAPPY_STREAM)) diff --git a/tests/test_timing_identity_contract.py b/tests/test_timing_identity_contract.py index f8bab325..730585c6 100644 --- a/tests/test_timing_identity_contract.py +++ b/tests/test_timing_identity_contract.py @@ -69,7 +69,7 @@ ToolEndEvent, ToolEndStatus, ) -from coder_eval.testing import assert_identity_closes +from coder_eval.testing import Replay, ScriptedClock, Tick, assert_identity_closes, replay # The two CLI harnesses (opencode, codex) report their stamps as epoch @@ -152,7 +152,7 @@ def now(self) -> datetime: return at(self.at_ms) -def _pi_turn(*, untile: bool = False) -> Turn: +def _pi_replay(*, untile: bool = False) -> Replay: """Two tiled windows around a tool, with a real head and a real tail. The tool closes INSIDE the first window rather than across the boundary — @@ -165,35 +165,32 @@ def _pi_turn(*, untile: bool = False) -> Turn: so that the sensor can be shown to catch it. See ``test_the_sensor_sees_a_window_that_stops_tiling``. """ - from coder_eval.agents.pi_agent import _PiTurnState - - payload = {"message": {"role": "assistant", "usage": {"input": 10, "output": 5}, "stopReason": "stop"}} - clock = _InjectedClock() - state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) - commands: list[CommandTelemetry] = [] - state.bind(lambda e: commands.append(e.tool) if isinstance(e, ToolEndEvent) else None) - - clock.at_ms = 500 # CLI boot: head - state.on_turn_start() - clock.at_ms = 700 - state.on_tool_execution_start({"toolCallId": "c1", "toolName": "bash", "args": {}}) - clock.at_ms = 1200 - state.on_tool_execution_end({"toolCallId": "c1", "result": "ok"}) - clock.at_ms = 2000 - state.on_turn_end(payload) - clock.at_ms = 2600 # the inter-turn gap, which window 2 tiles back over - state.on_turn_start() - if untile: - state.gen_mark = None - clock.at_ms = 3000 - state.on_turn_end(payload) - - return Turn( - started_ms=0.0, - ended_ms=3500.0, # process teardown after the last turn: tail - messages=list(state.messages), - commands=commands, - ) + from coder_eval.agents.pi_agent import _PiDecoder + + payload = {"type": "turn_end", "message": {"role": "assistant", "usage": {"input": 10, "output": 5}}} + + class _Untiling(_PiDecoder): + def on_turn_start(self) -> None: + super().on_turn_start() + if untile: + self.gen_mark = None + + stream = [ + Tick(500), # CLI boot: head + {"type": "turn_start"}, + Tick(700), + {"type": "tool_execution_start", "toolCallId": "c1", "toolName": "bash", "args": {}}, + Tick(1200), + {"type": "tool_execution_end", "toolCallId": "c1", "result": "ok"}, + Tick(2000), + payload, + Tick(2600), # the inter-turn gap, which window 2 tiles back over + {"type": "turn_start"}, + Tick(3000), + payload, + Tick(3500), # process teardown after the last turn: tail + ] + return replay(stream, _Untiling, clock=ScriptedClock(BASE), end=lambda d: d.end(AgentEndStatus.COMPLETED)) # -------------------------------------------------------------------------- @@ -571,7 +568,8 @@ def _monotonic() -> float: def test_pi_buckets_tile_the_turn(): - _assert_closes(_pi_turn()) + result = _pi_replay() + assert_identity_closes(result.record, started_at=result.started_at, ended_at=result.ended_at) def test_opencode_buckets_tile_the_turn(monkeypatch: pytest.MonkeyPatch): @@ -633,12 +631,12 @@ def test_the_sensor_sees_a_window_that_stops_tiling(): 600 ms is the scripted gap between one ``turn_end`` and the next ``turn_start`` — real model time, which untiling books to nothing. """ - healthy = _pi_turn() - mutated = _pi_turn(untile=True) + healthy = _pi_replay() + mutated = _pi_replay(untile=True) - def _generation_ms(turn: Turn) -> float: - return sum(m.generation_duration_ms or 0.0 for m in turn.messages if isinstance(m, AssistantMessage)) + def _generation_ms(result: Replay) -> float: + return sum(m.generation_duration_ms or 0.0 for m in result.record.messages if isinstance(m, AssistantMessage)) assert _generation_ms(healthy) - _generation_ms(mutated) == pytest.approx(600.0) with pytest.raises(AssertionError, match="booked nowhere"): - _assert_closes(mutated) + assert_identity_closes(mutated.record, started_at=mutated.started_at, ended_at=mutated.ended_at) From 111d36ed96afc29a0a7d9a758b88309ee7a94bd8 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 18:39:43 -0700 Subject: [PATCH 05/25] =?UTF-8?q?feat(agents):=205/10=20=E2=80=94=20Subpro?= =?UTF-8?q?cessJsonlAgent=20under=20Pi=20and=20OpenCode;=20OpenCode=20is?= =?UTF-8?q?=20cli=5Fepoch=5Fms;=20CE073?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nd-JSON CLI transport (spawn with stdin=DEVNULL, stderr drain, read loop, settle, kill/reap and their constants) lives once in agents/_transport/subprocess_jsonl.py. Pi moves onto it; OpenCode is ported onto TurnEmitter with windows bounded by the CLI envelope timestamp and tool spans from state.time, so TimingBasis.MIXED is deleted. The orchestrator's pre/post-run shell and the docker run spawn no longer inherit stdin, and CE073 requires every asyncio subprocess spawn to decide its stdin. A captured real OpenCode stream is a new golden. Every opencode scenario is now fictional-duration (ms-scripted CLI stamps); opencode_c_multi_step_tiling measures a window again. Reviewed golden diffs: result_summary: opencode_a/b/c result null -> the final reply; opencode_e -> null. sequence_number 1 -> 0: opencode_b, opencode_c, opencode_d. opencode_d orphan: execution_completed_at set -> null, error_message -> null. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 11 +- docs/TASK_DEFINITION_GUIDE.md | 4 +- docs/agents/HARNESS_PARITY.md | 38 +- docs/agents/OPENCODE.md | 13 +- pyproject.toml | 1 + src/coder_eval/agents/_transport/__init__.py | 6 + .../agents/_transport/subprocess_jsonl.py | 351 +++++++ src/coder_eval/agents/opencode_agent.py | 862 +++------------- src/coder_eval/agents/pi_agent.py | 333 +------ src/coder_eval/isolation/docker_runner.py | 1 + src/coder_eval/models/harness_contract.py | 3 +- src/coder_eval/orchestrator.py | 3 + src/coder_eval/spi.py | 3 + src/coder_eval/streaming/emitter.py | 12 +- .../expected/opencode_a_single_text_turn.json | 2 +- .../opencode_b_tool_call_resolved.json | 4 +- .../opencode_c_multi_step_tiling.json | 4 +- .../expected/opencode_d_orphaned_tool.json | 6 +- .../opencode_e_error_after_generation.json | 7 +- .../expected/opencode_f_captured_stream.json | 148 +++ .../golden_streams/opencode_fixtures.py | 79 +- tests/fixtures/opencode_happy_stream.jsonl | 8 + .../ce073_create_subprocess_explicit_stdin.py | 56 ++ tests/lint/runner.py | 4 +- tests/test_agent_golden_master.py | 27 +- tests/test_custom_lint.py | 41 + tests/test_harness_conformance.py | 6 +- tests/test_opencode_agent.py | 937 ++++++++++-------- tests/test_pi_agent.py | 4 +- tests/test_spi.py | 3 +- tests/test_subprocess_jsonl_agent.py | 198 ++++ tests/test_timing_identity_contract.py | 97 +- tests/test_turn_emitter.py | 16 + 33 files changed, 1692 insertions(+), 1596 deletions(-) create mode 100644 src/coder_eval/agents/_transport/__init__.py create mode 100644 src/coder_eval/agents/_transport/subprocess_jsonl.py create mode 100644 tests/_fixtures/golden_streams/expected/opencode_f_captured_stream.json create mode 100644 tests/fixtures/opencode_happy_stream.jsonl create mode 100644 tests/lint/rules/ce073_create_subprocess_explicit_stdin.py create mode 100644 tests/test_subprocess_jsonl_agent.py diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index e4d97786..326cd063 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -630,6 +630,9 @@ blocks with zero events until the 300 s `turn_timeout`. Measured on 2026-09-16: | the same with `< /dev/null` | `SUCCESS` in 10 s | So every CLI spawn passes `stdin=asyncio.subprocess.DEVNULL`, which gives an immediate EOF. +The same inheritance reached the task's `pre_run`/`post_run` shell commands (an authored +`read` hung the task) and the `docker run` CLI, so those pass it too, and CE073 requires +every asyncio subprocess spawn under `src/` to decide its stdin. ## Reaping the CLI harnesses @@ -665,9 +668,11 @@ orchestrator's mid-turn backstop calls `kill()`, and dropping the dir there woul resume across a retried turn. `_cleanup` always calls `stop()` after any `kill()`, so the tempdir is still reclaimed. -`_TERM_GRACE_SECONDS` is re-declared at the same value in both nd-JSON harnesses rather -than shared: the CLI-driver hoist that would unify their teardown constants and reducers is -a tracked follow-up. `STDOUT_LINE_LIMIT_BYTES`, which IS canonical, is imported. +Both nd-JSON harnesses run on `agents/_transport/subprocess_jsonl.py::SubprocessJsonlAgent`, +which owns this whole transport once: the spawn, the stderr drain, the read loop, the settle, +`kill` / `kill_sync` / the reap, and `_TERM_GRACE_SECONDS`, `_DRAIN_SECONDS`, `_SIGKILL` and +`_MAX_UNRECOGNIZED_TYPES`. A subclass keeps its argv, environment, session handling and its +decoder. ## The system_prompt_semantics marker diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 3013dee0..03324816 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -1565,7 +1565,9 @@ pre_run: | `timeout` | 30 | Maximum seconds to wait (1–300) | | `fail_on_error` | `true` | When true, failure aborts evaluation with `FinalStatus.ERROR` | -Commands run sequentially with `cwd` set to the sandbox directory. stdout and stderr are +Commands run sequentially with `cwd` set to the sandbox directory, with stdin on +`/dev/null`: a command that reads stdin gets end-of-file at once instead of waiting for +input nobody can type. stdout and stderr are captured in `pre_run_results` on the evaluation result (truncated to 100KB each). When a command fails with `fail_on_error: true`, remaining commands are skipped. diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 0cfa2935..5722ba42 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -55,7 +55,7 @@ Generated from each agent class's `contract` by `make parity-table`; CE069 fails | `disallowed_tools` | enforced | unsupported | enforced | enforced | enforced | unsupported | | `cooperative_stop` | yes | yes | yes | yes | yes | no | | `usage_granularity` | generation | turn | turn | step | step | turn | -| `timing_basis` | turn_clock | cli_epoch_ms | turn_clock | mixed | turn_clock | turn_clock | +| `timing_basis` | turn_clock | cli_epoch_ms | turn_clock | cli_epoch_ms | turn_clock | turn_clock | | `permission_modes` | acceptEdits, bypassPermissions, default, plan | — | bypassPermissions, plan | bypassPermissions, plan | bypassPermissions, plan | — | @@ -100,18 +100,18 @@ wall clock its numbers account for. | Field | claude-code | codex | antigravity | opencode | pi | |---|---|---|---|---|---| -| `generation_duration_ms` RAW window (the reducer's part) | harness clock: previous SDK event → this message | SDK item stamps | harness clock: previous flush → this flush | harness clock: previous `step_finish` → this one | harness clock: previous `turn_end` → this one | +| `generation_duration_ms` RAW window (the reducer's part) | harness clock: previous SDK event → this message | SDK item stamps | harness clock: previous flush → this flush | CLI envelope `timestamp`: previous `step_finish` → this one | harness clock: previous `turn_end` → this one | | tool time subtracted from it | centrally | centrally | centrally | centrally | centrally | | what the **first** window covers | the first `message_start`, so CLI boot + TTFT are OUTSIDE it | the first SDK item's own start, so CLI boot + TTFT are OUTSIDE it | the first MODEL-source `Step`, so dispatch + TTFT are OUTSIDE it | the first `step_start`, so CLI boot + TTFT are OUTSIDE it | the first `turn_start`, so CLI boot + TTFT are OUTSIDE it | | `harness_startup_ms` (turn head) | ~3.6 s — CLI boot fused with TTFT | ~3.1 s — CLI boot fused with TTFT | ~4.7 s — dispatch fused with TTFT (its harness process is spawned once at startup, not per turn) | ~2.5 s — CLI boot fused with TTFT | ~0.23 s — CLI boot fused with TTFT | | `harness_teardown_ms` (turn tail) | ~1.3 s | ~13 ms | ~7 ms | ~26 ms | ~19 ms | -| tool `duration_ms` source | measured around the tool result | SDK `completed_at_ms − started_at_ms`; the item's own `duration_ms` only as a fallback | measured ACTIVE → DONE | measured around the tool event | measured around the tool event | -| `execution_started_at` / `execution_completed_at` | derived from the measured duration | SDK stamps (both, or neither) | measured at ACTIVE / DONE | measured | measured | +| tool `duration_ms` source | measured around the tool result | SDK `completed_at_ms − started_at_ms`; the item's own `duration_ms` only as a fallback | measured ACTIVE → DONE | CLI `state.time.end − state.time.start` | measured around the tool event | +| `execution_started_at` / `execution_completed_at` | derived from the measured duration | SDK stamps (both, or neither) | measured at ACTIVE / DONE | CLI `state.time` stamps (none when absent) | measured | | `generation_completed_at` | set | `None` — see below | `None` | `None` | `None` | | `message_id` source | SDK `message_id`; `None` when the stream carries none; `subagent-` for a synthesized sub-agent terminal | synthetic `turn_id-msg-N`, shared across the sub-messages of one generation; `turn_id-subagent-N` for recovered sub-agent generations | synthetic `turn_id-msg-N`, one per generation | CLI `messageID`; `None` when absent | CLI `responseId`; `None` when absent | | `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | -| clock basis for recorded stamps | one `TurnClock` per turn | SDK epoch ms (`_ms_to_dt`) — the subprocess's own clock, unreachable from the host, for BOTH window bounds and tool spans | one `TurnClock` per turn | **MIXED**: window bounds on the host `datetime.now()` (`:362`, `:696`); tool spans on CLI epoch ms (`_epoch_ms_to_dt`, `:406`/`:462`) | one `TurnClock` per turn | -| turn bracket (`AgentStartEvent` / `AgentEndEvent`) stamp | the same `TurnClock` (**CE064**) | raw `datetime.now()` — consistent with its epoch-ms bounds | the same `TurnClock` (**CE064**) | raw `datetime.now()` — consistent with its epoch-ms tool spans | the same `TurnClock` (**CE064**) | +| clock basis for recorded stamps | one `TurnClock` per turn | SDK epoch ms (`_ms_to_dt`) — the subprocess's own clock, unreachable from the host, for BOTH window bounds and tool spans | one `TurnClock` per turn | CLI epoch ms (`timing_basis` `cli_epoch_ms`): envelope `timestamp` for window bounds, `state.time` for tool spans; the host clock only for a window bound whose event carries no stamp | one `TurnClock` per turn | +| turn bracket (`AgentStartEvent` / `AgentEndEvent`) stamp | the same `TurnClock` (**CE064**) | raw `datetime.now()` — consistent with its epoch-ms bounds | the same `TurnClock` (**CE064**) | raw `datetime.now()` — consistent with its epoch-ms stamps | the same `TurnClock` (**CE064**) | | window built by `timing.py::close_window` | yes | yes | yes | yes | yes | [^identity]: "yes" is load-bearing, and THREE sensors check it, each seeing @@ -249,24 +249,14 @@ generation that arrives as a tool result and is never streamed excludes the message from `subtract_tool_time` and from the head/tail bracket. A stamp no bucket reads has no basis to share. -Codex and OpenCode are **not** converted, and their reasons are DIFFERENT — they -were stated as one, and that reading described a state OpenCode is already in. - -**Codex** is genuinely single-basis: both its window bounds and its tool spans -come from `_ms_to_dt` over the CLI's own epoch milliseconds, which cannot be -re-derived host-side. Converting only the window bounds would put two bases -inside one `busy_ms` subtraction — relocating the defect instead of removing it — -so it stays whole, and keeps the naive-local exposure. - -**OpenCode is already mixed, today.** Its window bounds are host -`datetime.now()` (`opencode_agent.py:362` at `step_start`, `:696` at -`step_finish`) while its tool spans are CLI epoch ms (`:406`, assigned to -`execution_started_at` at `:420`, and `:462`), so the two bases already meet -inside one subtraction. The argument for leaving it is therefore not the Codex -one: it is that a monotonic-derived anchor would trade a narrow NTP exposure on -the window bounds for intra-turn drift against the CLI's own tool stamps, which -is the larger of the two. The mixed basis is recorded here rather than defended -as uniform. +Codex and OpenCode are **not** converted to a `TurnClock`: both are single-basis on the +CLI's own clock (`timing_basis` `cli_epoch_ms`). Codex takes its window bounds and tool +spans from `_ms_to_dt` over the SDK's epoch milliseconds; OpenCode takes its window bounds +from each event's envelope `timestamp` and its tool spans from `state.time`. Neither can be +re-derived host-side, and converting only the window bounds would put two bases inside one +`busy_ms` subtraction — relocating the defect instead of removing it. Both keep the +naive-local exposure. OpenCode falls back to the host clock only for a window bound whose +event carries no envelope stamp, and warns when it does. Deadlines on every harness stay on raw `time.monotonic()` and must — a deadline may not move when the wall clock steps. diff --git a/docs/agents/OPENCODE.md b/docs/agents/OPENCODE.md index 58bf66c7..0f3904af 100644 --- a/docs/agents/OPENCODE.md +++ b/docs/agents/OPENCODE.md @@ -201,7 +201,14 @@ Mapping from the CLI's event vocabulary onto `TurnRecord`: | `text` | `TextChunkEvent` + `agent_output` | | `tool_use` | `ToolStartEvent` + `ToolEndEvent` (one terminal event carries both) | | `step_finish` | `TurnEndEvent` + per-step tokens/cost, one `AssistantMessage` | -| `error` | `AgentCrashError` with the partial turn preserved | +| `error` | a `CRASHED` turn, its partial record kept | + +Timing uses the CLI's own clock (`timing_basis` `cli_epoch_ms`): every event's envelope +`timestamp` (epoch ms) bounds the generation windows, and a tool's `state.time.start` / +`.end` is its execution span. A tool with no `state.time.end` gets no completion stamp and +no duration. An event with no envelope `timestamp` bounds its window on the host clock, +with one warning per turn. The CLI runs with stdin on `/dev/null`: it reads a non-TTY +stdin to EOF before it emits anything, so an inherited open stdin would stall the turn. Token buckets come from `step_finish.tokens`. Two conventions for `tokens.input` exist in the wild, and the stream's own `total` arbitrates **per step**: @@ -226,13 +233,13 @@ reconciliation invariant exact: summing the four buckets across Real per-call cost rides on `step_finish.cost` and lands on `token_usage.total_cost_usd`, so runs are costed from the provider's own accounting rather than the static rate card. The rate card -(`calculate_cost` over the captured buckets) fills two gaps so the run total +(`pricing.price_turn` over the captured buckets) fills two gaps so the run total never books tokens with no money: a stream that reports **no** cost at all (a provider or auth mode that omits it, or a turn that died before its first `step_finish`), and a stream that reports **`cost: 0`** for tokens the rate card prices above zero — OpenCode reports 0 when its own model registry has no price for the model, or under subscription-style auth, and neither means the -tokens were free (the fallback logs a warning naming the substituted amount). A +tokens were free. A *non-zero* cost the CLI reported always wins, and a genuinely free model still resolves to $0 because its rate entry is absent or all-zero. diff --git a/pyproject.toml b/pyproject.toml index e421b64f..2c3d5c5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -305,6 +305,7 @@ external = [ "CE069", "CE070", "CE071", + "CE073", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/src/coder_eval/agents/_transport/__init__.py b/src/coder_eval/agents/_transport/__init__.py new file mode 100644 index 00000000..293d4599 --- /dev/null +++ b/src/coder_eval/agents/_transport/__init__.py @@ -0,0 +1,6 @@ +"""Transport bases shared by more than one harness adapter.""" + +from coder_eval.agents._transport.subprocess_jsonl import JsonlDecoder, SubprocessJsonlAgent + + +__all__ = ["JsonlDecoder", "SubprocessJsonlAgent"] diff --git a/src/coder_eval/agents/_transport/subprocess_jsonl.py b/src/coder_eval/agents/_transport/subprocess_jsonl.py new file mode 100644 index 00000000..878acec3 --- /dev/null +++ b/src/coder_eval/agents/_transport/subprocess_jsonl.py @@ -0,0 +1,351 @@ +"""``SubprocessJsonlAgent``: one CLI invocation per turn, its stdout read as nd-JSON events. + +The base owns the transport: the spawn, the concurrent stderr drain, the read loop +racing each line against exit and the turn deadline, the cooperative stop, the +settle that decides how the turn ended, and every reap. A subclass supplies the +argv, the environment and a ``JsonlDecoder`` that turns one event into +``TurnEmitter`` calls. + +Rationale: .claude/notes/agents.md § Reaping the CLI harnesses +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import os +import signal +import time +from abc import ABC, abstractmethod +from collections.abc import Callable +from typing import Any, ClassVar + +from coder_eval.agent import Agent +from coder_eval.errors.agent import format_timeout_reason +from coder_eval.isolation.docker_runner import STDOUT_LINE_LIMIT_BYTES +from coder_eval.models import AgentState, ApiRoute, BaseAgentConfig +from coder_eval.streaming.callbacks import StreamCallback +from coder_eval.streaming.emitter import TurnEmitter, TurnOutcome +from coder_eval.streaming.events import AgentEndStatus, StopReason, end_status_for + + +logger = logging.getLogger(__name__) + +# Grace between SIGTERM and SIGKILL, and the post-EOF exit grace when no deadline is set. +_TERM_GRACE_SECONDS = 5.0 + +# SIGKILL does not exist on Windows (where the process-group sweep is a no-op). +_SIGKILL: signal.Signals = getattr(signal, "SIGKILL", signal.SIGTERM) + +# How long to keep reading after the CLI exited: a child can hold the pipes open. +_DRAIN_SECONDS = 2.0 + +# How many distinct unrecognized event types the drift crash names. +_MAX_UNRECOGNIZED_TYPES = 8 + + +class JsonlDecoder(ABC): + """One turn's reducer: one decoded nd-JSON event in, ``TurnEmitter`` calls out. + + ``error`` is a terminal CLI or provider error the stream reported; the base + crashes the turn on it. When ``error_survives_stop`` is False, a requested stop + wins: the stream can clear its error later, so a cut may land on a stale one. + """ + + error_survives_stop: ClassVar[bool] = False + + def __init__(self, emitter: TurnEmitter) -> None: + self.emitter = emitter + self.error: str | None = None + + @abstractmethod + def __call__(self, event: dict[str, Any]) -> None: + """Reduce one event; never raises on unexpected payload shapes.""" + + @abstractmethod + def end(self, status: AgentEndStatus, *, reason: str | None = None) -> TurnOutcome: + """End the turn: ``emitter.fail(status, reason)`` for CRASHED / TIMEOUT, else ``emitter.finalize``.""" + + +class SubprocessJsonlAgent[ConfigT: BaseAgentConfig](Agent[ConfigT]): + """An adapter whose turn is one CLI process streaming nd-JSON on stdout.""" + + cli_name: ClassVar[str] + docs_page: ClassVar[str] + recognized_events: ClassVar[frozenset[str]] + decoder: ClassVar[type[JsonlDecoder]] + + def __init__( + self, + config: ConfigT, + route: ApiRoute | None = None, + *, + task_id: str = "unknown", + cost_log_tags: dict[str, str] | None = None, + ) -> None: + super().__init__(config, route, cost_log_tags=cost_log_tags) + self.task_id = task_id + self.working_directory: str | None = None + self._process: asyncio.subprocess.Process | None = None + # Process-group ids of every invocation this agent spawned, swept on + # kill()/kill_sync()/stop(). + self._spawned_pgids: list[int] = [] + self._state = AgentState.WORKING + + @abstractmethod + def argv(self, prompt: str) -> list[str]: + """The CLI command line for one turn; ``prompt`` is a distinct argv element.""" + + @abstractmethod + def env(self) -> dict[str, str]: + """The CLI's whole environment.""" + + def observe(self, event: dict[str, Any]) -> None: + """See every decoded event before the decoder does (session ids, for example).""" + return None + + def clean_exit_problem(self, decoder: JsonlDecoder) -> str | None: + """Why a clean, uncut exit that recognized events must still crash, or None.""" + return None + + async def communicate( + self, + user_input: str, + *, + iteration: int, + stream_callback: StreamCallback | None = None, + timeout: float | None = None, + should_stop: Callable[[], StopReason | None] | None = None, + ) -> TurnOutcome: + """Run one CLI invocation as one turn; see ``Agent.communicate``.""" + if self.working_directory is None: + raise RuntimeError(f"{type(self).__name__}.start() must be called before communicate()") + + emitter = self._open_emitter( + prompt=user_input, + iteration=iteration, + model=self.config.model, + task_id=self.task_id, + stream_callback=stream_callback, + ) + emitter.begin() + decoder = self.decoder(emitter) + vocabulary = _Vocabulary() + # Deadlines stay on `time.monotonic()`: a deadline must not move when the wall clock steps. + deadline = None if timeout is None else time.monotonic() + timeout + requested_stop: StopReason | None = None + stderr_drain: asyncio.Future[bytes] | None = None + # Bound OUTSIDE the try so `finally` can tell "never spawned" from "spawned". + proc: asyncio.subprocess.Process | None = None + try: + proc = await asyncio.create_subprocess_exec( + *self.argv(user_input), + # A CLI that reads a non-TTY stdin to EOF stalls on an inherited open one. + # Rationale: .claude/notes/agents.md § Why a CLI never inherits stdin + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=self.working_directory, + env=self.env(), + # One nd-JSON event can carry a whole tool result, past the 64 KiB default. + limit=STDOUT_LINE_LIMIT_BYTES, + # Own process group, so teardown can killpg a lingering child. + start_new_session=os.name == "posix", + ) + self._process = proc + if os.name == "posix": + self._spawned_pgids.append(proc.pid) + assert proc.stdout is not None + # Drained CONCURRENTLY, or a child that fills the pipe hangs the turn. + if proc.stderr is not None: + stderr_drain = asyncio.ensure_future(proc.stderr.read()) + + exit_waiter = asyncio.ensure_future(proc.wait()) + read_task: asyncio.Future[bytes] | None = None + try: + while True: + remaining = None if deadline is None else deadline - time.monotonic() + if remaining is not None and remaining <= 0: + return await self._time_out(decoder, timeout or 0.0) + if read_task is None: + read_task = asyncio.ensure_future(proc.stdout.readline()) + done, _pending = await asyncio.wait( + {read_task, exit_waiter}, timeout=remaining, return_when=asyncio.FIRST_COMPLETED + ) + if not done: + return await self._time_out(decoder, timeout or 0.0) + if not read_task.done(): + # Exited with the read pending: bound the tail, a child may hold the pipe. + try: + await asyncio.wait_for(asyncio.shield(read_task), _DRAIN_SECONDS) + except TimeoutError: + break + line = read_task.result() + read_task = None + if not line: + break + self._handle_line(line, decoder, vocabulary) + requested_stop = should_stop() if should_stop is not None else None + if requested_stop is not None: + await self.kill() + break + finally: + if read_task is not None: + read_task.cancel() + exit_waiter.cancel() + + return await self._settle( + proc, + decoder, + vocabulary, + stderr_drain, + requested_stop=requested_stop, + deadline=deadline, + timeout=timeout, + ) + except asyncio.CancelledError: + self._state = AgentState.ERROR + decoder.end(AgentEndStatus.CRASHED, reason="turn cancelled") + raise + except Exception as e: + # A spawn failure, a StreamReader ValueError past `limit`, a malformed payload. + logger.warning("%s: turn failed", self.cli_name.lower(), exc_info=True) + return self._crash(decoder, f"{self.cli_name} turn failed: {e!s}") + finally: + if stderr_drain is not None: + stderr_drain.cancel() + self._reap(proc) + self._process = None + + async def _settle( + self, + proc: asyncio.subprocess.Process, + decoder: JsonlDecoder, + vocabulary: _Vocabulary, + stderr_drain: asyncio.Future[bytes] | None, + *, + requested_stop: StopReason | None, + deadline: float | None, + timeout: float | None, + ) -> TurnOutcome: + """Reap the CLI once the read loop is done and end the turn. + + In order: no exit by the deadline is TIMEOUT (without one, CRASHED after the + grace); a stream error is CRASHED (unless a stop was requested and the decoder's + error does not survive one); a requested stop ends with its status; a non-zero + exit, no recognized event, or a subclass's ``clean_exit_problem`` is CRASHED; + else COMPLETED. + + Rationale: .claude/notes/agents.md § Why a clean exit can still be a crash + """ + remaining = None if deadline is None else max(0.0, deadline - time.monotonic()) + try: + await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_SECONDS if remaining is None else remaining) + except TimeoutError: + if remaining is not None: + return await self._time_out(decoder, timeout or 0.0) + await self.kill() + return self._crash( + decoder, f"{self.cli_name} closed its event stream but did not exit within {_TERM_GRACE_SECONDS:.0f}s" + ) + stderr_bytes = b"" + if stderr_drain is not None: + with contextlib.suppress(TimeoutError): + stderr_bytes = await asyncio.wait_for(asyncio.shield(stderr_drain), timeout=_DRAIN_SECONDS) + + if decoder.error is not None and (requested_stop is None or decoder.error_survives_stop): + return self._crash(decoder, f"{self.cli_name} error: {decoder.error}") + if requested_stop is not None: + return decoder.end(end_status_for(requested_stop)) + if proc.returncode not in (0, None): + detail = stderr_bytes.decode("utf-8", "replace").strip() or f"exit code {proc.returncode}" + return self._crash(decoder, f"{self.cli_name} exited non-zero: {detail}") + if vocabulary.recognized == 0: + seen = ", ".join(sorted(vocabulary.unrecognized)) or "none (stdout carried no JSON events)" + return self._crash( + decoder, + f"{self.cli_name} exited cleanly but the turn captured no recognized events. Unrecognized event " + + f"types seen: {seen}. The CLI's event schema may have changed — see {self.docs_page} before " + + "trusting any run from this CLI version.", + ) + problem = self.clean_exit_problem(decoder) + if problem is not None: + return self._crash(decoder, problem) + return decoder.end(AgentEndStatus.COMPLETED) + + def _crash(self, decoder: JsonlDecoder, message: str) -> TurnOutcome: + self._state = AgentState.ERROR + return decoder.end(AgentEndStatus.CRASHED, reason=message) + + async def _time_out(self, decoder: JsonlDecoder, timeout: float) -> TurnOutcome: + await self.kill() + self._state = AgentState.ERROR + return decoder.end(AgentEndStatus.TIMEOUT, reason=format_timeout_reason(timeout)) + + def _handle_line(self, line: bytes, decoder: JsonlDecoder, vocabulary: _Vocabulary) -> None: + """Parse one line and hand a JSON object to ``observe`` and the decoder; skip anything else.""" + raw = line.decode("utf-8", "replace").strip() + if not raw: + return + try: + event = json.loads(raw) + except json.JSONDecodeError: + logger.debug("%s: skipping non-JSON stdout line: %s", self.cli_name.lower(), raw[:200]) + return + if not isinstance(event, dict): + return + event_type = str(event.get("type") or "") + if event_type in self.recognized_events: + vocabulary.recognized += 1 + elif len(vocabulary.unrecognized) < _MAX_UNRECOGNIZED_TYPES: + vocabulary.unrecognized.add(event_type or "") + self.observe(event) + decoder(event) + + async def kill(self) -> None: + """SIGTERM the in-flight CLI, SIGKILL it after the grace, then sweep its process groups.""" + proc = self._process + if proc is not None and proc.returncode is None: + with contextlib.suppress(ProcessLookupError): + proc.terminate() + with contextlib.suppress(TimeoutError, asyncio.TimeoutError): + await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_SECONDS) + if proc.returncode is None: + with contextlib.suppress(ProcessLookupError): + proc.kill() + self._sweep_process_groups() + + def kill_sync(self) -> None: + """SIGKILL the in-flight CLI and its process groups (watchdog thread; must not await).""" + proc = self._process + if proc is not None and proc.returncode is None: + with contextlib.suppress(ProcessLookupError, PermissionError): + os.kill(proc.pid, _SIGKILL) + self._sweep_process_groups() + + def _sweep_process_groups(self) -> None: + """SIGKILL every process group this agent spawned (POSIX only); each holds one invocation's children.""" + if os.name != "posix": + return + for pgid in self._spawned_pgids: + with contextlib.suppress(ProcessLookupError, PermissionError, OSError): + os.killpg(pgid, _SIGKILL) + self._spawned_pgids.clear() + + def _reap(self, proc: asyncio.subprocess.Process | None) -> None: + """Kill a CLI still running as the turn unwinds; synchronous, so it survives a cancel.""" + if proc is None or proc.returncode is not None: + return + with contextlib.suppress(ProcessLookupError, PermissionError): + proc.kill() + self._sweep_process_groups() + + +class _Vocabulary: + """The drift check's evidence: how many events matched, and a sample of the types that did not.""" + + def __init__(self) -> None: + self.recognized = 0 + self.unrecognized: set[str] = set() diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index ab35dc59..98f0a5b2 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -1,9 +1,8 @@ """OpenCode agent implementation (the open-source terminal coding agent). Drives the ``opencode`` CLI in non-interactive mode, which streams -newline-delimited JSON events on stdout, and reduces that stream into the -standardized coder_eval event protocol so :class:`EventCollector` builds the -``TurnRecord``. +newline-delimited JSON events on stdout, and reduces that stream through one +``TurnEmitter`` per turn, on the CLI's own epoch-millisecond stamps. The CLI emits TWO envelope shapes on the same stream: the normal form carries its payload under ``part``, while the CLI's own error path emits a flat object @@ -18,60 +17,37 @@ from __future__ import annotations import asyncio -import contextlib import json import logging import os import shutil -import signal import tempfile -import time -from collections.abc import Callable from datetime import datetime from pathlib import Path -from typing import Any, Literal, NoReturn +from typing import Any -from coder_eval.agent import Agent -from coder_eval.errors import AgentCrashError, TurnTimeoutError -from coder_eval.isolation.docker_runner import STDOUT_LINE_LIMIT_BYTES +from coder_eval.agents._transport import JsonlDecoder, SubprocessJsonlAgent from coder_eval.models import ( READ_ONLY_DENIED_TOOLS, AgentKind, AgentState, ApiRoute, - AssistantMessage, - CommandTelemetry, ContentBlock, Enforcement, HarnessContract, OpenCodeAgentConfig, PermissionMode, - ResultSummary, TimingBasis, TokenUsage, ToolNameMap, - TranscriptMessage, - TurnRecord, UsageGranularity, ) from coder_eval.pricing import price_turn -from coder_eval.streaming.callbacks import StreamCallback, safe_emit -from coder_eval.streaming.collector import EventCollector -from coder_eval.streaming.emitter import TurnOutcome +from coder_eval.streaming.emitter import Generation, TurnEmitter, TurnOutcome from coder_eval.streaming.events import ( - AgentEndEvent, AgentEndStatus, - AgentStartEvent, - StopReason, - StreamEvent, - TextChunkEvent, - ToolEndEvent, ToolEndStatus, - ToolStartEvent, - TurnEndEvent, TurnEndStatus, - TurnStartEvent, - end_status_for, ) from coder_eval.timing import close_window @@ -80,21 +56,6 @@ logger = logging.getLogger(__name__) -# Grace period between SIGTERM and SIGKILL when tearing down the CLI subprocess. -# Doubles as the post-EOF exit grace in _settle_turn when no deadline is set. -_TERM_GRACE_SECONDS = 5.0 - -# SIGKILL does not exist on Windows (where the process-group sweep is a no-op -# anyway); resolve it dynamically so the module imports and typechecks on every -# platform, falling back to SIGTERM for the direct-pid kill_sync path. -_SIGKILL: signal.Signals = getattr(signal, "SIGKILL", signal.SIGTERM) - -# How long to keep draining stdout/stderr after the CLI has been reaped: -# `opencode run` leaves a server child holding the pipes open, so EOF never -# arrives on its own. -# Rationale: .claude/notes/agents.md § Reaping the CLI harnesses -_DRAIN_SECONDS = 2.0 - # The CLI's OWN compact vocabulary, captured from a live run — NOT the # `session.next.*` names in the server's OpenAPI schema, which describe # `opencode serve`'s HTTP/SSE surface. The two are not interchangeable. @@ -109,10 +70,6 @@ # Rationale: .claude/notes/agents.md § Why a clean exit can still be a crash _RECOGNIZED_EVENTS = frozenset({_STEP_START, _STEP_FINISH, _TEXT, _TOOL_USE, _ERROR}) -# How many distinct unrecognized event-type strings to retain for the crash -# message when the vocabulary check fails (diagnosis, not an exhaustive list). -_MAX_UNRECOGNIZED_TYPES = 8 - # OpenCode's native tool names -> the canonical (Claude) vocabulary that every # criterion is written against. Unknown tools pass through unchanged. # Rationale: .claude/notes/agents.md § Tool-name and argument normalization @@ -190,27 +147,21 @@ _PROMPT_FILE_NAME = "system_prompt.md" -# ToolEndStatus -> CommandTelemetry.result_status (the persisted tri-state). -_RESULT_STATUS: dict[ToolEndStatus, Literal["success", "error", "unknown"]] = { - ToolEndStatus.OK: "success", - ToolEndStatus.ERROR: "error", - ToolEndStatus.PERMISSION_DENIED: "error", - ToolEndStatus.UNRESOLVED: "unknown", -} - -def _unwrap(obj: dict[str, Any]) -> tuple[str, dict[str, Any]]: - """Normalize an OpenCode CLI event to ``(event_type, payload)``. +def _unwrap(obj: dict[str, Any]) -> tuple[str, dict[str, Any], datetime | None]: + """Normalize an OpenCode CLI event to ``(event_type, payload, envelope stamp)``. Every line carries its payload under ``part`` except the CLI's own error line, which is flat. Returning the top-level dict for that case is safe: the - accessors read named keys, never iterate. + accessors read named keys, never iterate. The envelope ``timestamp`` (epoch + ms) is the CLI's own stamp for the event; ``None`` when absent. """ event_type = str(obj.get("type") or "") + stamp = _epoch_ms_to_dt(obj.get("timestamp")) part = obj.get("part") if isinstance(part, dict): - return event_type, part - return event_type, obj + return event_type, part, stamp + return event_type, obj, stamp def _epoch_ms_to_dt(value: Any) -> datetime | None: @@ -237,157 +188,125 @@ def _canonical_params(tool_name: str, params: dict[str, Any]) -> dict[str, Any]: return {rename.get(key, key): value for key, value in params.items()} -class _OpenCodeTurnState: - """Per-``communicate()`` accumulator: events in, finalization payload out. +class _OpenCodeDecoder(JsonlDecoder): + """One turn's reducer: OpenCode's nd-JSON events in, ``TurnEmitter`` calls out. - Owns everything the terminal ``AgentEndEvent`` must carry (transcript - messages, cumulative usage, text output) plus the open-tool bookkeeping - needed to force-close orphans when a turn dies mid-flight. + Timing is the CLI's own: window bounds come from each event's envelope + ``timestamp`` and tool spans from ``state.time``. A missing envelope stamp + falls back to the host clock for a window bound, with one warning per turn. + The CLI's ``error`` event is final, so it crashes the turn even after a stop. """ - def __init__(self, *, task_id: str, iteration: int, user_input: str, model: str | None) -> None: - self.task_id = task_id - self.iteration = iteration - self.user_input = user_input - self.model = model + error_survives_stop = True - self.started_at = time.monotonic() - self.session_id: str | None = None - self.thread_id: str | None = None - - # Cumulative turn totals (summed across every inner step). + def __init__(self, emitter: TurnEmitter) -> None: + super().__init__(emitter) self.usage = TokenUsage() self.cost_usd: float = 0.0 self.saw_cost = False - - self.messages: list[TranscriptMessage] = [] - self.text_parts: list[str] = [] + self.stop_reason: str | None = None self.step_count = 0 # Steps the CLI reported as FINISHED, as opposed to `step_count`, which - # counts the ones it started. `_settle_turn` needs the distinction. + # counts the ones it started. `clean_exit_problem` needs the distinction. self.steps_finished = 0 - self.turn_id: str = "" - # True between a step's `step_start` and its `step_finish`. `finalize` - # needs it to close a TurnStartEvent the stream never got to close. - self.step_open = False + self.tool_count = 0 self.step_started_at: datetime | None = None # Where the NEXT generation window starts: the previous step's finish. - # None until the first step finishes, and deliberately so — everything - # before the first `step_start` is CLI process spawn, not model time. + # None until the first step finishes — everything before the first + # `step_start` is CLI process spawn, not model time. # Rationale: .claude/notes/agents.md § Per-harness generation marks self.gen_mark: datetime | None = None self.step_text_parts: list[str] = [] self.step_tool_ids: list[str] = [] - - # callID -> (telemetry, started_at) for tools awaiting a result. - self.open_tools: dict[str, CommandTelemetry] = {} - self.sequence = 0 - self.stop_reason: str | None = None - self.error_message: str | None = None - # Guards the one-terminal-event rule; see finalize(). - self.finalized = False - # Guards _warn_token_shape: one report per turn, not one per step. + # callID -> the latest canonical parameters of a call still awaiting its result. + self.open_tools: dict[str, dict[str, Any]] = {} + self._tool_names: dict[str, str] = {} self.warned_token_shape = False - # Vocabulary drift detection (see _settle_turn): how many events matched - # _RECOGNIZED_EVENTS, and a bounded sample of the types that did not. - self.recognized_events = 0 - self.unrecognized_types: set[str] = set() - - self._emit: Callable[[StreamEvent], None] = lambda _e: None - - def bind(self, emit: Callable[[StreamEvent], None]) -> None: - self._emit = emit - - def emit(self, event: StreamEvent) -> None: - self._emit(event) - - @property - def agent_output(self) -> str: - return "".join(self.text_parts) + self.warned_missing_stamp = False - # --- event handlers ---------------------------------------------------- + def __call__(self, event: dict[str, Any]) -> None: + event_type, part, stamp = _unwrap(event) + if event_type == _STEP_START: + self.on_step_start(part, stamp) + elif event_type == _TEXT: + self.on_text(part) + elif event_type == _TOOL_USE: + self.on_tool_use(part) + elif event_type == _STEP_FINISH: + self.on_step_finish(part, stamp) + elif event_type == _ERROR: + self.on_error(part) + else: + logger.debug("opencode: unhandled event type %r", event_type) - def on_step_start(self, part: dict[str, Any]) -> None: + def end(self, status: AgentEndStatus, *, reason: str | None = None) -> TurnOutcome: + """End the turn: ``fail`` for CRASHED / TIMEOUT (with ``reason``), else ``finalize``.""" + reported = self.usage.model_copy(update={"total_cost_usd": self.cost_usd if self.saw_cost else None}) + usage = self.usage.model_copy(update={"total_cost_usd": price_turn(reported, (self.emitter.model,))}) + if status is AgentEndStatus.CRASHED or status is AgentEndStatus.TIMEOUT: + return self.emitter.fail(status, reason or status.value, usage=usage) + return self.emitter.finalize(status, usage=usage, stop_reason=self.stop_reason) + + def _bound(self, stamp: datetime | None) -> datetime: + """A window bound: the CLI's envelope stamp, else the host clock (warned once per turn).""" + if stamp is not None: + return stamp + if not self.warned_missing_stamp: + self.warned_missing_stamp = True + logger.warning("opencode: an event carried no envelope timestamp; bounding its window on the host clock") + return self.emitter.now() + + def on_step_start(self, part: dict[str, Any], stamp: datetime | None) -> None: + if self.emitter.inner_turn_open: + self.emitter.end_inner_turn(TurnEndStatus.CRASHED) self.step_count += 1 - self.step_open = True - self.turn_id = str(part.get("messageID") or f"step_{self.step_count}") - self.step_started_at = datetime.now() + self.emitter.begin_inner_turn(str(part.get("messageID") or f"step_{self.step_count}")) + self.step_started_at = self._bound(stamp) self.step_text_parts = [] self.step_tool_ids = [] - # No per-step span list to reset here any more: the collector sees every - # span at once and clips each to the window it overlaps. - self.emit( - TurnStartEvent( - task_id=self.task_id, - thread_id=self.thread_id, - turn_id=self.turn_id, - model=self.model, - ) - ) def on_text(self, part: dict[str, Any]) -> None: """``text`` carries a COMPLETE assistant message, not a streaming delta.""" text = part.get("text") if not isinstance(text, str) or not text: return - self.text_parts.append(text) self.step_text_parts.append(text) - self.emit(TextChunkEvent(task_id=self.task_id, thread_id=self.thread_id, turn_id=self.turn_id, text=text)) + self.emitter.text(text) def on_tool_use(self, part: dict[str, Any]) -> None: """A ``tool_use`` event carries the tool's whole state under ``state``. - In practice the CLI emits one already-``completed`` event per call rather - than a call/result pair, so both ``ToolStart`` and ``ToolEnd`` are - synthesized here. A non-terminal state is still handled: the tool is left - open and closed by a later event for the same ``callID``, or force-closed - as ``unresolved``. Execution timestamps come from ``state.time``, so - ``duration_ms`` is the tool's real runtime, not our parse instant. + The CLI usually emits one already-``completed`` event per call. A + non-terminal state leaves the call open, to be closed by a later event for + the same ``callID`` or swept as ``unresolved``. The span is ``state.time``. """ state = part.get("state") state = state if isinstance(state, dict) else {} - call_id = str(part.get("callID") or f"call_{self.sequence + 1}") + call_id = str(part.get("callID") or f"call_{self.tool_count + 1}") time_val = state.get("time") times = time_val if isinstance(time_val, dict) else {} - started = _epoch_ms_to_dt(times.get("start")) params = state.get("input") params = params if isinstance(params, dict) else {} - telemetry = self.open_tools.get(call_id) - if telemetry is None: - self.sequence += 1 + if call_id not in self.open_tools: + self.tool_count += 1 raw_tool = str(part.get("tool") or "unknown") tool_name = _TOOL_NAME_MAP.get(raw_tool.lower(), raw_tool) - telemetry = CommandTelemetry( - tool_name=tool_name, - tool_id=call_id, - assistant_turn_index=self.step_count, - timestamp=started or datetime.now(), - execution_started_at=started, - parameters=_canonical_params(tool_name, params), - sequence_number=self.sequence, - ) - self.open_tools[call_id] = telemetry + canonical = _canonical_params(tool_name, params) + self.emitter.open_tool(call_id, tool_name, canonical, started_at=_epoch_ms_to_dt(times.get("start"))) + self.open_tools[call_id] = canonical self.step_tool_ids.append(call_id) - self.emit( - ToolStartEvent(task_id=self.task_id, thread_id=self.thread_id, turn_id=self.turn_id, tool=telemetry) - ) - else: - # A SECOND event for a call already open. The first routinely carries - # no `input` yet, so freezing its view would leave `parameters` - # permanently `{}` and zero every `command_executed` row while the run - # looked normal. Later evidence wins; absent evidence clears nothing. - if params: - telemetry.parameters = _canonical_params(telemetry.tool_name, params) - if started is not None: - telemetry.execution_started_at = started + self._tool_names[call_id] = tool_name + elif params: + # A SECOND event for an open call: the first routinely carries no + # `input` yet. Later evidence wins; absent evidence clears nothing. + self.open_tools[call_id] = _canonical_params(self._tool_names[call_id], params) status_text = str(state.get("status") or "").lower() + if status_text in ("pending", "running"): + return output = state.get("output") error_text = state.get("error") - if status_text in ("pending", "running"): - return # still in flight; a later event (or the orphan sweep) closes it - if status_text == "error" or error_text: message = str(error_text or output or "tool failed") denied = "permission" in message.lower() or "denied" in message.lower() @@ -395,53 +314,14 @@ def on_tool_use(self, part: dict[str, Any]) -> None: else: message = None status = ToolEndStatus.OK - - # `times` is the SAME dict read at the top: nothing between rebinds or - # mutates `state`. - self._close_tool( + self.emitter.close_tool( call_id, status=status, summary=output if isinstance(output, str) else None, error=message, + parameters=self.open_tools.pop(call_id), completed_at=_epoch_ms_to_dt(times.get("end")), - ) - - def _close_tool( - self, - call_id: str, - *, - status: ToolEndStatus, - summary: str | None, - error: str | None, - completed_at: datetime | None = None, - ) -> None: - telemetry = self.open_tools.pop(call_id, None) - if telemetry is None: - # A result with no matching call (shouldn't happen, but never drop it). - self.sequence += 1 - telemetry = CommandTelemetry( - tool_name="unknown", - tool_id=call_id, - assistant_turn_index=self.step_count, - timestamp=datetime.now(), - sequence_number=self.sequence, - ) - completed = completed_at or datetime.now() - telemetry.execution_completed_at = completed - if telemetry.execution_started_at is not None: - telemetry.duration_ms = (completed - telemetry.execution_started_at).total_seconds() * 1000 - telemetry.result_status = _RESULT_STATUS[status] - # Stored untruncated by design (sub-agent returns must survive whole). - telemetry.result_summary = summary - telemetry.error_message = error - self.emit( - ToolEndEvent( - task_id=self.task_id, - thread_id=self.thread_id, - turn_id=self.turn_id, - tool=telemetry, - status=status, - ) + started_at=_epoch_ms_to_dt(times.get("start")), ) def _warn_token_shape(self, message: str, *args: Any) -> None: @@ -540,9 +420,8 @@ def _fresh_input_slice( ) return raw_in - def on_step_finish(self, part: dict[str, Any]) -> None: + def on_step_finish(self, part: dict[str, Any], stamp: datetime | None) -> None: self.steps_finished += 1 - self.step_open = False tokens = part.get("tokens") tokens = tokens if isinstance(tokens, dict) else {} cache_val = tokens.get("cache") @@ -554,16 +433,14 @@ def on_step_finish(self, part: dict[str, Any]) -> None: step_cr = self._as_int("cache.read", cache.get("read") or 0) step_in = self._fresh_input_slice(tokens, raw_in, raw_out, step_reasoning, step_cw, step_cr) - # Reasoning bills at the output rate but is reported apart from `output`, - # so fold it into the turn total; the per-message record keeps it apart. - step_out = raw_out + step_reasoning - - self.usage = TokenUsage( - uncached_input_tokens=self.usage.uncached_input_tokens + step_in, - output_tokens=self.usage.output_tokens + step_out, - cache_creation_input_tokens=self.usage.cache_creation_input_tokens + step_cw, - cache_read_input_tokens=self.usage.cache_read_input_tokens + step_cr, + # Reasoning bills at the output rate but is reported apart from `output`. + step_delta = TokenUsage( + uncached_input_tokens=step_in, + output_tokens=raw_out + step_reasoning, + cache_creation_input_tokens=step_cw, + cache_read_input_tokens=step_cr, ) + self.usage += step_delta cost = part.get("cost") if isinstance(cost, int | float): self.cost_usd += float(cost) @@ -573,7 +450,7 @@ def on_step_finish(self, part: dict[str, Any]) -> None: if isinstance(finish, str) and finish: self.stop_reason = finish - completed = datetime.now() + completed = self._bound(stamp) step_start = self.step_started_at or completed blocks: list[ContentBlock] = [] step_text = "".join(self.step_text_parts) @@ -583,135 +460,46 @@ def on_step_finish(self, part: dict[str, Any]) -> None: blocks.append(ContentBlock(block_type="tool_use", sequence=i, tool_use_id=tool_id)) # Tile from the previous step's finish. The RAW window only. - window = close_window( - mark=self.gen_mark if self.gen_mark is not None else step_start, - now=completed, - item_start=step_start, - ) - self.messages.append( - AssistantMessage( - started_at=window.started_at, - completed_at=window.completed_at, - generation_duration_ms=window.duration_ms, - content_blocks=blocks, - tool_use_ids=list(self.step_tool_ids), - input_tokens=step_in, - output_tokens=step_out, - cache_creation_tokens=step_cw, - cache_read_tokens=step_cr, - reasoning_tokens=step_reasoning, - stop_reason=finish if isinstance(finish, str) else None, - model=self.model, - message_id=str(part.get("messageID") or "") or None, - ) + self.emitter.add_generation( + message_id=str(part.get("messageID") or "") or None, + window=close_window( + mark=self.gen_mark if self.gen_mark is not None else step_start, now=completed, item_start=step_start + ), + parts=[ + Generation( + blocks=blocks, + tokens=step_delta, + reasoning_tokens=step_reasoning, + stop_reason=finish if isinstance(finish, str) else None, + ) + ], ) - # A message was appended, so the next window starts where this one ended. - # Only `step_finish` advances the mark. self.gen_mark = completed # SPENT state, cleared HERE and not only in `on_step_start`: a second # `step_finish` with no intervening start would otherwise republish this - # step's whole span as the next one's. The `min()` in `close_window` still - # defends a genuinely OPEN step against a backwards clock, which is what - # it is for — this reducer's stamps are raw `datetime.now()`. + # step's whole span as the next one's. # Rationale: .claude/notes/agents.md § Per-harness generation marks self.step_started_at = None - self.emit( - TurnEndEvent( - task_id=self.task_id, - thread_id=self.thread_id, - turn_id=self.turn_id, - status=TurnEndStatus.COMPLETED, - tokens=TokenUsage( - uncached_input_tokens=step_in, - output_tokens=step_out, - cache_creation_input_tokens=step_cw, - cache_read_input_tokens=step_cr, - ), - ) - ) + if self.emitter.inner_turn_open: + self.emitter.end_inner_turn(TurnEndStatus.COMPLETED, tokens=step_delta) def on_error(self, part: dict[str, Any]) -> None: - """Record the CLI's own structured error, which ``_settle_turn`` crashes on. + """Record the CLI's own structured error, which the settle crashes on. - The payload is the flat envelope, and its shape varies: a nested - ``error.data.message`` when the CLI has one, otherwise the error's - ``name``. Anything else degrades to its string form rather than raising. + Its shape varies: a nested ``error.data.message`` when the CLI has one, + otherwise the error's ``name``; anything else degrades to its string form. """ error = part.get("error") if isinstance(error, dict): data = error.get("data") message = (data or {}).get("message") if isinstance(data, dict) else None - self.error_message = str(message or error.get("name") or "unknown error") + self.error = str(message or error.get("name") or "unknown error") else: - self.error_message = str(error or "unknown error") - - def close_open_tools(self) -> None: - """Force-close every tool still awaiting a result (crash/timeout orphans).""" - for call_id in list(self.open_tools): - self._close_tool(call_id, status=ToolEndStatus.UNRESOLVED, summary=None, error="no result observed") - - def finalize( - self, - status: AgentEndStatus, - *, - crashed: bool = False, - crash_reason: str | None = None, - ) -> None: - """Close orphaned tools and emit the terminal ``AgentEndEvent``. - - Idempotent: the protocol allows EXACTLY ONE ``AgentEndEvent`` per - ``communicate()``. - - Rationale: .claude/notes/agents.md § Shared turn lifecycle - """ - if self.finalized: - return - self.finalized = True - self.close_open_tools() - reported = self.usage.model_copy(update={"total_cost_usd": self.cost_usd if self.saw_cost else None}) - usage = self.usage.model_copy(update={"total_cost_usd": price_turn(reported, (self.model,))}) - # A step still open never received its `step_finish`; close it or the - # one-pair-per-inner-turn contract breaks. Completed steps already closed - # themselves, so this fires ONLY for the straggler. - if self.step_open: - self.step_open = False - self.emit( - TurnEndEvent( - task_id=self.task_id, - thread_id=self.thread_id, - turn_id=self.turn_id, - status=TurnEndStatus(status.value), - tokens=None, - ) - ) - self.emit( - AgentEndEvent( - task_id=self.task_id, - thread_id=self.thread_id, - status=status, - usage=usage, - iteration=self.iteration, - user_input=self.user_input, - agent_output=self.agent_output, - model_used=self.model, - assistant_turn_count=self.step_count, - messages=list(self.messages), - num_turns=self.step_count, - result_summary=ResultSummary( - is_error=crashed, - subtype=status.value, - stop_reason=self.stop_reason, - result=crash_reason or self.error_message, - ), - crashed=crashed, - crash_reason=crash_reason, - duration_seconds=time.monotonic() - self.started_at, - ) - ) + self.error = str(error or "unknown error") @AgentRegistry.register(AgentKind.OPENCODE, OpenCodeAgentConfig) -class OpenCodeAgent(Agent[OpenCodeAgentConfig]): +class OpenCodeAgent(SubprocessJsonlAgent[OpenCodeAgentConfig]): """Runs the ``opencode`` CLI as a subprocess, one invocation per turn.""" # `should_stop` is polled at every event boundary (tool-call granularity); @@ -726,10 +514,14 @@ class OpenCodeAgent(Agent[OpenCodeAgentConfig]): disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, usage_granularity=UsageGranularity.STEP, - timing_basis=TimingBasis.MIXED, + timing_basis=TimingBasis.CLI_EPOCH_MS, permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), ) tool_names = _TOOL_NAMES + cli_name = "OpenCode" + docs_page = "docs/agents/OPENCODE.md" + recognized_events = _RECOGNIZED_EVENTS + decoder = _OpenCodeDecoder def __init__( self, @@ -748,9 +540,7 @@ def __init__( Rationale: .claude/notes/agents.md § Why the constructors declare every kwarg """ - super().__init__(config, route, cost_log_tags=cost_log_tags) - self.task_id = task_id - self.working_directory: str | None = None + super().__init__(config, route, task_id=task_id, cost_log_tags=cost_log_tags) self._env_path_prepend: list[str] = [] self._plugin_tools_dir: str | None = None self._skill_dirs: list[str] = [] @@ -758,12 +548,6 @@ def __init__( # removed in stop(). Never inside the sandbox. self._prompt_dir: str | None = None self._session_id: str | None = None - self._process: asyncio.subprocess.Process | None = None - # Process-group ids of every invocation this agent spawned, swept on - # kill()/kill_sync()/stop(): signalling only the CLI pid orphans the - # server child `opencode run` leaves behind. - self._spawned_pgids: list[int] = [] - self._state = AgentState.WORKING # --- lifecycle --------------------------------------------------------- @@ -804,41 +588,6 @@ def _remove_prompt_dir(self) -> None: shutil.rmtree(self._prompt_dir, ignore_errors=True) self._prompt_dir = None - async def kill(self) -> None: - proc = self._process - if proc is not None and proc.returncode is None: - with contextlib.suppress(ProcessLookupError): - proc.terminate() - with contextlib.suppress(TimeoutError, asyncio.TimeoutError): - await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_SECONDS) - if proc.returncode is None: - with contextlib.suppress(ProcessLookupError): - proc.kill() - self._sweep_process_groups() - - def kill_sync(self) -> None: - """SIGKILL the in-flight CLI and its process group (watchdog thread; must not await).""" - proc = self._process - if proc is not None and proc.returncode is None: - with contextlib.suppress(ProcessLookupError, PermissionError): - os.kill(proc.pid, _SIGKILL) - self._sweep_process_groups() - - def _sweep_process_groups(self) -> None: - """SIGKILL every process group this agent spawned (POSIX only). - - Each invocation runs in its own session, so its pgid is the CLI's pid and - the group holds ONLY what that invocation spawned. The CLI itself gets - SIGTERM-then-SIGKILL first (see ``kill``); this reaps what survives. - Sessions persist on disk, so this does not lose ``--session`` continuity. - """ - if os.name != "posix": - return - for pgid in self._spawned_pgids: - with contextlib.suppress(ProcessLookupError, PermissionError, OSError): - os.killpg(pgid, _SIGKILL) - self._spawned_pgids.clear() - def get_environment_info(self) -> dict[str, Any]: # Base first so the `system_prompt_semantics` run marker is always # present (an absent marker reads as a pre-marker run). @@ -855,7 +604,7 @@ def get_environment_info(self) -> dict[str, Any]: # --- command construction --------------------------------------------- - def _build_argv(self, user_input: str) -> list[str]: + def argv(self, prompt: str) -> list[str]: argv = ["opencode", "run", "--format", "json"] if self.config.model: argv += ["-m", self.config.model] @@ -871,10 +620,10 @@ def _build_argv(self, user_input: str) -> list[str]: if self._session_id: argv += ["--session", self._session_id] argv.append("--") - argv.append(user_input) + argv.append(prompt) return argv - def _build_env(self) -> dict[str, str]: + def env(self) -> dict[str, str]: """The CLI's full environment: the host's, plus the sandbox's contributions. The PATH prepend is the mock-shadowing contract (``Agent.start``): the @@ -968,342 +717,31 @@ def _inject_config_content(self, env: dict[str, str]) -> None: # --- the turn ---------------------------------------------------------- - async def communicate( - self, - user_input: str, - *, - iteration: int, - stream_callback: StreamCallback | None = None, - timeout: float | None = None, - should_stop: Callable[[], StopReason | None] | None = None, - ) -> TurnOutcome: - """Run one turn; see ``Agent.communicate``.""" - return await self._legacy_outcome( - self._communicate_legacy, - user_input, - iteration=iteration, - stream_callback=stream_callback, - timeout=timeout, - should_stop=should_stop, - ) - - async def _communicate_legacy( - self, - user_input: str, - *, - stream_callback: StreamCallback | None = None, - timeout: float | None = None, - should_stop: Callable[[], StopReason | None] | None = None, - ) -> TurnRecord: - if self.working_directory is None: - raise RuntimeError("OpenCodeAgent.start() must be called before communicate()") - - self._begin_turn() - collector = EventCollector() - - def emit(event: StreamEvent) -> None: - collector.on_event(event) - if stream_callback is not None: - safe_emit(stream_callback, event) - - state = _OpenCodeTurnState( - task_id=self.task_id, - iteration=self._iteration, - user_input=user_input, - model=self.config.model, - ) - state.bind(emit) - - emit( - AgentStartEvent( - task_id=self.task_id, - prompt=user_input, - iteration=self._iteration, - model=self.config.model, - ) - ) - - deadline = None if timeout is None else time.monotonic() + timeout - requested_stop: StopReason | None = None - stderr_drain: asyncio.Future[bytes] | None = None - # Bound OUTSIDE the try so `finally` can tell "never spawned" from - # "spawned and possibly still running". - proc: asyncio.subprocess.Process | None = None - try: - proc = await asyncio.create_subprocess_exec( - *self._build_argv(user_input), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - cwd=self.working_directory, - env=self._build_env(), - # One nd-JSON event can carry a whole tool result, past - # StreamReader's default 64 KiB cap. - limit=STDOUT_LINE_LIMIT_BYTES, - # Own session/process group, so teardown can killpg the server - # child. POSIX-only knob; harmless False elsewhere. - start_new_session=os.name == "posix", - ) - self._process = proc - if os.name == "posix": - self._spawned_pgids.append(proc.pid) - assert proc.stdout is not None - - # Drain stderr CONCURRENTLY, or a child that fills the pipe blocks on - # write and hangs the turn to its deadline. - # Rationale: .claude/notes/agents.md § Reaping the CLI harnesses - if proc.stderr is not None: - stderr_drain = asyncio.ensure_future(proc.stderr.read()) - - # The server child INHERITS this stdout pipe, so it is not closed when - # the CLI exits: race each read against process exit, then drain. - exit_waiter = asyncio.ensure_future(proc.wait()) - read_task: asyncio.Future[bytes] | None = None - try: - while True: - remaining = None if deadline is None else deadline - time.monotonic() - if remaining is not None and remaining <= 0: - await self._timeout_turn(state, collector, timeout or 0.0) - - if read_task is None: - read_task = asyncio.ensure_future(proc.stdout.readline()) - done, _pending = await asyncio.wait( - {read_task, exit_waiter}, - timeout=remaining, - return_when=asyncio.FIRST_COMPLETED, - ) - if not done: - await self._timeout_turn(state, collector, timeout or 0.0) - if not read_task.done(): - # Exited with the read still pending: bound the tail rather - # than wait on the grandchild's open write end. - try: - await asyncio.wait_for(asyncio.shield(read_task), _DRAIN_SECONDS) - except TimeoutError: - break - line = read_task.result() - read_task = None - if not line: - break - - self._handle_line(line, state) - - requested_stop = should_stop() if should_stop is not None else None - if requested_stop is not None: - await self.kill() - break - finally: - if read_task is not None: - read_task.cancel() - exit_waiter.cancel() - - status = await self._settle_turn( - proc, - state, - collector, - stderr_drain, - requested_stop=requested_stop, - deadline=deadline, - timeout=timeout, - ) - state.finalize(status) - # Build BEFORE marking the turn clean: a failure in the reduction is a - # failed turn, and `_end_turn_ok` clears the rollback flag. - record = collector.build_turn_record() - self._end_turn_ok() - return record - - except (AgentCrashError, TurnTimeoutError): - # Already funneled through finalize by _crash_turn / _timeout_turn. - raise - except asyncio.CancelledError: - self._finalize_external_cancel(state.finalize) - self._capture_partial_turn(collector) - raise - except Exception as e: - # Everything the loop does NOT anticipate. Without this the exception - # escapes raw and breaks the pending-turn contract three ways: no - # AgentEndEvent, the telemetry dropped rather than parked, and - # `_iteration` left incremented. - self._crash_turn(state, collector, f"OpenCode turn failed: {e!s}", cause=e) - raise # unreachable (_crash_turn is NoReturn) — makes the no-fall-through explicit - finally: - if stderr_drain is not None: - stderr_drain.cancel() - self._reap_orphaned_cli(proc) - self._process = None - - def _reap_orphaned_cli(self, proc: asyncio.subprocess.Process | None) -> None: - """Kill a CLI that is still running as the turn unwinds. No-op otherwise. - - Deliberately synchronous: this runs while a ``CancelledError`` is - propagating, where any await can itself be cut short. Skipping the SIGTERM - courtesy is right for a turn that is already lost — :meth:`kill` still - owns every path with something left to flush. ``proc`` is ``None`` when - the spawn itself failed. - - Rationale: .claude/notes/agents.md § Reaping the CLI harnesses - """ - if proc is None or proc.returncode is not None: - return - with contextlib.suppress(ProcessLookupError, PermissionError): - proc.kill() - self._sweep_process_groups() - - async def _settle_turn( - self, - proc: asyncio.subprocess.Process, - state: _OpenCodeTurnState, - collector: EventCollector, - stderr_drain: asyncio.Future[bytes] | None, - *, - requested_stop: StopReason | None, - deadline: float | None, - timeout: float | None, - ) -> AgentEndStatus: - """Reap the CLI once the read loop is done and decide the turn's end status. - - Raises ``AgentCrashError`` (via :meth:`_crash_turn`) on a structured error, - on a death with neither a structured error nor an intentional stop, or on - a clean exit that captured no token telemetry. Raises ``TurnTimeoutError`` - when the deadline elapses while waiting for the exit. - """ - # Bound the reap: the read loop can end at EOF with the CLI still alive, - # and an unbounded wait here would outlive the turn deadline. - remaining = None if deadline is None else max(0.0, deadline - time.monotonic()) - try: - await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_SECONDS if remaining is None else remaining) - except TimeoutError: - if remaining is not None: - await self._timeout_turn(state, collector, timeout or 0.0) - await self.kill() - self._crash_turn( - state, - collector, - f"OpenCode closed its event stream but did not exit within {_TERM_GRACE_SECONDS:.0f}s", - ) - # Bounded for the same reason as the read loop: the inherited stderr pipe - # outlives the CLI. Shielded so the timeout doesn't kill it early. - stderr_bytes = b"" - if stderr_drain is not None: - with contextlib.suppress(TimeoutError): - stderr_bytes = await asyncio.wait_for(asyncio.shield(stderr_drain), timeout=_DRAIN_SECONDS) - - if state.error_message is not None: - self._crash_turn(state, collector, f"OpenCode error: {state.error_message}") - - # A non-zero exit with no structured error still means the turn died: - # surface stderr rather than reporting a silent empty success. - if proc.returncode not in (0, None) and requested_stop is None: - detail = stderr_bytes.decode("utf-8", "replace").strip() or f"exit code {proc.returncode}" - self._crash_turn(state, collector, f"OpenCode exited non-zero: {detail}") - - # A clean exit that captured NO token telemetry must not score. Keying on - # the token counts ALONE is what misses the second arm: an exit that - # recognized no events at all reaches the same silent-empty-success - # outcome. Intentional cuts are exempt — either can land before the first - # event, or mid-step. (The two arms are NOT interchangeable downstream; - # see the require_token_telemetry escape hatch below.) - # Rationale: .claude/notes/agents.md § Why a clean exit can still be a crash - nothing_recognized = state.recognized_events == 0 - finished_without_tokens = state.steps_finished > 0 and state.usage.is_empty() - if requested_stop is None and (nothing_recognized or finished_without_tokens): - if nothing_recognized: - seen = ", ".join(sorted(state.unrecognized_types)) or "none (stdout carried no JSON events)" - detail = f"It emitted no recognized events at all. Unrecognized event types seen: {seen}." - else: - detail = ( - f"It reported {state.steps_finished} finished step(s), none of which carried usable " - + f"token counts (cost reported: {'yes' if state.saw_cost else 'no'})." - ) - message = ( - f"OpenCode exited cleanly but the turn captured zero token telemetry. {detail} The CLI's " - + "event or token schema may have changed — see docs/agents/OPENCODE.md (Telemetry) before " - + "trusting any run from this CLI version." - ) - # Escape hatch for a provider/auth mode that reports no usage at all. - # Deliberately does NOT cover `nothing_recognized`: that arm is - # vocabulary drift, which no provider quirk explains. - if not self.config.require_token_telemetry and not nothing_recognized: - logger.warning("opencode: %s Scored anyway — require_token_telemetry is off.", message) - else: - self._crash_turn(state, collector, message) - - return end_status_for(requested_stop) if requested_stop is not None else AgentEndStatus.COMPLETED + def observe(self, event: dict[str, Any]) -> None: + """Remember the CLI's session id: it rides on the envelope and is replayed via ``--session``.""" + part = event.get("part") + session_id = event.get("sessionID") or (part.get("sessionID") if isinstance(part, dict) else None) + if isinstance(session_id, str) and session_id: + self._session_id = session_id - def _crash_turn( - self, - state: _OpenCodeTurnState, - collector: EventCollector, - message: str, - *, - cause: BaseException | None = None, - ) -> NoReturn: - """Park the crashed partial record and raise ``AgentCrashError``. + def clean_exit_problem(self, decoder: JsonlDecoder) -> str | None: + """A clean exit whose finished steps carried no token counts must not score. - ``cause`` preserves the ``__cause__`` link from an ``except ... as e``. - """ - state.close_open_tools() - try: - self._finalize_and_raise_crash(state.finalize, message, cause=cause) - finally: - self._capture_partial_turn(collector) + ``require_token_telemetry: false`` is the escape hatch for a provider or auth + mode that reports no usage at all: the turn is scored with a warning. - async def _timeout_turn( - self, - state: _OpenCodeTurnState, - collector: EventCollector, - timeout: float, - ) -> NoReturn: - """Kill the CLI, park the crashed partial record, raise ``TurnTimeoutError``. - - The partial record is captured immediately after, so ``pending_turn`` - carries everything observed before the deadline. + Rationale: .claude/notes/agents.md § Why a clean exit can still be a crash """ - await self.kill() - state.close_open_tools() - try: - self._finalize_and_raise_timeout(state.finalize, timeout) - finally: - self._capture_partial_turn(collector) - - def _handle_line(self, line: bytes, state: _OpenCodeTurnState) -> None: - """Parse one nd-JSON line and dispatch it. Never raises on bad input.""" - raw = line.decode("utf-8", "replace").strip() - if not raw: - return - try: - obj = json.loads(raw) - except json.JSONDecodeError: - # OpenCode interleaves non-JSON notices (the Bun AVX warning) on - # stdout; a malformed line must not kill the turn. - logger.debug("opencode: skipping non-JSON stdout line: %s", raw[:200]) - return - if not isinstance(obj, dict): - return - - event_type, part = _unwrap(obj) - if event_type in _RECOGNIZED_EVENTS: - state.recognized_events += 1 - elif len(state.unrecognized_types) < _MAX_UNRECOGNIZED_TYPES: - state.unrecognized_types.add(event_type or "") - - # sessionID rides on the envelope, not the part. - session_id = obj.get("sessionID") or part.get("sessionID") - if isinstance(session_id, str) and session_id: - if state.session_id is None: - state.session_id = session_id - state.thread_id = session_id - self._session_id = session_id - - if event_type == _STEP_START: - state.on_step_start(part) - elif event_type == _TEXT: - state.on_text(part) - elif event_type == _TOOL_USE: - state.on_tool_use(part) - elif event_type == _STEP_FINISH: - state.on_step_finish(part) - elif event_type == _ERROR: - state.on_error(part) - else: - logger.debug("opencode: unhandled event type %r", event_type) + assert isinstance(decoder, _OpenCodeDecoder) + if decoder.steps_finished == 0 or not decoder.usage.is_empty(): + return None + message = ( + "OpenCode exited cleanly but the turn captured zero token telemetry. It reported " + + f"{decoder.steps_finished} finished step(s), none of which carried usable token counts " + + f"(cost reported: {'yes' if decoder.saw_cost else 'no'}). The CLI's event or token schema may have " + + "changed — see docs/agents/OPENCODE.md (Telemetry) before trusting any run from this CLI version." + ) + if not self.config.require_token_telemetry: + logger.warning("opencode: %s Scored anyway — require_token_telemetry is off.", message) + return None + return message diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index fd044037..86f9296b 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -20,26 +20,17 @@ from __future__ import annotations -import asyncio -import contextlib -import json import logging import os import re import shutil -import signal import tempfile -import time -from collections.abc import Callable -from dataclasses import dataclass, field from datetime import datetime from pathlib import Path from typing import Any from uuid import uuid4 -from coder_eval.agent import Agent -from coder_eval.errors.agent import format_timeout_reason -from coder_eval.isolation.docker_runner import STDOUT_LINE_LIMIT_BYTES +from coder_eval.agents._transport import JsonlDecoder, SubprocessJsonlAgent from coder_eval.models import ( READ_ONLY_DENIED_TOOLS, AgentKind, @@ -56,9 +47,8 @@ UsageGranularity, ) from coder_eval.pricing import price_turn -from coder_eval.streaming.callbacks import StreamCallback from coder_eval.streaming.emitter import Generation, TurnEmitter, TurnOutcome -from coder_eval.streaming.events import AgentEndStatus, StopReason, ToolEndStatus, TurnEndStatus, end_status_for +from coder_eval.streaming.events import AgentEndStatus, ToolEndStatus, TurnEndStatus from coder_eval.timing import close_window from .registry import AgentRegistry @@ -66,26 +56,6 @@ logger = logging.getLogger(__name__) -# Grace period between SIGTERM and SIGKILL when tearing down the CLI subprocess. -# Doubles as the post-EOF exit grace in _settle_turn when no turn deadline is set. -# Re-declared at OpenCode's value rather than shared — see the notes. -# Rationale: .claude/notes/agents.md § Reaping the CLI harnesses -_TERM_GRACE_SECONDS = 5.0 - -# SIGKILL does not exist on Windows (where the process-group sweep is a no-op -# anyway); resolve it dynamically so the module imports and typechecks on every -# platform, falling back to SIGTERM for the direct-pid kill_sync path. -_SIGKILL: signal.Signals = getattr(signal, "SIGKILL", signal.SIGTERM) - -# How long to keep draining stdout/stderr after the CLI has been reaped: a -# print-mode CLI may leave an inherited pipe open, so every post-exit read is -# bounded. -_DRAIN_SECONDS = 2.0 - -# How many distinct unrecognized event-type strings to retain for the crash -# message when the vocabulary check fails (diagnosis, not an exhaustive list). -_MAX_UNRECOGNIZED_TYPES = 8 - # pi's native tool names -> the canonical (Claude) vocabulary every criterion is # written against. Unknown tools pass through unchanged. # Rationale: .claude/notes/agents.md § Tool-name and argument normalization @@ -176,7 +146,7 @@ def _result_text(result: Any) -> str | None: return str(result) -class _PiDecoder: +class _PiDecoder(JsonlDecoder): """One turn's reducer: Pi's nd-JSON events in, ``TurnEmitter`` calls out. Holds only what the emitter cannot know: where the next generation window @@ -185,12 +155,11 @@ class _PiDecoder: """ def __init__(self, emitter: TurnEmitter) -> None: - self.emitter = emitter + super().__init__(emitter) self.usage = TokenUsage() self.cost_usd: float = 0.0 self.saw_cost = False self.stop_reason: str | None = None - self.error: str | None = None self.turn_count = 0 self.tool_count = 0 self.open_tool_ids: set[str] = set() @@ -403,7 +372,7 @@ def on_turn_end(self, event: dict[str, Any]) -> None: @AgentRegistry.register(AgentKind.PI, PiAgentConfig) -class PiAgent(Agent[PiAgentConfig]): +class PiAgent(SubprocessJsonlAgent[PiAgentConfig]): """Runs the ``pi`` CLI as a subprocess, one invocation per turn.""" # `should_stop` is polled at every event boundary (tool-call granularity); @@ -422,6 +391,10 @@ class PiAgent(Agent[PiAgentConfig]): permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), ) tool_names = _TOOL_NAMES + cli_name = "Pi" + docs_page = "docs/agents/PI.md" + recognized_events = _RECOGNIZED_EVENTS + decoder = _PiDecoder def __init__( self, @@ -439,9 +412,7 @@ def __init__( Rationale: .claude/notes/agents.md § Why the constructors declare every kwarg """ - super().__init__(config, route, cost_log_tags=cost_log_tags) - self.task_id = task_id - self.working_directory: str | None = None + super().__init__(config, route, task_id=task_id, cost_log_tags=cost_log_tags) self._env_path_prepend: list[str] = [] self._plugin_tools_dir: str | None = None # The staged root's skills dir, passed to `pi --skill`. Assigned in start(). @@ -451,11 +422,6 @@ def __init__( # Rationale: .claude/notes/agents.md § Reaping the CLI harnesses self._session_id: str | None = None self._session_dir: str | None = None - self._process: asyncio.subprocess.Process | None = None - # Process-group ids of every invocation this agent spawned, swept on - # kill()/kill_sync()/stop(). - self._spawned_pgids: list[int] = [] - self._state = AgentState.WORKING # --- lifecycle --------------------------------------------------------- @@ -499,39 +465,6 @@ def _cleanup_session_dir(self) -> None: shutil.rmtree(self._session_dir, ignore_errors=True) self._session_dir = None - async def kill(self) -> None: - proc = self._process - if proc is not None and proc.returncode is None: - with contextlib.suppress(ProcessLookupError): - proc.terminate() - with contextlib.suppress(TimeoutError, asyncio.TimeoutError): - await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_SECONDS) - if proc.returncode is None: - with contextlib.suppress(ProcessLookupError): - proc.kill() - self._sweep_process_groups() - - def kill_sync(self) -> None: - """SIGKILL the in-flight CLI and its process group (watchdog thread; must not await).""" - proc = self._process - if proc is not None and proc.returncode is None: - with contextlib.suppress(ProcessLookupError, PermissionError): - os.kill(proc.pid, _SIGKILL) - self._sweep_process_groups() - - def _sweep_process_groups(self) -> None: - """SIGKILL every process group this agent spawned (POSIX only). - - Each invocation runs in its own session, so its pgid is the CLI's pid and - the group holds ONLY what that invocation spawned. - """ - if os.name != "posix": - return - for pgid in self._spawned_pgids: - with contextlib.suppress(ProcessLookupError, PermissionError, OSError): - os.killpg(pgid, _SIGKILL) - self._spawned_pgids.clear() - def get_environment_info(self) -> dict[str, Any]: # Spread the base first so the `system_prompt_semantics` run marker is # always present (CE046). @@ -546,7 +479,7 @@ def get_environment_info(self) -> dict[str, Any]: # --- command construction --------------------------------------------- - def _build_argv(self, user_input: str) -> list[str]: + def argv(self, prompt: str) -> list[str]: # -p exits after the run; --no-context-files + --no-approve isolate the # sandbox from host AGENTS.md/CLAUDE.md and project-local trust. # --session-dir + --session-id give cross-communicate() continuity — NOT @@ -575,8 +508,8 @@ def _build_argv(self, user_input: str) -> list[str]: argv += self._tool_flags() if self.config.system_prompt: argv += ["--append-system-prompt", self.config.system_prompt] - # user_input is a distinct argv element after `--` (never shell-interpolated). - argv += ["--", user_input] + # The prompt is a distinct argv element after `--` (never shell-interpolated). + argv += ["--", prompt] return argv def _tool_flags(self) -> list[str]: @@ -594,7 +527,7 @@ def _tool_flags(self) -> list[str]: return ["--tools", ",".join(sorted(allow))] if allow else ["--no-tools"] return ["--exclude-tools", ",".join(sorted(deny))] if deny else [] - def _build_env(self) -> dict[str, str]: + def env(self) -> dict[str, str]: """The CLI's full environment: the host's, plus the sandbox's contributions. The PATH prepend is the mock-shadowing contract (``Agent.start``): the @@ -609,241 +542,3 @@ def _build_env(self) -> dict[str, str]: if self._plugin_tools_dir and "PLUGIN_TOOLS_DIR" not in env: env["PLUGIN_TOOLS_DIR"] = self._plugin_tools_dir return env - - # --- the turn ---------------------------------------------------------- - - async def communicate( - self, - user_input: str, - *, - iteration: int, - stream_callback: StreamCallback | None = None, - timeout: float | None = None, - should_stop: Callable[[], StopReason | None] | None = None, - ) -> TurnOutcome: - """Run one ``pi -p`` invocation as one turn; see ``Agent.communicate``.""" - if self.working_directory is None: - raise RuntimeError("PiAgent.start() must be called before communicate()") - - emitter = self._open_emitter( - prompt=user_input, - iteration=iteration, - model=self.config.model, - task_id=self.task_id, - stream_callback=stream_callback, - ) - emitter.begin() - decoder = _PiDecoder(emitter) - vocabulary = _Vocabulary() - - # Deadlines stay on `time.monotonic()`, deliberately NOT the turn clock: - # a deadline must not move when the wall clock steps. - deadline = None if timeout is None else time.monotonic() + timeout - requested_stop: StopReason | None = None - stderr_drain: asyncio.Future[bytes] | None = None - # Bound OUTSIDE the try so `finally` can tell "never spawned" from - # "spawned and possibly still running". - proc: asyncio.subprocess.Process | None = None - try: - proc = await asyncio.create_subprocess_exec( - *self._build_argv(user_input), - # Pi reads a non-TTY stdin to EOF before it emits anything; an - # inherited, still-open stdin stalls the turn to its deadline. - # Rationale: .claude/notes/agents.md § Why a CLI never inherits stdin - stdin=asyncio.subprocess.DEVNULL, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - cwd=self.working_directory, - env=self._build_env(), - # One nd-JSON event can carry a whole tool result, past - # StreamReader's default 64 KiB cap. - limit=STDOUT_LINE_LIMIT_BYTES, - # Own session/process group, so teardown can killpg a lingering - # child without touching anything this invocation didn't spawn. - start_new_session=os.name == "posix", - ) - self._process = proc - if os.name == "posix": - self._spawned_pgids.append(proc.pid) - assert proc.stdout is not None - - # Drain stderr CONCURRENTLY, or a child that fills the pipe blocks on - # write and hangs the turn to its deadline. - # Rationale: .claude/notes/agents.md § Reaping the CLI harnesses - if proc.stderr is not None: - stderr_drain = asyncio.ensure_future(proc.stderr.read()) - - # An inherited pipe may never reach EOF, so race each read against - # process exit; a bounded drain then collects the tail. - exit_waiter = asyncio.ensure_future(proc.wait()) - read_task: asyncio.Future[bytes] | None = None - try: - while True: - remaining = None if deadline is None else deadline - time.monotonic() - if remaining is not None and remaining <= 0: - return await self._time_out(decoder, timeout or 0.0) - - if read_task is None: - read_task = asyncio.ensure_future(proc.stdout.readline()) - done, _pending = await asyncio.wait( - {read_task, exit_waiter}, - timeout=remaining, - return_when=asyncio.FIRST_COMPLETED, - ) - if not done: - return await self._time_out(decoder, timeout or 0.0) - if not read_task.done(): - try: - await asyncio.wait_for(asyncio.shield(read_task), _DRAIN_SECONDS) - except TimeoutError: - break - line = read_task.result() - read_task = None - if not line: - break - - self._handle_line(line, decoder, vocabulary) - - requested_stop = should_stop() if should_stop is not None else None - if requested_stop is not None: - await self.kill() - break - finally: - if read_task is not None: - read_task.cancel() - exit_waiter.cancel() - - return await self._settle_turn( - proc, - decoder, - vocabulary, - stderr_drain, - requested_stop=requested_stop, - deadline=deadline, - timeout=timeout, - ) - - except asyncio.CancelledError: - self._state = AgentState.ERROR - decoder.end(AgentEndStatus.CRASHED, reason="turn cancelled") - raise - except Exception as e: - # A spawn failure, a StreamReader ValueError past `limit`, a malformed payload. - logger.warning("pi: turn failed", exc_info=True) - return self._crash(decoder, f"Pi turn failed: {e!s}") - finally: - if stderr_drain is not None: - stderr_drain.cancel() - self._reap_orphaned_cli(proc) - self._process = None - - def _reap_orphaned_cli(self, proc: asyncio.subprocess.Process | None) -> None: - """Kill a CLI still running as the turn unwinds. No-op otherwise. - - Synchronous (no await) so it survives a ``CancelledError`` in flight. - ``proc`` is ``None`` when the spawn failed. - - Rationale: .claude/notes/agents.md § Reaping the CLI harnesses - """ - if proc is None or proc.returncode is not None: - return - with contextlib.suppress(ProcessLookupError, PermissionError): - proc.kill() - self._sweep_process_groups() - - async def _settle_turn( - self, - proc: asyncio.subprocess.Process, - decoder: _PiDecoder, - vocabulary: _Vocabulary, - stderr_drain: asyncio.Future[bytes] | None, - *, - requested_stop: StopReason | None, - deadline: float | None, - timeout: float | None, - ) -> TurnOutcome: - """Reap the CLI once the read loop is done and end the turn. - - CRASHED when the process died with neither an intentional stop nor a - recognized event stream, TIMEOUT when the deadline elapses while waiting - for the exit. - """ - remaining = None if deadline is None else max(0.0, deadline - time.monotonic()) - try: - await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_SECONDS if remaining is None else remaining) - except TimeoutError: - if remaining is not None: - return await self._time_out(decoder, timeout or 0.0) - await self.kill() - return self._crash( - decoder, f"Pi closed its event stream but did not exit within {_TERM_GRACE_SECONDS:.0f}s" - ) - stderr_bytes = b"" - if stderr_drain is not None: - with contextlib.suppress(TimeoutError): - stderr_bytes = await asyncio.wait_for(asyncio.shield(stderr_drain), timeout=_DRAIN_SECONDS) - - # A terminal provider error is infrastructure failure, not an agent - # failure, and `pi -p` exits 0 after exhausting retries. GATED on - # intentional cuts: a cut can fire before the clearing `turn_end` arrives. - # Rationale: .claude/notes/agents.md § Why a clean exit can still be a crash - if decoder.error is not None and requested_stop is None: - return self._crash(decoder, f"Pi error: {decoder.error}") - - if proc.returncode not in (0, None) and requested_stop is None: - detail = stderr_bytes.decode("utf-8", "replace").strip() or f"exit code {proc.returncode}" - return self._crash(decoder, f"Pi exited non-zero: {detail}") - - # A clean exit that recognized NO events is vocabulary drift. Intentional - # cuts are exempt: a stop can land before the first event. - if requested_stop is None and vocabulary.recognized == 0: - seen = ", ".join(sorted(vocabulary.unrecognized)) or "none (stdout carried no JSON events)" - return self._crash( - decoder, - "Pi exited cleanly but the turn captured no recognized events. Unrecognized event types seen: " - + f"{seen}. The CLI's event schema may have changed — see docs/agents/PI.md before trusting any " - + "run from this CLI version.", - ) - - return decoder.end(end_status_for(requested_stop) if requested_stop is not None else AgentEndStatus.COMPLETED) - - def _crash(self, decoder: _PiDecoder, message: str) -> TurnOutcome: - self._state = AgentState.ERROR - return decoder.end(AgentEndStatus.CRASHED, reason=message) - - async def _time_out(self, decoder: _PiDecoder, timeout: float) -> TurnOutcome: - """Kill the CLI and end the turn as a timeout.""" - await self.kill() - self._state = AgentState.ERROR - return decoder.end(AgentEndStatus.TIMEOUT, reason=format_timeout_reason(timeout)) - - def _handle_line(self, line: bytes, decoder: _PiDecoder, vocabulary: _Vocabulary) -> None: - """Parse one nd-JSON line and hand it to the decoder. Never raises on bad input. - - ``agent_end`` is NOT terminal — only ``agent_settled`` / stdout EOF is — so - it is recognized, ignored, and the read loop keeps going. - """ - raw = line.decode("utf-8", "replace").strip() - if not raw: - return - try: - obj = json.loads(raw) - except json.JSONDecodeError: - logger.debug("pi: skipping non-JSON stdout line: %s", raw[:200]) - return - if not isinstance(obj, dict): - return - event_type = str(obj.get("type") or "") - if event_type in _RECOGNIZED_EVENTS: - vocabulary.recognized += 1 - elif len(vocabulary.unrecognized) < _MAX_UNRECOGNIZED_TYPES: - vocabulary.unrecognized.add(event_type or "") - decoder(obj) - - -@dataclass -class _Vocabulary: - """The drift check's evidence: how many events matched Pi's vocabulary, and a sample of what did not.""" - - recognized: int = 0 - unrecognized: set[str] = field(default_factory=set) diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index 9b0b3764..2a4429ec 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -660,6 +660,7 @@ async def run(self) -> EvaluationResult: heartbeat_task = asyncio.create_task(_heartbeat_loop(heartbeat_path)) proc = await asyncio.create_subprocess_exec( *argv, + stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, limit=STDOUT_LINE_LIMIT_BYTES, diff --git a/src/coder_eval/models/harness_contract.py b/src/coder_eval/models/harness_contract.py index ff3f83cf..608b65db 100644 --- a/src/coder_eval/models/harness_contract.py +++ b/src/coder_eval/models/harness_contract.py @@ -32,7 +32,6 @@ class TimingBasis(StrEnum): TURN_CLOCK = "turn_clock" CLI_EPOCH_MS = "cli_epoch_ms" - MIXED = "mixed" class HarnessContract(BaseModel): @@ -68,7 +67,7 @@ class HarnessContract(BaseModel): description=( "Where recorded stamps come from: turn_clock (the TurnEmitter stamps the turn bracket, every tool " "and every window from one TurnClock) or cli_epoch_ms (the adapter passes the CLI's own stamps for " - "windows and main-thread tools). mixed is OpenCode's interim value." + "windows and main-thread tools)." ) ) permission_modes: frozenset[PermissionMode] | None = Field( diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index d683dad8..80b50557 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -2738,6 +2738,9 @@ async def _run_command_list( proc = await asyncio.create_subprocess_shell( cmd.command, cwd=str(sandbox_dir), + # An authored command that reads stdin must not stall the task. + # Rationale: .claude/notes/agents.md § Why a CLI never inherits stdin + stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, limit=self._POST_RUN_STREAM_LIMIT, diff --git a/src/coder_eval/spi.py b/src/coder_eval/spi.py index c18a57e9..2f1fc91f 100644 --- a/src/coder_eval/spi.py +++ b/src/coder_eval/spi.py @@ -8,6 +8,7 @@ from typing import Final from coder_eval.agent import Agent +from coder_eval.agents._transport import JsonlDecoder, SubprocessJsonlAgent from coder_eval.agents.registry import AgentRegistry from coder_eval.agents.watchdog import WatchdogFired, run_with_watchdog from coder_eval.errors import AgentConfigError, AgentCrashError, TurnTimeoutError @@ -72,6 +73,7 @@ "EventCollector", "Generation", "HarnessContract", + "JsonlDecoder", "LocalPluginConfig", "ModelPricing", "PermissionMode", @@ -80,6 +82,7 @@ "SPI_VERSION", "StopReason", "StreamCallback", + "SubprocessJsonlAgent", "SystemPromptMode", "TextChunkEvent", "TimingBasis", diff --git a/src/coder_eval/streaming/emitter.py b/src/coder_eval/streaming/emitter.py index 0524de1d..d77404f2 100644 --- a/src/coder_eval/streaming/emitter.py +++ b/src/coder_eval/streaming/emitter.py @@ -254,19 +254,27 @@ def close_tool( result_data: dict[str, Any] | list[Any] | None = None, parameters: dict[str, Any] | None = None, completed_at: datetime | None = _UNSET, + started_at: datetime | None = None, ) -> None: """Record a tool call's end; an unknown id synthesizes a ``tool_name="unknown"`` call. Only a resolved call is timed: ``UNRESOLVED`` keeps ``execution_started_at`` - and sets no completion stamp and no duration. + and sets no completion stamp and no duration. ``started_at`` is a CLI start + stamp that arrived only with the result (``CLI_EPOCH_MS``); it fills a call + opened without one and never replaces an existing start. Raises: - TypeError: the same basis rule as ``open_tool``, for ``completed_at``. + TypeError: the same basis rule as ``open_tool``, for ``completed_at``; or + ``started_at`` given under ``TURN_CLOCK``. """ if self._ended(): return + if started_at is not None and self._basis is TimingBasis.TURN_CLOCK: + raise TypeError("started_at= is not accepted under TimingBasis.TURN_CLOCK: the emitter stamps the clock") opened = self._open_tools.get(tool_id) stamp = self._stamp("completed_at", completed_at, self.now(), opened.parent_tool_id if opened else None) + if opened is not None and started_at is not None and opened.telemetry.execution_started_at is None: + opened.telemetry.execution_started_at = started_at self._close(tool_id, status, summary, error, result_data, parameters, stamp) def add_generation( diff --git a/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json index 594b2e72..2b1d35fa 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json @@ -41,7 +41,7 @@ "num_turns": 1, "result_summary": { "is_error": false, - "result": null, + "result": "All done.", "stop_reason": "stop", "subtype": "completed" }, diff --git a/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json index d5aa832d..aa5f6b5a 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json @@ -16,7 +16,7 @@ "result_status": "success", "result_summary": "print('hi')", "result_tokens": 3, - "sequence_number": 1, + "sequence_number": 0, "timestamp": "", "tool_id": "call_1", "tool_name": "Read" @@ -90,7 +90,7 @@ "num_turns": 2, "result_summary": { "is_error": false, - "result": null, + "result": "Created the file.", "stop_reason": "stop", "subtype": "completed" }, diff --git a/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json b/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json index 4b496314..d237c144 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json +++ b/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json @@ -16,7 +16,7 @@ "result_status": "success", "result_summary": "main.py", "result_tokens": 2, - "sequence_number": 1, + "sequence_number": 0, "timestamp": "", "tool_id": "call_1", "tool_name": "Bash" @@ -90,7 +90,7 @@ "num_turns": 2, "result_summary": { "is_error": false, - "result": null, + "result": "Listed it.", "stop_reason": "stop", "subtype": "completed" }, diff --git a/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json index 0b8d6284..1422c542 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json @@ -5,8 +5,8 @@ { "assistant_turn_index": 0, "duration_ms": null, - "error_message": "no result observed", - "execution_completed_at": "", + "error_message": null, + "execution_completed_at": null, "execution_started_at": null, "generation_completed_at": null, "parameters": { @@ -16,7 +16,7 @@ "result_status": "unknown", "result_summary": null, "result_tokens": 0, - "sequence_number": 1, + "sequence_number": 0, "timestamp": "", "tool_id": "call_1", "tool_name": "Bash" diff --git a/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json b/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json index 65a062a5..f89b5a28 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json +++ b/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json @@ -39,12 +39,7 @@ ], "model_used": "deepseek/deepseek-v4-pro", "num_turns": 1, - "result_summary": { - "is_error": true, - "result": "OpenCode error: 401 from the provider", - "stop_reason": "stop", - "subtype": "crashed" - }, + "result_summary": null, "timestamp": "", "token_usage": { "cache_creation_input_tokens": 0, diff --git a/tests/_fixtures/golden_streams/expected/opencode_f_captured_stream.json b/tests/_fixtures/golden_streams/expected/opencode_f_captured_stream.json new file mode 100644 index 00000000..63709697 --- /dev/null +++ b/tests/_fixtures/golden_streams/expected/opencode_f_captured_stream.json @@ -0,0 +1,148 @@ +{ + "agent_output": "I'll help you create the file and list the directory.DONE", + "assistant_turn_count": 2, + "commands": [ + { + "assistant_turn_index": 0, + "duration_ms": "", + "error_message": null, + "execution_completed_at": "", + "execution_started_at": "", + "generation_completed_at": null, + "parameters": { + "content": "hi", + "file_path": "/work/hello.txt" + }, + "result_data": null, + "result_status": "success", + "result_summary": "Wrote file successfully.", + "result_tokens": 6, + "sequence_number": 0, + "timestamp": "", + "tool_id": "toolu_016ZZZbGJz51bwQrFrxN4Sv2", + "tool_name": "Write" + }, + { + "assistant_turn_index": 0, + "duration_ms": "", + "error_message": null, + "execution_completed_at": "", + "execution_started_at": "", + "generation_completed_at": null, + "parameters": { + "command": "ls -la /work/" + }, + "result_data": null, + "result_status": "success", + "result_summary": "total 8\ndrwx------ 3 user staff 96 Sep 16 15:57 .\ndrwx------@ 39029 user staff 1248928 Sep 16 15:57 ..\n-rw-r--r-- 1 user staff 2 Sep 16 15:57 hello.txt\n", + "result_tokens": 45, + "sequence_number": 1, + "timestamp": "", + "tool_id": "toolu_01DYyaQT59GsS3Qyj9smkbrb", + "tool_name": "Bash" + } + ], + "crash_reason": null, + "crashed": false, + "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", + "iteration": 1, + "messages": [ + { + "cache_creation_tokens": 16336, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [ + { + "block_type": "text", + "is_error": false, + "sequence": 0, + "signature": null, + "text": "I'll help you create the file and list the directory.", + "thinking": null, + "tool_use_id": null + }, + { + "block_type": "tool_use", + "is_error": false, + "sequence": 1, + "signature": null, + "text": null, + "thinking": null, + "tool_use_id": "toolu_016ZZZbGJz51bwQrFrxN4Sv2" + }, + { + "block_type": "tool_use", + "is_error": false, + "sequence": 2, + "signature": null, + "text": null, + "thinking": null, + "tool_use_id": "toolu_01DYyaQT59GsS3Qyj9smkbrb" + } + ], + "generation_duration_ms": "", + "input_tokens": 3, + "message_id": "msg_0ac700f2b001SSGFoj9fVWpcYA", + "model": "deepseek/deepseek-v4-pro", + "output_tokens": 210, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "tool-calls", + "tool_use_ids": [ + "toolu_016ZZZbGJz51bwQrFrxN4Sv2", + "toolu_01DYyaQT59GsS3Qyj9smkbrb" + ] + }, + { + "cache_creation_tokens": 355, + "cache_read_tokens": 16336, + "completed_at": "", + "content_blocks": [ + { + "block_type": "text", + "is_error": false, + "sequence": 0, + "signature": null, + "text": "DONE", + "thinking": null, + "tool_use_id": null + } + ], + "generation_duration_ms": "", + "input_tokens": 7, + "message_id": "msg_0ac701b6a001f4VpkH4NL51mAe", + "model": "deepseek/deepseek-v4-pro", + "output_tokens": 5, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "stop", + "tool_use_ids": [] + } + ], + "model_used": "deepseek/deepseek-v4-pro", + "num_turns": 2, + "result_summary": { + "is_error": false, + "result": "DONE", + "stop_reason": "stop", + "subtype": "completed" + }, + "timestamp": "", + "token_usage": { + "cache_creation_input_tokens": 16691, + "cache_read_input_tokens": 16336, + "input_tokens": 33037, + "output_tokens": 215, + "total_cost_usd": "", + "uncached_input_tokens": 10 + }, + "tool_calls_exhausted": false, + "tool_union_ms": "", + "user_input": "do it" +} diff --git a/tests/_fixtures/golden_streams/opencode_fixtures.py b/tests/_fixtures/golden_streams/opencode_fixtures.py index 6e784da8..8a01710c 100644 --- a/tests/_fixtures/golden_streams/opencode_fixtures.py +++ b/tests/_fixtures/golden_streams/opencode_fixtures.py @@ -26,6 +26,7 @@ import os from dataclasses import dataclass from datetime import datetime +from pathlib import Path from typing import Any from unittest.mock import patch @@ -50,10 +51,20 @@ _REPLAY_LEAD_MS = 2 -def _evt(event_type: str, part: dict[str, Any]) -> str: - """One CLI event line: payload under ``part``, sessionID on the envelope.""" +def _evt(event_type: str, part: dict[str, Any], *, at_ms: int = 0) -> str: + """One CLI event line: payload under ``part``; sessionID and the CLI's own stamp on the envelope. + + ``at_ms`` places the event on the recorded timeline: the envelope stamp is what + bounds a generation window on this harness, so a scenario that should measure + one gives its events increasing stamps. + """ return json.dumps( - {"type": event_type, "timestamp": _T0_MS, "sessionID": SESSION, "part": {"sessionID": SESSION, **part}} + { + "type": event_type, + "timestamp": _T0_MS + at_ms, + "sessionID": SESSION, + "part": {"sessionID": SESSION, **part}, + } ) @@ -81,6 +92,27 @@ def shift(node: Any) -> Any: _STAMP_KEYS = frozenset({"timestamp", "start", "end"}) +def _starting_at_t0(lines: list[str]) -> list[str]: + """Shift a captured stream's stamps so its first envelope ``timestamp`` is ``_T0_MS``.""" + first = json.loads(lines[0])["timestamp"] + + def shift(node: Any) -> Any: + if isinstance(node, dict): + return { + k: (v - first + _T0_MS if k in _STAMP_KEYS and isinstance(v, int) else shift(v)) + for k, v in node.items() + } + if isinstance(node, list): + return [shift(v) for v in node] + return node + + return [json.dumps(shift(json.loads(line))) for line in lines] + + +_CAPTURED = Path(__file__).resolve().parents[2] / "fixtures" / "opencode_happy_stream.jsonl" +CAPTURED_STREAM = _CAPTURED.read_text(encoding="utf-8").splitlines() + + def _tokens(inp: int, out: int, *, write: int = 0, read: int = 0, reasoning: int = 0) -> dict[str, Any]: """Token payload in the NESTED convention (total = input+output+reasoning, cache counted inside `input`); see TestTokenShapeIsObservable for the flat one.""" @@ -94,7 +126,7 @@ def _tokens(inp: int, out: int, *, write: int = 0, read: int = 0, reasoning: int HAPPY_STREAM = [ - _evt("step_start", {"id": "prt_1", "messageID": "msg_1", "type": "step-start"}), + _evt("step_start", {"id": "prt_1", "messageID": "msg_1", "type": "step-start"}, at_ms=1400), _evt( "tool_use", { @@ -110,6 +142,7 @@ def _tokens(inp: int, out: int, *, write: int = 0, read: int = 0, reasoning: int "time": {"start": 1786663018214, "end": 1786663018231}, }, }, + at_ms=1435, ), _evt( "step_finish", @@ -120,9 +153,10 @@ def _tokens(inp: int, out: int, *, write: int = 0, read: int = 0, reasoning: int "cost": 0.001, "tokens": _tokens(100, 20, write=5, read=10), }, + at_ms=1440, ), - _evt("step_start", {"id": "prt_4", "messageID": "msg_2", "type": "step-start"}), - _evt("text", {"id": "prt_5", "messageID": "msg_2", "type": "text", "text": "Created the file."}), + _evt("step_start", {"id": "prt_4", "messageID": "msg_2", "type": "step-start"}, at_ms=1450), + _evt("text", {"id": "prt_5", "messageID": "msg_2", "type": "text", "text": "Created the file."}, at_ms=1460), _evt( "step_finish", { @@ -132,6 +166,7 @@ def _tokens(inp: int, out: int, *, write: int = 0, read: int = 0, reasoning: int "cost": 0.002, "tokens": _tokens(50, 30, read=40, reasoning=7), }, + at_ms=1470, ), ] @@ -249,8 +284,8 @@ def _build_catalogue() -> list[OpenCodeScenario]: OpenCodeScenario( name="a_single_text_turn", lines=[ - _evt("step_start", {"id": "prt_1", "messageID": "msg_1"}), - _evt("text", {"id": "prt_2", "messageID": "msg_1", "text": "All done."}), + _evt("step_start", {"id": "prt_1", "messageID": "msg_1"}, at_ms=0), + _evt("text", {"id": "prt_2", "messageID": "msg_1", "text": "All done."}, at_ms=1), _evt( "step_finish", { @@ -260,6 +295,7 @@ def _build_catalogue() -> list[OpenCodeScenario]: "cost": 0.001, "tokens": _tokens(100, 20), }, + at_ms=3, ), ], ) @@ -280,7 +316,7 @@ def _build_catalogue() -> list[OpenCodeScenario]: OpenCodeScenario( name="c_multi_step_tiling", lines=[ - _evt("step_start", {"id": "prt_1", "messageID": "msg_1"}), + _evt("step_start", {"id": "prt_1", "messageID": "msg_1"}, at_ms=0), _evt( "tool_use", { @@ -295,16 +331,19 @@ def _build_catalogue() -> list[OpenCodeScenario]: "time": {"start": _T0_MS, "end": _T0_MS + 5}, }, }, + at_ms=5, ), _evt( "step_finish", {"id": "prt_3", "messageID": "msg_1", "reason": "tool-calls", "tokens": _tokens(100, 20)}, + at_ms=6, ), - _evt("step_start", {"id": "prt_4", "messageID": "msg_2"}), - _evt("text", {"id": "prt_5", "messageID": "msg_2", "text": "Listed it."}), + _evt("step_start", {"id": "prt_4", "messageID": "msg_2"}, at_ms=8), + _evt("text", {"id": "prt_5", "messageID": "msg_2", "text": "Listed it."}, at_ms=9), _evt( "step_finish", {"id": "prt_6", "messageID": "msg_2", "reason": "stop", "tokens": _tokens(50, 30)}, + at_ms=12, ), ], ) @@ -323,7 +362,7 @@ def _build_catalogue() -> list[OpenCodeScenario]: OpenCodeScenario( name="d_orphaned_tool", lines=[ - _evt("step_start", {"id": "prt_1", "messageID": "msg_1"}), + _evt("step_start", {"id": "prt_1", "messageID": "msg_1"}, at_ms=0), _evt( "tool_use", { @@ -333,11 +372,13 @@ def _build_catalogue() -> list[OpenCodeScenario]: "callID": "call_1", "state": {"status": "pending", "input": {"command": "sleep 600"}}, }, + at_ms=1, ), - _evt("text", {"id": "prt_3", "messageID": "msg_1", "text": "Waiting."}), + _evt("text", {"id": "prt_3", "messageID": "msg_1", "text": "Waiting."}, at_ms=2), _evt( "step_finish", {"id": "prt_4", "messageID": "msg_1", "reason": "stop", "tokens": _tokens(100, 20)}, + at_ms=4, ), ], ) @@ -351,15 +392,17 @@ def _build_catalogue() -> list[OpenCodeScenario]: OpenCodeScenario( name="e_error_after_generation", lines=[ - _evt("step_start", {"id": "prt_1", "messageID": "msg_1"}), - _evt("text", {"id": "prt_2", "messageID": "msg_1", "text": "Starting."}), + _evt("step_start", {"id": "prt_1", "messageID": "msg_1"}, at_ms=0), + _evt("text", {"id": "prt_2", "messageID": "msg_1", "text": "Starting."}, at_ms=1), _evt( "step_finish", {"id": "prt_3", "messageID": "msg_1", "reason": "stop", "tokens": _tokens(100, 20)}, + at_ms=3, ), json.dumps( { "type": "error", + "timestamp": _T0_MS + 4, "sessionID": SESSION, "error": {"name": "ProviderAuthError", "data": {"message": "401 from the provider"}}, } @@ -369,6 +412,12 @@ def _build_catalogue() -> list[OpenCodeScenario]: ) ) + # (f) a real stream captured from `opencode run --format json` (1.18.30, Haiku via + # OpenRouter): two steps, a `write` and a `bash` call, the CLI's own envelope and + # `state.time` stamps. Session id and paths are scrubbed. The stamps are moved so + # the stream starts at `_T0_MS`, which `_rebase_lines` puts on the replay clock. + scenarios.append(OpenCodeScenario(name="f_captured_stream", lines=_starting_at_t0(CAPTURED_STREAM))) + return scenarios diff --git a/tests/fixtures/opencode_happy_stream.jsonl b/tests/fixtures/opencode_happy_stream.jsonl new file mode 100644 index 00000000..b14c28ae --- /dev/null +++ b/tests/fixtures/opencode_happy_stream.jsonl @@ -0,0 +1,8 @@ +{"type":"step_start","timestamp":1789599421824,"sessionID":"ses_captured","part":{"id":"prt_0ac70157a001ZT0k7RkMOES5fK","messageID":"msg_0ac700f2b001SSGFoj9fVWpcYA","sessionID":"ses_captured","type":"step-start"}} +{"type":"tool_use","timestamp":1789599422997,"sessionID":"ses_captured","part":{"type":"tool","tool":"write","callID":"toolu_016ZZZbGJz51bwQrFrxN4Sv2","state":{"status":"completed","input":{"filePath":"/work/hello.txt","content":"hi"},"output":"Wrote file successfully.","metadata":{"diagnostics":{},"filepath":"/work/hello.txt","exists":false,"truncated":false},"title":"work/hello.txt","time":{"start":1789599422988,"end":1789599422995}},"metadata":{"openrouter":{"reasoning_details":[]}},"id":"prt_0ac701686001Qw5kHqmnUNaw0i","sessionID":"ses_captured","messageID":"msg_0ac700f2b001SSGFoj9fVWpcYA"}} +{"type":"text","timestamp":1789599423294,"sessionID":"ses_captured","part":{"id":"prt_0ac70157e001dUrObdNiWMqhff","messageID":"msg_0ac700f2b001SSGFoj9fVWpcYA","sessionID":"ses_captured","type":"text","text":"I'll help you create the file and list the directory.","time":{"start":1789599421822,"end":1789599423291}}} +{"type":"tool_use","timestamp":1789599423340,"sessionID":"ses_captured","part":{"type":"tool","tool":"bash","callID":"toolu_01DYyaQT59GsS3Qyj9smkbrb","state":{"status":"completed","input":{"command":"ls -la /work/"},"output":"total 8\ndrwx------ 3 user staff 96 Sep 16 15:57 .\ndrwx------@ 39029 user staff 1248928 Sep 16 15:57 ..\n-rw-r--r-- 1 user staff 2 Sep 16 15:57 hello.txt\n","metadata":{"output":"total 8\ndrwx------ 3 user staff 96 Sep 16 15:57 .\ndrwx------@ 39029 user staff 1248928 Sep 16 15:57 ..\n-rw-r--r-- 1 user staff 2 Sep 16 15:57 hello.txt\n","exit":0,"truncated":false},"title":"ls -la /work/","time":{"start":1789599423281,"end":1789599423331}},"id":"prt_0ac701a0d0016U67B5JGdiRx7R","sessionID":"ses_captured","messageID":"msg_0ac700f2b001SSGFoj9fVWpcYA"}} +{"type":"step_finish","timestamp":1789599423340,"sessionID":"ses_captured","part":{"id":"prt_0ac701b65001oV8lzL5tJ354vF","reason":"tool-calls","messageID":"msg_0ac700f2b001SSGFoj9fVWpcYA","sessionID":"ses_captured","type":"step-finish","tokens":{"total":16549,"input":3,"output":210,"reasoning":0,"cache":{"write":16336,"read":0}},"cost":0.021473}} +{"type":"step_start","timestamp":1789599424051,"sessionID":"ses_captured","part":{"id":"prt_0ac701e2c001eq21OKMX4VsR60","messageID":"msg_0ac701b6a001f4VpkH4NL51mAe","sessionID":"ses_captured","type":"step-start"}} +{"type":"text","timestamp":1789599424098,"sessionID":"ses_captured","part":{"id":"prt_0ac701e30001wThYZ7I3muX1xY","messageID":"msg_0ac701b6a001f4VpkH4NL51mAe","sessionID":"ses_captured","type":"text","text":"DONE","time":{"start":1789599424048,"end":1789599424088}}} +{"type":"step_finish","timestamp":1789599424098,"sessionID":"ses_captured","part":{"id":"prt_0ac701e5b001GJolKGDIGkU7ZI","reason":"stop","messageID":"msg_0ac701b6a001f4VpkH4NL51mAe","sessionID":"ses_captured","type":"step-finish","tokens":{"total":16703,"input":7,"output":5,"reasoning":0,"cache":{"write":355,"read":16336}},"cost":0.00210935}} diff --git a/tests/lint/rules/ce073_create_subprocess_explicit_stdin.py b/tests/lint/rules/ce073_create_subprocess_explicit_stdin.py new file mode 100644 index 00000000..35f2fdf9 --- /dev/null +++ b/tests/lint/rules/ce073_create_subprocess_explicit_stdin.py @@ -0,0 +1,56 @@ +"""CE073: ``asyncio.create_subprocess_exec`` / ``create_subprocess_shell`` must pass ``stdin=``. + +The defect: ``pi`` and ``opencode`` both read a non-TTY stdin TO EOF before they emit +anything. Neither adapter passed ``stdin=``, so the CLI inherited the parent's stdin, and a +``coder-eval run`` whose own stdin was a pipe that stayed open (a backgrounded or +tool-spawned batch) stalled every turn with zero events until the 300 s ``turn_timeout``. +Reproduced end to end on 2026-09-16: stdin held open → ``ERROR`` after the timeout with 0 +commands; stdin on ``/dev/null`` → ``SUCCESS`` in 10 s. The same inheritance reached the +task's ``pre_run``/``post_run`` shell commands, where an authored ``read`` hangs the task. + +Fires, anywhere under ``src/coder_eval/``, on an ``asyncio.create_subprocess_exec`` / +``create_subprocess_shell`` call (attribute form or a bare imported name) with no +``stdin=`` keyword. It requires a DECISION, not ``DEVNULL``: ``stdin=PIPE`` for a caller +that writes to the child passes. Sibling of CE015 (``limit=``); one invariant per id. + +Blind spots: synchronous ``subprocess.run`` / ``Popen`` (which includes +``Sandbox.run_command``, running task-authored shell commands), ``loop.subprocess_exec`` / +``subprocess_shell``, ``anyio.open_process`` / ``run_process``, an explicit ``stdin=None`` +(which still inherits), and a call through ``**kwargs``. +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +_SRC_ROOT = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]") +_SPAWNERS = frozenset({"create_subprocess_exec", "create_subprocess_shell"}) + + +def _spawner_name(func: ast.expr) -> str | None: + if isinstance(func, ast.Attribute) and func.attr in _SPAWNERS: + return func.attr + if isinstance(func, ast.Name) and func.id in _SPAWNERS: + return func.id + return None + + +class CreateSubprocessExplicitStdin(BaseRule): + id = "CE073" + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(_SRC_ROOT.search(filepath)) + + def visit_Call(self, node: ast.Call) -> None: + name = _spawner_name(node.func) + if self._in_scope and name is not None and not any(kw.arg == "stdin" for kw in node.keywords): + self.violation( + node, + f"{name} without stdin= inherits this process's stdin; a child that reads it to EOF stalls " + + "while the parent's stdin stays open. Pass stdin= explicitly (asyncio.subprocess.DEVNULL " + + "unless you write to the child).", + ) + self.generic_visit(node) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index ddfb62ca..58f38c43 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -45,6 +45,7 @@ from tests.lint.rules.ce068_no_kind_names_in_kernel import NoKindNamesInKernel from tests.lint.rules.ce070_no_cap_or_skill_scan_in_adapters import NoCapOrSkillScanInAdapters from tests.lint.rules.ce071_price_turn_only import PriceTurnOnly +from tests.lint.rules.ce073_create_subprocess_explicit_stdin import CreateSubprocessExplicitStdin from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -68,7 +69,7 @@ # start meaning something new. # # Claim 074 next (069 is TestCE069HarnessParityTable, 070 is NoCapOrSkillScanInAdapters, 071 is -# PriceTurnOnly; 072 and 073 are reserved for the emitter sole-writer and subprocess-stdin rules). +# PriceTurnOnly, 073 is CreateSubprocessExplicitStdin; 072 is reserved for the emitter sole-writer rule). # NOTE 065 IS TAKEN and is not in ALL_RULES: doc-surface and # whole-tree rules are `@pytest.mark.lint` classes in tests/test_custom_lint.py # rather than BaseRules, so the `_rule_ids` uniqueness assert below cannot see @@ -129,6 +130,7 @@ NoKindNamesInKernel, NoCapOrSkillScanInAdapters, PriceTurnOnly, + CreateSubprocessExplicitStdin, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index e6f3faad..7835971f 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -63,26 +63,6 @@ # subtraction in codex_agent._flush_message. "codex_d_cross_flush_is_error", # flush lands before the tool completes: zero-width window "codex_e_orphan_tool", # the tool never completes, so the window never opens - # Same shape, reached from the opposite direction. This scenario injects - # a 5 ms CLI tool interval into a replay whose whole turn is well under - # one millisecond, so the tool spans BOTH windows entirely and the - # central subtraction takes each down to a measured 0.0. It is the tool - # interval that is fictional, not the subtraction — which is why the - # scenario is in FICTIONAL_DURATIONS too. - # - # BE HONEST ABOUT WHAT IS LEFT. With both exemptions on, this snapshot - # asserts neither the identity nor a positive window, and it does NOT - # record the tiling the scenario is named for — `SCRUB_KEYS` masks - # `started_at`, `completed_at` and `generation_duration_ms`, so nothing - # about where a window opened survives into the JSON. What it still - # pins is the STRUCTURE: two assistant messages, their content blocks, - # their token buckets, and one resolved command. OpenCode's tiling is - # asserted where it can be — `tests/test_timing_identity_contract.py` - # (scripted clock, ms-exact) and - # `tests/test_opencode_agent.py::TestGenerationWindowsTileTheTurn`. - # `pi_c_multi_turn_tiling` is the same scenario shape on a harness whose - # stamps come from its own clock, and it needs neither exemption. - "opencode_c_multi_step_tiling", } ) @@ -118,6 +98,13 @@ def _expect_window(harness: str, scenario_name: str) -> bool: "codex_f_collab_fallback", # 900 ms collab wait "codex_h_no_turn_completed_crash", # 200 ms of item time — see below "opencode_b_tool_call_resolved", # 17 ms tool interval + # OpenCode bounds every window and tool on the CLI's own envelope and + # `state.time` epoch stamps (timing_basis cli_epoch_ms), scripted in whole + # milliseconds, while the host bracket spans a sub-millisecond replay. + "opencode_a_single_text_turn", + "opencode_d_orphaned_tool", + "opencode_e_error_after_generation", + "opencode_f_captured_stream", # a real 2.3 s CLI timeline # 5 ms tool interval, injected as CLI epoch stamps. OpenCode takes its # tool bounds from the CLI payload rather than from its own clock, so # every scenario of this harness that resolves a tool injects them — diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index add96a63..4b9fa4d1 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4099,6 +4099,47 @@ def test_price_turn_is_allowed_in_an_adapter(self): assert not self._violations(source, "/repo/src/coder_eval/agents/x_agent.py") +class TestCE073CreateSubprocessExplicitStdin: + """CE073 — every asyncio subprocess spawn under src/coder_eval decides its stdin.""" + + SRC = "/repo/src/coder_eval/agents/x_agent.py" + + @staticmethod + def _violations(source: str, filepath: str) -> list: + import ast + + from tests.lint.rules.ce073_create_subprocess_explicit_stdin import CreateSubprocessExplicitStdin + + return list(CreateSubprocessExplicitStdin(filepath).check(ast.parse(source))) + + @pytest.mark.parametrize( + "source", + [ + "p = await asyncio.create_subprocess_exec('pi', stdout=PIPE, limit=1)", + "p = await asyncio.create_subprocess_shell('ls', limit=1)", + "p = await create_subprocess_exec('pi', limit=1)", + "p = await create_subprocess_shell('ls', limit=1)", + ], + ) + def test_a_spawn_without_stdin_violates(self, source: str): + found = self._violations(source, self.SRC) + assert found + assert "stdin=" in found[0].message + + @pytest.mark.parametrize( + "source", + [ + "p = await asyncio.create_subprocess_exec('pi', stdin=asyncio.subprocess.DEVNULL, limit=1)", + "p = await asyncio.create_subprocess_shell('ls', stdin=asyncio.subprocess.PIPE, limit=1)", + ], + ) + def test_an_explicit_stdin_passes(self, source: str): + assert not self._violations(source, self.SRC) + + def test_outside_src_is_out_of_scope(self): + assert not self._violations("p = await asyncio.create_subprocess_exec('x')", "/repo/tests/test_x.py") + + class TestCE054EnvInfoKeyRoundTrip: """CE054 fires when an environment_info key is read with no writer anywhere. diff --git a/tests/test_harness_conformance.py b/tests/test_harness_conformance.py index 64a10ef4..b340d26f 100644 --- a/tests/test_harness_conformance.py +++ b/tests/test_harness_conformance.py @@ -255,10 +255,10 @@ async def _opencode( monkeypatch.delenv("OPENCODE_CONFIG_CONTENT", raising=False) opencode = await _cli_agent(OpenCodeAgent, AgentKind.OPENCODE, tmp_path, monkeypatch, plugin_root, **agent) try: - raw = opencode._build_env().get("OPENCODE_CONFIG_CONTENT") + raw = opencode.env().get("OPENCODE_CONFIG_CONTENT") config = json.loads(raw) if raw else {} config["instructions_text"] = [Path(p).read_text(encoding="utf-8") for p in config.get("instructions", [])] - return config, opencode._build_argv(USER_TURN) + return config, opencode.argv(USER_TURN) finally: await opencode.stop() @@ -302,7 +302,7 @@ async def _pi_argv( ) -> list[str]: pi = await _cli_agent(PiAgent, AgentKind.PI, tmp_path, monkeypatch, plugin_root, **agent) try: - return pi._build_argv(USER_TURN) + return pi.argv(USER_TURN) finally: await pi.stop() diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 5eb26e71..a6e6af0d 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -3,7 +3,9 @@ The CLI is never invoked: ``asyncio.create_subprocess_exec`` is patched with a fake process that replays a newline-delimited JSON event stream, so the whole reduction path (nd-JSON -> standardized events -> ``TurnRecord``) is exercised -offline and without credentials. +offline and without credentials. Timing cases drive ``_OpenCodeDecoder`` +directly through ``coder_eval.testing.replay`` under ``cli_epoch_ms``: the +windows come from the scripted envelope stamps, never from the clock. The fixtures below mirror event lines CAPTURED FROM A LIVE ``opencode run --format json`` — the CLI's own compact vocabulary (``step_start`` / @@ -28,14 +30,14 @@ from coder_eval.agents import opencode_agent as agent_module from coder_eval.agents.opencode_agent import ( OpenCodeAgent, - _OpenCodeTurnState, + _OpenCodeDecoder, _unwrap, ) -from coder_eval.errors import AgentCrashError, TurnTimeoutError -from coder_eval.models import AssistantMessage, CommandTelemetry, OpenCodeAgentConfig, PermissionMode, TokenUsage +from coder_eval.errors.agent import format_timeout_reason +from coder_eval.models import AssistantMessage, OpenCodeAgentConfig, PermissionMode, TimingBasis, TurnRecord from coder_eval.orchestration.plugin_staging import stage_plugins from coder_eval.pricing import calculate_cost -from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.emitter import TurnOutcome from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -48,7 +50,10 @@ TurnEndStatus, TurnStartEvent, ) +from coder_eval.testing import Replay, ScriptedClock, Tick, assert_identity_closes, assert_stream_balanced, replay from tests._fixtures.golden_streams.opencode_fixtures import ( + _T0_MS, + CAPTURED_STREAM, HAPPY_STREAM, SESSION, _evt, @@ -85,19 +90,19 @@ async def fake_exec(*argv: str, **kwargs: Any) -> _FakeProcess: return _install -async def _run_outcome( +async def _run( agent: OpenCodeAgent, tmp_path: Any, prompt: str = "do the thing", *, plugin_root: Path | None = None, **kwargs: Any -): +) -> TurnOutcome: await agent.start(str(tmp_path), plugin_root=plugin_root) kwargs.setdefault("iteration", 1) return await agent.communicate(prompt, **kwargs) -async def _run( - agent: OpenCodeAgent, tmp_path: Any, prompt: str = "do the thing", *, plugin_root: Path | None = None, **kwargs: Any -): - outcome = await _run_outcome(agent, tmp_path, prompt, plugin_root=plugin_root, **kwargs) - return outcome.record_or_raise() +async def _record(agent: OpenCodeAgent, tmp_path: Any, prompt: str = "do the thing", **kwargs: Any) -> TurnRecord: + """The record of a turn that must not end CRASHED or TIMEOUT.""" + outcome = await _run(agent, tmp_path, prompt, **kwargs) + assert outcome.error is None, f"{outcome.status}: {outcome.error}" + return outcome.record def _agent(**overrides: Any) -> OpenCodeAgent: @@ -105,33 +110,107 @@ def _agent(**overrides: Any) -> OpenCodeAgent: return OpenCodeAgent(config, task_id="t1") +_BASE = datetime.fromtimestamp(_T0_MS / 1000) + + +def _at(ms: float) -> datetime: + """The naive-local instant of the CLI stamp ``_T0_MS + ms``, as the decoder converts it.""" + return datetime.fromtimestamp((_T0_MS + ms) / 1000) + + +def _event(event_type: str, part: dict[str, Any] | None = None, *, at_ms: int | None = 0) -> dict[str, Any]: + """One decoded CLI event; ``at_ms=None`` drops the envelope ``timestamp``.""" + event = json.loads(_evt(event_type, part or {}, at_ms=at_ms or 0)) + if at_ms is None: + del event["timestamp"] + return event + + +def _tool(call_id: str, status: str, *, at_ms: int, start: int | None = None, end: int | None = None) -> dict[str, Any]: + """A ``bash`` ``tool_use`` event whose ``state.time`` carries the given bounds, in ms after ``_T0_MS``.""" + times = {key: _T0_MS + ms for key, ms in (("start", start), ("end", end)) if ms is not None} + state: dict[str, Any] = {"status": status, "input": {"command": "ls"}} + if times: + state["time"] = times + return _event("tool_use", {"callID": call_id, "tool": "bash", "state": state}, at_ms=at_ms) + + +def _finish(at_ms: int | None) -> dict[str, Any]: + return _event("step_finish", {"reason": "stop", "tokens": {"input": 10, "output": 5}}, at_ms=at_ms) + + +def _replay( + stream: list[Any], *, status: AgentEndStatus = AgentEndStatus.COMPLETED, reason: str | None = None +) -> tuple[Replay, _OpenCodeDecoder]: + """Drive an `_OpenCodeDecoder` under `cli_epoch_ms` on a clock at `_BASE`; return the decoder too.""" + decoders: list[_OpenCodeDecoder] = [] + + def end(decoder: _OpenCodeDecoder) -> TurnOutcome: + decoders.append(decoder) + return decoder.end(status, reason=reason) + + result = replay(stream, _OpenCodeDecoder, clock=ScriptedClock(_BASE), basis=TimingBasis.CLI_EPOCH_MS, end=end) + return result, decoders[0] + + +def _assistants(result: Replay) -> list[AssistantMessage]: + return [m for m in result.record.messages if isinstance(m, AssistantMessage)] + + class TestEnvelopeNormalization: def test_part_envelope(self): """Normal events carry their payload under `part`.""" - t, part = _unwrap({"type": "step_finish", "sessionID": "s", "part": {"reason": "stop"}}) + t, part, stamp = _unwrap({"type": "step_finish", "sessionID": "s", "part": {"reason": "stop"}}) assert t == "step_finish" assert part["reason"] == "stop" + assert stamp is None def test_flat_envelope(self): """The CLI's own error path emits a flat object with no `part`.""" - t, props = _unwrap({"type": "error", "sessionID": "s", "error": {"name": "UnknownError"}}) + t, props, _ = _unwrap({"type": "error", "sessionID": "s", "error": {"name": "UnknownError"}}) assert t == "error" assert props["error"]["name"] == "UnknownError" + def test_the_envelope_stamp_is_returned(self): + """The envelope `timestamp` (epoch ms) is the CLI's own stamp for the event.""" + _, _, stamp = _unwrap({"type": "text", "timestamp": _T0_MS + 250, "part": {"text": "hi"}}) + assert stamp == _at(250) + class TestHappyPath: async def test_builds_turn_record(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + record = outcome.record + assert outcome.status is AgentEndStatus.COMPLETED assert record.crashed is False assert record.agent_output == "Created the file." assert record.assistant_turn_count == 2 assert record.model_used == "deepseek/deepseek-v4-pro" + async def test_the_result_summary_carries_the_final_reply(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await _record(_agent(), tmp_path) + + assert record.result_summary is not None + assert record.result_summary.is_error is False + assert record.result_summary.stop_reason == "stop" + assert record.result_summary.result == "Created the file." + + async def test_events_carry_no_session_thread_id(self, patch_exec, tmp_path): + """The session id is replayed via `--session`; it is not a sub-agent thread.""" + patch_exec(_FakeProcess(HAPPY_STREAM)) + recorder = _EventRecorder() + await _run(_agent(), tmp_path, stream_callback=recorder) + + assert recorder.events + assert all(e.thread_id is None for e in recorder.events) + assert_stream_balanced(recorder.events) + async def test_token_buckets_accumulate_across_steps(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) usage = record.token_usage assert usage is not None @@ -148,7 +227,7 @@ async def test_token_buckets_accumulate_across_steps(self, patch_exec, tmp_path) async def test_reconciliation_invariant(self, patch_exec, tmp_path): """Summing the four buckets across messages must equal token_usage exactly.""" patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) usage = record.token_usage assert usage is not None @@ -159,13 +238,14 @@ async def test_reconciliation_invariant(self, patch_exec, tmp_path): async def test_tool_call_captured(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert len(record.commands) == 1 cmd = record.commands[0] # Normalized to the canonical vocabulary criteria are written against. assert cmd.tool_name == "Read" assert cmd.tool_id == "call_1" + assert cmd.sequence_number == 0 assert cmd.result_status == "success" # ...including the ARGUMENT keys: the fixture's native `filePath` is # recorded under Claude's `file_path` (see TestCrossHarnessNormalization). @@ -176,7 +256,7 @@ async def test_tool_call_captured(self, patch_exec, tmp_path): async def test_messages_attributed_to_steps(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assistants = [m for m in record.messages if isinstance(m, AssistantMessage)] assert len(assistants) == 2 @@ -205,7 +285,7 @@ async def test_flat_convention_keeps_input_verbatim(self, patch_exec, tmp_path, ) patch_exec(_FakeProcess([step])) with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) usage = record.token_usage assert usage is not None @@ -219,7 +299,7 @@ async def test_nested_convention_subtracts_the_cache_buckets(self, patch_exec, t fresh slice must come back out or the cached portion is billed twice.""" patch_exec(_FakeProcess(HAPPY_STREAM)) # _tokens() builds nested totals with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) usage = record.token_usage assert usage is not None @@ -230,7 +310,7 @@ async def test_total_matching_neither_convention_warns(self, patch_exec, tmp_pat """nested=350, flat=8030, reported 8000 — the schema moved; keep `input`.""" patch_exec(_FakeProcess([self._step({"total": 8000, "input": 300, "output": 50, "cache": {"read": 7680}})])) with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) usage = record.token_usage assert usage is not None @@ -251,7 +331,7 @@ async def test_missing_total_with_cache_traffic_defaults_flat_but_warns(self, pa silent — but `input` is still taken verbatim, the live-verified convention.""" patch_exec(_FakeProcess([self._step({"input": 500, "output": 20, "cache": {"read": 200}})])) with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) usage = record.token_usage assert usage is not None @@ -263,7 +343,7 @@ async def test_missing_total_without_cache_traffic_is_silent(self, patch_exec, t """No arbiter but no cache either ⇒ the conventions agree; nothing to verify.""" patch_exec(_FakeProcess([self._step({"input": 500, "output": 20})])) with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) usage = record.token_usage assert usage is not None @@ -274,7 +354,7 @@ async def test_nested_total_contradicted_by_small_input_warns(self, patch_exec, """`total` says nested but input < cache: self-contradictory; keep `input`.""" patch_exec(_FakeProcess([self._step({"total": 350, "input": 300, "output": 50, "cache": {"read": 7680}})])) with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) usage = record.token_usage assert usage is not None @@ -286,7 +366,7 @@ class TestCostFallsBackToTheRateCard: async def test_stream_cost_wins_when_reported(self, patch_exec, tmp_path): """The provider's own accounting beats a static headline rate.""" patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert record.token_usage is not None assert record.token_usage.total_cost_usd == pytest.approx(0.003) # 0.001 + 0.002 @@ -298,7 +378,7 @@ async def test_missing_cost_is_priced_from_the_rate_card(self, patch_exec, tmp_p _evt("step_finish", {"id": "prt_2", "messageID": "msg_1", "reason": "stop", "tokens": _tokens(1000, 500)}), ] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert record.token_usage is not None expected = calculate_cost("deepseek/deepseek-v4-pro", uncached_input_tokens=1000, output_tokens=500) @@ -308,7 +388,7 @@ async def test_missing_cost_is_priced_from_the_rate_card(self, patch_exec, tmp_p async def test_unpriced_model_reports_no_cost(self, patch_exec, tmp_path): """`None` (not 0.0) so "unpriceable" stays distinct from "ran for free".""" patch_exec(_FakeProcess([_evt("step_finish", {"id": "p", "reason": "stop", "tokens": _tokens(10, 5)})])) - record = await _run(_agent(model="nowhere/not-a-real-model"), tmp_path) + record = await _record(_agent(model="nowhere/not-a-real-model"), tmp_path) assert record.token_usage is not None assert record.token_usage.total_cost_usd is None @@ -325,7 +405,7 @@ async def test_zero_reported_cost_on_a_priced_model_uses_the_rate_card(self, pat ] patch_exec(_FakeProcess(stream)) with caplog.at_level("DEBUG", logger="coder_eval.pricing"): - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) expected = calculate_cost("deepseek/deepseek-v4-pro", uncached_input_tokens=1000, output_tokens=500) assert expected is not None and expected > 0 @@ -343,7 +423,7 @@ async def test_zero_reported_cost_on_an_unpriced_model_stays_zero(self, patch_ex ), ] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(model="nowhere/not-a-real-model"), tmp_path) + record = await _record(_agent(model="nowhere/not-a-real-model"), tmp_path) assert record.token_usage is not None assert record.token_usage.total_cost_usd == 0.0 @@ -370,7 +450,7 @@ async def test_native_names_map_to_canonical(self, patch_exec, tmp_path): `parameters["command"]` only for that name — OpenCode's `bash` would match nothing and fall back to raw-JSON matching.""" patch_exec(_FakeProcess([self._tool_event("bash"), self._tool_event("write")])) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert [c.tool_name for c in record.commands] == ["Bash", "Write"] async def test_gpt_family_apply_patch_maps_to_write(self, patch_exec, tmp_path): @@ -385,20 +465,20 @@ async def test_gpt_family_apply_patch_maps_to_write(self, patch_exec, tmp_path): `_TOOL_ITEM_NAMES["apply_patch"] = "Write"`. """ patch_exec(_FakeProcess([self._tool_with_input("apply_patch", {"patchText": "*** Begin Patch\n"})])) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert [c.tool_name for c in record.commands] == ["Write"] async def test_unknown_tool_passes_through(self, patch_exec, tmp_path): """An unmapped tool still surfaces under its own name rather than vanishing.""" patch_exec(_FakeProcess([self._tool_event("some_new_tool")])) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert [c.tool_name for c in record.commands] == ["some_new_tool"] async def test_native_skill_tool_maps_to_canonical_skill(self, patch_exec, tmp_path): """`skill_triggered` keys on the canonical `Skill`; OpenCode emits lowercase `skill`, so without the mapping a real engagement scores as a miss.""" patch_exec(_FakeProcess([self._tool_event("skill")])) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert [c.tool_name for c in record.commands] == ["Skill"] @staticmethod @@ -441,7 +521,7 @@ async def test_argument_keys_map_to_canonical(self, patch_exec, tmp_path, tool, and scores 0 on OpenCode for identical agent behaviour. """ patch_exec(_FakeProcess([self._tool_with_input(tool, native)])) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert record.commands[0].parameters == expected @pytest.mark.parametrize( @@ -457,7 +537,7 @@ async def test_already_canonical_keys_are_left_alone(self, patch_exec, tmp_path, """The rename is per-tool: `path` means `file_path` on Read/Write/Edit and stays `path` on the search tools, which is exactly Claude's split.""" patch_exec(_FakeProcess([self._tool_with_input(tool, native)])) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert record.commands[0].parameters == native @@ -501,14 +581,16 @@ async def test_the_completion_supplies_the_parameters(self, patch_exec, tmp_path ] ) ) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert len(record.commands) == 1 # one tool, not two cmd = record.commands[0] assert cmd.tool_name == "Bash" assert cmd.parameters == {"command": "pytest -q"} assert cmd.result_status == "success" - assert cmd.execution_started_at is not None + assert cmd.execution_completed_at == datetime.fromtimestamp(1786663018231 / 1000.0) + assert cmd.execution_started_at is not None, "a start that arrives only with the result still times the call" + assert cmd.duration_ms is not None async def test_one_tool_start_end_pair_is_emitted(self, patch_exec, tmp_path): patch_exec( @@ -551,10 +633,12 @@ async def test_the_completion_time_end_becomes_execution_completed_at(self, patc ] ) ) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) cmd = record.commands[0] assert cmd.execution_completed_at == datetime.fromtimestamp(1786663018231 / 1000.0) + assert cmd.execution_started_at is not None, "a start that arrives only with the result still times the call" + assert cmd.duration_ms is not None assert cmd.execution_started_at == datetime.fromtimestamp(1786663018214 / 1000.0) assert cmd.duration_ms == pytest.approx(17.0) @@ -569,7 +653,7 @@ async def test_a_later_event_without_input_never_clears_what_we_have(self, patch ] ) ) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert record.commands[0].parameters == {"command": "ls"} @@ -866,6 +950,12 @@ async def test_explicit_line_limit_is_passed(self, patch_exec, tmp_path): await _run(_agent(), tmp_path) assert captured["kwargs"]["limit"] > 64 * 1024 + async def test_the_cli_never_inherits_stdin(self, patch_exec, tmp_path): + """OpenCode reads a non-TTY stdin to EOF before it emits; an inherited open stdin stalls the turn.""" + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(_agent(), tmp_path) + assert captured["kwargs"]["stdin"] is asyncio.subprocess.DEVNULL + class TestSessionContinuity: async def test_first_turn_omits_session(self, patch_exec, tmp_path): @@ -915,10 +1005,6 @@ async def test_session_id_recorded_after_a_turn(self, patch_exec, tmp_path): class TestErrorEventShapes: """`error` is the CLI's own flat envelope, and its payload shape varies.""" - @staticmethod - def _state() -> _OpenCodeTurnState: - return _OpenCodeTurnState(task_id="t1", iteration=1, user_input="p", model="m") - @pytest.mark.parametrize( ("payload", "expected"), [ @@ -932,9 +1018,8 @@ def _state() -> _OpenCodeTurnState: ], ) def test_message_extraction(self, payload, expected): - state = self._state() - state.on_error(payload) - assert state.error_message == expected + _, decoder = _replay([{"type": "error", "sessionID": SESSION, **payload}]) + assert decoder.error == expected class TestTokenCastsNeverRaise: @@ -966,7 +1051,7 @@ def _stream(tokens: dict[str, Any]) -> list[str]: async def test_a_non_numeric_bucket_warns_instead_of_crashing(self, patch_exec, tmp_path, tokens, caplog): patch_exec(_FakeProcess(self._stream(tokens))) with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert record.crashed is False assert "unexpected token accounting" in caplog.text @@ -977,7 +1062,7 @@ async def test_numeric_strings_are_still_accepted(self, patch_exec, tmp_path, ca """A stringly-typed but numeric count is a serialization detail, not drift.""" patch_exec(_FakeProcess(self._stream({"input": "100", "output": "20", "total": 120}))) with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert record.token_usage is not None assert record.token_usage.uncached_input_tokens == 100 @@ -988,7 +1073,7 @@ async def test_a_float_count_truncates(self, patch_exec, tmp_path, caplog): """JSON has one number type, so a provider may serialize a count as 100.0.""" patch_exec(_FakeProcess(self._stream({"input": 100.0, "output": 20.7, "total": 120}))) with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert record.token_usage is not None assert record.token_usage.uncached_input_tokens == 100 @@ -999,15 +1084,24 @@ async def test_a_bool_is_not_a_token_count(self, patch_exec, tmp_path, caplog): """`int(True) == 1` would book a phantom token.""" patch_exec(_FakeProcess(self._stream({"input": True, "output": 20}))) with caplog.at_level("WARNING"): - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert record.token_usage is not None assert record.token_usage.uncached_input_tokens == 0 assert "unexpected token accounting" in caplog.text +_ERROR_LINE = json.dumps( + { + "type": "error", + "sessionID": SESSION, + "error": {"name": "UnknownError", "data": {"message": "provider exploded"}}, + } +) + + class TestFailurePaths: - async def test_error_event_raises_and_parks_partial(self, patch_exec, tmp_path): + async def test_error_event_then_clean_exit_crashes_with_the_clis_message(self, patch_exec, tmp_path): stream = [ _evt("step_start", {"id": "prt_1", "messageID": "msg_1", "type": "step-start"}), _evt( @@ -1021,36 +1115,40 @@ async def test_error_event_raises_and_parks_partial(self, patch_exec, tmp_path): "state": {"status": "running", "input": {"command": "ls"}}, }, ), - json.dumps( - { - "type": "error", - "sessionID": SESSION, - "error": {"name": "UnknownError", "data": {"message": "provider exploded"}}, - } - ), + _ERROR_LINE, ] - patch_exec(_FakeProcess(stream)) + patch_exec(_FakeProcess(stream, returncode=0)) agent = _agent() - outcome = await _run_outcome(agent, tmp_path) + outcome = await _run(agent, tmp_path) assert outcome.status is AgentEndStatus.CRASHED - assert outcome.error is not None and "provider exploded" in outcome.error + assert outcome.error == "OpenCode error: provider exploded" partial = outcome.record assert partial.crashed is True + assert partial.result_summary is None # The in-flight tool was force-closed rather than dropped. assert [c.result_status for c in partial.commands] == ["unknown"] + async def test_a_stream_error_crashes_even_when_a_stop_was_requested(self, patch_exec, tmp_path): + """OpenCode's `error` event is final: an error read on the line a stop fires on is still a crash.""" + patch_exec(_RunningProcess([_ERROR_LINE])) + outcome = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOOL_CALL_CAP) + + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and outcome.error.startswith("OpenCode error:") + async def test_nonzero_exit_without_error_event_crashes(self, patch_exec, tmp_path): patch_exec(_FakeProcess([], returncode=1, stderr=b"boom: bad model")) - with pytest.raises(AgentCrashError, match="boom: bad model"): - await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error == "OpenCode exited non-zero: boom: bad model" async def test_malformed_line_is_skipped(self, patch_exec, tmp_path): """Non-JSON noise on stdout must not kill the turn.""" stream = ["warn: CPU lacks AVX support", *HAPPY_STREAM] patch_exec(_FakeProcess(stream)) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert record.crashed is False assert record.assistant_turn_count == 2 @@ -1088,7 +1186,7 @@ async def test_unrecognized_vocabulary_crashes_and_names_the_types(self, patch_e patch_exec(_FakeProcess(stream)) agent = _agent() - outcome = await _run_outcome(agent, tmp_path) + outcome = await _run(agent, tmp_path) assert outcome.status is AgentEndStatus.CRASHED assert outcome.error is not None and "no recognized events" in outcome.error @@ -1102,8 +1200,10 @@ async def test_unrecognized_vocabulary_crashes_and_names_the_types(self, patch_e async def test_empty_stdout_with_clean_exit_crashes(self, patch_exec, tmp_path): """Zero events at all is the same zero-telemetry hole as wrong vocabulary.""" patch_exec(_FakeProcess([], returncode=0)) - with pytest.raises(AgentCrashError, match="no recognized events"): - await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None + assert outcome.error.startswith("OpenCode exited cleanly but the turn captured no recognized events.") async def test_intentional_cuts_are_exempt(self, patch_exec, tmp_path): """A cooperative stop can land before the first recognized event; that is @@ -1111,7 +1211,7 @@ async def test_intentional_cuts_are_exempt(self, patch_exec, tmp_path): stream = [json.dumps({"id": "evt_1", "type": "session.next.idle", "properties": {"sessionID": SESSION}})] proc = _RunningProcess(stream) patch_exec(proc) - record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION) + record = await _record(_agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION) assert record.crashed is False @staticmethod @@ -1136,27 +1236,29 @@ async def test_finished_step_without_tokens_crashes(self, patch_exec, tmp_path): patch_exec(_FakeProcess(self._stream_without_tokens())) agent = _agent() - outcome = await _run_outcome(agent, tmp_path) + outcome = await _run(agent, tmp_path) assert outcome.status is AgentEndStatus.CRASHED assert outcome.error is not None assert "zero token telemetry" in outcome.error assert "1 finished step(s)" in outcome.error - assert outcome.record is not None # telemetry captured so far still parked + assert outcome.record.crashed is True # telemetry captured so far is still on the record + assert len(outcome.record.messages) >= 1 async def test_cost_without_tokens_still_crashes(self, patch_exec, tmp_path): """Reported cost does not excuse missing tokens: the USD gate might trip, but every token gate and aggregate is still silently blind.""" patch_exec(_FakeProcess(self._stream_without_tokens(cost=0.004))) - with pytest.raises(AgentCrashError, match="cost reported: yes"): - await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "cost reported: yes" in outcome.error async def test_require_token_telemetry_false_warns_and_scores(self, patch_exec, tmp_path, caplog): """The escape hatch, for a provider that genuinely reports no usage: crashing every turn there would make the harness unusable, not merely imprecise.""" patch_exec(_FakeProcess(self._stream_without_tokens())) with caplog.at_level("WARNING"): - record = await _run(_agent(require_token_telemetry=False), tmp_path) + record = await _record(_agent(require_token_telemetry=False), tmp_path) assert record.crashed is False assert "require_token_telemetry is off" in caplog.text @@ -1165,21 +1267,22 @@ async def test_the_hatch_never_relaxes_the_vocabulary_check(self, patch_exec, tm """Vocabulary drift has silently zeroed a whole run before, and no provider quirk explains it — so this arm stays fatal even with the hatch open.""" patch_exec(_FakeProcess([json.dumps({"type": "session.next.idle", "properties": {"sessionID": SESSION}})])) - with pytest.raises(AgentCrashError, match="no recognized events"): - await _run(_agent(require_token_telemetry=False), tmp_path) + outcome = await _run(_agent(require_token_telemetry=False), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "no recognized events" in outcome.error async def test_a_cut_before_any_step_finished_is_exempt(self, patch_exec, tmp_path): """The arm keys on a step the CLI reported FINISHED. A stop landing between a step's start and its `step_finish` is an intentional cut, not drift.""" proc = _RunningProcess([_evt("step_start", {"id": "prt_1", "messageID": "msg_1", "type": "step-start"})]) patch_exec(proc) - record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION) + record = await _record(_agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION) assert record.crashed is False async def test_real_tokens_are_never_condemned(self, patch_exec, tmp_path): """The guard must not fire on the ordinary path it lives beside.""" patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path) + record = await _record(_agent(), tmp_path) assert record.crashed is False assert record.token_usage is not None @@ -1215,7 +1318,7 @@ class TestUnexpectedErrorContract: instead of kept on the crashed record. """ - async def test_stream_error_becomes_a_crash_with_partial_parked(self, patch_exec, tmp_path): + async def test_stream_error_becomes_a_crash_with_the_crashed_partial(self, patch_exec, tmp_path): stream = [ _evt("step_start", {"id": "prt_1", "messageID": "msg_1", "type": "step-start"}), _evt( @@ -1233,10 +1336,10 @@ async def test_stream_error_becomes_a_crash_with_partial_parked(self, patch_exec patch_exec(_ExplodingProcess(stream)) agent = _agent() - outcome = await _run_outcome(agent, tmp_path) + outcome = await _run(agent, tmp_path) assert outcome.status is AgentEndStatus.CRASHED - assert outcome.error is not None and "OpenCode turn failed" in outcome.error + assert outcome.error is not None and outcome.error.startswith("OpenCode turn failed: ") partial = outcome.record assert partial.crashed is True # Telemetry captured before the failure survives, orphan tool force-closed. @@ -1251,16 +1354,17 @@ async def boom(*_argv: str, **_kwargs: Any): monkeypatch.setattr(asyncio, "create_subprocess_exec", boom) monkeypatch.setattr("shutil.which", lambda _name: "/usr/local/bin/opencode") - with pytest.raises(AgentCrashError, match="no fork for you"): - await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error == "OpenCode turn failed: no fork for you" async def test_terminal_event_is_emitted_exactly_once(self, patch_exec, tmp_path): """The protocol allows exactly one AgentEnd per communicate(), crash included.""" patch_exec(_ExplodingProcess([_evt("step_start", {"id": "prt_1", "messageID": "msg_1"})])) recorder = _EventRecorder() - with pytest.raises(AgentCrashError): - await _run(_agent(), tmp_path, stream_callback=recorder) + outcome = await _run(_agent(), tmp_path, stream_callback=recorder) + assert outcome.status is AgentEndStatus.CRASHED seen = recorder.events assert len([e for e in seen if isinstance(e, AgentStartEvent)]) == 1 @@ -1295,7 +1399,7 @@ class TestLeakedPipeDrain: async def test_completes_without_eof(self, patch_exec, tmp_path): """A stdout pipe that never closes must not stall the turn.""" patch_exec(_LeakyPipeProcess(HAPPY_STREAM)) - record = await asyncio.wait_for(_run(_agent(), tmp_path, timeout=300), timeout=30) + record = await asyncio.wait_for(_record(_agent(), tmp_path, timeout=300), timeout=30) assert record.crashed is False assert record.assistant_turn_count == 2 @@ -1327,7 +1431,7 @@ class TestStderrIsDrainedConcurrently: async def test_turn_completes_under_stderr_backpressure(self, patch_exec, tmp_path): patch_exec(_StderrBackpressureProcess(HAPPY_STREAM, stderr=b"noisy")) # Bounded so a regression fails here instead of hanging the suite. - record = await asyncio.wait_for(_run(_agent(), tmp_path, timeout=300), timeout=10) + record = await asyncio.wait_for(_record(_agent(), tmp_path, timeout=300), timeout=10) assert record.assistant_turn_count == 2 assert record.crashed is False @@ -1353,10 +1457,12 @@ async def test_early_criterion_ends_turn_stopped_early(self, patch_exec, tmp_pat proc = _RunningProcess(HAPPY_STREAM) patch_exec(proc) recorder = _EventRecorder() - record = await _run( + outcome = await _run( _agent(), tmp_path, should_stop=lambda: StopReason.EARLY_CRITERION, stream_callback=recorder ) + record = outcome.record + assert outcome.status is AgentEndStatus.STOPPED_EARLY assert record.crashed is False assert proc.terminated is True # Stopped at the first event boundary rather than draining the stream. @@ -1368,7 +1474,9 @@ async def test_tool_call_cap_ends_turn_tool_calls_exhausted(self, patch_exec, tm proc = _RunningProcess(HAPPY_STREAM) patch_exec(proc) recorder = _EventRecorder() - record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOOL_CALL_CAP, stream_callback=recorder) + record = await _record( + _agent(), tmp_path, should_stop=lambda: StopReason.TOOL_CALL_CAP, stream_callback=recorder + ) assert proc.terminated is True assert record.crashed is False @@ -1380,7 +1488,9 @@ async def test_token_budget_ends_turn_token_budget_exceeded(self, patch_exec, tm proc = _RunningProcess(HAPPY_STREAM) patch_exec(proc) recorder = _EventRecorder() - record = await _run(_agent(), tmp_path, should_stop=lambda: StopReason.TOKEN_BUDGET, stream_callback=recorder) + record = await _record( + _agent(), tmp_path, should_stop=lambda: StopReason.TOKEN_BUDGET, stream_callback=recorder + ) assert proc.terminated is True assert record.crashed is False @@ -1390,7 +1500,7 @@ async def test_token_budget_ends_turn_token_budget_exceeded(self, patch_exec, tm async def test_no_stop_is_uncapped(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) - record = await _run(_agent(), tmp_path, should_stop=lambda: None) + record = await _record(_agent(), tmp_path, should_stop=lambda: None) assert record.tool_calls_exhausted is False assert record.assistant_turn_count == 2 @@ -1403,7 +1513,7 @@ async def test_the_deciding_step_is_kept_whole(self, patch_exec, tmp_path): """ patch_exec(_RunningProcess(HAPPY_STREAM)) recorder = _EventRecorder() - record = await _run( + record = await _record( _agent(), tmp_path, should_stop=_stop_after(3, StopReason.TOOL_CALL_CAP), stream_callback=recorder ) @@ -1422,7 +1532,7 @@ async def test_the_deciding_step_is_kept_whole(self, patch_exec, tmp_path): async def test_an_intentional_stop_is_exempt_from_a_non_zero_exit(self, patch_exec, tmp_path): """Killing the CLI makes it exit non-zero; that must not crash an intentional stop.""" patch_exec(_RunningProcess(HAPPY_STREAM, returncode=-15, stderr=b"terminated")) - record = await _run(_agent(), tmp_path, should_stop=_stop_after(3, StopReason.TOOL_CALL_CAP)) + record = await _record(_agent(), tmp_path, should_stop=_stop_after(3, StopReason.TOOL_CALL_CAP)) assert record.crashed is False assert record.tool_calls_exhausted is True @@ -1480,8 +1590,8 @@ class _EofNoExitProcess(_HangingProcess): """Replays its lines, signals EOF — but never exits until killed. Models a CLI that closed its stream during shutdown and then wedged: the one - window where the read loop is already done, so only a bounded reap in - ``_settle_turn`` stands between the turn and an unbounded hang. + window where the read loop is already done, so only the bounded post-EOF + exit wait in the transport's settle stands between the turn and an unbounded hang. """ async def readline(self) -> bytes: @@ -1491,7 +1601,7 @@ async def readline(self) -> bytes: class TestTimeoutContract: - async def test_deadline_raises_turn_timeout_with_partial_parked(self, patch_exec, tmp_path): + async def test_deadline_returns_a_timeout_outcome_with_the_partial(self, patch_exec, tmp_path): """A wedged CLI must yield a TIMEOUT outcome with a crashed partial record, with exactly one terminal AgentEndEvent (status TIMEOUT) emitted.""" proc = _HangingProcess([_evt("step_start", {"id": "prt_1", "messageID": "msg_1"})]) @@ -1499,12 +1609,13 @@ async def test_deadline_raises_turn_timeout_with_partial_parked(self, patch_exec agent = _agent() recorder = _EventRecorder() - outcome = await _run_outcome(agent, tmp_path, timeout=0.2, stream_callback=recorder) + outcome = await _run(agent, tmp_path, timeout=0.2, stream_callback=recorder) assert outcome.status is AgentEndStatus.TIMEOUT + assert outcome.error == format_timeout_reason(0.2) partial = outcome.record - assert partial is not None assert partial.crashed is True + assert partial.result_summary is None assert proc.terminated is True # the CLI was torn down, not abandoned ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] assert len(ends) == 1 @@ -1517,12 +1628,11 @@ async def test_eof_without_exit_hits_the_deadline(self, patch_exec, tmp_path): patch_exec(proc) agent = _agent() - outcome = await asyncio.wait_for(_run_outcome(agent, tmp_path, timeout=0.3), timeout=10) + outcome = await asyncio.wait_for(_run(agent, tmp_path, timeout=0.3), timeout=10) assert outcome.status is AgentEndStatus.TIMEOUT # Everything parsed before the wedge survives on the partial record. partial = outcome.record - assert partial is not None assert partial.crashed is True assert partial.token_usage is not None assert partial.token_usage.output_tokens > 0 @@ -1530,19 +1640,23 @@ async def test_eof_without_exit_hits_the_deadline(self, patch_exec, tmp_path): async def test_eof_without_exit_and_no_deadline_crashes(self, patch_exec, monkeypatch, tmp_path): """With no turn deadline configured, the reap still gets a fixed grace — a stream-closed-but-wedged CLI is a crash, not an indefinite hang.""" - monkeypatch.setattr("coder_eval.agents.opencode_agent._TERM_GRACE_SECONDS", 0.1) + monkeypatch.setattr("coder_eval.agents._transport.subprocess_jsonl._TERM_GRACE_SECONDS", 0.1) proc = _EofNoExitProcess(HAPPY_STREAM) patch_exec(proc) - with pytest.raises(AgentCrashError, match="did not exit"): - await asyncio.wait_for(_run(_agent(), tmp_path), timeout=10) + outcome = await asyncio.wait_for(_run(_agent(), tmp_path), timeout=10) + + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None + assert outcome.error.startswith("OpenCode closed its event stream but did not exit within") + assert proc.terminated is True class TestExternalCancel: - async def test_cancel_parks_partial_and_reraises(self, patch_exec, tmp_path): + async def test_cancel_ends_the_turn_and_reraises(self, patch_exec, tmp_path): """The watchdog's CancelledError must not swallow captured telemetry: the - partial record is parked, the terminal event says CRASHED, and the - cancellation still propagates.""" + turn is ended, the terminal event says CRASHED, and the cancellation + still propagates.""" proc = _HangingProcess([_evt("step_start", {"id": "prt_1", "messageID": "msg_1"})]) patch_exec(proc) agent = _agent() @@ -1555,12 +1669,10 @@ async def test_cancel_parks_partial_and_reraises(self, patch_exec, tmp_path): with pytest.raises(asyncio.CancelledError): _ = await task # the await re-raises the cancellation; no value ever exists - partial = agent.pending_turn - assert partial is not None - assert partial.crashed is True ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] assert len(ends) == 1 assert ends[0].status is AgentEndStatus.CRASHED + assert ends[0].crashed is True assert ends[0].crash_reason == "turn cancelled" assert proc.killed is True # not abandoned mid-stream — see TestTurnAlwaysReapsTheCli @@ -1569,9 +1681,9 @@ class TestTurnEventsAreBalanced: """`Agent.communicate`'s contract is one TurnStart/TurnEnd pair per inner turn. `on_step_start` opens one per CLI step and `on_step_finish` closes it, but a - turn that dies (or is cut) between the two left the last TurnStartEvent open - forever — a task.log with `>>> Turn start` and no matching `--- Turn end`. - All three sibling agents close it from `finalize`. + turn that dies (or is cut) between the two must still close the last + TurnStartEvent — a task.log with `>>> Turn start` and no matching + `--- Turn end` is the defect. The emitter closes it at the end of the turn. """ @staticmethod @@ -1591,8 +1703,8 @@ async def test_a_timeout_closes_the_open_step(self, patch_exec, tmp_path): patch_exec(proc) recorder = _EventRecorder() - with pytest.raises(TurnTimeoutError): - await _run(_agent(), tmp_path, timeout=0.2, stream_callback=recorder) + outcome = await _run(_agent(), tmp_path, timeout=0.2, stream_callback=recorder) + assert outcome.status is AgentEndStatus.TIMEOUT assert self._pairs(recorder) == (1, 1) end = next(e for e in recorder.events if isinstance(e, TurnEndEvent)) @@ -1620,8 +1732,8 @@ async def test_a_crash_closes_the_open_step(self, patch_exec, tmp_path): patch_exec(_ExplodingProcess([_evt("step_start", {"id": "prt_1", "messageID": "msg_1"})])) recorder = _EventRecorder() - with pytest.raises(AgentCrashError): - await _run(_agent(), tmp_path, stream_callback=recorder) + outcome = await _run(_agent(), tmp_path, stream_callback=recorder) + assert outcome.status is AgentEndStatus.CRASHED assert self._pairs(recorder) == (1, 1) @@ -1638,17 +1750,33 @@ async def test_a_clean_cut_closes_the_open_step(self, patch_exec, tmp_path): assert end.status is TurnEndStatus.STOPPED_EARLY async def test_a_completed_step_is_never_closed_twice(self, patch_exec, tmp_path): - """Unlike the siblings, completed steps close themselves in `on_step_finish`, - so `finalize` must fire ONLY for a straggler.""" + """Completed steps close themselves in `on_step_finish`, so the end of the + turn must close ONLY a straggler.""" patch_exec(_FakeProcess(HAPPY_STREAM)) recorder = _EventRecorder() - record = await _run(_agent(), tmp_path, stream_callback=recorder) + record = await _record(_agent(), tmp_path, stream_callback=recorder) assert record.crashed is False starts, ends = self._pairs(recorder) assert starts == ends == 2 assert all(e.status is TurnEndStatus.COMPLETED for e in recorder.events if isinstance(e, TurnEndEvent)) + def test_a_dangling_step_is_closed_crashed_at_the_next_step_start(self): + result, _ = _replay( + [ + _event("step_start", {"messageID": "m1"}, at_ms=0), + _event("step_start", {"messageID": "m2"}, at_ms=100), + _finish(200), + ] + ) + + turn_ends = [e for e in result.events if isinstance(e, TurnEndEvent)] + assert [(e.turn_id, e.status) for e in turn_ends] == [ + ("m1", TurnEndStatus.CRASHED), + ("m2", TurnEndStatus.COMPLETED), + ] + assert_stream_balanced(result.events) + class TestTurnAlwaysReapsTheCli: """No exit from `communicate()` may leave the CLI running. @@ -1665,13 +1793,14 @@ class TestTurnAlwaysReapsTheCli: """ async def test_read_loop_crash_kills_the_cli(self, patch_exec, tmp_path): - """`_crash_turn` is synchronous and raises — nothing below it reaps.""" + """A read-loop crash ends the turn as an outcome, and `finally` still reaps the live CLI.""" proc = _ExplodingRunningProcess([_evt("step_start", {"id": "prt_1", "messageID": "msg_1"})]) patch_exec(proc) - with pytest.raises(AgentCrashError, match="OpenCode turn failed"): - await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and outcome.error.startswith("OpenCode turn failed: ") assert proc.killed is True async def test_external_cancel_kills_the_cli(self, patch_exec, tmp_path): @@ -1709,8 +1838,9 @@ async def boom(*_argv: str, **_kwargs: Any): monkeypatch.setattr(asyncio, "create_subprocess_exec", boom) monkeypatch.setattr("shutil.which", lambda _name: "/usr/local/bin/opencode") - with pytest.raises(AgentCrashError, match="no fork for you"): - await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED + assert outcome.error is not None and "no fork for you" in outcome.error @pytest.mark.skipif(os.name != "posix", reason="process-group teardown (killpg/SIGKILL) is POSIX-only by design") @@ -1738,8 +1868,8 @@ async def test_a_crashed_turn_sweeps_the_group_too(self, patch_exec, tmp_path): the pipes — across a retried batch, that is the leak that compounds.""" captured = patch_exec(_ExplodingRunningProcess([_evt("step_start", {"id": "prt_1", "messageID": "msg_1"})])) - with pytest.raises(AgentCrashError): - await _run(_agent(), tmp_path) + outcome = await _run(_agent(), tmp_path) + assert outcome.status is AgentEndStatus.CRASHED assert (4242, signal.SIGKILL) in captured["killpg"] @@ -1784,7 +1914,7 @@ def _failing_tool(error: str) -> str: async def test_tool_error_is_captured_not_dropped(self, patch_exec, tmp_path): recorder = _EventRecorder() patch_exec(_FakeProcess([self._failing_tool("boom: command exploded")])) - record = await _run(_agent(), tmp_path, stream_callback=recorder) + record = await _record(_agent(), tmp_path, stream_callback=recorder) [cmd] = record.commands assert cmd.result_status == "error" @@ -1795,180 +1925,239 @@ async def test_tool_error_is_captured_not_dropped(self, patch_exec, tmp_path): async def test_permission_denial_gets_its_own_status(self, patch_exec, tmp_path): recorder = _EventRecorder() patch_exec(_FakeProcess([self._failing_tool("Permission denied by policy")])) - record = await _run(_agent(), tmp_path, stream_callback=recorder) + record = await _record(_agent(), tmp_path, stream_callback=recorder) [cmd] = record.commands assert cmd.result_status == "error" # the persisted tri-state folds both [end] = [e for e in recorder.events if isinstance(e, ToolEndEvent)] assert end.status is ToolEndStatus.PERMISSION_DENIED - def test_orphan_result_is_never_dropped(self): - """A result with no matching call still surfaces as an `unknown` tool.""" - state = _OpenCodeTurnState(task_id="t", iteration=1, user_input="x", model=None) - events: list[Any] = [] - state.bind(events.append) + def test_an_unresolved_call_is_swept_never_dropped(self): + """A call the CLI never resolved still surfaces, as an `unknown` tool with no invented error.""" + result, decoder = _replay([_event("step_start", {"messageID": "m1"}), _tool("ghost", "pending", at_ms=1)]) - state._close_tool("ghost", status=ToolEndStatus.UNRESOLVED, summary=None, error="no result observed") - - [event] = events - assert isinstance(event, ToolEndEvent) - assert event.tool.tool_name == "unknown" + [event] = [e for e in result.events if isinstance(e, ToolEndEvent)] + assert event.status is ToolEndStatus.UNRESOLVED + assert event.tool.tool_id == "ghost" + assert event.tool.tool_name == "Bash" assert event.tool.result_status == "unknown" - assert event.tool.error_message == "no result observed" + assert event.tool.error_message is None + assert decoder.open_tools == {"ghost": {"command": "ls"}} + assert [c.tool_id for c in result.record.commands] == ["ghost"] + + +class TestTimingIsTheClisOwn: + """Under `cli_epoch_ms` every recorded bound is a CLI stamp, never a read of the host clock. + + Each case moves the scripted clock far away from the stamps, so a bound taken + from the clock instead of the stream lands seconds off and fails. + """ + + def test_window_bounds_are_the_envelope_stamps(self): + result, _ = _replay( + [ + Tick(7_000), + _event("step_start", {"messageID": "m1"}, at_ms=100), + Tick(50_000), + _finish(900), + ] + ) + + [message] = _assistants(result) + assert message.started_at == _at(100) + assert message.completed_at == _at(900) + assert message.generation_duration_ms == pytest.approx(800.0) + + def test_tool_span_is_state_time(self): + result, _ = _replay( + [ + _event("step_start", {"messageID": "m1"}, at_ms=0), + Tick(9_000), + _tool("c1", "completed", at_ms=600, start=200, end=450), + _finish(1000), + ] + ) + + [command] = result.record.commands + assert command.execution_started_at == _at(200) + assert command.execution_completed_at == _at(450) + assert command.duration_ms == pytest.approx(250.0) + + def test_a_resolved_tool_without_an_end_stamp_gets_no_completion_or_duration(self): + result, _ = _replay( + [ + _event("step_start", {"messageID": "m1"}, at_ms=0), + Tick(600), + _tool("c1", "completed", at_ms=600, start=200), + _finish(1000), + ] + ) + + [command] = result.record.commands + assert command.result_status == "success" + assert command.execution_started_at == _at(200) + assert command.execution_completed_at is None + assert command.duration_ms is None + + def test_an_orphan_has_no_completion_stamp(self): + """Force-closing is not observing a completion; the CLI's start stamp is kept.""" + result, _ = _replay( + [ + _event("step_start", {"messageID": "m1"}, at_ms=0), + _tool("c1", "running", at_ms=200, start=200), + _finish(1000), + Tick(4_000), + ] + ) + + [command] = result.record.commands + assert command.result_status == "unknown" + assert command.error_message is None + assert command.execution_started_at == _at(200) + assert command.execution_completed_at is None + assert command.duration_ms is None + + def test_a_missing_envelope_stamp_bounds_the_window_on_the_host_clock_and_warns_once(self, caplog): + with caplog.at_level("WARNING", logger="coder_eval.agents.opencode_agent"): + result, decoder = _replay( + [ + _event("step_start", {"messageID": "m1"}, at_ms=0), + Tick(700), + _finish(None), + Tick(1_200), + _event("step_start", {"messageID": "m2"}, at_ms=None), + Tick(1_500), + _finish(None), + ] + ) + + first, second = _assistants(result) + assert first.started_at == _at(0) + assert first.completed_at == _BASE + timedelta(milliseconds=700) + assert second.started_at == first.completed_at + assert second.completed_at == _BASE + timedelta(milliseconds=1_500) + assert decoder.warned_missing_stamp is True + warnings = [r for r in caplog.records if "no envelope timestamp" in r.getMessage()] + assert len(warnings) == 1 + + def test_the_captured_stream_tiles_on_its_own_stamps(self): + """A real `opencode run` stream: every window and tool bound is a CLI stamp, and the identity closes.""" + lines = [json.loads(line) for line in CAPTURED_STREAM] + first_ms = lines[0]["timestamp"] + last_ms = max(event.get("timestamp", first_ms) for event in lines) + origin = datetime.fromtimestamp(first_ms / 1000) + stream: list[Any] = [*lines, Tick(last_ms - first_ms)] + + decoders: list[_OpenCodeDecoder] = [] + + def end(decoder: _OpenCodeDecoder) -> TurnOutcome: + decoders.append(decoder) + return decoder.end(AgentEndStatus.COMPLETED) + + result = replay(stream, _OpenCodeDecoder, clock=ScriptedClock(origin), basis=TimingBasis.CLI_EPOCH_MS, end=end) + + assert result.outcome.status is AgentEndStatus.COMPLETED + assert decoders[0].warned_missing_stamp is False + assert [c.tool_name for c in result.record.commands] == ["Write", "Bash"] + assert all(c.duration_ms is not None for c in result.record.commands) + assert_stream_balanced(result.events) + assert_identity_closes(result.record, started_at=result.started_at, ended_at=result.ended_at) class TestGenerationWindowExcludesToolExecution: """A tool running inside a step is not model time — asserted where it is now DECIDED. - The reducer no longer subtracts anything. It publishes the RAW window, and + The decoder does not subtract anything. It publishes the RAW window, and `timing.subtract_tool_time` takes the tool union back out of it - once, for all five harnesses. So these cases drive the reducer and then a - real collector, and assert the PUBLISHED number — the one that reaches - `task.json` — rather than an intermediate the reducer used to own. + once, for all five harnesses. So these cases replay the decoder through a + real emitter and assert the PUBLISHED number — the one that reaches + `task.json` — rather than an intermediate the decoder used to own. They are not duplicates of `tests/test_event_collector.py::TestSubtractToolTime`: those pin the - arithmetic, these pin that THIS reducer hands the collector a window and a + arithmetic, these pin that THIS decoder hands the collector a window and a span set the arithmetic can be right about. """ - WINDOW_START = datetime(2026, 1, 1, 12, 0, 0) - WINDOW_END = datetime(2026, 1, 1, 12, 0, 1) # a 1000ms step - - def _finish_step(self, monkeypatch, spans, open_starts=()): - """Drive the reducer, then publish through a real collector. + def _finish_step(self, spans: list[tuple[int, int]], open_starts: tuple[int, ...] = ()) -> AssistantMessage: + """Replay one step stamped 0 -> 1000 ms with tool calls whose `state.time` is at the given offsets. `spans` are RESOLVED calls (both bounds); `open_starts` are calls that - never returned. An unresolved call now contributes NO span — it has no + never returned. An unresolved call contributes NO span — it has no `execution_completed_at`, and inventing one is what `None` exists to - prevent — where the reducer used to bound it at the window's end. That - is a real change and a better one: the collector sees every span at - once, so a call straddling a boundary is clipped to each window it - actually overlapped instead of approximated at the boundary. + prevent. The collector sees every span at once, so a call straddling a + boundary is clipped to each window it actually overlapped. """ - - class _Clock(datetime): - @staticmethod - def now(tz=None): - return TestGenerationWindowExcludesToolExecution.WINDOW_END - - state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="do it", model="deepseek/deepseek-v4-pro") - state.step_started_at = self.WINDOW_START - commands = [ - CommandTelemetry( - tool_name="bash", - tool_id=f"closed-{i}", - timestamp=started, - execution_started_at=started, - execution_completed_at=completed, - result_status="success", - ) + stream: list[Any] = [_event("step_start", {"messageID": "m1"}, at_ms=0)] + stream += [ + _tool(f"closed-{i}", "completed", at_ms=completed, start=started, end=completed) for i, (started, completed) in enumerate(spans) ] - commands += [ - CommandTelemetry(tool_name="bash", tool_id=f"open-{i}", timestamp=st, execution_started_at=st) - for i, st in enumerate(open_starts) - ] - monkeypatch.setattr(agent_module, "datetime", _Clock) - state.on_step_finish({"reason": "stop", "tokens": {"input": 100, "output": 20}}) - - collector = EventCollector() - collector.on_event(AgentStartEvent(task_id="t1", prompt="do it", iteration=1, timestamp=self.WINDOW_START)) - for command in commands: - collector.on_event(ToolEndEvent(task_id="t1", turn_id="s1", tool=command)) - collector.on_event( - AgentEndEvent( - task_id="t1", - status=AgentEndStatus.COMPLETED, - messages=list(state.messages), - usage=TokenUsage(), - timestamp=self.WINDOW_END, - ) - ) - published = [m for m in collector.build_turn_record().messages if m.role == "assistant"] + stream += [_tool(f"open-{i}", "running", at_ms=started, start=started) for i, started in enumerate(open_starts)] + stream.append(_finish(1000)) + + result, _ = _replay(stream) + published = _assistants(result) assert len(published) == 1 return published[0] - def test_tool_time_inside_the_step_is_subtracted(self, monkeypatch): - message = self._finish_step( - monkeypatch, - [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))], - ) + def test_tool_time_inside_the_step_is_subtracted(self): + message = self._finish_step([(200, 700)]) span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 - assert span_ms == pytest.approx(1000.0), "the reducer still publishes the whole window as its bounds" + assert span_ms == pytest.approx(1000.0), "the decoder still publishes the whole window as its bounds" assert message.generation_duration_ms == pytest.approx(500.0) - def test_a_step_with_no_tools_keeps_its_whole_window(self, monkeypatch): - assert self._finish_step(monkeypatch, []).generation_duration_ms == pytest.approx(1000.0) + def test_a_step_with_no_tools_keeps_its_whole_window(self): + assert self._finish_step([]).generation_duration_ms == pytest.approx(1000.0) - def test_concurrent_tools_are_subtracted_once(self, monkeypatch): + def test_concurrent_tools_are_subtracted_once(self): # Two overlapping 500ms tools occupy 600ms, not 1000ms. Summing them # would leave 0 generation for a step that generated 400. - message = self._finish_step( - monkeypatch, - [ - (self.WINDOW_START + timedelta(milliseconds=100), self.WINDOW_START + timedelta(milliseconds=600)), - (self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700)), - ], - ) + message = self._finish_step([(100, 600), (200, 700)]) assert message.generation_duration_ms == pytest.approx(400.0) - def test_the_window_never_goes_negative(self, monkeypatch): - message = self._finish_step( - monkeypatch, - [(self.WINDOW_START - timedelta(seconds=30), self.WINDOW_END + timedelta(seconds=30))], - ) + def test_the_window_never_goes_negative(self): + message = self._finish_step([(-30_000, 31_000)]) assert message.generation_duration_ms == 0.0 - def test_a_tool_still_open_at_the_boundary_contributes_no_span(self, monkeypatch): - """The behaviour that CHANGED with the move, stated rather than implied. + def test_a_tool_still_open_at_the_boundary_contributes_no_span(self): + """A call with no `execution_completed_at` was never timed. - The reducer used to bound a still-open call at the window's end and - subtract that slice. The collector cannot: a call with no - `execution_completed_at` was never timed. Its time is subtracted when it - RESOLVES, from whichever windows its real interval overlaps. + Its time is subtracted when it RESOLVES, from whichever windows its real + interval overlaps — never bounded at the window's end. """ - message = self._finish_step(monkeypatch, [], open_starts=[self.WINDOW_START + timedelta(milliseconds=600)]) + message = self._finish_step([], open_starts=(600,)) assert message.generation_duration_ms == pytest.approx(1000.0) - def test_a_resolved_tool_overlapping_an_unresolved_one_counts_only_the_resolved(self, monkeypatch): - message = self._finish_step( - monkeypatch, - [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))], - open_starts=[self.WINDOW_START + timedelta(milliseconds=500)], - ) + def test_a_resolved_tool_overlapping_an_unresolved_one_counts_only_the_resolved(self): + message = self._finish_step([(200, 700)], open_starts=(500,)) assert message.generation_duration_ms == pytest.approx(500.0) - def test_a_mark_later_than_the_step_start_does_not_invert_the_window(self, monkeypatch): - """The backwards-clock defence, pinned at the reducer, not in isolation. + def test_a_mark_later_than_the_step_start_does_not_invert_the_window(self): + """The backwards-stamp defence, pinned at the decoder, not in isolation. - `close_window`'s `min()` only fires if the reducer actually passes the + `close_window`'s `min()` only fires if the decoder actually passes the step's own start as `item_start`. Drop that argument and the window opens at the (later) mark instead, so the span shrinks — or inverts and - clamps to 0.0, publishing a fabricated instant generation. Nothing else - in this file fails when it is dropped, which is the whole reason it is - here: the mark is what the reducer still owns after the tool - subtraction moved to the collector. + clamps to 0.0, publishing a fabricated instant generation. The CLI's + stamps are not monotonic by contract: here the first step's + `step_finish` is stamped 400 ms AFTER the second step's `step_start`. """ - state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="do it", model="m") - state.step_started_at = self.WINDOW_START - # A mark 400ms AFTER this step began: the CLI's step_finish for the - # previous step landed late, or the clock stepped. - state.gen_mark = self.WINDOW_START + timedelta(milliseconds=400) - - class _Clock(datetime): - @staticmethod - def now(tz=None): - return TestGenerationWindowExcludesToolExecution.WINDOW_END - - monkeypatch.setattr(agent_module, "datetime", _Clock) - state.on_step_finish({"reason": "stop", "tokens": {"input": 100, "output": 20}}) - - message = next(m for m in state.messages if m.role == "assistant") - assert message.started_at == self.WINDOW_START + result, decoder = _replay( + [ + _event("step_start", {"messageID": "m0"}, at_ms=-500), + _finish(400), + _event("step_start", {"messageID": "m1"}, at_ms=0), + _finish(1000), + ] + ) + + _, message = _assistants(result) + assert decoder.gen_mark == _at(1000) + assert message.started_at == _at(0) assert message.generation_duration_ms == pytest.approx(1000.0) - def test_the_published_window_reconciles_to_its_own_bounds(self, monkeypatch): + def test_the_published_window_reconciles_to_its_own_bounds(self): """The collector subtracted exactly the spans the record carries. `scripts/timing/decompose_run.py` and the evalboard's Unaccounted cell @@ -1979,10 +2168,9 @@ def test_the_published_window_reconciles_to_its_own_bounds(self, monkeypatch): """ from coder_eval.timing import busy_ms - closed = [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))] - message = self._finish_step(monkeypatch, closed) + message = self._finish_step([(200, 700)]) span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 - expected = span_ms - busy_ms(closed, message.started_at, message.completed_at) + expected = span_ms - busy_ms([(_at(200), _at(700))], message.started_at, message.completed_at) assert message.generation_duration_ms == pytest.approx(expected) @@ -1995,235 +2183,146 @@ class TestGenerationWindowsTileTheTurn: carrying no tool at all (the Write inside them took 7 ms), attributed to nothing — 24% of the turn, on its own enough to hold OpenCode above the evalboard's 25% "Unaccounted" red threshold. - - Driven at the reducer for the same reason as the sibling class above: the - window is two `datetime.now()` reads, so only setting them explicitly - makes the arithmetic deterministic. """ - T0 = datetime(2026, 1, 1, 12, 0, 0) - - @staticmethod - def _finish_at(monkeypatch, state, *, step_start, now): - class _Clock(datetime): - @staticmethod - def now(tz=None): - return now - - state.step_started_at = step_start - monkeypatch.setattr(agent_module, "datetime", _Clock) - state.on_step_finish({"reason": "stop", "tokens": {"input": 100, "output": 20}}) - - def _two_steps(self, monkeypatch): - state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="do it", model="deepseek/deepseek-v4-pro") - # Step 1 runs T0 -> T0+1000. - self._finish_at(monkeypatch, state, step_start=self.T0, now=self.T0 + timedelta(milliseconds=1000)) - # 800ms of model time, then a step the CLI only announces at T0+1800. - self._finish_at( - monkeypatch, - state, - step_start=self.T0 + timedelta(milliseconds=1800), - now=self.T0 + timedelta(milliseconds=2000), + def _two_steps(self) -> list[AssistantMessage]: + # Step 1 runs 0 -> 1000; then 800ms of model time, then a step the CLI only announces at 1800. + result, _ = _replay( + [ + _event("step_start", {"messageID": "m1"}, at_ms=0), + _finish(1000), + _event("step_start", {"messageID": "m2"}, at_ms=1800), + _finish(2000), + ] ) - return [m for m in state.messages if m.role == "assistant"] + return _assistants(result) - def test_the_gap_before_a_step_is_its_generation_time(self, monkeypatch): - first, second = self._two_steps(monkeypatch) + def test_the_gap_before_a_step_is_its_generation_time(self): + first, second = self._two_steps() # Bounded by its own step_start, this window was 200ms and the 800ms # that produced it was attributed to nothing. assert second.generation_duration_ms == pytest.approx(1000.0) assert second.started_at == first.completed_at - def test_the_first_step_keeps_its_own_start(self, monkeypatch): + def test_the_first_step_keeps_its_own_start(self): """Everything before the first `step_start` is CLI spawn, not model time. Tiling the first window back to the turn's start would report Node's boot — 3.1 s of OpenCode's measured head — as generation. """ - first, _ = self._two_steps(monkeypatch) - assert first.started_at == self.T0 + first, _ = self._two_steps() + assert first.started_at == _at(0) assert first.generation_duration_ms == pytest.approx(1000.0) - def test_the_steps_leave_no_gap_between_them(self, monkeypatch): - first, second = self._two_steps(monkeypatch) + def test_the_steps_leave_no_gap_between_them(self): + first, second = self._two_steps() covered = (second.completed_at - first.started_at).total_seconds() * 1000.0 gen = sum(m.generation_duration_ms or 0.0 for m in (first, second)) assert gen == pytest.approx(covered) -_SPAN_EPOCH_MS = 1_800_000_000_000 -_SPAN_BASE = datetime.fromtimestamp(_SPAN_EPOCH_MS / 1000) - - -class _SteppedClock(datetime): - """A clock the test moves by hand, in ms from `_SPAN_BASE`. - - Subclasses `datetime` rather than stubbing it, because `_epoch_ms_to_dt` - calls `datetime.fromtimestamp` through the same module global and must keep - resolving to the real implementation — the CLI's epoch stamps and the - reducer's own `now()` reads have to land on ONE timeline for the span - arithmetic under test to mean anything. - """ - - at_ms = 0.0 - - @staticmethod - def now(tz=None): - return _SPAN_BASE + timedelta(milliseconds=_SteppedClock.at_ms) - - class TestToolSpansSurviveTheStepBoundary: """A tool that closes BETWEEN two steps still belongs to the next window. - This used to be a bookkeeping problem: a per-step span list, cleared at - `step_start` — after the window it feeds had already opened at `gen_mark` — - so a call closing in the gap had its span wiped before the next - `step_finish` could subtract it. That list is gone. - `timing.subtract_tool_time` sees every span at once and clips each - to the windows it overlaps, so the property now holds by construction - rather than by a reset rule. Kept, and re-pointed at the collector, because - the property is what matters: a future reducer change could still break it - by moving a mark or failing to emit the ToolEnd the collector reduces. + `timing.subtract_tool_time` sees every span at once and clips each to the + windows it overlaps, so the property holds by construction rather than by a + reset rule. Kept because the property is what matters: a future decoder + change could still break it by moving a mark or failing to close the tool + the collector reduces. It needs the NON-TERMINAL tool path to reach: the CLI normally emits one already-`completed` event per call, which closes inside the step that opened it. That is why the measured corpus reads 0.00% and a reproduction - has to drive the state object. + has to script the stream. """ - def _run(self, monkeypatch): - monkeypatch.setattr(agent_module, "datetime", _SteppedClock) - state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="go", model="m") - # The resolved telemetry leaves the state via ToolEnd; the identity - # case below reconciles against what was RECORDED, not against the - # clock the test scripted. - resolved: list[Any] = [] - state.bind(lambda e: resolved.append(e.tool) if isinstance(e, ToolEndEvent) else None) - - def tool(status, *, end_ms=None): - times = {"start": _SPAN_EPOCH_MS + 100} - if end_ms is not None: - times["end"] = _SPAN_EPOCH_MS + end_ms - state.on_tool_use({"callID": "c1", "tool": "bash", "state": {"status": status, "time": times}}) - - _SteppedClock.at_ms = 0 - state.on_step_start({"messageID": "m1"}) - _SteppedClock.at_ms = 100 - tool("running") # non-terminal: stays open across the boundary - _SteppedClock.at_ms = 1000 - state.on_step_finish({"reason": "stop", "tokens": {"input": 10, "output": 5}}) - _SteppedClock.at_ms = 1500 - tool("completed", end_ms=1500) # closes in the GAP between the steps - _SteppedClock.at_ms = 1600 - state.on_step_start({"messageID": "m2"}) - _SteppedClock.at_ms = 2000 - state.on_step_finish({"reason": "stop", "tokens": {"input": 10, "output": 5}}) - - # Published through the real collector: the reducer hands over raw - # windows, and the tool subtraction happens once, there. - collector = EventCollector() - collector.on_event(AgentStartEvent(task_id="t1", prompt="go", iteration=1, timestamp=_SPAN_BASE)) - for command in resolved: - collector.on_event(ToolEndEvent(task_id="t1", turn_id="s1", tool=command)) - collector.on_event( - AgentEndEvent( - task_id="t1", - status=AgentEndStatus.COMPLETED, - messages=list(state.messages), - usage=TokenUsage(), - timestamp=_SPAN_BASE + timedelta(milliseconds=2000), - ) - ) - published = [m for m in collector.build_turn_record().messages if m.role == "assistant"] - return resolved, published - - def test_the_gap_slice_of_a_straddling_call_is_not_published_as_generation(self, monkeypatch): - _, messages = self._run(monkeypatch) + def _run(self) -> Replay: + return _replay( + [ + _event("step_start", {"messageID": "m1"}, at_ms=0), + _tool("c1", "running", at_ms=100, start=100), # non-terminal: stays open across the boundary + _finish(1000), + _tool("c1", "completed", at_ms=1500, start=100, end=1500), # closes in the GAP between the steps + _event("step_start", {"messageID": "m2"}, at_ms=1600), + _finish(2000), + Tick(2000), + ] + )[0] + + def test_the_gap_slice_of_a_straddling_call_is_not_published_as_generation(self): + messages = _assistants(self._run()) assert len(messages) == 2 # Window 2 tiles 1000 -> 2000. c1 ran for 1000 -> 1500 of it, so 500ms # is model time. Before the reset moved, this published 1000.0 — a 100% # overstatement, with c1's own duration_ms counting the same 500ms. assert messages[1].generation_duration_ms == pytest.approx(500.0) - def test_the_call_is_subtracted_from_exactly_one_window(self, monkeypatch): - # Window 1 owns c1's 100 -> 1000 slice (it was open at that boundary - # and bounded there); window 2 owns 1000 -> 1500. Neither owns both. - _, messages = self._run(monkeypatch) + def test_the_call_is_subtracted_from_exactly_one_window(self): + # Window 1 owns c1's 100 -> 1000 slice; window 2 owns 1000 -> 1500. Neither owns both. + messages = _assistants(self._run()) assert messages[0].generation_duration_ms == pytest.approx(100.0) assert messages[1].generation_duration_ms == pytest.approx(500.0) - def test_the_four_bucket_identity_closes_exactly_across_the_boundary(self, monkeypatch): + def test_the_four_bucket_identity_closes_exactly_across_the_boundary(self): """generation + UNION(tool) accounts for the whole span, to the ms. The assertion the golden corpus CANNOT make: `_scrub.py` masks `generation_duration_ms` and both bounds to a placeholder, so a snapshot records that a window was measured and never what it - measured, and its identity check is an upper bound besides — so - under-accounting, the defect this phase fixes, passes it silently. + measured. """ from coder_eval.timing import busy_ms - resolved, messages = self._run(monkeypatch) + result = self._run() + messages = _assistants(result) lo, hi = messages[0].started_at, messages[1].completed_at generation_ms = sum(m.generation_duration_ms or 0.0 for m in messages) - command = next(c for c in resolved if c.tool_id == "c1") + command = next(c for c in result.record.commands if c.tool_id == "c1") + assert command.execution_started_at is not None and command.execution_completed_at is not None tool_ms = busy_ms([(command.execution_started_at, command.execution_completed_at)], lo, hi) assert generation_ms + tool_ms == pytest.approx((hi - lo).total_seconds() * 1000.0) + assert_identity_closes(result.record, started_at=result.started_at, ended_at=result.ended_at) - def test_a_duplicate_step_finish_does_not_republish_the_previous_window(self, monkeypatch): + def test_a_duplicate_step_finish_does_not_republish_the_previous_window(self): """A spent `step_started_at` must not seed the next window. `close_window`'s `min(mark, item_start)` pulls the window open to cover - the item's own start. That is the backwards-clock defence — which this - reducer genuinely needs, since its stamps are raw `datetime.now()` and - not on a `TurnClock`. But a start stamp left in place after its step was - published is not a backwards clock: it is a stale value BEFORE the mark, - so the guard reopens the next window at the previous step's start and - publishes that whole span again. Reproduced on Pi's identical twin - before the fix: 3000 ms of generation for a 2000 ms turn. + the item's own start. A start stamp left in place after its step was + published is a stale value BEFORE the mark, so the guard would reopen the + next window at the previous step's start and publish that whole span + again. Reproduced on Pi's identical twin before the fix: 3000 ms of + generation for a 2000 ms turn. """ - monkeypatch.setattr(agent_module, "datetime", _SteppedClock) - state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="go", model="m") - _SteppedClock.at_ms = 0 - state.on_step_start({"messageID": "m1"}) - _SteppedClock.at_ms = 1000 - state.on_step_finish({"reason": "stop", "tokens": {"input": 10, "output": 5}}) - _SteppedClock.at_ms = 2000 - state.on_step_finish({"reason": "stop", "tokens": {"input": 10, "output": 5}}) - - messages = [m for m in state.messages if m.role == "assistant"] + result, _ = _replay( + [_event("step_start", {"messageID": "m1"}, at_ms=0), _finish(1000), _finish(2000)] + ) # no intervening `step_start` + + messages = _assistants(result) assert len(messages) == 2 assert messages[1].started_at == messages[0].completed_at assert sum(m.generation_duration_ms or 0.0 for m in messages) == pytest.approx(2000.0) - def test_a_step_that_never_finishes_does_not_advance_the_mark(self, monkeypatch): - """The half of this that is still the reducer's job. + def test_a_step_that_never_finishes_does_not_advance_the_mark(self): + """The half of this that is still the decoder's job. - There is no span list to preserve any more — the collector reduces the - ToolEnd stream itself. What the reducer still owns is the MARK: a step - that published nothing must not advance it, or its time is handed to + There is no span list to preserve — the collector reduces the ToolEnd + stream itself. What the decoder still owns is the MARK: a step that + published nothing must not advance it, or its time is handed to whichever step finishes next. """ - monkeypatch.setattr(agent_module, "datetime", _SteppedClock) - state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="go", model="m") - _SteppedClock.at_ms = 0 - state.on_step_start({"messageID": "m1"}) - _SteppedClock.at_ms = 1000 - state.on_step_finish({"reason": "stop", "tokens": {"input": 10, "output": 5}}) - mark_after_flush = state.gen_mark - - _SteppedClock.at_ms = 1600 - state.on_step_start({"messageID": "m2"}) - _SteppedClock.at_ms = 1700 - state.on_tool_use( - { - "callID": "c2", - "tool": "bash", - "state": {"status": "running", "time": {"start": _SPAN_EPOCH_MS + 1700}}, - } + result, decoder = _replay( + [ + _event("step_start", {"messageID": "m1"}, at_ms=0), + _finish(1000), + _event("step_start", {"messageID": "m2"}, at_ms=1600), + _tool("c2", "running", at_ms=1700, start=1700), + Tick(1900), + ], + status=AgentEndStatus.CRASHED, + reason="turn cancelled", ) - _SteppedClock.at_ms = 1900 - state.close_open_tools() # crash/timeout orphan sweep — no message appended - assert state.gen_mark == mark_after_flush + assert decoder.gen_mark == _at(1000) + assert len(_assistants(result)) == 1 + assert result.outcome.status is AgentEndStatus.CRASHED diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index d0bac98c..a2abf2f2 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -669,9 +669,9 @@ async def test_no_exit_before_the_deadline_is_a_timeout(self, patch_exec, tmp_pa assert [e.status for e in recorder.events if isinstance(e, AgentEndEvent)] == [AgentEndStatus.TIMEOUT] async def test_no_exit_without_a_deadline_is_a_crash(self, patch_exec, tmp_path, monkeypatch): - from coder_eval.agents import pi_agent + from coder_eval.agents._transport import subprocess_jsonl - monkeypatch.setattr(pi_agent, "_TERM_GRACE_SECONDS", 0.1) + monkeypatch.setattr(subprocess_jsonl, "_TERM_GRACE_SECONDS", 0.1) proc = _EofButAliveProcess([_turn_start()]) patch_exec(proc) recorder = _EventRecorder() diff --git a/tests/test_spi.py b/tests/test_spi.py index bfcf8f95..81cab0fb 100644 --- a/tests/test_spi.py +++ b/tests/test_spi.py @@ -7,6 +7,7 @@ _ORIGINS = ( "coder_eval.agent", + "coder_eval.agents._transport", "coder_eval.agents.registry", "coder_eval.agents.watchdog", "coder_eval.errors", @@ -26,7 +27,7 @@ def test_spi_version_is_three() -> None: def test_the_emitter_surface_is_exported() -> None: assert {"TurnEmitter", "TurnOutcome", "Generation", "Window", "TimingBasis"} <= set(spi.__all__) - assert {"run_with_watchdog", "WatchdogFired"} <= set(spi.__all__) + assert {"run_with_watchdog", "WatchdogFired", "SubprocessJsonlAgent", "JsonlDecoder"} <= set(spi.__all__) def test_the_stop_channel_is_exported() -> None: diff --git a/tests/test_subprocess_jsonl_agent.py b/tests/test_subprocess_jsonl_agent.py new file mode 100644 index 00000000..f443fda1 --- /dev/null +++ b/tests/test_subprocess_jsonl_agent.py @@ -0,0 +1,198 @@ +"""``SubprocessJsonlAgent``: the shared nd-JSON CLI transport, against real child processes.""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +from typing import Any + +import pytest + +from coder_eval.agents._transport import JsonlDecoder, SubprocessJsonlAgent, subprocess_jsonl +from coder_eval.models import Enforcement, HarnessContract, PiAgentConfig, TimingBasis, UsageGranularity +from coder_eval.streaming.emitter import TurnOutcome +from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus, StopReason, StreamEvent, end_status_for + + +pytestmark = pytest.mark.skipif(os.name != "posix", reason="process-group teardown is POSIX-only") + + +class _Decoder(JsonlDecoder): + def __call__(self, event: dict[str, Any]) -> None: + if event.get("type") == "say": + self.emitter.text(str(event.get("text", ""))) + elif event.get("type") == "err": + self.error = str(event.get("message")) + + def end(self, status: AgentEndStatus, *, reason: str | None = None) -> TurnOutcome: + if status is AgentEndStatus.CRASHED or status is AgentEndStatus.TIMEOUT: + return self.emitter.fail(status, reason or status.value) + return self.emitter.finalize(status) + + +class _ScriptAgent(SubprocessJsonlAgent[PiAgentConfig]): + """Runs ``python -c