From cb31e8e165a91e748cd04825d87570abc46dc2d3 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 07:50:24 -0700 Subject: [PATCH 01/12] =?UTF-8?q?feat(agents):=201/6=20=E2=80=94=20declare?= =?UTF-8?q?=20a=20HarnessContract=20on=20every=20agent=20and=20validate=20?= =?UTF-8?q?registrations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the supports_cooperative_stop and system_prompt_semantics ClassVars with one frozen HarnessContract per agent class. AgentRegistry.register now rejects an agent without a contract, and a config class that is not a forbid-extra BaseAgentConfig whose type Literal names the kind. The base environment info records the contract. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 2 + .claude/notes/reporting.md | 9 +- docs/EXTENDING.md | 4 +- docs/REPORT_SCHEMA.md | 4 + docs/agents/CLAUDE_CODE.md | 2 +- src/coder_eval/agent.py | 29 ++--- src/coder_eval/agents/antigravity_agent.py | 19 ++- src/coder_eval/agents/claude_code_agent.py | 14 +- src/coder_eval/agents/codex_agent.py | 21 ++- src/coder_eval/agents/noop_agent.py | 11 +- src/coder_eval/agents/opencode_agent.py | 16 ++- src/coder_eval/agents/pi_agent.py | 19 ++- src/coder_eval/agents/registry.py | 38 +++++- src/coder_eval/models/__init__.py | 6 + src/coder_eval/models/harness_contract.py | 49 +++++++ src/coder_eval/orchestration/early_stop.py | 6 +- src/coder_eval/reports/helpers.py | 2 +- tests/fixtures/harness_stubs.py | 27 ++++ tests/fixtures/mock_agent.py | 3 + tests/fixtures/text_stub_agent.py | 3 + .../rules/ce046_env_info_spreads_super.py | 16 ++- tests/test_agent.py | 9 +- tests/test_agent_config_registry_dispatch.py | 3 + tests/test_agentless.py | 2 + tests/test_codex_agent.py | 6 +- tests/test_custom_lint.py | 9 +- tests/test_early_stop.py | 25 ++-- tests/test_harness_contract.py | 122 ++++++++++++++++++ tests/test_opencode_agent.py | 3 +- tests/test_pi_agent.py | 2 +- tests/test_plugins.py | 32 +++-- tests/test_registry.py | 7 +- tests/test_simulation_integration.py | 3 + tests/test_visible_turn_cap.py | 2 +- 34 files changed, 427 insertions(+), 98 deletions(-) create mode 100644 src/coder_eval/models/harness_contract.py create mode 100644 tests/fixtures/harness_stubs.py create mode 100644 tests/test_harness_contract.py diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 7265cf9a4..277b31398 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -599,6 +599,8 @@ field. **A trend dashboard must not pool scores across that boundary**, and an a marker reads as a pre-marker run — which is why every adapter spreads the base `get_environment_info()` first rather than emitting the marker conditionally (CE046). +The class default is the `system_prompt_semantics` field of the agent's `HarnessContract` +(the base emits `"unknown"` when the contract marks `system_prompt` unsupported). claude-code's is the only one derived per config rather than fixed, so it is computed from the resolved prompt value and never recomputed independently — the persisted regime cannot disagree with what was sent. diff --git a/.claude/notes/reporting.md b/.claude/notes/reporting.md index a694d49a8..2afca5a04 100644 --- a/.claude/notes/reporting.md +++ b/.claude/notes/reporting.md @@ -82,10 +82,11 @@ right after the counter bump at the top of `communicate()` and consumed by when partial-record assembly leaves `pending_turn` at None. That is why rollback is the caller's move, not the agent's: only the caller knows a turn failed. -Capability flags are declared rather than probed. `supports_cooperative_stop` gates -arming early-stop, so arming it on an agent that ignores `should_stop` is rejected at -resolution rather than silently never firing. `supports_cost_log_tags` and -`system_prompt_semantics` are declared for reasons of their own — see +Capabilities are declared rather than probed, on the agent's `HarnessContract`. +`contract.cooperative_stop` gates arming early-stop, so arming it on an agent that ignores +`should_stop` is rejected at resolution rather than silently never firing. +`supports_cost_log_tags` and `contract.system_prompt_semantics` are declared for reasons of +their own — see [agents.md](agents.md) § Why the constructors declare every kwarg and § The system_prompt_semantics marker. diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 401380d96..0aeacd682 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -108,8 +108,8 @@ turn, and `ToolStart`/`ToolEnd` per tool call (close orphaned tools with `status=unresolved`). Fan events through an internal `EventCollector` — it builds the returned `TurnRecord`, the single agent-agnostic capture path. -Set `supports_cooperative_stop: ClassVar[bool] = True` only if your `communicate()` -actually honors `should_stop` (needed for criterion-level `stop_early:` arming). Leaving it +Set `cooperative_stop=True` in your agent's `contract` only if your `communicate()` +actually honors `should_stop` (needed for criterion-level `stop_early:` arming). Setting it `False` means early stop is rejected at resolution for your agent — which is correct if you can't stop cooperatively. diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index b232a3d54..c50c6122d 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -153,6 +153,10 @@ score-comparable, so consumers should segment on it (absent ⇒ pre-append regime; `"unknown"` ⇒ current run, undeclared agent). Codex runs before the marker dropped `system_prompt` entirely and Antigravity always appended, so for those two the boundary is a reporting change, not a behavioral one. +`environment_info.harness_contract` is the agent class's declared contract: for +each of `system_prompt`, `plugin_skills`, `permission_mode`, `allowed_tools` and +`disallowed_tools`, `"enforced"` or `"unsupported"`, plus +`system_prompt_semantics` (the class default) and `cooperative_stop`. `sdk_options.system_prompt` is a `SystemPromptPreset` dict (`{type: "preset", preset: "claude_code", exclude_dynamic_sections: true, append?: str}`) on append-mode Claude Code runs and a plain string only in replace mode — it is diff --git a/docs/agents/CLAUDE_CODE.md b/docs/agents/CLAUDE_CODE.md index 84e2146c8..1c95197ac 100644 --- a/docs/agents/CLAUDE_CODE.md +++ b/docs/agents/CLAUDE_CODE.md @@ -205,7 +205,7 @@ carries a `stop_early:` block, a single-shot run ends cleanly at the next tool-call boundary once its **armed** criteria (those carrying a `stop_early:` block) are decided — so a raised `max_turns` isn't wasted on a smoke run. Early stop errors at -resolution for any agent that does not declare `supports_cooperative_stop`. See the +resolution for any agent whose contract does not declare `cooperative_stop`. See the [Task Definition Guide](../TASK_DEFINITION_GUIDE.md) for the full contract. ## Telemetry diff --git a/src/coder_eval/agent.py b/src/coder_eval/agent.py index 8fb5f3cec..64de10fdb 100644 --- a/src/coder_eval/agent.py +++ b/src/coder_eval/agent.py @@ -11,7 +11,7 @@ from .errors import AgentCrashError, TurnTimeoutError from .errors.agent import format_timeout_reason, truncate_crash_message from .models import AgentState as AgentState -from .models import BaseAgentConfig, SystemPromptSemantics, TurnRecord +from .models import BaseAgentConfig, HarnessContract, TurnRecord from .streaming.callbacks import StreamCallback from .streaming.collector import EventCollector from .streaming.events import AgentEndStatus @@ -70,22 +70,15 @@ def __init__(self, config: ClaudeCodeAgentConfig, ...): _iteration: int = 0 _iteration_was_incremented: bool = False - # Whether this agent honors the cooperative ``should_stop`` interrupt. Default - # False: arming early-stop on an agent that does not set it True is rejected at - # resolution rather than silently never firing. - supports_cooperative_stop: ClassVar[bool] = False - # Whether this agent's constructor accepts ``cost_log_tags``. The agent-agnostic # factory must only forward it to agents that set this True, or a route-driven # kwarg crashes every agent whose ``__init__`` lacks it. supports_cost_log_tags: ClassVar[bool] = False - # How this agent combines a configured ``system_prompt`` with its own default, - # recorded per run. Declared on the BASE so the marker is present on every run - # and "absent" reads as one thing only. An agent whose regime depends on its - # config overrides ``get_environment_info`` and emits the resolved value. + # Which uniform config fields this harness honors. No default: registration + # rejects a class that does not declare one. # Rationale: .claude/notes/agents.md § The system_prompt_semantics marker - system_prompt_semantics: ClassVar[SystemPromptSemantics] = "unknown" + contract: ClassVar[HarnessContract] def _begin_turn(self) -> None: """Mark the start of a ``communicate()`` turn: reset the pending slot and @@ -224,7 +217,7 @@ async def communicate( returned ``TurnRecord`` has ``max_turns_exhausted=True``. None defers to the underlying SDK default. should_stop: Cooperative early-stop poll. An implementation with - ``supports_cooperative_stop=True`` calls it at each safe message + ``contract.cooperative_stop`` calls it at each safe message boundary and, when it returns True, stops pulling further work and finalizes the turn cleanly (``crashed=False``, no raise). Agents that do not support it accept and ignore the argument. @@ -322,12 +315,16 @@ def get_environment_info(self) -> dict[str, Any]: operators. The orchestrator merges this into ``environment_info`` after the agent starts. - The base emits ``system_prompt_semantics`` (from the ClassVar of the same - name) so every agent — including out-of-tree SPI agents — records the - regime. Overrides should spread ``super().get_environment_info()`` rather + The base emits ``system_prompt_semantics`` (the contract's class default, or + ``"unknown"`` when the harness does not honor a system prompt) and the + ``harness_contract`` itself, so every agent — including out-of-tree SPI + agents — records both. Overrides should spread ``super().get_environment_info()`` rather than returning a bare dict, or that guarantee is lost for that agent. Returns: A flat dict of JSON-serializable keys to merge. """ - return {"system_prompt_semantics": self.system_prompt_semantics} + return { + "system_prompt_semantics": self.contract.system_prompt_semantics or "unknown", + "harness_contract": self.contract.model_dump(mode="json"), + } diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 1923c4742..4ff9641fd 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -25,7 +25,7 @@ from contextlib import AsyncExitStack from datetime import datetime from pathlib import Path -from typing import Any, ClassVar +from typing import Any from coder_eval.agent import Agent, AgentState from coder_eval.agents._logging import PrefixedAdapter @@ -45,7 +45,8 @@ CommandTelemetry, ContentBlock, DirectRoute, - SystemPromptSemantics, + Enforcement, + HarnessContract, TokenUsage, TranscriptMessage, TurnRecord, @@ -181,13 +182,19 @@ def _to_token_usage(usage: Any, model: str | None) -> TokenUsage: class AntigravityAgent(Agent[AntigravityAgentConfig]): """Implementation of the Agent interface for Google Antigravity (Gemini).""" - # The step loop has a between-steps guard where `should_stop` runs. - supports_cooperative_stop: ClassVar[bool] = True - + # The step loop has a between-steps guard where `should_stop` runs; # TemplatedSystemInstructions wraps system_instructions around the harness's # own prompt, and always has — so runs ARE comparable across the marker. # Rationale: .claude/notes/agents.md § The system_prompt_semantics marker - system_prompt_semantics: ClassVar[SystemPromptSemantics] = "append" + contract = HarnessContract( + system_prompt=Enforcement.ENFORCED, + system_prompt_semantics="append", + plugin_skills=Enforcement.ENFORCED, + permission_mode=Enforcement.UNSUPPORTED, + allowed_tools=Enforcement.UNSUPPORTED, + disallowed_tools=Enforcement.UNSUPPORTED, + cooperative_stop=True, + ) def __init__( self, diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index bd1fb002e..5b4fd1639 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -50,6 +50,8 @@ CommandTelemetry, ContentBlock, DirectRoute, + Enforcement, + HarnessContract, LiteLLMRoute, ResultSummary, SystemPromptSemantics, @@ -691,7 +693,15 @@ class ClaudeCodeAgent(Agent[ClaudeCodeAgentConfig]): """Implementation of the Agent interface for Claude Code using the SDK.""" # The message loop has a between-messages guard where `should_stop` runs. - supports_cooperative_stop: ClassVar[bool] = True + contract = HarnessContract( + system_prompt=Enforcement.ENFORCED, + system_prompt_semantics="append", + plugin_skills=Enforcement.ENFORCED, + permission_mode=Enforcement.ENFORCED, + allowed_tools=Enforcement.ENFORCED, + disallowed_tools=Enforcement.ENFORCED, + cooperative_stop=True, + ) # __init__ accepts cost_log_tags and stamps them into ANTHROPIC_CUSTOM_HEADERS # for the proxy-side actual-cost join (LiteLLM backend). @@ -1253,7 +1263,7 @@ def get_environment_info(self) -> dict[str, Any]: ``append`` = the claude_code preset with the configured prompt appended; ``replace`` = the configured prompt IS the entire system prompt (judge sub-agents). Unlike the other agents this is per-config, not fixed, so it - overrides the base ClassVar with the resolved value. + overrides the contract's class default with the resolved value. Rationale: .claude/notes/agents.md § The system_prompt_semantics marker """ diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 5611038ef..1a8b239b0 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -12,7 +12,7 @@ from collections.abc import Callable from datetime import datetime from pathlib import Path -from typing import Any, ClassVar, NamedTuple +from typing import Any, NamedTuple from urllib.parse import urlparse from coder_eval.agent import Agent, AgentState @@ -33,7 +33,8 @@ CommandTelemetry, ContentBlock, DirectRoute, - SystemPromptSemantics, + Enforcement, + HarnessContract, TokenUsage, TranscriptMessage, TurnRecord, @@ -761,12 +762,18 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso class CodexAgent(Agent[CodexAgentConfig]): """Implementation of the Agent interface for OpenAI Codex using the Codex SDK.""" - # The pump has a between-items guard where `should_stop` runs. - supports_cooperative_stop: ClassVar[bool] = True - - # `system_prompt` maps to developer_instructions, ON TOP of the base prompt. + # The pump has a between-items guard where `should_stop` runs; `system_prompt` + # maps to developer_instructions, ON TOP of the base prompt. # Rationale: .claude/notes/agents.md § The system_prompt_semantics marker - system_prompt_semantics: ClassVar[SystemPromptSemantics] = "append" + contract = HarnessContract( + system_prompt=Enforcement.ENFORCED, + system_prompt_semantics="append", + plugin_skills=Enforcement.ENFORCED, + permission_mode=Enforcement.UNSUPPORTED, + allowed_tools=Enforcement.UNSUPPORTED, + disallowed_tools=Enforcement.UNSUPPORTED, + cooperative_stop=True, + ) def __init__( self, diff --git a/src/coder_eval/agents/noop_agent.py b/src/coder_eval/agents/noop_agent.py index e39c45dff..f97088a04 100644 --- a/src/coder_eval/agents/noop_agent.py +++ b/src/coder_eval/agents/noop_agent.py @@ -19,7 +19,7 @@ from coder_eval.agent import Agent, AgentState from coder_eval.agents.registry import AgentRegistry -from coder_eval.models import AgentKind, ApiRoute, NoneAgentConfig, TurnRecord +from coder_eval.models import AgentKind, ApiRoute, Enforcement, HarnessContract, NoneAgentConfig, TurnRecord from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( @@ -44,6 +44,15 @@ class NoOpAgent(Agent[NoneAgentConfig]): from it. """ + contract = HarnessContract( + system_prompt=Enforcement.UNSUPPORTED, + plugin_skills=Enforcement.UNSUPPORTED, + permission_mode=Enforcement.UNSUPPORTED, + allowed_tools=Enforcement.UNSUPPORTED, + disallowed_tools=Enforcement.UNSUPPORTED, + cooperative_stop=False, + ) + def __init__(self, config: NoneAgentConfig, route: ApiRoute | None = None) -> None: self.config = config self.route = route diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index 399fda793..206776c29 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -27,7 +27,7 @@ import time from collections.abc import Callable from datetime import datetime -from typing import Any, ClassVar, Literal, NoReturn +from typing import Any, Literal, NoReturn from coder_eval.agent import Agent from coder_eval.errors import AgentCrashError, TurnTimeoutError @@ -39,10 +39,11 @@ AssistantMessage, CommandTelemetry, ContentBlock, + Enforcement, + HarnessContract, OpenCodeAgentConfig, PermissionMode, ResultSummary, - SystemPromptSemantics, TokenUsage, TranscriptMessage, TurnRecord, @@ -731,11 +732,16 @@ class OpenCodeAgent(Agent[OpenCodeAgentConfig]): """Runs the ``opencode`` CLI as a subprocess, one invocation per turn.""" # `should_stop` is polled at every event boundary (tool-call granularity). - supports_cooperative_stop: ClassVar[bool] = True - # No CLI knob for `system_prompt`, so the honest regime is `"unknown"`. # Rationale: .claude/notes/agents.md § The system_prompt_semantics marker - system_prompt_semantics: ClassVar[SystemPromptSemantics] = "unknown" + contract = HarnessContract( + system_prompt=Enforcement.UNSUPPORTED, + plugin_skills=Enforcement.ENFORCED, + permission_mode=Enforcement.UNSUPPORTED, + allowed_tools=Enforcement.UNSUPPORTED, + disallowed_tools=Enforcement.UNSUPPORTED, + cooperative_stop=True, + ) def __init__( self, diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 0f9814266..535b59b35 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -35,7 +35,7 @@ import time from collections.abc import Callable from datetime import datetime -from typing import Any, ClassVar, Literal, NoReturn +from typing import Any, Literal, NoReturn from uuid import uuid4 from coder_eval.agent import Agent @@ -49,9 +49,10 @@ AssistantMessage, CommandTelemetry, ContentBlock, + Enforcement, + HarnessContract, PiAgentConfig, ResultSummary, - SystemPromptSemantics, TokenUsage, TranscriptMessage, TurnRecord, @@ -679,12 +680,18 @@ def finalize( class PiAgent(Agent[PiAgentConfig]): """Runs the ``pi`` CLI as a subprocess, one invocation per turn.""" - # `should_stop` is polled at every event boundary (tool-call granularity). - supports_cooperative_stop: ClassVar[bool] = True - + # `should_stop` is polled at every event boundary (tool-call granularity); # `--append-system-prompt` appends to, never replaces, the CLI's own prompt. # Rationale: .claude/notes/agents.md § The system_prompt_semantics marker - system_prompt_semantics: ClassVar[SystemPromptSemantics] = "append" + contract = HarnessContract( + system_prompt=Enforcement.ENFORCED, + system_prompt_semantics="append", + plugin_skills=Enforcement.ENFORCED, + permission_mode=Enforcement.UNSUPPORTED, + allowed_tools=Enforcement.UNSUPPORTED, + disallowed_tools=Enforcement.UNSUPPORTED, + cooperative_stop=True, + ) def __init__( self, diff --git a/src/coder_eval/agents/registry.py b/src/coder_eval/agents/registry.py index 5770e5266..136f80e72 100644 --- a/src/coder_eval/agents/registry.py +++ b/src/coder_eval/agents/registry.py @@ -7,11 +7,11 @@ from collections.abc import Callable from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast +from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast, get_args # TYPE_CHECKING-only imports, so this module imports nothing from coder_eval at -# runtime and the dependency edge stays one-way (CodeQL py/cyclic-import). +# module load and the dependency edge stays one-way (CodeQL py/cyclic-import). # Rationale: .claude/notes/agents.md § Why the registry rejects a re-registration if TYPE_CHECKING: from coder_eval.agent import Agent @@ -21,6 +21,39 @@ AgentClassT = TypeVar("AgentClassT") +def _validate_registration(kind: str, agent_cls: type, config_class: type) -> None: + """Reject an ``(agent class, config class)`` pair the resolver cannot trust. + + Raises: + TypeError: the agent class declares no ``HarnessContract``, or the config + class is not a ``BaseAgentConfig`` with ``extra="forbid"`` whose ``type`` + Literal names ``kind``. + """ + from coder_eval.models import BaseAgentConfig, HarnessContract + + agent_name = agent_cls.__name__ + config_name = config_class.__name__ + if not isinstance(getattr(agent_cls, "contract", None), HarnessContract): + raise TypeError( + f"Agent kind {kind!r}: {agent_name} must declare `contract = HarnessContract(...)` " + + "as a class attribute, so the resolver knows which agent fields the harness honors." + ) + if not issubclass(config_class, BaseAgentConfig): + raise TypeError(f"Agent kind {kind!r}: config class {config_name} must subclass BaseAgentConfig.") + if config_class.model_config.get("extra") != "forbid": + raise TypeError( + f"Agent kind {kind!r}: config class {config_name} must keep extra='forbid', " + + "so an unknown agent key in YAML is an error rather than silently dropped." + ) + type_field = config_class.model_fields.get("type") + literal_kinds = {str(arg) for arg in get_args(type_field.annotation)} if type_field is not None else set() + if kind not in literal_kinds: + raise TypeError( + f"Agent kind {kind!r}: config class {config_name} must declare " + + f"`type: Literal[{kind!r}]` (its `type` annotation admits {sorted(literal_kinds)})." + ) + + @dataclass class AgentRegistration[ConfigT: BaseAgentConfig]: """Metadata for a registered agent. @@ -64,6 +97,7 @@ class ClaudeCodeAgent(Agent[ClaudeCodeAgentConfig]): def decorator(agent_cls: type[AgentClassT]) -> type[AgentClassT]: kind = str(agent_kind) + _validate_registration(kind, agent_cls, config_class) existing = cls._registry.get(kind) # Re-registering the SAME classes is legitimate (an idempotent # built-in reload); a DIFFERENT implementation for the same kind is a diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index a3aa4cf8d..2f485e5c7 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -91,6 +91,9 @@ VariantResult, ) +# Harness contract +from coder_eval.models.harness_contract import Enforcement, HarnessContract + # Judge from coder_eval.models.judge import JudgeVerdict @@ -244,6 +247,9 @@ "SystemPromptMode", "SystemPromptSemantics", "parse_agent_config", + # Harness contract + "Enforcement", + "HarnessContract", # Enums "AgentKind", "AgentState", diff --git a/src/coder_eval/models/harness_contract.py b/src/coder_eval/models/harness_contract.py new file mode 100644 index 000000000..6b3917ae8 --- /dev/null +++ b/src/coder_eval/models/harness_contract.py @@ -0,0 +1,49 @@ +"""What an agent harness honors of the uniform ``BaseAgentConfig`` fields.""" + +from __future__ import annotations + +from enum import StrEnum +from typing import Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from coder_eval.models.agent_config import SystemPromptMode + + +class Enforcement(StrEnum): + """Whether a harness honors a uniform agent field.""" + + ENFORCED = "enforced" + UNSUPPORTED = "unsupported" + + +class HarnessContract(BaseModel): + """The per-agent declaration of which uniform fields reach the harness. + + A task that sets a field this contract marks ``UNSUPPORTED`` is rejected at + resolution. Every registered agent class declares one. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + system_prompt: Enforcement = Field(description="Whether agent.system_prompt reaches the harness.") + system_prompt_semantics: SystemPromptMode | None = Field( + default=None, + description="How the prompt combines with the harness's own prompt. None iff system_prompt is unsupported.", + ) + plugin_skills: Enforcement = Field(description="Whether the skills of agent.plugins reach the harness.") + permission_mode: Enforcement = Field(description="Whether agent.permission_mode is honored.") + allowed_tools: Enforcement = Field(description="Whether agent.allowed_tools restricts the harness's tools.") + disallowed_tools: Enforcement = Field(description="Whether agent.disallowed_tools denies the harness's tools.") + cooperative_stop: bool = Field(description="Whether communicate() honors the should_stop poll.") + + @model_validator(mode="after") + def check_semantics_matches_prompt_support(self) -> Self: + """Require a semantics value exactly when the system prompt is enforced.""" + if (self.system_prompt is Enforcement.ENFORCED) != (self.system_prompt_semantics is not None): + raise ValueError( + "system_prompt_semantics must be set when system_prompt is 'enforced' and must be None " + + f"when it is 'unsupported' (got system_prompt={self.system_prompt.value!r}, " + + f"system_prompt_semantics={self.system_prompt_semantics!r})" + ) + return self diff --git a/src/coder_eval/orchestration/early_stop.py b/src/coder_eval/orchestration/early_stop.py index c3f23e46a..136231237 100644 --- a/src/coder_eval/orchestration/early_stop.py +++ b/src/coder_eval/orchestration/early_stop.py @@ -93,7 +93,7 @@ def validate_early_stop(task: TaskDefinition) -> None: 1. ``run_limits.stop_early: true`` (master arm removed) 2. armed together with ``simulation.enabled`` - 3. agent does not declare ``supports_cooperative_stop`` + 3. agent's contract does not declare ``cooperative_stop`` 4. degenerate ``stop_early_gate_threshold`` (``<= 0.0``) There are deliberately NO per-instance polarity guards, and no armed-but-empty @@ -153,11 +153,11 @@ def validate_early_stop(task: TaskDefinition) -> None: + "not registered (is the providing plugin installed and loaded?). " + "Disarm with run_limits.stop_early: false to bypass this check." ) - if not registration.agent_class.supports_cooperative_stop: + if not registration.agent_class.contract.cooperative_stop: supporting = ", ".join( kind for kind in AgentRegistry.list_kinds() - if (reg := AgentRegistry.get(kind)) is not None and reg.agent_class.supports_cooperative_stop + if (reg := AgentRegistry.get(kind)) is not None and reg.agent_class.contract.cooperative_stop ) raise EarlyStopConfigError( "criterion-level stop_early arming requires an agent that supports cooperative stopping " diff --git a/src/coder_eval/reports/helpers.py b/src/coder_eval/reports/helpers.py index 14f944afa..b923cd5ed 100644 --- a/src/coder_eval/reports/helpers.py +++ b/src/coder_eval/reports/helpers.py @@ -78,7 +78,7 @@ class VariantSeries(NamedTuple): # bookkeeping the reader did not ask for — `command_base_path` is a full PATH # string on every row, and the graded_by_* provenance keys only appear on a # re-graded row where they would read as facts about the run itself. -ENV_TABLE_EXCLUDE = frozenset({"installed_tools", "command_base_path", "reference_digest"}) +ENV_TABLE_EXCLUDE = frozenset({"installed_tools", "command_base_path", "reference_digest", "harness_contract"}) def is_env_table_key(key: str) -> bool: diff --git a/tests/fixtures/harness_stubs.py b/tests/fixtures/harness_stubs.py new file mode 100644 index 000000000..3b848e6f9 --- /dev/null +++ b/tests/fixtures/harness_stubs.py @@ -0,0 +1,27 @@ +"""Registration-valid building blocks for tests that register a throwaway agent kind.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import create_model + +from coder_eval.models import BaseAgentConfig, Enforcement, HarnessContract + + +def stub_contract(*, cooperative_stop: bool = True) -> HarnessContract: + """A contract that honors every uniform field.""" + return HarnessContract( + system_prompt=Enforcement.ENFORCED, + system_prompt_semantics="append", + plugin_skills=Enforcement.ENFORCED, + permission_mode=Enforcement.ENFORCED, + allowed_tools=Enforcement.ENFORCED, + disallowed_tools=Enforcement.ENFORCED, + cooperative_stop=cooperative_stop, + ) + + +def config_for_kind[C: BaseAgentConfig](kind: str, base: type[C] = BaseAgentConfig) -> type[C]: + """A ``base`` subclass whose ``type`` Literal names ``kind``, as registration requires.""" + return create_model(f"StubConfig_{kind.replace('-', '_')}", __base__=base, type=(Literal[kind], ...)) # type: ignore[valid-type] diff --git a/tests/fixtures/mock_agent.py b/tests/fixtures/mock_agent.py index 14ffbf0a8..c384a5e6d 100644 --- a/tests/fixtures/mock_agent.py +++ b/tests/fixtures/mock_agent.py @@ -9,6 +9,7 @@ from coder_eval.agent import Agent, AgentState from coder_eval.models import TaskDefinition, TurnRecord +from tests.fixtures.harness_stubs import stub_contract class MockAgent(Agent): @@ -24,6 +25,8 @@ class MockAgent(Agent): - "partial": Creates files but with incorrect content """ + contract = stub_contract() + def __init__(self, task: TaskDefinition, scenario: str = "success"): """Initialize mock agent with task definition. diff --git a/tests/fixtures/text_stub_agent.py b/tests/fixtures/text_stub_agent.py index e873fded6..b2fef05ac 100644 --- a/tests/fixtures/text_stub_agent.py +++ b/tests/fixtures/text_stub_agent.py @@ -11,11 +11,14 @@ from coder_eval.agent import Agent, AgentState from coder_eval.models import TurnRecord +from tests.fixtures.harness_stubs import stub_contract class TextStubAgent(Agent): """Canned-response Agent fake. Records every ``communicate`` prompt.""" + contract = stub_contract() + def __init__(self, responses: list[str]) -> None: self._responses = list(responses) self.calls: list[str] = [] diff --git a/tests/lint/rules/ce046_env_info_spreads_super.py b/tests/lint/rules/ce046_env_info_spreads_super.py index a1807a78c..15e09c133 100644 --- a/tests/lint/rules/ce046_env_info_spreads_super.py +++ b/tests/lint/rules/ce046_env_info_spreads_super.py @@ -1,7 +1,7 @@ """CE046: a ``get_environment_info`` override must spread the base result. ``Agent.get_environment_info`` (agent.py) emits the ``system_prompt_semantics`` -run marker from the ClassVar of the same name, so EVERY run — including +run marker from the agent's ``contract``, so EVERY run — including out-of-tree SPI agents — records which system-prompt regime built its prompts. Dashboards read an ABSENT marker as "a run from before the marker existed" and pool it into a legacy bucket, so an override that returns a bare dict does not @@ -20,8 +20,8 @@ body that neither * calls ``super().get_environment_info()`` (the override contract), nor - * references ``self.system_prompt_semantics`` (the base itself, which emits - the marker directly — exempt so the rule does not flag its own source). + * is the ``Agent`` base itself, which emits the marker from ``self.contract`` + directly — exempt so the rule does not flag its own source. ``# noqa: CE046`` if an agent genuinely must not record the marker (there is no such case today). @@ -49,12 +49,14 @@ def _spreads_super(node: ast.FunctionDef) -> bool: return False -def _emits_marker_directly(node: ast.FunctionDef) -> bool: - """True if the body reads ``self.system_prompt_semantics`` (the base itself).""" +def _emits_marker_directly(cls: ast.ClassDef, node: ast.FunctionDef) -> bool: + """True if this is the ``Agent`` base reading ``self.contract``.""" + if cls.name != "Agent": + return False for sub in ast.walk(node): if ( isinstance(sub, ast.Attribute) - and sub.attr == "system_prompt_semantics" + and sub.attr == "contract" and isinstance(sub.value, ast.Name) and sub.value.id == "self" ): @@ -71,7 +73,7 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None: isinstance(stmt, ast.FunctionDef) and stmt.name == _METHOD and not _spreads_super(stmt) - and not _emits_marker_directly(stmt) + and not _emits_marker_directly(node, stmt) ): self.violation( stmt, diff --git a/tests/test_agent.py b/tests/test_agent.py index 9636b1082..2b6cd8f25 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -473,12 +473,16 @@ def test_environment_info_reports_system_prompt_semantics(): trend dashboards can segment runs by prompt regime instead of pooling pre-/post-append-semantics scores.""" default_agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE)) - assert default_agent.get_environment_info() == {"system_prompt_semantics": "append"} + info = default_agent.get_environment_info() + assert info["system_prompt_semantics"] == "append" + assert info["harness_contract"] == ClaudeCodeAgent.contract.model_dump(mode="json") judge_like = ClaudeCodeAgent( parse_agent_config(type=AgentKind.CLAUDE_CODE, system_prompt="grader", system_prompt_mode="replace") ) - assert judge_like.get_environment_info() == {"system_prompt_semantics": "replace"} + judge_info = judge_like.get_environment_info() + assert judge_info["system_prompt_semantics"] == "replace" + assert judge_info["harness_contract"]["cooperative_stop"] is True def test_every_registered_agent_reports_prompt_semantics(): @@ -499,6 +503,7 @@ def get_environment_info(self) -> dict[str, object]: assert PluginAgent(parse_agent_config(type=AgentKind.NONE)).get_environment_info() == { "system_prompt_semantics": "unknown", + "harness_contract": NoOpAgent.contract.model_dump(mode="json"), "plugin_endpoint": "example.invalid", } diff --git a/tests/test_agent_config_registry_dispatch.py b/tests/test_agent_config_registry_dispatch.py index 145692666..316fb429e 100644 --- a/tests/test_agent_config_registry_dispatch.py +++ b/tests/test_agent_config_registry_dispatch.py @@ -20,6 +20,7 @@ parse_agent_config, ) from coder_eval.orchestration.config_merge import MergeError, validate_paths +from tests.fixtures.harness_stubs import stub_contract def _now(): @@ -146,6 +147,8 @@ def _registered_plugin_kind(): from coder_eval.agents.registry import AgentRegistry class _PluginAgent: + contract = stub_contract() + def __init__(self, config, route=None, **kwargs): self.config = config diff --git a/tests/test_agentless.py b/tests/test_agentless.py index 671086360..2001dcd66 100644 --- a/tests/test_agentless.py +++ b/tests/test_agentless.py @@ -293,6 +293,8 @@ async def test_run_executes_criteria_via_noop_agent(self, tmp_path: Path, monkey assert orch.result.agent_config is not None and orch.result.agent_config.type == AgentKind.NONE assert len(orch.result.success_criteria_results) == 3 assert all(r.score >= 0.9 for r in orch.result.success_criteria_results) + assert orch.result.environment_info is not None + assert orch.result.environment_info["harness_contract"] == NoOpAgent.contract.model_dump(mode="json") async def test_run_executes_pre_run(self, tmp_path: Path, monkeypatch) -> None: """pre_run still runs with the no-op agent — it's the only thing touching the sandbox. diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index 5f9654f05..fc8bcf794 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -338,7 +338,10 @@ def test_no_base_url_emits_only_prompt_semantics(self, monkeypatch): emitted (Codex appends system_prompt as developer_instructions).""" monkeypatch.delenv("CODEX_BASE_URL", raising=False) agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, model="gpt-5-codex")) - assert agent.get_environment_info() == {"system_prompt_semantics": "append"} + assert agent.get_environment_info() == { + "system_prompt_semantics": "append", + "harness_contract": CodexAgent.contract.model_dump(mode="json"), + } def test_azure_routing_recorded(self, monkeypatch): """Host (not full URL), wire_api, api-version, and the deployment-name marker @@ -350,6 +353,7 @@ def test_azure_routing_recorded(self, monkeypatch): info = agent.get_environment_info() assert info == { "system_prompt_semantics": "append", + "harness_contract": CodexAgent.contract.model_dump(mode="json"), "codex_base_url_host": "my-res.openai.azure.com", "codex_wire_api": "responses", "codex_api_version": "2025-04-01-preview", diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 4c1429104..7f6338d96 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -198,10 +198,17 @@ def test_allows_base_that_emits_marker_directly(self): # The base Agent.get_environment_info is the marker's source, not an override. src = ( "class Agent:\n def get_environment_info(self):\n" - " return {'system_prompt_semantics': self.system_prompt_semantics}" + " return {'system_prompt_semantics': self.contract.system_prompt_semantics}" ) assert not self._run(src) + def test_flags_override_that_reads_contract_without_spread(self): + src = ( + "class FooAgent:\n def get_environment_info(self):\n" + " return {'semantics': self.contract.system_prompt_semantics}" + ) + assert self._run(src) + def test_ignores_classes_without_the_method(self): assert not self._run("class FooAgent:\n def other(self):\n return {}") diff --git a/tests/test_early_stop.py b/tests/test_early_stop.py index 791098bd3..571277d73 100644 --- a/tests/test_early_stop.py +++ b/tests/test_early_stop.py @@ -47,7 +47,6 @@ from coder_eval.models import ( AgentKind, ApiBackend, - BaseAgentConfig, CommandExecutedCriterion, CommandTelemetry, CriterionResult, @@ -92,6 +91,7 @@ TurnStartEvent, ) from tests._fixtures.live_criteria import FROZEN_TS, make_command, make_turn +from tests.fixtures.harness_stubs import config_for_kind, stub_contract # --------------------------------------------------------------------------- # @@ -144,26 +144,22 @@ def _task( ) -class _DummyNoStopConfig(BaseAgentConfig): - """Config for the dummy non-supporting agent registered by the fixture below.""" - - class _DummyNoStopAgent: - """Agent stand-in that leaves ``supports_cooperative_stop`` at the default False. + """Agent stand-in whose contract declares ``cooperative_stop=False``. - ``validate_early_stop`` only reads the flag off the registered class, so no + ``validate_early_stop`` only reads the contract off the registered class, so no ``Agent`` machinery is needed. Guardrail 1 must keep rejecting agents that have not opted into the cooperative interrupt (all built-ins now support it). """ - supports_cooperative_stop = False + contract = stub_contract(cooperative_stop=False) @pytest.fixture def dummy_no_stop_kind() -> Iterator[str]: """Register a non-supporting agent kind for guardrail-1 tests, then clean up.""" kind = "dummy-no-stop" - AgentRegistry.register(kind, _DummyNoStopConfig)(_DummyNoStopAgent) + AgentRegistry.register(kind, config_for_kind(kind))(_DummyNoStopAgent) try: yield kind finally: @@ -792,11 +788,14 @@ def test_guardrail5_simulation_rejected(self) -> None: validate_early_stop(task) def test_guardrail1_non_supporting_agent_rejected(self, dummy_no_stop_kind: str) -> None: - # Codex/antigravity now support the cooperative interrupt, so guardrail 1 - # is exercised with a dummy agent that leaves the flag at False. + # Every built-in supports the cooperative interrupt, so guardrail 1 is + # exercised with a dummy agent whose contract declares cooperative_stop=False. task = _task(criteria=[_skill_crit("s", "s", stop_on_pass=True)], agent_type=dummy_no_stop_kind) - with pytest.raises(EarlyStopConfigError, match="cooperative stopping"): + with pytest.raises(EarlyStopConfigError, match="cooperative stopping") as exc: validate_early_stop(task) + supporting = str(exc.value).split("(", 1)[1].split(")", 1)[0] + assert "claude-code" in supporting + assert dummy_no_stop_kind not in supporting def test_guardrail3_agentless_task_rejected(self) -> None: # An armed task with no agent block at all: the diagnosis must point at @@ -809,7 +808,7 @@ def test_guardrail3_unregistered_agent_type_rejected(self) -> None: # An armed task whose agent type vanished from the registry (plugin not # installed/loaded) must fail with the plugin-pointing diagnosis. kind = "vanishing-agent" - AgentRegistry.register(kind, _DummyNoStopConfig)(_DummyNoStopAgent) + AgentRegistry.register(kind, config_for_kind(kind))(_DummyNoStopAgent) try: task = _task(criteria=[_skill_crit("s", "s", stop_on_pass=True)], agent_type=kind) finally: diff --git a/tests/test_harness_contract.py b/tests/test_harness_contract.py new file mode 100644 index 000000000..65bcf685d --- /dev/null +++ b/tests/test_harness_contract.py @@ -0,0 +1,122 @@ +"""The harness contract: the model, registration validation, and every built-in's declaration.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Literal + +import pytest +from pydantic import BaseModel, ConfigDict, ValidationError + +from coder_eval.agents.registry import AgentRegistry +from coder_eval.models import ( + AgentKind, + BaseAgentConfig, + ClaudeCodeAgentConfig, + Enforcement, + HarnessContract, +) +from coder_eval.plugins import ensure_plugins_loaded +from tests.fixtures.harness_stubs import config_for_kind, stub_contract + + +KIND = "contract-test-kind" + + +@pytest.fixture +def restored_registry() -> Iterator[None]: + saved = dict(AgentRegistry._registry) + try: + yield + finally: + AgentRegistry._registry.clear() + AgentRegistry._registry.update(saved) + + +class TestModel: + def test_enforced_prompt_requires_semantics(self) -> None: + with pytest.raises(ValidationError, match="system_prompt_semantics"): + HarnessContract(**{**stub_contract().model_dump(), "system_prompt_semantics": None}) + + def test_unsupported_prompt_rejects_semantics(self) -> None: + with pytest.raises(ValidationError, match="system_prompt_semantics"): + HarnessContract(**{**stub_contract().model_dump(), "system_prompt": Enforcement.UNSUPPORTED}) + + def test_unsupported_prompt_without_semantics_is_valid(self) -> None: + contract = HarnessContract( + **{**stub_contract().model_dump(), "system_prompt": "unsupported", "system_prompt_semantics": None} + ) + assert contract.system_prompt is Enforcement.UNSUPPORTED + + def test_contract_is_frozen(self) -> None: + contract = stub_contract() + with pytest.raises(ValidationError): + contract.cooperative_stop = False # type: ignore[misc] + + def test_unknown_field_rejected(self) -> None: + with pytest.raises(ValidationError, match="timing_basis"): + HarnessContract(**{**stub_contract().model_dump(), "timing_basis": "wall"}) + + +class _ContractAgent: + contract = stub_contract() + + +class TestRegistryValidation: + def test_missing_contract_rejected(self, restored_registry: None) -> None: + class NoContractAgent: + pass + + with pytest.raises(TypeError, match=rf"{KIND}.*NoContractAgent.*HarnessContract"): + AgentRegistry.register(KIND, config_for_kind(KIND))(NoContractAgent) + assert AgentRegistry.get(KIND) is None + + def test_dict_contract_rejected(self, restored_registry: None) -> None: + class DictContractAgent: + contract = stub_contract().model_dump() + + with pytest.raises(TypeError, match=rf"{KIND}.*DictContractAgent"): + AgentRegistry.register(KIND, config_for_kind(KIND))(DictContractAgent) + + def test_config_not_a_base_agent_config_rejected(self, restored_registry: None) -> None: + class ForeignConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + type: Literal["contract-test-kind"] + + with pytest.raises(TypeError, match=rf"{KIND}.*ForeignConfig.*BaseAgentConfig"): + AgentRegistry.register(KIND, ForeignConfig)(_ContractAgent) # type: ignore[type-var] + + def test_config_without_extra_forbid_rejected(self, restored_registry: None) -> None: + class LaxConfig(BaseAgentConfig): + model_config = ConfigDict(extra="ignore") + type: Literal["contract-test-kind"] # type: ignore[assignment] + + with pytest.raises(TypeError, match=rf"{KIND}.*LaxConfig.*extra='forbid'"): + AgentRegistry.register(KIND, LaxConfig)(_ContractAgent) + + def test_type_literal_not_naming_the_kind_rejected(self, restored_registry: None) -> None: + with pytest.raises(TypeError, match=rf"{KIND}.*ClaudeCodeAgentConfig.*Literal"): + AgentRegistry.register(KIND, ClaudeCodeAgentConfig)(_ContractAgent) + + def test_type_literal_covering_several_kinds_accepted(self, restored_registry: None) -> None: + class TwoKindConfig(BaseAgentConfig): + type: Literal["contract-test-kind", "other-kind"] # type: ignore[assignment] + + AgentRegistry.register(KIND, TwoKindConfig)(_ContractAgent) + AgentRegistry.register("other-kind", TwoKindConfig)(_ContractAgent) + assert AgentRegistry.get("other-kind") is not None + + def test_valid_pair_registers_idempotently(self, restored_registry: None) -> None: + config = config_for_kind(KIND) + AgentRegistry.register(KIND, config)(_ContractAgent) + AgentRegistry.register(KIND, config)(_ContractAgent) + registration = AgentRegistry.get(KIND) + assert registration is not None and registration.agent_class is _ContractAgent + + +@pytest.mark.parametrize("kind", [k for k in AgentKind if k is not AgentKind.UNKNOWN]) +def test_every_builtin_declares_a_contract(kind: AgentKind) -> None: + ensure_plugins_loaded() + registration = AgentRegistry.get(kind) + assert registration is not None + assert isinstance(registration.agent_class.contract, HarnessContract) diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 7b7b679ea..915368313 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -911,6 +911,7 @@ def test_carries_the_system_prompt_semantics_marker(self): prompt, so the honest value is `unknown`.""" info = _agent().get_environment_info() assert info["system_prompt_semantics"] == "unknown" + assert info["harness_contract"] == OpenCodeAgent.contract.model_dump(mode="json") assert info["opencode_model"] == "deepseek/deepseek-v4-pro" assert info["opencode_pure"] is True @@ -1358,7 +1359,7 @@ async def test_turn_completes_under_stderr_backpressure(self, patch_exec, tmp_pa class TestCooperativeStop: def test_capability_flag_is_declared(self): - assert OpenCodeAgent.supports_cooperative_stop is True + assert OpenCodeAgent.contract.cooperative_stop is True async def test_should_stop_ends_turn_cleanly(self, patch_exec, tmp_path): """A live subprocess must be torn down, and the turn must not be a crash.""" diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index f15e14457..e5e6e8f48 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -432,7 +432,7 @@ async def test_two_agent_cycles_reduce_to_one_agent_end(self, patch_exec, tmp_pa class TestCooperativeStop: def test_capability_flag_is_declared(self): - assert PiAgent.supports_cooperative_stop is True + assert PiAgent.contract.cooperative_stop is True async def test_should_stop_ends_turn_cleanly(self, patch_exec, tmp_path): proc = _RunningProcess(HAPPY_STREAM) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index d420cdf27..54a3f9070 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -7,6 +7,7 @@ import coder_eval.plugins as plugins from coder_eval.agents.registry import AgentRegistry from coder_eval.models import AgentKind, ClaudeCodeAgentConfig +from tests.fixtures.harness_stubs import config_for_kind, stub_contract PLUGIN_ENTRY_POINT_GROUP = plugins.PLUGIN_ENTRY_POINT_GROUP @@ -136,15 +137,16 @@ def test_registry_string_and_enum_keys_are_equivalent(): def test_register_accepts_raw_string_kind(): """Plugins register a kind that is not an AgentKind enum member.""" - class _Cfg(ClaudeCodeAgentConfig): - pass + cfg = config_for_kind("totally-custom-kind", ClaudeCodeAgentConfig) class _Agent: + contract = stub_contract() + def __init__(self, config, route=None, **kwargs): self.config = config try: - AgentRegistry.register("totally-custom-kind", _Cfg)(_Agent) + AgentRegistry.register("totally-custom-kind", cfg)(_Agent) reg = AgentRegistry.get("totally-custom-kind") assert reg is not None assert reg.agent_class is _Agent @@ -158,24 +160,25 @@ def test_register_rejects_conflicting_kind_collision(): last-write-win (which agent runs would then depend on entry-point discovery order — a reproducibility hole). The second registration raises.""" - class _CfgA(ClaudeCodeAgentConfig): - pass - - class _CfgB(ClaudeCodeAgentConfig): - pass + cfg_a = config_for_kind("collide-kind", ClaudeCodeAgentConfig) + cfg_b = config_for_kind("collide-kind", ClaudeCodeAgentConfig) class _AgentA: + contract = stub_contract() + def __init__(self, config, route=None, **kwargs): self.config = config class _AgentB: + contract = stub_contract() + def __init__(self, config, route=None, **kwargs): self.config = config try: - AgentRegistry.register("collide-kind", _CfgA)(_AgentA) + AgentRegistry.register("collide-kind", cfg_a)(_AgentA) with pytest.raises(ValueError, match="already registered"): - AgentRegistry.register("collide-kind", _CfgB)(_AgentB) + AgentRegistry.register("collide-kind", cfg_b)(_AgentB) # The incumbent is untouched — the conflict did not overwrite it. reg = AgentRegistry.get("collide-kind") assert reg is not None and reg.agent_class is _AgentA @@ -187,16 +190,17 @@ def test_register_same_kind_same_classes_is_idempotent(): """Re-registering the IDENTICAL classes (e.g. load_plugins(force=True) re-running register_builtins) is a legitimate no-op, not a collision.""" - class _Cfg(ClaudeCodeAgentConfig): - pass + cfg = config_for_kind("idem-kind", ClaudeCodeAgentConfig) class _Agent: + contract = stub_contract() + def __init__(self, config, route=None, **kwargs): self.config = config try: - AgentRegistry.register("idem-kind", _Cfg)(_Agent) - AgentRegistry.register("idem-kind", _Cfg)(_Agent) # no raise + AgentRegistry.register("idem-kind", cfg)(_Agent) + AgentRegistry.register("idem-kind", cfg)(_Agent) # no raise reg = AgentRegistry.get("idem-kind") assert reg is not None and reg.agent_class is _Agent finally: diff --git a/tests/test_registry.py b/tests/test_registry.py index 3db670873..d54d2143f 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -4,6 +4,7 @@ from coder_eval.agents.registry import AgentRegistry, create_agent from coder_eval.models import AgentKind, ClaudeCodeAgentConfig, parse_agent_config +from tests.fixtures.harness_stubs import config_for_kind, stub_contract def test_register_decorator_preserves_class_type(): @@ -13,13 +14,17 @@ def test_register_decorator_preserves_class_type(): # the type annotation erases to Any in the decorator signature. class FakeAgent: + contract = stub_contract() + def __init__(self, config, route=None, **kwargs): self.config = config # Register under a unique kind (not a built-in) to avoid the shadow-collision # guard, then roll it back so the process-global registry doesn't leak. try: - registration = AgentRegistry.register("fake-identity-kind", ClaudeCodeAgentConfig)(FakeAgent) + registration = AgentRegistry.register( + "fake-identity-kind", config_for_kind("fake-identity-kind", ClaudeCodeAgentConfig) + )(FakeAgent) assert registration is FakeAgent finally: AgentRegistry._registry.pop("fake-identity-kind", None) diff --git a/tests/test_simulation_integration.py b/tests/test_simulation_integration.py index 36a209127..cd8207178 100644 --- a/tests/test_simulation_integration.py +++ b/tests/test_simulation_integration.py @@ -25,6 +25,7 @@ ) from coder_eval.orchestrator import Orchestrator from coder_eval.simulation.user_simulator import UserSimulator +from tests.fixtures.harness_stubs import stub_contract from tests.fixtures.mock_agent import MockAgent from tests.fixtures.text_stub_agent import TextStubAgent @@ -105,6 +106,8 @@ async def communicate(self, user_input: str, **kwargs: object) -> TurnRecord: class _ExplodingAgent(Agent): """Agent fake whose communicate() always raises — exercises error-path handling.""" + contract = stub_contract() + def __init__(self, message: str = "simulator exploded") -> None: self._message = message self._state = AgentState.WORKING diff --git a/tests/test_visible_turn_cap.py b/tests/test_visible_turn_cap.py index 93cf5f090..cde8e7ae1 100644 --- a/tests/test_visible_turn_cap.py +++ b/tests/test_visible_turn_cap.py @@ -65,4 +65,4 @@ def test_collector_visible_turn_count_matches_the_built_record(): @pytest.mark.parametrize("agent_cls", [CodexAgent, AntigravityAgent]) def test_both_capped_agents_declare_cooperative_stop(agent_cls): """The turn cap reuses the cooperative-stop boundary, so both must support it.""" - assert agent_cls.supports_cooperative_stop is True + assert agent_cls.contract.cooperative_stop is True From 5a18f2ffad9a0f9f04e283d5d920a2da381087c1 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 07:54:51 -0700 Subject: [PATCH 02/12] =?UTF-8?q?refactor(agents):=202/6=20=E2=80=94=20dec?= =?UTF-8?q?lare=20cost=5Flog=5Ftags=20on=20the=20base=20Agent=20constructo?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent.__init__ is now concrete and takes cost_log_tags; every in-tree agent forwards it through super().__init__. The orchestrator forwards the tags on every LiteLLM route, so the supports_cost_log_tags capability gate is deleted. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 7 ++-- .claude/notes/reporting.md | 4 +-- src/coder_eval/agent.py | 22 ++++++++---- src/coder_eval/agents/antigravity_agent.py | 5 +-- src/coder_eval/agents/claude_code_agent.py | 19 ++++------ src/coder_eval/agents/codex_agent.py | 5 +-- src/coder_eval/agents/noop_agent.py | 7 ++-- src/coder_eval/agents/opencode_agent.py | 4 +-- src/coder_eval/agents/pi_agent.py | 4 +-- src/coder_eval/orchestrator.py | 15 +++----- tests/test_harness_contract.py | 11 ++++++ tests/test_litellm_route.py | 41 ++++++++++------------ tests/test_opencode_agent.py | 9 ++--- tests/test_pi_agent.py | 3 +- 14 files changed, 81 insertions(+), 75 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 277b31398..4ddde5ecb 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -122,9 +122,10 @@ budget again and re-hits the cap. `ended_cleanly` is the guard. `create_agent` calls `agent_class(config, route=route, **kwargs)` through a `cast(Any, ...)`, so pyright checks nothing at the call site; a `**_` sink would mean nothing checks it at runtime either. The orchestrator depends on that `TypeError` as a -signal — it gates `cost_log_tags` on `supports_cost_log_tags` precisely because the -agent-agnostic factory would otherwise forward it into constructors that do not declare -it. A mis-gated kwarg must be loud, not silently dropped. +signal: a kwarg forwarded into a constructor that does not declare it must be loud, not +silently dropped. `cost_log_tags` is declared on the base `Agent.__init__` and every +subclass forwards it, so the factory passes it on every LiteLLM route without a +capability gate. `route` is accepted for factory parity and deliberately unused by the CLI-driven harnesses: those CLIs own their own provider configuration. diff --git a/.claude/notes/reporting.md b/.claude/notes/reporting.md index 2afca5a04..5c0607cfa 100644 --- a/.claude/notes/reporting.md +++ b/.claude/notes/reporting.md @@ -85,8 +85,8 @@ caller's move, not the agent's: only the caller knows a turn failed. Capabilities are declared rather than probed, on the agent's `HarnessContract`. `contract.cooperative_stop` gates arming early-stop, so arming it on an agent that ignores `should_stop` is rejected at resolution rather than silently never firing. -`supports_cost_log_tags` and `contract.system_prompt_semantics` are declared for reasons of -their own — see +`contract.system_prompt_semantics` is declared for reasons of its own, and `cost_log_tags` +is a base-constructor kwarg — see [agents.md](agents.md) § Why the constructors declare every kwarg and § The system_prompt_semantics marker. diff --git a/src/coder_eval/agent.py b/src/coder_eval/agent.py index 64de10fdb..d36d22b64 100644 --- a/src/coder_eval/agent.py +++ b/src/coder_eval/agent.py @@ -11,7 +11,7 @@ from .errors import AgentCrashError, TurnTimeoutError from .errors.agent import format_timeout_reason, truncate_crash_message from .models import AgentState as AgentState -from .models import BaseAgentConfig, HarnessContract, TurnRecord +from .models import ApiRoute, BaseAgentConfig, HarnessContract, TurnRecord from .streaming.callbacks import StreamCallback from .streaming.collector import EventCollector from .streaming.events import AgentEndStatus @@ -70,16 +70,26 @@ def __init__(self, config: ClaudeCodeAgentConfig, ...): _iteration: int = 0 _iteration_was_incremented: bool = False - # Whether this agent's constructor accepts ``cost_log_tags``. The agent-agnostic - # factory must only forward it to agents that set this True, or a route-driven - # kwarg crashes every agent whose ``__init__`` lacks it. - supports_cost_log_tags: ClassVar[bool] = False - # Which uniform config fields this harness honors. No default: registration # rejects a class that does not declare one. # Rationale: .claude/notes/agents.md § The system_prompt_semantics marker contract: ClassVar[HarnessContract] + def __init__( + self, config: ConfigT, route: ApiRoute | None = None, *, cost_log_tags: dict[str, str] | None = None + ) -> None: + """Bind the resolved config and route. + + Args: + config: The agent's resolved config. + route: API routing; harnesses that own their provider config ignore it. + cost_log_tags: LiteLLM-only correlation headers the factory forwards on + every LiteLLM route. An agent that cannot stamp them keeps them unused. + """ + self.config = config + self.route = route + self.cost_log_tags = cost_log_tags + def _begin_turn(self) -> None: """Mark the start of a ``communicate()`` turn: reset the pending slot and bump the iteration counter so a mid-turn failure can be rolled back. diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 4ff9641fd..7d9222b72 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -202,6 +202,7 @@ def __init__( route: ApiRoute | None = None, *, instance_name: str = "antigravity", + cost_log_tags: dict[str, str] | None = None, ): """Initialize the Antigravity agent. @@ -210,9 +211,9 @@ def __init__( route: API routing configuration (unused — Antigravity authenticates via GEMINI_API_KEY against the Gemini Developer API; kept for parity). instance_name: Short label used to prefix this instance's log records. + cost_log_tags: LiteLLM correlation headers; accepted for factory parity, unused. """ - self.config = config - self.route = route or DirectRoute() + super().__init__(config, route or DirectRoute(), cost_log_tags=cost_log_tags) self.working_directory: Path | None = None # The live SDK Agent session + its AsyncExitStack. The exit-stack teardown # terminates the localharness subprocess, which is what stop()/kill() rely diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 5b4fd1639..b139211ce 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -10,7 +10,7 @@ from contextlib import suppress from datetime import datetime, timedelta from pathlib import Path -from typing import Any, ClassVar +from typing import Any from claude_agent_sdk import ( ClaudeAgentOptions, @@ -703,14 +703,13 @@ class ClaudeCodeAgent(Agent[ClaudeCodeAgentConfig]): cooperative_stop=True, ) - # __init__ accepts cost_log_tags and stamps them into ANTHROPIC_CUSTOM_HEADERS - # for the proxy-side actual-cost join (LiteLLM backend). - supports_cost_log_tags: ClassVar[bool] = True - # One warning per agent for a replace-mode config with no prompt: the resolver # runs on every query, and a per-turn repeat would bury the rest of task.log. _warned_prompt_mode_downgrade: bool = False + # Narrowed from the base: __init__ defaults a missing route to DirectRoute. + route: ApiRoute + def __init__( self, config: ClaudeCodeAgentConfig, @@ -737,12 +736,8 @@ def __init__( attribute each call's real cost back to this run. None on Direct/Bedrock. """ - self.config = config - self.route = route or DirectRoute() + super().__init__(config, route or DirectRoute(), cost_log_tags=cost_log_tags) self._extra_mcp_servers = extra_mcp_servers or {} - # Correlation headers stamped on every SDK->proxy request (LiteLLM only). - # This turn's iteration is appended per-communicate(). - self._cost_log_tags = cost_log_tags self.client: ClaudeSDKClient | None = None self.working_directory: Path | None = None # Turn-lifecycle bookkeeping lives on the Agent base class. @@ -1166,8 +1161,8 @@ def _build_claude_query( # Per-turn cost-correlation headers (LiteLLM only): the run/task tag plus # this turn's iteration, so the proxy-side cost log joins to the turn. cost_log_tags: dict[str, str] | None = None - if self._cost_log_tags is not None: - cost_log_tags = {**self._cost_log_tags, "x-ce-iteration": str(self._iteration)} + if self.cost_log_tags is not None: + cost_log_tags = {**self.cost_log_tags, "x-ce-iteration": str(self._iteration)} env, route_model = self._build_sdk_env( self.route, path_prepend=self._env_path_prepend, diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 1a8b239b0..f3914edad 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -781,6 +781,7 @@ def __init__( route: ApiRoute | None = None, *, instance_name: str = "codex", + cost_log_tags: dict[str, str] | None = None, ): """Initialize the Codex agent. @@ -788,9 +789,9 @@ def __init__( config: Agent configuration route: API routing configuration (unused for Codex, kept for interface compatibility) instance_name: Short label used to prefix this instance's log records + cost_log_tags: LiteLLM correlation headers; accepted for factory parity, unused """ - self.config = config - self.route = route or DirectRoute() + super().__init__(config, route or DirectRoute(), cost_log_tags=cost_log_tags) self.codex_client: Any = None self.thread: Any = None # Thread-cumulative snapshot as of the END of the last finalized turn: diff --git a/src/coder_eval/agents/noop_agent.py b/src/coder_eval/agents/noop_agent.py index f97088a04..47035f613 100644 --- a/src/coder_eval/agents/noop_agent.py +++ b/src/coder_eval/agents/noop_agent.py @@ -53,9 +53,10 @@ class NoOpAgent(Agent[NoneAgentConfig]): cooperative_stop=False, ) - def __init__(self, config: NoneAgentConfig, route: ApiRoute | None = None) -> None: - self.config = config - self.route = route + def __init__( + self, config: NoneAgentConfig, route: ApiRoute | None = None, *, cost_log_tags: dict[str, str] | None = None + ) -> None: + super().__init__(config, route, cost_log_tags=cost_log_tags) async def start( self, diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index 206776c29..a18a46aa5 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -749,6 +749,7 @@ def __init__( route: ApiRoute | None = None, *, task_id: str = "unknown", + cost_log_tags: dict[str, str] | None = None, ) -> None: """Every parameter the agent factory can pass is DECLARED, not absorbed. @@ -759,8 +760,7 @@ def __init__( Rationale: .claude/notes/agents.md § Why the constructors declare every kwarg """ - self.config = config - self.route = route + super().__init__(config, route, cost_log_tags=cost_log_tags) self.task_id = task_id self.working_directory: str | None = None self._env_path_prepend: list[str] = [] diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 535b59b35..5843613be 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -699,6 +699,7 @@ def __init__( route: ApiRoute | None = None, *, task_id: str = "unknown", + cost_log_tags: dict[str, str] | None = None, ) -> None: """Every parameter the agent factory can pass is DECLARED, not absorbed. @@ -708,8 +709,7 @@ def __init__( Rationale: .claude/notes/agents.md § Why the constructors declare every kwarg """ - self.config = config - self.route = route + super().__init__(config, route, cost_log_tags=cost_log_tags) self.task_id = task_id self.working_directory: str | None = None self._env_path_prepend: list[str] = [] diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index cdcfba490..732ebf621 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -1844,7 +1844,7 @@ async def _create_agent(self) -> Agent[Any]: ValueError: If agent type is not supported TypeError: If config doesn't match agent's expected type """ - from coder_eval.agents import AgentRegistry, create_agent + from coder_eval.agents import create_agent from coder_eval.plugins import ensure_plugins_loaded # create_agent no longer self-loads (keeping plugins -> registry one-way), @@ -1853,18 +1853,11 @@ async def _create_agent(self) -> Agent[Any]: assert self.task.agent is not None assert self.task.agent.type is not None # LiteLLM only: correlation headers so a proxy-side cost callback can - # attribute each call back to this task-run. GATED ON AGENT CAPABILITY, not - # on the route — the route is settings-derived and independent of agent - # type, so the agent-agnostic factory would otherwise forward the kwarg - # into constructors that do not declare it. + # attribute each call back to this task-run. Every agent accepts the kwarg + # through the base constructor. # Rationale: .claude/notes/agents.md § Why the constructors declare every kwarg kwargs: dict[str, Any] = {} - registration = AgentRegistry.get(self.task.agent.type) - if ( - isinstance(self.route, LiteLLMRoute) - and registration is not None - and registration.agent_class.supports_cost_log_tags - ): + if isinstance(self.route, LiteLLMRoute): kwargs["cost_log_tags"] = { "x-ce-run-id": self._cost_correlation_run_id, "x-ce-task-id": self._log_task_id, diff --git a/tests/test_harness_contract.py b/tests/test_harness_contract.py index 65bcf685d..2978467b0 100644 --- a/tests/test_harness_contract.py +++ b/tests/test_harness_contract.py @@ -15,6 +15,7 @@ ClaudeCodeAgentConfig, Enforcement, HarnessContract, + parse_agent_config, ) from coder_eval.plugins import ensure_plugins_loaded from tests.fixtures.harness_stubs import config_for_kind, stub_contract @@ -120,3 +121,13 @@ def test_every_builtin_declares_a_contract(kind: AgentKind) -> None: registration = AgentRegistry.get(kind) assert registration is not None assert isinstance(registration.agent_class.contract, HarnessContract) + + +@pytest.mark.parametrize("kind", [k for k in AgentKind if k is not AgentKind.UNKNOWN]) +def test_every_builtin_accepts_cost_log_tags(kind: AgentKind) -> None: + ensure_plugins_loaded() + registration = AgentRegistry.get(kind) + assert registration is not None + tags = {"x-ce-run-id": "r"} + agent = registration.agent_class(parse_agent_config(type=kind), cost_log_tags=tags) + assert agent.cost_log_tags == tags diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 087d6dc33..8d1bd195b 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -556,29 +556,26 @@ def test_cost_log_tags_ignored_on_non_litellm_routes(self): env_d, _ = ClaudeCodeAgent._build_sdk_env(DirectRoute(), cost_log_tags=tags) assert "ANTHROPIC_CUSTOM_HEADERS" not in env_d - def test_cost_log_tags_gated_on_agent_capability_not_route(self): - """Regression: cost_log_tags is a Claude-only constructor kwarg, but the - route that triggers it (LiteLLM) is agent-independent. The agent-agnostic - create_agent factory must forward it ONLY to agents that declare - supports_cost_log_tags — otherwise a none/codex/antigravity task crashes - with TypeError under API_BACKEND=litellm.""" - from coder_eval.agents import AgentRegistry, create_agent - from coder_eval.models import NoneAgentConfig - from coder_eval.plugins import ensure_plugins_loaded - - ensure_plugins_loaded() - # Capability contract the orchestrator gate reads. - assert AgentRegistry.get(AgentKind.CLAUDE_CODE).agent_class.supports_cost_log_tags is True - assert AgentRegistry.get(AgentKind.NONE).agent_class.supports_cost_log_tags is False + @pytest.mark.parametrize("kind", [AgentKind.NONE, AgentKind.CODEX]) + async def test_orchestrator_factory_forwards_tags_on_litellm(self, kind: AgentKind, tmp_path): + """The case the old capability gate protected: a none/codex task on a + LiteLLM route constructs with the forwarded tags instead of crashing.""" + from coder_eval.models import FileExistsCriterion, SandboxConfig, TaskDefinition, parse_agent_config + from coder_eval.orchestrator import Orchestrator - route = LiteLLMRoute(model="deepseek/deepseek-v4-pro") - # A none-agent constructs fine on a LiteLLM route (the gate omits the kwarg)... - assert create_agent(AgentKind.NONE, NoneAgentConfig(type=AgentKind.NONE), route=route) is not None - # ...and it WOULD crash if the kwarg were forwarded — exactly what the gate prevents. - with pytest.raises(TypeError): - create_agent( - AgentKind.NONE, NoneAgentConfig(type=AgentKind.NONE), route=route, cost_log_tags={"x-ce-run-id": "r"} - ) + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt=None if kind is AgentKind.NONE else "hi", + agent=parse_agent_config(type=kind), + sandbox=SandboxConfig(driver="tempdir"), + success_criteria=[FileExistsCriterion(description="c", path="x")], + ) + orchestrator = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") + orchestrator.route = LiteLLMRoute(model="m") + agent = await orchestrator._create_agent() + assert agent.cost_log_tags is not None + assert set(agent.cost_log_tags) == {"x-ce-run-id", "x-ce-task-id", "x-ce-attempt"} def test_cost_log_tags_reject_header_injection(self): # A task_id/variant_id carrying a CR/LF would inject extra headers into every diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 915368313..0b8c6023c 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -573,15 +573,12 @@ def test_route_is_accepted_positionally_and_by_keyword(self): assert OpenCodeAgent(config, None).route is None def test_an_undeclared_kwarg_raises_instead_of_vanishing(self): - """The orchestrator gates `cost_log_tags` on `supports_cost_log_tags` - precisely because an ungated forward must crash with TypeError. A `**_` - sink defeated that: a mis-gated kwarg would be silently dropped, yielding - runs with no cost correlation and no error. + """A `**_` sink would silently drop a kwarg the factory forwards by + mistake, yielding a run with no error; a declared signature raises. """ config = OpenCodeAgentConfig(type="opencode", model="deepseek/deepseek-v4-pro") - assert OpenCodeAgent.supports_cost_log_tags is False with pytest.raises(TypeError): - OpenCodeAgent(config, cost_log_tags={"x-ce-run-id": "r1"}) # type: ignore[call-arg] + OpenCodeAgent(config, not_a_kwarg={"x-ce-run-id": "r1"}) # type: ignore[call-arg] def test_a_mistyped_task_id_raises_instead_of_defaulting(self): config = OpenCodeAgentConfig(type="opencode", model="deepseek/deepseek-v4-pro") diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index e5e6e8f48..b6d6dc3d0 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -662,9 +662,8 @@ def test_route_accepted_positionally_and_by_keyword(self): def test_undeclared_kwarg_raises(self): config = PiAgentConfig(type="pi", model="openrouter/moonshotai/kimi-k3") - assert PiAgent.supports_cost_log_tags is False with pytest.raises(TypeError): - PiAgent(config, cost_log_tags={"x": "y"}) # type: ignore[call-arg] + PiAgent(config, not_a_kwarg={"x": "y"}) # type: ignore[call-arg] class TestRegistry: From 6ee47a88198afe0794f74970a021909c023c3fac Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 08:10:39 -0700 Subject: [PATCH 03/12] =?UTF-8?q?feat(agents):=203/6=20=E2=80=94=20honor?= =?UTF-8?q?=20tool=20restrictions=20natively=20on=20Pi,=20OpenCode=20and?= =?UTF-8?q?=20Antigravity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pi maps allowed_tools / disallowed_tools / permission_mode plan to --tools, --no-tools and --exclude-tools. OpenCode writes explicit permission rules and an instructions file for system_prompt into OPENCODE_CONFIG_CONTENT and always passes --auto. Antigravity builds SDK tool-call policies. Every map is the inverse of the adapter's telemetry map. The Codex enabled_tools / disabled_tools forward was a no-op (they are MCP-server keys) and is deleted with its task. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 37 +++++ docs/TASK_DEFINITION_GUIDE.md | 6 +- docs/agents/ANTIGRAVITY.md | 48 +++--- docs/agents/CODEX.md | 60 +------- docs/agents/HARNESS_PARITY.md | 29 +--- docs/agents/OPENCODE.md | 37 +++-- docs/agents/PI.md | 60 +++----- src/coder_eval/agents/antigravity_agent.py | 45 ++++-- src/coder_eval/agents/codex_agent.py | 26 ---- src/coder_eval/agents/opencode_agent.py | 143 +++++++++++++----- src/coder_eval/agents/pi_agent.py | 48 +++--- src/coder_eval/models/__init__.py | 2 + src/coder_eval/models/agent_config.py | 16 +- src/coder_eval/models/enums.py | 6 +- tasks/agents/codex_disallowed_tools_test.yaml | 31 ---- tasks/agents/codex_hello_world.yaml | 4 - tasks/agents/codex_parallel_commands.yaml | 4 - tasks/agents/codex_parallel_single_gen.yaml | 4 - tasks/agents/codex_string_utils.yaml | 5 - tasks/agents/codex_subagent_test.yaml | 4 - tests/test_antigravity_agent.py | 92 +++++++++-- tests/test_codex_agent.py | 34 +---- tests/test_codex_agent_unit.py | 34 +---- tests/test_opencode_agent.py | 124 +++++++++++++-- tests/test_pi_agent.py | 63 +++++--- 25 files changed, 557 insertions(+), 405 deletions(-) delete mode 100644 tasks/agents/codex_disallowed_tools_test.yaml diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 4ddde5ecb..0dd8c80b8 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -474,6 +474,9 @@ shell-aware `parameters["command"]` extraction in `criteria/command_executed.py` to raw-JSON matching — so the same task scores differently per harness. Unknown names pass through unchanged. +The Claude-to-native maps the uniform tool fields use are derived by inverting these, never +written twice. + Three cases are worth knowing: - **OpenCode's tool set varies by MODEL within the one harness.** A live 174-task run @@ -494,6 +497,35 @@ any key that FIRST appears at DONE is treated as a result — which matters beca `skill_triggered` substring-searches every parameter value, so a leaked result could false-positive. +## The uniform fields, per harness + +`permission_mode`, `allowed_tools` and `disallowed_tools` stay on `BaseAgentConfig` as one +interface, and each harness honors them where the pinned CLI or SDK has a verified mechanism +(spike of 2026-09-16). Their meaning is the same everywhere: an allowlist permits only the +named tools, a deny always wins, and `plan` denies the Write, Edit and Bash equivalents +(`READ_ONLY_DENIED_TOOLS`, one declaration for every adapter). An empty `allowed_tools: []` +restricts nothing, because Claude Code passes `[]` as "no `--allowedTools` flag"; the same +YAML must not mean "all tools" on one harness and "no tools" on the others. + +- **Pi** (0.85.1): `--tools ` is an allowlist and `--exclude-tools ` a denylist over + the lowercase built-ins. The denied set is subtracted before `--tools` is emitted, and an + allowlist that maps to nothing becomes `--no-tools`. +- **OpenCode** (1.18.30): the `permission` config accepts `"*": "deny"` plus per-key + `allow` / `deny`, and `--auto` approves only what is not explicitly denied — so `plan` is + explicit denies and `--auto` is passed on every run. `instructions` files are read by + `session/instruction.ts::system()` and spread into the SYSTEM messages, so + `system_prompt` is a temp file listed there (append). The file lives outside the sandbox + for the same reason as the skill paths below. The permission keys are coarser than tool + names (`edit` governs every write-shaped tool); that four-entry table is the one literal. + `"*"` also matches non-tool permissions (`external_directory`, `doom_loop`), which the CLI + merges before config rules and `--auto` used to approve, so an allowlist re-allows them. + OpenCode applies the LAST matching rule, so our rules are placed after inherited ones. +- **Antigravity** (0.1.8): `hooks/policy.py` buckets specific rules above wildcard ones and + deny above allow, so rule order does not matter. `finish` is always allowed under an + allowlist because the harness ends a turn with it; whether `deny_all()` reaches it could + not be probed offline, and allowing it is the safe direction. +- **Codex**: no mechanism (next section), so all three rows are unsupported. + ## Codex runs full-access on every permission mode `coder_eval` owns the isolation boundary either way — a docker container or an ephemeral @@ -510,6 +542,11 @@ The consequence is stated loudly at `start()` for EVERY mode, not just Codex — none of them do. Adversarial or untrusted evals belong on the docker driver; the tempdir/host driver is a working directory, not a confinement boundary. +Tool restriction is not available either. `strings` on the pinned codex-cli 0.39.0 binary +shows `enabled_tools` / `disabled_tools` only inside `RawMcpServerConfig` (beside +`bearer_token_env_var`, `startup_timeout_sec`); there is no top-level key. The adapter's old +top-level `config.enabled_tools` forward therefore never restricted a tool, and was deleted. + Approval mode is `deny_all` on every permission mode too. The SDK offers only two: `auto_review`, which puts a SERVER-SIDE reviewer in the loop that can spuriously return "declined" under gateway load — files silently not written, a failure mode Claude has no diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 845f98d4d..bafe807c3 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -107,9 +107,9 @@ sandbox, no API call. Use it to park a task that is blocked on something outside (an upstream bug, a missing service) without deleting the YAML and losing its history. ```yaml -task_id: "codex_disallowed_tools_test" -# Blocked: the Codex SDK doesn't enforce disallowed_tools via config. Re-enable -# once upstream ships the fix. +task_id: "flaky_upstream_service" +# Blocked: the mocked service times out under CI load (issue #123). Re-enable +# once the mock is fixed. skip: true ``` diff --git a/docs/agents/ANTIGRAVITY.md b/docs/agents/ANTIGRAVITY.md index 875058696..7588fed8a 100644 --- a/docs/agents/ANTIGRAVITY.md +++ b/docs/agents/ANTIGRAVITY.md @@ -136,18 +136,28 @@ can't be resolved or if zero skills are discovered. ## Permissions & tools — important differences -**Antigravity ignores `permission_mode`, `allowed_tools`, and `disallowed_tools`.** -The local harness runs in a single unconditional mode: every tool call (including -`run_command`) is approved via an allow-all policy, and file tools are restricted to -the configured `workspaces` (the sandbox working directory plus any skill roots). - -The trust boundary for an Antigravity run is therefore the **sandbox**, not the -agent config. Run untrusted tasks under the [Docker driver](../DOCKER_ISOLATION.md); -the `tempdir` driver is not a security boundary. This mirrors the reality that the -`bypassPermissions`-equivalent behavior is always on for this backend. - -Those inherited fields still exist on the config for schema uniformity but have no -runtime effect here — don't rely on them to gate Antigravity. +The uniform fields become SDK tool-call policies (`google.antigravity.hooks.policy`): + +| Field | Policies | +|---|---| +| none set | `allow_all()` — every tool call, including `run_command`, is approved | +| `allowed_tools` | `deny_all()`, then `allow(tool)` for each mapped tool and for `finish` | +| `disallowed_tools` | `deny(tool)` for each mapped tool | +| `permission_mode: plan` | `deny` on `create_file`, `edit_file` and `run_command` (read-only) | + +Claude tool names map to harness tools by inverting the telemetry map (`Bash` → +`run_command`, `Write` → `create_file`, `Edit` → `edit_file`, `Read` → `view_file`, +…). A specific rule outranks the wildcard one in the SDK, and a specific deny +outranks a specific allow, so a denied tool stays denied. `finish` is always allowed +under an allowlist and never denied, because the harness ends a turn with it. An empty +`allowed_tools: []` restricts nothing, as on Claude Code. A name with no +Antigravity equivalent restricts nothing. File tools also stay restricted to the +configured `workspaces` (the sandbox working directory plus any skill roots); an +out-of-workspace path is a specific deny. + +Policies decide which calls run; a denied tool is still visible to the model. They do +not confine what a permitted `run_command` can do, so the trust boundary for an +untrusted run is still the **sandbox**: use the [Docker driver](../DOCKER_ISOLATION.md). ## Telemetry @@ -159,7 +169,7 @@ as every other agent. canonical Claude-style names so cross-agent criteria work: `run_command` → `Bash`, `create_file` → `Write`, `edit_file` → `Edit`, `view_file` → `Read`, `search_directory` → `Grep`, `find_file` → `Glob`, `list_directory` → `LS`, - `start_subagent` → `Task`, `search_web` → `WebSearch`. Argument keys are also + `start_subagent` → `Agent`, `search_web` → `WebSearch`. Argument keys are also normalized (e.g. `command_line` → `command`) so `command_executed` criteria key on the same params across agents. - **Tokens.** Gemini usage maps to Coder Eval's four buckets: uncached input, cache @@ -184,16 +194,12 @@ as every other agent. 3. **`kill_sync()` is best-effort.** The SDK's cancel/disconnect are async-only, so the watchdog's synchronous kill only flips agent state to `ERROR`; real teardown happens on the subsequent async `stop()`. -4. **`permission_mode` does not confine the harness.** Every mode runs - `policy.allow_all()`; coder_eval's write boundary is the sandbox driver, and a - headless eval has no human to approve anything. -5. **`allowed_tools` / `disallowed_tools` are not read.** The harness runs with its - full builtin tool set, so an Antigravity run has tools (web search, subagents, - URL fetch) that the same task file denies on Claude Code and Codex. -6. **`max_turns` counts visible turns.** One `communicate()` is a single SDK turn here, +4. **Denied tools stay visible.** A policy denial rejects the call after the model + makes it, so a denied tool can still cost tokens on a retry. +5. **`max_turns` counts visible turns.** One `communicate()` is a single SDK turn here, so the cap counts resolved tool calls instead, enforced on the step loop. See [Run-Limit Parity](HARNESS_PARITY.md). -7. **Shell commands over ~10s are moved to the background.** The localharness has a +6. **Shell commands over ~10s are moved to the background.** The localharness has a 10-second maximum synchronous wait; past it the command becomes a background task and the model gets a task id, not a result. The turn polls for that result instead of finalizing on an idle step stream, so slow work does complete — but the wait is diff --git a/docs/agents/CODEX.md b/docs/agents/CODEX.md index 2d6a131cb..c83f3e8e7 100644 --- a/docs/agents/CODEX.md +++ b/docs/agents/CODEX.md @@ -179,18 +179,11 @@ On failure, the agent: ### Permission and Tool Mapping -The agent maps `permission_mode` to the Codex SDK's `Sandbox`. The approval mode is **uniformly `deny_all`** for every mode — the trust boundary is the sandbox, which does vary by mode: +Codex runs with `sandbox: full-access` and `approval_mode: deny_all` on every run. Its own OS sandbox fails silently on the hosts Coder Eval runs on, so the isolation boundary is the task's driver: use `driver: docker` for untrusted evals. -| `permission_mode` | `sandbox` | `approval_mode` | -|-------------------|-----------|-----------------| -| `bypassPermissions` | `full-access` | `deny_all` | -| `acceptEdits` | `workspace-write` | `deny_all` | -| `default` | `workspace-write` | `deny_all` | -| `plan` | `read-only` | `deny_all` | +`deny_all` means *run autonomously, never prompt, no server-side reviewer*. Coder Eval uses it because the alternative (`auto_review`) adds a server-side reviewer that can spuriously return `declined` under gateway load. -`deny_all` means *run autonomously, never prompt, no server-side reviewer*: in-sandbox operations execute directly and only escalations beyond the sandbox are refused. Coder Eval uses it for every mode because the alternative (`auto_review`) adds a server-side reviewer that can spuriously return `declined` under gateway load. - -`allowed_tools` / `disallowed_tools` are normalized (`Bash` → `shell`, `Write`/`Edit` → `apply_patch`, etc.) and passed as `enabled_tools` / `disabled_tools` in the thread `config`. **Note:** the Codex SDK does not currently enforce `disabled_tools`; do not rely on it as a security boundary (the agent logs a warning when it is set). +Codex honors none of `permission_mode`, `allowed_tools` and `disallowed_tools`. Its config has no top-level key that restricts the built-in `shell` / `apply_patch` tools (`enabled_tools` / `disabled_tools` exist only per MCP server), so Coder Eval forwards nothing for them. ### Skills Discovery @@ -215,8 +208,7 @@ The Codex SDK is synchronous. The agent uses `_run_async()` helper to detect and | **Model Selection** | Direct via `--model` or config | `agent.model` pinned into `thread_start` | | **System prompt** | `system_prompt` appended to the default prompt (SDK `claude_code` preset) | `system_prompt` passed as `developer_instructions` on top of the Codex base prompt | | **Session Resume** | `--resume {session_id}` | Via thread ID | -| **Permissions** | `permission_mode` + `allowed_tools` | `permission_mode` → sandbox/approval + `allowed_tools`/`disallowed_tools` → thread config | -| **Tool Enforcement** | Not enforced by Coder Eval wrapper | `enabled_tools` honored; `disabled_tools` NOT enforced by the SDK | +| **Permissions** | `permission_mode` + `allowed_tools` + `disallowed_tools` | Not supported; always full-access | | **`max_turns`** | Native SDK turn cap (assistant messages) | Visible-turn cap (tool calls), enforced on the notification pump | | **Early stop** | Supported (cooperative `should_stop`, polled between messages) | Supported — polled after each streamed notification; the in-flight turn is interrupted best-effort | @@ -259,55 +251,11 @@ Run the included test tasks: # Basic functionality test coder-eval run tasks/agents/codex_hello_world.yaml -# Tool restriction test (verifies disallowed_tools enforcement) -coder-eval run tasks/agents/codex_disallowed_tools_test.yaml - # Skills discovery test (requires PLUGIN_PATH environment variable) export PLUGIN_PATH=~/path/to/skills coder-eval run tasks/agents/codex_skills_test.yaml ``` -Example unit test to verify agent setup: - -```python -import pytest -from coder_eval.models import AgentKind, AgentConfig -from coder_eval.agents.codex_agent import CodexAgent -from coder_eval.agent import AgentState - -def test_codex_agent_initialization(): - """Verify CodexAgent can be instantiated with valid config.""" - config = AgentConfig( - type=AgentKind.CODEX, - permission_mode="acceptEdits", - allowed_tools=["Bash", "Read", "Write"], - ) - agent = CodexAgent(config) - assert agent.get_state() == AgentState.WORKING - assert agent.config.type == AgentKind.CODEX - -def test_tool_name_mapping(): - """Verify Claude Code tool names map to Codex SDK names.""" - from coder_eval.agents.codex_agent import _CLAUDE_TO_CODEX_TOOL_MAP - - assert _CLAUDE_TO_CODEX_TOOL_MAP["Bash"] == "shell" - assert _CLAUDE_TO_CODEX_TOOL_MAP["Write"] == "apply_patch" - assert _CLAUDE_TO_CODEX_TOOL_MAP["Edit"] == "apply_patch" - assert _CLAUDE_TO_CODEX_TOOL_MAP["Read"] == "shell" - -def test_permission_mode_mapping(): - """Verify permission_mode maps to a sandbox; approval is uniformly deny_all.""" - from coder_eval.agents.codex_agent import ( - _CODEX_APPROVAL_MODE, - _PERMISSION_MODE_TO_SANDBOX, - ) - - assert _PERMISSION_MODE_TO_SANDBOX["acceptEdits"] == "workspace-write" - assert _PERMISSION_MODE_TO_SANDBOX["plan"] == "read-only" - # Approval is the same for every permission mode — no per-mode mapping. - assert _CODEX_APPROVAL_MODE == "deny_all" -``` - ## References - [Codex SDK Documentation](https://developers.openai.com/codex/sdk) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index ec0919ee5..7d3d88235 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -704,29 +704,12 @@ both via the same `_plugin_skill_dirs` resolver — so both **can** run activati suites. A plugin's non-skill assets (agents/hooks/commands/MCP servers) are dropped on both. See [OpenCode](OPENCODE.md) and [Pi § plugins](PI.md#known-limitations). -## Pi enforces `system_prompt` but not the tool allowlists - -- **`system_prompt` is ENFORCED** (`--append-system-prompt`, semantics `append`) — a - small win over OpenCode, which drops it. -- **`allowed_tools` / `disallowed_tools` are NOT enforced.** Pi's built-in tools are - lowercase (`bash`/`read`/`write`/`edit`/`grep`/`find`/`ls`), but the shared config - default (`experiments/default.yaml`) sets Claude-namespaced names - (`Bash`/`Read`/`Write`/…). Forwarding those to `--tools` would allowlist tools that - do not exist in Pi and strip the agent of ALL tools — so, like OpenCode (drops them), - Codex (forwards `disallowed_tools` without SDK enforcement), and Antigravity (does not - read them), Pi ignores them and runs with its full native toolset. A task that needs a - restricted Pi toolset would have to name Pi's lowercase tools — a documented follow-up. -- **`permission_mode` is NOT enforced** — Pi headless print mode auto-runs tools and - exposes only project-file trust (`--approve` / `--no-approve`), no tool-approval - mode; the sandbox driver is the isolation boundary (same as Codex/Antigravity). -- **`system_prompt_file` is NOT read** (use inline `system_prompt`), matching - Codex/Antigravity. -- **Built-in auto-retry.** Pi retries a transient/provider error *internally* (another - `agent_start` cycle in the same invocation, flagged `willRetry: true`), which the - harness folds into one turn. The internal retry is bounded by - `turn_timeout` / `task_timeout`. - -Full detail: [Pi](PI.md). +## Agent fields per harness + +Claude Code, Pi, OpenCode and Antigravity honor `system_prompt`, `permission_mode`, +`allowed_tools` and `disallowed_tools` natively. Codex honors `system_prompt` only. See +each harness page for the mechanism: [Pi](PI.md#config-fields-and-their-pi-flags), +[OpenCode](OPENCODE.md#permissions), [Antigravity](ANTIGRAVITY.md), [Codex](CODEX.md). ## Reproducing diff --git a/docs/agents/OPENCODE.md b/docs/agents/OPENCODE.md index 09327d30f..17d188f7f 100644 --- a/docs/agents/OPENCODE.md +++ b/docs/agents/OPENCODE.md @@ -170,10 +170,32 @@ skills under test otherwise looks entirely normal. ## Permissions -Every `permission_mode` except `plan` passes `--auto`, auto-approving tool use. -This is required for unattended evaluation — without it OpenCode blocks on an -interactive approval prompt and the turn runs to its timeout. Use -`permission_mode: plan` when you explicitly want approvals withheld. +Every run passes `--auto`, which auto-approves each permission that is not +explicitly denied. Without it OpenCode blocks on an interactive approval prompt and +the turn runs to its timeout. + +The uniform fields become OpenCode `permission` rules, merged into +`OPENCODE_CONFIG_CONTENT` beside `skills.paths` (an inherited value is merged, and +our keys win): + +| Field | `permission` rules | +|---|---| +| `allowed_tools` | `"*": "deny"`, `"allow"` for `external_directory` and `doom_loop`, then `"allow"` for each mapped key | +| `disallowed_tools` | `"deny"` for each mapped key (a deny always wins) | +| `permission_mode: plan` | `edit: "deny"`, `bash: "deny"` (read-only) | + +Claude tool names map to permission keys by inverting the telemetry map. OpenCode's +keys are coarser than its tools: `edit` governs `write`, `edit`, `patch`, +`multiedit` and `apply_patch`, so `Write` and `Edit` restrict together. A name with +no OpenCode equivalent restricts nothing; an allowlist of only such names denies +every tool. An empty `allowed_tools: []` restricts nothing, as on Claude Code. Our +rules are placed after every inherited rule, because OpenCode applies the last +matching rule. A string rule such as `read: "allow"` replaces the CLI's default +`.env` read deny, which is acceptable inside a sandbox. + +`system_prompt` is written to a temporary file outside the sandbox and listed in +`instructions`, which OpenCode appends to its own system messages (semantics +`append`). The file is removed at `stop()`. ## Telemetry @@ -310,11 +332,8 @@ any other provider credential can be added via `sandbox.env_passthrough_extra`. ## Known limitations -- **`allowed_tools` / `disallowed_tools` / `system_prompt` / `system_prompt_file` - are not enforced.** The CLI exposes no equivalent knob, so these are - dropped — `start()` logs a warning naming each one it saw (`experiments/default.yaml` - sets `allowed_tools` on every task, so expect it on a default run). Do not rely on - them as a boundary here. +- **Tool restrictions are not a confinement boundary.** They limit which tools the + model may call, not what a permitted `bash` call can do. - **Only the *skills* half of a `plugins:` entry is honored** (see below). A Claude plugin's agents, hooks, commands and MCP servers have no OpenCode equivalent and are still dropped. diff --git a/docs/agents/PI.md b/docs/agents/PI.md index eb5151971..7b35bf089 100644 --- a/docs/agents/PI.md +++ b/docs/agents/PI.md @@ -25,14 +25,10 @@ and lets `EventCollector` build the `TurnRecord`, exactly like every other harness. Because Pi is model-agnostic, this is a cheap way to evaluate a broad set of -open-weight models (Kimi, DeepSeek, GLM, …) through a single agent. Pi **enforces -`system_prompt`** (via `--append-system-prompt`) — a small win over OpenCode. It -does **not** enforce `allowed_tools` / `disallowed_tools`: the shared config -default uses Claude-namespaced tool names (`Bash`/`Read`/`Write`/…) that do not -match Pi's lowercase built-ins (`bash`/`read`/`write`/…), so forwarding them would -leave the agent with no tools at all. Like OpenCode / Codex / Antigravity, Pi -therefore runs with its full native toolset and warns that these fields are -unenforced. +open-weight models (Kimi, DeepSeek, GLM, …) through a single agent. Pi honors +`system_prompt` (via `--append-system-prompt`), `allowed_tools`, +`disallowed_tools` and `permission_mode: plan` (via `--tools` / +`--exclude-tools`). ## Setup @@ -112,38 +108,29 @@ Pi's reasoning effort, forwarded as `--thinking`. Accepts the seven-value set `off` / `minimal` / `low` / `medium` / `high` / `xhigh` / `max` (a strict superset of the Antigravity `thinking_level`), defaulting to `medium`. -### Enforced config fields - -Pi forwards these config knobs to real CLI flags: +### Config fields and their Pi flags | Field | Pi flag | |---|---| | `system_prompt` | `--append-system-prompt ` (appended, semantics `append`) | +| `allowed_tools` | `--tools `, or `--no-tools` when no name has a Pi equivalent | +| `disallowed_tools` | `--exclude-tools ` (subtracted from `--tools` when both are set) | +| `permission_mode: plan` | the Write, Edit and Bash equivalents added to the denied set | | `thinking_level` | `--thinking ` | | `model` | `--model ` | +| `plugins` | `--skill ` per resolved skills dir | -### Unenforced config fields - -`allowed_tools` / `disallowed_tools` are **not** forwarded. The shared config -default (`experiments/default.yaml`) sets Claude-namespaced tool names -(`Bash`/`Read`/`Write`/`Edit`/`Glob`/`Grep`/`Skill`), but Pi's built-in tools are -lowercase and differently named (`bash`/`read`/`write`/`edit`/`grep`/`find`/`ls`). -Passing the PascalCase names to `--tools` would allowlist tools that do not exist -in Pi, leaving the agent with **zero** tools. So — like OpenCode, Codex, and -Antigravity — Pi ignores these fields, runs with its full native toolset, and -warns at `start()` that they are unenforced. `permission_mode` and -`system_prompt_file` are unenforced too (see below); `plugins` **is** honored for -its skills half (each resolved skills dir → a `--skill ` arg). +Claude tool names map to Pi's lowercase built-ins by inverting the telemetry map +(`Bash` → `bash`, `Edit` → `edit,multiedit,patch`, `Glob` → `find`, `LS` → +`list,ls`, …). A name with no Pi equivalent restricts nothing; an allowlist of only +such names disables every tool. An empty `allowed_tools: []` restricts nothing, as on +Claude Code. ## Permissions -Pi headless print mode auto-runs tools; it exposes only project-file trust -(`--approve` / `--no-approve`), not a tool-approval mode. Coder Eval always -passes `--no-approve` so the run never blocks. **`permission_mode` is therefore -not enforced** — the sandbox driver is the isolation boundary (the same posture -as Codex and Antigravity). `start()` logs a warning naming `permission_mode` -(and any other unenforced field it saw) so a task never silently believes it was -constrained. +`permission_mode: plan` is read-only: the Write, Edit and Bash equivalents are +denied. Every other mode runs autonomously. Pi headless print mode auto-runs +tools, and Coder Eval always passes `--no-approve` so the run never blocks. ## Multi-turn and simulation @@ -226,16 +213,14 @@ export OPENROUTER_API_KEY="sk-or-..." uv run coder-eval run tasks/pi_smoke_test.yaml --driver docker ``` -**Docker is the recommended driver for untrusted / adversarial Pi runs.** Pi's -`permission_mode` is unenforced — headless print mode auto-runs tools — so the -**container is the confinement boundary**. Under `tempdir` there is no such +**Docker is the recommended driver for untrusted / adversarial Pi runs.** Tool +restrictions limit which tools the model may call, not what a permitted `bash` +call can do, so the **container is the confinement boundary**. Under `tempdir` there is no such boundary; the agent runs with the host's own permissions. Prefer `--driver docker` whenever the task prompt or workspace is not fully trusted. ## Known limitations -- **`permission_mode` is not enforced.** Pi headless print mode auto-runs tools; - the sandbox driver is the isolation boundary (same as Codex/Antigravity). - **`plugins` skills are injected via `--skill`.** Each `type: local` plugin root is resolved to its skills dir (`/skills`, holding `/SKILL.md`) and passed to the CLI as a `--skill ` argument — the same `_plugin_skill_dirs` @@ -249,9 +234,8 @@ docker` whenever the task prompt or workspace is not fully trusted. suite scores recall 0 even though the skill ran — see [Harness Parity § plugin-path depth](HARNESS_PARITY.md). A plugin's non-skill assets (agents/hooks/commands/MCP) are not wired. -- **`system_prompt_file` is not read.** Use `system_prompt` (inline) instead — it - is enforced via `--append-system-prompt`. `system_prompt_file` is warned about - at `start()` (matching Codex/Antigravity, which also do not read the file form). +- **`system_prompt_file` is not read by the adapter.** Use `system_prompt` (inline) + instead — it is enforced via `--append-system-prompt`. - **`max_turns` counts Pi's native agent-loop turns.** One `turn_start` = one agent-loop step; `max_turns: N` allows N complete turns, then the run finalizes cleanly as `max_turns_exhausted`. See diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 7d9222b72..6e4c13eda 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -38,6 +38,7 @@ truncate_crash_message, ) from coder_eval.models import ( + READ_ONLY_DENIED_TOOLS, AgentKind, AntigravityAgentConfig, ApiRoute, @@ -47,6 +48,7 @@ DirectRoute, Enforcement, HarnessContract, + PermissionMode, TokenUsage, TranscriptMessage, TurnRecord, @@ -114,13 +116,23 @@ "search_directory": "Grep", "find_file": "Glob", "list_directory": "LS", - "start_subagent": "Task", + "start_subagent": "Agent", "search_web": "WebSearch", + "read_url_content": "WebFetch", "generate_image": "GenerateImage", "ask_question": "AskUser", "finish": "Finish", } +# Inverse of _ANTIGRAVITY_TO_CLAUDE_TOOL_MAP: each Claude name -> its harness tools. +_CLAUDE_TO_ANTIGRAVITY_TOOLS: dict[str, tuple[str, ...]] = { + claude: tuple(sorted(tool for tool, name in _ANTIGRAVITY_TO_CLAUDE_TOOL_MAP.items() if name == claude)) + for claude in set(_ANTIGRAVITY_TO_CLAUDE_TOOL_MAP.values()) +} + +# The harness ends a turn by calling `finish`, so an allowlist never denies it. +_TURN_END_TOOL = "finish" + # Tool-call arg keys the harness ADDS at completion (the result payload), not # model-supplied inputs. The STATIC backstop; ``_params`` also strips any key # that first appears at DONE. A leaked result would false-positive @@ -190,9 +202,9 @@ class AntigravityAgent(Agent[AntigravityAgentConfig]): system_prompt=Enforcement.ENFORCED, system_prompt_semantics="append", plugin_skills=Enforcement.ENFORCED, - permission_mode=Enforcement.UNSUPPORTED, - allowed_tools=Enforcement.UNSUPPORTED, - disallowed_tools=Enforcement.UNSUPPORTED, + permission_mode=Enforcement.ENFORCED, + allowed_tools=Enforcement.ENFORCED, + disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, ) @@ -308,6 +320,24 @@ def _harness_env(self) -> dict[str, str] | None: merged = os.pathsep.join([*self._env_path_prepend, os.environ.get(path_key) or ""]) return {path_key: merged} + def _policies(self, policy: Any) -> list[Any]: + """Tool-call policies from the uniform tool fields, built with the SDK's ``policy`` module. + + No allowlist approves every call (autonomous execution; the SDK default + would deny ``run_command``). A specific deny outranks a specific allow in + the SDK, so a denied or ``plan``-denied tool stays denied. + """ + if not self.config.allowed_tools: + policies = [policy.allow_all()] + else: + allowed = {t for name in self.config.allowed_tools for t in _CLAUDE_TO_ANTIGRAVITY_TOOLS.get(name, ())} + policies = [policy.deny_all(), *(policy.allow(t) for t in sorted(allowed | {_TURN_END_TOOL}))] + deny_names = list(self.config.disallowed_tools or []) + if self.config.permission_mode is PermissionMode.PLAN: + deny_names += READ_ONLY_DENIED_TOOLS + denied = {t for name in deny_names for t in _CLAUDE_TO_ANTIGRAVITY_TOOLS.get(name, ())} - {_TURN_END_TOOL} + return policies + [policy.deny(t) for t in sorted(denied)] + async def start( self, working_directory: str, @@ -353,12 +383,7 @@ async def start( # workspace_only policy — see _resolve_workspaces for why the skill # roots must be in here and not only in ``skills_paths``. workspaces=self._resolve_workspaces(skills_paths), - # Autonomous execution: approve every tool call, which the default - # policy would deny. ``permission_mode`` is deliberately NOT mapped - # here — it does not confine this agent, exactly as on Codex, and - # docs/agents/HARNESS_PARITY.md says so rather than leaving it - # silent. The isolation boundary is the driver. - policies=[policy.allow_all()], + policies=self._policies(policy), system_instructions=self.config.system_prompt or None, # Skill discovery: the search-path roots that parent the skill dirs. skills_paths=skills_paths, diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index f3914edad..e1403b7b4 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -60,16 +60,6 @@ logger = logging.getLogger(__name__) -# Tool name mapping: Claude Code SDK names → Codex SDK names -_CLAUDE_TO_CODEX_TOOL_MAP: dict[str, str] = { - "Bash": "shell", - "Write": "apply_patch", - "Edit": "apply_patch", - "Read": "shell", - "Grep": "shell", - "Glob": "shell", -} - # Approval mode — the SAME for every permission mode. Despite the name this is # the "run autonomously, never prompt, no reviewer" mode: in-sandbox operations # execute directly, and only escalations BEYOND the sandbox are refused. The @@ -1400,24 +1390,8 @@ def _build_thread_options(self) -> dict[str, Any]: self._log.debug(f"Permission mode {permission_mode} → sandbox={sandbox_name}, approval_mode={approval_name}") - # Build config dict for tool enforcement tool_config: dict[str, Any] = {} - if self.config.allowed_tools: - enabled_tools = [_CLAUDE_TO_CODEX_TOOL_MAP.get(tool, tool) for tool in self.config.allowed_tools] - tool_config["enabled_tools"] = enabled_tools - normalized = ", ".join(enabled_tools) - self._log.debug(f"Allowed tools (normalized): {normalized}") - - if self.config.disallowed_tools: - disabled_tools = [_CLAUDE_TO_CODEX_TOOL_MAP.get(tool, tool) for tool in self.config.disallowed_tools] - tool_config["disabled_tools"] = disabled_tools - normalized = ", ".join(disabled_tools) - self._log.warning( - f"disallowed_tools ({normalized}) is passed to Codex but NOT enforced by the SDK; " - + "do not rely on it as a security boundary." - ) - # The codex binary has no base-URL env var: a model provider must be # defined in config and selected, with env_key naming the key's variable. # For Azure, CODEX_API_VERSION adds the required query param and diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index a18a46aa5..e532ea6fa 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -24,15 +24,18 @@ import os import shutil import signal +import tempfile import time from collections.abc import Callable from datetime import datetime +from pathlib import Path from typing import Any, Literal, NoReturn from coder_eval.agent import Agent from coder_eval.errors import AgentCrashError, TurnTimeoutError from coder_eval.isolation.docker_runner import STDOUT_LINE_LIMIT_BYTES from coder_eval.models import ( + READ_ONLY_DENIED_TOOLS, AgentKind, AgentState, ApiRoute, @@ -149,23 +152,38 @@ "Skill": {"name": "skill"}, } -# Config fields the OpenCode CLI has no equivalent knob for. `experiments/default.yaml` -# sets `allowed_tools` on every task, so start() warns once rather than letting a -# task believe it constrained the agent. `plugins` is NOT here: its skills half is -# honored. Per-harness table: docs/agents/HARNESS_PARITY.md. -_UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ( - "system_prompt", - "system_prompt_file", - "allowed_tools", - "disallowed_tools", -) +# OpenCode's permission keys are coarser than its tool names: `edit` governs every +# write-shaped tool. +_PERMISSION_KEY_FOR_TOOL: dict[str, str] = { + "write": "edit", + "patch": "edit", + "multiedit": "edit", + "apply_patch": "edit", +} + +# Permissions `"*"` also matches that are not tools. An allowlist re-allows them, so it +# restricts tools only; `--auto` approved them before. +_NON_TOOL_PERMISSIONS: tuple[str, ...] = ("external_directory", "doom_loop") + +# Inverse of _TOOL_NAME_MAP: each Claude name -> the permission keys that govern it. +_CLAUDE_TO_OPENCODE_PERMISSION: dict[str, tuple[str, ...]] = { + claude: tuple( + sorted({_PERMISSION_KEY_FOR_TOOL.get(tool, tool) for tool, name in _TOOL_NAME_MAP.items() if name == claude}) + ) + for claude in set(_TOOL_NAME_MAP.values()) +} + +# `system_prompt_file` is inlined into `system_prompt` before the agent runs. +_UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ("system_prompt_file",) -# Skill injection: each plugin root's skills dir is merged into `skills.paths` -# through this variable, which OpenCode applies as a final local-scope layer. +# Skill paths, the system-prompt `instructions` file and tool `permission` rules are +# merged through this variable, which OpenCode applies as a final local-scope layer. # Only the SKILLS half of a plugin is honored. # Rationale: .claude/notes/agents.md § Skills, per harness _CONFIG_CONTENT_ENV = "OPENCODE_CONFIG_CONTENT" +_PROMPT_FILE_NAME = "system_prompt.md" + # ToolEndStatus -> CommandTelemetry.result_status (the persisted tri-state). _RESULT_STATUS: dict[ToolEndStatus, Literal["success", "error", "unknown"]] = { ToolEndStatus.OK: "success", @@ -731,15 +749,16 @@ def finalize( class OpenCodeAgent(Agent[OpenCodeAgentConfig]): """Runs the ``opencode`` CLI as a subprocess, one invocation per turn.""" - # `should_stop` is polled at every event boundary (tool-call granularity). - # No CLI knob for `system_prompt`, so the honest regime is `"unknown"`. + # `should_stop` is polled at every event boundary (tool-call granularity); + # `system_prompt` joins the CLI's own system messages as an `instructions` file. # Rationale: .claude/notes/agents.md § The system_prompt_semantics marker contract = HarnessContract( - system_prompt=Enforcement.UNSUPPORTED, + system_prompt=Enforcement.ENFORCED, + system_prompt_semantics="append", plugin_skills=Enforcement.ENFORCED, - permission_mode=Enforcement.UNSUPPORTED, - allowed_tools=Enforcement.UNSUPPORTED, - disallowed_tools=Enforcement.UNSUPPORTED, + permission_mode=Enforcement.ENFORCED, + allowed_tools=Enforcement.ENFORCED, + disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, ) @@ -766,6 +785,9 @@ def __init__( self._env_path_prepend: list[str] = [] self._plugin_tools_dir: str | None = None self._skill_dirs: list[str] = [] + # Holds the system prompt as an `instructions` file; created in start(), + # removed in stop(). Never inside the sandbox. + self._prompt_dir: str | None = None self._session_id: str | None = None self._process: asyncio.subprocess.Process | None = None # Process-group ids of every invocation this agent spawned, swept on @@ -811,6 +833,7 @@ async def start( + "WITHOUT them (see docs/agents/OPENCODE.md).", len(self.config.plugins), ) + await asyncio.to_thread(self._write_prompt_file) self.working_directory = working_directory self._env_path_prepend = list(env_path_prepend or []) self._plugin_tools_dir = plugin_tools_dir @@ -819,8 +842,20 @@ async def start( async def stop(self) -> None: await self.kill() + self._remove_prompt_dir() self._mark_stopped() + def _write_prompt_file(self) -> None: + self._remove_prompt_dir() + if self.config.system_prompt: + self._prompt_dir = tempfile.mkdtemp(prefix="coder-eval-opencode-") + Path(self._prompt_dir, _PROMPT_FILE_NAME).write_text(self.config.system_prompt, encoding="utf-8") + + def _remove_prompt_dir(self) -> None: + if self._prompt_dir is not None: + shutil.rmtree(self._prompt_dir, ignore_errors=True) + self._prompt_dir = None + async def kill(self) -> None: proc = self._process if proc is not None and proc.returncode is None: @@ -886,10 +921,9 @@ def _build_argv(self, user_input: str) -> list[str]: argv += ["--variant", self.config.variant] if self.config.pure: argv.append("--pure") - # PLAN is the one mode that must not auto-approve side effects; every - # other runs unattended, where an approval prompt would simply hang. - if self.config.permission_mode is not PermissionMode.PLAN: - argv.append("--auto") + # Auto-approves only what `permission` does not explicitly deny; an + # approval prompt would hang an unattended run. + argv.append("--auto") if self._session_id: argv += ["--session", self._session_id] argv.append("--") @@ -916,17 +950,37 @@ def _build_env(self) -> dict[str, str]: env["PATH"] = os.pathsep.join([*self._env_path_prepend, env.get("PATH", "")]) if self._plugin_tools_dir and "PLUGIN_TOOLS_DIR" not in env: env["PLUGIN_TOOLS_DIR"] = self._plugin_tools_dir - self._inject_skill_paths(env) + self._inject_config_content(env) return env - def _inject_skill_paths(self, env: dict[str, str]) -> None: - """Merge the resolved skill directories into ``OPENCODE_CONFIG_CONTENT``. + def _permission_config(self) -> dict[str, str] | None: + """OpenCode ``permission`` rules for the uniform tool fields; None when none is set. - No plugins means the variable is left exactly as inherited. An inherited - value is appended to, never clobbered: the host may legitimately configure - OpenCode through the same seam. + An allowlist denies ``*`` and allows the mapped keys. Denies are written + last, so a disallowed or ``plan``-denied key always wins. """ - if not self._skill_dirs: + deny_names = list(self.config.disallowed_tools or []) + if self.config.permission_mode is PermissionMode.PLAN: + deny_names += READ_ONLY_DENIED_TOOLS + permission: dict[str, str] = {} + if self.config.allowed_tools: + permission["*"] = "deny" + permission.update(dict.fromkeys(_NON_TOOL_PERMISSIONS, "allow")) + for name in self.config.allowed_tools: + permission.update(dict.fromkeys(_CLAUDE_TO_OPENCODE_PERMISSION.get(name, ()), "allow")) + for name in deny_names: + permission.update(dict.fromkeys(_CLAUDE_TO_OPENCODE_PERMISSION.get(name, ()), "deny")) + return permission or None + + def _inject_config_content(self, env: dict[str, str]) -> None: + """Merge skill paths, the prompt file and permission rules into ``OPENCODE_CONFIG_CONTENT``. + + With none of the three the variable is left exactly as inherited. An + inherited value is merged, never clobbered: the host may legitimately + configure OpenCode through the same seam. Our entries win per key. + """ + permission = self._permission_config() + if not (self._skill_dirs or self._prompt_dir or permission): return config: dict[str, Any] = {} inherited = env.get(_CONFIG_CONTENT_ENV) @@ -935,7 +989,7 @@ def _inject_skill_paths(self, env: dict[str, str]) -> None: parsed = json.loads(inherited) except json.JSONDecodeError: logger.warning( - "opencode: inherited %s is not valid JSON; replacing it with the injected skill paths.", + "opencode: inherited %s is not valid JSON; replacing it with the injected config.", _CONFIG_CONTENT_ENV, ) else: @@ -943,14 +997,31 @@ def _inject_skill_paths(self, env: dict[str, str]) -> None: config = parsed else: logger.warning( - "opencode: inherited %s is not a JSON object; replacing it with the injected skill paths.", + "opencode: inherited %s is not a JSON object; replacing it with the injected config.", _CONFIG_CONTENT_ENV, ) - skills = config.get("skills") - skills = dict(skills) if isinstance(skills, dict) else {} - existing = [path for path in skills.get("paths", []) if isinstance(path, str)] - skills["paths"] = existing + [path for path in self._skill_dirs if path not in existing] - config["skills"] = skills + if self._skill_dirs: + skills = config.get("skills") + skills = dict(skills) if isinstance(skills, dict) else {} + existing = [path for path in skills.get("paths", []) if isinstance(path, str)] + skills["paths"] = existing + [path for path in self._skill_dirs if path not in existing] + config["skills"] = skills + if self._prompt_dir is not None: + prompt_file = str(Path(self._prompt_dir, _PROMPT_FILE_NAME)) + inherited_files = config.get("instructions") + inherited_files = inherited_files if isinstance(inherited_files, list) else [] + config["instructions"] = [f for f in inherited_files if isinstance(f, str) and f != prompt_file] + [ + prompt_file + ] + if permission: + # OpenCode applies the LAST matching rule, so ours go after every inherited one. + inherited_rules = config.get("permission") + kept = ( + {k: v for k, v in inherited_rules.items() if k not in permission} + if isinstance(inherited_rules, dict) + else {} + ) + config["permission"] = {**kept, **permission} env[_CONFIG_CONTENT_ENV] = json.dumps(config) # --- the turn ---------------------------------------------------------- diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 5843613be..e9440f9bd 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -43,6 +43,7 @@ from coder_eval.errors import AgentCrashError, TurnTimeoutError from coder_eval.isolation.docker_runner import STDOUT_LINE_LIMIT_BYTES from coder_eval.models import ( + READ_ONLY_DENIED_TOOLS, AgentKind, AgentState, ApiRoute, @@ -51,6 +52,7 @@ ContentBlock, Enforcement, HarnessContract, + PermissionMode, PiAgentConfig, ResultSummary, TokenUsage, @@ -135,20 +137,14 @@ }, } -# Config fields Pi does NOT enforce. `experiments/default.yaml` sets -# `permission_mode` and `allowed_tools` on every task, so start() warns once -# rather than letting a task believe it constrained the agent. `system_prompt` -# and `plugins` ARE supported, so neither is here. Per-harness table: -# docs/agents/HARNESS_PARITY.md. -_UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ( - "permission_mode", - "system_prompt_file", - # Forwarding these to --tools would allowlist nonexistent tools and strip the - # agent of ALL tools: Pi's built-ins are lowercase. - # Rationale: .claude/notes/agents.md § Harness run-limit parity - "allowed_tools", - "disallowed_tools", -) +# Inverse of _TOOL_NAME_MAP: each Claude name -> every Pi tool it stands for. +_CLAUDE_TO_PI_TOOLS: dict[str, tuple[str, ...]] = { + claude: tuple(sorted(pi for pi, name in _TOOL_NAME_MAP.items() if name == claude)) + for claude in set(_TOOL_NAME_MAP.values()) +} + +# `system_prompt_file` is inlined into `system_prompt` before the agent runs. +_UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ("system_prompt_file",) # The full recognized Pi vocabulary (from `pi` 0.84.4). A clean exit that # recognized NOTHING from this set is vocabulary drift and is crashed, not scored. @@ -687,9 +683,9 @@ class PiAgent(Agent[PiAgentConfig]): system_prompt=Enforcement.ENFORCED, system_prompt_semantics="append", plugin_skills=Enforcement.ENFORCED, - permission_mode=Enforcement.UNSUPPORTED, - allowed_tools=Enforcement.UNSUPPORTED, - disallowed_tools=Enforcement.UNSUPPORTED, + permission_mode=Enforcement.ENFORCED, + allowed_tools=Enforcement.ENFORCED, + disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, ) @@ -864,14 +860,28 @@ def _build_argv(self, user_input: str) -> list[str]: # Additive skill load (from agent.plugins): Pi lists each skill's # name+description in the system prompt and reads SKILL.md on demand. argv += ["--skill", skill_dir] - # allowed_tools / disallowed_tools are NOT forwarded — see - # _UNSUPPORTED_CONFIG_FIELDS. Pi runs with its full native toolset. + argv += self._tool_flags() if self.config.system_prompt: argv += ["--append-system-prompt", self.config.system_prompt] # user_input is a distinct argv element after `--` (never shell-interpolated). argv += ["--", user_input] return argv + def _tool_flags(self) -> list[str]: + """``--tools`` / ``--no-tools`` / ``--exclude-tools`` from the uniform tool fields. + + A deny always wins: denied names are subtracted from the allowlist, and + ``permission_mode: plan`` denies the Write, Edit and Bash equivalents. + """ + deny_names = list(self.config.disallowed_tools or []) + if self.config.permission_mode is PermissionMode.PLAN: + deny_names += READ_ONLY_DENIED_TOOLS + deny = {pi for name in deny_names for pi in _CLAUDE_TO_PI_TOOLS.get(name, ())} + if self.config.allowed_tools: + allow = {pi for name in self.config.allowed_tools for pi in _CLAUDE_TO_PI_TOOLS.get(name, ())} - deny + return ["--tools", ",".join(sorted(allow))] if allow else ["--no-tools"] + return ["--exclude-tools", ",".join(sorted(deny))] if deny else [] + def _build_env(self) -> dict[str, str]: """The CLI's full environment: the host's, plus the sandbox's contributions. diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index 2f485e5c7..db6981473 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -70,6 +70,7 @@ UiPathEvalCriterion, ) from coder_eval.models.enums import ( + READ_ONLY_DENIED_TOOLS, AgentKind, AgentState, ApiBackend, @@ -257,6 +258,7 @@ "FinalStatus", "PermissionMode", "PreservationMode", + "READ_ONLY_DENIED_TOOLS", # Criteria "BaseSuccessCriterion", "ClassificationMatchCriterion", diff --git a/src/coder_eval/models/agent_config.py b/src/coder_eval/models/agent_config.py index 5b74a746b..c16b9916d 100644 --- a/src/coder_eval/models/agent_config.py +++ b/src/coder_eval/models/agent_config.py @@ -325,9 +325,10 @@ class OpenCodeAgentConfig(BaseAgentConfig): on stdout. ``model`` is OpenCode's ``provider/model`` form (e.g. ``deepseek/deepseek-v4-pro``) and is passed through verbatim via ``-m``. - Permission handling is derived from the inherited ``permission_mode``: every - mode except :attr:`PermissionMode.PLAN` passes ``--auto`` so an unattended - eval run never blocks on an interactive approval prompt. + Every run passes ``--auto``. ``allowed_tools`` / ``disallowed_tools`` and + ``permission_mode: plan`` (read-only) become explicit ``permission`` denies, and + ``system_prompt`` is appended as an ``instructions`` file, both merged into + ``OPENCODE_CONFIG_CONTENT``. See ``docs/agents/OPENCODE.md``. """ type: Literal[AgentKind.OPENCODE] # type: ignore[assignment] @@ -376,11 +377,10 @@ class PiAgentConfig(BaseAgentConfig): Each ``communicate()`` is one ``pi`` subprocess, so a dialog task relies on a per-agent ``--session-dir`` + stable ``--session-id`` for continuity. - ``system_prompt`` IS enforced. ``allowed_tools`` / ``disallowed_tools`` are NOT - forwarded (the shared defaults name Claude-namespaced tools that do not exist in - Pi's toolset, so forwarding them would strip the agent of ALL tools), nor is - ``permission_mode``, nor is ``system_prompt_file`` read -- both warned about at - ``start()``. ``plugins`` skills ARE injected. See ``docs/agents/PI.md``. + ``system_prompt`` is appended (``--append-system-prompt``). ``allowed_tools`` / + ``disallowed_tools`` become ``--tools`` / ``--exclude-tools`` over Pi's lowercase + built-ins, and ``permission_mode: plan`` denies the Write, Edit and Bash + equivalents. ``plugins`` skills are injected. See ``docs/agents/PI.md``. Rationale: .claude/notes/agents.md § Pi """ diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index 6baa96dba..805768f91 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -1,7 +1,7 @@ """Enumeration types for coder_eval.""" from enum import StrEnum -from typing import Literal +from typing import Final, Literal class FinalStatus(StrEnum): @@ -119,6 +119,10 @@ class PermissionMode(StrEnum): BYPASS_PERMISSIONS = "bypassPermissions" +READ_ONLY_DENIED_TOOLS: Final[tuple[str, ...]] = ("Write", "Edit", "Bash") +"""The Claude tool names every harness that maps ``permission_mode: plan`` denies.""" + + class PreservationMode(StrEnum): """How a task's sandbox is persisted (or not) after execution. diff --git a/tasks/agents/codex_disallowed_tools_test.yaml b/tasks/agents/codex_disallowed_tools_test.yaml deleted file mode 100644 index 46f956728..000000000 --- a/tasks/agents/codex_disallowed_tools_test.yaml +++ /dev/null @@ -1,31 +0,0 @@ -task_id: codex_disallowed_tools_test -description: | - Test that disallowed_tools (Write, Edit, Bash) prevents file creation. - NOTE: Currently skipped - Codex SDK doesn't enforce disabled_tools through config parameter. - This is a known limitation that requires Codex SDK enhancement. - -tags: - - codex - - permissions - - skipped - -skip: true - -agent: - type: codex - model: gpt-5.5 - permission_mode: acceptEdits - disallowed_tools: - - Write - - Edit - - Bash - -initial_prompt: | - Please create a file called `restricted.txt` with the content "This should not exist". - You can use any tools available to you. - -success_criteria: - - type: run_command - description: File should NOT exist (disallowed tools enforcement test) - command: test ! -f restricted.txt - expected_exit_code: 0 diff --git a/tasks/agents/codex_hello_world.yaml b/tasks/agents/codex_hello_world.yaml index 25609e13c..3c0bb6a32 100644 --- a/tasks/agents/codex_hello_world.yaml +++ b/tasks/agents/codex_hello_world.yaml @@ -12,10 +12,6 @@ agent: type: codex model: gpt-5.5 permission_mode: acceptEdits - allowed_tools: - - Bash - - Read - - Write initial_prompt: | Create a simple Python script called `hello.py` that prints "Hello, Codex!" when run. diff --git a/tasks/agents/codex_parallel_commands.yaml b/tasks/agents/codex_parallel_commands.yaml index 380cc0e6c..94c6928ab 100644 --- a/tasks/agents/codex_parallel_commands.yaml +++ b/tasks/agents/codex_parallel_commands.yaml @@ -16,10 +16,6 @@ agent: type: codex model: gpt-5.5 permission_mode: acceptEdits - allowed_tools: - - Bash - - Read - - Write initial_prompt: | Run these two independent commands IN PARALLEL — issue them together as two diff --git a/tasks/agents/codex_parallel_single_gen.yaml b/tasks/agents/codex_parallel_single_gen.yaml index 143c71c3c..5fb73c0ad 100644 --- a/tasks/agents/codex_parallel_single_gen.yaml +++ b/tasks/agents/codex_parallel_single_gen.yaml @@ -17,10 +17,6 @@ agent: type: codex model: gpt-5.5 permission_mode: acceptEdits - allowed_tools: - - Bash - - Read - - Write initial_prompt: | Gather two completely independent pieces of system information at once. Issue diff --git a/tasks/agents/codex_string_utils.yaml b/tasks/agents/codex_string_utils.yaml index a025ed24c..3c626b47b 100644 --- a/tasks/agents/codex_string_utils.yaml +++ b/tasks/agents/codex_string_utils.yaml @@ -13,11 +13,6 @@ agent: type: codex model: gpt-5.5 permission_mode: acceptEdits - allowed_tools: - - Bash - - Read - - Write - - Grep initial_prompt: | Create a Python module named `string_utils.py` with the following functions: diff --git a/tasks/agents/codex_subagent_test.yaml b/tasks/agents/codex_subagent_test.yaml index 3cc827384..5f32e90d0 100644 --- a/tasks/agents/codex_subagent_test.yaml +++ b/tasks/agents/codex_subagent_test.yaml @@ -15,10 +15,6 @@ agent: type: codex model: gpt-5.5 permission_mode: acceptEdits - allowed_tools: - - Bash - - Read - - Write initial_prompt: | Delegate this to a SUB-AGENT (spawn one via the Agent tool). Instruct the diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index 117c8f892..93bc9d349 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -490,6 +490,7 @@ def _install_fake_sdk(monkeypatch, sdk_agent_cls) -> None: hooks = ModuleType("google.antigravity.hooks") hooks.policy = SimpleNamespace( allow_all=lambda: SimpleNamespace(kind="allow_all"), + deny_all=lambda: SimpleNamespace(kind="deny_all"), deny=lambda tool, **kw: SimpleNamespace(kind="deny", tool=tool), allow=lambda tool, **kw: SimpleNamespace(kind="allow", tool=tool), ) @@ -1329,26 +1330,58 @@ async def __aexit__(self, *exc): assert configs[0].env is None -# --- permission_mode ---------------------------------------------------------------- -# -# The local harness has one mode: policies are hardcoded to allow_all, so no -# permission_mode confines it. These pin that as intended behavior rather than an -# oversight — the write boundary is the sandbox driver, and a headless eval has -# nobody to approve anything. +# --- permission_mode and tool fields -------------------------------------------------- def _agent(**cfg) -> AntigravityAgent: return AntigravityAgent(parse_agent_config(type="antigravity", **cfg)) -@pytest.mark.parametrize("mode", ["default", "acceptEdits", "plan", "bypassPermissions"]) -async def test_permission_mode_never_confines_the_harness(monkeypatch, tmp_path, mode: str): - """permission_mode is not honored here: every mode stays fully autonomous. +def _policy_pairs(**cfg) -> list[tuple[str, str | None]]: + policy = SimpleNamespace( + allow_all=lambda: SimpleNamespace(kind="allow_all"), + deny_all=lambda: SimpleNamespace(kind="deny_all"), + deny=lambda tool: SimpleNamespace(kind="deny", tool=tool), + allow=lambda tool: SimpleNamespace(kind="allow", tool=tool), + ) + return [(p.kind, getattr(p, "tool", None)) for p in _agent(**cfg)._policies(policy)] - coder_eval's write boundary is the driver (docker container / ephemeral tempdir), - not the agent — same deliberate stance as Codex. A mode that silently switched the - policy list would make an A/B across harnesses incomparable. - """ + +@pytest.mark.parametrize( + ("cfg", "expected"), + [ + ({}, [("allow_all", None)]), + ({"allowed_tools": ["Bash"]}, [("deny_all", None), ("allow", "finish"), ("allow", "run_command")]), + ({"disallowed_tools": ["Bash"]}, [("allow_all", None), ("deny", "run_command")]), + ( + {"permission_mode": "plan"}, + [("allow_all", None), ("deny", "create_file"), ("deny", "edit_file"), ("deny", "run_command")], + ), + ({"allowed_tools": ["Skill"]}, [("deny_all", None), ("allow", "finish")]), + ({"allowed_tools": []}, [("allow_all", None)]), + ( + {"allowed_tools": ["Read"], "disallowed_tools": ["Finish"]}, + [("deny_all", None), ("allow", "finish"), ("allow", "view_file")], + ), + ( + {"allowed_tools": ["Bash", "Read"], "disallowed_tools": ["Bash"]}, + [ + ("deny_all", None), + ("allow", "finish"), + ("allow", "run_command"), + ("allow", "view_file"), + ("deny", "run_command"), + ], + ), + ], +) +def test_policies_map_the_uniform_fields(cfg: dict[str, Any], expected: list[tuple[str, str | None]]): + assert _policy_pairs(**cfg) == expected + + +@pytest.mark.parametrize("mode", ["default", "acceptEdits", "bypassPermissions"]) +async def test_non_plan_modes_stay_autonomous(monkeypatch, tmp_path, mode: str): + """Only `plan` confines the harness; every other mode approves every call.""" configs: list[Any] = [] class _FakeSdkAgent: @@ -1368,6 +1401,39 @@ async def __aexit__(self, *exc): assert [p.kind for p in configs[0].policies] == ["allow_all"] +async def test_start_hands_the_policies_to_the_sdk(monkeypatch, tmp_path): + configs: list[Any] = [] + + class _FakeSdkAgent: + def __init__(self, cfg): + configs.append(cfg) + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + _install_fake_sdk(monkeypatch, _FakeSdkAgent) + + await _agent(allowed_tools=["Bash"], permission_mode="plan").start(str(tmp_path)) + + assert [(p.kind, getattr(p, "tool", None)) for p in configs[0].policies] == [ + ("deny_all", None), + ("allow", "finish"), + ("allow", "run_command"), + ("deny", "create_file"), + ("deny", "edit_file"), + ("deny", "run_command"), + ] + + +def test_inverse_tool_map_covers_every_claude_name(): + from coder_eval.agents.antigravity_agent import _ANTIGRAVITY_TO_CLAUDE_TOOL_MAP, _CLAUDE_TO_ANTIGRAVITY_TOOLS + + assert set(_CLAUDE_TO_ANTIGRAVITY_TOOLS) == set(_ANTIGRAVITY_TO_CLAUDE_TOOL_MAP.values()) + + # --- max_turns visible-turn cap ----------------------------------------------------- # # max_turns was accepted and never read on this backend, so a task capping turns ran diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index fc8bcf794..d71bcce23 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -397,33 +397,14 @@ def test_build_thread_options_with_plan(self): assert options["sandbox"] == Sandbox.full_access assert options["approval_mode"] == ApprovalMode.deny_all - def test_build_thread_options_with_allowed_tools(self): - """_build_thread_options includes enabled_tools from allowed_tools.""" - config = parse_agent_config( - type=AgentKind.CODEX, - allowed_tools=["Bash", "Read", "Write"], - ) - agent = CodexAgent(config) - - options = agent._build_thread_options() - - assert options is not None - assert "config" in options - assert options["config"]["enabled_tools"] == ["shell", "shell", "apply_patch"] - - def test_build_thread_options_with_disallowed_tools(self): - """_build_thread_options includes disabled_tools from disallowed_tools.""" - config = parse_agent_config( - type=AgentKind.CODEX, - disallowed_tools=["Write", "Edit", "Bash"], - ) - agent = CodexAgent(config) + def test_allowed_tools_do_not_reach_the_thread_config(self, monkeypatch): + """No top-level Codex config key restricts its built-in tools, so nothing is forwarded.""" + monkeypatch.delenv("CODEX_BASE_URL", raising=False) + config = parse_agent_config(type=AgentKind.CODEX, allowed_tools=["Bash"], disallowed_tools=["Write"]) - options = agent._build_thread_options() + options = CodexAgent(config)._build_thread_options() - assert options is not None - assert "config" in options - assert options["config"]["disabled_tools"] == ["apply_patch", "apply_patch", "shell"] + assert "config" not in options def test_build_thread_options_with_no_permission_mode(self): """_build_thread_options is full-access/deny_all even without a permission_mode.""" @@ -439,7 +420,7 @@ def test_build_thread_options_with_no_permission_mode(self): assert options["approval_mode"] == ApprovalMode.deny_all def test_build_thread_options_with_permission_and_tools(self): - """_build_thread_options combines permission_mode and tool config.""" + """permission_mode plan with tools set still runs full-access.""" from openai_codex.api import ApprovalMode, Sandbox # pyright: ignore[reportPrivateImportUsage] config = parse_agent_config( @@ -454,7 +435,6 @@ def test_build_thread_options_with_permission_and_tools(self): assert options is not None assert options["sandbox"] == Sandbox.full_access assert options["approval_mode"] == ApprovalMode.deny_all - assert options["config"]["enabled_tools"] == ["shell", "shell"] @pytest.mark.asyncio diff --git a/tests/test_codex_agent_unit.py b/tests/test_codex_agent_unit.py index ee480f638..1fcec84cc 100644 --- a/tests/test_codex_agent_unit.py +++ b/tests/test_codex_agent_unit.py @@ -1,8 +1,7 @@ """SDK-independent unit tests for CodexAgent. -These tests exercise pure-logic seams of ``codex_agent.py`` — the static -Claude→Codex tool-name map and the per-turn ``_CodexTurnState`` list-mutation -contract — that need NO Codex SDK. ``codex_agent`` imports ``openai_codex`` +These tests exercise pure-logic seams of ``codex_agent.py`` — the per-turn +``_CodexTurnState`` list-mutation contract — that need NO Codex SDK. ``codex_agent`` imports ``openai_codex`` only lazily (inside ``start`` / ``_build_thread_options`` / the turn-completed handler), so the module imports cleanly without the extra and these tests run in the base Quality Gate. @@ -20,7 +19,7 @@ import pytest -from coder_eval.agents.codex_agent import _CLAUDE_TO_CODEX_TOOL_MAP, CodexAgent +from coder_eval.agents.codex_agent import CodexAgent from coder_eval.models import AgentKind, parse_agent_config @@ -28,33 +27,6 @@ def _item_notification(method: str, root: SimpleNamespace) -> SimpleNamespace: return SimpleNamespace(method=method, payload=SimpleNamespace(item=SimpleNamespace(root=root))) -class TestToolNameMapping: - """The static Claude→Codex tool-name map (pure dict, no SDK).""" - - def test_bash_maps_to_shell(self): - assert _CLAUDE_TO_CODEX_TOOL_MAP["Bash"] == "shell" - - def test_write_maps_to_apply_patch(self): - assert _CLAUDE_TO_CODEX_TOOL_MAP["Write"] == "apply_patch" - - def test_edit_maps_to_apply_patch(self): - assert _CLAUDE_TO_CODEX_TOOL_MAP["Edit"] == "apply_patch" - - def test_read_maps_to_shell(self): - """Read maps to shell in Codex (no dedicated read tool).""" - assert _CLAUDE_TO_CODEX_TOOL_MAP["Read"] == "shell" - - def test_grep_maps_to_shell(self): - assert _CLAUDE_TO_CODEX_TOOL_MAP["Grep"] == "shell" - - def test_glob_maps_to_shell(self): - assert _CLAUDE_TO_CODEX_TOOL_MAP["Glob"] == "shell" - - def test_all_tools_mapped(self): - expected_tools = {"Bash", "Write", "Edit", "Read", "Grep", "Glob"} - assert expected_tools.issubset(set(_CLAUDE_TO_CODEX_TOOL_MAP.keys())) - - class TestCodexTurnState: """Unit tests for the per-turn state object extracted from _run_turn_with_streaming.""" diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 0b8c6023c..2ec85a904 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -20,6 +20,7 @@ import os import signal from datetime import datetime, timedelta +from pathlib import Path from typing import Any import pytest @@ -829,18 +830,15 @@ async def test_unusable_inherited_config_is_replaced_with_a_warning( await _run(_agent(plugins=[{"type": "local", "path": str(root)}]), tmp_path / "sandbox") assert _injected_skill_paths(captured) == [str(root / "skills")] - assert "replacing it with the injected skill paths" in caplog.text + assert "replacing it with the injected config" in caplog.text class TestUnsupportedConfigIsAnnounced: - async def test_start_warns_about_unenforced_fields(self, patch_exec, tmp_path, caplog): - """`experiments/default.yaml` sets allowed_tools on every task; the CLI has no - equivalent knob, so silence would let a task believe it was constrained.""" + async def test_enforced_fields_do_not_warn(self, patch_exec, tmp_path, caplog): patch_exec(_FakeProcess(HAPPY_STREAM)) with caplog.at_level("WARNING"): await _agent(allowed_tools=["Bash"], system_prompt="be terse").start(str(tmp_path)) - assert "allowed_tools" in caplog.text - assert "system_prompt" in caplog.text + assert "NOT enforced" not in caplog.text async def test_no_warning_when_nothing_is_dropped(self, patch_exec, tmp_path, caplog): patch_exec(_FakeProcess(HAPPY_STREAM)) @@ -861,10 +859,114 @@ async def test_defaults_include_auto_and_pure(self, patch_exec, tmp_path): assert argv[argv.index("-m") + 1] == "deepseek/deepseek-v4-pro" assert argv[-1] == "do the thing" - async def test_plan_mode_withholds_auto(self, patch_exec, tmp_path): + async def test_plan_mode_keeps_auto_and_denies_writes(self, patch_exec, tmp_path): + """`plan` is explicit denies, so the run stays unattended instead of hanging.""" captured = patch_exec(_FakeProcess(HAPPY_STREAM)) await _run(_agent(permission_mode=PermissionMode.PLAN), tmp_path) - assert "--auto" not in captured["argv"] + assert "--auto" in captured["argv"] + config = json.loads(captured["kwargs"]["env"]["OPENCODE_CONFIG_CONTENT"]) + assert config["permission"] == {"edit": "deny", "bash": "deny"} + + +_NON_TOOL_ALLOWS = {"external_directory": "allow", "doom_loop": "allow"} + + +class TestPermissionConfig: + @pytest.mark.parametrize( + ("cfg", "expected"), + [ + ({}, None), + ({"allowed_tools": []}, None), + ({"allowed_tools": ["Bash"]}, {"*": "deny", **_NON_TOOL_ALLOWS, "bash": "allow"}), + ({"disallowed_tools": ["Bash"]}, {"bash": "deny"}), + ({"permission_mode": "plan"}, {"edit": "deny", "bash": "deny"}), + ({"allowed_tools": ["TodoWrite", "NotATool"]}, {"*": "deny", **_NON_TOOL_ALLOWS, "todowrite": "allow"}), + ({"allowed_tools": ["NotATool"]}, {"*": "deny", **_NON_TOOL_ALLOWS}), + ( + {"allowed_tools": ["Bash", "Write"], "disallowed_tools": ["Bash"]}, + {"*": "deny", **_NON_TOOL_ALLOWS, "bash": "deny", "edit": "allow"}, + ), + ( + {"allowed_tools": ["Bash", "Read"], "permission_mode": "plan"}, + {"*": "deny", **_NON_TOOL_ALLOWS, "bash": "deny", "read": "allow", "edit": "deny"}, + ), + ], + ) + def test_shapes(self, cfg: dict[str, Any], expected: dict[str, str] | None): + assert _agent(**cfg)._permission_config() == expected + + def test_inverse_map_covers_every_claude_name(self): + assert set(agent_module._CLAUDE_TO_OPENCODE_PERMISSION) == set(agent_module._TOOL_NAME_MAP.values()) + + def test_wildcard_deny_comes_first(self): + assert next(iter(_agent(allowed_tools=["Read"])._permission_config() or {})) == "*" + + def test_write_shaped_tools_share_the_edit_key(self): + assert agent_module._CLAUDE_TO_OPENCODE_PERMISSION["Write"] == ("edit",) + assert agent_module._CLAUDE_TO_OPENCODE_PERMISSION["Edit"] == ("edit",) + + +class TestSystemPromptInstructions: + async def test_prompt_reaches_the_cli_as_an_instructions_file(self, patch_exec, tmp_path): + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + agent = _agent(system_prompt="be terse") + await _run(agent, tmp_path / "sandbox") + try: + (prompt_file,) = json.loads(captured["kwargs"]["env"]["OPENCODE_CONFIG_CONTENT"])["instructions"] + assert Path(prompt_file).read_text(encoding="utf-8") == "be terse" + assert not prompt_file.startswith(str(tmp_path / "sandbox")) + finally: + await agent.stop() + + async def test_stop_removes_the_prompt_dir(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + agent = _agent(system_prompt="be terse") + await agent.start(str(tmp_path)) + prompt_dir = agent._prompt_dir + assert prompt_dir is not None and os.path.isdir(prompt_dir) + await agent.stop() + assert not os.path.exists(prompt_dir) + assert agent._prompt_dir is None + + async def test_inherited_wildcard_allow_cannot_outrank_our_allowlist(self, patch_exec, tmp_path, monkeypatch): + monkeypatch.setenv("OPENCODE_CONFIG_CONTENT", json.dumps({"permission": {"*": "allow", "webfetch": "allow"}})) + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(_agent(allowed_tools=["Read"]), tmp_path) + rules = list(json.loads(captured["kwargs"]["env"]["OPENCODE_CONFIG_CONTENT"])["permission"].items()) + assert rules[0] == ("webfetch", "allow") + assert rules[1] == ("*", "deny") + + async def test_no_prompt_writes_no_file(self, patch_exec, tmp_path): + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + agent = _agent() + await _run(agent, tmp_path) + assert agent._prompt_dir is None + assert "OPENCODE_CONFIG_CONTENT" not in captured["kwargs"]["env"] + + async def test_inherited_config_merges_all_three_keys(self, patch_exec, tmp_path, monkeypatch): + root = _skill_repo(tmp_path / "plug") + monkeypatch.setenv( + "OPENCODE_CONFIG_CONTENT", + json.dumps( + { + "skills": {"paths": ["/host/skills"]}, + "instructions": ["/host/AGENTS.md"], + "permission": {"read": "deny", "bash": "allow", "webfetch": "deny"}, + } + ), + ) + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + agent = _agent( + plugins=[{"type": "local", "path": str(root)}], system_prompt="be terse", disallowed_tools=["Bash"] + ) + await _run(agent, tmp_path / "sandbox") + await agent.stop() + + config = json.loads(captured["kwargs"]["env"]["OPENCODE_CONFIG_CONTENT"]) + assert config["skills"]["paths"] == ["/host/skills", str(root / "skills")] + assert config["instructions"][0] == "/host/AGENTS.md" + assert config["instructions"][-1].endswith("system_prompt.md") + assert list(config["permission"].items()) == [("read", "deny"), ("webfetch", "deny"), ("bash", "deny")] async def test_variant_and_pure_off(self, patch_exec, tmp_path): captured = patch_exec(_FakeProcess(HAPPY_STREAM)) @@ -904,10 +1006,10 @@ async def test_second_turn_resumes_session(self, patch_exec, tmp_path): class TestEnvironmentInfo: def test_carries_the_system_prompt_semantics_marker(self): """The base contract: every agent's env-info records the regime, so a run - is never mis-bucketed as pre-marker. OpenCode cannot touch the system - prompt, so the honest value is `unknown`.""" + is never mis-bucketed as pre-marker. OpenCode appends the system prompt + as an `instructions` file.""" info = _agent().get_environment_info() - assert info["system_prompt_semantics"] == "unknown" + assert info["system_prompt_semantics"] == "append" assert info["harness_contract"] == OpenCodeAgent.contract.model_dump(mode="json") assert info["opencode_model"] == "deepseek/deepseek-v4-pro" assert info["opencode_pure"] is True diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index b6d6dc3d0..a0cc487e1 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -251,19 +251,12 @@ async def test_base_flags_and_prompt(self, patch_exec, tmp_path): assert argv[-2] == "--" assert argv[-1] == "do the thing" - async def test_tools_are_not_forwarded_but_system_prompt_is(self, patch_exec, tmp_path): - """allowed_tools/disallowed_tools are NOT forwarded: the shared config default - sets Claude-namespaced tool names (Bash/Read/...) that do not exist in Pi - (lowercase bash/read/...), so `--tools` would strip the agent of ALL tools. - Only `system_prompt` (a free-text string with no namespace) is forwarded.""" + async def test_tool_flags_and_system_prompt_are_forwarded(self, patch_exec, tmp_path): captured = patch_exec(_FakeProcess(HAPPY_STREAM)) - await _run( - _agent(allowed_tools=["Read", "Write"], disallowed_tools=["Bash"], system_prompt="be terse"), - tmp_path, - ) + await _run(_agent(allowed_tools=["Read", "Bash"], system_prompt="be terse"), tmp_path) argv = captured["argv"] - assert "--tools" not in argv - assert "--exclude-tools" not in argv + assert argv[argv.index("--tools") + 1] == "bash,read" + assert argv.index("--tools") > argv.index("--thinking") assert argv[argv.index("--append-system-prompt") + 1] == "be terse" async def test_user_input_is_a_post_dashdash_argv_element(self, patch_exec, tmp_path): @@ -280,6 +273,40 @@ async def test_explicit_line_limit_is_passed(self, patch_exec, tmp_path): assert captured["kwargs"]["limit"] > 64 * 1024 +class TestToolFlags: + def test_allowlist_maps_claude_names_to_pi_tools(self): + assert _agent(allowed_tools=["Bash", "Read"])._tool_flags() == ["--tools", "bash,read"] + + def test_denylist_expands_to_every_equivalent(self): + assert _agent(disallowed_tools=["Edit"])._tool_flags() == ["--exclude-tools", "edit,multiedit,patch"] + + def test_plan_denies_write_edit_and_bash(self): + assert _agent(permission_mode="plan")._tool_flags() == [ + "--exclude-tools", + "bash,edit,multiedit,patch,write", + ] + + def test_allowlist_with_no_pi_equivalent_disables_every_tool(self): + assert _agent(allowed_tools=["Skill"])._tool_flags() == ["--no-tools"] + + def test_deny_wins_over_allow_without_a_separate_exclude(self): + assert _agent(allowed_tools=["Bash", "Read"], disallowed_tools=["Bash"])._tool_flags() == ["--tools", "read"] + + def test_plan_wins_over_an_allowed_bash(self): + assert _agent(allowed_tools=["Bash", "Read"], permission_mode="plan")._tool_flags() == ["--tools", "read"] + + def test_no_fields_emit_no_flags(self): + assert _agent()._tool_flags() == [] + + def test_empty_allowlist_restricts_nothing(self): + assert _agent(allowed_tools=[])._tool_flags() == [] + + def test_inverse_map_covers_every_claude_name(self): + from coder_eval.agents.pi_agent import _CLAUDE_TO_PI_TOOLS, _TOOL_NAME_MAP + + assert set(_CLAUDE_TO_PI_TOOLS) == set(_TOOL_NAME_MAP.values()) + + class TestSessionContinuity: async def test_successive_calls_reuse_the_same_session(self, patch_exec, tmp_path): """Multi-turn / simulation stitching: both invocations carry the SAME id+dir.""" @@ -371,8 +398,8 @@ class TestUnsupportedConfigIsAnnounced: async def test_start_warns_about_unenforced_fields(self, patch_exec, tmp_path, caplog): patch_exec(_FakeProcess(HAPPY_STREAM)) with caplog.at_level("WARNING"): - await _agent(permission_mode="plan").start(str(tmp_path)) - assert "permission_mode" in caplog.text + await _agent(system_prompt_file="prompt.md").start(str(tmp_path)) + assert "system_prompt_file" in caplog.text assert "NOT enforced" in caplog.text async def test_plugins_that_do_not_resolve_warn_loudly(self, patch_exec, tmp_path, caplog): @@ -385,19 +412,13 @@ async def test_plugins_that_do_not_resolve_warn_loudly(self, patch_exec, tmp_pat # plugins is no longer named in the "NOT enforced" warning. assert "plugins" not in "".join(r.message for r in caplog.records if "NOT enforced" in r.message) - async def test_unenforced_fields_warn_but_system_prompt_does_not(self, patch_exec, tmp_path, caplog): - """allowed_tools/disallowed_tools are unenforced (Claude-namespaced default cannot - map to Pi's lowercase toolset) and MUST warn when set. `system_prompt` IS enforced - (--append-system-prompt) and must never appear in the unenforced-fields warning.""" + async def test_enforced_fields_do_not_warn(self, patch_exec, tmp_path, caplog): patch_exec(_FakeProcess(HAPPY_STREAM)) with caplog.at_level("WARNING"): await _agent(allowed_tools=["Read"], disallowed_tools=["Bash"], system_prompt="be terse").start( str(tmp_path) ) - warning = "".join(r.message for r in caplog.records if "NOT enforced" in r.message) - assert "allowed_tools" in warning - assert "disallowed_tools" in warning - assert "system_prompt" not in warning # the enforced field is never named + assert "NOT enforced" not in caplog.text class TestAutoRetry: From 923141e1f9e19817e513b99a3fad35abada3d7b2 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 08:23:20 -0700 Subject: [PATCH 04/12] =?UTF-8?q?feat(orchestration):=204/6=20=E2=80=94=20?= =?UTF-8?q?per-kind=20by=5Ftype=20defaults=20and=20the=20resolution-time?= =?UTF-8?q?=20harness=20contract=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An experiment's defaults.agent may carry by_type., applied below the task and selected by the final agent kind, so Claude-only defaults move out of the shared baseline. A gated agent field set on a harness whose contract marks it unsupported now raises HarnessContractError, which plan, run and export treat as a hard config error. The sdk_options guard is derived from the registry, a -D system_prompt_file is inlined after layer 5, and every unenforced-field warning is deleted. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/orchestration.md | 17 ++ docs/AB_EXPERIMENTS.md | 24 ++ docs/TASK_DEFINITION_GUIDE.md | 14 +- docs/USER_GUIDE.md | 2 + docs/agents/CODEX.md | 15 +- experiments/default.yaml | 36 +-- experiments/model-comparison.yaml | 4 +- experiments/plugin-comparison.yaml | 4 +- experiments/prompt-mutations-example.yaml | 4 +- src/coder_eval/agents/codex_agent.py | 40 +-- src/coder_eval/agents/opencode_agent.py | 10 - src/coder_eval/agents/pi_agent.py | 10 - src/coder_eval/cli/export_command.py | 17 +- src/coder_eval/cli/plan_command.py | 8 +- src/coder_eval/config.py | 4 +- src/coder_eval/isolation/docker_runner.py | 13 +- src/coder_eval/models/experiment.py | 8 +- src/coder_eval/orchestration/config_merge.py | 4 +- src/coder_eval/orchestration/early_stop.py | 42 ++-- src/coder_eval/orchestration/experiment.py | 99 +++++++- .../orchestration/harness_contract.py | 91 +++++++ src/coder_eval/orchestration/overrides.py | 37 ++- src/coder_eval/orchestration/task_loader.py | 3 + tasks/agents/codex_hello_world.yaml | 1 - tasks/agents/codex_parallel_commands.yaml | 1 - tasks/agents/codex_parallel_single_gen.yaml | 1 - tasks/agents/codex_skills_test.yaml | 1 - tasks/agents/codex_string_utils.yaml | 1 - tasks/agents/codex_subagent_test.yaml | 1 - tests/test_byoa_plugin.py | 14 ++ tests/test_config_precedence.py | 4 +- tests/test_early_stop.py | 7 +- tests/test_experiment_resolver.py | 50 ++++ tests/test_harbor_experiment_packager.py | 13 + tests/test_harness_contract.py | 229 +++++++++++++++++- tests/test_merge_characterization.py | 5 +- tests/test_opencode_agent.py | 21 -- tests/test_overrides_engine.py | 10 +- tests/test_pi_agent.py | 19 +- tests/test_plan_command.py | 25 ++ 40 files changed, 690 insertions(+), 219 deletions(-) create mode 100644 src/coder_eval/orchestration/harness_contract.py diff --git a/.claude/notes/orchestration.md b/.claude/notes/orchestration.md index 733aab932..683229049 100644 --- a/.claude/notes/orchestration.md +++ b/.claude/notes/orchestration.md @@ -26,6 +26,23 @@ agent.type=…` last-win rather than hard-error (the `-D` value wins). Tools, plugins, and SDK options are `-D`-only. +- **Per-kind defaults (`by_type`)**: an experiment's `defaults.agent` may carry + `by_type: {: {...}}`. Each entry is inserted as its own layer directly above the + experiment layer that carries it, so it stays BELOW the task. The entry is selected by the + FINAL kind across all five layers (`cli_agent_type` gives the CLI half), so `--type pi` + never inherits Claude-only values, which would otherwise be rejected by the contract + check. A kind that is not installed is tolerated (DEBUG log): a shared experiment YAML + can name a plugin kind that only some hosts have. It is not allowed on a task or a + variant, since both already know their kind. + +- **The harness contract check** is the second hard resolution-time rejection, beside + early stop. Both raise a `TaskResolutionError`, which `resolve_all_tasks` re-raises + instead of demoting to a skipped task and `plan` turns into a non-zero exit. A field + counts as SET only if a layer wrote it with a non-null value, so a model default and + the default experiment's `plugins: null` never trip it. A `-D + agent.system_prompt_file` is inlined against `Path.cwd()` after layer 5, so no adapter + and no container mount ever sees a prompt file. + ## Execute vs. run: the grading switch - **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` diff --git a/docs/AB_EXPERIMENTS.md b/docs/AB_EXPERIMENTS.md index afe35d66b..eee67bf81 100644 --- a/docs/AB_EXPERIMENTS.md +++ b/docs/AB_EXPERIMENTS.md @@ -113,6 +113,30 @@ it (the unification invariant). The per-field strategy is: Variants set the sandbox driver via `driver:` and add templates via `template_sources:` (top-level fields); they don't set a full `sandbox:` block. +### Per-kind defaults with by_type + +Layers 1 and 2 may carry `by_type:` inside their `agent:` block. Each entry is a +sub-layer that applies only when the resolved agent kind matches, and it sits +directly above its own layer, so it stays **below the task**: + +```yaml +defaults: + agent: + type: claude-code + by_type: + claude-code: + model: claude-sonnet-4-6 + permission_mode: acceptEdits + pi: + model: openrouter/moonshotai/kimi-k3 +``` + +The kind is the final `agent.type` across all five layers, so `--type pi` selects +the `pi` entry and never inherits the `claude-code` one. An entry for a kind that is +not installed is ignored. `by_type` is not allowed on a task or a variant (a task +knows its kind; a variant sets its fields directly), and an entry must not set +`type`. Lineage records such a value with `source_detail: by_type.`. + ## What a Variant Can Override From `ExperimentVariant` (`coder_eval/models/experiment.py`): diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index bafe807c3..06f82754b 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -188,7 +188,14 @@ an error. - `plan` — Agent proposes changes, waits for approval - `bypassPermissions` — No permission checks (use with caution) -> **Codex note:** `permission_mode` confines the **`claude-code`** agent only. The **`codex`** agent always runs full-access regardless of the mode — its in-process OS sandbox is redundant given Coder Eval's docker/tempdir isolation and unusable on our CI hosts (and on Windows). Run adversarial or untrusted Codex evals under the **docker driver**, which is the OS-level write boundary; the tempdir/host driver is a working directory, not a confinement boundary. +**Fields a harness cannot honor are rejected.** Every agent declares which of +`system_prompt`, `plugins`, `permission_mode`, `allowed_tools` and +`disallowed_tools` it honors. A task that sets one of them on a harness that marks +it unsupported fails at resolution, and `coder-eval plan` exits non-zero. A field +that only a lower layer's default sets does not count, and neither does a value of +`null`. Put harness-specific values under `by_type` in the experiment (see +[A/B Experiments](AB_EXPERIMENTS.md#per-kind-defaults-with-by_type)). Per-harness table: +[Harness Parity](agents/HARNESS_PARITY.md). **Agent Types:** - `claude-code` (default) — Claude Code SDK agent. Supports `sdk_options`, `claude_settings`, and all permission modes. @@ -227,7 +234,10 @@ Contract (enforced at load): a `type: none` task must declare no `initial_prompt / `initial_prompt_file` and no enabled `simulation` (no agent reads them), and every criterion must be agent-independent — criteria that inspect the agent trajectory (`command_executed`, `skill_triggered`, `reference_comparison`, -`commands_efficiency`) are rejected. A worked example lives at +`commands_efficiency`) are rejected. The no-op agent honors none of the gated agent +fields, so a `type: none` task that sets `plugins`, `system_prompt`, +`permission_mode`, `allowed_tools` or `disallowed_tools` is rejected too. A worked +example lives at [`tasks/agentless_smoke_test.yaml`](https://github.com/UiPath/coder_eval/blob/main/tasks/agentless_smoke_test.yaml). ## Run Limits diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 2d93d36a5..c561e8107 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -161,6 +161,8 @@ coder-eval plan tasks/*.yaml # validate specific tasks ``` Checks task syntax, required CLI tools, API keys, and schema validity without executing. +It exits non-zero on a hard configuration error, including an agent field the chosen +harness does not support (see [Harness Parity](agents/HARNESS_PARITY.md)). | Flag | Description | | --- | --- | diff --git a/docs/agents/CODEX.md b/docs/agents/CODEX.md index c83f3e8e7..a9ad99a76 100644 --- a/docs/agents/CODEX.md +++ b/docs/agents/CODEX.md @@ -85,13 +85,6 @@ Specify Codex in task YAML: ```yaml agent: type: codex - permission_mode: acceptEdits - allowed_tools: - - Bash - - Read - - Write - disallowed_tools: - - Edit plugins: - type: local path: "$PLUGIN_PATH" @@ -102,11 +95,9 @@ success_criteria: description: "Solution file must exist" ``` -Valid `permission_mode` values: -- `default` - Standard access, requires approval on failure -- `acceptEdits` - Automatically accept file edits, no filesystem restrictions -- `plan` - Read-only sandbox, approval required for any changes -- `bypassPermissions` - Full access, no approvals needed +A Codex task that sets `permission_mode`, `allowed_tools` or `disallowed_tools` is +rejected at resolution: Codex honors none of them (see +[Permission and Tool Mapping](#permission-and-tool-mapping)). ### Skills (SKILL.md) diff --git a/experiments/default.yaml b/experiments/default.yaml index b9469a4bd..ae364b978 100644 --- a/experiments/default.yaml +++ b/experiments/default.yaml @@ -43,22 +43,6 @@ defaults: # none (plus any plugin-registered kind). claude-code is the run-wide default. type: claude-code - # Permission mode for agent actions: - # default - ask for confirmation on each action - # acceptEdits - auto-accept file edits, confirm other actions - # plan - read-only, no file modifications allowed - # bypassPermissions - auto-accept all actions (use with caution) - permission_mode: acceptEdits - - # Specific model to use (null = defer to Claude Code CLI default) - # Examples: claude-sonnet-4-6, claude-opus-4-6 - model: claude-sonnet-4-6 - - # Allowed tools — restrict which tools the agent can use (null = all tools) - # Most tasks use ["Bash"] or ["Bash", "Read", "Write"]. This default - # provides the common set needed for general-purpose coding tasks. - allowed_tools: ["Bash", "Read", "Write", "Edit", "Glob", "Grep", "Skill"] - # Claude Code plugins to load (null = none) # `path` must be a PLUGIN ROOT: a directory holding skills/, so the skill sits at # /skills//SKILL.md. One level deeper loads nothing on claude-code. @@ -69,6 +53,26 @@ defaults: # Example: ["*.log", "__pycache__"] ignore_patterns: [] + # Per-kind defaults. Applied only when the resolved `type` matches, BELOW the + # task, so `--type pi` does not inherit Claude-only values. + by_type: + claude-code: + # Permission mode for agent actions: + # default - ask for confirmation on each action + # acceptEdits - auto-accept file edits, confirm other actions + # plan - read-only, no file modifications allowed + # bypassPermissions - auto-accept all actions (use with caution) + permission_mode: acceptEdits + + # Specific model to use (null = defer to Claude Code CLI default) + # Examples: claude-sonnet-4-6, claude-opus-4-6 + model: claude-sonnet-4-6 + + # Allowed tools — restrict which tools the agent can use (null = all tools) + # Most tasks use ["Bash"] or ["Bash", "Read", "Write"]. This default + # provides the common set needed for general-purpose coding tasks. + allowed_tools: ["Bash", "Read", "Write", "Edit", "Glob", "Grep", "Skill"] + # Claude Code SDK pass-through options — anything coder_eval doesn't own # directly (e.g. `effort`, `max_thinking_tokens`). Keys must be # ClaudeAgentOptions fields and must not be framework-managed (model, diff --git a/experiments/model-comparison.yaml b/experiments/model-comparison.yaml index f8df1bd17..83ebb1d67 100644 --- a/experiments/model-comparison.yaml +++ b/experiments/model-comparison.yaml @@ -10,7 +10,9 @@ description: "Compare Sonnet 4.6 vs Opus 4.6 on coding tasks" defaults: agent: type: claude-code - permission_mode: bypassPermissions + by_type: + claude-code: + permission_mode: bypassPermissions # Drop /{node_modules,.npm-prefix} so preserved artifacts stay slim (MST-9674). post_run: diff --git a/experiments/plugin-comparison.yaml b/experiments/plugin-comparison.yaml index 9faab36c8..fa42808e5 100644 --- a/experiments/plugin-comparison.yaml +++ b/experiments/plugin-comparison.yaml @@ -20,7 +20,9 @@ description: "Compare agent performance with vs without a plugin" defaults: agent: type: claude-code - permission_mode: bypassPermissions + by_type: + claude-code: + permission_mode: bypassPermissions # Drop /{node_modules,.npm-prefix} so preserved artifacts stay slim (MST-9674). post_run: diff --git a/experiments/prompt-mutations-example.yaml b/experiments/prompt-mutations-example.yaml index e1c391f4a..914988bdd 100644 --- a/experiments/prompt-mutations-example.yaml +++ b/experiments/prompt-mutations-example.yaml @@ -12,7 +12,9 @@ description: "Example of prompt mutations for A/B testing prompt phrasing" defaults: agent: type: claude-code - permission_mode: acceptEdits + by_type: + claude-code: + permission_mode: acceptEdits # Drop /{node_modules,.npm-prefix} so preserved artifacts stay slim (MST-9674). post_run: diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index e1403b7b4..77f2d90ba 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -848,9 +848,6 @@ async def start( # Set up skills from plugin_tools_dir or plugins config self._setup_skills(plugin_tools_dir) - # Log permission and tool configuration - self._log_config_enforcement() - except ImportError as e: raise RuntimeError("Codex SDK not installed. Install with: pip install 'coder-eval[codex]'") from e except Exception as e: @@ -1373,22 +1370,13 @@ def _build_thread_options(self) -> dict[str, Any]: if self.config.system_prompt is not None: options["developer_instructions"] = self.config.system_prompt - permission_mode = self.config.permission_mode.value - approval_mode_str = _CODEX_APPROVAL_MODE - - # ALWAYS full-access: permission_mode does NOT confine Codex. The docker - # driver is the only OS-level write boundary; the tempdir/host driver is a - # working directory, not a confinement boundary, so adversarial or - # untrusted evals belong on docker. _log_config_enforcement says so. + # ALWAYS full-access: Codex honors no permission_mode (its contract rejects + # the field). The docker driver is the only OS-level write boundary; the + # tempdir/host driver is a working directory, not a confinement boundary, so + # adversarial or untrusted evals belong on docker. # Rationale: .claude/notes/agents.md § Codex runs full-access on every permission mode options["sandbox"] = Sandbox.full_access - options["approval_mode"] = ApprovalMode(approval_mode_str) - - # For logging, use the enum names (which use underscores) - sandbox_name = options["sandbox"].name - approval_name = options["approval_mode"].name - - self._log.debug(f"Permission mode {permission_mode} → sandbox={sandbox_name}, approval_mode={approval_name}") + options["approval_mode"] = ApprovalMode(_CODEX_APPROVAL_MODE) tool_config: dict[str, Any] = {} @@ -1427,24 +1415,6 @@ def _build_thread_options(self) -> dict[str, Any]: return options - def _log_config_enforcement(self) -> None: - """Log configuration settings.""" - if self.config.allowed_tools: - self._log.debug(f"Allowed tools: {', '.join(self.config.allowed_tools)}") - - if self.config.disallowed_tools: - self._log.debug(f"Disallowed tools: {', '.join(self.config.disallowed_tools)}") - - self._log.debug(f"Permission mode: {self.config.permission_mode.value}") - # Fires for EVERY mode, not just bypassPermissions, so operators are not - # misled that plan/acceptEdits/default confine Codex — none of them do. - self._log.warning( - "[SECURITY] Codex runs full-access on every permission_mode " - + f"(configured: {self.config.permission_mode.value}); permission_mode does not confine it. " - + "OS-level isolation of untrusted code is the docker driver's job — use it for " - + "adversarial or untrusted evals; the tempdir/host driver is not a confinement boundary." - ) - def _format_turn_result(self, turn_result: Any) -> str: """Format a Codex Turn to a readable string — fallback when no text streamed. diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index e532ea6fa..825edf0de 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -173,9 +173,6 @@ for claude in set(_TOOL_NAME_MAP.values()) } -# `system_prompt_file` is inlined into `system_prompt` before the agent runs. -_UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ("system_prompt_file",) - # Skill paths, the system-prompt `instructions` file and tool `permission` rules are # merged through this variable, which OpenCode applies as a final local-scope layer. # Only the SKILLS half of a plugin is honored. @@ -810,13 +807,6 @@ async def start( "The 'opencode' CLI was not found on PATH." + " Install it with `npm install -g opencode-ai` (or see https://opencode.ai/docs/)." ) - ignored = [f for f in _UNSUPPORTED_CONFIG_FIELDS if getattr(self.config, f, None)] - if ignored: - logger.warning( - "opencode: %s set but NOT enforced — the CLI has no equivalent knob, so the run is " - + "unconstrained by them; do not rely on them as a boundary (see docs/agents/OPENCODE.md).", - ", ".join(ignored), - ) self._skill_dirs = _plugin_skill_dirs(self.config.plugins, log=logger) if self._skill_dirs: logger.info( diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index e9440f9bd..056b65759 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -143,9 +143,6 @@ for claude in set(_TOOL_NAME_MAP.values()) } -# `system_prompt_file` is inlined into `system_prompt` before the agent runs. -_UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ("system_prompt_file",) - # The full recognized Pi vocabulary (from `pi` 0.84.4). A clean exit that # recognized NOTHING from this set is vocabulary drift and is crashed, not scored. # Rationale: .claude/notes/agents.md § Why a clean exit can still be a crash @@ -738,13 +735,6 @@ async def start( "The 'pi' CLI was not found on PATH." + " Install it with `npm install -g @earendil-works/pi-coding-agent` (see https://pi.dev/)." ) - ignored = [f for f in _UNSUPPORTED_CONFIG_FIELDS if getattr(self.config, f, None)] - if ignored: - logger.warning( - "pi: %s set but NOT enforced — the CLI has no equivalent knob in JSON print mode, so the run is " - + "unconstrained by them; do not rely on them as a boundary (see docs/agents/PI.md).", - ", ".join(ignored), - ) # Resolve `agent.plugins` -> skills dirs and load them via `pi --skill`. # Loudly logs when plugins were declared but nothing resolved (the run # would otherwise measure the model WITHOUT the skill under test). diff --git a/src/coder_eval/cli/export_command.py b/src/coder_eval/cli/export_command.py index 5afd69deb..ee334440f 100644 --- a/src/coder_eval/cli/export_command.py +++ b/src/coder_eval/cli/export_command.py @@ -24,6 +24,7 @@ from ..harbor.experiment_packager import export_experiment from ..harbor.packager import CriteriaNotExportableError, TaskNotExportableError, export_task +from ..orchestration.harness_contract import TaskResolutionError from .console import console from .run_helpers import expand_task_files @@ -95,12 +96,16 @@ def export_command( return all_task_files = expand_task_files(task_files) - exp_result = export_experiment( - all_task_files, - experiment, - output_dir, - allow_credentials=allow_credentials, - ) + try: + exp_result = export_experiment( + all_task_files, + experiment, + output_dir, + allow_credentials=allow_credentials, + ) + except TaskResolutionError as e: + console.print(f"[red]✗[/] config error - {e}") + raise typer.Exit(1) from e for exported in exp_result.exported: console.print(f"[green]✓[/] Exported → {exported.out_dir} (format: {format})") diff --git a/src/coder_eval/cli/plan_command.py b/src/coder_eval/cli/plan_command.py index b994d74d2..879bc4d7a 100644 --- a/src/coder_eval/cli/plan_command.py +++ b/src/coder_eval/cli/plan_command.py @@ -72,8 +72,9 @@ def run_plan(*, task_files: list[Path] | None = None, experiment: Path | None = check_api_keys() # Lazy import to avoid circular dependency at module level - from ..orchestration.early_stop import EarlyStopConfigError, validate_early_stop + from ..orchestration.early_stop import validate_early_stop from ..orchestration.experiment import DEFAULT_EXPERIMENT_PATH, load_experiment, resolve_task_for_variant + from ..orchestration.harness_contract import TaskResolutionError, validate_harness_contract from ..orchestration.run_limits import validate_run_limits # Always load experiment (defaults to experiments/default.yaml) @@ -143,6 +144,7 @@ def run_plan(*, task_files: list[Path] | None = None, experiment: Path | None = resolved, _lineage, _ = resolve_task_for_variant(default_exp, task, exp_def, variant) # Early-stop guardrails (no-op unless a criterion carries a stop_early: block). validate_early_stop(resolved) + validate_harness_contract(resolved) for message in validate_run_limits(resolved): console.print( f" [yellow]⚠[/yellow] [yellow]Variant '{variant.variant_id}': {message}[/yellow]" @@ -151,10 +153,10 @@ def run_plan(*, task_files: list[Path] | None = None, experiment: Path | None = agent_model = resolved.agent.model if resolved.agent else None model_str = f" ({agent_model})" if agent_model else "" console.print(f" [dim]Variant '{variant.variant_id}': {agent_type}{model_str}[/dim]") - except EarlyStopConfigError as e: + except TaskResolutionError as e: # A hard config error (unlike generic per-variant resolution # failures, which stay soft): flip the plan exit code. - console.print(f" [red]Variant '{variant.variant_id}': early-stop config error - {e}[/red]") + console.print(f" [red]Variant '{variant.variant_id}': config error - {e}[/red]") all_valid = False except Exception as e: console.print(f" [red]Variant '{variant.variant_id}': resolution failed - {e}[/red]") diff --git a/src/coder_eval/config.py b/src/coder_eval/config.py index 651788145..946e6d141 100644 --- a/src/coder_eval/config.py +++ b/src/coder_eval/config.py @@ -41,8 +41,8 @@ # pydantic-settings silently ignores unknown env vars, so without this guard a # stale knob would quietly stop having any effect. Fail loud with a migration hint. _REMOVED_DEFAULT_KNOBS = { - "DEFAULT_AGENT_MODEL": "agent.model", - "DEFAULT_PERMISSION_MODE": "agent.permission_mode", + "DEFAULT_AGENT_MODEL": "agent.by_type.claude-code.model", + "DEFAULT_PERMISSION_MODE": "agent.by_type.claude-code.permission_mode", "DEFAULT_MAX_TURNS": "run_limits.max_turns", } diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index fff43caf5..b32b6b1b0 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -1427,13 +1427,10 @@ def _build_argv( # Rationale: .claude/notes/isolation.md § Extra mounts and reserved destinations sensitive_sources = self._sensitive_source_paths() - def _auto_mount(raw_path: str | None, *, dir_only: bool = True) -> None: + def _auto_mount(raw_path: str | None) -> None: if not raw_path: return - resolved = Path(os.path.expandvars(os.path.expanduser(raw_path))).resolve() - # File paths get mounted as the parent dir so a single -v covers - # the file; container-side reads still resolve at the same path. - target = resolved if (dir_only or resolved.is_dir()) else resolved.parent + target = Path(os.path.expandvars(os.path.expanduser(raw_path))).resolve() if target in mounted or not target.is_dir(): return for sensitive in sensitive_sources: @@ -1457,12 +1454,6 @@ def _auto_mount(raw_path: str | None, *, dir_only: bool = True) -> None: if isinstance(source, TemplateDirSource): _auto_mount(source.path) - # Defensive: normally inlined into system_prompt by load_task / experiment - # resolution, but a variant could inject an absolute path that survives. - agent_cfg = self.rt.task.agent - if agent_cfg and agent_cfg.system_prompt_file: - _auto_mount(agent_cfg.system_prompt_file, dir_only=False) - # HAZARD: task.reference.directory is deliberately NOT auto-mounted at its # host path. That would bind the REAL tree in beside the shielded copy, so # the mode-000 window would leave it readable through $TASK_DIR. diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index c72e3ea01..078b4bc81 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -105,7 +105,13 @@ class ExperimentDefaults(BaseModel): ge=1, description="Default number of replicates across all variants. None = 1 (no repetition).", ) - agent: dict[str, Any] | None = Field(default=None, description="Partial agent config defaults") + agent: dict[str, Any] | None = Field( + default=None, + description=( + "Partial agent config defaults. May carry `by_type: {: {...}}`, a sub-layer applied only " + "when the resolved agent kind matches; it sits below the task." + ), + ) checker_context: dict[str, dict[str, Any]] | None = Field( default=None, description=( diff --git a/src/coder_eval/orchestration/config_merge.py b/src/coder_eval/orchestration/config_merge.py index 77c2355ba..ac957376b 100644 --- a/src/coder_eval/orchestration/config_merge.py +++ b/src/coder_eval/orchestration/config_merge.py @@ -29,8 +29,6 @@ from ..models import ( BaseAgentConfig, - ClaudeCodeAgentConfig, - CodexAgentConfig, ConfigLineageEntry, RunLimits, SandboxConfig, @@ -346,7 +344,7 @@ def merge_layers( @overload def resolve_root( root: Literal["agent"], layers: Sequence[Layer], *, lineage: dict[str, ConfigLineageEntry] | None = ... -) -> ClaudeCodeAgentConfig | CodexAgentConfig | BaseAgentConfig | None: +) -> BaseAgentConfig | None: """Resolve the ``agent`` root to its concrete agent-config model.""" diff --git a/src/coder_eval/orchestration/early_stop.py b/src/coder_eval/orchestration/early_stop.py index 136231237..69964c260 100644 --- a/src/coder_eval/orchestration/early_stop.py +++ b/src/coder_eval/orchestration/early_stop.py @@ -34,6 +34,7 @@ LiveSuccessCriterion, StopEarlyPolicy, ) +from coder_eval.orchestration.harness_contract import TaskResolutionError, registration_for from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( AgentStartEvent, @@ -57,13 +58,12 @@ logger = logging.getLogger(__name__) -class EarlyStopConfigError(ValueError): +class EarlyStopConfigError(TaskResolutionError): """Raised when a task arms early-stop in a way v1 cannot honor. - Subclasses ``ValueError`` so the run path's resolve -> ``typer.BadParameter`` - conversion covers it transparently; the ``plan`` command catches this - subclass specifically to flip its exit code (generic per-variant resolution - failures intentionally stay soft). + A ``TaskResolutionError`` (so a ``ValueError``): the run path aborts on it and + the ``plan`` command flips its exit code, while generic per-variant resolution + failures stay soft. """ @@ -103,6 +103,7 @@ def validate_early_stop(task: TaskDefinition) -> None: Raises: EarlyStopConfigError: on any unsupported armed configuration. + HarnessContractError: an armed task has no agent type, or an unregistered one. """ limits = task.run_limits # (1) The master arm no longer exists; arming moved onto the criteria. A @@ -128,31 +129,16 @@ def validate_early_stop(task: TaskDefinition) -> None: + "for dialog-mode criteria stopping, or disarm with run_limits.stop_early: false." ) - # (3) The agent must honor the cooperative interrupt. Lazily import the - # registry + plugin loader so this module stays free of runtime coder_eval - # imports at load time. + # (3) The agent must honor the cooperative interrupt. from coder_eval.agents.registry import AgentRegistry - from coder_eval.plugins import ensure_plugins_loaded - ensure_plugins_loaded() - agent_type = str(task.agent.type) if task.agent is not None and task.agent.type is not None else None - if agent_type is None: - # Distinct from an unregistered type: there is no agent block at all, - # so pointing at plugin loading would send the user the wrong way. - raise EarlyStopConfigError( - "criterion-level stop_early arming requires an agent block with a registered type; " - + "this task resolves without one. " - + "Disarm with run_limits.stop_early: false to bypass this check." - ) - registration = AgentRegistry.get(agent_type) - if registration is None: - # Not the same failure as an agent that opted out of cooperative stop: - # an unregistered type usually means a plugin is not installed/loaded. - raise EarlyStopConfigError( - f"criterion-level stop_early arming requires a registered agent type; {agent_type!r} is " - + "not registered (is the providing plugin installed and loaded?). " - + "Disarm with run_limits.stop_early: false to bypass this check." - ) + registration = registration_for( + task, + requirement="criterion-level stop_early arming", + hint="Disarm with run_limits.stop_early: false to bypass this check.", + ) + assert task.agent is not None + agent_type = str(task.agent.type) if not registration.agent_class.contract.cooperative_stop: supporting = ", ".join( kind diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index fb8a583f1..b0a4c287e 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -11,7 +11,7 @@ import importlib.resources import logging import re -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Any, Literal @@ -41,6 +41,7 @@ from ..path_utils import build_task_run_dir from .config import BatchRunConfig from .config_merge import ConfigSource, Layer, merge_layers, resolve_root +from .overrides import cli_agent_type from .task_loader import ( expand_dataset, load_task, @@ -374,6 +375,55 @@ def _build_sandbox_layers( return sandbox_layers +def _split_by_type( + patch: dict[str, Any] | None, layer: ConfigSource +) -> tuple[dict[str, Any] | None, dict[str, dict[str, Any]]]: + """Separate an experiment layer's ``by_type`` sub-layers from its plain agent fields. + + Raises: + ValueError: ``by_type`` is not a mapping of kind -> mapping, or an entry sets ``type``. + """ + if not patch or "by_type" not in patch: + return patch, {} + by_type = patch["by_type"] + if not isinstance(by_type, Mapping) or not all( + isinstance(kind, str) and isinstance(entry, Mapping) for kind, entry in by_type.items() + ): + raise ValueError(f"{layer} agent.by_type must map each agent kind to a mapping of agent fields") + for kind, entry in by_type.items(): + if "type" in entry: + raise ValueError(f"{layer} agent.by_type.{kind} must not set 'type'; the entry is selected by it") + rest = {key: value for key, value in patch.items() if key != "by_type"} + return rest, {kind: dict(entry) for kind, entry in by_type.items()} + + +def _effective_agent_kind( + default_agent: Mapping[str, Any] | None, + exp_defaults_agent: Mapping[str, Any] | None, + task_agent: Mapping[str, Any] | None, + variant_agent: Mapping[str, Any] | None, + config: BatchRunConfig | None, +) -> str | None: + """The agent kind all five layers resolve to: the highest layer that sets ``type``.""" + if config is not None and (kind := cli_agent_type(config.overrides, config.agent_type)) is not None: + return kind + for patch in (variant_agent, task_agent, exp_defaults_agent, default_agent): + if patch and patch.get("type") is not None: + return str(patch["type"]) + return None + + +def _log_unregistered_by_type_kinds(*by_types: Mapping[str, Any]) -> None: + from coder_eval.agents.registry import AgentRegistry + from coder_eval.plugins import ensure_plugins_loaded + + ensure_plugins_loaded() + for by_type in by_types: + for kind in by_type: + if AgentRegistry.get(kind) is None: + logger.debug("agent.by_type.%s names no registered agent kind; the entry is never applied", kind) + + def resolve_task_for_variant( default_experiment: ExperimentDefinition, task: TaskDefinition, @@ -385,10 +435,14 @@ def resolve_task_for_variant( Precedence (lowest to highest): 1. default_experiment.defaults.agent (global baseline defaults) + 1b. its by_type[] 2. experiment.defaults.agent (experiment-wide defaults, below task) + 2b. its by_type[] 3. task.agent (task-explicit fields only via exclude_unset) 4. variant.agent (per-variant overrides, highest) + ```` is the agent type all five layers resolve to, CLI included. + After resolution, CLI overrides (layer 5) are applied separately by _apply_cli_overrides(). @@ -405,10 +459,17 @@ def resolve_task_for_variant( # Experiment-side dicts pass through verbatim; the task agent is dumped with # exclude_unset so Pydantic defaults don't leak into the merge. Timing belongs # under run_limits — a legacy `max_turns` under `agent:` fails loudly. - default_agent = default_experiment.defaults.agent if default_experiment.defaults else None - exp_defaults_agent = experiment.defaults.agent if experiment.defaults else None + default_agent, default_by_type = _split_by_type( + default_experiment.defaults.agent if default_experiment.defaults else None, "default" + ) + exp_defaults_agent, exp_by_type = _split_by_type( + experiment.defaults.agent if experiment.defaults else None, "experiment-defaults" + ) variant_agent_clean = variant.agent task_agent = task.agent.model_dump(exclude_unset=True) if task.agent else None + kind = _effective_agent_kind(default_agent, exp_defaults_agent, task_agent, variant_agent_clean, config) + if default_by_type or exp_by_type: + _log_unregistered_by_type_kinds(default_by_type, exp_by_type) # All three `-D`-reachable roots through the SAME generic resolver the CLI # layer uses, with lineage emitted as a side effect. Type is enforced AFTER @@ -420,15 +481,22 @@ def resolve_task_for_variant( # Rationale: .claude/notes/orchestration.md § No-op tasks need no special case anywhere resolved_agent: AgentConfig | BaseAgentConfig | None agent_layers: list[Layer] = [] - agent_specs: list[tuple[ConfigSource, dict[str, Any] | None]] = [ - ("default", default_agent), - ("experiment-defaults", exp_defaults_agent), - ("task", task_agent), - ("variant", variant_agent_clean), + by_type_detail = f"by_type.{kind}" + agent_specs: list[tuple[ConfigSource, dict[str, Any] | None, str | None]] = [ + ("default", default_agent, None), + ("default", default_by_type.get(kind) if kind else None, by_type_detail), + ("experiment-defaults", exp_defaults_agent, None), + ("experiment-defaults", exp_by_type.get(kind) if kind else None, by_type_detail), + ("task", task_agent, None), + ("variant", variant_agent_clean, None), ] - for source, patch in agent_specs: + for source, patch, detail in agent_specs: if patch: - agent_layers.append(Layer(source=source, patch=patch)) + agent_layers.append(Layer(source=source, patch=patch, detail=detail)) + if config is not None and cli_agent_type(config.overrides, config.agent_type) is not None: + # The CLI kind selected the by_type entry, so layers 1-4 must validate against + # its config class too. Lineage-silent: layer 5 records the type itself. + agent_layers.append(Layer(source="cli", patch={"type": kind}, record_lineage=False)) resolved_agent = resolve_root("agent", agent_layers, lineage=lineage) assert resolved_agent is not None # parse_agent_config always returns a model @@ -546,6 +614,11 @@ def _apply_cli_overrides( + "Set it in the task YAML, the experiment, or via --type." ) + # A `-D agent.system_prompt_file` is relative to the invoking directory. After + # this no adapter ever sees a prompt file. + task.agent = resolve_agent_system_prompt(task.agent, Path.cwd()) + assert task.agent.system_prompt_file is None + def resolve_task_files( task: TaskDefinition, @@ -600,7 +673,8 @@ def resolve_all_tasks( Raises: ValueError: If duplicate task IDs are found after resolution. """ - from .early_stop import EarlyStopConfigError, validate_early_stop + from .early_stop import validate_early_stop + from .harness_contract import TaskResolutionError, validate_harness_contract resolved: list[ResolvedTask] = [] skipped: list[SkippedTask] = [] @@ -671,6 +745,7 @@ def resolve_all_tasks( # Once the task is fully resolved, so the -D kill switch is # already merged. No-op unless armed. validate_early_stop(resolved_task) + validate_harness_contract(resolved_task) # Fan-out: simulation n_trials takes precedence over experiment repeats # when simulation is active; otherwise use experiment-level repeats. @@ -696,7 +771,7 @@ def resolve_all_tasks( ) # A deliberate hard stop: never demoted to skipped, so a misconfigured # run fails loudly instead of quietly shrinking the suite. - except EarlyStopConfigError: + except TaskResolutionError: raise # NARROW, matching the load/expand block above. except (FileNotFoundError, OSError, ValueError, yaml.YAMLError) as exc: diff --git a/src/coder_eval/orchestration/harness_contract.py b/src/coder_eval/orchestration/harness_contract.py new file mode 100644 index 000000000..19f872ea6 --- /dev/null +++ b/src/coder_eval/orchestration/harness_contract.py @@ -0,0 +1,91 @@ +"""Resolution-time check that a task sets no agent field its harness cannot honor.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from coder_eval.models import Enforcement + + +if TYPE_CHECKING: + from coder_eval.agents.registry import AgentRegistration + from coder_eval.models import TaskDefinition + + +class TaskResolutionError(ValueError): + """A hard configuration error found at resolution: never demoted to a skipped task.""" + + +class HarnessContractError(TaskResolutionError): + """The task's agent config sets a field its harness declares unsupported, or names no registered harness.""" + + +# BaseAgentConfig field -> the HarnessContract row that gates it. +_GATED: dict[str, str] = { + "system_prompt": "system_prompt", + "plugins": "plugin_skills", + "permission_mode": "permission_mode", + "allowed_tools": "allowed_tools", + "disallowed_tools": "disallowed_tools", +} + + +def registration_for(task: TaskDefinition, *, requirement: str, hint: str = "") -> AgentRegistration[Any]: + """The registry entry for the task's resolved agent kind. + + Args: + task: A resolved task. + requirement: What needs the registration, opening each error message. + hint: A sentence appended to each error message. + + Raises: + HarnessContractError: the task has no agent type, or its kind is not registered. + """ + from coder_eval.agents.registry import AgentRegistry + from coder_eval.plugins import ensure_plugins_loaded + + ensure_plugins_loaded() + suffix = f" {hint}" if hint else "" + if task.agent is None or task.agent.type is None: + raise HarnessContractError( + f"{requirement} requires an agent block with a registered type; this task resolves without one.{suffix}" + ) + kind = str(task.agent.type) + registration = AgentRegistry.get(kind) + if registration is None: + raise HarnessContractError( + f"{requirement} requires a registered agent type; {kind!r} is not registered " + + f"(is the providing plugin installed and loaded?).{suffix}" + ) + return registration + + +def validate_harness_contract(task: TaskDefinition) -> None: + """Reject a gated agent field that is set on a harness whose contract marks it unsupported. + + A field is set when a config layer wrote it and its value is not None. A task + without an agent type returns silently; the layer-5 type guard reports that. + + Raises: + HarnessContractError: on the first unsupported field that is set, or an unregistered kind. + """ + if task.agent is None or task.agent.type is None: + return + from coder_eval.agents.registry import AgentRegistry + + contract = registration_for(task, requirement="The harness contract check").agent_class.contract + kind = str(task.agent.type) + for field, row in _GATED.items(): + is_set = field in task.agent.model_fields_set and getattr(task.agent, field) is not None + if is_set and getattr(contract, row) is Enforcement.UNSUPPORTED: + honoring = [ + k + for k in AgentRegistry.list_kinds() + if (reg := AgentRegistry.get(k)) is not None + and getattr(reg.agent_class.contract, row) is Enforcement.ENFORCED + ] + raise HarnessContractError( + f"agent.{field} is set but the {kind!r} harness does not support it " + + "(see docs/agents/HARNESS_PARITY.md). Remove the field, or move it under " + + f"by_type. in the experiment for a harness that honors it ({', '.join(honoring) or 'none'})." + ) diff --git a/src/coder_eval/orchestration/overrides.py b/src/coder_eval/orchestration/overrides.py index de1f55d93..7ff772217 100644 --- a/src/coder_eval/orchestration/overrides.py +++ b/src/coder_eval/orchestration/overrides.py @@ -26,7 +26,7 @@ import yaml from pydantic import ValidationError -from ..models import AgentKind, ConfigLineageEntry, TaskDefinition +from ..models import ConfigLineageEntry, TaskDefinition from .config_merge import ALLOWED_OVERRIDE_ROOTS, Layer, MergeError, RootName, resolve_root @@ -89,6 +89,29 @@ def _assign_nested(patch: dict[str, Any], segments: list[str], value: Any) -> No cursor[segments[-1]] = value +def cli_agent_type(overrides: Mapping[str, Any], agent_type: str | None) -> str | None: + """The agent kind layer 5 selects: an explicit ``-D agent.type`` beats ``--type``; None if neither.""" + kind = overrides.get("agent.type", agent_type) + return None if kind is None else str(kind) + + +def _check_sdk_options_supported(kind: str | None) -> None: + """Reject ``-D agent.sdk_options.*`` unless the becoming kind's config class declares ``sdk_options``.""" + from coder_eval.agents.registry import AgentRegistry + from coder_eval.plugins import ensure_plugins_loaded + + ensure_plugins_loaded() + registration = AgentRegistry.get(kind) if kind is not None else None + if registration is None or "sdk_options" not in registration.config_class.model_fields: + supporting = ", ".join( + k + for k in AgentRegistry.list_kinds() + if (reg := AgentRegistry.get(k)) is not None and "sdk_options" in reg.config_class.model_fields + ) + where = "no agent type is set" if kind is None else f"the {kind!r} agent config" + raise OverrideError(f"sdk_options is not a field of {where}; it is supported by: {supporting}.") + + def apply_overrides( task: TaskDefinition, overrides: Mapping[str, Any], @@ -132,16 +155,10 @@ def apply_overrides( if agent_patch: assert task.agent is not None, f"Task '{task.task_id}' has no agent config" - # Preserve the friendly "sdk_options only for claude-code" message before - # reconstruction, keyed on the type the agent is *becoming*. + # A friendly message before reconstruction, keyed on the kind the agent is *becoming*. if "sdk_options" in agent_patch: - becoming = agent_patch.get("type", task.agent.type) - type_value = becoming.value if isinstance(becoming, AgentKind) else becoming - if type_value != AgentKind.CLAUDE_CODE.value: - where = "no agent type is set" if type_value is None else f"agent type {type_value}" - raise OverrideError( - f"sdk_options cannot be used with {where}. This option is only supported for claude-code agents." - ) + becoming = cli_agent_type(overrides, agent_type) or task.agent.type + _check_sdk_options_supported(None if becoming is None else str(becoming)) # Seed with only the explicitly-set fields (exclude_unset) so switching the # agent subclass via --type doesn't drag subclass-only defaults into a model # that forbids them. diff --git a/src/coder_eval/orchestration/task_loader.py b/src/coder_eval/orchestration/task_loader.py index 02e6b5d1b..6dcb38226 100644 --- a/src/coder_eval/orchestration/task_loader.py +++ b/src/coder_eval/orchestration/task_loader.py @@ -231,6 +231,9 @@ def resolve_agent_system_prompt[T: AgentConfig | BaseAgentConfig | None](agent_c rejects. A single ``model_copy(update=...)`` applies both edits at once so no half-updated state is ever validated. + Three call sites, one per base directory: ``load_task`` (task-relative), variant + file resolution (experiment-relative) and the layer-5 tail (``Path.cwd()``). + Args: agent_config: Config to resolve; ``None`` and configs without a ``system_prompt_file`` are returned unchanged. diff --git a/tasks/agents/codex_hello_world.yaml b/tasks/agents/codex_hello_world.yaml index 3c0bb6a32..ae91c1a68 100644 --- a/tasks/agents/codex_hello_world.yaml +++ b/tasks/agents/codex_hello_world.yaml @@ -11,7 +11,6 @@ tags: agent: type: codex model: gpt-5.5 - permission_mode: acceptEdits initial_prompt: | Create a simple Python script called `hello.py` that prints "Hello, Codex!" when run. diff --git a/tasks/agents/codex_parallel_commands.yaml b/tasks/agents/codex_parallel_commands.yaml index 94c6928ab..2123c7b44 100644 --- a/tasks/agents/codex_parallel_commands.yaml +++ b/tasks/agents/codex_parallel_commands.yaml @@ -15,7 +15,6 @@ tags: agent: type: codex model: gpt-5.5 - permission_mode: acceptEdits initial_prompt: | Run these two independent commands IN PARALLEL — issue them together as two diff --git a/tasks/agents/codex_parallel_single_gen.yaml b/tasks/agents/codex_parallel_single_gen.yaml index 5fb73c0ad..668c363a3 100644 --- a/tasks/agents/codex_parallel_single_gen.yaml +++ b/tasks/agents/codex_parallel_single_gen.yaml @@ -16,7 +16,6 @@ tags: agent: type: codex model: gpt-5.5 - permission_mode: acceptEdits initial_prompt: | Gather two completely independent pieces of system information at once. Issue diff --git a/tasks/agents/codex_skills_test.yaml b/tasks/agents/codex_skills_test.yaml index acddfc874..1e7a7614a 100644 --- a/tasks/agents/codex_skills_test.yaml +++ b/tasks/agents/codex_skills_test.yaml @@ -9,7 +9,6 @@ tags: agent: type: codex model: gpt-5.5 - permission_mode: acceptEdits plugins: - type: local path: "$PLUGIN_PATH" diff --git a/tasks/agents/codex_string_utils.yaml b/tasks/agents/codex_string_utils.yaml index 3c626b47b..9ea147483 100644 --- a/tasks/agents/codex_string_utils.yaml +++ b/tasks/agents/codex_string_utils.yaml @@ -12,7 +12,6 @@ tags: agent: type: codex model: gpt-5.5 - permission_mode: acceptEdits initial_prompt: | Create a Python module named `string_utils.py` with the following functions: diff --git a/tasks/agents/codex_subagent_test.yaml b/tasks/agents/codex_subagent_test.yaml index 5f32e90d0..8534ebe26 100644 --- a/tasks/agents/codex_subagent_test.yaml +++ b/tasks/agents/codex_subagent_test.yaml @@ -14,7 +14,6 @@ tags: agent: type: codex model: gpt-5.5 - permission_mode: acceptEdits initial_prompt: | Delegate this to a SUB-AGENT (spawn one via the Agent tool). Instruct the diff --git a/tests/test_byoa_plugin.py b/tests/test_byoa_plugin.py index 3a1e3faba..9ee5fb2ee 100644 --- a/tests/test_byoa_plugin.py +++ b/tests/test_byoa_plugin.py @@ -89,3 +89,17 @@ def test_create_agent_wrong_config_type_raises(discovered_demo_plugin): claude_cfg = parse_agent_config(type="claude-code") with pytest.raises(TypeError, match=r"--type.*mismatch"): create_agent("byoa-demo", claude_cfg) + + +def test_sdk_options_override_is_accepted_for_a_plugin_config_that_declares_it(discovered_demo_plugin): + from coder_eval.orchestration.overrides import apply_overrides + + task = TaskDefinition( + task_id="t1", + description="d", + initial_prompt="hi", + agent={"type": "claude-code"}, + success_criteria=CRIT, + ) + apply_overrides(task, {"agent.sdk_options.effort": "high"}, agent_type="byoa-demo") + assert task.agent is not None and task.agent.sdk_options == {"effort": "high"} # type: ignore[attr-defined] diff --git a/tests/test_config_precedence.py b/tests/test_config_precedence.py index d968fda3c..d309e0e36 100644 --- a/tests/test_config_precedence.py +++ b/tests/test_config_precedence.py @@ -399,8 +399,8 @@ def test_resolve_route_bedrock_missing_token_asserts(): @pytest.mark.parametrize( ("var_name", "replacement"), [ - ("DEFAULT_AGENT_MODEL", "agent.model"), - ("DEFAULT_PERMISSION_MODE", "agent.permission_mode"), + ("DEFAULT_AGENT_MODEL", "agent.by_type.claude-code.model"), + ("DEFAULT_PERMISSION_MODE", "agent.by_type.claude-code.permission_mode"), ("DEFAULT_MAX_TURNS", "run_limits.max_turns"), ], ) diff --git a/tests/test_early_stop.py b/tests/test_early_stop.py index 571277d73..89787cb46 100644 --- a/tests/test_early_stop.py +++ b/tests/test_early_stop.py @@ -76,6 +76,7 @@ validate_early_stop, ) from coder_eval.orchestration.experiment import load_experiment, resolve_all_tasks +from coder_eval.orchestration.harness_contract import HarnessContractError from coder_eval.orchestrator import Orchestrator, build_task_event from coder_eval.reports import ReportGenerator from coder_eval.reports.html import _render_criteria, _render_header @@ -801,7 +802,7 @@ def test_guardrail3_agentless_task_rejected(self) -> None: # An armed task with no agent block at all: the diagnosis must point at # the missing agent block, not at plugin loading. task = _task(criteria=[_skill_crit("s", "s", stop_on_pass=True)]).model_copy(update={"agent": None}) - with pytest.raises(EarlyStopConfigError, match="agent block"): + with pytest.raises(HarnessContractError, match="agent block"): validate_early_stop(task) def test_guardrail3_unregistered_agent_type_rejected(self) -> None: @@ -813,7 +814,7 @@ def test_guardrail3_unregistered_agent_type_rejected(self) -> None: task = _task(criteria=[_skill_crit("s", "s", stop_on_pass=True)], agent_type=kind) finally: AgentRegistry._registry.pop(kind, None) - with pytest.raises(EarlyStopConfigError, match="not registered"): + with pytest.raises(HarnessContractError, match="not registered"): validate_early_stop(task) def test_guardrail1_armed_codex_accepts(self) -> None: @@ -1067,7 +1068,7 @@ def test_plan_surface_flips_exit_code_on_master_arm(self, tmp_path: Path) -> Non task_file = _write_task_yaml(tmp_path, criterion_yaml=_UNARMED_CRITERION, stop_early=True) printed, exit_code = self._run_plan(task_file, tmp_path) assert exit_code == 1 - assert "early-stop config error" in printed + assert "config error" in printed assert "has been removed" in printed def test_plan_surface_accepts_valid_armed_task(self, tmp_path: Path) -> None: diff --git a/tests/test_experiment_resolver.py b/tests/test_experiment_resolver.py index ec82aadb1..84e102f02 100644 --- a/tests/test_experiment_resolver.py +++ b/tests/test_experiment_resolver.py @@ -1132,3 +1132,53 @@ def test_judge_nested_sdk_options_validator_runs(self): prompt="grade", agent=parse_agent_config(type="claude-code", sdk_options={"hooks": {}}), ) + + +def _write_task(tmp_path, body: str): + task_file = tmp_path / "task.yaml" + task_file.write_text( + "task_id: contract-task\ndescription: d\ninitial_prompt: do it\nsandbox:\n driver: tempdir\n" + + body + + "success_criteria:\n - type: file_exists\n path: out.txt\n description: c\n" + ) + return task_file + + +def _resolve_all(tmp_path, task_file, **config): + from coder_eval.orchestration.config import BatchRunConfig + from coder_eval.orchestration.experiment import resolve_all_tasks + + single = [ExperimentVariant(variant_id="default")] + return resolve_all_tasks( + task_files=[task_file], + experiment=ExperimentDefinition(experiment_id="exp", variants=single), + default_experiment=ExperimentDefinition(experiment_id="default", variants=single), + config=BatchRunConfig(run_dir=tmp_path / "runs", **config), + ) + + +class TestHarnessContractAtResolution: + def test_unsupported_field_aborts_instead_of_skipping(self, tmp_path): + from coder_eval.orchestration.harness_contract import HarnessContractError + + task_file = _write_task(tmp_path, "agent:\n type: codex\n permission_mode: acceptEdits\n") + with pytest.raises(HarnessContractError, match=r"agent\.permission_mode.*'codex'"): + _resolve_all(tmp_path, task_file) + + def test_cli_prompt_file_is_inlined_after_layer_five(self, tmp_path, monkeypatch): + (tmp_path / "prompt.md").write_text("be terse\n") + monkeypatch.chdir(tmp_path) + task_file = _write_task(tmp_path, "agent:\n type: claude-code\n") + resolved, skipped = _resolve_all(tmp_path, task_file, overrides={"agent.system_prompt_file": "prompt.md"}) + assert not skipped + agent = resolved[0].task.agent + assert agent is not None + assert agent.system_prompt == "be terse" + assert agent.system_prompt_file is None + + def test_missing_cli_prompt_file_is_a_resolution_failure(self, tmp_path, monkeypatch): + """An ordinary per-task failure: with no other task left, resolution refuses the empty run.""" + monkeypatch.chdir(tmp_path) + good = _write_task(tmp_path, "agent:\n type: claude-code\n") + with pytest.raises(ValueError, match="system_prompt_file not found"): + _resolve_all(tmp_path, good, overrides={"agent.system_prompt_file": "missing.md"}) diff --git a/tests/test_harbor_experiment_packager.py b/tests/test_harbor_experiment_packager.py index 2be63d64f..c3150ef5d 100644 --- a/tests/test_harbor_experiment_packager.py +++ b/tests/test_harbor_experiment_packager.py @@ -306,3 +306,16 @@ def test_traversal_variant_id_is_refused(self, tmp_path: Path) -> None: # Nothing should have been written outside out_dir. assert not (tmp_path.parent / "tmp" / "pwned").exists() + + +def test_unsupported_agent_field_is_a_clean_cli_error(tmp_path: Path) -> None: + from typer.testing import CliRunner + + from coder_eval.cli import app + + task_file = _write_task(tmp_path, {"agent": {"type": "codex", "permission_mode": "acceptEdits"}}) + exp_file = _write_experiment(tmp_path, {"experiment_id": "e", "variants": [{"variant_id": "v"}]}) + result = CliRunner().invoke(app, ["export", str(task_file), "-e", str(exp_file), "-o", str(tmp_path / "out")]) + assert result.exit_code == 1 + assert "config error" in result.output + assert "agent.permission_mode" in result.output diff --git a/tests/test_harness_contract.py b/tests/test_harness_contract.py index 2978467b0..f6061ebf4 100644 --- a/tests/test_harness_contract.py +++ b/tests/test_harness_contract.py @@ -1,9 +1,10 @@ -"""The harness contract: the model, registration validation, and every built-in's declaration.""" +"""The harness contract: the model, registration validation, the resolution check, and ``by_type``.""" from __future__ import annotations from collections.abc import Iterator -from typing import Literal +from pathlib import Path +from typing import Any, Literal import pytest from pydantic import BaseModel, ConfigDict, ValidationError @@ -14,9 +15,28 @@ BaseAgentConfig, ClaudeCodeAgentConfig, Enforcement, + ExperimentDefaults, + ExperimentDefinition, + ExperimentVariant, + FileExistsCriterion, HarnessContract, + SandboxConfig, + TaskDefinition, parse_agent_config, ) +from coder_eval.orchestration.config import BatchRunConfig +from coder_eval.orchestration.config_merge import MergeError +from coder_eval.orchestration.experiment import ( + DEFAULT_EXPERIMENT_PATH, + _apply_cli_overrides, + load_experiment, + resolve_task_for_variant, +) +from coder_eval.orchestration.harness_contract import ( + HarnessContractError, + TaskResolutionError, + validate_harness_contract, +) from coder_eval.plugins import ensure_plugins_loaded from tests.fixtures.harness_stubs import config_for_kind, stub_contract @@ -131,3 +151,208 @@ def test_every_builtin_accepts_cost_log_tags(kind: AgentKind) -> None: tags = {"x-ce-run-id": "r"} agent = registration.agent_class(parse_agent_config(type=kind), cost_log_tags=tags) assert agent.cost_log_tags == tags + + +def _task(kind: str, **agent_fields: Any) -> TaskDefinition: + prompt = None if kind == AgentKind.NONE else "do it" + return TaskDefinition( + task_id="t", + description="d", + initial_prompt=prompt, + agent=parse_agent_config(type=kind, **agent_fields), + sandbox=SandboxConfig(driver="tempdir"), + success_criteria=[FileExistsCriterion(description="c", path="out.txt")], + ) + + +_GATED_VALUES: dict[str, Any] = { + "system_prompt": "be terse", + "plugins": [{"type": "local", "path": "/plugins/p"}], + "permission_mode": "plan", + "allowed_tools": ["Bash"], + "disallowed_tools": ["Bash"], +} + + +class TestValidateHarnessContract: + @pytest.mark.parametrize("field", list(_GATED_VALUES)) + def test_unsupported_field_is_rejected(self, field: str) -> None: + with pytest.raises(HarnessContractError) as exc: + validate_harness_contract(_task(AgentKind.NONE, **{field: _GATED_VALUES[field]})) + message = str(exc.value) + assert f"agent.{field}" in message + assert "'none'" in message + assert "docs/agents/HARNESS_PARITY.md" in message + assert "claude-code" in message.split("honors it", 1)[1] + + @pytest.mark.parametrize("field", list(_GATED_VALUES)) + def test_enforced_field_is_accepted(self, field: str) -> None: + validate_harness_contract(_task(AgentKind.CLAUDE_CODE, **{field: _GATED_VALUES[field]})) + + def test_the_error_is_a_task_resolution_error(self) -> None: + with pytest.raises(TaskResolutionError): + validate_harness_contract(_task(AgentKind.CODEX, permission_mode="acceptEdits")) + + def test_ungated_field_passes_on_a_harness_that_supports_nothing(self) -> None: + validate_harness_contract(_task(AgentKind.NONE, model="m", ignore_patterns=["*.log"])) + + def test_unset_default_permission_mode_passes(self) -> None: + task = _task(AgentKind.CODEX) + assert "permission_mode" not in task.agent.model_fields_set # type: ignore[union-attr] + validate_harness_contract(task) + + def test_explicit_empty_allowlist_is_set(self) -> None: + with pytest.raises(HarnessContractError, match=r"agent\.allowed_tools"): + validate_harness_contract(_task(AgentKind.CODEX, allowed_tools=[])) + + def test_null_plugins_from_default_is_not_set(self) -> None: + default = ExperimentDefinition( + experiment_id="default", + defaults=ExperimentDefaults(agent={"type": "claude-code", "plugins": None}), + variants=[ExperimentVariant(variant_id="default")], + ) + resolved, _, _ = resolve_task_for_variant(default, _task(AgentKind.CODEX), default, default.variants[0]) + assert resolved.agent is not None and "plugins" in resolved.agent.model_fields_set + validate_harness_contract(resolved) + + def test_task_without_agent_type_is_left_to_the_type_guard(self) -> None: + validate_harness_contract(_task(AgentKind.CODEX).model_copy(update={"agent": None})) + + def test_unregistered_kind_is_rejected(self, restored_registry: None) -> None: + AgentRegistry.register(KIND, config_for_kind(KIND))(_ContractAgent) + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="do it", + agent={"type": KIND}, + success_criteria=[FileExistsCriterion(description="c", path="out.txt")], + ) + AgentRegistry._registry.pop(KIND) + with pytest.raises(HarnessContractError, match="not registered"): + validate_harness_contract(task) + + +def _default_experiment(**by_type: dict[str, Any]) -> ExperimentDefinition: + agent: dict[str, Any] = {"type": "claude-code", "plugins": None} + if by_type: + agent["by_type"] = by_type + return ExperimentDefinition( + experiment_id="default", + defaults=ExperimentDefaults(agent=agent), + variants=[ExperimentVariant(variant_id="default")], + ) + + +def _resolve( + default: ExperimentDefinition, + task: TaskDefinition, + experiment: ExperimentDefinition | None = None, + **config: Any, +) -> tuple[TaskDefinition, dict[str, Any]]: + experiment = experiment or ExperimentDefinition(experiment_id="exp", variants=[ExperimentVariant(variant_id="v")]) + batch = BatchRunConfig(run_dir=Path("."), **config) + resolved, lineage, _ = resolve_task_for_variant(default, task, experiment, experiment.variants[0], batch) + _apply_cli_overrides(resolved, batch, lineage) + return resolved, lineage + + +def _bare_task(**agent: Any) -> TaskDefinition: + return TaskDefinition( + task_id="t", + description="d", + initial_prompt="do it", + agent=agent or None, + success_criteria=[FileExistsCriterion(description="c", path="out.txt")], + ) + + +class TestByType: + def test_default_by_type_reaches_the_matching_kind_with_lineage(self) -> None: + default = _default_experiment(**{"claude-code": {"model": "claude-sonnet-4-6"}}) + resolved, lineage = _resolve(default, _bare_task()) + assert resolved.agent is not None and resolved.agent.model == "claude-sonnet-4-6" + assert lineage["agent.model"].source == "default" + assert lineage["agent.model"].source_detail == "by_type.claude-code" + + def test_cli_type_does_not_inherit_another_kinds_entry(self) -> None: + default = _default_experiment(**{"claude-code": {"model": "claude-sonnet-4-6", "permission_mode": "plan"}}) + resolved, _ = _resolve(default, _bare_task(), agent_type="pi") + assert resolved.agent is not None and str(resolved.agent.type) == "pi" + assert resolved.agent.model is None + assert "permission_mode" not in resolved.agent.model_fields_set + + def test_cli_kind_selects_the_entry_over_the_task_type(self) -> None: + default = _default_experiment(pi={"model": "m-pi"}, codex={"model": "m-codex"}) + resolved, _ = _resolve(default, _bare_task(type="pi"), agent_type="codex") + assert resolved.agent is not None and resolved.agent.model == "m-codex" + + def test_explicit_dash_d_type_beats_dash_dash_type(self) -> None: + default = _default_experiment(pi={"model": "m-pi"}, codex={"model": "m-codex"}) + resolved, _ = _resolve(default, _bare_task(), agent_type="codex", overrides={"agent.type": "pi"}) + assert resolved.agent is not None and resolved.agent.model == "m-pi" + + def test_cli_kind_entry_may_carry_fields_only_that_kind_declares(self) -> None: + default = _default_experiment(pi={"thinking_level": "high"}) + resolved, lineage = _resolve(default, _bare_task(), agent_type="pi") + assert resolved.agent is not None and resolved.agent.thinking_level == "high" # type: ignore[attr-defined] + assert lineage["agent.type"].source_detail == "--type" + + def test_cli_kind_drops_nothing_from_a_task_of_another_kind(self) -> None: + default = _default_experiment(**{"claude-code": {"sdk_options": {"effort": "high"}}}) + resolved, _ = _resolve(default, _bare_task(type="pi"), agent_type="claude-code") + assert resolved.agent is not None and resolved.agent.sdk_options == {"effort": "high"} # type: ignore[attr-defined] + + def test_task_value_beats_by_type(self) -> None: + default = _default_experiment(**{"claude-code": {"model": "claude-sonnet-4-6"}}) + resolved, lineage = _resolve(default, _bare_task(type="claude-code", model="task-model")) + assert resolved.agent is not None and resolved.agent.model == "task-model" + assert lineage["agent.model"].source == "task" + + def test_experiment_by_type_beats_default_by_type(self) -> None: + default = _default_experiment(**{"claude-code": {"model": "default-model"}}) + experiment = ExperimentDefinition( + experiment_id="exp", + defaults=ExperimentDefaults(agent={"by_type": {"claude-code": {"model": "exp-model"}}}), + variants=[ExperimentVariant(variant_id="v")], + ) + resolved, lineage = _resolve(default, _bare_task(), experiment) + assert resolved.agent is not None and resolved.agent.model == "exp-model" + assert lineage["agent.model"].source == "experiment-defaults" + assert lineage["agent.model"].source_detail == "by_type.claude-code" + + def test_unregistered_kind_is_tolerated(self) -> None: + default = _default_experiment(**{"not-installed": {"model": "x"}, "claude-code": {"model": "m"}}) + resolved, _ = _resolve(default, _bare_task()) + assert resolved.agent is not None and resolved.agent.model == "m" + + @pytest.mark.parametrize("by_type", ["pi", {"pi": "not-a-mapping"}]) + def test_non_mapping_is_rejected(self, by_type: Any) -> None: + default = _default_experiment() + assert default.defaults is not None and default.defaults.agent is not None + default.defaults.agent["by_type"] = by_type + with pytest.raises(ValueError, match=r"default agent\.by_type must map"): + _resolve(default, _bare_task()) + + def test_entry_setting_type_is_rejected(self) -> None: + default = _default_experiment(pi={"type": "codex"}) + with pytest.raises(ValueError, match=r"by_type\.pi must not set 'type'"): + _resolve(default, _bare_task()) + + def test_by_type_on_a_task_is_rejected(self) -> None: + with pytest.raises(ValidationError, match="by_type"): + _bare_task(type="codex", by_type={"codex": {"model": "m"}}) + + def test_by_type_on_a_variant_is_rejected(self) -> None: + default = _default_experiment() + experiment = ExperimentDefinition( + experiment_id="exp", + variants=[ExperimentVariant(variant_id="v", agent={"by_type": {"codex": {"model": "m"}}})], + ) + with pytest.raises(MergeError, match="by_type"): + _resolve(default, _bare_task(), experiment) + + def test_the_shipped_default_experiment_resolves_every_builtin_kind(self) -> None: + default = load_experiment(DEFAULT_EXPERIMENT_PATH) + for kind in (k for k in AgentKind if k not in (AgentKind.UNKNOWN, AgentKind.NONE)): + resolved, _ = _resolve(default, _bare_task(), agent_type=str(kind)) + validate_harness_contract(resolved) diff --git a/tests/test_merge_characterization.py b/tests/test_merge_characterization.py index 2326b7290..bdb27b252 100644 --- a/tests/test_merge_characterization.py +++ b/tests/test_merge_characterization.py @@ -306,7 +306,10 @@ def test_system_prompt_file_override_clears_sibling(self): def test_sdk_options_on_codex_raises_friendly(self): task = _live_task(agent={"type": "codex"}) - with pytest.raises(OverrideError, match="only supported for claude-code"): + with pytest.raises( + OverrideError, + match="sdk_options is not a field of the 'codex' agent config; it is supported by: claude-code", + ): apply_overrides(task, {"agent.sdk_options.effort": "high"}) def test_lineage_cli_source_for_touched_paths_only(self): diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 2ec85a904..cfb4e2b2e 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -735,13 +735,6 @@ async def test_unresolved_path_warns_and_injects_nothing(self, patch_exec, tmp_p assert "env var likely unset" in caplog.text assert "0 skill path(s) resolved" in caplog.text - async def test_plugins_are_no_longer_announced_as_unenforced(self, patch_exec, tmp_path, caplog): - root = _skill_repo(tmp_path / "plug") - patch_exec(_FakeProcess(HAPPY_STREAM)) - with caplog.at_level("WARNING"): - await _agent(plugins=[{"type": "local", "path": str(root)}]).start(str(tmp_path / "sandbox")) - assert "NOT enforced" not in caplog.text - async def test_resolved_paths_are_recorded_for_audit(self, patch_exec, tmp_path): root = _skill_repo(tmp_path / "plug") patch_exec(_FakeProcess(HAPPY_STREAM)) @@ -833,20 +826,6 @@ async def test_unusable_inherited_config_is_replaced_with_a_warning( assert "replacing it with the injected config" in caplog.text -class TestUnsupportedConfigIsAnnounced: - async def test_enforced_fields_do_not_warn(self, patch_exec, tmp_path, caplog): - patch_exec(_FakeProcess(HAPPY_STREAM)) - with caplog.at_level("WARNING"): - await _agent(allowed_tools=["Bash"], system_prompt="be terse").start(str(tmp_path)) - assert "NOT enforced" not in caplog.text - - async def test_no_warning_when_nothing_is_dropped(self, patch_exec, tmp_path, caplog): - patch_exec(_FakeProcess(HAPPY_STREAM)) - with caplog.at_level("WARNING"): - await _agent().start(str(tmp_path)) - assert "NOT enforced" not in caplog.text - - class TestArgvConstruction: async def test_defaults_include_auto_and_pure(self, patch_exec, tmp_path): captured = patch_exec(_FakeProcess(HAPPY_STREAM)) diff --git a/tests/test_overrides_engine.py b/tests/test_overrides_engine.py index 6a1753f5d..5ba6fe344 100644 --- a/tests/test_overrides_engine.py +++ b/tests/test_overrides_engine.py @@ -144,12 +144,18 @@ def test_agent_type_injection_switches_subclass(self): def test_sdk_options_on_codex_raises(self): task = _make_task(agent=parse_agent_config(type="codex")) - with pytest.raises(OverrideError, match="only supported for claude-code"): + with pytest.raises( + OverrideError, + match="sdk_options is not a field of the 'codex' agent config; it is supported by: claude-code", + ): apply_overrides(task, {"agent.sdk_options.effort": "high"}) def test_sdk_options_with_agent_type_codex_raises(self): task = _make_task(agent=parse_agent_config(type="claude-code")) - with pytest.raises(OverrideError, match="only supported for claude-code"): + with pytest.raises( + OverrideError, + match="sdk_options is not a field of the 'codex' agent config; it is supported by: claude-code", + ): apply_overrides(task, {"agent.sdk_options.effort": "high"}, agent_type="codex") def test_unknown_root_raises(self): diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index a0cc487e1..9b90b7d93 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -394,14 +394,7 @@ async def test_host_environment_is_inherited_whole(self, patch_exec, tmp_path, m assert captured["kwargs"]["env"]["OPENROUTER_API_KEY"] == "sk-test" -class TestUnsupportedConfigIsAnnounced: - async def test_start_warns_about_unenforced_fields(self, patch_exec, tmp_path, caplog): - patch_exec(_FakeProcess(HAPPY_STREAM)) - with caplog.at_level("WARNING"): - await _agent(system_prompt_file="prompt.md").start(str(tmp_path)) - assert "system_prompt_file" in caplog.text - assert "NOT enforced" in caplog.text - +class TestPluginWarnings: async def test_plugins_that_do_not_resolve_warn_loudly(self, patch_exec, tmp_path, caplog): """plugins IS supported now (-> --skill), but a path that resolves to no skills must warn — else the run silently measures the model WITHOUT the skill.""" @@ -409,16 +402,6 @@ async def test_plugins_that_do_not_resolve_warn_loudly(self, patch_exec, tmp_pat with caplog.at_level("WARNING"): await _agent(plugins=[{"type": "local", "path": "/no/such/dir"}]).start(str(tmp_path)) assert "0 skill dir(s) resolved" in caplog.text or "did not resolve" in caplog.text - # plugins is no longer named in the "NOT enforced" warning. - assert "plugins" not in "".join(r.message for r in caplog.records if "NOT enforced" in r.message) - - async def test_enforced_fields_do_not_warn(self, patch_exec, tmp_path, caplog): - patch_exec(_FakeProcess(HAPPY_STREAM)) - with caplog.at_level("WARNING"): - await _agent(allowed_tools=["Read"], disallowed_tools=["Bash"], system_prompt="be terse").start( - str(tmp_path) - ) - assert "NOT enforced" not in caplog.text class TestAutoRetry: diff --git a/tests/test_plan_command.py b/tests/test_plan_command.py index f8796fb9b..dfeb03e93 100644 --- a/tests/test_plan_command.py +++ b/tests/test_plan_command.py @@ -311,3 +311,28 @@ def test_plan_exits_on_explicit_experiment_load_failure(self, tmp_path: Path) -> printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "Bad experiment" in printed + + +class TestPlanCommandHarnessContract: + def test_unsupported_agent_field_flips_the_exit_code(self, tmp_path: Path) -> None: + task_file = tmp_path / "task.yaml" + task_file.write_text( + "task_id: contract-task\ndescription: d\ninitial_prompt: do it\n" + + "agent:\n type: codex\n permission_mode: acceptEdits\n" + + "sandbox:\n driver: tempdir\n" + + "success_criteria:\n - type: file_exists\n path: out.txt\n description: c\n" + ) + exp_file = tmp_path / "experiment.yaml" + exp_file.write_text("experiment_id: contract\nvariants:\n - variant_id: default\n") + with ( + patch("coder_eval.cli.plan_command.check_tools"), + patch("coder_eval.cli.plan_command.check_api_keys"), + patch(f"{_EXP}.DEFAULT_EXPERIMENT_PATH", tmp_path / "missing.yaml"), + patch("coder_eval.cli.plan_command.console") as mock_console, + pytest.raises(typer.Exit) as exc, + ): + run_plan(task_files=[task_file], experiment=exp_file) + printed = " ".join(str(call) for call in mock_console.print.call_args_list) + assert exc.value.exit_code == 1 + assert "config error" in printed + assert "agent.permission_mode" in printed From a45d4dada55c7ef45b76a574a9de07b58e741adc Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 10:15:09 -0700 Subject: [PATCH 05/12] =?UTF-8?q?feat(agents):=203b/6=20=E2=80=94=20declar?= =?UTF-8?q?e=20permission=5Fmodes=20and=20a=20closed=20tool-name=20map=20p?= =?UTF-8?q?er=20harness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HarnessContract gains permission_modes, the permission_mode values a harness honors with their Claude Code meaning. ToolNameMap maps every name in CANONICAL_TOOL_NAMES to the harness's native tools (an empty tuple for a tool the harness lacks), is validated at registration, and replaces the silent .get(name, ()) reads. Pi and OpenCode smoke tasks move to bypassPermissions. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 12 +++ docs/REPORT_SCHEMA.md | 3 +- src/coder_eval/agent.py | 6 +- src/coder_eval/agents/antigravity_agent.py | 16 ++-- src/coder_eval/agents/claude_code_agent.py | 5 + src/coder_eval/agents/opencode_agent.py | 21 +++-- src/coder_eval/agents/pi_agent.py | 16 ++-- src/coder_eval/agents/registry.py | 16 +++- src/coder_eval/models/__init__.py | 5 +- src/coder_eval/models/enums.py | 31 +++++++ src/coder_eval/models/harness_contract.py | 93 ++++++++++++++++++- tasks/opencode_smoke_test.yaml | 2 +- tasks/pi_smoke_test.yaml | 2 +- tests/fixtures/harness_stubs.py | 8 +- tests/test_antigravity_agent.py | 13 ++- tests/test_harness_contract.py | 102 +++++++++++++++++++++ tests/test_opencode_agent.py | 14 ++- tests/test_pi_agent.py | 9 +- 18 files changed, 324 insertions(+), 50 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 0dd8c80b8..f9c485d1d 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -507,6 +507,18 @@ named tools, a deny always wins, and `plan` denies the Write, Edit and Bash equi restricts nothing, because Claude Code passes `[]` as "no `--allowedTools` flag"; the same YAML must not mean "all tools" on one harness and "no tools" on the others. +One meaning per field is not enough; each VALUE needs one too (`c/harness-architecture-comparison.md` +§ 6, P0-1 and P0-2). Two defects made that concrete. The inverse tool maps were read with +`.get(name, ())`, so a typo or a name the harness lacks silently restricted nothing. And +`permission_mode: default` meant "ask for approval" on Claude Code but "run autonomously" on the +other harnesses. So the contract lists the `permission_modes` a harness honors, and `tool_names` is a +`ToolNameMap` that is total and closed over `CANONICAL_TOOL_NAMES`: a canonical name the harness has +no tool for maps to `()` explicitly, and a missing row fails at adapter import. `Task` and `Agent` both +name the subagent tool (`TOOL_NAME_ALIASES`), so `from_inverse` gives `Task` the natives of `Agent`; +otherwise the older spelling, which the corpus still uses, would restrict nothing. Pi, OpenCode and +Antigravity honor only `plan` and `bypassPermissions` until a native mechanism with the Claude Code +meaning of `default` / `acceptEdits` is verified. + - **Pi** (0.85.1): `--tools ` is an allowlist and `--exclude-tools ` a denylist over the lowercase built-ins. The denied set is subtracted before `--tools` is emitted, and an allowlist that maps to nothing becomes `--no-tools`. diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index c50c6122d..b1f1a8cd1 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -156,7 +156,8 @@ those two the boundary is a reporting change, not a behavioral one. `environment_info.harness_contract` is the agent class's declared contract: for each of `system_prompt`, `plugin_skills`, `permission_mode`, `allowed_tools` and `disallowed_tools`, `"enforced"` or `"unsupported"`, plus -`system_prompt_semantics` (the class default) and `cooperative_stop`. +`system_prompt_semantics` (the class default), `cooperative_stop`, and +`permission_modes` (the sorted `permission_mode` values the harness honors, or `null`). `sdk_options.system_prompt` is a `SystemPromptPreset` dict (`{type: "preset", preset: "claude_code", exclude_dynamic_sections: true, append?: str}`) on append-mode Claude Code runs and a plain string only in replace mode — it is diff --git a/src/coder_eval/agent.py b/src/coder_eval/agent.py index d36d22b64..1e4bdb7b1 100644 --- a/src/coder_eval/agent.py +++ b/src/coder_eval/agent.py @@ -11,7 +11,7 @@ from .errors import AgentCrashError, TurnTimeoutError from .errors.agent import format_timeout_reason, truncate_crash_message from .models import AgentState as AgentState -from .models import ApiRoute, BaseAgentConfig, HarnessContract, TurnRecord +from .models import ApiRoute, BaseAgentConfig, HarnessContract, ToolNameMap, TurnRecord from .streaming.callbacks import StreamCallback from .streaming.collector import EventCollector from .streaming.events import AgentEndStatus @@ -75,6 +75,10 @@ def __init__(self, config: ClaudeCodeAgentConfig, ...): # Rationale: .claude/notes/agents.md § The system_prompt_semantics marker contract: ClassVar[HarnessContract] + # Canonical tool name -> native tools. Registration requires one exactly when the + # contract enforces allowed_tools or disallowed_tools. + tool_names: ClassVar[ToolNameMap | None] = None + def __init__( self, config: ConfigT, route: ApiRoute | None = None, *, cost_log_tags: dict[str, str] | None = None ) -> None: diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 6e4c13eda..86320ea56 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -50,6 +50,7 @@ HarnessContract, PermissionMode, TokenUsage, + ToolNameMap, TranscriptMessage, TurnRecord, ) @@ -124,11 +125,10 @@ "finish": "Finish", } -# Inverse of _ANTIGRAVITY_TO_CLAUDE_TOOL_MAP: each Claude name -> its harness tools. -_CLAUDE_TO_ANTIGRAVITY_TOOLS: dict[str, tuple[str, ...]] = { - claude: tuple(sorted(tool for tool, name in _ANTIGRAVITY_TO_CLAUDE_TOOL_MAP.items() if name == claude)) - for claude in set(_ANTIGRAVITY_TO_CLAUDE_TOOL_MAP.values()) -} +# The canonical tool names Antigravity has no tool for. +_ANTIGRAVITY_NO_EQUIVALENT: frozenset[str] = frozenset({"NotebookEdit", "Skill", "TodoWrite", "ToolSearch"}) + +_TOOL_NAMES = ToolNameMap.from_inverse(_ANTIGRAVITY_TO_CLAUDE_TOOL_MAP, no_equivalent=_ANTIGRAVITY_NO_EQUIVALENT) # The harness ends a turn by calling `finish`, so an allowlist never denies it. _TURN_END_TOOL = "finish" @@ -206,7 +206,9 @@ class AntigravityAgent(Agent[AntigravityAgentConfig]): allowed_tools=Enforcement.ENFORCED, disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, + permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), ) + tool_names = _TOOL_NAMES def __init__( self, @@ -330,12 +332,12 @@ def _policies(self, policy: Any) -> list[Any]: if not self.config.allowed_tools: policies = [policy.allow_all()] else: - allowed = {t for name in self.config.allowed_tools for t in _CLAUDE_TO_ANTIGRAVITY_TOOLS.get(name, ())} + allowed = {t for name in self.config.allowed_tools for t in _TOOL_NAMES.names[name]} policies = [policy.deny_all(), *(policy.allow(t) for t in sorted(allowed | {_TURN_END_TOOL}))] deny_names = list(self.config.disallowed_tools or []) if self.config.permission_mode is PermissionMode.PLAN: deny_names += READ_ONLY_DENIED_TOOLS - denied = {t for name in deny_names for t in _CLAUDE_TO_ANTIGRAVITY_TOOLS.get(name, ())} - {_TURN_END_TOOL} + denied = {t for name in deny_names for t in _TOOL_NAMES.names[name]} - {_TURN_END_TOOL} return policies + [policy.deny(t) for t in sorted(denied)] async def start( diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index b139211ce..f02c02ff2 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -43,6 +43,7 @@ ) from coder_eval.formatting import format_messages, format_payload from coder_eval.models import ( + CANONICAL_TOOL_NAMES, AgentKind, ApiRoute, BedrockRoute, @@ -53,9 +54,11 @@ Enforcement, HarnessContract, LiteLLMRoute, + PermissionMode, ResultSummary, SystemPromptSemantics, TokenUsage, + ToolNameMap, TranscriptMessage, TurnRecord, to_bedrock_inference_profile, @@ -701,7 +704,9 @@ class ClaudeCodeAgent(Agent[ClaudeCodeAgentConfig]): allowed_tools=Enforcement.ENFORCED, disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, + permission_modes=frozenset(PermissionMode), ) + tool_names = ToolNameMap(names={name: (name,) for name in CANONICAL_TOOL_NAMES}, mcp_names=True) # One warning per agent for a replace-mode config with no prompt: the resolver # runs on every query, and a per-turn repeat would bury the rest of task.log. diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index 825edf0de..de1d0cd0c 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -48,6 +48,7 @@ PermissionMode, ResultSummary, TokenUsage, + ToolNameMap, TranscriptMessage, TurnRecord, ) @@ -122,6 +123,7 @@ "grep": "Grep", "list": "LS", "webfetch": "WebFetch", + "websearch": "WebSearch", "todowrite": "TodoWrite", "todoread": "TodoRead", "task": "Agent", @@ -165,12 +167,15 @@ # restricts tools only; `--auto` approved them before. _NON_TOOL_PERMISSIONS: tuple[str, ...] = ("external_directory", "doom_loop") -# Inverse of _TOOL_NAME_MAP: each Claude name -> the permission keys that govern it. +# The canonical tool names OpenCode has no tool for. +_OPENCODE_NO_EQUIVALENT: frozenset[str] = frozenset({"NotebookEdit", "ToolSearch"}) + +_TOOL_NAMES = ToolNameMap.from_inverse(_TOOL_NAME_MAP, no_equivalent=_OPENCODE_NO_EQUIVALENT) + +# Each canonical tool name -> the permission keys that govern its OpenCode tools. _CLAUDE_TO_OPENCODE_PERMISSION: dict[str, tuple[str, ...]] = { - claude: tuple( - sorted({_PERMISSION_KEY_FOR_TOOL.get(tool, tool) for tool, name in _TOOL_NAME_MAP.items() if name == claude}) - ) - for claude in set(_TOOL_NAME_MAP.values()) + canonical: tuple(sorted({_PERMISSION_KEY_FOR_TOOL.get(tool, tool) for tool in natives})) + for canonical, natives in _TOOL_NAMES.names.items() } # Skill paths, the system-prompt `instructions` file and tool `permission` rules are @@ -757,7 +762,9 @@ class OpenCodeAgent(Agent[OpenCodeAgentConfig]): allowed_tools=Enforcement.ENFORCED, disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, + permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), ) + tool_names = _TOOL_NAMES def __init__( self, @@ -957,9 +964,9 @@ def _permission_config(self) -> dict[str, str] | None: permission["*"] = "deny" permission.update(dict.fromkeys(_NON_TOOL_PERMISSIONS, "allow")) for name in self.config.allowed_tools: - permission.update(dict.fromkeys(_CLAUDE_TO_OPENCODE_PERMISSION.get(name, ()), "allow")) + permission.update(dict.fromkeys(_CLAUDE_TO_OPENCODE_PERMISSION[name], "allow")) for name in deny_names: - permission.update(dict.fromkeys(_CLAUDE_TO_OPENCODE_PERMISSION.get(name, ()), "deny")) + permission.update(dict.fromkeys(_CLAUDE_TO_OPENCODE_PERMISSION[name], "deny")) return permission or None def _inject_config_content(self, env: dict[str, str]) -> None: diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 056b65759..43aaf81a5 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -56,6 +56,7 @@ PiAgentConfig, ResultSummary, TokenUsage, + ToolNameMap, TranscriptMessage, TurnRecord, ) @@ -137,11 +138,10 @@ }, } -# Inverse of _TOOL_NAME_MAP: each Claude name -> every Pi tool it stands for. -_CLAUDE_TO_PI_TOOLS: dict[str, tuple[str, ...]] = { - claude: tuple(sorted(pi for pi, name in _TOOL_NAME_MAP.items() if name == claude)) - for claude in set(_TOOL_NAME_MAP.values()) -} +# The canonical tool names Pi has no tool for. +_PI_NO_EQUIVALENT: frozenset[str] = frozenset({"NotebookEdit", "Skill", "ToolSearch", "WebSearch"}) + +_TOOL_NAMES = ToolNameMap.from_inverse(_TOOL_NAME_MAP, no_equivalent=_PI_NO_EQUIVALENT) # The full recognized Pi vocabulary (from `pi` 0.84.4). A clean exit that # recognized NOTHING from this set is vocabulary drift and is crashed, not scored. @@ -684,7 +684,9 @@ class PiAgent(Agent[PiAgentConfig]): allowed_tools=Enforcement.ENFORCED, disallowed_tools=Enforcement.ENFORCED, cooperative_stop=True, + permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), ) + tool_names = _TOOL_NAMES def __init__( self, @@ -866,9 +868,9 @@ def _tool_flags(self) -> list[str]: deny_names = list(self.config.disallowed_tools or []) if self.config.permission_mode is PermissionMode.PLAN: deny_names += READ_ONLY_DENIED_TOOLS - deny = {pi for name in deny_names for pi in _CLAUDE_TO_PI_TOOLS.get(name, ())} + deny = {pi for name in deny_names for pi in _TOOL_NAMES.names[name]} if self.config.allowed_tools: - allow = {pi for name in self.config.allowed_tools for pi in _CLAUDE_TO_PI_TOOLS.get(name, ())} - deny + allow = {pi for name in self.config.allowed_tools for pi in _TOOL_NAMES.names[name]} - deny return ["--tools", ",".join(sorted(allow))] if allow else ["--no-tools"] return ["--exclude-tools", ",".join(sorted(deny))] if deny else [] diff --git a/src/coder_eval/agents/registry.py b/src/coder_eval/agents/registry.py index 136f80e72..177c7e83e 100644 --- a/src/coder_eval/agents/registry.py +++ b/src/coder_eval/agents/registry.py @@ -25,19 +25,29 @@ def _validate_registration(kind: str, agent_cls: type, config_class: type) -> No """Reject an ``(agent class, config class)`` pair the resolver cannot trust. Raises: - TypeError: the agent class declares no ``HarnessContract``, or the config + TypeError: the agent class declares no ``HarnessContract``, its ``tool_names`` + presence does not match the contract's tool-list rows, or the config class is not a ``BaseAgentConfig`` with ``extra="forbid"`` whose ``type`` Literal names ``kind``. """ - from coder_eval.models import BaseAgentConfig, HarnessContract + from coder_eval.models import BaseAgentConfig, Enforcement, HarnessContract, ToolNameMap agent_name = agent_cls.__name__ config_name = config_class.__name__ - if not isinstance(getattr(agent_cls, "contract", None), HarnessContract): + contract = getattr(agent_cls, "contract", None) + if not isinstance(contract, HarnessContract): raise TypeError( f"Agent kind {kind!r}: {agent_name} must declare `contract = HarnessContract(...)` " + "as a class attribute, so the resolver knows which agent fields the harness honors." ) + lists_enforced = Enforcement.ENFORCED in (contract.allowed_tools, contract.disallowed_tools) + tool_names = getattr(agent_cls, "tool_names", None) + has_map = isinstance(tool_names, ToolNameMap) if lists_enforced else tool_names is None + if not has_map: + raise TypeError( + f"Agent kind {kind!r}: {agent_name} must declare `tool_names = ToolNameMap(...)` exactly when its " + + "contract enforces allowed_tools or disallowed_tools, and leave it None otherwise." + ) if not issubclass(config_class, BaseAgentConfig): raise TypeError(f"Agent kind {kind!r}: config class {config_name} must subclass BaseAgentConfig.") if config_class.model_config.get("extra") != "forbid": diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index db6981473..2df3fbc64 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -70,6 +70,7 @@ UiPathEvalCriterion, ) from coder_eval.models.enums import ( + CANONICAL_TOOL_NAMES, READ_ONLY_DENIED_TOOLS, AgentKind, AgentState, @@ -93,7 +94,7 @@ ) # Harness contract -from coder_eval.models.harness_contract import Enforcement, HarnessContract +from coder_eval.models.harness_contract import Enforcement, HarnessContract, ToolNameMap # Judge from coder_eval.models.judge import JudgeVerdict @@ -251,6 +252,7 @@ # Harness contract "Enforcement", "HarnessContract", + "ToolNameMap", # Enums "AgentKind", "AgentState", @@ -258,6 +260,7 @@ "FinalStatus", "PermissionMode", "PreservationMode", + "CANONICAL_TOOL_NAMES", "READ_ONLY_DENIED_TOOLS", # Criteria "BaseSuccessCriterion", diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index 805768f91..adcbfa7e8 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -119,9 +119,40 @@ class PermissionMode(StrEnum): BYPASS_PERMISSIONS = "bypassPermissions" +CANONICAL_TOOL_NAMES: Final[frozenset[str]] = frozenset( + { + "Agent", + "Bash", + "Edit", + "Glob", + "Grep", + "NotebookEdit", + "Read", + "Skill", + "Task", + "TodoWrite", + "ToolSearch", + "WebFetch", + "WebSearch", + "Write", + } +) +"""The tool names ``allowed_tools`` / ``disallowed_tools`` accept. + +The coding tools the Claude Code CLI 2.1.216 (claude-agent-sdk 0.2.124) lists in its +``system/init`` message, plus ``Agent``, ``Glob``, ``Grep`` and ``TodoWrite``, which +that CLI does not list but the task corpus names. ``Task`` and ``Agent`` both name the +subagent tool. +""" + +TOOL_NAME_ALIASES: Final[dict[str, str]] = {"Task": "Agent"} +"""Canonical tool name -> the canonical name of the same tool (an older spelling).""" + READ_ONLY_DENIED_TOOLS: Final[tuple[str, ...]] = ("Write", "Edit", "Bash") """The Claude tool names every harness that maps ``permission_mode: plan`` denies.""" +assert set(READ_ONLY_DENIED_TOOLS) <= CANONICAL_TOOL_NAMES, "READ_ONLY_DENIED_TOOLS must be canonical tool names" + class PreservationMode(StrEnum): """How a task's sandbox is persisted (or not) after execution. diff --git a/src/coder_eval/models/harness_contract.py b/src/coder_eval/models/harness_contract.py index 6b3917ae8..06c56c27b 100644 --- a/src/coder_eval/models/harness_contract.py +++ b/src/coder_eval/models/harness_contract.py @@ -2,12 +2,14 @@ from __future__ import annotations +from collections.abc import Mapping from enum import StrEnum from typing import Self -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator from coder_eval.models.agent_config import SystemPromptMode +from coder_eval.models.enums import CANONICAL_TOOL_NAMES, TOOL_NAME_ALIASES, PermissionMode class Enforcement(StrEnum): @@ -20,7 +22,8 @@ class Enforcement(StrEnum): class HarnessContract(BaseModel): """The per-agent declaration of which uniform fields reach the harness. - A task that sets a field this contract marks ``UNSUPPORTED`` is rejected at + A task that sets a field this contract marks ``UNSUPPORTED``, or a + ``permission_mode`` value outside ``permission_modes``, is rejected at resolution. Every registered agent class declares one. """ @@ -29,13 +32,23 @@ class HarnessContract(BaseModel): system_prompt: Enforcement = Field(description="Whether agent.system_prompt reaches the harness.") system_prompt_semantics: SystemPromptMode | None = Field( default=None, - description="How the prompt combines with the harness's own prompt. None iff system_prompt is unsupported.", + description=( + "How the prompt combines with the harness's own system prompt, through the system or developer " + "instruction channel of the model request, never the user turn. None iff system_prompt is unsupported." + ), ) plugin_skills: Enforcement = Field(description="Whether the skills of agent.plugins reach the harness.") permission_mode: Enforcement = Field(description="Whether agent.permission_mode is honored.") allowed_tools: Enforcement = Field(description="Whether agent.allowed_tools restricts the harness's tools.") disallowed_tools: Enforcement = Field(description="Whether agent.disallowed_tools denies the harness's tools.") cooperative_stop: bool = Field(description="Whether communicate() honors the should_stop poll.") + permission_modes: frozenset[PermissionMode] | None = Field( + default=None, + description=( + "The permission_mode values honored, each with its Claude Code meaning. None iff permission_mode " + "is unsupported." + ), + ) @model_validator(mode="after") def check_semantics_matches_prompt_support(self) -> Self: @@ -47,3 +60,77 @@ def check_semantics_matches_prompt_support(self) -> Self: + f"system_prompt_semantics={self.system_prompt_semantics!r})" ) return self + + @model_validator(mode="after") + def check_modes_match_permission_support(self) -> Self: + """Require a non-empty value set exactly when permission_mode is enforced.""" + enforced = self.permission_mode is Enforcement.ENFORCED + if enforced != (self.permission_modes is not None) or self.permission_modes == frozenset(): + raise ValueError( + "permission_modes must be a non-empty set when permission_mode is 'enforced' and None when it " + + f"is 'unsupported' (got permission_mode={self.permission_mode.value!r}, " + + f"permission_modes={self.permission_modes!r})" + ) + return self + + @field_serializer("permission_modes") + def _sorted_modes(self, modes: frozenset[PermissionMode] | None) -> list[str] | None: + return None if modes is None else sorted(str(mode) for mode in modes) + + +class ToolNameMap(BaseModel): + """Each canonical tool name -> the harness's native tools it stands for. + + Total and closed over ``CANONICAL_TOOL_NAMES``: an empty tuple means the harness + has no such tool, so no name is ever silently dropped. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + names: dict[str, tuple[str, ...]] = Field(description="Canonical tool name -> native tool names.") + mcp_names: bool = Field(default=False, description="Whether mcp____ names reach the harness.") + + @model_validator(mode="after") + def check_total_over_canonical_names(self) -> Self: + """Require exactly one row per canonical tool name.""" + missing = sorted(CANONICAL_TOOL_NAMES - set(self.names)) + extra = sorted(set(self.names) - CANONICAL_TOOL_NAMES) + if missing or extra: + raise ValueError( + f"ToolNameMap must map every canonical tool name exactly (missing={missing}, extra={extra})" + ) + return self + + @classmethod + def from_inverse( + cls, + forward: Mapping[str, str], + *, + no_equivalent: frozenset[str], + mcp_names: bool = False, + ) -> ToolNameMap: + """Invert an adapter's native -> canonical telemetry map. + + Args: + forward: Native tool name -> canonical name. Values outside + ``CANONICAL_TOOL_NAMES`` are telemetry-only and dropped. An alias in + ``TOOL_NAME_ALIASES`` shares its target's natives. + no_equivalent: The canonical names the harness has no tool for. + mcp_names: Whether MCP tool names reach the harness natively. + + Raises: + ValueError: a canonical name is both mapped and in ``no_equivalent``, or in neither. + """ + mapped = { + canonical: tuple(sorted(native for native, name in forward.items() if name == canonical)) + for canonical in set(forward.values()) & CANONICAL_TOOL_NAMES + } + mapped |= {alias: mapped[target] for alias, target in TOOL_NAME_ALIASES.items() if target in mapped} + both = sorted(set(mapped) & no_equivalent) + neither = sorted(CANONICAL_TOOL_NAMES - set(mapped) - no_equivalent) + if both or neither: + raise ValueError( + "each canonical tool name must be mapped or listed in no_equivalent, exactly once " + + f"(in both: {both}; in neither: {neither})" + ) + return cls(names={**mapped, **dict.fromkeys(no_equivalent, ())}, mcp_names=mcp_names) diff --git a/tasks/opencode_smoke_test.yaml b/tasks/opencode_smoke_test.yaml index d5e9f20a5..61b3877d2 100644 --- a/tasks/opencode_smoke_test.yaml +++ b/tasks/opencode_smoke_test.yaml @@ -21,7 +21,7 @@ agent: # step_finish event and the harness folds it into token_usage.total_cost_usd # (falling back to the rate card when the stream omits or zeroes it). model: "openrouter/deepseek/deepseek-v4-pro" - permission_mode: "acceptEdits" + permission_mode: "bypassPermissions" success_criteria: - type: "file_exists" diff --git a/tasks/pi_smoke_test.yaml b/tasks/pi_smoke_test.yaml index f35084dee..ad5c42f26 100644 --- a/tasks/pi_smoke_test.yaml +++ b/tasks/pi_smoke_test.yaml @@ -22,7 +22,7 @@ agent: # rate card only when the stream omits it). Spike-verified working model; # swappable to any Pi-addressable provider/id. model: "openrouter/moonshotai/kimi-k3" - permission_mode: "acceptEdits" + permission_mode: "bypassPermissions" success_criteria: - type: "file_exists" diff --git a/tests/fixtures/harness_stubs.py b/tests/fixtures/harness_stubs.py index 3b848e6f9..6d4bbe25b 100644 --- a/tests/fixtures/harness_stubs.py +++ b/tests/fixtures/harness_stubs.py @@ -10,14 +10,14 @@ def stub_contract(*, cooperative_stop: bool = True) -> HarnessContract: - """A contract that honors every uniform field.""" + """A contract that honors the system prompt and skills only, so it needs no ``tool_names``.""" return HarnessContract( system_prompt=Enforcement.ENFORCED, system_prompt_semantics="append", plugin_skills=Enforcement.ENFORCED, - permission_mode=Enforcement.ENFORCED, - allowed_tools=Enforcement.ENFORCED, - disallowed_tools=Enforcement.ENFORCED, + permission_mode=Enforcement.UNSUPPORTED, + allowed_tools=Enforcement.UNSUPPORTED, + disallowed_tools=Enforcement.UNSUPPORTED, cooperative_stop=cooperative_stop, ) diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index 93bc9d349..cfe8b8616 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -1359,10 +1359,6 @@ def _policy_pairs(**cfg) -> list[tuple[str, str | None]]: ), ({"allowed_tools": ["Skill"]}, [("deny_all", None), ("allow", "finish")]), ({"allowed_tools": []}, [("allow_all", None)]), - ( - {"allowed_tools": ["Read"], "disallowed_tools": ["Finish"]}, - [("deny_all", None), ("allow", "finish"), ("allow", "view_file")], - ), ( {"allowed_tools": ["Bash", "Read"], "disallowed_tools": ["Bash"]}, [ @@ -1428,10 +1424,13 @@ async def __aexit__(self, *exc): ] -def test_inverse_tool_map_covers_every_claude_name(): - from coder_eval.agents.antigravity_agent import _ANTIGRAVITY_TO_CLAUDE_TOOL_MAP, _CLAUDE_TO_ANTIGRAVITY_TOOLS +def test_tool_names_cover_the_canonical_vocabulary(): + from coder_eval.models import CANONICAL_TOOL_NAMES - assert set(_CLAUDE_TO_ANTIGRAVITY_TOOLS) == set(_ANTIGRAVITY_TO_CLAUDE_TOOL_MAP.values()) + assert AntigravityAgent.tool_names is not None + assert set(AntigravityAgent.tool_names.names) == CANONICAL_TOOL_NAMES + assert "Finish" not in AntigravityAgent.tool_names.names + assert AntigravityAgent.tool_names.names["Bash"] == ("run_command",) # --- max_turns visible-turn cap ----------------------------------------------------- diff --git a/tests/test_harness_contract.py b/tests/test_harness_contract.py index f6061ebf4..37297ef1a 100644 --- a/tests/test_harness_contract.py +++ b/tests/test_harness_contract.py @@ -11,6 +11,8 @@ from coder_eval.agents.registry import AgentRegistry from coder_eval.models import ( + CANONICAL_TOOL_NAMES, + READ_ONLY_DENIED_TOOLS, AgentKind, BaseAgentConfig, ClaudeCodeAgentConfig, @@ -20,8 +22,10 @@ ExperimentVariant, FileExistsCriterion, HarnessContract, + PermissionMode, SandboxConfig, TaskDefinition, + ToolNameMap, parse_agent_config, ) from coder_eval.orchestration.config import BatchRunConfig @@ -79,6 +83,89 @@ def test_unknown_field_rejected(self) -> None: HarnessContract(**{**stub_contract().model_dump(), "timing_basis": "wall"}) +class TestPermissionModes: + def _contract(self, **fields: Any) -> HarnessContract: + return HarnessContract(**{**stub_contract().model_dump(), **fields}) + + def test_enforced_permission_mode_requires_a_value_set(self) -> None: + with pytest.raises(ValidationError, match="permission_modes"): + self._contract(permission_mode="enforced") + + def test_unsupported_permission_mode_rejects_a_value_set(self) -> None: + with pytest.raises(ValidationError, match="permission_modes"): + self._contract(permission_modes={PermissionMode.PLAN}) + + def test_empty_value_set_is_rejected(self) -> None: + with pytest.raises(ValidationError, match="permission_modes"): + self._contract(permission_mode="enforced", permission_modes=set()) + + def test_value_set_dumps_sorted(self) -> None: + contract = self._contract( + permission_mode="enforced", permission_modes={PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS} + ) + assert contract.model_dump(mode="json")["permission_modes"] == ["bypassPermissions", "plan"] + + def test_only_claude_code_honors_default_or_accept_edits(self) -> None: + ensure_plugins_loaded() + for kind in (k for k in AgentKind if k is not AgentKind.UNKNOWN): + registration = AgentRegistry.get(kind) + assert registration is not None + modes = registration.agent_class.contract.permission_modes or frozenset() + if kind is not AgentKind.CLAUDE_CODE: + assert not modes & {PermissionMode.DEFAULT, PermissionMode.ACCEPT_EDITS}, kind + + +def _identity_names() -> dict[str, tuple[str, ...]]: + return {name: (name,) for name in CANONICAL_TOOL_NAMES} + + +class TestToolNameMap: + def test_missing_canonical_name_is_rejected(self) -> None: + names = _identity_names() + del names["Bash"] + with pytest.raises(ValidationError, match="missing=\\['Bash'\\]"): + ToolNameMap(names=names) + + def test_extra_name_is_rejected(self) -> None: + with pytest.raises(ValidationError, match="extra=\\['LS'\\]"): + ToolNameMap(names={**_identity_names(), "LS": ("ls",)}) + + def test_from_inverse_rejects_a_name_both_mapped_and_absent(self) -> None: + forward = {name.lower(): name for name in CANONICAL_TOOL_NAMES} + with pytest.raises(ValueError, match="in both: \\['Bash'\\]"): + ToolNameMap.from_inverse(forward, no_equivalent=frozenset({"Bash"})) + + def test_from_inverse_rejects_a_gap(self) -> None: + forward = {name.lower(): name for name in CANONICAL_TOOL_NAMES - {"Skill"}} + with pytest.raises(ValueError, match="in neither: \\['Skill'\\]"): + ToolNameMap.from_inverse(forward, no_equivalent=frozenset()) + + def test_from_inverse_drops_telemetry_only_names_and_groups_natives(self) -> None: + forward = {name.lower(): name for name in CANONICAL_TOOL_NAMES - {"Edit"}} + forward |= {"patch": "Edit", "edit": "Edit", "ls": "LS"} + tool_names = ToolNameMap.from_inverse(forward, no_equivalent=frozenset()) + assert tool_names.names["Edit"] == ("edit", "patch") + assert "LS" not in tool_names.names + + def test_no_equivalent_maps_to_an_empty_tuple(self) -> None: + forward = {name.lower(): name for name in CANONICAL_TOOL_NAMES - {"Skill"}} + assert ToolNameMap.from_inverse(forward, no_equivalent=frozenset({"Skill"})).names["Skill"] == () + + def test_an_alias_shares_its_targets_natives(self) -> None: + forward = {name.lower(): name for name in CANONICAL_TOOL_NAMES - {"Task"}} + assert ToolNameMap.from_inverse(forward, no_equivalent=frozenset()).names["Task"] == ("agent",) + + def test_claude_code_honors_every_mode_and_names_tools_natively(self) -> None: + from coder_eval.agents.claude_code_agent import ClaudeCodeAgent + + assert ClaudeCodeAgent.contract.permission_modes == frozenset(PermissionMode) + assert ClaudeCodeAgent.tool_names is not None and ClaudeCodeAgent.tool_names.mcp_names is True + assert all(natives == (name,) for name, natives in ClaudeCodeAgent.tool_names.names.items()) + + def test_read_only_denied_tools_are_canonical(self) -> None: + assert set(READ_ONLY_DENIED_TOOLS) <= CANONICAL_TOOL_NAMES + + class _ContractAgent: contract = stub_contract() @@ -127,6 +214,21 @@ class TwoKindConfig(BaseAgentConfig): AgentRegistry.register("other-kind", TwoKindConfig)(_ContractAgent) assert AgentRegistry.get("other-kind") is not None + def test_enforced_tool_lists_require_tool_names(self, restored_registry: None) -> None: + class NoMapAgent: + contract = HarnessContract(**{**stub_contract().model_dump(), "allowed_tools": "enforced"}) + + with pytest.raises(TypeError, match=rf"{KIND}.*NoMapAgent.*tool_names"): + AgentRegistry.register(KIND, config_for_kind(KIND))(NoMapAgent) + + def test_unsupported_tool_lists_reject_tool_names(self, restored_registry: None) -> None: + class StrayMapAgent: + contract = stub_contract() + tool_names = ToolNameMap(names=_identity_names()) + + with pytest.raises(TypeError, match=rf"{KIND}.*StrayMapAgent.*tool_names"): + AgentRegistry.register(KIND, config_for_kind(KIND))(StrayMapAgent) + def test_valid_pair_registers_idempotently(self, restored_registry: None) -> None: config = config_for_kind(KIND) AgentRegistry.register(KIND, config)(_ContractAgent) diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index cfb4e2b2e..b2ad525cd 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -859,8 +859,8 @@ class TestPermissionConfig: ({"allowed_tools": ["Bash"]}, {"*": "deny", **_NON_TOOL_ALLOWS, "bash": "allow"}), ({"disallowed_tools": ["Bash"]}, {"bash": "deny"}), ({"permission_mode": "plan"}, {"edit": "deny", "bash": "deny"}), - ({"allowed_tools": ["TodoWrite", "NotATool"]}, {"*": "deny", **_NON_TOOL_ALLOWS, "todowrite": "allow"}), - ({"allowed_tools": ["NotATool"]}, {"*": "deny", **_NON_TOOL_ALLOWS}), + ({"allowed_tools": ["TodoWrite", "NotebookEdit"]}, {"*": "deny", **_NON_TOOL_ALLOWS, "todowrite": "allow"}), + ({"allowed_tools": ["NotebookEdit"]}, {"*": "deny", **_NON_TOOL_ALLOWS}), ( {"allowed_tools": ["Bash", "Write"], "disallowed_tools": ["Bash"]}, {"*": "deny", **_NON_TOOL_ALLOWS, "bash": "deny", "edit": "allow"}, @@ -874,8 +874,14 @@ class TestPermissionConfig: def test_shapes(self, cfg: dict[str, Any], expected: dict[str, str] | None): assert _agent(**cfg)._permission_config() == expected - def test_inverse_map_covers_every_claude_name(self): - assert set(agent_module._CLAUDE_TO_OPENCODE_PERMISSION) == set(agent_module._TOOL_NAME_MAP.values()) + def test_permission_map_covers_the_canonical_vocabulary(self): + from coder_eval.models import CANONICAL_TOOL_NAMES + + assert OpenCodeAgent.tool_names is not None + assert set(OpenCodeAgent.tool_names.names) == CANONICAL_TOOL_NAMES + assert set(agent_module._CLAUDE_TO_OPENCODE_PERMISSION) == CANONICAL_TOOL_NAMES + assert agent_module._CLAUDE_TO_OPENCODE_PERMISSION["NotebookEdit"] == () + assert agent_module._CLAUDE_TO_OPENCODE_PERMISSION["Task"] == ("task",) def test_wildcard_deny_comes_first(self): assert next(iter(_agent(allowed_tools=["Read"])._permission_config() or {})) == "*" diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 9b90b7d93..4dcee023b 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -301,10 +301,13 @@ def test_no_fields_emit_no_flags(self): def test_empty_allowlist_restricts_nothing(self): assert _agent(allowed_tools=[])._tool_flags() == [] - def test_inverse_map_covers_every_claude_name(self): - from coder_eval.agents.pi_agent import _CLAUDE_TO_PI_TOOLS, _TOOL_NAME_MAP + def test_tool_names_cover_the_canonical_vocabulary(self): + from coder_eval.models import CANONICAL_TOOL_NAMES - assert set(_CLAUDE_TO_PI_TOOLS) == set(_TOOL_NAME_MAP.values()) + assert PiAgent.tool_names is not None + assert set(PiAgent.tool_names.names) == CANONICAL_TOOL_NAMES + assert PiAgent.tool_names.names["Edit"] == ("edit", "multiedit", "patch") + assert PiAgent.tool_names.names["Skill"] == () class TestSessionContinuity: From e2592cb670bbbdf3de219550d23c6b4a4b63a4bf Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 10:26:19 -0700 Subject: [PATCH 06/12] =?UTF-8?q?feat(orchestration):=204r/6=20=E2=80=94?= =?UTF-8?q?=20reject=20permission=5Fmode=20values=20and=20tool=20names=20a?= =?UTF-8?q?=20harness=20cannot=20honor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The resolution-time contract check now also rejects a permission_mode value outside the harness's permission_modes and a tool-list name outside CANONICAL_TOOL_NAMES, with a did-you-mean hint. Claude Code keeps its native mcp__ names and permission rule syntax. Orchestrator setup runs the check for library and in-container runs, but not for a re-grade. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/orchestration.md | 5 +- docs/TASK_DEFINITION_GUIDE.md | 11 ++- docs/agents/ANTIGRAVITY.md | 3 + docs/agents/OPENCODE.md | 5 +- docs/agents/PI.md | 8 +- .../orchestration/harness_contract.py | 92 +++++++++++++++---- src/coder_eval/orchestrator.py | 5 + tests/test_experiment_resolver.py | 14 +++ tests/test_harness_contract.py | 67 ++++++++++++++ tests/test_plan_command.py | 29 ++++-- 10 files changed, 208 insertions(+), 31 deletions(-) diff --git a/.claude/notes/orchestration.md b/.claude/notes/orchestration.md index 683229049..800adb40b 100644 --- a/.claude/notes/orchestration.md +++ b/.claude/notes/orchestration.md @@ -39,7 +39,10 @@ early stop. Both raise a `TaskResolutionError`, which `resolve_all_tasks` re-raises instead of demoting to a skipped task and `plan` turns into a non-zero exit. A field counts as SET only if a layer wrote it with a non-null value, so a model default and - the default experiment's `plugins: null` never trip it. A `-D + the default experiment's `plugins: null` never trip it. For a set field on an enforcing + harness it also checks the VALUE: a `permission_mode` outside `contract.permission_modes` + and a tool name outside `CANONICAL_TOOL_NAMES` are rejected, because the adapters index + their total `ToolNameMap` and would otherwise meet the name mid-run. A `-D agent.system_prompt_file` is inlined against `Path.cwd()` after layer 5, so no adapter and no container mount ever sees a prompt file. diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 06f82754b..c7516e854 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -193,7 +193,16 @@ an error. `disallowed_tools` it honors. A task that sets one of them on a harness that marks it unsupported fails at resolution, and `coder-eval plan` exits non-zero. A field that only a lower layer's default sets does not count, and neither does a value of -`null`. Put harness-specific values under `by_type` in the experiment (see +`null`. Values are checked too: a `permission_mode` value the harness does not +honor (for example `acceptEdits` on Pi, OpenCode or Antigravity, which honor only +`plan` and `bypassPermissions`) is rejected, and every `allowed_tools` / +`disallowed_tools` name must be a canonical tool name (`Agent`, `Bash`, `Edit`, +`Glob`, `Grep`, `NotebookEdit`, `Read`, `Skill`, `Task`, `TodoWrite`, `ToolSearch`, +`WebFetch`, `WebSearch`, `Write`). Claude Code alone also accepts `mcp__` / +`mcp____` names and permission rules such as `Bash(git status:*)`. +A task that sets `permission_mode: acceptEdits` for Claude Code is therefore rejected +when run with `--type pi`; move the value under `by_type.claude-code` instead. +Put harness-specific values under `by_type` in the experiment (see [A/B Experiments](AB_EXPERIMENTS.md#per-kind-defaults-with-by_type)). Per-harness table: [Harness Parity](agents/HARNESS_PARITY.md). diff --git a/docs/agents/ANTIGRAVITY.md b/docs/agents/ANTIGRAVITY.md index 7588fed8a..ad0a1e4ae 100644 --- a/docs/agents/ANTIGRAVITY.md +++ b/docs/agents/ANTIGRAVITY.md @@ -144,6 +144,9 @@ The uniform fields become SDK tool-call policies (`google.antigravity.hooks.poli | `allowed_tools` | `deny_all()`, then `allow(tool)` for each mapped tool and for `finish` | | `disallowed_tools` | `deny(tool)` for each mapped tool | | `permission_mode: plan` | `deny` on `create_file`, `edit_file` and `run_command` (read-only) | +| `permission_mode: bypassPermissions` | no extra rule | + +`default` and `acceptEdits` have no Antigravity meaning and are rejected at resolution. Claude tool names map to harness tools by inverting the telemetry map (`Bash` → `run_command`, `Write` → `create_file`, `Edit` → `edit_file`, `Read` → `view_file`, diff --git a/docs/agents/OPENCODE.md b/docs/agents/OPENCODE.md index 17d188f7f..c5610df92 100644 --- a/docs/agents/OPENCODE.md +++ b/docs/agents/OPENCODE.md @@ -89,7 +89,7 @@ agent: type: "opencode" # provider/model, exactly as `opencode models` prints it. model: "openrouter/deepseek/deepseek-v4-pro" - permission_mode: "acceptEdits" + permission_mode: "bypassPermissions" variant: "high" # optional: provider reasoning effort pure: true # optional (default): run with --pure, no host plugins ``` @@ -183,6 +183,9 @@ our keys win): | `allowed_tools` | `"*": "deny"`, `"allow"` for `external_directory` and `doom_loop`, then `"allow"` for each mapped key | | `disallowed_tools` | `"deny"` for each mapped key (a deny always wins) | | `permission_mode: plan` | `edit: "deny"`, `bash: "deny"` (read-only) | +| `permission_mode: bypassPermissions` | no rule; `--auto` approves every permitted tool | + +`default` and `acceptEdits` have no OpenCode meaning and are rejected at resolution. Claude tool names map to permission keys by inverting the telemetry map. OpenCode's keys are coarser than its tools: `edit` governs `write`, `edit`, `patch`, diff --git a/docs/agents/PI.md b/docs/agents/PI.md index 7b35bf089..83c09c87d 100644 --- a/docs/agents/PI.md +++ b/docs/agents/PI.md @@ -78,7 +78,7 @@ agent: type: "pi" # provider-prefixed model id; no separate --provider needed. model: "openrouter/moonshotai/kimi-k3" - permission_mode: "acceptEdits" + permission_mode: "bypassPermissions" thinking_level: "medium" # optional: reasoning effort (see below) ``` @@ -129,8 +129,10 @@ Claude Code. ## Permissions `permission_mode: plan` is read-only: the Write, Edit and Bash equivalents are -denied. Every other mode runs autonomously. Pi headless print mode auto-runs -tools, and Coder Eval always passes `--no-approve` so the run never blocks. +denied. `permission_mode: bypassPermissions` runs every permitted tool without +approval. `default` and `acceptEdits` have no Pi meaning and are rejected at +resolution. Pi headless print mode auto-runs tools, and Coder Eval always passes +`--no-approve` so the run never blocks. ## Multi-turn and simulation diff --git a/src/coder_eval/orchestration/harness_contract.py b/src/coder_eval/orchestration/harness_contract.py index 19f872ea6..3d8e5f85a 100644 --- a/src/coder_eval/orchestration/harness_contract.py +++ b/src/coder_eval/orchestration/harness_contract.py @@ -2,9 +2,12 @@ from __future__ import annotations +import difflib +import re +from collections.abc import Callable from typing import TYPE_CHECKING, Any -from coder_eval.models import Enforcement +from coder_eval.models import CANONICAL_TOOL_NAMES, Enforcement, HarnessContract, PermissionMode, ToolNameMap if TYPE_CHECKING: @@ -20,6 +23,11 @@ class HarnessContractError(TaskResolutionError): """The task's agent config sets a field its harness declares unsupported, or names no registered harness.""" +# `mcp__` (every tool of a server) or `mcp____`. +_MCP_NAME = re.compile(r"mcp__[^_]\S*") +# A Claude Code permission rule: `Bash(git status:*)`, `Read(./src/**)`. +_RULE_SPECIFIER = re.compile(r"(?P[A-Za-z]+)\(.*\)") + # BaseAgentConfig field -> the HarnessContract row that gates it. _GATED: dict[str, str] = { "system_prompt": "system_prompt", @@ -61,31 +69,77 @@ def registration_for(task: TaskDefinition, *, requirement: str, hint: str = "") def validate_harness_contract(task: TaskDefinition) -> None: - """Reject a gated agent field that is set on a harness whose contract marks it unsupported. + """Reject agent config the task's harness cannot honor with its documented meaning. - A field is set when a config layer wrote it and its value is not None. A task - without an agent type returns silently; the layer-5 type guard reports that. + Three checks, in order: a gated field set on a harness whose contract marks it + unsupported; a ``permission_mode`` value outside the contract's + ``permission_modes``; a tool-list name outside ``CANONICAL_TOOL_NAMES`` (or an + ``mcp__`` name the harness cannot address). A field is set when a config layer + wrote it and its value is not None. A task without an agent type returns + silently; the layer-5 type guard reports that. Raises: - HarnessContractError: on the first unsupported field that is set, or an unregistered kind. + HarnessContractError: on the first violation, or an unregistered kind. """ if task.agent is None or task.agent.type is None: return - from coder_eval.agents.registry import AgentRegistry - - contract = registration_for(task, requirement="The harness contract check").agent_class.contract + registration = registration_for(task, requirement="The harness contract check") + contract = registration.agent_class.contract kind = str(task.agent.type) - for field, row in _GATED.items(): - is_set = field in task.agent.model_fields_set and getattr(task.agent, field) is not None - if is_set and getattr(contract, row) is Enforcement.UNSUPPORTED: - honoring = [ - k - for k in AgentRegistry.list_kinds() - if (reg := AgentRegistry.get(k)) is not None - and getattr(reg.agent_class.contract, row) is Enforcement.ENFORCED - ] + set_fields = [ + field for field in _GATED if field in task.agent.model_fields_set and getattr(task.agent, field) is not None + ] + for field in set_fields: + row = _GATED[field] + if getattr(contract, row) is Enforcement.UNSUPPORTED: + honoring = _honoring_kinds(lambda c, row=row: getattr(c, row) is Enforcement.ENFORCED) raise HarnessContractError( f"agent.{field} is set but the {kind!r} harness does not support it " - + "(see docs/agents/HARNESS_PARITY.md). Remove the field, or move it under " - + f"by_type. in the experiment for a harness that honors it ({', '.join(honoring) or 'none'})." + + "(see docs/agents/HARNESS_PARITY.md). Remove the field, or move it under by_type. " + + f"in the experiment for a harness that honors it ({honoring})." ) + if "permission_mode" in set_fields: + _check_permission_value(task.agent.permission_mode, contract.permission_modes or frozenset(), kind) + tool_names = registration.agent_class.tool_names + for field in ("allowed_tools", "disallowed_tools"): + if field in set_fields and tool_names is not None: + _check_tool_names(field, getattr(task.agent, field), tool_names, kind) + + +def _honoring_kinds(honors: Callable[[HarnessContract], bool]) -> str: + from coder_eval.agents.registry import AgentRegistry + + kinds = [ + k for k in AgentRegistry.list_kinds() if (reg := AgentRegistry.get(k)) and honors(reg.agent_class.contract) + ] + return ", ".join(kinds) or "none" + + +def _check_permission_value(value: PermissionMode, honored: frozenset[PermissionMode], kind: str) -> None: + if value not in honored: + raise HarnessContractError( + f"agent.permission_mode={str(value)!r} has no documented meaning on the {kind!r} harness, which " + + f"honors {sorted(str(m) for m in honored)} (see docs/agents/HARNESS_PARITY.md). Use one of those, " + + "or move the value under by_type. for a harness that honors it " + + f"({_honoring_kinds(lambda c: value in (c.permission_modes or frozenset()))})." + ) + + +def _check_tool_names(field: str, names: list[str], tool_names: ToolNameMap, kind: str) -> None: + def accepted(name: str) -> bool: + if tool_names.mcp_names and _MCP_NAME.fullmatch(name): + return True + rule = _RULE_SPECIFIER.fullmatch(name) + # A permission rule reaches only a harness that speaks canonical names natively. + base = rule["tool"] if rule and tool_names.names.get(rule["tool"]) == (rule["tool"],) else name + return base in CANONICAL_TOOL_NAMES + + unknown = sorted(name for name in set(names) if not accepted(name)) + if unknown: + hints = {name: difflib.get_close_matches(name, CANONICAL_TOOL_NAMES, n=1) for name in unknown} + did_you_mean = "; ".join(f"{name!r} -> did you mean {hint[0]!r}?" for name, hint in hints.items() if hint) + raise HarnessContractError( + f"agent.{field} names unknown tool(s) {unknown} for the {kind!r} harness. Accepted names: " + + f"{sorted(CANONICAL_TOOL_NAMES)} (see docs/agents/HARNESS_PARITY.md)." + + (f" {did_you_mean}" if did_you_mean else "") + ) diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 732ebf621..f1a6a3f0d 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -67,6 +67,7 @@ ) from .orchestration.early_stop import EarlyStopWatcher, early_stop_active, validate_early_stop from .orchestration.evaluation import resolve_reference_dir, stage_reference_dir +from .orchestration.harness_contract import validate_harness_contract from .orchestration.run_limits import validate_run_limits from .path_utils import ( TASK_JSON_FILENAME, @@ -1496,6 +1497,10 @@ async def _setup(self) -> None: self._record_route_environment_info() return + # After the evaluate-only return: a re-grade builds no agent, so a recorded + # config from before the contract existed stays gradable. + validate_harness_contract(self.task) + # validate_api_keys exempts the no-op agent internally — it makes no API # call, so it needs no agent keys. assert self.task.agent is not None and self.task.agent.type is not None diff --git a/tests/test_experiment_resolver.py b/tests/test_experiment_resolver.py index 84e102f02..526f19f1f 100644 --- a/tests/test_experiment_resolver.py +++ b/tests/test_experiment_resolver.py @@ -1165,6 +1165,20 @@ def test_unsupported_field_aborts_instead_of_skipping(self, tmp_path): with pytest.raises(HarnessContractError, match=r"agent\.permission_mode.*'codex'"): _resolve_all(tmp_path, task_file) + def test_undeclared_permission_value_aborts(self, tmp_path): + from coder_eval.orchestration.harness_contract import HarnessContractError + + task_file = _write_task(tmp_path, "agent:\n type: pi\n permission_mode: acceptEdits\n") + with pytest.raises(HarnessContractError, match=r"permission_mode='acceptEdits'.*'pi'"): + _resolve_all(tmp_path, task_file) + + def test_unknown_tool_name_aborts(self, tmp_path): + from coder_eval.orchestration.harness_contract import HarnessContractError + + task_file = _write_task(tmp_path, "agent:\n type: opencode\n allowed_tools: [Bassh]\n") + with pytest.raises(HarnessContractError, match="did you mean 'Bash'"): + _resolve_all(tmp_path, task_file) + def test_cli_prompt_file_is_inlined_after_layer_five(self, tmp_path, monkeypatch): (tmp_path / "prompt.md").write_text("be terse\n") monkeypatch.chdir(tmp_path) diff --git a/tests/test_harness_contract.py b/tests/test_harness_contract.py index 37297ef1a..35c3f722a 100644 --- a/tests/test_harness_contract.py +++ b/tests/test_harness_contract.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from collections.abc import Iterator from pathlib import Path from typing import Any, Literal @@ -9,6 +10,7 @@ import pytest from pydantic import BaseModel, ConfigDict, ValidationError +from coder_eval.agents.pi_agent import PiAgent from coder_eval.agents.registry import AgentRegistry from coder_eval.models import ( CANONICAL_TOOL_NAMES, @@ -334,6 +336,71 @@ def test_unregistered_kind_is_rejected(self, restored_registry: None) -> None: validate_harness_contract(task) +_NON_CLAUDE_ENFORCING = [AgentKind.PI, AgentKind.OPENCODE, AgentKind.ANTIGRAVITY] + + +class TestValueChecks: + @pytest.mark.parametrize("kind", _NON_CLAUDE_ENFORCING) + @pytest.mark.parametrize("mode", ["acceptEdits", "default"]) + def test_claude_only_modes_are_rejected(self, kind: AgentKind, mode: str) -> None: + with pytest.raises(HarnessContractError) as exc: + validate_harness_contract(_task(kind, permission_mode=mode)) + message = str(exc.value) + assert f"agent.permission_mode={mode!r}" in message + assert "['bypassPermissions', 'plan']" in message + assert "claude-code" in message.split("honors it", 1)[1] + + @pytest.mark.parametrize("kind", _NON_CLAUDE_ENFORCING) + @pytest.mark.parametrize("mode", ["plan", "bypassPermissions"]) + def test_declared_modes_are_accepted(self, kind: AgentKind, mode: str) -> None: + validate_harness_contract(_task(kind, permission_mode=mode)) + + @pytest.mark.parametrize("mode", list(PermissionMode)) + def test_claude_code_accepts_every_mode(self, mode: PermissionMode) -> None: + validate_harness_contract(_task(AgentKind.CLAUDE_CODE, permission_mode=mode)) + + @pytest.mark.parametrize("field", ["allowed_tools", "disallowed_tools"]) + def test_unknown_tool_name_is_rejected_with_a_suggestion(self, field: str) -> None: + with pytest.raises(HarnessContractError) as exc: + validate_harness_contract(_task(AgentKind.PI, **{field: ["Read", "Bassh"]})) + message = str(exc.value) + assert f"agent.{field} names unknown tool(s) ['Bassh']" in message + assert "did you mean 'Bash'?" in message + + @pytest.mark.parametrize("name", ["mcp__github__create_issue", "mcp__my_server__do_it", "mcp__n8n-mcp"]) + def test_mcp_name_is_accepted_only_where_the_harness_addresses_it(self, name: str) -> None: + validate_harness_contract(_task(AgentKind.CLAUDE_CODE, allowed_tools=[name])) + with pytest.raises(HarnessContractError, match=re.escape(name)): + validate_harness_contract(_task(AgentKind.PI, allowed_tools=[name])) + + def test_malformed_mcp_name_is_rejected(self) -> None: + with pytest.raises(HarnessContractError, match="mcp____x"): + validate_harness_contract(_task(AgentKind.CLAUDE_CODE, allowed_tools=["mcp____x"])) + + @pytest.mark.parametrize("rule", ["Bash(git status:*)", "Read(./src/**)"]) + def test_permission_rule_syntax_is_accepted_only_on_claude_code(self, rule: str) -> None: + validate_harness_contract(_task(AgentKind.CLAUDE_CODE, allowed_tools=[rule])) + with pytest.raises(HarnessContractError, match="unknown tool"): + validate_harness_contract(_task(AgentKind.PI, allowed_tools=[rule])) + + @pytest.mark.parametrize("kind", [AgentKind.OPENCODE, AgentKind.ANTIGRAVITY, AgentKind.CLAUDE_CODE]) + def test_unknown_name_is_rejected_on_every_enforcing_harness(self, kind: AgentKind) -> None: + with pytest.raises(HarnessContractError, match=r"unknown tool\(s\) \['LS'\]"): + validate_harness_contract(_task(kind, disallowed_tools=["LS"])) + + def test_a_name_the_harness_lacks_is_accepted(self) -> None: + assert PiAgent.tool_names is not None and PiAgent.tool_names.names["Skill"] == () + validate_harness_contract(_task(AgentKind.PI, allowed_tools=["Skill"])) + + def test_an_unset_default_mode_is_not_checked(self) -> None: + validate_harness_contract(_task(AgentKind.PI)) + + def test_a_task_level_claude_mode_is_rejected_under_a_cli_kind(self) -> None: + resolved, _ = _resolve(_default_experiment(), _bare_task(permission_mode="acceptEdits"), agent_type="pi") + with pytest.raises(HarnessContractError, match="permission_mode='acceptEdits'"): + validate_harness_contract(resolved) + + def _default_experiment(**by_type: dict[str, Any]) -> ExperimentDefinition: agent: dict[str, Any] = {"type": "claude-code", "plugins": None} if by_type: diff --git a/tests/test_plan_command.py b/tests/test_plan_command.py index dfeb03e93..01aa2836e 100644 --- a/tests/test_plan_command.py +++ b/tests/test_plan_command.py @@ -314,25 +314,42 @@ def test_plan_exits_on_explicit_experiment_load_failure(self, tmp_path: Path) -> class TestPlanCommandHarnessContract: - def test_unsupported_agent_field_flips_the_exit_code(self, tmp_path: Path) -> None: + @staticmethod + def _plan(tmp_path: Path, agent_yaml: str) -> tuple[int, str]: task_file = tmp_path / "task.yaml" task_file.write_text( "task_id: contract-task\ndescription: d\ninitial_prompt: do it\n" - + "agent:\n type: codex\n permission_mode: acceptEdits\n" + + agent_yaml + "sandbox:\n driver: tempdir\n" + "success_criteria:\n - type: file_exists\n path: out.txt\n description: c\n" ) exp_file = tmp_path / "experiment.yaml" exp_file.write_text("experiment_id: contract\nvariants:\n - variant_id: default\n") + exit_code = 0 with ( patch("coder_eval.cli.plan_command.check_tools"), patch("coder_eval.cli.plan_command.check_api_keys"), patch(f"{_EXP}.DEFAULT_EXPERIMENT_PATH", tmp_path / "missing.yaml"), patch("coder_eval.cli.plan_command.console") as mock_console, - pytest.raises(typer.Exit) as exc, ): - run_plan(task_files=[task_file], experiment=exp_file) - printed = " ".join(str(call) for call in mock_console.print.call_args_list) - assert exc.value.exit_code == 1 + try: + run_plan(task_files=[task_file], experiment=exp_file) + except typer.Exit as exc: + exit_code = exc.exit_code + return exit_code, " ".join(str(call) for call in mock_console.print.call_args_list) + + def test_unsupported_agent_field_flips_the_exit_code(self, tmp_path: Path) -> None: + exit_code, printed = self._plan(tmp_path, "agent:\n type: codex\n permission_mode: acceptEdits\n") + assert exit_code == 1 assert "config error" in printed assert "agent.permission_mode" in printed + + def test_undeclared_permission_value_flips_the_exit_code(self, tmp_path: Path) -> None: + exit_code, printed = self._plan(tmp_path, "agent:\n type: pi\n permission_mode: acceptEdits\n") + assert exit_code == 1 + assert "has no documented meaning" in printed + + def test_declared_permission_value_is_valid(self, tmp_path: Path) -> None: + exit_code, printed = self._plan(tmp_path, "agent:\n type: pi\n permission_mode: plan\n") + assert exit_code == 0 + assert "All tasks are valid!" in printed From 8af5f4e9260c42f60502ffc2237a8cfbd1cf6e27 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 10:29:09 -0700 Subject: [PATCH 07/12] =?UTF-8?q?feat(lint):=205/6=20=E2=80=94=20CE068=20k?= =?UTF-8?q?eeps=20kind=20names=20out=20of=20the=20kernel;=20add=20the=20co?= =?UTF-8?q?der=5Feval.spi=20surface?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CE068 flags a concrete agent config class or an AgentKind member (other than UNKNOWN) named in orchestration/, streaming/ or timing.py. coder_eval.spi re-exports the stable plugin surface with SPI_VERSION = 1, and EXTENDING.md documents the harness contract a plugin agent must declare. Co-Authored-By: Claude Opus 5 (1M context) --- docs/EXTENDING.md | 65 +++++++++++++-- pyproject.toml | 1 + src/coder_eval/spi.py | 82 +++++++++++++++++++ tests/fixtures/byoa_demo_plugin/byoa_demo.py | 2 + tests/lint/rules/_layers.py | 8 +- .../rules/ce068_no_kind_names_in_kernel.py | 72 ++++++++++++++++ tests/lint/runner.py | 5 +- tests/test_custom_lint.py | 52 ++++++++++++ tests/test_spi.py | 36 ++++++++ 9 files changed, 314 insertions(+), 9 deletions(-) create mode 100644 src/coder_eval/spi.py create mode 100644 tests/lint/rules/ce068_no_kind_names_in_kernel.py create mode 100644 tests/test_spi.py diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 0aeacd682..7d4cce47d 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -40,10 +40,15 @@ and skipped; only a failing *built-in* registration is fatal. ### The `register` hook +Import everything from `coder_eval.spi`, the stable plugin surface, and check its +version in the hook. `SPI_VERSION` changes whenever an exported name changes its +signature. + ```python -from coder_eval.agents.registry import AgentRegistry +from coder_eval.spi import SPI_VERSION, AgentRegistry def register(registry: type[AgentRegistry]) -> None: + assert SPI_VERSION == 1, f"my-agent supports coder_eval SPI 1, not {SPI_VERSION}" # Bind type string → config class → agent class. registry.register("my-agent", MyAgentConfig)(MyAgent) # Optionally contribute pricing here too (see §3): @@ -59,6 +64,11 @@ class MyAgent(Agent[MyAgentConfig]): ... ``` +Registration validates the pair and raises `TypeError` if the agent class declares no +`contract`, if its `tool_names` presence does not match its contract, or if the config +class is not a `BaseAgentConfig` with `extra="forbid"` whose `type` Literal names the +kind. + Registration is **anti-shadow**: re-registering the same `(agent_class, config_class)` pair is a no-op, but claiming an existing `agent.type` with a *different* implementation raises `ValueError`. Two plugins can never silently fight @@ -70,7 +80,7 @@ Subclass `BaseAgentConfig` with your own `type` discriminator: ```python from typing import Literal -from coder_eval.models import BaseAgentConfig # importable from coder_eval.models +from coder_eval.spi import BaseAgentConfig class MyAgentConfig(BaseAgentConfig): type: Literal["my-agent"] = "my-agent" @@ -80,8 +90,54 @@ class MyAgentConfig(BaseAgentConfig): The factory `create_agent(kind, config, …)` raises `TypeError` if the passed config isn't an instance of the registered `config_class`, so keep them paired. +### The harness contract + +Every agent class declares which uniform `BaseAgentConfig` fields reach its harness. +A task that sets a field the contract marks `UNSUPPORTED`, a `permission_mode` value +outside `permission_modes`, or a tool name outside `CANONICAL_TOOL_NAMES` is rejected +at resolution, so `coder-eval plan` fails before any run. This is a JSONL CLI agent +that appends a system prompt and honors `plan` and tool lists natively: + +```python +from coder_eval.spi import Agent, Enforcement, HarnessContract, PermissionMode, ToolNameMap + +# native tool name -> canonical (Claude) name; also used for telemetry +_TOOL_NAME_MAP = {"bash": "Bash", "read": "Read", "write": "Write", "edit": "Edit", "task": "Agent"} + +class MyAgent(Agent[MyAgentConfig]): + contract = HarnessContract( + system_prompt=Enforcement.ENFORCED, + system_prompt_semantics="append", + plugin_skills=Enforcement.UNSUPPORTED, + permission_mode=Enforcement.ENFORCED, + permission_modes=frozenset({PermissionMode.PLAN, PermissionMode.BYPASS_PERMISSIONS}), + allowed_tools=Enforcement.ENFORCED, + disallowed_tools=Enforcement.ENFORCED, + cooperative_stop=True, + ) + tool_names = ToolNameMap.from_inverse( + _TOOL_NAME_MAP, + no_equivalent=frozenset({"Glob", "Grep", "NotebookEdit", "Skill", "TodoWrite", "ToolSearch", "WebFetch", "WebSearch"}), + ) +``` + +- `system_prompt_semantics` `append` / `replace` mean the text reaches the model through + the system or developer instruction channel. A harness that can only prefix the user + turn declares `system_prompt=UNSUPPORTED`. +- Declare a `permission_modes` value only if the harness gives it the Claude Code + meaning (`plan` is read-only, `bypassPermissions` runs every permitted tool). +- `tool_names` is required exactly when a tool-list row is `ENFORCED`. It must map every + canonical name; list a name your harness has no tool for in `no_equivalent`. +- Set `cooperative_stop=True` only if your `communicate()` honors `should_stop` + (needed for criterion-level `stop_early:` arming). `False` means early stop is + rejected at resolution for your agent. + ### The `Agent` ABC — implementation checklist +Call `super().__init__(config, route, cost_log_tags=cost_log_tags)` first in your +`__init__`, and declare `cost_log_tags` as a keyword-only parameter: the factory passes +it on every LiteLLM route. + Implement these three abstract methods: - [ ] `async def start(self, working_directory, *, env_path_prepend=None, plugin_tools_dir=None) -> None` @@ -108,11 +164,6 @@ turn, and `ToolStart`/`ToolEnd` per tool call (close orphaned tools with `status=unresolved`). Fan events through an internal `EventCollector` — it builds the returned `TurnRecord`, the single agent-agnostic capture path. -Set `cooperative_stop=True` in your agent's `contract` only if your `communicate()` -actually honors `should_stop` (needed for criterion-level `stop_early:` arming). Setting it -`False` means early stop is rejected at resolution for your agent — which is correct -if you can't stop cooperatively. - ### Worked example The in-tree worked example is the BYOA test fixture at diff --git a/pyproject.toml b/pyproject.toml index cafde0ba8..6d17f1848 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -302,6 +302,7 @@ external = [ "CE064", "CE065", "CE066", + "CE068", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/src/coder_eval/spi.py b/src/coder_eval/spi.py new file mode 100644 index 000000000..3163bc42a --- /dev/null +++ b/src/coder_eval/spi.py @@ -0,0 +1,82 @@ +"""The stable import surface for plugin agents. + +A plugin imports only from this module and checks ``SPI_VERSION`` in its +``register(registry)`` hook. Any signature change to a name exported here bumps +``SPI_VERSION``; adding a name does not. +""" + +from typing import Final + +from coder_eval.agent import Agent +from coder_eval.agents.registry import AgentRegistry +from coder_eval.errors import AgentConfigError, AgentCrashError, TurnTimeoutError +from coder_eval.models import ( + CANONICAL_TOOL_NAMES, + READ_ONLY_DENIED_TOOLS, + AgentState, + ApiRoute, + BaseAgentConfig, + Enforcement, + HarnessContract, + LocalPluginConfig, + PermissionMode, + SystemPromptMode, + ToolNameMap, + TurnRecord, +) +from coder_eval.pricing import register_pricing +from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback +from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentEndStatus, + AgentStartEvent, + TextChunkEvent, + ToolEndEvent, + ToolEndStatus, + ToolStartEvent, + TurnEndEvent, + TurnEndStatus, + TurnStartEvent, +) +from coder_eval.timing import TurnClock, close_window + + +SPI_VERSION: Final[int] = 1 + +__all__ = [ # noqa: RUF022 - plain sort, pinned by tests/test_spi.py + "Agent", + "AgentConfigError", + "AgentCrashError", + "AgentEndEvent", + "AgentEndStatus", + "AgentRegistry", + "AgentStartEvent", + "AgentState", + "ApiRoute", + "BaseAgentConfig", + "CANONICAL_TOOL_NAMES", + "CompositeStreamCallback", + "Enforcement", + "EventCollector", + "HarnessContract", + "LocalPluginConfig", + "PermissionMode", + "READ_ONLY_DENIED_TOOLS", + "SPI_VERSION", + "StreamCallback", + "SystemPromptMode", + "TextChunkEvent", + "ToolEndEvent", + "ToolEndStatus", + "ToolNameMap", + "ToolStartEvent", + "TurnClock", + "TurnEndEvent", + "TurnEndStatus", + "TurnRecord", + "TurnStartEvent", + "TurnTimeoutError", + "close_window", + "register_pricing", +] diff --git a/tests/fixtures/byoa_demo_plugin/byoa_demo.py b/tests/fixtures/byoa_demo_plugin/byoa_demo.py index f9754a222..d62532f50 100644 --- a/tests/fixtures/byoa_demo_plugin/byoa_demo.py +++ b/tests/fixtures/byoa_demo_plugin/byoa_demo.py @@ -22,6 +22,7 @@ from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.agents.registry import AgentRegistry from coder_eval.models import ClaudeCodeAgentConfig +from coder_eval.spi import SPI_VERSION DEMO_KIND = "byoa-demo" @@ -43,4 +44,5 @@ def register(registry: type[AgentRegistry]) -> None: ``registry`` is the ``AgentRegistry`` class (not an instance). """ + assert SPI_VERSION == 1, f"byoa_demo supports coder_eval SPI 1, not {SPI_VERSION}" registry.register(DEMO_KIND, DemoAgentConfig)(DemoAgent) diff --git a/tests/lint/rules/_layers.py b/tests/lint/rules/_layers.py index 034336775..0e9288b65 100644 --- a/tests/lint/rules/_layers.py +++ b/tests/lint/rules/_layers.py @@ -1,4 +1,4 @@ -"""The package-layer predicates, declared once and shared by CE004 and CE066. +"""The package-layer predicates, declared once and shared by CE004, CE066 and CE068. Both rules ask where a file sits in ``src/coder_eval/``, and a second copy of the answer is how a package added to one regex silently escapes the other. So @@ -53,6 +53,7 @@ _PKG = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]") _CLI = re.compile(_PKG.pattern + r"cli[/\\]") _REPORTS = re.compile(_PKG.pattern + r"reports[/\\]") +_KERNEL = re.compile(_PKG.pattern + r"(?:orchestration[/\\]|streaming[/\\]|timing\.py$)") def is_package_path(filepath: str) -> bool: @@ -70,6 +71,11 @@ def is_core_path(filepath: str) -> bool: return is_package_path(filepath) and not is_cli_path(filepath) and not _REPORTS.search(filepath) +def is_kernel_path(filepath: str) -> bool: + """Whether ``filepath`` is in CE068's agent-agnostic kernel: ``orchestration/``, ``streaming/``, ``timing.py``.""" + return bool(_KERNEL.search(filepath)) + + def _containing_package(filepath: str) -> list[str] | None: """The dotted parts of the package a module lives in, rooted at ``coder_eval``. diff --git a/tests/lint/rules/ce068_no_kind_names_in_kernel.py b/tests/lint/rules/ce068_no_kind_names_in_kernel.py new file mode 100644 index 000000000..b0ec039f0 --- /dev/null +++ b/tests/lint/rules/ce068_no_kind_names_in_kernel.py @@ -0,0 +1,72 @@ +"""CE068: the kernel names no concrete agent kind. + +``orchestration/``, ``streaming/`` and ``timing.py`` are the kernel every agent, +in-tree or plugin, runs through. A branch keyed on one kind there is a behaviour a +plugin kind cannot get. The motivating defect: ``orchestration/overrides.py`` +accepted ``-D agent.sdk_options.*`` only when the type was ``AgentKind.CLAUDE_CODE``, +so the out-of-tree Delegate agent, whose config also declares ``sdk_options``, +needed a monkeypatch (``coder_eval_uipath/_overrides_patch.py``). The guard now +asks the registry whether the config class declares the field. + +Fires on, inside the kernel: + + * a ``from ... import`` of a concrete agent config class. The set is derived + from the ``AgentConfig`` union at rule-import time, never listed, so a new + in-tree kind is covered on arrival; + * an ``AgentKind.`` attribute read, except ``AgentKind.UNKNOWN`` (the + batch sentinel for a task that failed to load, which names no harness). + +Blind spots: a kind reached through ``getattr`` or a string literal +(``type == "codex"``) is invisible here, and ``AgentKind`` used as a bare name +(``isinstance(x, AgentKind)``) is allowed. The string-dispatch half is +``no_type_name_string_dispatch``'s job. +""" + +import ast +import typing + +from coder_eval.models import AgentConfig +from tests.lint.rules._layers import is_kernel_path +from tests.lint.rules.base import BaseRule + + +def _config_class_names() -> frozenset[str]: + union = typing.get_args(AgentConfig.__value__)[0] + return frozenset(cls.__name__ for cls in typing.get_args(union)) + + +CONFIG_CLASS_NAMES = _config_class_names() +_ALLOWED_MEMBERS = frozenset({"UNKNOWN"}) +_FIX = "ask the agent registry (a config class's fields or an agent's HarnessContract) instead" + + +class NoKindNamesInKernel(BaseRule): + id = "CE068" + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_kernel = is_kernel_path(filepath) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + if self._in_kernel: + for alias in node.names: + if alias.name in CONFIG_CLASS_NAMES: + self.violation( + node, + f"architectural violation: concrete agent config '{alias.name}' imported into the " + f"agent-agnostic kernel — {_FIX}", + ) + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute) -> None: + if ( + self._in_kernel + and isinstance(node.value, ast.Name) + and node.value.id == "AgentKind" + and node.attr not in _ALLOWED_MEMBERS + ): + self.violation( + node, + f"architectural violation: 'AgentKind.{node.attr}' named in the agent-agnostic kernel — {_FIX}", + ) + self.generic_visit(node) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index 81ddadc6d..3e3b07488 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -42,6 +42,7 @@ from tests.lint.rules.ce063_no_busy_ms_in_agents import NoBusyMsInAgents from tests.lint.rules.ce064_turn_bracket_on_the_clock import TurnBracketOnTheClock from tests.lint.rules.ce066_no_report_imports_in_core import NoReportImportsInCore +from tests.lint.rules.ce068_no_kind_names_in_kernel import NoKindNamesInKernel from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -64,7 +65,8 @@ # comment carrying 062 in an older branch, review or commit message must never # start meaning something new. # -# Claim 068 next. NOTE 065 IS TAKEN and is not in ALL_RULES: doc-surface and +# Claim 070 next (069 is TestCE069HarnessParityTable). NOTE 065 IS TAKEN and is +# not in ALL_RULES: doc-surface and # whole-tree rules are `@pytest.mark.lint` classes in tests/test_custom_lint.py # rather than BaseRules, so the `_rule_ids` uniqueness assert below cannot see # them. Enumerating them here is how this note fell behind CE044, so grep @@ -121,6 +123,7 @@ NoBusyMsInAgents, TurnBracketOnTheClock, NoReportImportsInCore, + NoKindNamesInKernel, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 7f6338d96..236bf17af 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -2162,6 +2162,58 @@ def test_the_reports_package_is_in_scope(self): assert self._violations("from ..cli import run_command", "/repo/src/coder_eval/reports/markdown.py") +@pytest.mark.lint +class TestCE068NoKindNamesInKernel: + """CE068 — orchestration/, streaming/ and timing.py name no concrete agent kind.""" + + @staticmethod + def _violations(source: str, filepath: str) -> list: + import ast + + from tests.lint.rules.ce068_no_kind_names_in_kernel import NoKindNamesInKernel + + return list(NoKindNamesInKernel(filepath).check(ast.parse(source))) + + ORCHESTRATION = "/repo/src/coder_eval/orchestration/x.py" + STREAMING = "/repo/src/coder_eval/streaming/x.py" + TIMING = "/repo/src/coder_eval/timing.py" + + def test_a_concrete_config_import_in_orchestration_violates(self): + found = self._violations("from coder_eval.models import ClaudeCodeAgentConfig", self.ORCHESTRATION) + assert len(found) == 1 + assert "registry" in found[0].message + + def test_the_relative_spelling_violates_too(self): + assert self._violations("from ..models import CodexAgentConfig", self.ORCHESTRATION) + + @pytest.mark.parametrize("path", [STREAMING, TIMING]) + def test_a_kind_member_read_violates(self, path: str): + assert self._violations("x = AgentKind.CLAUDE_CODE", path) + + def test_the_unknown_sentinel_is_allowed(self): + assert not self._violations("x = AgentKind.UNKNOWN", self.ORCHESTRATION) + + def test_the_base_config_and_the_union_alias_are_allowed(self): + assert not self._violations("from coder_eval.models import AgentConfig, BaseAgentConfig", self.ORCHESTRATION) + + def test_bare_agent_kind_use_is_allowed(self): + assert not self._violations("ok = isinstance(x, AgentKind)", self.ORCHESTRATION) + + @pytest.mark.parametrize("path", ["/repo/src/coder_eval/agents/x.py", "/repo/src/coder_eval/cli/x.py"]) + def test_outside_the_kernel_is_allowed(self, path: str): + assert not self._violations("from coder_eval.models import ClaudeCodeAgentConfig\nx = AgentKind.PI", path) + + def test_the_config_class_set_is_derived_from_the_union(self): + import typing + + from coder_eval.models import AgentConfig + from tests.lint.rules.ce068_no_kind_names_in_kernel import CONFIG_CLASS_NAMES + + members = typing.get_args(typing.get_args(AgentConfig.__value__)[0]) + assert {cls.__name__ for cls in members} == CONFIG_CLASS_NAMES + assert "PiAgentConfig" in CONFIG_CLASS_NAMES + + @pytest.mark.lint class TestCE066NoReportImportsInCore: """CE066 — core may import only the reports package's public writers. diff --git a/tests/test_spi.py b/tests/test_spi.py new file mode 100644 index 000000000..0dc6b0346 --- /dev/null +++ b/tests/test_spi.py @@ -0,0 +1,36 @@ +"""``coder_eval.spi``: the plugin import surface re-exports, unchanged, the objects it names.""" + +import importlib + +import coder_eval.spi as spi + + +_ORIGINS = ( + "coder_eval.agent", + "coder_eval.agents.registry", + "coder_eval.errors", + "coder_eval.models", + "coder_eval.pricing", + "coder_eval.streaming.callbacks", + "coder_eval.streaming.collector", + "coder_eval.streaming.events", + "coder_eval.timing", +) + + +def test_spi_version_is_one() -> None: + assert spi.SPI_VERSION == 1 + + +def test_all_is_sorted_and_unique() -> None: + assert spi.__all__ == sorted(spi.__all__) + assert len(set(spi.__all__)) == len(spi.__all__) + + +def test_every_export_is_the_origin_object() -> None: + origins = [importlib.import_module(name) for name in _ORIGINS] + for name in spi.__all__: + if name == "SPI_VERSION": + continue + exported = getattr(spi, name) + assert any(getattr(module, name, None) is exported for module in origins), name From ca15864c7d156ab92c0917262674bbc2ec092081 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 10:36:43 -0700 Subject: [PATCH 08/12] =?UTF-8?q?docs(agents):=206/6=20=E2=80=94=20generat?= =?UTF-8?q?e=20the=20agent-field=20contract=20tables=20and=20add=20the=20h?= =?UTF-8?q?arness=20conformance=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit make parity-table renders the HarnessContract and ToolNameMap of every in-tree agent into docs/agents/HARNESS_PARITY.md, and CE069 fails on drift. The conformance test derives every rejection from the contracts and requires one offline probe per enforced cell and declared permission mode, including a check that a system prompt never rides the user turn. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 6 + CLAUDE.md | 16 +- Makefile | 5 +- README.md | 2 +- docs/agents/ANTIGRAVITY.md | 2 + docs/agents/CLAUDE_CODE.md | 3 +- docs/agents/CODEX.md | 5 +- docs/agents/HARNESS_PARITY.md | 59 ++++- docs/agents/OPENCODE.md | 2 + docs/agents/PI.md | 2 + docs/index.md | 2 +- docs/llms.txt | 2 +- mkdocs.yml | 2 +- pyproject.toml | 1 + tests/lint/harness_parity.py | 101 ++++++++ tests/test_custom_lint.py | 31 +++ tests/test_harness_conformance.py | 416 ++++++++++++++++++++++++++++++ 17 files changed, 637 insertions(+), 20 deletions(-) create mode 100644 tests/lint/harness_parity.py create mode 100644 tests/test_harness_conformance.py diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index f9c485d1d..40ddbc9cd 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -82,6 +82,12 @@ intentionally brief and out of scope; trimming for DISPLAY belongs in the render table's to state, not this file's. Full table + rationale: docs/agents/HARNESS_PARITY.md. + The agent-field half of parity is now the `HarnessContract` each agent class declares: + a field, `permission_mode` value or tool name a harness cannot honor is a resolution + error, and `make parity-table` renders the contract (CE069 checks it), so the page can no + longer drift from the adapters. The run-limit half is still the hand-written table above; + Plan 2 moves it onto the contract. + ## Shared turn lifecycle Every adapter drives the same skeleton, on the base class: `_begin_turn()` resets the diff --git a/CLAUDE.md b/CLAUDE.md index b92f39b64..2ad1f2f0a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,8 +86,10 @@ Each entry is a pointer. Full rationale: `.claude/notes/` (index: `.claude/notes Defense-in-depth, not a boundary — the known gaps are documented in the notes. Authoring reference: [Reference Solutions](docs/TASK_DEFINITION_GUIDE.md#reference-solutions). - **Harness run-limit parity**: a shared config field must mean the same thing on every - backend, or the divergence is documented. Table: - [Run-Limit Parity](docs/agents/HARNESS_PARITY.md). Caps are authored under + backend, or the divergence is documented. Every agent declares a `HarnessContract`; a + base field the harness marks unsupported is rejected at resolution. Table: + [Run-Limit Parity](docs/agents/HARNESS_PARITY.md) § Agent-field contract + (generated). Caps are authored under [Run Limits](docs/TASK_DEFINITION_GUIDE.md#run-limits). - **Execute vs. run**: `execute` is `run` with grading off — rows finalize as `NOT_GRADED` and leave both sides of every rate. Per-command behaviour: @@ -162,6 +164,7 @@ make evalboard-verify # the JS half: tsc --noEmit + vitest + next build make docs-indexes # README/docs index tables from the mkdocs nav (CE028) make plugin-reference # the plugin's criteria reference from the models (CE033) make pricing-mirror # the evalboard's rate table from pricing.py (CE065) +make parity-table # the agent-field contract tables from the agent classes (CE069) make docs-budget # per-file comment budget + docstring essay check (fails `make verify`) ``` @@ -215,6 +218,10 @@ A few rules constrain routine edits, so they are worth knowing before you start: metric, statistic or serializer pulled out of `reports*` is what put `turn_time_buckets` and the run.json serializer in a rendering module; they now live in `result_metrics.py`, `stats.py` and `run_record.py`. +- **CE068** keeps `orchestration/`, `streaming/` and `timing.py` free of concrete agent + config classes and `AgentKind` members (except `UNKNOWN`); ask the registry instead. +- **CE069** diffs the generated contract tables in `docs/agents/HARNESS_PARITY.md` against + the agent classes. Regenerate with `make parity-table`. **Docs index SSOT.** `nav:` plus `extra.docs_index` in `mkdocs.yml` are the single source of truth for `README.md`'s Documentation table, `docs/index.md`'s "Where to go @@ -252,8 +259,9 @@ A live criterion also needs `ContractCase`s (CE036) and `make plugin-reference`. **A new agent**: agents register through the plugin SPI (entry-point group `coder_eval.plugins`) — there is no closed enum or dispatch to edit, and in-tree and -third-party agents take the same path. A new agent must be named on every onboarding -surface CE047 tracks, and its run-limit behaviour recorded in +third-party agents take the same path. It declares a `HarnessContract` (registration +fails without one) and imports from `coder_eval.spi`. A new agent must be named on every +onboarding surface CE047 tracks, and its run-limit behaviour recorded in [Run-Limit Parity](docs/agents/HARNESS_PARITY.md). **Model pricing**: `register_pricing(YOUR_RATES)` from the same `register(registry)` diff --git a/Makefile b/Makefile index 44b2d3f44..4f619730b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install format check typecheck test test-live test-smoke verify verify-noextra evalboard-verify clean run lint docs-indexes plugin-reference pricing-mirror docs-budget docker-image docker-image-full coder-eval-runtime docker-images +.PHONY: help install format check typecheck test test-live test-smoke verify verify-noextra evalboard-verify clean run lint docs-indexes plugin-reference pricing-mirror parity-table docs-budget docker-image docker-image-full coder-eval-runtime docker-images # Single source of the installed coder-eval version (used to tag the docker # images). Referenced lazily inside the docker recipes, so it doesn't run on @@ -39,6 +39,9 @@ plugin-reference: ## Regenerate the plugin's bundled criteria reference from th pricing-mirror: ## Regenerate the evalboard's rate table from pricing.py (SSOT) uv run python -m tests.lint.pricing_mirror +parity-table: ## Regenerate the agent-field contract table from the agent classes (SSOT) + uv run python -m tests.lint.harness_parity + docs-budget: ## Report the docstring/comment prose budget and check it against the baseline uv run python -m tests.lint.prose_budget diff --git a/README.md b/README.md index e0b78755a..fc350c394 100644 --- a/README.md +++ b/README.md @@ -253,7 +253,7 @@ The step's exit code is coder-eval's own: non-zero on any failed task. | [Antigravity (Gemini)](docs/agents/ANTIGRAVITY.md) | Running the Google Antigravity / Gemini agent | | [OpenCode](docs/agents/OPENCODE.md) | Running the OpenCode agent on open-weight models | | [Pi](docs/agents/PI.md) | Running the Pi agent on open-weight models | -| [Run-Limit Parity](docs/agents/HARNESS_PARITY.md) | What each run_limits field means on every harness | +| [Run-Limit Parity](docs/agents/HARNESS_PARITY.md) | What each run_limits and agent field means on every harness | | [A/B Experiments](docs/AB_EXPERIMENTS.md) | Compare models / tools / prompts across the same tasks | | [Bring Your Own Dataset](docs/DATASETS.md) | Fan a single task out over a dataset | | [Dialog Mode](docs/DIALOG_MODE.md) | Evaluate agents in multi-turn conversation via a simulated user | diff --git a/docs/agents/ANTIGRAVITY.md b/docs/agents/ANTIGRAVITY.md index ad0a1e4ae..b54543d03 100644 --- a/docs/agents/ANTIGRAVITY.md +++ b/docs/agents/ANTIGRAVITY.md @@ -136,6 +136,8 @@ can't be resolved or if zero skills are discovered. ## Permissions & tools — important differences +Per-field contract and tool names (generated): [Harness Parity § Agent-field contract](HARNESS_PARITY.md#agent-field-contract). + The uniform fields become SDK tool-call policies (`google.antigravity.hooks.policy`): | Field | Policies | diff --git a/docs/agents/CLAUDE_CODE.md b/docs/agents/CLAUDE_CODE.md index 1c95197ac..12356ee90 100644 --- a/docs/agents/CLAUDE_CODE.md +++ b/docs/agents/CLAUDE_CODE.md @@ -70,7 +70,8 @@ auto-qualified to a regional inference profile (`eu.` / `us.` / `apac.` / `globa ## Agent config surface All fields live under `agent:` in a task (or an experiment variant). Only `type` is -required; everything else has a default. +required; everything else has a default. What each uniform field means on every +harness (generated): [Harness Parity § Agent-field contract](HARNESS_PARITY.md#agent-field-contract). ```yaml agent: diff --git a/docs/agents/CODEX.md b/docs/agents/CODEX.md index a9ad99a76..35f2f184d 100644 --- a/docs/agents/CODEX.md +++ b/docs/agents/CODEX.md @@ -170,6 +170,9 @@ On failure, the agent: ### Permission and Tool Mapping +Per-field contract (generated): [Harness Parity § Agent-field contract](HARNESS_PARITY.md#agent-field-contract). `permission_mode`, `allowed_tools` and +`disallowed_tools` are rejected at load on Codex. + Codex runs with `sandbox: full-access` and `approval_mode: deny_all` on every run. Its own OS sandbox fails silently on the hosts Coder Eval runs on, so the isolation boundary is the task's driver: use `driver: docker` for untrusted evals. `deny_all` means *run autonomously, never prompt, no server-side reviewer*. Coder Eval uses it because the alternative (`auto_review`) adds a server-side reviewer that can spuriously return `declined` under gateway load. @@ -209,7 +212,7 @@ Run-limit semantics per harness: [Run-Limit Parity](HARNESS_PARITY.md). 1. **Tool-name collapse** - Codex reports shell tools (`Read`/`Grep`/`Bash`) all as shell commands, surfaced as `Bash` telemetry; name-keyed criteria that distinguish these tools aren't meaningful across agents. 2. **`skill_triggered` criterion** - Codex has no distinct `Skill` tool (it engages a skill by reading its files via shell), so the criterion detects Codex engagement from that file-read signal (a command referencing `skills//`) instead of a `Skill` tool call. The file-read signal is weaker than Claude's explicit invocation. -3. **`disallowed_tools`** - passed to the SDK but not enforced; not a security boundary. +3. **`permission_mode`, `allowed_tools`, `disallowed_tools`** - Codex honors none of them; a Codex task that sets any of them is rejected at load. 4. **Authentication** - Requires `CODEX_API_KEY` in the environment (point it at whichever endpoint's key you use — OpenAI, gateway, or Azure); the agent calls `login_api_key` when a key is present. `OPENAI_API_KEY`/`AZURE_OPENAI_API_KEY` are NOT read. 5. **Model field** - `TurnRecord.model_used` reflects the pinned `agent.model`; the Codex `Turn` payload itself doesn't carry the resolved model. 6. **Skills with Windows paths** - Symlink creation may fail on Windows; agent falls back to copying (slower). diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 7d3d88235..e641c7200 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -5,8 +5,8 @@ was the field that broke that promise hardest: Claude Code enforced it, and Code Antigravity accepted it and never read it, so `max_turns: 6` ran capped on one backend and unbounded on the other two. -This page is the contract for what each run limit means per harness, plus the shared -`agent` fields whose meaning still differs across them. +This page is the contract for what each run limit means per harness, plus what each +shared `agent` field means on each harness. ## The table @@ -17,6 +17,54 @@ This page is the contract for what each run limit means per harness, plus the sh | `run_limits.task_timeout` | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | | `run_limits.stop_early` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` (event granularity) | cooperative `should_stop` (event granularity — Pi streams incrementally) | +## Agent-field contract + +Generated from each agent class's `contract` by `make parity-table`; CE069 fails the build on drift. + + +| field | claude-code | codex | antigravity | opencode | pi | none | +| --- | --- | --- | --- | --- | --- | --- | +| `system_prompt` | enforced | enforced | enforced | enforced | enforced | unsupported | +| `system_prompt_semantics` | append | append | append | append | append | — | +| `plugin_skills` | enforced | enforced | enforced | enforced | enforced | unsupported | +| `permission_mode` | enforced | unsupported | enforced | enforced | enforced | unsupported | +| `allowed_tools` | enforced | unsupported | enforced | enforced | enforced | unsupported | +| `disallowed_tools` | enforced | unsupported | enforced | enforced | enforced | unsupported | +| `cooperative_stop` | yes | yes | yes | yes | yes | no | +| `permission_modes` | acceptEdits, bypassPermissions, default, plan | — | bypassPermissions, plan | bypassPermissions, plan | bypassPermissions, plan | — | + + +A task that sets a field a harness marks `unsupported`, or a `permission_mode` value +outside that harness's `permission_modes`, is rejected at resolution and `coder-eval plan` +exits non-zero. `system_prompt_semantics` `append` / `replace` mean the system or +developer instruction channel of the model request, never the user turn. + +### Tool names + +Every `allowed_tools` / `disallowed_tools` name must be one of these canonical names. +Each cell is the native tool the name restricts on that harness; `none` means the +harness has no such tool, so the name permits or denies nothing there. Generated from +each agent class's `tool_names` by `make parity-table`. + + +| tool | claude-code | antigravity | opencode | pi | +| --- | --- | --- | --- | --- | +| `Agent` | `Agent` | `start_subagent` | `task` | `task` | +| `Bash` | `Bash` | `run_command` | `bash` | `bash` | +| `Edit` | `Edit` | `edit_file` | `edit`, `multiedit`, `patch` | `edit`, `multiedit`, `patch` | +| `Glob` | `Glob` | `find_file` | `glob` | `find` | +| `Grep` | `Grep` | `search_directory` | `grep` | `grep` | +| `NotebookEdit` | `NotebookEdit` | none | none | none | +| `Read` | `Read` | `view_file` | `read` | `read` | +| `Skill` | `Skill` | none | `skill` | none | +| `Task` | `Task` | `start_subagent` | `task` | `task` | +| `TodoWrite` | `TodoWrite` | none | `todowrite` | `todowrite` | +| `ToolSearch` | `ToolSearch` | none | none | none | +| `WebFetch` | `WebFetch` | `read_url_content` | `webfetch` | `webfetch` | +| `WebSearch` | `WebSearch` | `search_web` | `websearch` | none | +| `Write` | `Write` | `create_file` | `apply_patch`, `write` | `write` | + + ## Timing capture What each harness records about *when* things happened, and how much of a task's @@ -704,13 +752,6 @@ both via the same `_plugin_skill_dirs` resolver — so both **can** run activati suites. A plugin's non-skill assets (agents/hooks/commands/MCP servers) are dropped on both. See [OpenCode](OPENCODE.md) and [Pi § plugins](PI.md#known-limitations). -## Agent fields per harness - -Claude Code, Pi, OpenCode and Antigravity honor `system_prompt`, `permission_mode`, -`allowed_tools` and `disallowed_tools` natively. Codex honors `system_prompt` only. See -each harness page for the mechanism: [Pi](PI.md#config-fields-and-their-pi-flags), -[OpenCode](OPENCODE.md#permissions), [Antigravity](ANTIGRAVITY.md), [Codex](CODEX.md). - ## Reproducing `tasks/run_limits/` holds one fixture per limit: `max_turns_cap.yaml` asks for more diff --git a/docs/agents/OPENCODE.md b/docs/agents/OPENCODE.md index c5610df92..a17e92b79 100644 --- a/docs/agents/OPENCODE.md +++ b/docs/agents/OPENCODE.md @@ -170,6 +170,8 @@ skills under test otherwise looks entirely normal. ## Permissions +Per-field contract and tool names (generated): [Harness Parity § Agent-field contract](HARNESS_PARITY.md#agent-field-contract). + Every run passes `--auto`, which auto-approves each permission that is not explicitly denied. Without it OpenCode blocks on an interactive approval prompt and the turn runs to its timeout. diff --git a/docs/agents/PI.md b/docs/agents/PI.md index 83c09c87d..f6c048656 100644 --- a/docs/agents/PI.md +++ b/docs/agents/PI.md @@ -110,6 +110,8 @@ superset of the Antigravity `thinking_level`), defaulting to `medium`. ### Config fields and their Pi flags +Per-field contract and tool names (generated): [Harness Parity § Agent-field contract](HARNESS_PARITY.md#agent-field-contract). + | Field | Pi flag | |---|---| | `system_prompt` | `--append-system-prompt ` (appended, semantics `append`) | diff --git a/docs/index.md b/docs/index.md index d3fc34ae7..439a8ad83 100644 --- a/docs/index.md +++ b/docs/index.md @@ -87,7 +87,7 @@ New here? Start with **[Tutorial 01 — Your First Evaluation](tutorials/01-firs | [Antigravity (Gemini)](agents/ANTIGRAVITY.md) | Running the Google Antigravity / Gemini agent | | [OpenCode](agents/OPENCODE.md) | Running the OpenCode agent on open-weight models | | [Pi](agents/PI.md) | Running the Pi agent on open-weight models | -| [Run-Limit Parity](agents/HARNESS_PARITY.md) | What each run_limits field means on every harness | +| [Run-Limit Parity](agents/HARNESS_PARITY.md) | What each run_limits and agent field means on every harness | | [A/B Experiments](AB_EXPERIMENTS.md) | Compare models / tools / prompts across the same tasks | | [Bring Your Own Dataset](DATASETS.md) | Fan a single task out over a dataset | | [Dialog Mode](DIALOG_MODE.md) | Evaluate agents in multi-turn conversation via a simulated user | diff --git a/docs/llms.txt b/docs/llms.txt index 9a8adaa68..816b95269 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -32,7 +32,7 @@ and A/B plumbing. - [Antigravity (Gemini)](https://coder-eval.com/docs/agents/antigravity): Running the Google Antigravity / Gemini agent - [OpenCode](https://coder-eval.com/docs/agents/opencode): Running the OpenCode agent on open-weight models - [Pi](https://coder-eval.com/docs/agents/pi): Running the Pi agent on open-weight models -- [Run-Limit Parity](https://coder-eval.com/docs/agents/harness-parity): What each run_limits field means on every harness +- [Run-Limit Parity](https://coder-eval.com/docs/agents/harness-parity): What each run_limits and agent field means on every harness - [A/B Experiments](https://coder-eval.com/docs/ab-experiments): Compare models / tools / prompts across the same tasks - [Bring Your Own Dataset](https://coder-eval.com/docs/datasets): Fan a single task out over a dataset - [Dialog Mode](https://coder-eval.com/docs/dialog-mode): Evaluate agents in multi-turn conversation via a simulated user diff --git a/mkdocs.yml b/mkdocs.yml index d01681d14..1b4b1d531 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -86,7 +86,7 @@ extra: agents/ANTIGRAVITY.md: "Running the Google Antigravity / Gemini agent" agents/OPENCODE.md: "Running the OpenCode agent on open-weight models" agents/PI.md: "Running the Pi agent on open-weight models" - agents/HARNESS_PARITY.md: "What each run_limits field means on every harness" + agents/HARNESS_PARITY.md: "What each run_limits and agent field means on every harness" AB_EXPERIMENTS.md: "Compare models / tools / prompts across the same tasks" DATASETS.md: "Fan a single task out over a dataset" DIALOG_MODE.md: "Evaluate agents in multi-turn conversation via a simulated user" diff --git a/pyproject.toml b/pyproject.toml index 6d17f1848..d1f100381 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -303,6 +303,7 @@ external = [ "CE065", "CE066", "CE068", + "CE069", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/tests/lint/harness_parity.py b/tests/lint/harness_parity.py new file mode 100644 index 000000000..18873c8da --- /dev/null +++ b/tests/lint/harness_parity.py @@ -0,0 +1,101 @@ +"""CE069 — the agent-field contract tables are generated from the agent classes. + +Each in-tree agent class declares a ``HarnessContract`` and, when it honors the tool +lists, a ``ToolNameMap``. Those declarations are the single source of truth for what a +uniform agent field means on a harness; a hand-written table beside them drifts the +first time a row flips. ``write()`` renders two Markdown tables into +``docs/agents/HARNESS_PARITY.md`` between their marker pairs, ``make parity-table`` +calls it, and CE069 (``check()``) re-renders and diffs against disk. + +Columns are the in-tree kinds in ``AgentKind`` order. A plugin kind registered in the +test process is not rendered: its run record carries its own contract. + +Wired as ``tests/test_custom_lint.py::TestCE069HarnessParityTable``. +""" + +from __future__ import annotations + +from pathlib import Path + +from coder_eval.agents.registry import AgentRegistry +from coder_eval.models import CANONICAL_TOOL_NAMES, AgentKind, HarnessContract +from coder_eval.plugins import ensure_plugins_loaded +from tests.lint.doc_indexes import _replace_between +from tests.lint.generated import diff_all, write_all + + +CONTRACT_START = "" +CONTRACT_END = "" +TOOLS_START = "" +TOOLS_END = "" +_DOC = Path("docs/agents/HARNESS_PARITY.md") + + +def _kinds() -> list[AgentKind]: + return [kind for kind in AgentKind if kind is not AgentKind.UNKNOWN] + + +def _agent_class(kind: AgentKind) -> type: + ensure_plugins_loaded() + registration = AgentRegistry.get(kind) + assert registration is not None, f"in-tree kind {kind!r} is not registered" + return registration.agent_class + + +def _cell(value: object) -> str: + if value is None: + return "—" + if isinstance(value, bool): + return "yes" if value else "no" + if isinstance(value, frozenset): + return ", ".join(sorted(str(v) for v in value)) + return str(value) + + +def _row(label: str, cells: list[str]) -> str: + return f"| {label} | " + " | ".join(cells) + " |" + + +def render_table() -> str: + """One row per ``HarnessContract`` field, one column per in-tree kind.""" + kinds = _kinds() + contracts = [_agent_class(kind).contract for kind in kinds] + lines = [_row("field", [str(k) for k in kinds]), _row("---", ["---"] * len(kinds))] + lines += [ + _row(f"`{field}`", [_cell(getattr(contract, field)) for contract in contracts]) + for field in HarnessContract.model_fields + ] + return "\n".join(lines) + + +def render_tool_table() -> str: + """One row per canonical tool name, one column per kind that maps tool names.""" + mapped = [(kind, names) for kind in _kinds() if (names := _agent_class(kind).tool_names) is not None] + lines = [_row("tool", [str(k) for k, _ in mapped]), _row("---", ["---"] * len(mapped))] + lines += [ + _row(f"`{name}`", [", ".join(f"`{n}`" for n in tool_names.names[name]) or "none" for _, tool_names in mapped]) + for name in sorted(CANONICAL_TOOL_NAMES) + ] + return "\n".join(lines) + + +def _rendered_files(repo_root: Path) -> dict[Path, str]: + doc = repo_root / _DOC + text = _replace_between(doc.read_text(encoding="utf-8"), CONTRACT_START, CONTRACT_END, render_table()) + return {doc: _replace_between(text, TOOLS_START, TOOLS_END, render_tool_table())} + + +def write(repo_root: Path) -> list[Path]: + """Regenerate both tables in place. Returns the files touched.""" + return write_all(_rendered_files(repo_root)) + + +def check(repo_root: Path) -> dict[str, str]: + """Unified diff per file whose generated content differs from disk (empty = clean).""" + return diff_all(_rendered_files(repo_root)) + + +if __name__ == "__main__": + root = Path(__file__).resolve().parents[2] + for p in write(root): + print(f"wrote {p}") diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 236bf17af..5c94214c2 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -2162,6 +2162,37 @@ def test_the_reports_package_is_in_scope(self): assert self._violations("from ..cli import run_command", "/repo/src/coder_eval/reports/markdown.py") +@pytest.mark.lint +class TestCE069HarnessParityTable: + """CE069 — the agent-field contract tables are generated from the agent classes.""" + + REPO_ROOT = Path(__file__).parent.parent + + def test_repo_tables_match_generated_output(self): + from tests.lint.harness_parity import check + + findings = check(self.REPO_ROOT) + assert not findings, ( + "\nThe agent-field contract tables drifted from the agent classes — run `make parity-table` " + "to regenerate:\n\n" + "\n\n".join(f"{path}:\n{diff}" for path, diff in sorted(findings.items())) + ) + + def test_both_marker_pairs_exist(self): + from tests.lint.harness_parity import CONTRACT_END, CONTRACT_START, TOOLS_END, TOOLS_START + + text = (self.REPO_ROOT / "docs/agents/HARNESS_PARITY.md").read_text(encoding="utf-8") + for marker in (CONTRACT_START, CONTRACT_END, TOOLS_START, TOOLS_END): + assert marker in text + + def test_render_pins_the_header_and_a_known_cell(self): + from tests.lint.harness_parity import render_table + + lines = render_table().splitlines() + assert lines[0] == "| field | claude-code | codex | antigravity | opencode | pi | none |" + system_prompt = next(line for line in lines if line.startswith("| `system_prompt` |")) + assert system_prompt.split(" | ")[5] == "enforced" + + @pytest.mark.lint class TestCE068NoKindNamesInKernel: """CE068 — orchestration/, streaming/ and timing.py name no concrete agent kind.""" diff --git a/tests/test_harness_conformance.py b/tests/test_harness_conformance.py new file mode 100644 index 000000000..b633c0e8a --- /dev/null +++ b/tests/test_harness_conformance.py @@ -0,0 +1,416 @@ +"""Every in-tree harness honors what its contract claims, and rejects what it does not. + +The agent class is the single source of truth: rejection cases are derived from each +``HarnessContract``, and every ENFORCED cell (every declared ``permission_modes`` +value, for ``permission_mode``) must have a probe below that proves the value reaches +the native call, offline. A ``system_prompt`` probe also proves the user turn reaches +the harness unchanged, so an adapter that prefixes the user message cannot claim +``append``. Each tool-list probe includes a name the harness has no tool for. +""" + +from __future__ import annotations + +import json +from collections.abc import Awaitable, Callable +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from unittest.mock import patch + +import pytest + +from coder_eval.agents.antigravity_agent import AntigravityAgent +from coder_eval.agents.claude_code_agent import ClaudeCodeAgent +from coder_eval.agents.codex_agent import CodexAgent +from coder_eval.agents.opencode_agent import OpenCodeAgent +from coder_eval.agents.pi_agent import PiAgent +from coder_eval.agents.registry import AgentRegistry +from coder_eval.models import ( + AgentKind, + Enforcement, + FileExistsCriterion, + HarnessContract, + PermissionMode, + SandboxConfig, + TaskDefinition, + parse_agent_config, +) +from coder_eval.orchestration.harness_contract import HarnessContractError, validate_harness_contract +from coder_eval.plugins import ensure_plugins_loaded +from tests.test_antigravity_agent import _install_fake_sdk + + +MARKER = "CONFORMANCE-MARKER-7f3a" +USER_TURN = "do the task" +_FIELDS = ("system_prompt", "plugin_skills", "permission_mode", "allowed_tools", "disallowed_tools") +_CONFIG_FIELD = {"plugin_skills": "plugins"} +_KINDS = [kind for kind in AgentKind if kind is not AgentKind.UNKNOWN] + +type Probe = Callable[[Path, pytest.MonkeyPatch], Awaitable[None]] + + +def _contract(kind: AgentKind) -> HarnessContract: + ensure_plugins_loaded() + registration = AgentRegistry.get(kind) + assert registration is not None + return registration.agent_class.contract + + +def _task(kind: AgentKind, **agent: Any) -> TaskDefinition: + return TaskDefinition( + task_id="t", + description="d", + initial_prompt=None if kind is AgentKind.NONE else USER_TURN, + agent=parse_agent_config(type=kind, **agent), + sandbox=SandboxConfig(driver="tempdir"), + success_criteria=[FileExistsCriterion(description="c", path="out.txt")], + ) + + +def _plugin_root(tmp_path: Path) -> Path: + skill = tmp_path / "plugin" / "skills" / "probe-skill" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text("---\nname: probe-skill\ndescription: d\n---\n", encoding="utf-8") + return tmp_path / "plugin" + + +def _plugins(tmp_path: Path) -> list[dict[str, str]]: + return [{"type": "local", "path": str(_plugin_root(tmp_path))}] + + +_GATED_VALUES: dict[str, Any] = { + "system_prompt": MARKER, + "plugins": [{"type": "local", "path": "/plugins/p"}], + "permission_mode": "plan", + "allowed_tools": ["Bash"], + "disallowed_tools": ["Bash"], +} + + +# --- rejections, derived from the contracts ---------------------------------------- + + +@pytest.mark.parametrize( + ("kind", "field"), + [(k, f) for k in _KINDS for f in _FIELDS if getattr(_contract(k), f) is Enforcement.UNSUPPORTED], +) +def test_unsupported_field_is_rejected(kind: AgentKind, field: str) -> None: + config_field = _CONFIG_FIELD.get(field, field) + with pytest.raises(HarnessContractError, match=rf"agent\.{config_field}.*{kind.value!r}"): + validate_harness_contract(_task(kind, **{config_field: _GATED_VALUES[config_field]})) + + +@pytest.mark.parametrize( + ("kind", "mode"), + [ + (k, m) + for k in _KINDS + if _contract(k).permission_mode is Enforcement.ENFORCED + for m in PermissionMode + if m not in (_contract(k).permission_modes or frozenset()) + ], +) +def test_undeclared_permission_value_is_rejected(kind: AgentKind, mode: PermissionMode) -> None: + with pytest.raises(HarnessContractError, match="has no documented meaning"): + validate_harness_contract(_task(kind, permission_mode=mode)) + + +@pytest.mark.parametrize( + "kind", [k for k in _KINDS if AgentRegistry.get(k) and AgentRegistry.get(k).agent_class.tool_names] +) +def test_unknown_tool_name_is_rejected(kind: AgentKind) -> None: + with pytest.raises(HarnessContractError, match="did you mean 'Bash'"): + validate_harness_contract(_task(kind, allowed_tools=["Bassh"])) + + +# --- probes: the value reaches the native call -------------------------------------- + + +async def _claude(tmp_path: Path, **agent: Any) -> tuple[Any, str]: + captured: dict[str, Any] = {} + + async def fake_query(prompt: str, options: Any): + captured["prompt"], captured["options"] = prompt, options + yield type( + "ResultMessage", + (), + {"session_id": "s", "usage": {}, "total_cost_usd": 0.0, "num_turns": 1, "is_error": False, "result": "ok"}, + )() + + claude = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE, **agent)) + await claude.start(str(tmp_path)) + with patch("coder_eval.agents.claude_code_agent.query", fake_query): + await claude.communicate(USER_TURN) + return captured["options"], captured["prompt"] + + +async def _probe_claude_system_prompt(tmp_path: Path, _mp: pytest.MonkeyPatch) -> None: + options, prompt = await _claude(tmp_path, system_prompt=MARKER) + assert MARKER in str(options.system_prompt) + assert MARKER not in prompt + + +async def _probe_claude_plugins(tmp_path: Path, _mp: pytest.MonkeyPatch) -> None: + root = _plugin_root(tmp_path) + options, _ = await _claude(tmp_path, plugins=[{"type": "local", "path": str(root)}]) + assert [p["path"] for p in options.plugins] == [str(root)] + + +def _claude_mode(mode: PermissionMode) -> Probe: + async def probe(tmp_path: Path, _mp: pytest.MonkeyPatch) -> None: + options, _ = await _claude(tmp_path, permission_mode=mode) + assert options.permission_mode == mode.value + + return probe + + +async def _probe_claude_allowed(tmp_path: Path, _mp: pytest.MonkeyPatch) -> None: + options, _ = await _claude(tmp_path, allowed_tools=["Bash"]) + assert options.allowed_tools == ["Bash"] + + +async def _probe_claude_disallowed(tmp_path: Path, _mp: pytest.MonkeyPatch) -> None: + options, _ = await _claude(tmp_path, disallowed_tools=["Bash"]) + assert "Bash" in options.disallowed_tools + + +async def _probe_codex_system_prompt(_tmp: Path, _mp: pytest.MonkeyPatch) -> None: + from tests.test_codex_agent import _FakeThread, _started_agent, _turn_completed + + turn_inputs: list[str] = [] + + class _RecordingThread(_FakeThread): + def turn(self, user_input: str): # type: ignore[override] + turn_inputs.append(user_input) + return super().turn(user_input) + + codex = _started_agent(parse_agent_config(type=AgentKind.CODEX, system_prompt=MARKER), [_turn_completed()]) + options = codex._build_thread_options() + codex.thread = _RecordingThread([_turn_completed()]) + await codex.communicate(USER_TURN) + assert options["developer_instructions"] == MARKER + assert turn_inputs == [USER_TURN] + + +async def _probe_codex_plugins(tmp_path: Path, _mp: pytest.MonkeyPatch) -> None: + codex = CodexAgent(parse_agent_config(type=AgentKind.CODEX, plugins=_plugins(tmp_path))) + codex.working_directory = tmp_path / "work" + codex.working_directory.mkdir() + codex._setup_skills(None) + assert (codex.working_directory / ".agents" / "skills" / "probe-skill" / "SKILL.md").exists() + + +async def _antigravity_config(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, **agent: Any) -> Any: + configs: list[Any] = [] + + class _FakeSdkAgent: + def __init__(self, cfg: Any) -> None: + configs.append(cfg) + + async def __aenter__(self) -> _FakeSdkAgent: + return self + + async def __aexit__(self, *exc: object) -> bool: + return False + + _install_fake_sdk(monkeypatch, _FakeSdkAgent) + await AntigravityAgent(parse_agent_config(type=AgentKind.ANTIGRAVITY, **agent)).start(str(tmp_path)) + return configs[0] + + +def _policy_pairs(cfg: Any) -> list[tuple[str, str | None]]: + return [(p.kind, getattr(p, "tool", None)) for p in cfg.policies] + + +async def _probe_antigravity_system_prompt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from tests._fixtures.golden_streams.antigravity_fixtures import _FakeConversation, _step, _usage + + cfg = await _antigravity_config(tmp_path, monkeypatch, system_prompt=MARKER) + assert cfg.system_instructions == MARKER + + sent: list[str] = [] + + class _RecordingConversation(_FakeConversation): + async def send(self, prompt: str, **kwargs: Any) -> None: + sent.append(prompt) + + done = _step("TEXT_RESPONSE", "DONE", content="ok", complete=True, usage=_usage(10, 0, 1, 0)) + agent = AntigravityAgent(parse_agent_config(type=AgentKind.ANTIGRAVITY, system_prompt=MARKER)) + agent.working_directory = tmp_path + agent._sdk_agent = SimpleNamespace(conversation=_RecordingConversation([done]), is_started=True) + await agent.communicate(USER_TURN) + assert sent == [USER_TURN] + + +async def _probe_antigravity_plugins(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root = _plugin_root(tmp_path) + cfg = await _antigravity_config(tmp_path / "work", monkeypatch, plugins=[{"type": "local", "path": str(root)}]) + assert str(root / "skills") in cfg.skills_paths + + +async def _probe_antigravity_plan(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + cfg = await _antigravity_config(tmp_path, monkeypatch, permission_mode="plan") + assert {("deny", t) for t in ("create_file", "edit_file", "run_command")} <= set(_policy_pairs(cfg)) + + +async def _probe_antigravity_bypass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + cfg = await _antigravity_config(tmp_path, monkeypatch, permission_mode="bypassPermissions") + assert _policy_pairs(cfg) == [("allow_all", None)] + + +async def _probe_antigravity_allowed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + cfg = await _antigravity_config(tmp_path, monkeypatch, allowed_tools=["Bash", "NotebookEdit"]) + assert _policy_pairs(cfg) == [("deny_all", None), ("allow", "finish"), ("allow", "run_command")] + + +async def _probe_antigravity_disallowed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + cfg = await _antigravity_config(tmp_path, monkeypatch, disallowed_tools=["Bash", "NotebookEdit"]) + assert _policy_pairs(cfg) == [("allow_all", None), ("deny", "run_command")] + + +async def _cli_agent(cls: type, kind: AgentKind, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, **agent: Any): + monkeypatch.setattr("shutil.which", lambda name: f"/usr/local/bin/{name}") + cli = cls(parse_agent_config(type=kind, model="provider/model", **agent), task_id="t1") + await cli.start(str(tmp_path / "work")) + return cli + + +async def _opencode(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, **agent: Any) -> tuple[dict[str, Any], list[str]]: + monkeypatch.delenv("OPENCODE_CONFIG_CONTENT", raising=False) + opencode = await _cli_agent(OpenCodeAgent, AgentKind.OPENCODE, tmp_path, monkeypatch, **agent) + try: + raw = opencode._build_env().get("OPENCODE_CONFIG_CONTENT") + config = json.loads(raw) if raw else {} + config["instructions_text"] = [Path(p).read_text(encoding="utf-8") for p in config.get("instructions", [])] + return config, opencode._build_argv(USER_TURN) + finally: + await opencode.stop() + + +async def _probe_opencode_system_prompt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + config, argv = await _opencode(tmp_path, monkeypatch, system_prompt=MARKER) + assert config["instructions_text"] == [MARKER] + assert MARKER not in " ".join(argv) + + +async def _probe_opencode_plugins(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root = _plugin_root(tmp_path) + config, _ = await _opencode(tmp_path, monkeypatch, plugins=[{"type": "local", "path": str(root)}]) + assert config["skills"]["paths"] == [str(root / "skills")] + + +async def _probe_opencode_plan(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + config, argv = await _opencode(tmp_path, monkeypatch, permission_mode="plan") + assert config["permission"] == {"edit": "deny", "bash": "deny"} + assert "--auto" in argv + + +async def _probe_opencode_bypass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + config, argv = await _opencode(tmp_path, monkeypatch, permission_mode="bypassPermissions") + assert "permission" not in config + assert "--auto" in argv + + +async def _probe_opencode_allowed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + config, _ = await _opencode(tmp_path, monkeypatch, allowed_tools=["Bash", "NotebookEdit"]) + assert config["permission"] == {"*": "deny", "external_directory": "allow", "doom_loop": "allow", "bash": "allow"} + + +async def _probe_opencode_disallowed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + config, _ = await _opencode(tmp_path, monkeypatch, disallowed_tools=["Bash", "NotebookEdit"]) + assert config["permission"] == {"bash": "deny"} + + +async def _pi_argv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, **agent: Any) -> list[str]: + pi = await _cli_agent(PiAgent, AgentKind.PI, tmp_path, monkeypatch, **agent) + try: + return pi._build_argv(USER_TURN) + finally: + await pi.stop() + + +def _flag(argv: list[str], flag: str) -> str: + return argv[argv.index(flag) + 1] + + +async def _probe_pi_system_prompt(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + argv = await _pi_argv(tmp_path, monkeypatch, system_prompt=MARKER) + assert _flag(argv, "--append-system-prompt") == MARKER + assert MARKER not in argv[argv.index("--") + 1 :] + + +async def _probe_pi_plugins(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root = _plugin_root(tmp_path) + argv = await _pi_argv(tmp_path, monkeypatch, plugins=[{"type": "local", "path": str(root)}]) + assert _flag(argv, "--skill") == str(root / "skills") + + +async def _probe_pi_plan(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + argv = await _pi_argv(tmp_path, monkeypatch, permission_mode="plan") + assert _flag(argv, "--exclude-tools") == "bash,edit,multiedit,patch,write" + + +async def _probe_pi_bypass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + argv = await _pi_argv(tmp_path, monkeypatch, permission_mode="bypassPermissions") + assert "--tools" not in argv and "--exclude-tools" not in argv + + +async def _probe_pi_allowed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + assert _flag(await _pi_argv(tmp_path, monkeypatch, allowed_tools=["Bash", "Skill"]), "--tools") == "bash" + + +async def _probe_pi_disallowed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + argv = await _pi_argv(tmp_path, monkeypatch, disallowed_tools=["Bash", "Skill"]) + assert _flag(argv, "--exclude-tools") == "bash" + + +_PROBES: dict[tuple[str, str], Probe] = { + ("claude-code", "system_prompt"): _probe_claude_system_prompt, + ("claude-code", "plugin_skills"): _probe_claude_plugins, + **{("claude-code", f"permission_mode={m.value}"): _claude_mode(m) for m in PermissionMode}, + ("claude-code", "allowed_tools"): _probe_claude_allowed, + ("claude-code", "disallowed_tools"): _probe_claude_disallowed, + ("codex", "system_prompt"): _probe_codex_system_prompt, + ("codex", "plugin_skills"): _probe_codex_plugins, + ("antigravity", "system_prompt"): _probe_antigravity_system_prompt, + ("antigravity", "plugin_skills"): _probe_antigravity_plugins, + ("antigravity", "permission_mode=plan"): _probe_antigravity_plan, + ("antigravity", "permission_mode=bypassPermissions"): _probe_antigravity_bypass, + ("antigravity", "allowed_tools"): _probe_antigravity_allowed, + ("antigravity", "disallowed_tools"): _probe_antigravity_disallowed, + ("opencode", "system_prompt"): _probe_opencode_system_prompt, + ("opencode", "plugin_skills"): _probe_opencode_plugins, + ("opencode", "permission_mode=plan"): _probe_opencode_plan, + ("opencode", "permission_mode=bypassPermissions"): _probe_opencode_bypass, + ("opencode", "allowed_tools"): _probe_opencode_allowed, + ("opencode", "disallowed_tools"): _probe_opencode_disallowed, + ("pi", "system_prompt"): _probe_pi_system_prompt, + ("pi", "plugin_skills"): _probe_pi_plugins, + ("pi", "permission_mode=plan"): _probe_pi_plan, + ("pi", "permission_mode=bypassPermissions"): _probe_pi_bypass, + ("pi", "allowed_tools"): _probe_pi_allowed, + ("pi", "disallowed_tools"): _probe_pi_disallowed, +} + + +def _enforced_cells() -> set[tuple[str, str]]: + cells: set[tuple[str, str]] = set() + for kind in _KINDS: + contract = _contract(kind) + for field in _FIELDS: + if getattr(contract, field) is not Enforcement.ENFORCED: + continue + if field == "permission_mode": + cells |= {(kind.value, f"permission_mode={m.value}") for m in contract.permission_modes or ()} + else: + cells.add((kind.value, field)) + return cells + + +def test_every_enforced_cell_has_exactly_one_probe() -> None: + assert set(_PROBES) == _enforced_cells() + + +@pytest.mark.parametrize("cell", sorted(_PROBES)) +async def test_probe(cell: tuple[str, str], tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + await _PROBES[cell](tmp_path, monkeypatch) From d264f150402e2afee472441bbbab3f9f83e2db40 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 10:49:24 -0700 Subject: [PATCH 09/12] fix: code review fixes for the harness contract and SPI plan - Docker staging and the Harbor export write only the agent fields a layer set, so a reloaded non-Claude task no longer claims the permission_mode default and fails the contract check in the container (round-trip tests added). - An empty allowed_tools / disallowed_tools restricts nothing and is not a set field. - OpenCode keeps a host rule for external_directory / doom_loop under an allowlist. - coder_eval.spi exports CommandTelemetry, TokenUsage, TranscriptMessage, ResultSummary and ModelPricing, which a plugin needs to emit events and prices. - Correct stale notes and docs: the Codex start() warning, the simulator's allowed_tools=[] claim, plan meaning, resolution vs load, and Pi's ls. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/notes/agents.md | 8 +++--- docs/DIALOG_MODE.md | 5 ++-- docs/EXTENDING.md | 2 +- docs/TASK_DEFINITION_GUIDE.md | 6 ++--- docs/agents/CODEX.md | 4 +-- docs/agents/OPENCODE.md | 3 ++- docs/agents/PI.md | 6 ++--- src/coder_eval/agent.py | 2 +- src/coder_eval/agents/opencode_agent.py | 12 ++++----- src/coder_eval/harbor/packager.py | 4 ++- src/coder_eval/isolation/docker_runner.py | 7 ++++- .../orchestration/harness_contract.py | 22 +++++++++++---- src/coder_eval/simulation/user_simulator.py | 8 +++--- src/coder_eval/spi.py | 11 +++++++- .../expected/environment/task.yaml | 4 --- tests/test_execute_command.py | 27 +++++++++++++++++++ tests/test_harbor_packager.py | 17 ++++++++++++ tests/test_harness_contract.py | 6 ++--- tests/test_opencode_agent.py | 9 +++++++ 19 files changed, 120 insertions(+), 43 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 40ddbc9cd..6c2a5e2f3 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -555,10 +555,10 @@ and scores 0 with no loud error. Dropping to full-access matches claude-code and Antigravity, which run with no in-agent OS sandbox; it also keeps network on, so tool installs work without extra sandbox config. -The consequence is stated loudly at `start()` for EVERY mode, not just -`bypassPermissions`, so operators are not misled that plan/acceptEdits/default confine -Codex — none of them do. Adversarial or untrusted evals belong on the docker driver; the -tempdir/host driver is a working directory, not a confinement boundary. +Codex's contract therefore marks `permission_mode` unsupported, so a Codex task that sets +any mode is rejected at resolution rather than believing plan/acceptEdits/default confine +it. Adversarial or untrusted evals belong on the docker driver; the tempdir/host driver is +a working directory, not a confinement boundary. Tool restriction is not available either. `strings` on the pinned codex-cli 0.39.0 binary shows `enabled_tools` / `disabled_tools` only inside `RawMcpServerConfig` (beside diff --git a/docs/DIALOG_MODE.md b/docs/DIALOG_MODE.md index 9fff5dc9f..0f18bdd28 100644 --- a/docs/DIALOG_MODE.md +++ b/docs/DIALOG_MODE.md @@ -57,8 +57,9 @@ The mechanics: from its persona and goal — closer to a cold-start user. - Each exchange is one user message plus the agent's full response (the agent may make many tool calls inside a single exchange). -- The simulator is a **tools-disabled Claude Code agent** with `allowed_tools: []`, an explicit - deny-list, and no plugins or settings sources. It is pure text-in / text-out, and it **cannot see +- The simulator is a **tools-disabled Claude Code agent**: an explicit deny-list of every + built-in tool, and no plugins or settings sources. (Its `allowed_tools: []` restricts + nothing on Claude Code; the deny-list is the safeguard.) It is pure text-in / text-out, and it **cannot see the sandbox** — no files, no terminal, no agent reasoning. Only what the agent writes in the chat. - The simulator resolves its own `ApiRoute` independently of `checker_context.api_route` (that override is judge-only — see [Checker Context](TASK_DEFINITION_GUIDE.md#checker-context)) — same diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 7d4cce47d..c4307de56 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -301,7 +301,7 @@ Plugins that run their own models contribute USD rates through `register_pricing there is **no** separate entry-point group; call it from the same `register()` hook. ```python -from coder_eval.pricing import ModelPricing, register_pricing +from coder_eval.spi import ModelPricing, register_pricing # Rates are per MILLION tokens: (input, output, cache_write, cache_read) MY_RATES = { diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index c7516e854..c4e1126fb 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -185,15 +185,15 @@ an error. **Permission Modes:** - `default` — Default permission handling - `acceptEdits` — Auto-accept file edits (recommended for evaluations) -- `plan` — Agent proposes changes, waits for approval +- `plan` — Read-only: the Write, Edit and Bash tools are denied - `bypassPermissions` — No permission checks (use with caution) **Fields a harness cannot honor are rejected.** Every agent declares which of `system_prompt`, `plugins`, `permission_mode`, `allowed_tools` and `disallowed_tools` it honors. A task that sets one of them on a harness that marks it unsupported fails at resolution, and `coder-eval plan` exits non-zero. A field -that only a lower layer's default sets does not count, and neither does a value of -`null`. Values are checked too: a `permission_mode` value the harness does not +whose value is the schema default counts only if a layer wrote it; `null` and an empty +tool list never count. Values are checked too: a `permission_mode` value the harness does not honor (for example `acceptEdits` on Pi, OpenCode or Antigravity, which honor only `plan` and `bypassPermissions`) is rejected, and every `allowed_tools` / `disallowed_tools` name must be a canonical tool name (`Agent`, `Bash`, `Edit`, diff --git a/docs/agents/CODEX.md b/docs/agents/CODEX.md index 35f2f184d..65f43052a 100644 --- a/docs/agents/CODEX.md +++ b/docs/agents/CODEX.md @@ -171,7 +171,7 @@ On failure, the agent: ### Permission and Tool Mapping Per-field contract (generated): [Harness Parity § Agent-field contract](HARNESS_PARITY.md#agent-field-contract). `permission_mode`, `allowed_tools` and -`disallowed_tools` are rejected at load on Codex. +`disallowed_tools` are rejected at resolution on Codex. Codex runs with `sandbox: full-access` and `approval_mode: deny_all` on every run. Its own OS sandbox fails silently on the hosts Coder Eval runs on, so the isolation boundary is the task's driver: use `driver: docker` for untrusted evals. @@ -212,7 +212,7 @@ Run-limit semantics per harness: [Run-Limit Parity](HARNESS_PARITY.md). 1. **Tool-name collapse** - Codex reports shell tools (`Read`/`Grep`/`Bash`) all as shell commands, surfaced as `Bash` telemetry; name-keyed criteria that distinguish these tools aren't meaningful across agents. 2. **`skill_triggered` criterion** - Codex has no distinct `Skill` tool (it engages a skill by reading its files via shell), so the criterion detects Codex engagement from that file-read signal (a command referencing `skills//`) instead of a `Skill` tool call. The file-read signal is weaker than Claude's explicit invocation. -3. **`permission_mode`, `allowed_tools`, `disallowed_tools`** - Codex honors none of them; a Codex task that sets any of them is rejected at load. +3. **`permission_mode`, `allowed_tools`, `disallowed_tools`** - Codex honors none of them; a Codex task that sets any of them is rejected at resolution. 4. **Authentication** - Requires `CODEX_API_KEY` in the environment (point it at whichever endpoint's key you use — OpenAI, gateway, or Azure); the agent calls `login_api_key` when a key is present. `OPENAI_API_KEY`/`AZURE_OPENAI_API_KEY` are NOT read. 5. **Model field** - `TurnRecord.model_used` reflects the pinned `agent.model`; the Codex `Turn` payload itself doesn't carry the resolved model. 6. **Skills with Windows paths** - Symlink creation may fail on Windows; agent falls back to copying (slower). diff --git a/docs/agents/OPENCODE.md b/docs/agents/OPENCODE.md index a17e92b79..d06b39c05 100644 --- a/docs/agents/OPENCODE.md +++ b/docs/agents/OPENCODE.md @@ -195,7 +195,8 @@ keys are coarser than its tools: `edit` governs `write`, `edit`, `patch`, no OpenCode equivalent restricts nothing; an allowlist of only such names denies every tool. An empty `allowed_tools: []` restricts nothing, as on Claude Code. Our rules are placed after every inherited rule, because OpenCode applies the last -matching rule. A string rule such as `read: "allow"` replaces the CLI's default +matching rule. A host rule for `external_directory` or `doom_loop` is kept, so a tool +allowlist never loosens it. A string rule such as `read: "allow"` replaces the CLI's default `.env` read deny, which is acceptable inside a sandbox. `system_prompt` is written to a temporary file outside the sandbox and listed in diff --git a/docs/agents/PI.md b/docs/agents/PI.md index f6c048656..eec02eb4d 100644 --- a/docs/agents/PI.md +++ b/docs/agents/PI.md @@ -123,9 +123,9 @@ Per-field contract and tool names (generated): [Harness Parity § Agent-field co | `plugins` | `--skill ` per resolved skills dir | Claude tool names map to Pi's lowercase built-ins by inverting the telemetry map -(`Bash` → `bash`, `Edit` → `edit,multiedit,patch`, `Glob` → `find`, `LS` → -`list,ls`, …). A name with no Pi equivalent restricts nothing; an allowlist of only -such names disables every tool. An empty `allowed_tools: []` restricts nothing, as on +(`Bash` → `bash`, `Edit` → `edit,multiedit,patch`, `Glob` → `find`, …). A canonical +name with no Pi equivalent restricts nothing; an allowlist of only such names disables +every tool. Pi's `ls` has no canonical name, so any allowlist denies it. An empty `allowed_tools: []` restricts nothing, as on Claude Code. ## Permissions diff --git a/src/coder_eval/agent.py b/src/coder_eval/agent.py index 1e4bdb7b1..739f21d3e 100644 --- a/src/coder_eval/agent.py +++ b/src/coder_eval/agent.py @@ -72,7 +72,7 @@ def __init__(self, config: ClaudeCodeAgentConfig, ...): # Which uniform config fields this harness honors. No default: registration # rejects a class that does not declare one. - # Rationale: .claude/notes/agents.md § The system_prompt_semantics marker + # Rationale: .claude/notes/agents.md § The uniform fields, per harness contract: ClassVar[HarnessContract] # Canonical tool name -> native tools. Registration requires one exactly when the diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index de1d0cd0c..3d7122627 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -164,7 +164,7 @@ } # Permissions `"*"` also matches that are not tools. An allowlist re-allows them, so it -# restricts tools only; `--auto` approved them before. +# restricts tools only. _NON_TOOL_PERMISSIONS: tuple[str, ...] = ("external_directory", "doom_loop") # The canonical tool names OpenCode has no tool for. @@ -1012,13 +1012,11 @@ def _inject_config_content(self, env: dict[str, str]) -> None: ] if permission: # OpenCode applies the LAST matching rule, so ours go after every inherited one. + # A host rule for a non-tool key is kept: a tool allowlist must not loosen it. inherited_rules = config.get("permission") - kept = ( - {k: v for k, v in inherited_rules.items() if k not in permission} - if isinstance(inherited_rules, dict) - else {} - ) - config["permission"] = {**kept, **permission} + inherited = inherited_rules if isinstance(inherited_rules, dict) else {} + ours = {k: v for k, v in permission.items() if not (k in _NON_TOOL_PERMISSIONS and k in inherited)} + config["permission"] = {**{k: v for k, v in inherited.items() if k not in ours}, **ours} env[_CONFIG_CONTENT_ENV] = json.dumps(config) # --- the turn ---------------------------------------------------------- diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py index 5f83c46cb..125906932 100644 --- a/src/coder_eval/harbor/packager.py +++ b/src/coder_eval/harbor/packager.py @@ -574,7 +574,9 @@ def _write_agent_phase_task_yaml( sandbox_dict["driver"] = "tempdir" sandbox_dict.pop("docker", None) agent_dict = ( - task.agent.model_dump(mode="json", exclude_none=True) if task.agent is not None else {"type": "claude-code"} + task.agent.model_dump(mode="json", exclude_none=True, exclude_unset=True) + if task.agent is not None + else {"type": "claude-code"} ) payload: dict[str, object] = { "task_id": task.task_id, diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index b32b6b1b0..96729474a 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -701,7 +701,12 @@ async def _stage_inputs(self, input_dir: Path) -> None: task_yaml_in = input_dir / "task.yaml" def _dump_task_yaml() -> str: - return yaml.safe_dump(self.rt.task.model_dump(mode="json"), sort_keys=False) + payload = self.rt.task.model_dump(mode="json") + if self.rt.task.agent is not None: + # Only the fields a layer wrote: the reloaded task must not claim a + # model default (e.g. permission_mode) the harness contract rejects. + payload["agent"] = self.rt.task.agent.model_dump(mode="json", exclude_unset=True) + return yaml.safe_dump(payload, sort_keys=False) task_yaml_text = await asyncio.to_thread(_dump_task_yaml) await asyncio.to_thread(task_yaml_in.write_text, task_yaml_text, encoding="utf-8") diff --git a/src/coder_eval/orchestration/harness_contract.py b/src/coder_eval/orchestration/harness_contract.py index 3d8e5f85a..495c06b7b 100644 --- a/src/coder_eval/orchestration/harness_contract.py +++ b/src/coder_eval/orchestration/harness_contract.py @@ -7,7 +7,14 @@ from collections.abc import Callable from typing import TYPE_CHECKING, Any -from coder_eval.models import CANONICAL_TOOL_NAMES, Enforcement, HarnessContract, PermissionMode, ToolNameMap +from coder_eval.models import ( + CANONICAL_TOOL_NAMES, + BaseAgentConfig, + Enforcement, + HarnessContract, + PermissionMode, + ToolNameMap, +) if TYPE_CHECKING: @@ -75,7 +82,8 @@ def validate_harness_contract(task: TaskDefinition) -> None: unsupported; a ``permission_mode`` value outside the contract's ``permission_modes``; a tool-list name outside ``CANONICAL_TOOL_NAMES`` (or an ``mcp__`` name the harness cannot address). A field is set when a config layer - wrote it and its value is not None. A task without an agent type returns + wrote it with a value other than None or an empty tool list (which restricts + nothing). A task without an agent type returns silently; the layer-5 type guard reports that. Raises: @@ -86,9 +94,7 @@ def validate_harness_contract(task: TaskDefinition) -> None: registration = registration_for(task, requirement="The harness contract check") contract = registration.agent_class.contract kind = str(task.agent.type) - set_fields = [ - field for field in _GATED if field in task.agent.model_fields_set and getattr(task.agent, field) is not None - ] + set_fields = [field for field in _GATED if _is_set(task.agent, field)] for field in set_fields: row = _GATED[field] if getattr(contract, row) is Enforcement.UNSUPPORTED: @@ -106,6 +112,12 @@ def validate_harness_contract(task: TaskDefinition) -> None: _check_tool_names(field, getattr(task.agent, field), tool_names, kind) +def _is_set(agent: BaseAgentConfig, field: str) -> bool: + """A layer wrote the field with a value that means something: not None, and not an empty tool list.""" + value = getattr(agent, field) + return field in agent.model_fields_set and value is not None and value != [] + + def _honoring_kinds(honors: Callable[[HarnessContract], bool]) -> str: from coder_eval.agents.registry import AgentRegistry diff --git a/src/coder_eval/simulation/user_simulator.py b/src/coder_eval/simulation/user_simulator.py index 27f79d3b6..0696583f0 100644 --- a/src/coder_eval/simulation/user_simulator.py +++ b/src/coder_eval/simulation/user_simulator.py @@ -114,8 +114,8 @@ def _extract_system_prompt(config: SimulationConfig, task_description: str, init _OPENER_NUDGE = "Begin the conversation now: send your opening message as the user to the coding agent." -# SECURITY: ``allowed_tools=[]`` is the primary safeguard; this list pins the -# property against a future SDK change that reinterprets an empty allow-list. +# SECURITY: this deny list is the safeguard. ``allowed_tools=[]`` restricts nothing on +# Claude Code (no ``--allowedTools`` flag is sent), so it cannot be relied on. _SIMULATOR_DISALLOWED_TOOLS: list[str] = [ "Bash", "Read", @@ -192,8 +192,8 @@ def __init__( # BEDROCK_MODEL decide who the simulated user was, so an A/B varying the # subject model silently varied the interlocutor too. self._model = self._resolve_model(config.model, route) - # SECURITY: allowed_tools=[] is the primary guarantee that the simulator - # cannot touch files or run commands; the deny list is the backstop. + # SECURITY: _SIMULATOR_DISALLOWED_TOOLS is what keeps the simulator off files + # and commands; allowed_tools=[] restricts nothing on Claude Code. from coder_eval.models import ClaudeCodeAgentConfig agent_config = parse_agent_config( diff --git a/src/coder_eval/spi.py b/src/coder_eval/spi.py index 3163bc42a..0c918f397 100644 --- a/src/coder_eval/spi.py +++ b/src/coder_eval/spi.py @@ -16,15 +16,19 @@ AgentState, ApiRoute, BaseAgentConfig, + CommandTelemetry, Enforcement, HarnessContract, LocalPluginConfig, PermissionMode, + ResultSummary, SystemPromptMode, + TokenUsage, ToolNameMap, + TranscriptMessage, TurnRecord, ) -from coder_eval.pricing import register_pricing +from coder_eval.pricing import ModelPricing, register_pricing from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( @@ -56,21 +60,26 @@ "ApiRoute", "BaseAgentConfig", "CANONICAL_TOOL_NAMES", + "CommandTelemetry", "CompositeStreamCallback", "Enforcement", "EventCollector", "HarnessContract", "LocalPluginConfig", + "ModelPricing", "PermissionMode", "READ_ONLY_DENIED_TOOLS", + "ResultSummary", "SPI_VERSION", "StreamCallback", "SystemPromptMode", "TextChunkEvent", + "TokenUsage", "ToolEndEvent", "ToolEndStatus", "ToolNameMap", "ToolStartEvent", + "TranscriptMessage", "TurnClock", "TurnEndEvent", "TurnEndStatus", diff --git a/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml b/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml index d45b9d7dc..9d4acfe0b 100644 --- a/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml +++ b/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml @@ -3,10 +3,6 @@ description: A canonical task exercising every C2 mapping row at once (golden fi — do not edit casually). agent: type: claude-code - permission_mode: acceptEdits - ignore_patterns: [] - system_prompt_mode: append - sdk_options: {} sandbox: driver: tempdir python: diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py index 6e3775ad5..2d5a29429 100644 --- a/tests/test_execute_command.py +++ b/tests/test_execute_command.py @@ -303,3 +303,30 @@ def test_execute_help_explains_the_refused_flags() -> None: output = _strip_ansi(result.output) for flag in _DELIBERATELY_ABSENT_FROM_EXECUTE: assert flag in output, f"execute's help should explain why {flag} is unavailable" + + +@pytest.mark.parametrize("kind", ["pi", "codex", "none"]) +async def test_staged_task_reloads_without_claiming_model_defaults(tmp_path: Path, kind: str) -> None: + """The container reloads task.yaml and re-runs the harness contract check, so the + stage must not write a model default (permission_mode) as if a layer had set it.""" + from coder_eval.isolation.docker_runner import DockerRunner + from coder_eval.models import ResolvedTask, TaskDefinition + from coder_eval.orchestration.harness_contract import validate_harness_contract + from coder_eval.orchestration.task_loader import load_task + + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt=None if kind == "none" else "p", + agent={"type": kind}, + sandbox={"driver": "docker"}, + success_criteria=[{"type": "file_exists", "path": "x.txt", "description": "x"}], + ) + rt = ResolvedTask( + task=task, task_file=tmp_path / "t.yaml", run_dir=tmp_path / "run", variant_id="default", original_task_id="t" + ) + staged = tmp_path / "input" + staged.mkdir() + await DockerRunner(rt)._stage_inputs(staged) + reloaded, _ = load_task(staged / "task.yaml") + validate_harness_contract(reloaded) diff --git a/tests/test_harbor_packager.py b/tests/test_harbor_packager.py index 22038a52a..a7be38c1d 100644 --- a/tests/test_harbor_packager.py +++ b/tests/test_harbor_packager.py @@ -173,6 +173,23 @@ def test_never_sets_agent_type_none(self, tmp_path: Path) -> None: emitted = yaml.safe_load((out_dir / "tests" / "task.yaml").read_text(encoding="utf-8")) assert emitted["agent"]["type"] != "none" + @pytest.mark.parametrize("kind", ["pi", "codex"]) + def test_reloaded_agent_config_passes_the_harness_contract(self, tmp_path: Path, kind: str) -> None: + """`coder-eval execute` re-runs the contract check on this file, so a model default + (permission_mode) must not be written as if a layer had set it.""" + from coder_eval.orchestration.harness_contract import validate_harness_contract + from coder_eval.orchestration.task_loader import load_task + + task_file = _write_task(tmp_path, {"agent": {"type": kind}}) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + assert "permission_mode" not in emitted["agent"] + reloaded, _ = load_task(out_dir / "environment" / "task.yaml") + validate_harness_contract(reloaded) + def test_reloads_as_a_valid_task_definition(self, tmp_path: Path) -> None: task_file = _write_task(tmp_path) out_dir = tmp_path / "out" diff --git a/tests/test_harness_contract.py b/tests/test_harness_contract.py index 35c3f722a..ba8829cc2 100644 --- a/tests/test_harness_contract.py +++ b/tests/test_harness_contract.py @@ -305,9 +305,9 @@ def test_unset_default_permission_mode_passes(self) -> None: assert "permission_mode" not in task.agent.model_fields_set # type: ignore[union-attr] validate_harness_contract(task) - def test_explicit_empty_allowlist_is_set(self) -> None: - with pytest.raises(HarnessContractError, match=r"agent\.allowed_tools"): - validate_harness_contract(_task(AgentKind.CODEX, allowed_tools=[])) + @pytest.mark.parametrize("field", ["allowed_tools", "disallowed_tools"]) + def test_an_empty_tool_list_restricts_nothing_so_it_is_not_set(self, field: str) -> None: + validate_harness_contract(_task(AgentKind.CODEX, **{field: []})) def test_null_plugins_from_default_is_not_set(self) -> None: default = ExperimentDefinition( diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index b2ad525cd..2a9d42acf 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -921,6 +921,15 @@ async def test_inherited_wildcard_allow_cannot_outrank_our_allowlist(self, patch assert rules[0] == ("webfetch", "allow") assert rules[1] == ("*", "deny") + async def test_an_allowlist_keeps_a_host_rule_for_a_non_tool_permission(self, patch_exec, tmp_path, monkeypatch): + monkeypatch.setenv("OPENCODE_CONFIG_CONTENT", json.dumps({"permission": {"external_directory": "deny"}})) + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(_agent(allowed_tools=["Read"]), tmp_path) + rules = json.loads(captured["kwargs"]["env"]["OPENCODE_CONFIG_CONTENT"])["permission"] + assert rules["external_directory"] == "deny" + assert rules["doom_loop"] == "allow" + assert next(iter(rules)) == "external_directory" + async def test_no_prompt_writes_no_file(self, patch_exec, tmp_path): captured = patch_exec(_FakeProcess(HAPPY_STREAM)) agent = _agent() From 3039db673c494c69b2dcf339b346177ff576e452 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 10:49:33 -0700 Subject: [PATCH 10/12] chore: defer three harness candidates from the harness contract plan Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index e550452ad..d5d28a553 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -962,3 +962,6 @@ re-derive from scratch. leaves them stale with nothing failing. Needs a backtick-path extractor scoped to one section, which is the narrow case of the prose-path candidate above. — caught during the reports consolidation rebase. +- [ ] A `TaskDefinition` serialized for a later reload (docker `_stage_inputs`, Harbor `environment/task.yaml`) must dump `agent` with `exclude_unset=True`, or the reload marks model defaults as set and the harness contract check rejects the task — two round-trip tests guard today's two sites, but nothing flags a third `task.model_dump(` written for reload; needs a call-site classifier, not a name match — caught in the harness-contract final review. +- [ ] A real-SDK Antigravity policy test: run `policy.enforce(agent._policies(real_policy))` to prove deny-beats-allow and `finish` approval against the installed SDK instead of a SimpleNamespace fake — nothing exercises the SDK's own bucket precedence; needs study of the hook-evaluation API — caught in the harness-contract Phase 3 review. +- [ ] OpenCode: warn when an inherited `OPENCODE_CONFIG_CONTENT` `permission` / `instructions` value is not a dict / list and is replaced — today it is dropped silently; small, but needs a decision on warn vs. keep — caught in the harness-contract Phase 3 review. From 4b0fabf921a6f94253efb432d4bc714107f05116 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 11:00:40 -0700 Subject: [PATCH 11/12] fix(agents): four review findings in the turn cap, the drain retry, teardown and the early-stop ceiling - Pi and OpenCode stop at the (N+1)th turn_start / step_start before it is counted or emitted, so max_turns=N records N turns (it recorded N+1). - Antigravity retries a receive_steps() RuntimeError only when no step was pulled yet; a later error is a real failure and is no longer re-pulled. - A failed Antigravity harness teardown is logged instead of swallowed silently. - The early-stop ceiling fails closed at zero armed weight instead of dividing by zero, matching armed_criteria_passed. Co-Authored-By: Claude Opus 5 (1M context) --- src/coder_eval/agents/antigravity_agent.py | 25 ++++++--- src/coder_eval/agents/opencode_agent.py | 16 +++--- src/coder_eval/agents/pi_agent.py | 14 ++--- src/coder_eval/orchestration/early_stop.py | 3 ++ tests/test_antigravity_agent.py | 63 ++++++++++++++++++++++ tests/test_early_stop.py | 9 ++++ tests/test_opencode_agent.py | 11 ++++ tests/test_pi_agent.py | 13 +++++ 8 files changed, 136 insertions(+), 18 deletions(-) diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 86320ea56..be76208c6 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -424,14 +424,18 @@ async def _drain( A cooperative-stop ``break`` can leave the SDK connection "receiving" for a short bounded window, and the NEXT ``receive_steps()`` call raises ``RuntimeError`` inside it. The retry below yields an event-loop turn for - the already-scheduled generator finalizer to run. + the already-scheduled generator finalizer to run. Only an error raised + before this attempt pulled a step is retried; a later one is a real failure, + and retrying it would pull and emit the same steps again. Rationale: .claude/notes/agents.md § The receive_steps re-entrancy window """ for attempt in range(_RECEIVE_STEPS_REENTRY_RETRIES): + pulled = False try: async with contextlib.aclosing(conversation.receive_steps()) as steps: async for step in steps: + pulled = True state.process_step(step) if should_stop is not None and should_stop(): state.stopped_early_hit = True @@ -449,7 +453,7 @@ async def _drain( break return except RuntimeError: - if attempt == _RECEIVE_STEPS_REENTRY_RETRIES - 1: + if pulled or attempt == _RECEIVE_STEPS_REENTRY_RETRIES - 1: raise self._log.debug( "receive_steps() re-entrancy guard still set from a prior drain; retrying (attempt %d)", @@ -695,13 +699,22 @@ def _conversation_or_none(self) -> Any: return None async def _teardown(self) -> None: - """Close the SDK Agent context (reaps the localharness subprocess).""" + """Close the SDK Agent context (reaps the localharness subprocess). + + Never raises. A failed close is logged: the exit stack has already popped + its callbacks, so it cannot be retried, and the harness may still be running. + """ stack = self._exit_stack self._exit_stack = None self._sdk_agent = None - if stack is not None: - with contextlib.suppress(Exception): - await stack.aclose() + if stack is None: + return + try: + await stack.aclose() + except Exception: + self._log.warning( + "Antigravity harness teardown failed; the harness process may still be running", exc_info=True + ) class _AntigravityTurnState: diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index 3d7122627..5eb935631 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -1120,10 +1120,9 @@ def emit(event: StreamEvent) -> None: if not line: break - self._handle_line(line, state) + self._handle_line(line, state, max_turns=max_turns) - if max_turns is not None and state.step_count > max_turns: - state.max_turns_exhausted = True + if state.max_turns_exhausted: await self.kill() break if should_stop is not None and should_stop(): @@ -1309,8 +1308,11 @@ async def _timeout_turn( finally: self._capture_partial_turn(collector) - def _handle_line(self, line: bytes, state: _OpenCodeTurnState) -> None: - """Parse one nd-JSON line and dispatch it. Never raises on bad input.""" + def _handle_line(self, line: bytes, state: _OpenCodeTurnState, *, max_turns: int | None = None) -> None: + """Parse one nd-JSON line and dispatch it. Never raises on bad input. + + A ``step_start`` past ``max_turns`` sets ``state.max_turns_exhausted`` instead of opening a step. + """ raw = line.decode("utf-8", "replace").strip() if not raw: return @@ -1338,7 +1340,9 @@ def _handle_line(self, line: bytes, state: _OpenCodeTurnState) -> None: state.thread_id = session_id self._session_id = session_id - if event_type == _STEP_START: + if event_type == _STEP_START and max_turns is not None and state.step_count >= max_turns: + state.max_turns_exhausted = True + elif event_type == _STEP_START: state.on_step_start(part) elif event_type == _TEXT: state.on_text(part) diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 43aaf81a5..2c89bbadd 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -995,10 +995,9 @@ def emit(event: StreamEvent) -> None: if not line: break - self._handle_line(line, state) + self._handle_line(line, state, max_turns=max_turns) - if max_turns is not None and state.turn_count > max_turns: - state.max_turns_exhausted = True + if state.max_turns_exhausted: await self.kill() break if should_stop is not None and should_stop(): @@ -1152,11 +1151,12 @@ async def _timeout_turn( finally: self._capture_partial_turn(collector) - def _handle_line(self, line: bytes, state: _PiTurnState) -> None: + def _handle_line(self, line: bytes, state: _PiTurnState, *, max_turns: int | None = None) -> None: """Parse one nd-JSON line and dispatch it. Never raises on bad input. ``agent_end`` is NOT terminal — only ``agent_settled`` / stdout EOF is — so - it is recognized, ignored, and the read loop keeps going. + it is recognized, ignored, and the read loop keeps going. A ``turn_start`` + past ``max_turns`` sets ``state.max_turns_exhausted`` instead of opening a turn. """ raw = line.decode("utf-8", "replace").strip() if not raw: @@ -1177,7 +1177,9 @@ def _handle_line(self, line: bytes, state: _PiTurnState) -> None: elif len(state.unrecognized_types) < _MAX_UNRECOGNIZED_TYPES: state.unrecognized_types.add(event_type or "") - if event_type == "turn_start": + if event_type == "turn_start" and max_turns is not None and state.turn_count >= max_turns: + state.max_turns_exhausted = True + elif event_type == "turn_start": state.on_turn_start() elif event_type == "message_update": state.on_message_update(obj) diff --git a/src/coder_eval/orchestration/early_stop.py b/src/coder_eval/orchestration/early_stop.py index 69964c260..d74e2c310 100644 --- a/src/coder_eval/orchestration/early_stop.py +++ b/src/coder_eval/orchestration/early_stop.py @@ -357,6 +357,9 @@ def _ceiling(self, verdicts: list[LiveVerdict]) -> float: gate_threshold`` means the gate is mathematically guaranteed to fail no matter how the trajectory continues. """ + if self._armed_weight <= 0.0: + # Fails closed, as `armed_criteria_passed` does for the same unreachable case. + return 0.0 return sum(c.weight for (c, _checker), v in zip(self._armed, verdicts, strict=True) if v != "fail") / ( self._armed_weight ) diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index cfe8b8616..d7a2d368f 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -1087,6 +1087,69 @@ async def test_communicate_recovers_from_transient_reentrancy_after_cooperative_ assert tr.agent_output == "second turn" +async def test_a_failed_harness_teardown_is_logged_and_stop_still_completes(caplog): + """The SDK's exit stack pops each callback before running it, so a failed close + cannot be retried; it must at least be visible, since the harness may be left running.""" + import logging + from contextlib import AsyncExitStack + + closes = 0 + + async def _failing_close(*_exc: object) -> None: + nonlocal closes + closes += 1 + raise OSError("harness did not exit") + + stack = AsyncExitStack() + stack.push_async_exit(_failing_close) + agent = AntigravityAgent(parse_agent_config(type="antigravity")) + agent._exit_stack = stack + agent._sdk_agent = SimpleNamespace(conversation=None, is_started=True) + + with caplog.at_level(logging.WARNING): + await agent.stop() + await agent.stop() + + assert closes == 1 + assert "harness did not exit" in caplog.text + assert agent.get_state() == agent_module.AgentState.FINISHED + + +async def test_a_runtime_error_after_a_step_is_not_retried_as_reentrancy(monkeypatch): + """Only an error raised before the first step is the re-entrancy window. A + RuntimeError while processing a pulled step is a real failure: retrying it would + re-pull the stream and emit the same steps again.""" + from coder_eval.errors import AgentCrashError + + conversation_pulls = 0 + + class _Conversation: + last_response = "" + + async def send(self, prompt, **kwargs): + return None + + async def receive_steps(self): + nonlocal conversation_pulls + conversation_pulls += 1 + yield _step("TEXT_RESPONSE", "DONE", content="done", complete=True, usage=_usage(5, 0, 1, 0)) + + async def cancel(self): + return None + + def _boom(self, step): + raise RuntimeError("reducer bug") + + monkeypatch.setattr(agent_module._AntigravityTurnState, "process_step", _boom) + agent = AntigravityAgent(parse_agent_config(type="antigravity")) + agent.working_directory = __import__("pathlib").Path("/tmp") + agent._sdk_agent = SimpleNamespace(conversation=_Conversation(), is_started=True) + + with pytest.raises(AgentCrashError, match="reducer bug"): + await agent.communicate("do it") + assert conversation_pulls == 1 + + async def test_communicate_poll_budget_exhausted_finalizes_via_existing_timeout_path(monkeypatch): """A watchdog timeout landing during the poll loop's re-drain (not the first drain) must surface as TurnTimeoutError via the SAME existing exception diff --git a/tests/test_early_stop.py b/tests/test_early_stop.py index 89787cb46..a5359ad42 100644 --- a/tests/test_early_stop.py +++ b/tests/test_early_stop.py @@ -1719,6 +1719,15 @@ def test_ceiling_bound_fires_fail_stop_when_high_weight_criterion_fails(self) -> assert watcher.info is not None assert watcher.info.reason == EarlyStopReason.CRITERION_FAILED + def test_zero_armed_weight_fails_closed_instead_of_dividing_by_zero(self) -> None: + """The model rejects weight=0 on an armed criterion; a copy that skips validation + must still not crash the watcher, and must agree with the final gate (closed).""" + armed = _skill_crit("date-teller", "date-teller", stop_on_fail=True).model_copy(update={"weight": 0.0}) + watcher = EarlyStopWatcher( + "t", [(armed, _watcher([_skill_crit("x", "x", stop_on_fail=True)])._armed[0][1])], max_turns=20 + ) + assert watcher._ceiling(["undecided"]) == 0.0 + def test_default_gate_threshold_fires_fail_stop_on_any_weight(self) -> None: # At the default gate_threshold=1.0, even the low-weight criterion's # failure alone must still fire — byte-for-byte the pre-weighting rule. diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 2a9d42acf..d1604441a 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -1516,6 +1516,17 @@ async def test_the_deciding_step_is_kept_whole(self, patch_exec, tmp_path): assert usage.cache_creation_input_tokens == 5 assert usage.cache_read_input_tokens == 10 + async def test_the_step_past_the_cap_is_never_admitted(self, patch_exec, tmp_path): + """The cap stops at the (N+1)th `step_start`, before it is counted or emitted.""" + patch_exec(_FakeProcess(HAPPY_STREAM)) + recorder = _EventRecorder() + record = await _run(_agent(), tmp_path, max_turns=1, stream_callback=recorder) + + assert record.max_turns_exhausted is True + assert record.assistant_turn_count == 1 + assert len([e for e in recorder.events if isinstance(e, TurnStartEvent)]) == 1 + assert len([e for e in recorder.events if isinstance(e, TurnEndEvent)]) == 1 + class _HangingProcess(_FakeProcess): """Emits nothing and never exits until it is signaled. diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 4dcee023b..391e5048d 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -485,6 +485,19 @@ async def test_the_deciding_turn_is_kept_whole(self, patch_exec, tmp_path): assert usage.uncached_input_tokens == 406 # turn 1's input exactly assert usage.output_tokens == 77 # 69 + 8 reasoning + async def test_the_turn_past_the_cap_is_never_admitted(self, patch_exec, tmp_path): + """The cap stops at the (N+1)th `turn_start`, before it is counted or emitted.""" + patch_exec(_FakeProcess(HAPPY_STREAM)) + recorder = _EventRecorder() + record = await _run(_agent(), tmp_path, max_turns=1, stream_callback=recorder) + + assert record.max_turns_exhausted is True + assert record.assistant_turn_count == 1 + starts = [e for e in recorder.events if isinstance(e, TurnStartEvent)] + ends = [e for e in recorder.events if isinstance(e, TurnEndEvent)] + assert len(starts) == 1 + assert len(ends) == 1 + async def test_no_cap_is_uncapped(self, patch_exec, tmp_path): patch_exec(_FakeProcess(HAPPY_STREAM)) record = await _run(_agent(), tmp_path) From 474638fcfd08a3b5e31f465f6a533b3b99c644bc Mon Sep 17 00:00:00 2001 From: uipreliga Date: Wed, 16 Sep 2026 11:44:42 -0700 Subject: [PATCH 12/12] fix(limits): clarify docstring for RunLimits class to better describe task caps Co-Authored-By: Claude --- src/coder_eval/models/limits.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/coder_eval/models/limits.py b/src/coder_eval/models/limits.py index 48bdb48c2..b06029f18 100644 --- a/src/coder_eval/models/limits.py +++ b/src/coder_eval/models/limits.py @@ -16,13 +16,15 @@ class RunLimits(BaseModel): - """Run-time caps that abort a task when exceeded. + """Run-time caps on a task. Unifies structural caps (max_turns, task_timeout, turn_timeout) and - budget caps (tokens, USD). Budget caps are checked after each completed - agent turn and are cumulative across all turns of a single task; they - apply to the subject agent only — judge and simulator token spend are - not counted. + budget caps (tokens, USD). Structural caps stop the task. Budget caps are + checked after each completed agent turn and are cumulative across all + turns of a single task: a single-iteration task finishes and is then + marked over budget, and a dialog stops after the turn that crossed the + budget. Budgets apply to the subject agent only — judge and simulator + token spend are not counted. Any subset of fields is valid; an empty block is legal. """