diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc5170..98f44af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,11 +15,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 absent or rejected structured output. - All built-in adapters and public fake runtimes now validate returned values against `output_schema` locally, including textual JSON fallbacks. +- `AgentResult.is_success` provides one canonical success predicate. ### Changed - Pydantic structured-output parsing now requests strict validation, so values such as `"42"` no longer coerce into integer fields. +- BREAKING: task and value-object constructors now reject blank identities, + non-positive execution hints, invalid budgets, ambiguous session inputs, + duplicate MCP names, and conflicting tool filters. +- BREAKING: unknown `Usage` values, including cost, are `None` rather than zero; + an explicit provider-reported zero remains `0`/`0.0`. ### Fixed diff --git a/README.md b/README.md index 5f79f9d..a2c0d63 100644 --- a/README.md +++ b/README.md @@ -143,9 +143,11 @@ silently dropping it. `AgentResult` returns output, finish reason (see `FinishReason`), locally validated structured output, usage, cost, session id, tool-call audits, and -provider metadata. `parsed_output_available` distinguishes an absent payload -from valid JSON `null` (both use `parsed_output=None`). `artifacts` is a reserved -field: no built-in runtime populates it yet, so it is always an empty tuple today. +provider metadata. `is_success` is true only for a natural, error-free +completion. Unknown usage and cost fields are `None`; reported zero remains +distinct. `parsed_output_available` distinguishes an absent payload from valid +JSON `null` (both use `parsed_output=None`). `artifacts` is a reserved field: no +built-in runtime populates it yet, so it is always an empty tuple today. ## Docs diff --git a/docs/api-stability.md b/docs/api-stability.md index 278090b..5c10907 100644 --- a/docs/api-stability.md +++ b/docs/api-stability.md @@ -33,6 +33,13 @@ import the names from the top-level package instead. convention and compare by value. The wrappers remain plain `dict` subclasses, so `dataclasses.asdict`, `pickle`, `copy.deepcopy`, and JSON serialization keep working. +- **Constructors enforce domain invariants.** Goals and identifiers are + non-blank; execution hints are positive; counts and monetary values are + finite and non-negative; session start and resume inputs are mutually + exclusive; MCP server names and tool filters are unambiguous. +- **Unknown telemetry is not zero.** Every `Usage` field is nullable. `None` + means the provider did not report a value; `0` and `0.0` mean it explicitly + reported zero. - **Unsupported inputs raise, they are not dropped.** An adapter that cannot honor a task field raises `UnsupportedTaskInputError`; the one exception is vendor-option drift, which is recorded in diff --git a/docs/providers.md b/docs/providers.md index c7ac3fa..3683f9a 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -38,6 +38,10 @@ vocabulary (for example `claude-agent-sdk` 0.2.x accepts `low`/`medium`/`high`/`xhigh`/`max`), and an SDK too old to accept `effort` at all records the drop in `AgentResult.metadata["dropped_options"]`. +Usage fields are `None` when a provider omits its usage breakdown or cost. +Provider-reported zeros remain numeric zero, so callers can distinguish a free +or zero-token run from missing telemetry. + Claude uses the `claude-agent-sdk` package and maps working directory, permissions, filesystem access (a `READ_ONLY` filesystem forces `plan` mode), MCP servers, sessions, structured output, tool allow/deny lists, runtime diff --git a/src/agent_runtime_kit/_types.py b/src/agent_runtime_kit/_types.py index b089da5..5d45eb3 100644 --- a/src/agent_runtime_kit/_types.py +++ b/src/agent_runtime_kit/_types.py @@ -5,6 +5,7 @@ from collections.abc import Mapping from dataclasses import dataclass, field from enum import Enum +from math import isfinite from pathlib import Path from typing import Any, Generic, NoReturn, Protocol, TypeVar, cast, runtime_checkable from uuid import uuid4 @@ -81,6 +82,44 @@ def _coerce_enum(enum_cls: type[_EnumT], value: Any, field_name: str) -> _EnumT: raise ValueError(f"invalid {field_name} {value!r}; valid values: {valid}") from None +def _require_nonblank(value: str, field_name: str) -> None: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field_name} must be a non-empty string") + + +def _require_unique_nonblank(values: tuple[str, ...], field_name: str) -> None: + for value in values: + _require_nonblank(value, field_name) + duplicates = sorted({value for value in values if values.count(value) > 1}) + if duplicates: + raise ValueError(f"{field_name} contains duplicates: {duplicates}") + + +def _tuple_value(value: Any, field_name: str) -> tuple[Any, ...]: + if isinstance(value, (str, bytes)): + raise ValueError(f"{field_name} must be a sequence, not a scalar string") + try: + return tuple(value) + except TypeError as exc: + raise ValueError(f"{field_name} must be an iterable") from exc + + +def _validate_optional_count(value: int | None, field_name: str) -> None: + if value is None: + return + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{field_name} must be a non-negative integer or None") + + +def _validate_optional_amount(value: float | None, field_name: str) -> None: + if value is None: + return + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{field_name} must be a non-negative finite number or None") + if value < 0 or not isfinite(float(value)): + raise ValueError(f"{field_name} must be a non-negative finite number or None") + + class AgentRuntimeKind(str, Enum): """Supported runtime families.""" @@ -257,6 +296,17 @@ class McpServerConfig: env: Mapping[str, str] = field(default_factory=dict) def __post_init__(self) -> None: + _require_nonblank(self.name, "McpServerConfig.name") + _require_nonblank(self.command, "McpServerConfig.command") + object.__setattr__(self, "args", _tuple_value(self.args, "McpServerConfig.args")) + if not all(isinstance(arg, str) for arg in self.args): + raise ValueError("McpServerConfig.args must contain only strings") + if not isinstance(self.env, Mapping): + raise ValueError("McpServerConfig.env must be a mapping") + for key, value in self.env.items(): + _require_nonblank(key, "McpServerConfig.env key") + if not isinstance(value, str): + raise ValueError("McpServerConfig.env values must be strings") object.__setattr__(self, "env", _freeze_mapping(self.env)) @@ -285,6 +335,25 @@ def __post_init__(self) -> None: "filesystem", _coerce_enum(FilesystemAccess, self.filesystem, "filesystem"), ) + object.__setattr__( + self, + "allowed_tools", + _tuple_value(self.allowed_tools, "PermissionProfile.allowed_tools"), + ) + object.__setattr__( + self, + "disallowed_tools", + _tuple_value(self.disallowed_tools, "PermissionProfile.disallowed_tools"), + ) + _require_unique_nonblank(self.allowed_tools, "PermissionProfile.allowed_tools") + _require_unique_nonblank(self.disallowed_tools, "PermissionProfile.disallowed_tools") + overlap = sorted(set(self.allowed_tools) & set(self.disallowed_tools)) + if overlap: + raise ValueError( + "PermissionProfile cannot both allow and disallow tools: " + ", ".join(overlap) + ) + if self.network is not None and not isinstance(self.network, bool): + raise ValueError("PermissionProfile.network must be bool or None") @dataclass(frozen=True) @@ -298,6 +367,14 @@ class ToolCallAudit: duration_ms: int = 0 def __post_init__(self) -> None: + _require_nonblank(self.tool_name, "ToolCallAudit.tool_name") + _require_nonblank(self.status, "ToolCallAudit.status") + if ( + isinstance(self.duration_ms, bool) + or not isinstance(self.duration_ms, int) + or self.duration_ms < 0 + ): + raise ValueError("ToolCallAudit.duration_ms must be a non-negative integer") object.__setattr__(self, "arguments", _freeze_mapping(self.arguments)) @@ -310,6 +387,8 @@ class ArtifactRef: metadata: Mapping[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: + _require_nonblank(self.uri, "ArtifactRef.uri") + _require_nonblank(self.kind, "ArtifactRef.kind") object.__setattr__(self, "metadata", _freeze_mapping(self.metadata)) @@ -325,6 +404,14 @@ class SessionResumeState: session_id: str transcript: tuple[Any, ...] = () + def __post_init__(self) -> None: + _require_nonblank(self.session_id, "SessionResumeState.session_id") + object.__setattr__( + self, + "transcript", + _tuple_value(self.transcript, "SessionResumeState.transcript"), + ) + @dataclass(frozen=True) class Usage: @@ -332,18 +419,24 @@ class Usage: ``input_tokens`` counts prompt tokens excluding Anthropic-style cache reads and cache creation, which are reported separately in ``cache_read_tokens`` and - ``cache_creation_tokens``. ``total_tokens`` is the vendor-reported total when the - runtime provides one, and ``None`` when it does not (so "unknown" is - distinguishable from zero). ``cost_usd`` is ``0.0`` when the provider reports no - cost. + ``cache_creation_tokens``. Every field is ``None`` when unknown, so a reported + zero remains distinguishable from missing telemetry. """ - input_tokens: int = 0 - output_tokens: int = 0 - cache_read_tokens: int = 0 - cache_creation_tokens: int = 0 + input_tokens: int | None = None + output_tokens: int | None = None + cache_read_tokens: int | None = None + cache_creation_tokens: int | None = None total_tokens: int | None = None - cost_usd: float = 0.0 + cost_usd: float | None = None + + def __post_init__(self) -> None: + _validate_optional_count(self.input_tokens, "Usage.input_tokens") + _validate_optional_count(self.output_tokens, "Usage.output_tokens") + _validate_optional_count(self.cache_read_tokens, "Usage.cache_read_tokens") + _validate_optional_count(self.cache_creation_tokens, "Usage.cache_creation_tokens") + _validate_optional_count(self.total_tokens, "Usage.total_tokens") + _validate_optional_amount(self.cost_usd, "Usage.cost_usd") @dataclass(frozen=True) @@ -375,6 +468,30 @@ class AgentTask: metadata: Mapping[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: + _require_nonblank(self.goal, "AgentTask.goal") + _require_nonblank(self.task_id, "AgentTask.task_id") + if self.model is not None: + _require_nonblank(self.model, "AgentTask.model") + if self.reasoning_effort is not None: + _require_nonblank(self.reasoning_effort, "AgentTask.reasoning_effort") + if isinstance(self.sdk_executions, bool) or not isinstance(self.sdk_executions, int): + raise ValueError("AgentTask.sdk_executions must be a positive integer") + if self.sdk_executions < 1: + raise ValueError("AgentTask.sdk_executions must be a positive integer") + _validate_optional_amount(self.budget_usd, "AgentTask.budget_usd") + if self.session_id is not None: + _require_nonblank(self.session_id, "AgentTask.session_id") + if self.session_id is not None and self.resume_from is not None: + raise ValueError("AgentTask.session_id and resume_from are mutually exclusive") + object.__setattr__( + self, + "mcp_servers", + _tuple_value(self.mcp_servers, "AgentTask.mcp_servers"), + ) + server_names = tuple(server.name for server in self.mcp_servers) + duplicates = sorted({name for name in server_names if server_names.count(name) > 1}) + if duplicates: + raise ValueError(f"AgentTask.mcp_servers contains duplicate names: {duplicates}") object.__setattr__(self, "metadata", _freeze_mapping(self.metadata)) if self.output_schema is not None: # Local import avoids an import cycle: _schema's typed errors import @@ -405,16 +522,39 @@ class AgentResult: parsed_output_available: bool = False def __post_init__(self) -> None: + _require_nonblank(self.finish_reason, "AgentResult.finish_reason") + if self.error is not None: + _require_nonblank(self.error, "AgentResult.error") + if self.finish_reason == FinishReason.DONE: + raise ValueError("AgentResult.error requires a non-success finish_reason") + if isinstance(self.rounds, bool) or not isinstance(self.rounds, int) or self.rounds < 0: + raise ValueError("AgentResult.rounds must be a non-negative integer") + object.__setattr__( + self, + "tool_calls", + _tuple_value(self.tool_calls, "AgentResult.tool_calls"), + ) + object.__setattr__( + self, + "artifacts", + _tuple_value(self.artifacts, "AgentResult.artifacts"), + ) object.__setattr__(self, "metadata", _freeze_mapping(self.metadata)) if self.parsed_output is not None and not self.parsed_output_available: object.__setattr__(self, "parsed_output_available", True) @property - def cost_usd(self) -> float: + def cost_usd(self) -> float | None: """Return the reported task cost in USD.""" return self.usage.cost_usd + @property + def is_success(self) -> bool: + """Whether the runtime completed naturally without an error.""" + + return self.finish_reason == FinishReason.DONE and self.error is None + _ParsedT = TypeVar("_ParsedT") diff --git a/src/agent_runtime_kit/adapters/_common.py b/src/agent_runtime_kit/adapters/_common.py index eb15a39..05da845 100644 --- a/src/agent_runtime_kit/adapters/_common.py +++ b/src/agent_runtime_kit/adapters/_common.py @@ -6,6 +6,7 @@ import os from collections.abc import Iterable, Mapping from importlib import metadata, util +from math import isfinite from typing import Any from agent_runtime_kit._errors import UnsupportedTaskInputError @@ -259,13 +260,27 @@ def field_value(value: Any, name: str, default: Any = None) -> Any: return getattr(value, name, default) -def optional_int(value: Any) -> int: - """Coerce a vendor-reported value to ``int``, treating ``None``/junk as 0.""" +def optional_int(value: Any) -> int | None: + """Coerce a non-negative vendor count, preserving unknown as None.""" + if value is None or isinstance(value, bool): + return None + if isinstance(value, float): + if not isfinite(value) or not value.is_integer(): + return None + if isinstance(value, str) and not value.strip(): + return None try: - return int(value or 0) - except (TypeError, ValueError): - return 0 + parsed = int(value) + except (TypeError, ValueError, OverflowError): + return None + if not isinstance(value, (int, float, str)): + try: + if value != parsed: + return None + except Exception: + return None + return parsed if parsed >= 0 else None def optional_str(value: Any) -> str | None: diff --git a/src/agent_runtime_kit/adapters/antigravity.py b/src/agent_runtime_kit/adapters/antigravity.py index a9e16ba..abc8216 100644 --- a/src/agent_runtime_kit/adapters/antigravity.py +++ b/src/agent_runtime_kit/adapters/antigravity.py @@ -918,14 +918,24 @@ def _default_data_dir() -> Path: def _usage_from(value: Any) -> Usage: + if value is None: + return Usage() prompt_tokens = optional_int(getattr(value, "prompt_token_count", None)) output_tokens = optional_int(getattr(value, "candidates_token_count", None)) thoughts = optional_int(getattr(value, "thoughts_token_count", None)) cache_read = optional_int(getattr(value, "cached_content_token_count", None)) total = optional_int(getattr(value, "total_token_count", None)) return Usage( - input_tokens=max(prompt_tokens - cache_read, 0), - output_tokens=output_tokens + thoughts, + input_tokens=( + max(prompt_tokens - cache_read, 0) + if prompt_tokens is not None and cache_read is not None + else None + ), + output_tokens=( + output_tokens + thoughts + if output_tokens is not None and thoughts is not None + else None + ), cache_read_tokens=cache_read, total_tokens=total, ) diff --git a/src/agent_runtime_kit/adapters/claude.py b/src/agent_runtime_kit/adapters/claude.py index b8d541e..68959d6 100644 --- a/src/agent_runtime_kit/adapters/claude.py +++ b/src/agent_runtime_kit/adapters/claude.py @@ -7,6 +7,7 @@ import logging import os from collections.abc import AsyncIterator, Iterable, Mapping +from math import isfinite from typing import Any from agent_runtime_kit._errors import AgentRuntimeUnavailableError @@ -31,6 +32,7 @@ filter_supported_kwargs, fingerprint_value, metadata_str, + optional_int, optional_str, output_schema_from, package_availability, @@ -470,8 +472,8 @@ def _translate_messages( tool_calls: list[ToolCallAudit] = [] tool_use_ids: list[str | None] = [] usage = Usage() - cost_usd = 0.0 - session_id = task.session_id + cost_usd: float | None = None + session_id = _conversation_id(task) rounds = 0 error: str | None = None finish_reason = "done" @@ -495,7 +497,15 @@ def _translate_messages( if result_text and not content_parts: content_parts.append(str(result_text)) structured_output = field_value(message, "structured_output", structured_output) - cost_usd = float(field_value(message, "total_cost_usd", cost_usd) or cost_usd) + reported_cost = field_value(message, "total_cost_usd") + if reported_cost is not None: + try: + candidate_cost = float(reported_cost) + except (TypeError, ValueError): + pass + else: + if candidate_cost >= 0 and isfinite(candidate_cost): + cost_usd = candidate_cost usage = _usage_from(field_value(message, "usage"), current=usage) rounds = int(field_value(message, "num_turns", rounds) or rounds) session_id = optional_str(field_value(message, "session_id")) or session_id @@ -682,11 +692,20 @@ def _assistant_content( def _usage_from(value: Any, *, current: Usage) -> Usage: if not isinstance(value, Mapping): return current - input_tokens = int(value.get("input_tokens") or current.input_tokens) - output_tokens = int(value.get("output_tokens") or current.output_tokens) - cache_creation = int(value.get("cache_creation_input_tokens") or current.cache_creation_tokens) - cache_read = int(value.get("cache_read_input_tokens") or current.cache_read_tokens) - total = input_tokens + output_tokens + cache_creation + cache_read + input_tokens = optional_int(value.get("input_tokens")) + output_tokens = optional_int(value.get("output_tokens")) + cache_creation = optional_int(value.get("cache_creation_input_tokens")) + cache_read = optional_int(value.get("cache_read_input_tokens")) + input_tokens = current.input_tokens if input_tokens is None else input_tokens + output_tokens = current.output_tokens if output_tokens is None else output_tokens + cache_creation = current.cache_creation_tokens if cache_creation is None else cache_creation + cache_read = current.cache_read_tokens if cache_read is None else cache_read + components = (input_tokens, output_tokens, cache_creation, cache_read) + total = ( + sum(component for component in components if component is not None) + if all(component is not None for component in components) + else None + ) return Usage( input_tokens=input_tokens, output_tokens=output_tokens, diff --git a/src/agent_runtime_kit/adapters/codex.py b/src/agent_runtime_kit/adapters/codex.py index fd71eb2..66cb5cb 100644 --- a/src/agent_runtime_kit/adapters/codex.py +++ b/src/agent_runtime_kit/adapters/codex.py @@ -584,7 +584,7 @@ def _tool_audit(item: Any) -> ToolCallAudit | None: arguments={"command": str(field_value(item, "command", ""))}, result_preview=str(field_value(item, "aggregated_output", "") or "")[:256], status=_tool_status(item), - duration_ms=optional_int(field_value(item, "duration_ms")), + duration_ms=optional_int(field_value(item, "duration_ms")) or 0, ) if item_type in {_MCP_ITEM, _DYNAMIC_ITEM}: return ToolCallAudit( @@ -592,7 +592,7 @@ def _tool_audit(item: Any) -> ToolCallAudit | None: arguments=_tool_arguments(field_value(item, "arguments")), result_preview=str(field_value(item, "result", "") or "")[:256], status=_tool_status(item), - duration_ms=optional_int(field_value(item, "duration_ms")), + duration_ms=optional_int(field_value(item, "duration_ms")) or 0, ) if item_type == _WEB_SEARCH_ITEM: return ToolCallAudit( @@ -626,7 +626,9 @@ def _codex_usage(value: Any) -> Usage: breakdown = field_value(value, "total") if breakdown is None and isinstance(value, Mapping): breakdown = value.get("total", value) - input_tokens = optional_int(field_value(breakdown, "input_tokens")) + if breakdown is None: + return Usage() + raw_input_tokens = optional_int(field_value(breakdown, "input_tokens")) output_tokens = optional_int(field_value(breakdown, "output_tokens")) cached = optional_int(field_value(breakdown, "cached_input_tokens")) total_tokens = optional_int(field_value(breakdown, "total_tokens")) @@ -634,11 +636,18 @@ def _codex_usage(value: Any) -> Usage: # (and the Antigravity adapter) excludes cache reads from input_tokens and # reports them separately. Subtract rather than double-count across the two # fields; the raw value still backs the vendor-style total fallback. + input_tokens = ( + max(raw_input_tokens - cached, 0) + if raw_input_tokens is not None and cached is not None + else None + ) + if total_tokens is None and raw_input_tokens is not None and output_tokens is not None: + total_tokens = raw_input_tokens + output_tokens return Usage( - input_tokens=max(input_tokens - cached, 0), + input_tokens=input_tokens, output_tokens=output_tokens, cache_read_tokens=cached, - total_tokens=total_tokens or input_tokens + output_tokens, + total_tokens=total_tokens, ) diff --git a/tests/test_adapter_common.py b/tests/test_adapter_common.py index 6bb978b..f32020f 100644 --- a/tests/test_adapter_common.py +++ b/tests/test_adapter_common.py @@ -10,7 +10,11 @@ OutputSchemaError, UnsupportedTaskInputError, ) -from agent_runtime_kit.adapters._common import filter_supported_kwargs, output_schema_from +from agent_runtime_kit.adapters._common import ( + filter_supported_kwargs, + optional_int, + output_schema_from, +) def test_filter_supported_kwargs_rejects_required_key_hidden_by_var_kwargs() -> None: @@ -125,3 +129,22 @@ def test_output_schema_from_validates_legacy_aliases_and_nested_mutation() -> No with pytest.raises(OutputSchemaError, match="invalid output_schema"): output_schema_from(task.output_schema, task.metadata) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (0, 0), + (1.0, 1), + ("2", 2), + (1.9, None), + (float("inf"), None), + (-1, None), + (True, None), + ("not-a-number", None), + ], +) +def test_optional_int_never_truncates_or_invents_vendor_counts( + value: object, expected: int | None +) -> None: + assert optional_int(value) == expected diff --git a/tests/test_antigravity_adapter.py b/tests/test_antigravity_adapter.py index 8bd14cf..c74cec2 100644 --- a/tests/test_antigravity_adapter.py +++ b/tests/test_antigravity_adapter.py @@ -227,6 +227,77 @@ async def test_antigravity_runtime_runs_with_injected_sdk(tmp_path: Path) -> Non assert FakeAgent.last_config.kwargs["api_key"] == "key" +@pytest.mark.asyncio +async def test_antigravity_usage_is_unknown_when_sdk_omits_metadata(tmp_path: Path) -> None: + class NoUsageAgent(FakeAgent): + async def chat(self, prompt: str) -> FakeResponse: + response = await super().chat(prompt) + response.usage_metadata = None + return response + + runtime = AntigravityAgentRuntime( + api_key="key", + data_dir=tmp_path, + agent_cls=NoUsageAgent, + config_cls=FakeConfig, + types_module=FakeTypes, + policy_module=FakePolicy, + ) + + result = await runtime.run(AgentTask(goal="x")) + + assert result.usage.input_tokens is None + assert result.usage.output_tokens is None + assert result.usage.total_tokens is None + + +@pytest.mark.asyncio +async def test_antigravity_partial_and_zero_usage_preserve_presence(tmp_path: Path) -> None: + class PartialUsage: + prompt_token_count = 5 + candidates_token_count = 7 + total_token_count = 12 + + class ZeroUsage: + prompt_token_count = 0 + candidates_token_count = 0 + thoughts_token_count = 0 + cached_content_token_count = 0 + total_token_count = 0 + + class UsageAgent(FakeAgent): + usage: Any = None + + async def chat(self, prompt: str) -> FakeResponse: + response = await super().chat(prompt) + response.usage_metadata = self.usage + return response + + UsageAgent.usage = PartialUsage() + partial_runtime = AntigravityAgentRuntime( + api_key="key", + data_dir=tmp_path, + agent_cls=UsageAgent, + config_cls=FakeConfig, + types_module=FakeTypes, + policy_module=FakePolicy, + ) + partial = await partial_runtime.run(AgentTask(goal="x")) + + assert partial.usage.input_tokens is None + assert partial.usage.output_tokens is None + assert partial.usage.cache_read_tokens is None + assert partial.usage.total_tokens == 12 + + UsageAgent.usage = ZeroUsage() + zero = await partial_runtime.run(AgentTask(goal="x")) + + assert zero.usage.input_tokens == 0 + assert zero.usage.output_tokens == 0 + assert zero.usage.cache_read_tokens == 0 + assert zero.usage.total_tokens == 0 + + @pytest.mark.asyncio async def test_antigravity_rejects_structured_output_that_misses_schema(tmp_path: Path) -> None: runtime = make_runtime(data_dir=tmp_path) diff --git a/tests/test_claude_adapter.py b/tests/test_claude_adapter.py index e79d97e..5800075 100644 --- a/tests/test_claude_adapter.py +++ b/tests/test_claude_adapter.py @@ -144,6 +144,43 @@ async def test_claude_runtime_runs_with_injected_sdk() -> None: assert sink.events[-1]["name"] == "agent.task.completed" +@pytest.mark.asyncio +async def test_claude_usage_distinguishes_unknown_from_reported_zero() -> None: + unknown_runtime = ClaudeAgentRuntime( + query_func=make_query([assistant("done"), result_message()]), + options_cls=FakeClaudeOptions, + ) + unknown = await unknown_runtime.run(AgentTask(goal="x")) + + assert unknown.usage.input_tokens is None + assert unknown.usage.cost_usd is None + + zero_runtime = ClaudeAgentRuntime( + query_func=make_query( + [ + assistant( + "done", + usage={ + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + ), + result_message(total_cost_usd=0.0), + ] + ), + options_cls=FakeClaudeOptions, + ) + reported_zero = await zero_runtime.run(AgentTask(goal="x")) + + assert reported_zero.usage.input_tokens == 0 + assert reported_zero.usage.output_tokens == 0 + assert reported_zero.usage.cache_read_tokens == 0 + assert reported_zero.usage.total_tokens == 0 + assert reported_zero.usage.cost_usd == 0.0 + + @pytest.mark.asyncio async def test_claude_runtime_can_reuse_process_until_closed() -> None: FakeClaudeClient.messages = [assistant("ok"), result_message()] @@ -518,6 +555,25 @@ async def test_claude_resume_from_session_id() -> None: assert RECORDED["options"].resume == "sess-1" +@pytest.mark.asyncio +async def test_claude_result_session_falls_back_to_resume_state_when_sdk_omits_it() -> None: + runtime = ClaudeAgentRuntime( + query_func=make_query( + [ + assistant("ok"), + {"type": "ResultMessage", "num_turns": 1}, + ] + ), + options_cls=FakeClaudeOptions, + ) + + result = await runtime.run( + AgentTask(goal="x", resume_from=SessionResumeState(session_id="resume-123")) + ) + + assert result.session_id == "resume-123" + + @pytest.mark.asyncio async def test_claude_records_dropped_options() -> None: @dataclass diff --git a/tests/test_codex_adapter.py b/tests/test_codex_adapter.py index b950d27..7055201 100644 --- a/tests/test_codex_adapter.py +++ b/tests/test_codex_adapter.py @@ -180,7 +180,13 @@ async def test_codex_rejects_structured_output_that_misses_schema() -> None: { "status": "completed", "final_response": '{"ok": "bad"}', - "usage": {"total": {"input_tokens": 4, "output_tokens": 2}}, + "usage": { + "total": { + "input_tokens": 4, + "output_tokens": 2, + "cached_input_tokens": 0, + } + }, } ) schema = { @@ -705,7 +711,12 @@ async def test_codex_usage_prefers_per_turn_over_cumulative() -> None: "final_response": "ok", "usage": { "total": {"input_tokens": 1000, "output_tokens": 2000, "total_tokens": 3000}, - "last": {"input_tokens": 4, "output_tokens": 6, "total_tokens": 10}, + "last": { + "input_tokens": 4, + "output_tokens": 6, + "cached_input_tokens": 0, + "total_tokens": 10, + }, }, } runtime = make_runtime(run_result) @@ -717,6 +728,56 @@ async def test_codex_usage_prefers_per_turn_over_cumulative() -> None: assert result.usage.total_tokens == 10 +@pytest.mark.asyncio +async def test_codex_usage_is_unknown_when_sdk_omits_breakdown() -> None: + runtime = make_runtime({"status": "completed", "final_response": "ok"}) + + result = await runtime.run(AgentTask(goal="x")) + + assert result.usage.input_tokens is None + assert result.usage.output_tokens is None + assert result.usage.total_tokens is None + assert result.usage.cost_usd is None + + +@pytest.mark.asyncio +async def test_codex_partial_usage_preserves_unknown_cache_and_normalized_input() -> None: + partial_runtime = make_runtime( + { + "status": "completed", + "final_response": "ok", + "usage": {"total": {"input_tokens": 4, "output_tokens": 2}}, + } + ) + partial = await partial_runtime.run(AgentTask(goal="x")) + + assert partial.usage.input_tokens is None + assert partial.usage.cache_read_tokens is None + assert partial.usage.output_tokens == 2 + assert partial.usage.total_tokens == 6 + + zero_runtime = make_runtime( + { + "status": "completed", + "final_response": "ok", + "usage": { + "total": { + "input_tokens": 0, + "output_tokens": 0, + "cached_input_tokens": 0, + "total_tokens": 0, + } + }, + } + ) + zero = await zero_runtime.run(AgentTask(goal="x")) + + assert zero.usage.input_tokens == 0 + assert zero.usage.output_tokens == 0 + assert zero.usage.cache_read_tokens == 0 + assert zero.usage.total_tokens == 0 + + @pytest.mark.asyncio async def test_codex_session_id_falls_back_to_task_when_sdk_omits_it() -> None: # Thread with a falsy id -> result session_id should still reflect the resume id. diff --git a/tests/test_core.py b/tests/test_core.py index c417a27..3e09dbc 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -23,8 +23,10 @@ PermissionProfile, RuntimeAvailability, RuntimeNotRegisteredError, + SessionResumeState, ToolCallAudit, UnsupportedTaskInputError, + Usage, create_default_registry, runtime_kind_value, ) @@ -109,6 +111,75 @@ def test_agent_task_model_fields_are_keyword_only() -> None: assert AgentTask("g", model="m-1", reasoning_effort="high").model == "m-1" +@pytest.mark.parametrize("goal", ["", " "]) +def test_agent_task_rejects_blank_goal(goal: str) -> None: + with pytest.raises(ValueError, match="AgentTask.goal"): + AgentTask(goal=goal) + + +@pytest.mark.parametrize("sdk_executions", [0, -1, True]) +def test_agent_task_requires_positive_sdk_execution_hint(sdk_executions: object) -> None: + with pytest.raises(ValueError, match="sdk_executions"): + AgentTask(goal="g", sdk_executions=sdk_executions) # type: ignore[arg-type] + + +@pytest.mark.parametrize("budget", [-1.0, float("inf"), float("nan"), True]) +def test_agent_task_rejects_invalid_budget(budget: object) -> None: + with pytest.raises(ValueError, match="budget_usd"): + AgentTask(goal="g", budget_usd=budget) # type: ignore[arg-type] + + +def test_agent_task_rejects_ambiguous_session_and_duplicate_mcp_names() -> None: + with pytest.raises(ValueError, match="mutually exclusive"): + AgentTask( + goal="g", + session_id="session-1", + resume_from=SessionResumeState(session_id="session-1"), + ) + + duplicate_servers = ( + McpServerConfig(name="repo", command="first"), + McpServerConfig(name="repo", command="second"), + ) + with pytest.raises(ValueError, match="duplicate names"): + AgentTask(goal="g", mcp_servers=duplicate_servers) + + +def test_value_objects_reject_blank_or_conflicting_identity() -> None: + with pytest.raises(ValueError, match="McpServerConfig.name"): + McpServerConfig(name=" ", command="mcp") + with pytest.raises(ValueError, match="McpServerConfig.command"): + McpServerConfig(name="repo", command="") + with pytest.raises(ValueError, match="SessionResumeState.session_id"): + SessionResumeState(session_id=" ") + with pytest.raises(ValueError, match="both allow and disallow"): + PermissionProfile(allowed_tools=("Read",), disallowed_tools=("Read",)) + with pytest.raises(ValueError, match="duplicates"): + PermissionProfile(allowed_tools=("Read", "Read")) + with pytest.raises(ValueError, match="network"): + PermissionProfile(network="yes") # type: ignore[arg-type] + with pytest.raises(ValueError, match="scalar string"): + PermissionProfile(allowed_tools="Read") # type: ignore[arg-type] + with pytest.raises(ValueError, match="scalar string"): + McpServerConfig(name="repo", command="mcp", args="--root") # type: ignore[arg-type] + with pytest.raises(ValueError, match="args must contain only strings"): + McpServerConfig(name="repo", command="mcp", args=(1,)) # type: ignore[arg-type] + with pytest.raises(ValueError, match="env values"): + McpServerConfig(name="repo", command="mcp", env={"PORT": 3}) # type: ignore[dict-item] + with pytest.raises(ValueError, match="scalar string"): + SessionResumeState(session_id="s", transcript="text") # type: ignore[arg-type] + + +def test_sequence_value_fields_are_canonicalized_to_tuples() -> None: + profile = PermissionProfile(allowed_tools=["Read"]) # type: ignore[arg-type] + server = McpServerConfig(name="repo", command="mcp", args=["serve"]) # type: ignore[arg-type] + resume = SessionResumeState(session_id="s", transcript=[{"x": 1}]) # type: ignore[arg-type] + + assert profile.allowed_tools == ("Read",) + assert server.args == ("serve",) + assert resume.transcript == ({"x": 1},) + + def test_agent_task_does_not_leak_caller_mapping() -> None: source = {"model": "x"} task = AgentTask(goal="g", metadata=source) @@ -205,6 +276,42 @@ def test_finish_reason_enum_compares_as_string() -> None: assert format(FinishReason.FAILED) == "failed" +def test_agent_result_success_is_explicit_and_rounds_are_non_negative() -> None: + assert AgentResult(output="ok").is_success is True + assert AgentResult(output="", finish_reason="failed").is_success is False + assert ( + AgentResult(output="", finish_reason="failed", error="vendor failed").is_success is False + ) + + with pytest.raises(ValueError, match="rounds"): + AgentResult(output="", rounds=-1) + with pytest.raises(ValueError, match="AgentResult.error"): + AgentResult(output="", finish_reason="failed", error=" ") + with pytest.raises(ValueError, match="non-success"): + AgentResult(output="", error="contradictory") + + +def test_usage_distinguishes_unknown_from_reported_zero_and_rejects_negative_values() -> None: + unknown = Usage() + reported_zero = Usage( + input_tokens=0, + output_tokens=0, + cache_read_tokens=0, + cache_creation_tokens=0, + total_tokens=0, + cost_usd=0.0, + ) + + assert unknown.input_tokens is None + assert unknown.cost_usd is None + assert reported_zero.input_tokens == 0 + assert reported_zero.cost_usd == 0.0 + with pytest.raises(ValueError, match="input_tokens"): + Usage(input_tokens=-1) + with pytest.raises(ValueError, match="cost_usd"): + Usage(cost_usd=float("inf")) + + def test_coerce_returns_namespaced_string_and_rejects_blank() -> None: assert AgentRuntimeKind.coerce("claude-agent-sdk") is AgentRuntimeKind.CLAUDE_AGENT_SDK assert AgentRuntimeKind.coerce("x-myorg-agent") == "x-myorg-agent" diff --git a/tests/test_mestre_compatibility.py b/tests/test_mestre_compatibility.py index 98d5364..ba735dc 100644 --- a/tests/test_mestre_compatibility.py +++ b/tests/test_mestre_compatibility.py @@ -21,6 +21,7 @@ def test_public_task_can_represent_mestre_vendor_lane_fields(tmp_path: Path) -> None: sink = RecordingEventSink() + start_task = AgentTask(goal="start a conversation", session_id="session-new") task = AgentTask( task_id="mestre-task", goal="execute the vendor lane task", @@ -36,7 +37,6 @@ def test_public_task_can_represent_mestre_vendor_lane_fields(tmp_path: Path) -> ), sdk_executions=2, budget_usd=1.25, - session_id="session-1", resume_from=SessionResumeState(session_id="session-1", transcript=({"x": 1},)), output_schema={"type": "object", "properties": {"ok": {"type": "boolean"}}}, metadata={ @@ -48,9 +48,12 @@ def test_public_task_can_represent_mestre_vendor_lane_fields(tmp_path: Path) -> ) assert task.task_id == "mestre-task" + assert start_task.session_id == "session-new" + assert start_task.resume_from is None assert task.event_sink is sink assert task.mcp_servers[0].name == "repo" assert task.resume_from is not None + assert task.session_id is None assert task.metadata["prompt_context_receipt"] == {"schema_version": 1}