From d45100ba26f1d604186a279216359db0f41979f1 Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Sun, 16 Aug 2026 01:36:49 +0530 Subject: [PATCH 1/6] feat(agents): local integration_test authoring agent Implements the 01-coding-template's documented authoring flow locally: 'hud eval integration_test' now pre-stages the golden solution by replaying every Task.validation tool call through the task's own MCP capabilities, ends the trace, and lets the environment's scenario graders run. The CLI enforces the authoring gate afterwards: every run must grade Reward 1.0, otherwise the eval exits non-zero naming the failing reward/raw grade. Replaces the #585 error path with the real implementation. No LLM calls; timeout_seconds bounds the staging pass. --- hud/agents/__init__.py | 2 +- hud/agents/integration_test.py | 157 +++++++++++++++++++++++++++++++++ hud/agents/types.py | 18 ++++ hud/cli/eval.py | 26 ++++++ hud/eval/run.py | 5 ++ hud/types.py | 37 ++++++-- 6 files changed, 239 insertions(+), 6 deletions(-) create mode 100644 hud/agents/integration_test.py diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index 7bcaa56bb..67c7c2565 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -112,7 +112,7 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent: kwargs.setdefault("model", model_id) # cls/config_cls are matched unions; the pairing is correct by construction. config = agent_type.config_cls(**kwargs) - return agent_type.cls(cast("Any", config)) + return cast("GatewayAgent", agent_type.cls(cast("Any", config))) _LAZY_EXPORTS = { diff --git a/hud/agents/integration_test.py b/hud/agents/integration_test.py new file mode 100644 index 000000000..53bad3c64 --- /dev/null +++ b/hud/agents/integration_test.py @@ -0,0 +1,157 @@ +"""The local authoring agent: ``integration_test``. + +The 01-coding-template documents ``hud eval integration_test`` as the +shipping check for a task: pre-stage the golden solution (``Task.validation``), +let the environment's scenario graders run, and require Reward 1.0. This is +the *local* implementation of that agent — no LLM, no platform. It replays +every validation tool call through the task's own MCP capabilities (the bash +capability for the coding template), then ends the trace with an empty answer +so the environment grades the staged workspace. + +The Reward-1.0 gate lives in the CLI (``hud.cli.eval``): grading happens after +the agent finishes, so the agent itself cannot see the reward. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING, Any + +from hud.agents.base import Agent +from hud.agents.types import IntegrationTestConfig, ToolStep +from hud.capabilities import MCPClient +from hud.types import MCPToolCall, MCPToolResult +from hud.utils.time import now_iso + +if TYPE_CHECKING: + from hud.eval.run import Run + +logger = logging.getLogger(__name__) + +DEFAULT_VALIDATION_TIMEOUT_SECONDS = 120.0 + + +class IntegrationTestAgent(Agent): + """Pre-stage ``Task.validation``, then yield to the scenario graders. + + Stateless per run; one instance can drive concurrent rollouts. + """ + + def __init__(self, config: IntegrationTestConfig) -> None: + self.config = config + + async def __call__(self, run: Run) -> None: + validation = list(getattr(run, "validation", None) or []) + if not validation: + logger.warning("integration_test: task has no Task.validation — nothing staged") + return + + connections: dict[str, MCPClient] = {} + manifest = run.client.manifest + if manifest is not None: + for cap in manifest.bindings: + if cap.protocol != MCPClient.protocol: + continue + opened = await run.client.open(cap.name) + if isinstance(opened, MCPClient): + connections[cap.name] = opened + + if not connections: + logger.warning( + "integration_test: no MCP capabilities to stage the golden solution through" + ) + + timeout = self.config.timeout_seconds or DEFAULT_VALIDATION_TIMEOUT_SECONDS + deadline = asyncio.timeout(timeout) + try: + async with deadline: + for entry in validation: + call = self._coerce_call(entry) + if call is None: + continue + result = await self._dispatch(connections, call) + run.record(ToolStep(call=call, result=result, started_at=now_iso())) + except TimeoutError: + run.trace.status = "error" + run.trace.stop_reason = "timeout" + logger.warning("integration_test: validation staging timed out after %gs", timeout) + + @staticmethod + def _coerce_call(entry: Any) -> MCPToolCall | None: + if isinstance(entry, MCPToolCall): + return entry + if isinstance(entry, dict): + try: + return MCPToolCall.model_validate(entry) + except Exception as exc: # surface as a warning, not a crash + logger.warning("integration_test: skipping invalid validation step: %s", exc) + return None + logger.warning("integration_test: skipping unsupported validation step %r", entry) + return None + + async def _dispatch( + self, + connections: dict[str, MCPClient], + call: MCPToolCall, + ) -> MCPToolResult: + """Run one validation tool call against whichever capability serves it. + + Most tasks declare a single MCP capability (``bash``); try each + connected client in turn and stop at the first non-"unknown tool" + result so a task with several capabilities still routes correctly. + """ + from mcp.types import TextContent + + raw_args = call.arguments or {} + if not isinstance(raw_args, dict): + from mcp.types import TextContent + + return MCPToolResult( + content=[ + TextContent( + type="text", + text="the validation step's arguments arrived as a string " + "and were not executed", + ) + ], + isError=True, + ) + args: dict[str, Any] = raw_args + last: MCPToolResult | None = None + for name, client in connections.items(): + try: + result = await client.call_tool(call.name, args) + except Exception as exc: + logger.warning( + "integration_test: capability %r failed for %r: %s", + name, + call.name, + exc, + ) + last = MCPToolResult( + content=[TextContent(type="text", text=f"tool error: {exc}")], + isError=True, + ) + continue + text = getattr(result, "content", None) + unknown = any( + getattr(item, "type", None) == "text" + and str(getattr(item, "text", "")).startswith("unknown tool:") + for item in (text or []) + ) + if not unknown: + return result + last = result + + if last is not None: + return last + return MCPToolResult( + content=[ + TextContent( + type="text", + text=(f"unknown tool: {call.name!r} — no connected MCP capability serves it"), + ) + ], + isError=True, + ) diff --git a/hud/agents/types.py b/hud/agents/types.py index 562066e2f..9aa3e5d9e 100644 --- a/hud/agents/types.py +++ b/hud/agents/types.py @@ -70,6 +70,24 @@ class AgentConfig(BaseModel): model_client: Any = None +# ----------------------------------------------------------------------------- +# Integration test +# ----------------------------------------------------------------------------- + + +class IntegrationTestConfig(AgentConfig): + """Configuration for the local authoring agent. + + No model: the agent pre-stages the golden solution (``Task.validation``) + through the task's own MCP capabilities and lets the environment's + scenario graders run on completion. ``timeout_seconds`` bounds the whole + staging pass. + """ + + model_name: str = "Integration Test" + model: str = "integration_test" + + # ----------------------------------------------------------------------------- # Claude # ----------------------------------------------------------------------------- diff --git a/hud/cli/eval.py b/hud/cli/eval.py index 8b3f4ccd1..8ed71233a 100644 --- a/hud/cli/eval.py +++ b/hud/cli/eval.py @@ -301,6 +301,10 @@ def _parse_agent_type(cls, v: Any) -> AgentType | None: try: return AgentType(v) except ValueError: + if v == "integration_test": + # Local implementation: pre-stages Task.validation and + # lets the scenario graders run. No LLM calls. + return AgentType.INTEGRATION_TEST valid = [e.value for e in AgentType] raise ValueError( f"Invalid agent: {v}. Must be one of: {', '.join(valid)}" @@ -686,6 +690,25 @@ def _build_agent(cfg: EvalConfig) -> Any: return cast("Any", cfg.agent_type.cls)(config=config) +def _enforce_integration_test_reward(runs: list[Any]) -> None: + """The authoring gate: every grader must return Reward 1.0. + + A lower score means the golden solution does not pass the task's own + hidden graders — the task is not shippable. Surface the failing grades + and exit non-zero so the authoring loop fails loudly. + """ + bad = [] + for run in runs: + if run.grade.is_error: + bad.append(f"{run.slug or run.trace.trace_id}: grading errored ({run.grade.info})") + elif run.reward < 1.0: + raw_grade = run.grade.raw + bad.append(f"{run.slug or run.trace.trace_id}: reward {run.reward:g} (raw={raw_grade})") + if bad: + hud_console.error("integration_test: golden must score 1.0 — " + "; ".join(bad)) + raise typer.Exit(1) + + def _python_defines_environment(path: Path) -> bool: """Return True when ``path`` constructs a v6 :class:`~hud.environment.Environment`.""" try: @@ -1004,3 +1027,6 @@ def eval_command( from hud.cli.utils.display import display_runs display_runs(runs, name=cfg.source or "", elapsed=elapsed) + + if cfg.agent_type is AgentType.INTEGRATION_TEST: + _enforce_integration_test_reward(runs) diff --git a/hud/eval/run.py b/hud/eval/run.py index 229bb2d9b..9400f5840 100644 --- a/hud/eval/run.py +++ b/hud/eval/run.py @@ -166,6 +166,10 @@ def __init__( #: The task slug this run came from (set by the rollout engine). Lets #: ``Job.results`` key runs back to their task without positional zip. self.slug: str | None = None + #: The task's golden pre-staging steps (``Task.validation``). The + #: ``integration_test`` authoring agent replays them through the task's + #: own capabilities before the scenario graders run. + self.validation: list[Any] | None = None # Written by :func:`rollout` once placement is acquired. self._runtime: str | None = None @@ -442,6 +446,7 @@ async def _drive() -> None: task.args, best_effort_grade=task.verifier is not None, ) + live.validation = task.validation live._runtime = addr.url # the placement record for the receipt async with live: # start on enter; complete on exit run = live # bound only once live: an earlier failure synthesizes diff --git a/hud/types.py b/hud/types.py index 3b530a6e0..f0dcc8583 100644 --- a/hud/types.py +++ b/hud/types.py @@ -46,13 +46,22 @@ from hud.agents.claude import ClaudeAgent from hud.agents.gemini import GeminiAgent + from hud.agents.integration_test import IntegrationTestAgent from hud.agents.openai import OpenAIAgent from hud.agents.openai_compatible import OpenAIChatAgent - from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig - - AgentClass: TypeAlias = type[ClaudeAgent | GeminiAgent | OpenAIAgent | OpenAIChatAgent] + from hud.agents.types import ( + ClaudeConfig, + GeminiConfig, + IntegrationTestConfig, + OpenAIChatConfig, + OpenAIConfig, + ) + + AgentClass: TypeAlias = type[ + ClaudeAgent | GeminiAgent | OpenAIAgent | OpenAIChatAgent | IntegrationTestAgent + ] AgentConfigClass: TypeAlias = type[ - ClaudeConfig | GeminiConfig | OpenAIConfig | OpenAIChatConfig + ClaudeConfig | GeminiConfig | OpenAIConfig | OpenAIChatConfig | IntegrationTestConfig ] T = TypeVar("T") @@ -63,6 +72,7 @@ class AgentType(StrEnum): OPENAI = "openai" GEMINI = "gemini" OPENAI_COMPATIBLE = "openai_compatible" + INTEGRATION_TEST = "integration_test" @property def cls(self) -> AgentClass: @@ -83,11 +93,21 @@ def cls(self) -> AgentClass: from hud.agents import OpenAIChatAgent return OpenAIChatAgent + case AgentType.INTEGRATION_TEST: + from hud.agents.integration_test import IntegrationTestAgent + + return IntegrationTestAgent @property def config_cls(self) -> AgentConfigClass: """Get config class without importing agent (avoids SDK dependency).""" - from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig + from hud.agents.types import ( + ClaudeConfig, + GeminiConfig, + IntegrationTestConfig, + OpenAIChatConfig, + OpenAIConfig, + ) match self: case AgentType.CLAUDE: @@ -98,6 +118,10 @@ def config_cls(self) -> AgentConfigClass: return GeminiConfig case AgentType.OPENAI_COMPATIBLE: return OpenAIChatConfig + case AgentType.INTEGRATION_TEST: + return IntegrationTestConfig + case _: + raise ValueError(f"no config class for agent type {self!r}") @property def gateway_provider(self) -> str: @@ -111,6 +135,9 @@ def gateway_provider(self) -> str: return "gemini" case AgentType.OPENAI_COMPATIBLE: return "openai" + case AgentType.INTEGRATION_TEST: + # Not a gateway shortcut: the authoring agent makes no LLM calls. + raise ValueError("integration_test is not a gateway agent") @classmethod def of(cls, agent: object) -> AgentType | None: From d13314d626bc9ef89abf46311f3a8f5fac943bfc Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Sun, 16 Aug 2026 01:36:49 +0530 Subject: [PATCH 2/6] test(agents): integration_test dispatch + reward gate - validation calls dispatch through MCP capabilities and record ToolSteps - empty/invalid validation entries degrade safely - EvalConfig accepts integration_test - the reward gate exits non-zero below 1.0 and on grading errors --- hud/agents/tests/test_integration_test.py | 94 +++++++++++++++++++++++ hud/cli/tests/test_eval_config.py | 33 ++++++++ 2 files changed, 127 insertions(+) create mode 100644 hud/agents/tests/test_integration_test.py diff --git a/hud/agents/tests/test_integration_test.py b/hud/agents/tests/test_integration_test.py new file mode 100644 index 000000000..3fc3cc670 --- /dev/null +++ b/hud/agents/tests/test_integration_test.py @@ -0,0 +1,94 @@ +"""The local authoring agent (integration_test).""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock + +import pytest + +from hud.agents.integration_test import IntegrationTestAgent +from hud.agents.types import IntegrationTestConfig, ToolStep +from hud.types import MCPToolCall + + +def _fake_run(*, validation: list[Any] | None, tools: dict[str, str]) -> Any: + """Minimal Run stand-in: manifest + open() returning per-capability MCP clients.""" + + class FakeClient: + async def call_tool(self, name: str, args: dict[str, Any]): + served = tools.get(name) + if served is None: + from mcp.types import TextContent + + from hud.types import MCPToolResult + + return MCPToolResult( + content=[TextContent(type="text", text=f"unknown tool: {name!r}")], + isError=True, + ) + from mcp.types import TextContent + + from hud.types import MCPToolResult + + return MCPToolResult(content=[TextContent(type="text", text=served)]) + + run = SimpleNamespace() + run.validation = validation + run.recorded = [] # type: list[Any] + run.trace = SimpleNamespace(status="running", stop_reason=None) + + def record(step): + run.recorded.append(step) + + run.record = record + run.client = SimpleNamespace() + run.client.manifest = SimpleNamespace( + bindings=[ + SimpleNamespace(name="bash", protocol="mcp"), + ] + ) + run.client.open = AsyncMock(return_value=FakeClient()) + return run + + +@pytest.mark.asyncio +async def test_dispatches_validation_calls_and_records_tool_steps() -> None: + agent = IntegrationTestAgent(IntegrationTestConfig()) + run = _fake_run( + validation=[ + MCPToolCall(name="bash", arguments={"command": "echo 'golden' > answer.txt"}), + {"name": "bash", "arguments": {"command": "chmod +x answer.txt"}}, + ], + tools={"bash": "ok"}, + ) + + await agent(run) + + assert len(run.recorded) == 2 + assert all(isinstance(step, ToolStep) for step in run.recorded) + assert run.recorded[0].call.name == "bash" + + +@pytest.mark.asyncio +async def test_empty_validation_is_a_noop() -> None: + agent = IntegrationTestAgent(IntegrationTestConfig()) + run = _fake_run(validation=[], tools={}) + + await agent(run) + + assert run.recorded == [] + + +@pytest.mark.asyncio +async def test_invalid_entries_are_skipped_without_crashing() -> None: + agent = IntegrationTestAgent(IntegrationTestConfig()) + run = _fake_run( + validation=[{"not": "a tool call"}, 42, MCPToolCall(name="bash", arguments={})], + tools={"bash": "ok"}, + ) + + await agent(run) + + assert len(run.recorded) == 1 diff --git a/hud/cli/tests/test_eval_config.py b/hud/cli/tests/test_eval_config.py index fad19d88f..0db573c1a 100644 --- a/hud/cli/tests/test_eval_config.py +++ b/hud/cli/tests/test_eval_config.py @@ -25,6 +25,13 @@ def test_is_bedrock_arn() -> None: assert _is_bedrock_arn(None) is False +def test_parse_agent_type_accepts_integration_test() -> None: + from hud.types import AgentType + + cfg = EvalConfig(agent_type="integration_test") + assert cfg.agent_type is AgentType.INTEGRATION_TEST + + def test_parse_agent_type_accepts_known_value() -> None: cfg = EvalConfig(agent_type="openai") assert cfg.agent_type is not None @@ -352,3 +359,29 @@ def test_spawn_target_json_uses_parent_directory(tmp_path: Path) -> None: def test_spawn_target_directory_is_served_as_is(tmp_path: Path) -> None: assert eval_mod._spawn_target(tmp_path) == tmp_path.resolve() + + +def test_integration_test_reward_gate_enforces_one_point_zero(monkeypatch) -> None: + from types import SimpleNamespace + + import typer + + from hud.cli.eval import _enforce_integration_test_reward + + def run(reward: float, is_error: bool = False) -> SimpleNamespace: + r = SimpleNamespace() + r.slug = "my-task" + r.reward = reward + r.grade = SimpleNamespace(is_error=is_error, info={}, raw={"score": reward}) + return r + + # Reward 1.0 passes. + _enforce_integration_test_reward([run(1.0)]) + + # Reward < 1.0 exits non-zero. + with pytest.raises(typer.Exit): + _enforce_integration_test_reward([run(0.5)]) + + # Grading errors exit non-zero too. + with pytest.raises(typer.Exit): + _enforce_integration_test_reward([run(0.0, is_error=True)]) From 0cefdbb34a524fa581ca22d95a9ebc67bd1d9f24 Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Sun, 16 Aug 2026 14:58:45 +0530 Subject: [PATCH 3/6] feat(agents): stage golden solutions over SSH workspaces + gateway guard Two gaps from Bugbot review of the integration_test agent: 1. SSH-published workspaces: the agent now opens ssh/2 bindings alongside mcp ones and runs bash validation steps via 'bash -lc ' over SSHClient.run, converting the completed process into a ToolStep result (exit code -> isError). 2. create_agent now rejects INTEGRATION_TEST instead of hiding the non-LLM agent behind the gateway cast, mirroring gateway_provider. --- hud/agents/__init__.py | 6 ++++ hud/agents/integration_test.py | 54 ++++++++++++++++++++++++++++------ 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index 67c7c2565..5fb50ea34 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -109,6 +109,12 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent: ) raise ValueError(f"Model {requested_model!r} not found in {source}.{hint}") + if agent_type is AgentType.INTEGRATION_TEST: + raise ValueError( + "integration_test is the local authoring agent, not a gateway model: " + "it makes no LLM calls. Run it with `hud eval integration_test`." + ) + kwargs.setdefault("model", model_id) # cls/config_cls are matched unions; the pairing is correct by construction. config = agent_type.config_cls(**kwargs) diff --git a/hud/agents/integration_test.py b/hud/agents/integration_test.py index 53bad3c64..b6a58047e 100644 --- a/hud/agents/integration_test.py +++ b/hud/agents/integration_test.py @@ -16,11 +16,11 @@ import asyncio import logging -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from hud.agents.base import Agent from hud.agents.types import IntegrationTestConfig, ToolStep -from hud.capabilities import MCPClient +from hud.capabilities import MCPClient, SSHClient from hud.types import MCPToolCall, MCPToolResult from hud.utils.time import now_iso @@ -47,15 +47,16 @@ async def __call__(self, run: Run) -> None: logger.warning("integration_test: task has no Task.validation — nothing staged") return - connections: dict[str, MCPClient] = {} + connections: dict[str, MCPClient | SSHClient] = {} manifest = run.client.manifest if manifest is not None: for cap in manifest.bindings: - if cap.protocol != MCPClient.protocol: + if cap.protocol not in (MCPClient.protocol, SSHClient.protocol): continue opened = await run.client.open(cap.name) - if isinstance(opened, MCPClient): - connections[cap.name] = opened + # open() resolves through the capability registry, so the + # client type matches the protocol we filtered on above. + connections[cap.name] = cast("MCPClient | SSHClient", opened) if not connections: logger.warning( @@ -92,7 +93,7 @@ def _coerce_call(entry: Any) -> MCPToolCall | None: async def _dispatch( self, - connections: dict[str, MCPClient], + connections: dict[str, MCPClient | SSHClient], call: MCPToolCall, ) -> MCPToolResult: """Run one validation tool call against whichever capability serves it. @@ -100,6 +101,8 @@ async def _dispatch( Most tasks declare a single MCP capability (``bash``); try each connected client in turn and stop at the first non-"unknown tool" result so a task with several capabilities still routes correctly. + SSH-published workspaces (protocol ``ssh/2``) run the same golden + ``bash`` steps via ``bash -lc`` over the SSH connection. """ from mcp.types import TextContent @@ -121,7 +124,10 @@ async def _dispatch( last: MCPToolResult | None = None for name, client in connections.items(): try: - result = await client.call_tool(call.name, args) + if isinstance(client, SSHClient): + result = await self._run_over_ssh(client, call) + else: + result = await client.call_tool(call.name, args) except Exception as exc: logger.warning( "integration_test: capability %r failed for %r: %s", @@ -150,8 +156,38 @@ async def _dispatch( content=[ TextContent( type="text", - text=(f"unknown tool: {call.name!r} — no connected MCP capability serves it"), + text=( + f"unknown tool: {call.name!r} — no connected MCP " + "or SSH capability serves it" + ), ) ], isError=True, ) + + @staticmethod + async def _run_over_ssh(client: SSHClient, call: MCPToolCall) -> MCPToolResult: + from mcp.types import TextContent + + raw_args = call.arguments or {} + command = raw_args.get("command") if isinstance(raw_args, dict) else None + if not isinstance(command, str): + return MCPToolResult( + content=[ + TextContent( + type="text", + text=f"validation step {call.name!r} has no bash command to run", + ) + ], + isError=True, + ) + completed = await client.run("bash", "-lc", command) + output_parts = [p for p in (completed.stdout, completed.stderr) if p] + output = "".join( + p.decode("utf-8", errors="replace") if isinstance(p, bytes) else str(p) + for p in output_parts + ) + return MCPToolResult( + content=[TextContent(type="text", text=output.strip() or "(no output)")], + isError=completed.returncode != 0, + ) From f7c84bd443135b54905d99315afcc69e1f99496c Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Sun, 16 Aug 2026 14:58:45 +0530 Subject: [PATCH 4/6] test(agents): SSH staging dispatch + create_agent guard - bash validation over an ssh/2 workspace records a ToolStep with the command output and success state - non-zero SSH exit codes surface as error results - create_agent('integration_test') raises instead of constructing --- hud/agents/tests/test_base.py | 6 +++ hud/agents/tests/test_integration_test.py | 65 +++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/hud/agents/tests/test_base.py b/hud/agents/tests/test_base.py index 350a3c544..42ce25418 100644 --- a/hud/agents/tests/test_base.py +++ b/hud/agents/tests/test_base.py @@ -83,6 +83,12 @@ def gateway_api_key(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("hud.agents.settings.api_key", "test-key") +def test_create_agent_rejects_integration_test() -> None: + # The authoring agent is not a gateway shortcut; it makes no LLM calls. + with pytest.raises(ValueError, match="local authoring agent"): + create_agent("integration_test") + + def test_create_agent_unknown_model_raises(monkeypatch: pytest.MonkeyPatch) -> None: # No gateway models available -> a bare unknown model can't be resolved. monkeypatch.setattr("hud.agents.list_gateway_models", list) diff --git a/hud/agents/tests/test_integration_test.py b/hud/agents/tests/test_integration_test.py index 3fc3cc670..70dd8716f 100644 --- a/hud/agents/tests/test_integration_test.py +++ b/hud/agents/tests/test_integration_test.py @@ -92,3 +92,68 @@ async def test_invalid_entries_are_skipped_without_crashing() -> None: await agent(run) assert len(run.recorded) == 1 + + +def _fake_run_ssh(*, validation: list[Any]) -> Any: + """Run stand-in whose workspace is published over SSH (ssh/2).""" + + from hud.capabilities import SSHClient + + class FakeSSHClient(SSHClient): + def __init__(self) -> None: + pass + + async def run(self, *args: object, **kwargs: Any) -> Any: + from types import SimpleNamespace + + assert args[:3] == ("bash", "-lc", "echo 'golden' > answer.txt") + return SimpleNamespace(stdout=b"staged\n", stderr=b"", returncode=0) + + run = _fake_run(validation=validation, tools={}) + run.client.manifest = SimpleNamespace( + bindings=[SimpleNamespace(name="workspace", protocol="ssh/2")] + ) + run.client.open = AsyncMock(return_value=FakeSSHClient()) + return run + + +@pytest.mark.asyncio +async def test_dispatches_validation_over_ssh_workspace() -> None: + from hud.agents.types import ToolStep + + agent = IntegrationTestAgent(IntegrationTestConfig()) + run = _fake_run_ssh( + validation=[MCPToolCall(name="bash", arguments={"command": "echo 'golden' > answer.txt"})] + ) + + await agent(run) + + assert len(run.recorded) == 1 + step = run.recorded[0] + assert isinstance(step, ToolStep) + assert step.result.isError is False + assert step.result.content[0].text == "staged" + + +@pytest.mark.asyncio +async def test_ssh_failure_surfaces_as_error_result() -> None: + from types import SimpleNamespace + + from hud.capabilities import SSHClient + + class FailingSSHClient(SSHClient): + def __init__(self) -> None: + pass + + async def run(self, *args: object, **kwargs: Any) -> Any: + return SimpleNamespace(stdout=b"", stderr=b"no such file", returncode=127) + + run = _fake_run(validation=[MCPToolCall(name="bash", arguments={"command": "nope"})], tools={}) + run.client.manifest = SimpleNamespace( + bindings=[SimpleNamespace(name="workspace", protocol="ssh/2")] + ) + run.client.open = AsyncMock(return_value=FailingSSHClient()) + + await IntegrationTestAgent(IntegrationTestConfig())(run) + + assert run.recorded[0].result.isError is True From ed3fdeca507395835f8e608ce015b2a451b65117 Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Sun, 16 Aug 2026 16:59:32 +0530 Subject: [PATCH 5/6] fix(agents): single-string remote command for SSH staging Pass the golden bash step to SSHClient.run as one remote command string (asyncssh shlex-splits it, preserving the command's own quoting, and ships it to 'bash -lc' as a single argument) instead of three argv parts, which can lose grouping on space-join. --- hud/agents/integration_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hud/agents/integration_test.py b/hud/agents/integration_test.py index b6a58047e..613298482 100644 --- a/hud/agents/integration_test.py +++ b/hud/agents/integration_test.py @@ -181,7 +181,10 @@ async def _run_over_ssh(client: SSHClient, call: MCPToolCall) -> MCPToolResult: ], isError=True, ) - completed = await client.run("bash", "-lc", command) + # Single remote command string: asyncssh shlex-splits it, preserving + # the command's own quoting, and ships `command` to `bash -lc` as one + # argument (the codebase idiom; avoids multi-arg space-join hazards). + completed = await client.run(f"bash -lc {command}") output_parts = [p for p in (completed.stdout, completed.stderr) if p] output = "".join( p.decode("utf-8", errors="replace") if isinstance(p, bytes) else str(p) From c9601f20b0874ab80643548aa4d1f88a9fb8c29d Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Sun, 16 Aug 2026 16:59:32 +0530 Subject: [PATCH 6/6] test(agents): expect the single-string SSH command form --- hud/agents/tests/test_integration_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hud/agents/tests/test_integration_test.py b/hud/agents/tests/test_integration_test.py index 70dd8716f..8d8b95287 100644 --- a/hud/agents/tests/test_integration_test.py +++ b/hud/agents/tests/test_integration_test.py @@ -106,7 +106,7 @@ def __init__(self) -> None: async def run(self, *args: object, **kwargs: Any) -> Any: from types import SimpleNamespace - assert args[:3] == ("bash", "-lc", "echo 'golden' > answer.txt") + assert args == ("bash -lc echo 'golden' > answer.txt",) return SimpleNamespace(stdout=b"staged\n", stderr=b"", returncode=0) run = _fake_run(validation=validation, tools={})