From 339452b7c772187590db55e971f6a62fe127047f Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:22:39 -0700 Subject: [PATCH 01/10] feat(harbor): harden adapted task execution - preserve actor and verifier placement requirements on their own task rows - resolve deterministic Compose interpolation and artifact destinations and exclusions - return partial adaptation results with structured per-task findings - resolve zero- and multi-port sidecars without reserving workspace names - transfer actor files across independently placed verifier runtimes - route Compose control ports through network namespace owners - enforce runtime disk requirements and remote daemon socket paths - preserve isolated self-hostname and child-process discovery - parse multiline Dockerfile stages and canonical Compose recipes --- docs/v6/experimental/compose.mdx | 7 +- docs/v6/experimental/harbor.mdx | 35 +- .../v6/experimental/verifier-environments.mdx | 26 +- docs/v6/internals/walkthrough.mdx | 4 +- docs/v6/reference/runtime.mdx | 21 +- hud/__init__.py | 2 + hud/environment/egress.py | 22 +- hud/environment/namespace.py | 60 +- hud/environment/tests/test_workspace.py | 177 +- hud/environment/workspace.py | 74 +- hud/eval/__init__.py | 2 + hud/eval/_runtime_protocols.py | 212 --- hud/eval/docker-seccomp.json | 12 +- hud/eval/run.py | 188 +- hud/eval/runtime.py | 1692 ----------------- hud/eval/runtime/__init__.py | 49 + hud/eval/{ => runtime}/compose.py | 168 +- hud/eval/runtime/core.py | 587 ++++++ hud/eval/runtime/daytona.py | 439 +++++ hud/eval/runtime/docker.py | 323 ++++ hud/eval/runtime/hosted.py | 192 ++ hud/eval/runtime/hud.py | 232 +++ hud/eval/runtime/modal.py | 398 ++++ hud/eval/task.py | 3 + hud/eval/tests/test_docker_provider.py | 303 ++- hud/eval/tests/test_hosted.py | 36 +- hud/eval/tests/test_local_runtime.py | 6 +- hud/eval/tests/test_rollout.py | 140 +- hud/eval/tests/test_task.py | 15 +- hud/integrations/harbor/Dockerfile | 5 +- hud/integrations/harbor/__init__.py | 4 +- hud/integrations/harbor/adapt.py | 720 ++++--- hud/integrations/harbor/env.py | 447 +++-- hud/integrations/harbor/install.sh | 5 +- .../{compose.yaml => docker-compose.yaml} | 0 .../environment/compose.yaml | 11 - .../environment/docker-compose.yaml | 11 + .../sidecar-reachability/solution/solve.sh | 27 +- .../tasks/sidecar-reachability/task.toml | 10 +- .../sidecar-reachability/tests/Dockerfile | 6 +- .../tasks/sidecar-reachability/tests/test.sh | 17 +- .../tasks/verifier-lifecycle/tests/test.sh | 4 + .../harbor/tests/test_contract.py | 408 ++-- .../harbor/tests/test_integration.py | 126 +- hud/tests/test_init.py | 1 + hud/tests/test_init_module.py | 1 + pyproject.toml | 1 + 47 files changed, 4584 insertions(+), 2645 deletions(-) delete mode 100644 hud/eval/_runtime_protocols.py delete mode 100644 hud/eval/runtime.py create mode 100644 hud/eval/runtime/__init__.py rename hud/eval/{ => runtime}/compose.py (69%) create mode 100644 hud/eval/runtime/core.py create mode 100644 hud/eval/runtime/daytona.py create mode 100644 hud/eval/runtime/docker.py create mode 100644 hud/eval/runtime/hosted.py create mode 100644 hud/eval/runtime/hud.py create mode 100644 hud/eval/runtime/modal.py rename hud/integrations/harbor/tests/tasks/hello-mcp/environment/{compose.yaml => docker-compose.yaml} (100%) delete mode 100644 hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/compose.yaml create mode 100644 hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/docker-compose.yaml diff --git a/docs/v6/experimental/compose.mdx b/docs/v6/experimental/compose.mdx index e7582f52e..ad694959d 100644 --- a/docs/v6/experimental/compose.mdx +++ b/docs/v6/experimental/compose.mdx @@ -30,9 +30,10 @@ supervised startup order - are the container boundaries and `depends_on` conditi A Compose row selects its document with `RuntimeConfig.compose`; `compose_project` names the root that travels when the project is serialized and uploaded. Everything the build needs - contexts, -bind mounts, `env_file`, configs, secrets - must live under that root. Host environment -interpolation, `include`, and `extends` are not part of the serialized contract: the document that -uploads is the document that runs. +bind mounts, `env_file`, configs, secrets - must live under that root. Interpolation resolves from +the `.env` beside the Compose file and from defaults in the document; HUD does not read values from +the process environment, so an unbound variable is rejected. `include` and `extends` are not part +of the serialized contract: the document that uploads is the document that runs. Every runnable service has an `image`, a `build`, or both; paths are relative to the project. An optional `build.sh` beside the document is a preparation hook - it resolves or builds prerequisite diff --git a/docs/v6/experimental/harbor.mdx b/docs/v6/experimental/harbor.mdx index 14aa6348f..83aa55b94 100644 --- a/docs/v6/experimental/harbor.mdx +++ b/docs/v6/experimental/harbor.mdx @@ -15,19 +15,24 @@ images. `export()` materializes HUD tasks as self-contained Harbor folders. A Harbor source is either one task directory or a dataset directory containing task directories. Each task contains `task.toml` and `instruction.md`, plus an environment image or build recipe and a `tests/` verifier. +Multi-container sources use Harbor's `environment/docker-compose.yaml` recipe +name; other Compose-like filenames remain ordinary environment files. ```python from hud.integrations import harbor from hud.eval import DockerRuntime -taskset = harbor.adapt("./terminal-bench") -job = await taskset.run(agent, runtime=DockerRuntime()) +result = harbor.adapt("./terminal-bench") +job = await result.taskset.run(agent, runtime=DockerRuntime()) ``` Each returned row carries its instruction and grading configuration in `args`, Harbor metadata in `columns`, resource requirements in `runtime_config`, and, when declared, a [`Task.verifier`](/v6/experimental/verifier-environments). The row's Compose path points into a generated project under `.hud-adapt/`. +Tasks that cannot be adapted appear in `result.failures`, with every detectable +finding classified by a stable code. Other tasks in the dataset are still +packaged. ```text .hud-adapt// @@ -97,18 +102,24 @@ runtime isolation are described in The adapter handles: - an `environment/Dockerfile`, an `environment.docker_image`, or a Compose - project with a `main` service; -- Compose sidecars with one declared TCP endpoint each; + project; when authored Compose omits `main`, the environment recipe supplies + that agent service; +- Compose sidecars with any number of declared TCP endpoints; - HTTP MCP servers (`sse` and `streamable-http`), healthchecks, network modes, - allowlists, phase users, environment variables, and CPU/memory/GPU requests; + allowlists, phase users, environment variables, and placement requirements; - inline verifier scripts and separate verifier Dockerfiles; -- verifier collect hooks and declared artifact paths from `main` or a sidecar. +- verifier collect hooks and declared artifact paths from `main` or a sidecar, + including directory exclusions and host-side destinations. -Unsupported declarations fail during adaptation. These include non-Linux and -TPU environments, stdio MCP servers, skills directories, multi-step tasks, -Compose interpolation/include/extends, and sidecars without exactly one usable -TCP endpoint. A project also fails if its main service has neither an image nor -a valid build recipe. +Placement requirements are copied to the task that declares them: the main +environment configures the actor task, and a separate verifier environment can +configure its verifier task independently. The selected runtime decides whether +it can provision those requirements. + +Unsupported declarations fail during adaptation. These include stdio MCP +servers, skills directories, multi-step tasks, Compose variables not bound by +the project `.env` or document defaults, `include`, and `extends`. A project +also fails if its main service has neither an image nor a valid build recipe. ## Export HUD tasks to Harbor @@ -156,7 +167,7 @@ covers that review. | Function | Contract | | --- | --- | -| `harbor.adapt(path, *, hud_requirement="hud") -> Taskset` | Package one Harbor task or a dataset as generated Compose environments and native task rows. | +| `harbor.adapt(path, *, hud_requirement="hud") -> AdaptResult` | Package valid tasks as generated Compose environments and native task rows, and return structured failures for the rest. | | `await harbor.export(source, out_dir, *, answer_file=..., timeout_sec=600) -> list[Path]` | Write HUD task rows as Harbor task directories. | diff --git a/docs/v6/experimental/verifier-environments.mdx b/docs/v6/experimental/verifier-environments.mdx index bddc92518..131171571 100644 --- a/docs/v6/experimental/verifier-environments.mdx +++ b/docs/v6/experimental/verifier-environments.mdx @@ -45,12 +45,12 @@ verifier = Environment("judge") @actor.template(id="solve") async def solve(): answer = yield "Write the secret to the target system." - yield 0.0 # the verifier task supplies the authoritative reward + yield {"score": 0.0, "answer": answer} @verifier.template(id="verify") async def verify(expected: str): - answer = yield "" - yield 1.0 if answer == expected else 0.0 + actor_result = yield "" + yield 1.0 if actor_result["answer"] == expected else 0.0 task = solve() task.verifier = verify(expected="secret") @@ -75,16 +75,17 @@ sequenceDiagram Engine->>Actor: close connection + clean up Engine->>Judge: provision + connect Engine->>Judge: tasks.start(verifier) - Engine->>Judge: tasks.grade(answer) + Engine->>Judge: tasks.grade(actor result) Judge-->>Engine: authoritative evaluation Engine->>Judge: close connection + clean up ``` -The actor task is graded to complete its generator lifecycle, but that grade is best-effort: when -the verifier phase begins, the actor grade is cleared, and the verifier evaluation replaces it as -the run's grade of record. Agent failures and actor-grading failures are recorded on the trace -while the verifier still runs when the phase boundary can be reached; a verifier provisioning or -grading failure leaves the run errored and ungraded. +The actor task is graded to complete its generator lifecycle. Its full result frame is sent to the +verifier, whose evaluation replaces it as the run's grade of record. An actor result must include +the numeric `score` required of every task result plus whatever fields its verifier consumes. If +actor grading fails, the engine supplies an error result containing the submitted `answer` so an +authoritative verifier can still run. A verifier provisioning or grading failure leaves the run +errored and ungraded. If both rows name the same environment and the verifier has no row-level `runtime_config`, the engine keeps the actor connection and substrate alive and starts the verifier task on that control @@ -101,10 +102,9 @@ client-driven provider such as `LocalRuntime`, `DockerRuntime`, or a custom prov | Actor | Actor environment setup, the agent loop, actor task teardown | Actor capabilities and state | Provisional; retained only when no verifier exists | | Verifier | Verifier setup and grading; no agent loop | None through HUD's agent interface | Authoritative | -The engine forwards the final answer (`run.trace.content`) to the verifier. Files, processes, -sockets, and environment memory do not cross between distinct substrates automatically - any -graded state transfer is an explicit adapter or provider contract, such as an artifact snapshot, -object-store reference, or shared service endpoint. +The engine forwards the actor task's result to the verifier unchanged. Files, processes, sockets, +and environment memory do not cross between distinct substrates automatically - the actor result +must carry any artifact reference, object-store key, or shared service endpoint the verifier needs. ## Harbor verifier environments diff --git a/docs/v6/internals/walkthrough.mdx b/docs/v6/internals/walkthrough.mdx index 78e669952..8b93fe670 100644 --- a/docs/v6/internals/walkthrough.mdx +++ b/docs/v6/internals/walkthrough.mdx @@ -668,7 +668,7 @@ The score travels back: `TaskRunner.grade` -> server reply -> `client.grade` -> When `task.verifier` is present, the actor grade is provisional. A verifier in the same environment with no row-level runtime configuration starts on the existing client. Otherwise the actor client and provider exit before the same provider is called with the verifier row. `_verify` starts that -task without an agent loop and immediately grades it with `run.trace.content`; its evaluation +task without an agent loop and immediately grades it with the actor result frame; its evaluation replaces the actor grade. The full phase contract is documented in [verifier environments](/v6/experimental/verifier-environments#provisioning-order). @@ -694,7 +694,7 @@ Every hop above, in order: 7. **Checkpoint** - `run` holds a live client (manifest + suspended runner) and the prompt. 8. `await agent(run)` - agent opens capabilities via `run.client`, loops, fills `run.trace` (answer on `trace.content`). 9. `Run.__aexit__` - `client.grade` -> `tasks.grade` -> `TaskRunner.grade` resumes the generator to the second yield -> provisional `score` -> `run.grade.reward`. -10. Optional verifier - reuse the live substrate or finish actor cleanup and acquire the verifier substrate; start and grade the verifier with `trace.content`; replace the actor grade. +10. Optional verifier - reuse the live substrate or finish actor cleanup and acquire the verifier substrate; start and grade the verifier with the actor result; replace the actor grade. 11. Unwind - close the active client, stop the substrate (`serve` runs `env.stop()`), `trace_exit`, return the graded `Run` to `Taskset.run`. diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index 4f5c6da75..8f3335615 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -57,15 +57,21 @@ Compose project, hardware, and timeouts. Set it on the runtime (`runtime_config= supports. ```python -from hud.eval import RuntimeConfig, RuntimeResources, RuntimeGPU, RuntimeLimits +from hud.eval import RuntimeConfig, RuntimeGPU, RuntimeLimits, RuntimeResources, RuntimeTPU RuntimeConfig( image="my-env", - resources=RuntimeResources(cpu=4, memory_mb=8192, gpu=RuntimeGPU(type="A100", count=1)), + resources=RuntimeResources( + cpu=4, + memory_mb=8192, + storage_mb=32768, + gpu=RuntimeGPU(type=["H100", "A100"], count=1), + ), limits=RuntimeLimits(startup_timeout_s=300, run_timeout_s=1800), ) RuntimeConfig(compose="./compose.yaml") +RuntimeConfig(resources=RuntimeResources(tpu=RuntimeTPU(type="v5", topology="2x2"))) ``` | Field | Description | @@ -74,12 +80,15 @@ RuntimeConfig(compose="./compose.yaml") | `compose` | Local path to a Compose file, or its serialized `ComposeConfig`. Mutually exclusive with `image`. | | `compose_project` | Local project root for upload, or a serialized `ComposeProjectRef`. Requires `compose`. | | `compose_service_access` | Mount the runtime's Docker socket into Compose `main` at `/media/hud/docker.sock`. Requires `compose`. | -| `resources` | `RuntimeResources(cpu, memory_mb, gpu=RuntimeGPU(type, count))`. | +| `resources` | Placement requests: CPU, memory, disk, acceptable GPU types and count, OS, or TPU slice. | | `limits` | `RuntimeLimits(startup_timeout_s, run_timeout_s)`. | -Support differs per runtime: `DockerRuntime`, `ModalRuntime`, and `DaytonaRuntime` accept it (Docker -ignores `limits`; Daytona ignores `run_timeout_s` and resource overrides when booting from a snapshot). -`LocalRuntime` and `HUDRuntime` reject a per-task `runtime_config`. +Support differs per runtime. Providers reject unsupported requirements except `storage_mb`, which +is best effort: `DockerRuntime` admits against available disk and `DaytonaRuntime` provisions +enough whole GiB, while providers without disk sizing proceed with their default capacity. +Daytona accepts a list of GPU alternatives. Docker ignores `limits`; Daytona rejects +`run_timeout_s` and resource overrides when booting from an already-built snapshot. `LocalRuntime` +rejects a per-task `runtime_config`. ## Runtime directory diff --git a/hud/__init__.py b/hud/__init__.py index 3d5f646ef..675324e3d 100644 --- a/hud/__init__.py +++ b/hud/__init__.py @@ -23,6 +23,7 @@ RuntimeGPU, RuntimeLimits, RuntimeResources, + RuntimeTPU, SubprocessRuntime, SyncPlan, Task, @@ -48,6 +49,7 @@ "RuntimeGPU", "RuntimeLimits", "RuntimeResources", + "RuntimeTPU", "SubprocessRuntime", "SyncPlan", "Task", diff --git a/hud/environment/egress.py b/hud/environment/egress.py index 0f949b251..d9499b3bd 100644 --- a/hud/environment/egress.py +++ b/hud/environment/egress.py @@ -196,18 +196,23 @@ def bind_addresses( ("127.0.0.1", VISITOR_PORT), *(("127.0.0.1", port) for port in reserved_ports), } - addresses: dict[str, str] = {} + ports_by_name: dict[str, list[int]] = {} for peer in peers: - if peer.name in addresses: - raise ValueError(f"two peers are called {peer.name!r}") + ports = ports_by_name.setdefault(peer.name, []) + if peer.port in ports: + raise ValueError(f"peer {peer.name!r} declares port {peer.port} twice") + ports.append(peer.port) + + addresses: dict[str, str] = {} + for name, ports in ports_by_name.items(): for index in range(1, 256): host = f"127.0.0.{index}" - if (host, peer.port) not in taken: + if all((host, port) not in taken for port in ports): break else: - raise ValueError(f"too many peers on port {peer.port}") - taken.add((host, peer.port)) - addresses[peer.name] = host + raise ValueError(f"no loopback address can route peer {name!r}") + taken.update((host, port) for port in ports) + addresses[name] = host return addresses @@ -229,7 +234,7 @@ def hosts_text( lines = "".join( [ *(f"127.0.0.1\t{name}\n" for name in local_aliases), - *(f"{addresses[peer.name]}\t{peer.name}\n" for peer in peers), + *(f"{host}\t{name}\n" for name, host in addresses.items()), ] ) return f"{base.rstrip(chr(10))}\n{lines}" if base.strip() else lines @@ -510,6 +515,7 @@ def handle(self) -> None: class _UnixServer(socketserver.ThreadingUnixStreamServer): daemon_threads = True + request_queue_size = socket.SOMAXCONN def get_request(self) -> tuple[socket.socket, tuple[str, int]]: # A unix peer has no address; the handler wants one to log. diff --git a/hud/environment/namespace.py b/hud/environment/namespace.py index e6ccd27b9..d35353096 100644 --- a/hud/environment/namespace.py +++ b/hud/environment/namespace.py @@ -10,6 +10,7 @@ import os import pty import shutil +import signal import socket import struct import sys @@ -41,6 +42,25 @@ async def read_bwrap_pid(info_read: int) -> int: return int(document["child-pid"]) +def _child_pids(pid: int) -> list[int]: + children_file = Path(f"/proc/{pid}/task/{pid}/children") + try: + return [int(child) for child in children_file.read_text().split()] + except OSError: + children = [] + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + _, separator, suffix = (entry / "stat").read_text().rpartition(")") + fields = suffix.split() + if separator and int(fields[1]) == pid: + children.append(int(entry.name)) + except (OSError, IndexError, ValueError): + continue + return sorted(children) + + async def install_identity_map( info_read: int, block_write: int, @@ -53,11 +73,10 @@ async def install_identity_map( if launcher_pid is not None: pid = launcher_pid for _ in range(launcher_depth): - children_file = Path(f"/proc/{pid}/task/{pid}/children") - children = (await asyncio.to_thread(children_file.read_text)).split() + children = await asyncio.to_thread(_child_pids, pid) if len(children) != 1: raise RuntimeError(f"sandbox launcher {pid} has {len(children)} children") - pid = int(children[0]) + pid = children[0] await asyncio.to_thread(_map_identities, pid) await asyncio.to_thread(os.write, block_write, b"\n") return pid @@ -134,11 +153,14 @@ def __init__(self, socket_path: Path) -> None: async def connect(self) -> None: if self._connection is not None: return + self._connection = await self._open_connection() + + async def _open_connection(self) -> asyncssh.SSHClientConnection: sock = socket.socket(socket.AF_UNIX) sock.setblocking(False) try: await asyncio.get_running_loop().sock_connect(sock, str(self.socket_path)) - self._connection = await asyncssh.connect( + return await asyncssh.connect( sock=sock, username="hud", known_hosts=None, @@ -220,12 +242,16 @@ async def spawn( return NamespaceProcess(process) async def terminate_sessions(self) -> None: - connection = self._require_connection() - result = await connection.run( - json.dumps({"operation": "terminate_sessions"}), - check=False, - encoding=None, - ) + connection = await self._open_connection() + try: + result = await connection.run( + json.dumps({"operation": "terminate_sessions"}), + check=False, + encoding=None, + ) + finally: + connection.close() + await connection.wait_closed() if result.returncode != 0: stderr = result.stderr or b"" assert isinstance(stderr, bytes) @@ -295,6 +321,7 @@ def __init__( self.map_identities = map_identities self.ports = ports self.holders: dict[Literal["session", "environment"], tuple[ProcessGroup, int]] = {} + self.session_used = False self.forwarders: list[asyncio.AbstractServer] = [] async def serve(self) -> None: @@ -368,7 +395,7 @@ async def _handle(self, process: asyncssh.SSHServerProcess[bytes]) -> None: else: process.exit(await self._spawn(request, process)) except Exception as exc: - process.stderr.write(str(exc).encode()) + process.stderr.write(f"{type(exc).__name__}: {exc}".encode()) process.exit(1) await process.wait_closed() @@ -443,10 +470,15 @@ async def _holder_error(self, holder: ProcessGroup) -> str: return detail.decode(errors="replace").strip() or "sandbox holder did not become ready" async def _terminate_sessions(self) -> None: + if not self.session_used: + return + self.session_used = False held = self.holders.pop("session", None) if held is not None: - holder, _ = held - await holder.terminate() + holder, holder_pid = held + with contextlib.suppress(ProcessLookupError): + os.kill(holder_pid, signal.SIGKILL) + await holder.wait() async def _spawn( self, @@ -537,6 +569,8 @@ async def _spawn( stderr=process.stderr, send_eof=False, ) + if scope == "session": + self.session_used = True wait_task = asyncio.create_task(process.wait()) closed_task = asyncio.create_task(channel.channel.wait_closed()) try: diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index ff6c62769..cc63fb657 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -8,6 +8,7 @@ import json import os import shutil +import signal import socket import sys import tempfile @@ -25,7 +26,7 @@ from hud.capabilities import SSHClient from hud.environment import namespace as namespace_mod from hud.environment import workspace as workspace_mod -from hud.environment.egress import Peer, _field, _Unrelayable +from hud.environment.egress import Peer, _field, _UnixServer, _Unrelayable from hud.environment.workspace import Bubblewrap, Mount, Workspace from hud.utils.process import ProcessResult @@ -447,6 +448,50 @@ async def visiting(allowed): assert kwargs["env"]["HTTPS_PROXY"] == "http://visitor" +@pytest.mark.asyncio +async def test_run_uses_a_disposable_writable_hosts_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "hosts" + source.write_text("127.0.0.1 localhost\n", encoding="utf-8") + ws = Workspace(tmp_path / "root") + ws._hosts_path = source + monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) + monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=7)) + + @contextlib.asynccontextmanager + async def visiting(allowed): + assert allowed == () + yield {} + + mounted: Path | None = None + + async def spawn(argv: list[str], **kwargs: Any) -> Any: + nonlocal mounted + del kwargs + hosts_index = argv.index("/etc/hosts") + assert argv[hosts_index - 2] == "--bind" + mounted = Path(argv[hosts_index - 1]) + assert mounted.read_text("utf-8") == source.read_text("utf-8") + assert mounted.parent.stat().st_mode & 0o777 == 0o711 + mounted.write_text("127.0.0.1 verifier-added\n", encoding="utf-8") + return SimpleNamespace(complete=AsyncMock(return_value=ProcessResult(0, b"passed", b""))) + + monkeypatch.setattr(ws, "visiting", visiting) + ws._namespace = cast("Any", SimpleNamespace(spawn=spawn)) + + result = await ws.run(["test.sh"], identity=None, writable_hosts=True) + + assert result.stdout == b"passed" + assert mounted is not None and not await asyncio.to_thread(mounted.exists) + assert not mounted.parent.exists() + assert await asyncio.to_thread(source.read_text, "utf-8") == "127.0.0.1 localhost\n" + + +def test_peer_forwarders_use_the_substrate_listen_backlog() -> None: + assert _UnixServer.request_queue_size == socket.SOMAXCONN + + @pytest.mark.asyncio async def test_visiting_none_uses_the_workspace_egress( tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -461,6 +506,10 @@ async def test_visiting_none_uses_the_workspace_egress( async with ws.visiting(None) as environment: assert environment == {"HTTPS_PROXY": "http://workspace"} + ws.allowed_hosts = frozenset({"pypi.org"}) + async with ws.visiting({"pypi.org"}) as environment: + assert environment == {"HTTPS_PROXY": "http://workspace"} + @pytest.mark.asyncio async def test_visiting_uses_the_reserved_workspace_bridge( @@ -563,6 +612,59 @@ async def test_terminate_sessions_preserves_the_namespace_host(tmp_path: Path) - assert ws._namespace is not None +@pytest.mark.asyncio +async def test_namespace_management_does_not_share_process_connections( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + host = namespace_mod.NamespaceHost(tmp_path / "namespace.sock") + process_connection = SimpleNamespace(run=AsyncMock()) + host._connection = cast("Any", process_connection) + management = SimpleNamespace( + run=AsyncMock(return_value=SimpleNamespace(returncode=0)), + close=Mock(), + wait_closed=AsyncMock(), + ) + open_connection = AsyncMock(return_value=management) + monkeypatch.setattr(host, "_open_connection", open_connection) + + await host.terminate_sessions() + + open_connection.assert_awaited_once_with() + process_connection.run.assert_not_awaited() + management.close.assert_called_once_with() + management.wait_closed.assert_awaited_once_with() + + +@pytest.mark.asyncio +async def test_namespace_host_only_terminates_a_used_session_holder( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + host = namespace_mod._NamespaceHost( + tmp_path / "namespace.sock", + setup_loopback=False, + holder_argv=[], + bwrap="bwrap", + launcher_depth=0, + map_identities=False, + ports=frozenset(), + ) + holder = AsyncMock() + host.holders["session"] = (holder, 7) + kill = Mock() + monkeypatch.setattr(os, "kill", kill) + + await host._terminate_sessions() + holder.terminate.assert_not_awaited() + kill.assert_not_called() + + host.session_used = True + await host._terminate_sessions() + kill.assert_called_once_with(7, signal.SIGKILL) + holder.wait.assert_awaited_once_with() + + @pytest.mark.asyncio async def test_run_can_use_a_fresh_no_network_sandbox( tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -753,8 +855,17 @@ def test_a_peer_answers_at_the_address_the_task_expects() -> None: proxies = bind_addresses([Peer("agent", 3128), Peer("verifier", 3129)]) assert proxies == {"agent": "127.0.0.2", "verifier": "127.0.0.2"} - with pytest.raises(ValueError, match="two peers are called"): - bind_addresses([Peer("db", 5432), Peer("db", 6379)]) + service = bind_addresses([Peer("db", 5432), Peer("db", 6379)]) + assert service == {"db": "127.0.0.1"} + + service_with_reserved_port = bind_addresses( + [Peer("db", 5432), Peer("db", 6379)], + reserved_ports={5432}, + ) + assert service_with_reserved_port == {"db": "127.0.0.2"} + + with pytest.raises(ValueError, match="declares port 5432 twice"): + bind_addresses([Peer("db", 5432), Peer("db", 5432)]) def test_workspace_names_are_added_to_the_substrates_hosts_rather_than_replacing_it() -> None: @@ -772,6 +883,12 @@ def test_workspace_names_are_added_to_the_substrates_hosts_rather_than_replacing assert "127.0.0.1\tmain" in text assert text.endswith("127.0.0.1\tdb\n") + same_service = hosts_text( + [Peer("db", 5432), Peer("db", 6379)], + "", + ) + assert same_service == "127.0.0.1\tdb\n" + @pytest.mark.asyncio async def test_a_declared_peer_is_a_name_sessions_resolve( @@ -813,6 +930,23 @@ async def test_a_declared_peer_is_a_name_sessions_resolve( await sharing.stop() +def test_private_workspace_preserves_localhost_without_peers( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + hosts = tmp_path / "hosts" + ws = Workspace( + tmp_path / "root", + allowed_hosts=set(), + hosts_path=hosts, + ) + monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) + + ws._prepare_runtime() + + assert "/etc/hosts" in ws.bwrap_argv(["true"]) + assert "localhost" in hosts.read_text("utf-8") + + def test_a_peer_is_reached_directly_rather_than_through_the_proxy() -> None: """The proxy resolves names out on the substrate, where a peer's name means nothing and its address is something else entirely.""" @@ -1389,3 +1523,40 @@ def test_the_proxy_refuses_to_relay_a_header_it_cannot_represent() -> None: ): with pytest.raises(_Unrelayable): _field(name, value) + + +def test_private_workspace_maps_its_own_hostname_to_loopback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(socket, "gethostname", lambda: "container-self") + ws = Workspace( + tmp_path / "root", + allowed_hosts=set(), + local_aliases=["main"], + credentials_dir=tmp_path / "keys", + hosts_path=tmp_path / "hosts", + ) + + generated = ws._write_hosts().read_text("utf-8") + + assert "127.0.0.1\tmain" in generated + assert "127.0.0.1\tcontainer-self" in generated + + +def test_child_pid_discovery_falls_back_to_proc_stat(monkeypatch: pytest.MonkeyPatch) -> None: + entries = [Path("/proc/101"), Path("/proc/102"), Path("/proc/self")] + + def read_text(path: Path, *args: object, **kwargs: object) -> str: + if path.name == "children": + raise OSError("unavailable") + if str(path) == "/proc/101/stat": + return "101 (child one) S 100 0 0 0" + if str(path) == "/proc/102/stat": + return "102 (other) S 99 0 0 0" + raise OSError("gone") + + monkeypatch.setattr(Path, "iterdir", lambda path: iter(entries)) + monkeypatch.setattr(Path, "read_text", read_text) + + assert namespace_mod._child_pids(100) == [101] diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index e1a3dfc22..4790594be 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -450,7 +450,7 @@ async def visiting(self, allowed: Collection[str] | None) -> AsyncIterator[dict[ if await self.sandbox_pid() is None or not self.owns_netns: yield {} return - if allowed is None: + if allowed is None or frozenset(allowed) == self.allowed_hosts: yield self._egress.environment() if self._egress is not None else {} return if not allowed: @@ -546,7 +546,7 @@ def _prepare_runtime(self) -> None: os.lchown(self.root, self._shell_uid, gid) self._host_key, self._host_pubkey_str = self._load_or_generate_host_key() self._authorized_keys_path = self._ensure_authorized_keys_file() - if (self.peers or self.local_aliases) and self.owns_netns and self._bwrap is not None: + if self.owns_netns and self._bwrap is not None: self._hosts_path = self._write_hosts() self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -723,6 +723,7 @@ async def run( no_new_privs: bool = True, max_wait: float | None = None, scope: Literal["session", "environment"] = "session", + writable_hosts: bool = False, ) -> ProcessResult: """Run a captured command against this workspace. @@ -734,25 +735,53 @@ async def run( command's lifetime. ``isolated=True`` gives it a fresh no-network namespace instead. ``mounts`` can replace the session's mounts where an operation is allowed to see paths hidden from sessions. + ``writable_hosts`` gives a trusted command a private, disposable copy + of the workspace's hosts file. """ bwrap = self._bwrap if bwrap is None: raise RuntimeError("workspace commands require bwrap") + if isolated and writable_hosts: + raise ValueError("writable hosts require the workspace network") process_env = dict(env or {}) if not isolated: - async with self.visiting(allowed_hosts) as visitor_env: - process_env.update(visitor_env) - process = await self.launch( - command, - mounts=mounts, - env=process_env, - cwd=cwd, - identity=identity, - inherit_workspace_env=inherit_workspace_env, - no_new_privs=no_new_privs, - scope=scope, - ) - return await process.complete(max_wait=max_wait) + writable_hosts_path: Path | None = None + hosts_dir: Path | None = None + try: + if writable_hosts: + hosts_dir = Path( + tempfile.mkdtemp( + prefix="hosts-", + dir=self._credentials_dir().parent, + ) + ) + hosts_dir.chmod(0o711) + writable_hosts_path = hosts_dir / "hosts" + shutil.copyfile(self._hosts_path or Path("/etc/hosts"), writable_hosts_path) + writable_hosts_path.chmod(0o644) + command_mounts = self.mounts if mounts is None else mounts + if writable_hosts_path is not None: + command_mounts = [ + *command_mounts, + Mount("rw", src=str(writable_hosts_path), dst="/etc/hosts"), + ] + async with self.visiting(allowed_hosts) as visitor_env: + process_env.update(visitor_env) + process = await self.launch( + command, + mounts=command_mounts, + env=process_env, + cwd=cwd, + identity=identity, + inherit_workspace_env=inherit_workspace_env, + no_new_privs=no_new_privs, + scope=scope, + ) + return await process.complete(max_wait=max_wait) + finally: + if hosts_dir is not None: + with contextlib.suppress(FileNotFoundError): + shutil.rmtree(hosts_dir) if allowed_hosts: raise ValueError("an isolated workspace command has no network") @@ -947,9 +976,14 @@ def bwrap_argv( for mount in self._system_mounts: argv.extend(mount.to_bwrap_args(bind_devices=bind_host_devices)) argv.extend(["--bind", str(self.root), self._guest_path]) - for m in self.mounts if mounts is None else mounts: + selected_mounts = self.mounts if mounts is None else mounts + for m in selected_mounts: argv.extend(m.to_bwrap_args(bind_devices=bind_host_devices)) - if mount_hosts and self._hosts_path is not None: + if ( + mount_hosts + and self._hosts_path is not None + and not any(mount.dst == "/etc/hosts" for mount in selected_mounts) + ): # Last, so it survives whatever the caller mounted over /etc: a # peer the task can address by port but not by name is not at the # address the task expects. @@ -1349,13 +1383,17 @@ def _write_hosts(self) -> Path: process arguments do not disclose that path. """ substrate = Path("/etc/hosts") + local_aliases = {*self.local_aliases, socket.gethostname()} + peer_names = {peer.name for peer in self.peers} + if collision := local_aliases & peer_names: + raise ValueError(f"workspace local alias conflicts with peer {sorted(collision)[0]!r}") path = self._configured_hosts_path or self._credentials_dir() / "hosts" path.parent.mkdir(parents=True, exist_ok=True) path.write_text( hosts_text( self.peers, substrate.read_text() if substrate.is_file() else "", - local_aliases=sorted(self.local_aliases), + local_aliases=sorted(local_aliases), reserved_ports=self.ports, ), encoding="utf-8", diff --git a/hud/eval/__init__.py b/hud/eval/__init__.py index ee53010fa..f1fd9bdc2 100644 --- a/hud/eval/__init__.py +++ b/hud/eval/__init__.py @@ -46,6 +46,7 @@ RuntimeGPU, RuntimeLimits, RuntimeResources, + RuntimeTPU, Shared, SubprocessRuntime, ) @@ -70,6 +71,7 @@ "RuntimeGPU", "RuntimeLimits", "RuntimeResources", + "RuntimeTPU", "Shared", "SubprocessRuntime", "SyncPlan", diff --git a/hud/eval/_runtime_protocols.py b/hud/eval/_runtime_protocols.py deleted file mode 100644 index 332e53769..000000000 --- a/hud/eval/_runtime_protocols.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Structural contracts for lazily imported runtime provider SDKs.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Protocol, TypeVar - -if TYPE_CHECKING: - from collections.abc import Awaitable, Mapping, Sequence - from contextlib import AbstractAsyncContextManager - from pathlib import Path - -T_co = TypeVar("T_co", covariant=True) - - -class AioMethod(Protocol[T_co]): - async def aio(self, *args: object, **kwargs: object) -> T_co: ... - - -class ModalImage(Protocol): - build: AioMethod[None] - - def env(self, variables: Mapping[str, str]) -> ModalImage: ... - - -class _ModalImageFactory(Protocol): - def from_id(self, image_id: str) -> ModalImage: ... - - def from_registry(self, image: str) -> ModalImage: ... - - def from_name(self, name: str) -> ModalImage: ... - - -class _ModalAppFactory(Protocol): - lookup: AioMethod[object] - - -class _ModalStream(Protocol): - read: AioMethod[str] - - -class _ModalProcess(Protocol): - wait: AioMethod[int] - stderr: _ModalStream - - -class _ModalFilesystem(Protocol): - copy_from_local: AioMethod[None] - - -class _ModalTunnel(Protocol): - tcp_socket: tuple[str, int] - - -class _ModalSandbox(Protocol): - object_id: str - wait_until_ready: AioMethod[None] - filesystem: _ModalFilesystem - exec: AioMethod[_ModalProcess] - tunnels: AioMethod[dict[int, _ModalTunnel]] - terminate: AioMethod[None] - - -class _ModalSandboxFactory(Protocol): - create: AioMethod[_ModalSandbox] - - -class _ModalProbeFactory(Protocol): - def with_tcp(self, port: int) -> object: ... - - -class ModalModule(Protocol): - Image: _ModalImageFactory - App: _ModalAppFactory - Sandbox: _ModalSandboxFactory - Probe: _ModalProbeFactory - - -class DaytonaContextEntry(Protocol): - @property - def source_path(self) -> str | Path: ... - - @property - def archive_path(self) -> str | Path: ... - - -class DaytonaImage(Protocol): - @property - def _context_list(self) -> Sequence[DaytonaContextEntry]: ... - - def dockerfile(self) -> str: ... - - -class _DaytonaBuildInfo(Protocol): - dockerfile_content: str - context_hashes: Sequence[str] | None - - -class DaytonaSnapshot(Protocol): - image_name: str - build_info: _DaytonaBuildInfo | None - - -class ObjectStorage(Protocol): - async def _compute_hash_for_path_md5( - self, - source_path: str | Path, - archive_path: str | Path, - ) -> str: ... - - -class ObjectStorageModule(Protocol): - AsyncObjectStorage: type[ObjectStorage] - - -class _DaytonaResources(Protocol): - cpu: int | None - memory: int | None - gpu: int | None - gpu_type: Sequence[object] | None - - -class _DaytonaSessionCommand(Protocol): - cmd_id: str - - -class _DaytonaSessionLogs(Protocol): - stderr: str | None - output: str | None - stdout: str | None - - -class _DaytonaProcess(Protocol): - async def create_session(self, session: str) -> object: ... - - async def execute_session_command( - self, - session: str, - request: object, - ) -> _DaytonaSessionCommand: ... - - async def get_session_command_logs( - self, - session: str, - command_id: str, - ) -> _DaytonaSessionLogs: ... - - -class _DaytonaSshAccess(Protocol): - token: str - - -class _DaytonaSandbox(Protocol): - id: str - process: _DaytonaProcess - - async def create_ssh_access(self, *, expires_in_minutes: int) -> _DaytonaSshAccess: ... - - -class _DaytonaSnapshotClient(Protocol): - async def get(self, name: str) -> DaytonaSnapshot: ... - - async def delete(self, snapshot: DaytonaSnapshot) -> object: ... - - async def create(self, params: object) -> object: ... - - -class _DaytonaCreate(Protocol): - def __call__( - self, - params: object, - *, - timeout: int, - ) -> Awaitable[_DaytonaSandbox]: ... - - -class _DaytonaClient(Protocol): - snapshot: _DaytonaSnapshotClient - create: _DaytonaCreate - - async def delete(self, sandbox: _DaytonaSandbox) -> object: ... - - -class _DaytonaFactory(Protocol): - def __call__(self) -> AbstractAsyncContextManager[_DaytonaClient]: ... - - -class _ObjectFactory(Protocol): - def __call__(self, *args: object, **kwargs: object) -> object: ... - - -class _ResourcesFactory(Protocol): - def __call__(self, *args: object, **kwargs: object) -> _DaytonaResources: ... - - -class _GpuTypeFactory(Protocol): - def __call__(self, value: str) -> object: ... - - -class _DaytonaImageFactory(Protocol): - def base(self, image: str) -> object: ... - - -class DaytonaModule(Protocol): - AsyncDaytona: _DaytonaFactory - CreateSandboxFromImageParams: _ObjectFactory - CreateSandboxFromSnapshotParams: _ObjectFactory - CreateSnapshotParams: _ObjectFactory - DaytonaNotFoundError: type[Exception] - GpuType: _GpuTypeFactory - Image: _DaytonaImageFactory - Resources: _ResourcesFactory - SessionExecuteRequest: _ObjectFactory diff --git a/hud/eval/docker-seccomp.json b/hud/eval/docker-seccomp.json index 3b1ee17e0..b9e82ad8f 100644 --- a/hud/eval/docker-seccomp.json +++ b/hud/eval/docker-seccomp.json @@ -12,10 +12,20 @@ "perf_event_open", "process_vm_readv", "process_vm_writev", - "ptrace", "userfaultfd" ], "action": "SCMP_ACT_ERRNO" + }, + { + "names": [ + "ptrace" + ], + "action": "SCMP_ACT_ERRNO", + "excludes": { + "caps": [ + "CAP_SYS_PTRACE" + ] + } } ] } diff --git a/hud/eval/run.py b/hud/eval/run.py index 229bb2d9b..dca443403 100644 --- a/hud/eval/run.py +++ b/hud/eval/run.py @@ -24,9 +24,11 @@ import asyncio import contextlib import logging +import tempfile import traceback import uuid from dataclasses import dataclass, field +from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, Self, cast import mcp.types as mcp_types @@ -113,10 +115,6 @@ def from_dict(cls, data: dict[str, Any]) -> Grade: raise HudProtocolError(-32603, "tasks.grade: result must include a numeric 'score'") raw_info = data.get("info") raw = dict(data) - if isinstance(subscores := data.get("subscores"), list): - raw["subscores"] = [ - SubScore.model_validate(subscore).to_summary() for subscore in subscores - ] return cls( reward=float(score), done=bool(data.get("done", True)), @@ -194,8 +192,13 @@ def reward(self) -> float: @property def evaluation(self) -> dict[str, Any]: - """The persistence-safe evaluation dict (``grade.raw``).""" - return self.grade.raw + """A persistence-safe view of the task's evaluation result.""" + evaluation = dict(self.grade.raw) + if isinstance(subscores := evaluation.get("subscores"), list): + evaluation["subscores"] = [ + SubScore.model_validate(subscore).to_summary() for subscore in subscores + ] + return evaluation @property def trace_id(self) -> str | None: @@ -291,6 +294,16 @@ async def __aexit__( raise detail = "".join(traceback.format_exception_only(grade_exc)).strip() logger.warning("best-effort grade failed: %s", detail) + self.grade = Grade( + content=detail, + is_error=True, + raw={ + "score": 0.0, + "answer": self.trace.content, + "content": detail, + "isError": True, + }, + ) self.trace.status = "error" self.record(Step(source="system", error=f"[grading] {detail}")) return False @@ -326,7 +339,12 @@ def failed(cls, error: str) -> Run: return run -async def _verify(run: Run, client: HudClient, task: Task) -> None: +async def _verify( + run: Run, + client: HudClient, + task: Task, + actor_result: dict[str, Any], +) -> None: """Run an agent-less verifier task and make its evaluation authoritative.""" started_at = now_iso() started = await client.start_task(task.id, task.args) @@ -343,7 +361,7 @@ async def _verify(run: Run, client: HudClient, task: Task) -> None: ) ) - answer = {"answer": run.trace.content} + answer = {"answer": actor_result} started_at = now_iso() evaluation = await client.grade(answer) run.grade = Grade.from_dict(evaluation) @@ -432,76 +450,92 @@ async def rollout( async def _drive() -> None: nonlocal client, run, _phase - async with runtime(task) as addr: - _phase = "starting task" - async with connect(addr) as actor_client: - client = actor_client - live = Run( - actor_client, - task.id, - task.args, - best_effort_grade=task.verifier is not None, - ) - live._runtime = addr.url # the placement record for the receipt - async with live: # start on enter; complete on exit - run = live # bound only once live: an earlier failure synthesizes - _phase = "agent loop" - try: - async with file_tracking_observer(actor_client): - if agent_timeout is None: - await agent(run) - else: - deadline = asyncio.timeout(agent_timeout) - try: - async with deadline: - await agent(run) - except TimeoutError: - if not deadline.expired(): - raise - detail = f"agent timed out after {agent_timeout:g}s" - logger.warning(detail) - run.trace.status = "error" - run.trace.stop_reason = "timeout" - run.record(Step(source="system", error=detail)) - except Exception as exc: - if task.verifier is None: - raise - detail = "".join(traceback.format_exception_only(exc)).strip() - logger.warning("rollout failed mid-run (%s): %s", _phase, detail) - run.trace.status = "error" - run.record(Step(source="system", error=f"[{_phase}] {detail}")) - _phase = "grading" - - verifier = task.verifier - if verifier is not None: - # The verifier is authoritative. Once its phase begins, - # an actor-side grade must not survive a verifier failure. - live.grade = Grade() - if ( - verifier is not None - and verifier.env == task.env - and verifier.runtime_config is None - ): - _phase = "verifying" - await _verify(live, actor_client, verifier) - _phase = "cleanup" - return - - _phase = "actor cleanup" - - if rollout_expired: - return + actor_result: dict[str, Any] = {} verifier = task.verifier - if verifier is not None: - _phase = "provisioning verifier" - async with ( - runtime(verifier) as verifier_addr, - connect(verifier_addr) as verifier_client, - ): - client = verifier_client - _phase = "verifying" - await _verify(live, verifier_client, verifier) - _phase = "cleanup" + shared_verifier = ( + verifier is not None + and verifier.env == task.env + and verifier.runtime_config is None + ) + transfer_handoff = verifier is not None and verifier.requires_handoff + with tempfile.TemporaryDirectory(prefix="hud-handoff-") as directory: + handoff = Path(directory) / "handoff.tar.gz" + async with runtime(task) as addr: + if transfer_handoff and not shared_verifier and addr.handoff is None: + raise ValueError("the actor runtime cannot transfer verifier handoff files") + _phase = "starting task" + async with connect(addr) as actor_client: + client = actor_client + live = Run( + actor_client, + task.id, + task.args, + best_effort_grade=task.verifier is not None, + ) + live._runtime = addr.url # the placement record for the receipt + async with live: # start on enter; complete on exit + run = live # bound only once live: an earlier failure synthesizes + _phase = "agent loop" + try: + async with file_tracking_observer(actor_client): + if agent_timeout is None: + await agent(run) + else: + deadline = asyncio.timeout(agent_timeout) + try: + async with deadline: + await agent(run) + except TimeoutError: + if not deadline.expired(): + raise + detail = f"agent timed out after {agent_timeout:g}s" + logger.warning(detail) + run.trace.status = "error" + run.trace.stop_reason = "timeout" + run.record(Step(source="system", error=detail)) + except Exception as exc: + if task.verifier is None: + raise + detail = "".join(traceback.format_exception_only(exc)).strip() + logger.warning("rollout failed mid-run (%s): %s", _phase, detail) + run.trace.status = "error" + run.record(Step(source="system", error=f"[{_phase}] {detail}")) + _phase = "grading" + + if verifier is not None: + actor_result = live.grade.raw + # The verifier is authoritative. Once its phase begins, + # an actor-side grade must not survive a verifier failure. + live.grade = Grade() + if shared_verifier: + assert verifier is not None + _phase = "verifying" + await _verify(live, actor_client, verifier, actor_result) + _phase = "cleanup" + return + + _phase = "actor cleanup" + if transfer_handoff: + assert addr.handoff is not None + await addr.handoff.export_to(handoff) + + if rollout_expired: + return + if verifier is not None: + _phase = "provisioning verifier" + async with runtime(verifier) as verifier_addr: + if transfer_handoff and verifier_addr.handoff is None: + raise ValueError( + "the verifier runtime cannot receive actor handoff files" + ) + if transfer_handoff: + assert verifier_addr.handoff is not None + await verifier_addr.handoff.import_from(handoff) + async with connect(verifier_addr) as verifier_client: + client = verifier_client + _phase = "verifying" + await _verify(live, verifier_client, verifier, actor_result) + _phase = "cleanup" driver = asyncio.create_task(_drive()) try: diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py deleted file mode 100644 index ec893f448..000000000 --- a/hud/eval/runtime.py +++ /dev/null @@ -1,1692 +0,0 @@ -"""Provider: server placement — where the env's control channel comes from. - -A :class:`Provider` brings up the *server* (the env's control channel) for one -rollout and yields its connectable :class:`Runtime`; the agent loop drives it -from this process (:func:`hud.eval.run.rollout`). The channel is location -transparent, so "co-located" (loopback) and "split" (agent here, env -elsewhere) are the same code, differing only in the url. - -- :class:`LocalRuntime` — serve a fresh env per rollout, in this process, - from any pointer to it: a ``.py`` source path, a live module-level - :class:`Environment` (its declaring file is the recipe), or a - ``(task) -> Environment`` constructor. -- :class:`SubprocessRuntime` — serve the row's env from a ``.py`` source in a - child process, when the env should not share the orchestrator's fate. -- :class:`DockerRuntime` — starts an image or Compose environment whose primary - service serves the channel. -- ``Runtime(url)`` — the ``nullcontext`` of providers: yields itself, a - *borrowed, shared* substrate provisioned elsewhere (env served anywhere — - a cloud sandbox, another host — that this process connects to). - -The provider contract is structural (anything callable as ``(task) -> async -context manager of Runtime``), so per-task heterogeneity (this row on 1 GPU, -that one on 4, different images) is just a provider that reads the row. - -The delegated placement — :class:`HostedRuntime`, running the whole rollout -off-box on a HUD sandbox — also lives here; the scheduler (:meth:`Taskset.run`) -chooses between it and providers. A hosted box's own driver is itself a -``Provider`` (its ``DockerRuntime``) driven by the same ``rollout`` atom — -co-location all the way down. -""" - -from __future__ import annotations - -import asyncio -import contextlib -import importlib -import logging -import os -import shlex -import sys -import uuid -from collections import deque -from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext -from dataclasses import dataclass, field -from pathlib import Path -from typing import TYPE_CHECKING, Any, Protocol, Self, cast -from urllib.parse import urlsplit, urlunsplit - -import httpx -from pydantic import BaseModel, ConfigDict, Field, model_validator - -from hud.telemetry.context import get_current_trace_id -from hud.types import Step -from hud.utils.docker import docker as _docker -from hud.utils.platform import PlatformClient -from hud.utils.process import ProcessGroup, create_process_group_exec - -from .compose import ComposeConfig, ComposeProject, ComposeProjectRef, ComposeSource -from .run import Grade, Run, rollout - -if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence - - from hud.agents.base import Agent - from hud.environment.env import Environment - - from ._runtime_protocols import ( - DaytonaImage, - DaytonaModule, - DaytonaSnapshot, - ModalImage, - ModalModule, - ObjectStorageModule, - ) - from .task import Task - -logger = logging.getLogger("hud.eval.runtime") - -_MODAL_COMPOSE_CPU = 4.0 -_MODAL_COMPOSE_MEMORY_MB = 8192 - - -async def _prepare_compose_project(compose: Path) -> bool: - script = compose.parent / "build.sh" - if not script.is_file(): - return False - process = await create_process_group_exec( - "sh", - str(script), - cwd=str(compose.parent), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - result = await process.complete() - if result.returncode != 0: - detail = (result.stderr or result.stdout).decode("utf-8", "replace").strip() - raise RuntimeError(f"Compose project build failed: {detail}") - return True - - -class RuntimeGPU(BaseModel): - """Requested GPU resources, provider-neutral where possible.""" - - model_config = ConfigDict(extra="forbid") - - type: str | None = Field(default=None, min_length=1) - count: int = Field(default=1, ge=1) - - -class RuntimeResources(BaseModel): - """Requested compute resources for a runtime.""" - - model_config = ConfigDict(extra="forbid") - - cpu: float | None = Field(default=None, gt=0) - memory_mb: int | None = Field(default=None, gt=0) - gpu: RuntimeGPU | None = None - - -class RuntimeLimits(BaseModel): - """Runtime lifecycle limits in seconds.""" - - model_config = ConfigDict(extra="forbid") - - startup_timeout_s: int | None = Field(default=None, gt=0) - run_timeout_s: int | None = Field(default=None, gt=0) - - -class RuntimeConfig(BaseModel): - """Typed task-environment launch requirements. - - ``Task.runtime_config`` is requested construction input. ``Runtime.config`` - is the effective config used to construct a runtime. - - ``compose`` and ``compose_project`` are authored as local paths; platform - task records carry them as the serialized compose document and a - :class:`ComposeProjectRef`. Both forms validate; only the path form is - runnable by local providers. - """ - - model_config = ConfigDict(extra="forbid") - - image: str | None = Field(default=None, min_length=1) - compose: Path | ComposeConfig | None = None - compose_project: Path | ComposeProjectRef | None = None - compose_service_access: bool | None = None - resources: RuntimeResources | None = None - limits: RuntimeLimits | None = None - - @model_validator(mode="after") - def validate_source(self) -> Self: - if self.image is not None and self.compose is not None: - raise ValueError("runtime_config accepts either image or compose, not both") - if self.compose_project is not None and self.compose is None: - raise ValueError("compose_project requires runtime_config.compose") - if self.compose_service_access and self.compose is None: - raise ValueError("compose_service_access requires runtime_config.compose") - return self - - def with_overrides(self, override: RuntimeConfig | None) -> RuntimeConfig: - if override is None: - return self - config = self.model_dump() - changes = override.model_dump(exclude_unset=True) - if override.image is not None: - config["compose"] = None - config["compose_project"] = None - config["compose_service_access"] = None - elif override.compose is not None: - config["image"] = None - config["compose_project"] = None - return RuntimeConfig.model_validate(config | changes) - - def request_payload(self) -> dict[str, Any]: - payload = self.model_dump(mode="json", exclude_unset=True) - source = self.compose_source() - if source is not None: - payload.update(source.request_payload()) - return payload - - def compose_source(self) -> ComposeSource | None: - """The authored or wire-form Compose source, when configured.""" - if self.compose is None: - return None - return ComposeSource(self.compose, self.compose_project) - - -class Provider(Protocol): - """Server placement: called with the task row being placed, acquire one - fresh env substrate for it and yield its connectable :class:`Runtime`. - - A provider brings up the *server* (the env's control channel) wherever it - lives — a local subprocess, a container, a cloud sandbox — and the agent - loop drives it from this process (:func:`hud.eval.run.rollout`). The - channel is location-transparent, so "co-located" (loopback) and "split" - (agent here, env elsewhere) are the same code, differing only in the url. - """ - - def __call__(self, task: Task, /) -> AbstractAsyncContextManager[Runtime]: ... - - -@dataclass(frozen=True) -class Runtime: - """The connectable address of a provisioned substrate. - - ``url`` is the control-channel address (``tcp://127.0.0.1:7000`` for a - local process, ``tcp://sandbox-abc.hud.so:443`` for a hosted box). - ``params`` carries connection-time data a transport may need (auth token, - sandbox id). ``config`` is the effective runtime configuration used to - construct the runtime. Constructed directly, it is also a provider — the - borrowed, shared case: it yields itself with a no-op lifecycle, since - whoever provisioned the substrate owns its teardown. - """ - - url: str - params: dict[str, Any] = field(default_factory=dict) - config: RuntimeConfig | None = None - - def __call__(self, task: Task) -> AbstractAsyncContextManager[Runtime]: - return nullcontext(self) - - -class Shared: - """Lease provider: at most ``width`` concurrent rollouts share one substrate. - - The substrate boots lazily on the first lease and lives for the enclosing - ``async with`` scope — one boot however many rollouts flow through, torn - down deterministically at scope exit. ``width`` is the substrate's real - capacity (e.g. a vectorized sim's slot count): lease ``width + 1`` waits - for a slot instead of erroring, so the scheduler needs no pairing — - ``group`` and ``max_concurrent`` keep their ordinary meanings. - - ``Taskset.run`` scopes a context-manager placement to the call, so - ``runtime=Shared(DockerRuntime(...), width=8)`` works bare; open the scope - yourself to keep the substrate warm across several calls:: - - async with Shared(DockerRuntime("hud-isaac-env"), width=8) as rt: - await taskset.run(agent, runtime=rt, group=8) - """ - - def __init__(self, inner: Provider, *, width: int) -> None: - if width < 1: - raise ValueError("Shared width must be >= 1") - self.inner = inner - self.width = width - self._sem = asyncio.Semaphore(width) - self._boot = asyncio.Lock() - self._addr: Runtime | None = None - self._stack: contextlib.AsyncExitStack | None = None - self._opens = 0 - - async def __aenter__(self) -> Self: - self._opens += 1 - return self - - async def __aexit__(self, *exc: object) -> None: - self._opens -= 1 - if self._opens == 0 and self._stack is not None: - stack, self._stack, self._addr = self._stack, None, None - await stack.aclose() - - @asynccontextmanager - async def __call__(self, task: Task) -> AsyncIterator[Runtime]: - if self._opens == 0: - raise RuntimeError( - "Shared substrates outlive single rollouts; lease inside the scope " - "(Taskset.run opens it for you, or wrap calls in `async with Shared(...)`)" - ) - async with self._sem: - async with self._boot: - if self._addr is None: - # First leaseholder boots. A failed boot fails only its own - # rollout (nothing entered the stack); the next lease retries. - stack = contextlib.AsyncExitStack() - self._addr = await stack.enter_async_context(self.inner(task)) - self._stack = stack - addr = self._addr - yield addr - - -def _modal_image_from_uri(modal: ModalModule, image_uri: str) -> ModalImage: - modal_uri_prefix = "modal://" - if image_uri.startswith(modal_uri_prefix): - return modal.Image.from_id(image_uri.removeprefix(modal_uri_prefix)) - return modal.Image.from_registry(image_uri) - - -#: DockerRuntime always serves HUD environments, so this is part of the -#: provider contract rather than a per-image option. This is intentionally a -#: default-allow compatibility profile: Workspace's bwrap sessions need the -#: namespace and mount syscalls, while unrelated kernel interfaces stay denied. -_DOCKER_SECCOMP_PROFILE = Path(__file__).with_name("docker-seccomp.json") -_DOCKER_SECURITY_ARGS = ( - "--security-opt", - f"seccomp={_DOCKER_SECCOMP_PROFILE}", - # Docker exposes system-path masking only as an all-or-nothing option; - # bwrap replaces the container's proc and dev mounts while building a wall. - "--security-opt", - "systempaths=unconfined", -) - - -class LocalRuntime: - """The local provider: serve a fresh env per rollout, in this process. - - *source* points at the env in whatever form you have: - - - a ``.py`` file or directory — imported fresh per acquisition (sibling - imports resolve); *env* pins one name when several are declared, - defaulting to the placed task's env - - a live :class:`~hud.environment.Environment` — shorthand for its - declaring file; the instance itself is never served - - a ``(task) -> Environment`` callable — called per acquisition with the - placed row - - :: - - runtime = LocalRuntime("env.py") - runtime = LocalRuntime(env) - runtime = LocalRuntime(lambda task: build_env(task.env)) - - ``ready_timeout`` bounds ``@env.initialize`` startup. Freshness covers - the env's own source; modules it imports are cached as usual and shared - across rollouts. Hooks share this process's event loop, so blocking env - code stalls concurrent rollouts — use :class:`SubprocessRuntime` or - :class:`DockerRuntime` for process isolation, and ``Runtime(url)`` to - attach to a substrate served elsewhere. - """ - - def __init__( - self, - source: str | Path | Environment | Callable[[Task], Environment], - *, - env: str | None = None, - ready_timeout: float = 120.0, - ) -> None: - from hud.environment.env import Environment as _Environment - - self.ready_timeout = ready_timeout - # A live instance may have been mutated since its module was imported; - # verify the fresh copy still declares its templates, so drift fails - # at acquisition with the cause named instead of "unknown task" later. - expected_templates: frozenset[str] = frozenset() - if isinstance(source, _Environment): - file = _declaring_file(source, env or source.name) - if file is None: - raise TypeError( - f"LocalRuntime: env {source.name!r} is not rebuilt by importing " - "any file this process has loaded (constructed in a function or " - "notebook cell, or declared inside a package using relative " - "imports); pass its constructor instead: " - "LocalRuntime(lambda task: )" - ) - expected_templates = frozenset(source.tasks) - source, env = file, env or source.name - self._source_dir: Path | None = None - if isinstance(source, (str, Path)): - path, pinned = Path(source).resolve(), env - self._source_dir = path if path.is_dir() else path.parent - from hud.environment import load_environment - - def _load(task: Task) -> _Environment: - loaded = load_environment(path, name=pinned or task.env) - missing = expected_templates - loaded.tasks.keys() - if missing: - raise ValueError( - f"env {loaded.name!r} loaded from {path} lacks template(s) " - f"{sorted(missing)} present on the live instance — it was " - "modified after import; pass a constructor instead: " - "LocalRuntime(lambda task: )" - ) - return loaded - - self._build: Callable[[Task], _Environment] = _load - elif callable(source): - if env is not None: - raise TypeError("LocalRuntime: env= applies only to source paths") - self._build = source - else: - raise TypeError( - f"LocalRuntime: expected a source path, a live Environment, or a " - f"(task) -> Environment constructor; got {source!r}" - ) - - @asynccontextmanager - async def __call__(self, task: Task) -> AsyncIterator[Runtime]: - from hud.environment.env import Environment as _Environment - - if task.runtime_config is not None: - raise ValueError("LocalRuntime does not support task runtime_config") - # The source dir stays importable for the whole acquisition, not just - # the initial import, so a template can lazily import a sibling - # module at run time (as it could under the child-process runtime). - # Always insert-and-remove one entry: balanced under concurrency. - if self._source_dir is not None: - sys.path.insert(0, str(self._source_dir)) - try: - try: - env = self._build(task) - except RuntimeError as e: - # The source ran an event loop at import — usually an unguarded - # top-level run call; name the actual mistake. - if "running event loop" not in str(e): - raise - raise RuntimeError( - "the env source ran async code while being imported to place a " - 'rollout — guard top-level run calls with `if __name__ == "__main__":`' - ) from e - if not isinstance(env, _Environment): - raise TypeError(f"LocalRuntime: constructor returned {env!r}, not an Environment") - async with _local(env, ready_timeout=self.ready_timeout) as runtime: - yield runtime - finally: - if self._source_dir is not None: - with contextlib.suppress(ValueError): - sys.path.remove(str(self._source_dir)) - - -def _live_envs() -> Iterator[tuple[Environment, str]]: - """Envs declared in loaded, file-backed modules' globals, with their files. - - The in-memory counterpart of scanning ``.py`` sources on disk - (:func:`~hud.environment.load_environment`): an env found here can be - served fresh by re-importing its file. Envs in modules without a file - (a notebook ``__main__``) are not yielded — re-import could not - reconstruct them. - """ - from hud.environment.env import Environment as _Environment - - for module in list(sys.modules.values()): - module_file = getattr(module, "__file__", None) - module_vars = getattr(module, "__dict__", None) - if not module_file or not isinstance(module_vars, dict): - continue - for value in list(module_vars.values()): - if isinstance(value, _Environment): - yield value, module_file - - -def _declaring_file(env: Environment, name: str) -> Path | None: - """A file whose fresh import re-declares *env*, else None. - - Candidate files hold the instance in their module globals, but a holder - may be a re-exporter (``from .env import env`` in a package - ``__init__``, a tasks file re-exporting its env): validate each by - loading it fresh — a declarer yields a *new* instance under *name*, a - re-exporter yields the same live one (or fails to import standalone). - ``__init__.py`` holders are tried last. - """ - from hud.environment import load_environment - - candidates = dict.fromkeys(Path(file) for live, file in _live_envs() if live is env) - for file in sorted(candidates, key=lambda f: f.name == "__init__.py"): - try: - probe = load_environment(file, name=name) - except Exception as e: - logger.debug("candidate %s does not rebuild env %r: %s", file, name, e) - continue - if probe is not env: - return file - return None - - -def _declared_env(name: str) -> Environment | None: - """The one live env named *name*, else None; two distinct ones raise. - - The same instance re-exported across modules is one match; distinct envs - claiming one name are ambiguous. - """ - matches = {id(env): env for env, _ in _live_envs() if env.name == name} - if len(matches) > 1: - files = sorted({file for env, file in _live_envs() if env.name == name}) - raise ValueError( - f"env name {name!r} is declared by multiple live environments " - f"({', '.join(files)}); pass runtime= explicitly — the exact " - "instance disambiguates: runtime=LocalRuntime(env)" - ) - return next(iter(matches.values()), None) - - -def _declared_names(source: Path) -> set[str]: - """Env names a ``.py`` source (file or directory) itself declares. - - A fresh execution of the source yields *new* instances for envs it - declares; an env it merely imports is the already-live one and does not - count — importing the source again could not rebuild it. - """ - from hud.environment.env import Environment as _Environment - from hud.utils.modules import iter_modules - - live = {id(env) for env, _ in _live_envs()} - return { - value.name - for module in iter_modules(source) - for value in vars(module).values() - if isinstance(value, _Environment) and id(value) not in live - } - - -class SubprocessRuntime: - """The child-process provider: serve the placed row's env from *path*. - - Each acquisition runs ``python -m hud.environment.server --env - name`` — the same serving entry point a container CMD runs — on an - ephemeral loopback port, yields its :class:`Runtime`, and terminates the - child on exit. *path* is a ``.py`` file or a directory of them. The served - env is the placed task's ``env`` name (so a mixed-env taskset works - against one source), unless *env* pins one explicitly; placing a row whose - env the source does not define fails loudly in the child. - - The child's working directory is the source's directory, so sibling - imports and relative data paths resolve; ``@env.initialize`` daemons start - in the child and die with it. Because the source is re-imported in the - child, a script spawning itself (``SubprocessRuntime(__file__)``) must keep - top-level run calls under ``if __name__ == "__main__":``. - """ - - def __init__( - self, - path: str | Path, - *, - env: str | None = None, - ready_timeout: float = 120.0, - ) -> None: - self.source = Path(path).resolve() - self.env = env - self.ready_timeout = ready_timeout - - @asynccontextmanager - async def __call__(self, task: Task) -> AsyncIterator[Runtime]: - if task.runtime_config is not None: - raise ValueError("SubprocessRuntime does not support task runtime_config") - if not self.source.exists(): - raise FileNotFoundError(f"SubprocessRuntime: source not found: {self.source}") - cmd = [sys.executable, "-m", "hud.environment.server", str(self.source)] - cmd += ["--env", self.env or task.env] - proc = await create_process_group_exec( - *cmd, - term_timeout=10.0, - stdout=asyncio.subprocess.PIPE, - # Capture stderr (don't inherit it): under concurrent rollouts an - # inherited fd interleaves every child's output unattributably, so a - # crash-before-serving leaves no traceable diagnostic. We keep a - # bounded tail and attach it to the failure below. - stderr=asyncio.subprocess.PIPE, - cwd=self.source if self.source.is_dir() else self.source.parent, - ) - assert proc.stderr is not None - # Drain stderr into a bounded tail from the start: it never blocks on a - # full pipe, and the last lines survive if the child dies early. - stderr_tail: deque[str] = deque(maxlen=50) - capture = asyncio.create_task(_capture(proc.stderr, stderr_tail)) - try: - assert proc.stdout is not None - port = await asyncio.wait_for(_read_port(proc.stdout), self.ready_timeout) - if port is None: - raise RuntimeError(await _exit_detail(proc, self.source, capture, stderr_tail)) - drain = asyncio.create_task(_drain(proc.stdout)) - try: - yield Runtime(f"tcp://127.0.0.1:{port}") - finally: - drain.cancel() - with contextlib.suppress(asyncio.CancelledError): - await drain - finally: - capture.cancel() - with contextlib.suppress(asyncio.CancelledError): - await capture - await proc.terminate() - - -class DockerRuntime: - """Start a HUD environment from an image or a Docker Compose file. - - An image is started with ``docker run``. A Compose file is started unchanged - except for a small provider override that publishes the ``main`` service's - control-channel port and applies HUD's nested-workspace security profile. - """ - - def __init__( - self, - image: str | None = None, - *, - port: int = 8765, - run_args: Sequence[str] = (), - runtime_config: RuntimeConfig | dict[str, Any] | None = None, - ) -> None: - self.port = port - self.run_args = tuple(run_args) - config = RuntimeConfig(image=image) if image is not None else RuntimeConfig() - if runtime_config is not None: - config = config.with_overrides(RuntimeConfig.model_validate(runtime_config)) - self.runtime_config = config if config.model_dump(exclude_none=True) else None - self._compose_preparation_locks: dict[Path, asyncio.Lock] = {} - - @asynccontextmanager - async def __call__(self, task: Task) -> AsyncIterator[Runtime]: - config = (self.runtime_config or RuntimeConfig()).with_overrides(task.runtime_config) - if config.limits is not None and config.limits.model_dump(exclude_none=True): - raise ValueError("DockerRuntime does not support runtime_config limits") - compose_source = config.compose_source() - if compose_source is not None: - if self.run_args: - raise ValueError("DockerRuntime run_args apply only to image environments") - compose = compose_source.runnable_path("DockerRuntime") - resources = config.resources - if ( - resources is not None - and resources.gpu is not None - and resources.gpu.type is not None - ): - raise ValueError("DockerRuntime cannot select Compose GPUs by type") - service_socket = None - if config.compose_service_access: - endpoint = os.environ.get("DOCKER_HOST") - if not endpoint: - endpoint, _ = await _docker( - "context", - "inspect", - "--format", - "{{.Endpoints.docker.Host}}", - ) - endpoint = endpoint.strip() - parsed = urlsplit(endpoint) - if parsed.scheme != "unix" or not parsed.path: - raise ValueError( - "DockerRuntime Compose service access requires a local Unix " - f"Docker endpoint, got {endpoint!r}" - ) - service_socket = parsed.path - project_files = ComposeProject(compose) - project = f"hud-{uuid.uuid4().hex[:12]}" - lock = self._compose_preparation_locks.setdefault(compose, asyncio.Lock()) - async with lock: - prepared = await _prepare_compose_project(compose) - with project_files.stage( - f"127.0.0.1::{self.port}", - seccomp=_DOCKER_SECCOMP_PROFILE, - service_socket=service_socket, - cpu=resources.cpu if resources is not None else None, - memory_mb=resources.memory_mb if resources is not None else None, - gpu_count=( - resources.gpu.count - if resources is not None and resources.gpu is not None - else None - ), - ) as files: - command = ( - "compose", - "--project-name", - project, - "--file", - str(files.compose), - "--file", - str(files.override), - "--file", - str(files.ports), - ) - try: - await _docker( - *command, - "up", - "--detach", - "--no-build" if prepared else "--build", - "--remove-orphans", - ) - mapping, _ = await _docker(*command, "port", "main", str(self.port)) - if not mapping.strip(): - logs_out, logs_err = await _docker( - *command, "logs", "--tail", "40", "main", check=False - ) - raise RuntimeError( - f"Compose main service exited before serving port {self.port}:\n" - f"{(logs_err or logs_out).strip()}" - ) - host_port = int(mapping.strip().splitlines()[0].rsplit(":", 1)[1]) - yield Runtime( - f"tcp://127.0.0.1:{host_port}", - config=config if config.model_dump(exclude_none=True) else None, - ) - finally: - await _docker( - *command, - "down", - "--volumes", - "--remove-orphans", - check=False, - ) - return - if config.image is None: - raise ValueError( - "DockerRuntime requires runtime_config.image or runtime_config.compose" - ) - - resource_args: list[str] = [] - resources = config.resources - if resources is not None: - if resources.cpu is not None: - cpu = ( - str(int(resources.cpu)) - if isinstance(resources.cpu, float) and resources.cpu.is_integer() - else str(resources.cpu) - ) - resource_args.extend(("--cpus", cpu)) - if resources.memory_mb is not None: - resource_args.extend(("--memory", f"{resources.memory_mb}m")) - if resources.gpu is not None: - if resources.gpu.type is not None: - raise ValueError("DockerRuntime cannot select GPUs by type") - resource_args.extend(("--gpus", str(resources.gpu.count))) - - out, _ = await _docker( - "run", - "--detach", - *self.run_args, - *resource_args, - *_DOCKER_SECURITY_ARGS, - "--publish", - f"127.0.0.1::{self.port}", - config.image, - ) - container = out.strip() - try: - mapping, _ = await _docker("port", container, str(self.port)) - if not mapping.strip(): - logs_out, logs_err = await _docker("logs", "--tail", "40", container, check=False) - raise RuntimeError( - f"container for image {config.image!r} exited before serving port " - f"{self.port}:\n{(logs_err or logs_out).strip()}", - ) - host_port = int(mapping.strip().splitlines()[0].rsplit(":", 1)[1]) - yield Runtime(f"tcp://127.0.0.1:{host_port}", config=config) - finally: - # check=False: teardown must not shadow the run's own error, and - # rm -f only fails when the daemon itself is broken. - await _docker("rm", "--force", container, check=False) - - -class ModalRuntime: - """The Modal provider: each acquisition ``Sandbox.create``s a fresh container. - - The cloud :class:`DockerRuntime` — boots a sandbox from a pre-built image, - exposes the env's control channel as a raw-TCP tunnel (``unencrypted_ports``, - the only kind :func:`hud.clients.connect` dials), yields its :class:`Runtime`, - terminates on exit. Acquisitions are independent, so a batch fans out into - isolated containers (one ``sb-…`` id each). - - The image resolves once (so concurrent rollouts can't race a build): pass a - published name — ``ModalRuntime("hud-libero-env")``, the preferred durable - handle — or, as an escape hatch, an ``Image`` to build lazily on first use. - Requires the ``modal`` extra and a configured token. - """ - - def __init__( - self, - image_name: str | None = None, - *, - image: ModalImage | None = None, - command: Sequence[str] | None = None, - app_name: str = "hud-envs", - workdir: str | None = None, - port: int = 8765, - runtime_config: RuntimeConfig | dict[str, Any] | None = None, - env_vars: Mapping[str, str] | None = None, - ) -> None: - self.image_name = image_name - self.port = port - self.env_vars = dict(env_vars or {}) - self.workdir = workdir - # Default CMD mirrors the scaffolded Dockerfile.hud entrypoint. Leave - # workdir unset by default so Modal preserves the image WORKDIR. - self.command = ( - tuple(command) - if command is not None - else ( - "hud", - "serve", - "env.py", - "--host", - "0.0.0.0", # noqa: S104 - serving inside the sandbox; the tunnel is the only ingress - "--port", - str(port), - ) - ) - self.app_name = app_name - config = None - if runtime_config is not None: - config = RuntimeConfig.model_validate(runtime_config) - self.runtime_config = config - # Resolved (named) or built-once (from Dockerfile) image, behind a lock so - # concurrent first acquisitions build/look up exactly once. - self._image = image - self._resolved: ModalImage | None = None - self._image_lock = asyncio.Lock() - - @asynccontextmanager - async def __call__(self, task: Task) -> AsyncIterator[Runtime]: - config = (self.runtime_config or RuntimeConfig()).with_overrides(task.runtime_config) - compose_source = config.compose_source() - compose = ( - compose_source.runnable_path("ModalRuntime") if compose_source is not None else None - ) - modal = cast("ModalModule", importlib.import_module("modal")) - - app = None - if compose is not None: - image = modal.Image.from_registry("docker:28.3.3-dind") - elif config.image is not None: - image = _modal_image_from_uri(modal, config.image) - elif self.image_name is not None: - image = modal.Image.from_name(self.image_name) - elif self._image is None: - raise ValueError( - "ModalRuntime requires image=, image_name=, runtime_config.image, " - "or runtime_config.compose" - ) - else: - if self._resolved is None: - async with self._image_lock: - if self._resolved is None: - app = await modal.App.lookup.aio( - self.app_name, - create_if_missing=True, - ) - await self._image.build.aio(app=app) - self._resolved = self._image - image = self._resolved - if self.env_vars and compose is None: - image = image.env(self.env_vars) - - if app is None: - app = await modal.App.lookup.aio(self.app_name, create_if_missing=True) - - sandbox_kwargs: dict[str, Any] = {} - resources = config.resources - if compose is not None: - sandbox_kwargs["cpu"] = max( - resources.cpu if resources is not None and resources.cpu is not None else 0, - _MODAL_COMPOSE_CPU, - ) - sandbox_kwargs["memory"] = max( - ( - resources.memory_mb - if resources is not None and resources.memory_mb is not None - else 0 - ), - _MODAL_COMPOSE_MEMORY_MB, - ) - else: - if resources is not None and resources.cpu is not None: - sandbox_kwargs["cpu"] = resources.cpu - if resources is not None and resources.memory_mb is not None: - sandbox_kwargs["memory"] = resources.memory_mb - if resources is not None and resources.gpu is not None: - gpu_type = resources.gpu.type or "any" - gpu = gpu_type if resources.gpu.count == 1 else f"{gpu_type}:{resources.gpu.count}" - sandbox_kwargs["gpu"] = gpu - - run_timeout = 3600 - ready_timeout = 600 - if config.limits is not None: - run_timeout = config.limits.run_timeout_s or run_timeout - ready_timeout = config.limits.startup_timeout_s or ready_timeout - - sb = await modal.Sandbox.create.aio( - *(() if compose is not None else self.command), - app=app, - image=image, - workdir=None if compose is not None else self.workdir, - unencrypted_ports=[self.port], - readiness_probe=(None if compose is not None else modal.Probe.with_tcp(self.port)), - # Modal types both timeouts as int seconds; floats raise at proto encode. - timeout=run_timeout, - **({"experimental_options": {"vm_runtime": True}} if compose is not None else {}), - **sandbox_kwargs, - ) - try: - if compose is None: - await sb.wait_until_ready.aio(timeout=ready_timeout) - else: - project = ComposeProject(compose) - with project.stage( - f"{self.port}:{self.port}", - seccomp="/hud/docker-seccomp.json", - service_socket=( - "/var/run/docker.sock" if config.compose_service_access else None - ), - env_vars=self.env_vars, - cpu=resources.cpu if resources is not None else None, - memory_mb=resources.memory_mb if resources is not None else None, - gpu_count=( - resources.gpu.count - if resources is not None and resources.gpu is not None - else None - ), - archive=True, - ) as files: - assert files.archive is not None - await sb.filesystem.copy_from_local.aio(files.archive, "/hud/project.tar.gz") - await sb.filesystem.copy_from_local.aio(files.override, "/hud/override.json") - await sb.filesystem.copy_from_local.aio(files.ports, "/hud/ports.yaml") - await sb.filesystem.copy_from_local.aio( - _DOCKER_SECCOMP_PROFILE, "/hud/docker-seccomp.json" - ) - command = ( - "mkdir -p /hud/project && " - "tar -xzf /hud/project.tar.gz -C /hud/project && " - "until docker info >/dev/null 2>&1; do sleep 1; done && " - "BUILD_FLAG=--build && " - "if [ -f /hud/project/build.sh ]; then " - "sh /hud/project/build.sh && BUILD_FLAG=--no-build; fi && " - "docker compose --project-directory /hud/project " - f"--file /hud/project/{shlex.quote(compose.name)} " - "--file /hud/override.json --file /hud/ports.yaml " - 'up --detach "$BUILD_FLAG" --remove-orphans' - ) - process = await sb.exec.aio("sh", "-c", command, timeout=ready_timeout) - if await process.wait.aio() != 0: - error = (await process.stderr.read.aio()).strip() - raise RuntimeError(f"Modal Compose startup failed: {error}") - host, port = (await sb.tunnels.aio())[self.port].tcp_socket - yield Runtime( - f"tcp://{host}:{port}", - params={ - "provider": "modal", - "instance_id": sb.object_id, - **({"ready_timeout": ready_timeout} if compose is not None else {}), - }, - config=config if config.model_dump(exclude_none=True) else None, - ) - finally: - # check-free teardown: never shadow the run's own error. - if compose is not None: - with contextlib.suppress(Exception): - process = await sb.exec.aio( - "docker", - "compose", - "--project-directory", - "/hud/project", - "--file", - f"/hud/project/{compose.name}", - "--file", - "/hud/override.json", - "--file", - "/hud/ports.yaml", - "down", - "--volumes", - "--remove-orphans", - timeout=30, - ) - await process.wait.aio() - with contextlib.suppress(Exception): - await sb.terminate.aio() - - -async def _snapshot_is_current( - snapshot: DaytonaSnapshot, - image: str | DaytonaImage, -) -> bool: - """Whether *snapshot* was built from *image* as it exists right now. - - Daytona records what a snapshot was built from — the registry ref, or for a - built ``Image`` its Dockerfile text plus the hashes of the context it - uploads (``build_info``). The hashes are recomputed here with the SDK's own - hasher so they are comparable to what ``snapshot.create`` would upload. - ``_context_list`` is the same private attribute ``snapshot.create`` reads. - """ - if isinstance(image, str): - return bool(snapshot.image_name == image) - build = snapshot.build_info - if build is None: - return False - object_storage = cast( - "ObjectStorageModule", - importlib.import_module("daytona._async.object_storage"), - ) - AsyncObjectStorage = object_storage.AsyncObjectStorage - - # The hasher is an instance method only for code organization; credentials - # are needed to upload, not to hash, so skip the credentialed __init__. - storage = AsyncObjectStorage.__new__(AsyncObjectStorage) - hashes = [ - await storage._compute_hash_for_path_md5(entry.source_path, entry.archive_path) - for entry in image._context_list - ] - return bool( - build.dockerfile_content == image.dockerfile() - and list(build.context_hashes or []) == hashes - ) - - -class DaytonaRuntime: - """The Daytona provider: each acquisition creates a fresh sandbox from a snapshot. - - The Daytona runtime boots a sandbox from a pre-built *snapshot* - (the durable handle, the snapshot equivalent of Modal's image name), starts the - env's control channel inside it, then reaches it over an SSH local-forward: - Daytona exposes services only as HTTPS previews, but :func:`hud.clients.connect` - dials ``tcp://``, so the raw control channel is tunneled over SSH to a local - port. Yields its :class:`Runtime`, deletes the sandbox on exit. - - Pass a snapshot name — ``DaytonaRuntime("hud-libero-env")`` — optionally with an - ``image`` (Dockerfile/registry ref) to build that snapshot if it is missing. - With *image*, an existing snapshot is compared against the image's recorded - build (Dockerfile plus context hashes) and rebuilt under the same name when - they differ, so editing the env never silently reuses the snapshot built - before the edit. - Resources (cpu/memory/gpu) live on the snapshot, not here. *workdir* defaults to - ``/app`` (the scaffolded ``Dockerfile.hud`` WORKDIR) since a Daytona session - starts in ``~``, not the image's WORKDIR; override only for a non-standard layout. - Requires the ``daytona`` extra and ``DAYTONA_API_KEY``. - """ - - def __init__( - self, - snapshot_name: str | None = None, - *, - image: str | DaytonaImage | None = None, - command: str | None = None, - workdir: str | None = "/app", - port: int = 8765, - ssh_host: str = "ssh.app.daytona.io", - ssh_expires_minutes: int = 24 * 60, - runtime_config: RuntimeConfig | dict[str, Any] | None = None, - ) -> None: - self.snapshot_name = snapshot_name - # Default command serves on *port*, so the SSH forward target always - # matches what's listening; override only for a non-default layout. - self.command = ( - command or f'PATH="$PWD/.venv/bin:$PATH" hud serve env.py --host 0.0.0.0 --port {port}' - ) - self.workdir = workdir - self.port = port - self.ssh_host = ssh_host - self.ssh_expires_minutes = ssh_expires_minutes - config = None - if runtime_config is not None: - config = RuntimeConfig.model_validate(runtime_config) - self.runtime_config = config - # Resolve each snapshot name against the image once; lock so concurrent - # first acquisitions resolve exactly once. - self._image = image - self._resolved: set[str] = set() - self._snapshot_lock = asyncio.Lock() - - @asynccontextmanager - async def __call__(self, task: Task) -> AsyncIterator[Runtime]: - import asyncssh - - daytona_sdk = cast("DaytonaModule", importlib.import_module("daytona")) - AsyncDaytona = daytona_sdk.AsyncDaytona - CreateSandboxFromImageParams = daytona_sdk.CreateSandboxFromImageParams - CreateSandboxFromSnapshotParams = daytona_sdk.CreateSandboxFromSnapshotParams - CreateSnapshotParams = daytona_sdk.CreateSnapshotParams - DaytonaNotFoundError = daytona_sdk.DaytonaNotFoundError - GpuType = daytona_sdk.GpuType - Image = daytona_sdk.Image - Resources = daytona_sdk.Resources - SessionExecuteRequest = daytona_sdk.SessionExecuteRequest - - async with AsyncDaytona() as daytona: - config = (self.runtime_config or RuntimeConfig()).with_overrides(task.runtime_config) - if config.compose is not None: - raise ValueError("DaytonaRuntime does not support runtime_config.compose") - if config.limits is not None and config.limits.run_timeout_s is not None: - raise ValueError("DaytonaRuntime does not support runtime_config.run_timeout_s") - - daytona_resources = None - if config.resources is not None: - resource_kwargs: dict[str, Any] = {} - if config.resources.cpu is not None: - # Daytona allocates whole cores; truncating resizes silently. - if ( - isinstance(config.resources.cpu, float) - and not config.resources.cpu.is_integer() - ): - raise ValueError( - "DaytonaRuntime needs a whole number of CPUs, got " - f"{config.resources.cpu}" - ) - resource_kwargs["cpu"] = int(config.resources.cpu) - if config.resources.memory_mb is not None: - resource_kwargs["memory"] = max( - 1, - (config.resources.memory_mb + 1023) // 1024, - ) - if config.resources.gpu is not None: - resource_kwargs["gpu"] = config.resources.gpu.count - if config.resources.gpu.type is not None: - resource_kwargs["gpu_type"] = [GpuType(config.resources.gpu.type)] - if resource_kwargs: - daytona_resources = Resources(**resource_kwargs) - - if config.image is not None: - sandbox_params = CreateSandboxFromImageParams( - image=Image.base(config.image), - ephemeral=True, - auto_stop_interval=0, - resources=daytona_resources, - ) - else: - snapshot_name = self.snapshot_name - snapshot_image = self._image - if snapshot_name is None: - raise ValueError( - "DaytonaRuntime requires snapshot_name or runtime_config.image" - ) - if daytona_resources is not None and snapshot_image is None: - raise ValueError( - "DaytonaRuntime cannot resize an already-built snapshot: resources " - "are fixed when it is built, so pass image= to build one" - ) - if snapshot_image is not None: - if daytona_resources is not None: - # Sizing is baked in at build time, so each sizing is its - # own snapshot under a readable suffix (env-4cpu-8gb). - sizing = [] - if daytona_resources.cpu: - sizing.append(f"{daytona_resources.cpu}cpu") - if daytona_resources.memory: - sizing.append(f"{daytona_resources.memory}gb") - if daytona_resources.gpu: - sizing.append(f"{daytona_resources.gpu}gpu") - sizing.extend( - str(getattr(t, "value", t)).lower() - for t in daytona_resources.gpu_type or [] - ) - snapshot_name = "-".join([snapshot_name, *sizing]) - async with self._snapshot_lock: - if snapshot_name not in self._resolved: - try: - existing = await daytona.snapshot.get(snapshot_name) - except DaytonaNotFoundError: - existing = None - if existing is not None and not await _snapshot_is_current( - existing, snapshot_image - ): - logger.info( - "Daytona snapshot %s is stale; rebuilding", snapshot_name - ) - await daytona.snapshot.delete(existing) - # Deletion frees the name asynchronously (~10s - # observed); creating before it lands conflicts. - async with asyncio.timeout(120): - while True: - try: - await daytona.snapshot.get(snapshot_name) - except DaytonaNotFoundError: - break - await asyncio.sleep(0.5) - existing = None - if existing is None: - logger.info("building Daytona snapshot %s", snapshot_name) - await daytona.snapshot.create( - CreateSnapshotParams( - name=snapshot_name, - image=snapshot_image, - resources=daytona_resources, - ) - ) - self._resolved.add(snapshot_name) - sandbox_params = CreateSandboxFromSnapshotParams( - snapshot=snapshot_name, - ephemeral=True, - auto_stop_interval=0, - ) - - create_timeout = 120 - if config.limits is not None and config.limits.startup_timeout_s is not None: - create_timeout = config.limits.startup_timeout_s - # ephemeral: these sandboxes are per-rollout and deleted on exit anyway, - # and some regions only permit ephemeral sandboxes. - sandbox = await daytona.create( - sandbox_params, - timeout=create_timeout, - ) - try: - # Start the env server in a background session (the snapshot's CMD is - # not the sandbox's main process). connect() retries the handshake, - # so we don't poll for readiness here. - session: str = "hud-serve" - await sandbox.process.create_session(session) - cmd = f"cd {self.workdir} && {self.command}" if self.workdir else self.command - session_command = await sandbox.process.execute_session_command( - session, SessionExecuteRequest(command=cmd, run_async=True) - ) - ssh = await sandbox.create_ssh_access(expires_in_minutes=self.ssh_expires_minutes) - async with asyncssh.connect( - self.ssh_host, username=ssh.token, known_hosts=None - ) as conn: - listener = await conn.forward_local_port("127.0.0.1", 0, "127.0.0.1", self.port) - try: - yield Runtime( - f"tcp://127.0.0.1:{listener.get_port()}", - params={"provider": "daytona", "instance_id": sandbox.id}, - config=config if config.model_dump(exclude_none=True) else None, - ) - except (EOFError, OSError) as exc: - # Why it died only exists inside the sandbox, and the - # sandbox may already be gone. - try: - logs = await sandbox.process.get_session_command_logs( - session, session_command.cmd_id - ) - output = (logs.stderr or logs.output or logs.stdout or "").strip() - except Exception as log_exc: - exc.add_note(f"env output unavailable: {log_exc}") - else: - exc.add_note( - f"env output in sandbox {sandbox.id}:\n{output}" - if output - else "env printed nothing" - ) - raise - finally: - try: - await daytona.delete(sandbox) - except Exception: - # Swallowing this is how a billable sandbox outlives its process. - logger.warning( - "failed to delete Daytona sandbox %s; it may still be running", - sandbox.id, - exc_info=True, - ) - - -@asynccontextmanager -async def _local(env: Environment, *, ready_timeout: float | None = None) -> AsyncIterator[Runtime]: - """Substrate-side serving: a live env owned by *this* process, as a runtime. - - One env lifecycle (start → serve → stop) around one bound control - channel; ``ready_timeout`` bounds ``env.start()`` (initialize - hooks/daemons). ``LocalRuntime`` enters this per acquisition with the - fresh env it built; test harnesses enter it directly with a live one. - """ - from hud.environment.server import _shutdown, bind - - # start() inside the try: a failed or timed-out initialize hook still gets - # its already-started daemons torn down by stop() (best-effort per hook). - try: - started = env.start() - await (asyncio.wait_for(started, ready_timeout) if ready_timeout is not None else started) - server = await bind(env, "127.0.0.1", 0) - host, port = server.sockets[0].getsockname()[:2] - serve_task = asyncio.create_task(server.serve_forever()) - try: - yield Runtime(f"tcp://{host}:{port}") - finally: - serve_task.cancel() - await _shutdown(server) - with contextlib.suppress(asyncio.CancelledError): - await serve_task - finally: - await env.stop() - - -async def _read_port(stdout: asyncio.StreamReader) -> int | None: - """Read the child's stdout until it announces its port; ``None`` if stdout - hits EOF first (the child exited before serving — caller builds the error).""" - # Imported lazily: a module-level import would pre-load hud.environment.server - # in every `python -m hud.environment.server` child, tripping runpy's - # found-in-sys.modules RuntimeWarning on each spawned rollout. - from hud.environment.server import PORT_ANNOUNCEMENT - - while True: - line = await stdout.readline() - if not line: - return None - text = line.decode("utf-8", "replace").strip() - if text.startswith(PORT_ANNOUNCEMENT): - return int(text.removeprefix(PORT_ANNOUNCEMENT)) - - -async def _exit_detail( - proc: ProcessGroup, - source: Path, - capture: asyncio.Task[None], - stderr_tail: deque[str], -) -> str: - """Message for a child that exited before serving, with its captured stderr - tail. The child is gone, so its stderr is at EOF — let the capture finish so - the traceback it wrote on the way out is included, not raced past.""" - code = await proc.wait() - with contextlib.suppress(TimeoutError): - await asyncio.wait_for(asyncio.shield(capture), 2.0) - tail = "\n".join(stderr_tail).strip() - detail = f":\n{tail}" if tail else " (no stderr captured)" - return f"spawned env exited with code {code} before serving (source: {source}){detail}" - - -async def _capture(stream: asyncio.StreamReader, sink: deque[str]) -> None: - """Drain a child stream into a bounded tail so it never blocks on a full pipe - and its last lines survive for diagnostics.""" - while line := await stream.readline(): - sink.append(line.decode("utf-8", "replace").rstrip()) - - -async def _drain(stream: asyncio.StreamReader) -> None: - """Keep consuming the child's stdout so it never blocks on a full pipe.""" - while await stream.read(65536): - pass - - -#: Platform trace statuses that end a hosted rollout. -_TERMINAL_TRACE_STATUSES = frozenset({"completed", "error", "cancelled"}) -_RUNTIME_READY_TIMEOUT = 300.0 - - -class HUDRuntime: - """HUD tunnel placement: local agent loop against a HUD-hosted environment. - - The SDK creates a runtime session by environment name, exposes the remote - control channel through a local TCP listener, and lets the normal rollout - atom drive it from this process. - """ - - def __init__(self, *, run_timeout: float = 3600.0, runtime_url: str | None = None) -> None: - self.run_timeout = run_timeout - self.runtime_url = runtime_url - self._warned_unsupported_config = False - - async def run( - self, - task: Task, - agent: Agent, - *, - job_id: str, - group_id: str | None = None, - trace_id: str | None = None, - ) -> Run: - return await rollout( - task, - agent, - runtime=self, - trace_id=trace_id, - job_id=job_id, - group_id=group_id, - rollout_timeout=self.run_timeout, - ) - - def __call__(self, task: Task) -> AbstractAsyncContextManager[Runtime]: - return self._runtime_session(task) - - @asynccontextmanager - async def _runtime_session(self, task: Task) -> AsyncIterator[Runtime]: - from hud.settings import settings as sdk_settings - - if task.runtime_config is not None: - # The lease resolves the env by name: a stamped image is - # provenance and rides along. Declared cpu/memory are best-effort - # on the platform's substrate (warned once, not fatal — loaders - # stamp them on every row), but a GPU or explicit limits change - # what the task *is*; running without them would grade a - # different environment than declared. - resources = task.runtime_config.resources - if (resources is not None and resources.gpu is not None) or ( - task.runtime_config.limits is not None - and task.runtime_config.limits.model_dump(exclude_none=True) - ): - raise ValueError( - "HUDRuntime cannot honor this task's declared GPU/limits on an " - "already-deployed env; run it on a placement that provisions them" - ) - softly_ignored = task.runtime_config.model_dump( - exclude_none=True, exclude={"image", "compose"} - ) - if softly_ignored and not self._warned_unsupported_config: - self._warned_unsupported_config = True - logger.warning( - "HUDRuntime cannot honor task runtime_config %s on an " - "already-deployed env; rollouts proceed on the platform's " - "defaults", - sorted(softly_ignored), - ) - api_key = sdk_settings.api_key - if not api_key: - raise RuntimeError("HUD runtime tunnel requires HUD_API_KEY") - runtime_url = (self.runtime_url or sdk_settings.hud_runtime_url).rstrip("/") - session_id = await self._create_runtime_session(runtime_url, api_key, task) - server: asyncio.Server | None = None - try: - server = await asyncio.start_server( - lambda reader, writer: self._forward_runtime_connection( - runtime_url, - api_key, - session_id, - reader, - writer, - ), - "127.0.0.1", - 0, - ) - port = server.sockets[0].getsockname()[1] - yield Runtime( - f"tcp://127.0.0.1:{port}", - params={ - "session_id": session_id, - "gateway_url": runtime_url, - "ready_timeout": min(self.run_timeout, _RUNTIME_READY_TIMEOUT), - }, - ) - finally: - if server is not None: - server.close() - await server.wait_closed() - await self._delete_runtime_session(runtime_url, api_key, session_id) - - async def _create_runtime_session(self, runtime_url: str, api_key: str, task: Task) -> str: - payload: dict[str, Any] = {"environment": task.env} - trace_id = get_current_trace_id() - if trace_id is not None: - with contextlib.suppress(ValueError): - payload["trace_id"] = str(uuid.UUID(trace_id)) - async with httpx.AsyncClient(timeout=30.0) as client: - resp = await client.post( - f"{runtime_url}/runtime/sessions", - headers={"Authorization": f"Bearer {api_key}"}, - json=payload, - ) - resp.raise_for_status() - body = resp.json() - session_id = body.get("id") - if not isinstance(session_id, str): - raise RuntimeError("Runtime gateway did not return a session id") - return session_id - - async def _delete_runtime_session( - self, runtime_url: str, api_key: str, session_id: str - ) -> None: - async with httpx.AsyncClient(timeout=15.0) as client: - with contextlib.suppress(Exception): - await client.delete( - f"{runtime_url}/runtime/sessions/{session_id}", - headers={"Authorization": f"Bearer {api_key}"}, - ) - - async def _forward_runtime_connection( - self, - runtime_url: str, - api_key: str, - session_id: str, - reader: asyncio.StreamReader, - writer: asyncio.StreamWriter, - ) -> None: - import websockets - - ws_url = _runtime_tunnel_ws_url(runtime_url, session_id) - try: - async with websockets.connect( - ws_url, - additional_headers={"Authorization": f"Bearer {api_key}"}, - max_size=None, - ) as websocket: - await _splice_websocket(reader, writer, websocket) - finally: - if not writer.is_closing(): - writer.close() - with contextlib.suppress(Exception): - await writer.wait_closed() - - -class HostedRuntime: - """HUD-hosted placement: runs the rollout on a leased box and returns its ``Run``. - - The *client-elsewhere* placement. Where a :class:`Provider` yields a channel - this process drives, ``HostedRuntime`` runs the whole rollout off-box: the - platform leases an instance, brings the env's container up on it, and runs - the agent right next to it (the instance-side driver is just - :func:`hud.eval.run.rollout` over a ``DockerRuntime`` — co-location all the - way down). This process only submits the rollout and polls the trace to - completion, folding the result into a :class:`~hud.eval.run.Run`. Because - the agent runs remotely, its identity travels via :func:`_agent_spec`. - - ``run_timeout`` bounds one rollout end to end, including instance - provisioning (a cold EC2 boot plus image pull), queueing, and the agent - run itself. A local cancel (Ctrl-C) requests a platform-side cancel before - propagating, so abandoned rollouts do not hold instances open. - """ - - def __init__( - self, - *, - poll_interval: float = 5.0, - run_timeout: float = 3600.0, - ) -> None: - self.poll_interval = poll_interval - self.run_timeout = run_timeout - self._cancellations: set[asyncio.Task[None]] = set() - - async def run( - self, - task: Task, - agent: Agent, - *, - job_id: str, - group_id: str | None = None, - trace_id: str | None = None, - ) -> Run: - """Submit one rollout, await its terminal trace, and fold it into a ``Run``. - - The platform owns the trace lifecycle (the instance-side driver reports - enter/exit and streams telemetry), so this never double-reports. - Failures isolating one rollout from its batch (submit rejected, the - env/model unresolved) surface as :meth:`Run.failed`; a timeout or a - local cancel propagate, having first asked the platform to release the - lease. - """ - trace_id = trace_id or uuid.uuid4().hex - try: - if task.verifier is not None: - raise ValueError( - "HostedRuntime does not support verifier tasks until hosted rollouts " - "can keep both phases in one runtime scope" - ) - async with asyncio.timeout(self.run_timeout): - state = await self._submit_and_await( - task, agent, job_id=job_id, group_id=group_id, trace_id=trace_id - ) - except asyncio.CancelledError: - self._cancel_later(trace_id) - raise - except TimeoutError: - self._cancel_later(trace_id) - detail = f"hosted rollout {trace_id} did not finish within {self.run_timeout:g}s" - logger.warning(detail) - run = Run.failed(detail) - run.trace.stop_reason = "timeout" - except Exception as exc: - logger.warning("hosted rollout failed to launch: %s", exc) - run = Run.failed(str(exc)) - else: - run = self._fold(state, trace_id) - run.trace.trace_id = trace_id - run.job_id = job_id - run.group_id = group_id - return run - - async def _submit_and_await( - self, - task: Task, - agent: Agent, - *, - job_id: str, - group_id: str | None, - trace_id: str, - ) -> dict[str, Any]: - from hud.agents.tool_agent import ToolAgent - - if not isinstance(agent, ToolAgent): - raise ValueError( - f"hosted execution requires a gateway agent that can serialize its " - f"identity (Claude/OpenAI/Gemini/OpenAIChat); got {type(agent).__name__}" - ) - spec = agent.hosted_spec() - if task.agent_config: - spec = { - **spec, - "config": {**spec.get("config", {}), **task.agent_config}, - } - platform = PlatformClient.from_settings() - if not platform.api_key: - raise RuntimeError("HUD-hosted execution requires HUD_API_KEY") - payload: dict[str, Any] = { - # The SDK's hex ids travel as canonical UUID strings. - "trace_id": str(uuid.UUID(trace_id)), - "job_id": str(uuid.UUID(job_id)), - "env": task.env, - "task": task.id, - "slug": task.slug, - "args": task.args, - "agent": spec, - } - if group_id is not None: - payload["group_id"] = group_id - if task.runtime_config is not None: - runtime_config = task.runtime_config.request_payload() - if runtime_config: - payload["runtime_config"] = runtime_config - await platform.apost("/rollouts/submit", json=payload) - return await self._await_terminal(platform, payload["trace_id"]) - - @staticmethod - def _fold(state: dict[str, Any], trace_id: str) -> Run: - """Build the local view of a remotely-executed rollout from its trace state.""" - run = Run(None, "", {}) - # The poll loop only returns terminal states, so the status is one of - # the trace vocabulary; anything else would be a platform bug. - status = state.get("status") - run.trace.status = status if status in ("completed", "error", "cancelled") else "error" - error = state.get("error") - if error: - run.record(Step(source="system", error=str(error))) - reward = state.get("reward") - ungraded_failure = run.trace.status in ("error", "cancelled") and reward is None - grade_error = str(error) if error else None - if ungraded_failure and grade_error is None: - grade_error = ( - "rollout was cancelled before grading" - if run.trace.status == "cancelled" - else "rollout failed before grading" - ) - run.grade = Grade( - reward=float(reward) if reward is not None else 0.0, - is_error=ungraded_failure, - content=grade_error, - raw={"score": float(reward)} if reward is not None else {}, - ) - run._runtime = f"hud://trace/{trace_id}" - return run - - async def _await_terminal(self, platform: PlatformClient, trace_id: str) -> dict[str, Any]: - while True: - state: dict[str, Any] = await platform.aget(f"/trace/{trace_id}") - if state.get("status") in _TERMINAL_TRACE_STATUSES: - return state - await asyncio.sleep(self.poll_interval) - - async def _cancel(self, platform: PlatformClient, trace_id: str) -> None: - # The platform also bounds instances by max runtime; this just releases - # the lease promptly. Never shadow the caller's outcome. - try: - await platform.apost("/rollouts/cancel", json={"trace_id": trace_id}) - except Exception as exc: - logger.warning("hosted rollout %s cancel failed: %s", trace_id, exc) - - def _cancel_later(self, trace_id: str) -> None: - task = asyncio.create_task( - self._cancel(PlatformClient.from_settings(), str(uuid.UUID(trace_id))) - ) - self._cancellations.add(task) - task.add_done_callback(self._cancellations.discard) - - -def _runtime_tunnel_ws_url(runtime_url: str, session_id: str) -> str: - parts = urlsplit(runtime_url.rstrip("/")) - scheme = "wss" if parts.scheme == "https" else "ws" - path = f"{parts.path.rstrip('/')}/runtime/tunnels/{session_id}" - return urlunsplit((scheme, parts.netloc, path, "", "")) - - -async def _splice_websocket( - reader: asyncio.StreamReader, - writer: asyncio.StreamWriter, - websocket: Any, -) -> None: - async def tcp_to_ws() -> None: - while data := await reader.read(65536): - await websocket.send(data) - - async def ws_to_tcp() -> None: - async for message in websocket: - data = message.encode("utf-8") if isinstance(message, str) else message - writer.write(data) - await writer.drain() - - tasks = [ - asyncio.create_task(tcp_to_ws()), - asyncio.create_task(ws_to_tcp()), - ] - try: - done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) - for task in pending: - task.cancel() - done_results = await asyncio.gather(*done, return_exceptions=True) - await asyncio.gather(*pending, return_exceptions=True) - finally: - for task in tasks: - if not task.done(): - task.cancel() - await asyncio.gather(*tasks, return_exceptions=True) - - for result in done_results: - if isinstance(result, BaseException): - raise result - - -__all__ = [ - "DaytonaRuntime", - "DockerRuntime", - "HUDRuntime", - "HostedRuntime", - "LocalRuntime", - "ModalRuntime", - "Provider", - "Runtime", - "RuntimeConfig", - "RuntimeGPU", - "RuntimeLimits", - "RuntimeResources", - "Shared", - "SubprocessRuntime", -] diff --git a/hud/eval/runtime/__init__.py b/hud/eval/runtime/__init__.py new file mode 100644 index 000000000..3ec438a2d --- /dev/null +++ b/hud/eval/runtime/__init__.py @@ -0,0 +1,49 @@ +"""Runtime placement and provider configuration.""" + +from .core import ( + LocalRuntime, + Provider, + Runtime, + RuntimeConfig, + RuntimeGPU, + RuntimeLimits, + RuntimeResources, + RuntimeTPU, + Shared, + SubprocessRuntime, +) +from .core import ( + _declared_env as _declared_env, +) +from .core import ( + _declared_names as _declared_names, +) +from .core import ( + _local as _local, +) +from .daytona import DaytonaRuntime +from .docker import DockerRuntime +from .hosted import HostedRuntime +from .hud import HUDRuntime +from .hud import ( + _splice_websocket as _splice_websocket, +) +from .modal import ModalRuntime + +__all__ = [ + "DaytonaRuntime", + "DockerRuntime", + "HUDRuntime", + "HostedRuntime", + "LocalRuntime", + "ModalRuntime", + "Provider", + "Runtime", + "RuntimeConfig", + "RuntimeGPU", + "RuntimeLimits", + "RuntimeResources", + "RuntimeTPU", + "Shared", + "SubprocessRuntime", +] diff --git a/hud/eval/compose.py b/hud/eval/runtime/compose.py similarity index 69% rename from hud/eval/compose.py rename to hud/eval/runtime/compose.py index 28b178f74..4950fe14d 100644 --- a/hud/eval/compose.py +++ b/hud/eval/runtime/compose.py @@ -14,12 +14,127 @@ from typing import TYPE_CHECKING, Any import yaml +from dotenv import dotenv_values from pydantic import BaseModel, ConfigDict, Field, field_validator +from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode if TYPE_CHECKING: from collections.abc import Iterator, Mapping +_COMPOSE_VARIABLE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") + + +class ComposeUnboundVariableError(ValueError): + """A Compose variable depends on values outside the packaged project.""" + + +def _interpolate_compose_value(value: str, environment: Mapping[str, str]) -> str: + result: list[str] = [] + index = 0 + while index < len(value): + marker = value.find("$", index) + if marker < 0: + result.append(value[index:]) + break + result.append(value[index:marker]) + if marker + 1 >= len(value): + result.append("$") + break + following = value[marker + 1] + if following == "$": + result.append("$$") + index = marker + 2 + continue + if following == "{": + depth = 1 + end = marker + 2 + while end < len(value) and depth: + if value.startswith("${", end): + depth += 1 + end += 2 + continue + if value[end] == "}": + depth -= 1 + if depth == 0: + break + end += 1 + if depth: + raise ValueError("invalid Compose interpolation: unclosed variable") + expression = value[marker + 2 : end] + result.append(_resolve_compose_variable(expression, environment)) + index = end + 1 + continue + match = _COMPOSE_VARIABLE.match(value, marker + 1) + if match is None: + result.append("$") + index = marker + 1 + continue + name = match.group() + if name not in environment: + raise ComposeUnboundVariableError( + f"Compose variable {name!r} is not set by the project .env" + ) + result.append(environment[name]) + index = match.end() + return "".join(result) + + +def _resolve_compose_variable(expression: str, environment: Mapping[str, str]) -> str: + match = _COMPOSE_VARIABLE.match(expression) + if match is None: + raise ValueError(f"invalid Compose interpolation expression {expression!r}") + name = match.group() + suffix = expression[match.end() :] + value = environment.get(name) + if not suffix: + if value is None: + raise ComposeUnboundVariableError( + f"Compose variable {name!r} is not set by the project .env" + ) + return value + + operator = next( + (item for item in (":-", ":?", ":+", "-", "?", "+") if suffix.startswith(item)), None + ) + if operator is None: + raise ValueError(f"invalid Compose interpolation expression {expression!r}") + operand = suffix[len(operator) :] + is_set = value is not None + is_nonempty = is_set and value != "" + if operator == ":-": + return value if is_nonempty else _interpolate_compose_value(operand, environment) + if operator == "-": + return value if is_set else _interpolate_compose_value(operand, environment) + if operator == ":+": + return _interpolate_compose_value(operand, environment) if is_nonempty else "" + if operator == "+": + return _interpolate_compose_value(operand, environment) if is_set else "" + if (operator == ":?" and not is_nonempty) or (operator == "?" and not is_set): + detail = operand or f"Compose variable {name!r} is required" + raise ComposeUnboundVariableError(detail) + assert value is not None + return value + + +def _interpolate_compose_node( + node: Node, + environment: Mapping[str, str], + seen: set[int], +) -> None: + if id(node) in seen: + return + seen.add(id(node)) + if isinstance(node, MappingNode): + for _, value in node.value: + _interpolate_compose_node(value, environment, seen) + elif isinstance(node, SequenceNode): + for value in node.value: + _interpolate_compose_node(value, environment, seen) + elif isinstance(node, ScalarNode) and node.tag == "tag:yaml.org,2002:str" and node.style != "'": + node.value = _interpolate_compose_value(node.value, environment) + + class ComposeHealthcheck(BaseModel): model_config = ConfigDict(extra="allow") @@ -52,6 +167,7 @@ class ComposeService(BaseModel): command: list[str] | None = None working_dir: str | None = None healthcheck: ComposeHealthcheck | None = None + network_mode: str | None = None expose: list[str] = Field(default_factory=list) ports: list[ComposePort] = Field(default_factory=list) volumes: list[str | dict[str, Any]] = Field(default_factory=list) @@ -126,6 +242,14 @@ def argv(self) -> list[str]: raise RuntimeError(f"image defaults were not resolved for {self.image!r}") return [*self.entrypoint, *self.command] + @property + def tcp_ports(self) -> set[int]: + ports = { + int(value) for exposed in self.expose if (value := exposed.partition("/")[0]).isdigit() + } + ports.update(port.target for port in self.ports if port.protocol == "tcp") + return ports + def shell_command(self) -> str: command = shlex.join(self.argv) if self.working_dir: @@ -142,13 +266,42 @@ class ComposeConfig(BaseModel): services: dict[str, ComposeService] networks: dict[str, dict[str, Any] | None] = Field(default_factory=dict) + def network_owner(self, service: str) -> str: + """Service whose network namespace and published ports *service* uses.""" + seen: set[str] = set() + current = service + while True: + if current in seen: + raise ValueError(f"Compose network_mode service cycle includes {current!r}") + seen.add(current) + try: + mode = self.services[current].network_mode + except KeyError: + raise ValueError( + f"Compose network_mode references unknown service {current!r}" + ) from None + if mode is None or not mode.startswith("service:"): + return current + current = mode.removeprefix("service:") + @classmethod def from_file(cls, path: Path) -> ComposeConfig: """Load a self-contained authored Compose document without Docker.""" source = path.read_text(encoding="utf-8") - if re.search(r"(? Iterator[ComposeLaunchFiles]: main: dict[str, Any] = { - "security_opt": [f"seccomp={seccomp}", "systempaths=unconfined"], + "security_opt": [ + f"seccomp={seccomp}", + "systempaths=unconfined", + "apparmor=unconfined", + ], } if service_socket is not None: main["volumes"] = [ @@ -347,7 +505,7 @@ def stage( ) ports = root / "ports.yaml" ports.write_text( - f'services:\n main:\n ports: !override ["{published_port}"]\n', + f'services:\n {port_service}:\n ports: !override ["{published_port}"]\n', encoding="utf-8", ) archive_path = None diff --git a/hud/eval/runtime/core.py b/hud/eval/runtime/core.py new file mode 100644 index 000000000..e95c89a9a --- /dev/null +++ b/hud/eval/runtime/core.py @@ -0,0 +1,587 @@ +"""Runtime configuration, addresses, sharing, and local placement.""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import sys +from collections import deque +from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any, Protocol, Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from hud.utils.process import ProcessGroup, create_process_group_exec + +from .compose import ComposeConfig, ComposeProjectRef, ComposeSource + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Iterator + + from hud.environment.env import Environment + from hud.eval.task import Task + +logger = logging.getLogger("hud.eval.runtime") + + +class RuntimeGPU(BaseModel): + """Requested GPU resources, provider-neutral where possible.""" + + model_config = ConfigDict(extra="forbid") + + type: str | list[str] | None = Field(default=None, min_length=1) + count: int = Field(default=1, ge=1) + + @property + def acceptable_types(self) -> list[str]: + if self.type is None: + return [] + return [self.type] if isinstance(self.type, str) else self.type + + +class RuntimeTPU(BaseModel): + """Requested TPU slice.""" + + model_config = ConfigDict(extra="forbid") + + type: str = Field(min_length=1) + topology: str = Field(pattern=r"^[1-9]\d*(?:x[1-9]\d*)+$") + + +class RuntimeResources(BaseModel): + """Provider-neutral runtime placement requests.""" + + model_config = ConfigDict(extra="forbid") + + cpu: float | None = Field(default=None, gt=0) + memory_mb: int | None = Field(default=None, gt=0) + storage_mb: int | None = Field(default=None, gt=0) + gpu: RuntimeGPU | None = None + os: str | None = Field(default=None, min_length=1) + tpu: RuntimeTPU | None = None + + def _require_support(self, provider: str, supported: set[str]) -> None: + unsupported = self.model_dump(exclude_none=True).keys() - supported + unsupported.discard("storage_mb") + if unsupported: + fields = ", ".join(f"runtime_config.resources.{name}" for name in sorted(unsupported)) + raise ValueError(f"{provider} does not support {fields}") + + +class RuntimeLimits(BaseModel): + """Runtime lifecycle limits in seconds.""" + + model_config = ConfigDict(extra="forbid") + + startup_timeout_s: int | None = Field(default=None, gt=0) + run_timeout_s: int | None = Field(default=None, gt=0) + + +class RuntimeConfig(BaseModel): + """Typed task-environment launch requirements. + + ``Task.runtime_config`` is requested construction input. ``Runtime.config`` + is the effective config used to construct a runtime. + + ``compose`` and ``compose_project`` are authored as local paths; platform + task records carry them as the serialized compose document and a + :class:`ComposeProjectRef`. Both forms validate; only the path form is + runnable by local providers. + """ + + model_config = ConfigDict(extra="forbid") + + image: str | None = Field(default=None, min_length=1) + compose: Path | ComposeConfig | None = None + compose_project: Path | ComposeProjectRef | None = None + compose_service_access: bool | None = None + resources: RuntimeResources | None = None + limits: RuntimeLimits | None = None + + @model_validator(mode="after") + def validate_source(self) -> Self: + if self.image is not None and self.compose is not None: + raise ValueError("runtime_config accepts either image or compose, not both") + if self.compose_project is not None and self.compose is None: + raise ValueError("compose_project requires runtime_config.compose") + if self.compose_service_access and self.compose is None: + raise ValueError("compose_service_access requires runtime_config.compose") + return self + + def with_overrides(self, override: RuntimeConfig | None) -> RuntimeConfig: + if override is None: + return self + config = self.model_dump() + changes = override.model_dump(exclude_unset=True) + if override.image is not None: + config["compose"] = None + config["compose_project"] = None + config["compose_service_access"] = None + elif override.compose is not None: + config["image"] = None + config["compose_project"] = None + return RuntimeConfig.model_validate(config | changes) + + def request_payload(self) -> dict[str, Any]: + payload = self.model_dump(mode="json", exclude_unset=True) + source = self.compose_source() + if source is not None: + payload.update(source.request_payload()) + return payload + + def compose_source(self) -> ComposeSource | None: + """The authored or wire-form Compose source, when configured.""" + if self.compose is None: + return None + return ComposeSource(self.compose, self.compose_project) + + +class Provider(Protocol): + """Server placement: called with the task row being placed, acquire one + fresh env substrate for it and yield its connectable :class:`Runtime`. + + A provider brings up the *server* (the env's control channel) wherever it + lives — a local subprocess, a container, a cloud sandbox — and the agent + loop drives it from this process (:func:`hud.eval.run.rollout`). The + channel is location-transparent, so "co-located" (loopback) and "split" + (agent here, env elsewhere) are the same code, differing only in the url. + """ + + def __call__(self, task: Task, /) -> AbstractAsyncContextManager[Runtime]: ... + + +class HandoffEndpoint(Protocol): + """Provider-owned transfer of the runtime handoff namespace.""" + + async def export_to(self, destination: Path) -> None: ... + + async def import_from(self, source: Path) -> None: ... + + +@dataclass(frozen=True) +class Runtime: + """The connectable address of a provisioned substrate. + + ``url`` is the control-channel address (``tcp://127.0.0.1:7000`` for a + local process, ``tcp://sandbox-abc.hud.so:443`` for a hosted box). + ``params`` carries connection-time data a transport may need (auth token, + sandbox id). ``config`` is the effective runtime configuration used to + construct the runtime. Constructed directly, it is also a provider — the + borrowed, shared case: it yields itself with a no-op lifecycle, since + whoever provisioned the substrate owns its teardown. + """ + + url: str + params: dict[str, Any] = field(default_factory=dict) + config: RuntimeConfig | None = None + handoff: HandoffEndpoint | None = field(default=None, repr=False, compare=False) + + def __call__(self, task: Task) -> AbstractAsyncContextManager[Runtime]: + return nullcontext(self) + + +class Shared: + """Lease provider: at most ``width`` concurrent rollouts share one substrate. + + The substrate boots lazily on the first lease and lives for the enclosing + ``async with`` scope — one boot however many rollouts flow through, torn + down deterministically at scope exit. ``width`` is the substrate's real + capacity (e.g. a vectorized sim's slot count): lease ``width + 1`` waits + for a slot instead of erroring, so the scheduler needs no pairing — + ``group`` and ``max_concurrent`` keep their ordinary meanings. + + ``Taskset.run`` scopes a context-manager placement to the call, so + ``runtime=Shared(DockerRuntime(...), width=8)`` works bare; open the scope + yourself to keep the substrate warm across several calls:: + + async with Shared(DockerRuntime("hud-isaac-env"), width=8) as rt: + await taskset.run(agent, runtime=rt, group=8) + """ + + def __init__(self, inner: Provider, *, width: int) -> None: + if width < 1: + raise ValueError("Shared width must be >= 1") + self.inner = inner + self.width = width + self._sem = asyncio.Semaphore(width) + self._boot = asyncio.Lock() + self._addr: Runtime | None = None + self._stack: contextlib.AsyncExitStack | None = None + self._opens = 0 + + async def __aenter__(self) -> Self: + self._opens += 1 + return self + + async def __aexit__(self, *exc: object) -> None: + self._opens -= 1 + if self._opens == 0 and self._stack is not None: + stack, self._stack, self._addr = self._stack, None, None + await stack.aclose() + + @asynccontextmanager + async def __call__(self, task: Task) -> AsyncIterator[Runtime]: + if self._opens == 0: + raise RuntimeError( + "Shared substrates outlive single rollouts; lease inside the scope " + "(Taskset.run opens it for you, or wrap calls in `async with Shared(...)`)" + ) + async with self._sem: + async with self._boot: + if self._addr is None: + # First leaseholder boots. A failed boot fails only its own + # rollout (nothing entered the stack); the next lease retries. + stack = contextlib.AsyncExitStack() + self._addr = await stack.enter_async_context(self.inner(task)) + self._stack = stack + addr = self._addr + yield addr + + +class LocalRuntime: + """The local provider: serve a fresh env per rollout, in this process. + + *source* points at the env in whatever form you have: + + - a ``.py`` file or directory — imported fresh per acquisition (sibling + imports resolve); *env* pins one name when several are declared, + defaulting to the placed task's env + - a live :class:`~hud.environment.Environment` — shorthand for its + declaring file; the instance itself is never served + - a ``(task) -> Environment`` callable — called per acquisition with the + placed row + + :: + + runtime = LocalRuntime("env.py") + runtime = LocalRuntime(env) + runtime = LocalRuntime(lambda task: build_env(task.env)) + + ``ready_timeout`` bounds ``@env.initialize`` startup. Freshness covers + the env's own source; modules it imports are cached as usual and shared + across rollouts. Hooks share this process's event loop, so blocking env + code stalls concurrent rollouts — use :class:`SubprocessRuntime` or + :class:`DockerRuntime` for process isolation, and ``Runtime(url)`` to + attach to a substrate served elsewhere. + """ + + def __init__( + self, + source: str | Path | Environment | Callable[[Task], Environment], + *, + env: str | None = None, + ready_timeout: float = 120.0, + ) -> None: + from hud.environment.env import Environment as _Environment + + self.ready_timeout = ready_timeout + # A live instance may have been mutated since its module was imported; + # verify the fresh copy still declares its templates, so drift fails + # at acquisition with the cause named instead of "unknown task" later. + expected_templates: frozenset[str] = frozenset() + if isinstance(source, _Environment): + file = _declaring_file(source, env or source.name) + if file is None: + raise TypeError( + f"LocalRuntime: env {source.name!r} is not rebuilt by importing " + "any file this process has loaded (constructed in a function or " + "notebook cell, or declared inside a package using relative " + "imports); pass its constructor instead: " + "LocalRuntime(lambda task: )" + ) + expected_templates = frozenset(source.tasks) + source, env = file, env or source.name + self._source_dir: Path | None = None + if isinstance(source, (str, Path)): + path, pinned = Path(source).resolve(), env + self._source_dir = path if path.is_dir() else path.parent + from hud.environment import load_environment + + def _load(task: Task) -> _Environment: + loaded = load_environment(path, name=pinned or task.env) + missing = expected_templates - loaded.tasks.keys() + if missing: + raise ValueError( + f"env {loaded.name!r} loaded from {path} lacks template(s) " + f"{sorted(missing)} present on the live instance — it was " + "modified after import; pass a constructor instead: " + "LocalRuntime(lambda task: )" + ) + return loaded + + self._build: Callable[[Task], _Environment] = _load + elif callable(source): + if env is not None: + raise TypeError("LocalRuntime: env= applies only to source paths") + self._build = source + else: + raise TypeError( + f"LocalRuntime: expected a source path, a live Environment, or a " + f"(task) -> Environment constructor; got {source!r}" + ) + + @asynccontextmanager + async def __call__(self, task: Task) -> AsyncIterator[Runtime]: + from hud.environment.env import Environment as _Environment + + if task.runtime_config is not None: + raise ValueError("LocalRuntime does not support task runtime_config") + # The source dir stays importable for the whole acquisition, not just + # the initial import, so a template can lazily import a sibling + # module at run time (as it could under the child-process runtime). + # Always insert-and-remove one entry: balanced under concurrency. + if self._source_dir is not None: + sys.path.insert(0, str(self._source_dir)) + try: + try: + env = self._build(task) + except RuntimeError as e: + # The source ran an event loop at import — usually an unguarded + # top-level run call; name the actual mistake. + if "running event loop" not in str(e): + raise + raise RuntimeError( + "the env source ran async code while being imported to place a " + 'rollout — guard top-level run calls with `if __name__ == "__main__":`' + ) from e + if not isinstance(env, _Environment): + raise TypeError(f"LocalRuntime: constructor returned {env!r}, not an Environment") + async with _local(env, ready_timeout=self.ready_timeout) as runtime: + yield runtime + finally: + if self._source_dir is not None: + with contextlib.suppress(ValueError): + sys.path.remove(str(self._source_dir)) + + +def _live_envs() -> Iterator[tuple[Environment, str]]: + """Envs declared in loaded, file-backed modules' globals, with their files. + + The in-memory counterpart of scanning ``.py`` sources on disk + (:func:`~hud.environment.load_environment`): an env found here can be + served fresh by re-importing its file. Envs in modules without a file + (a notebook ``__main__``) are not yielded — re-import could not + reconstruct them. + """ + from hud.environment.env import Environment as _Environment + + for module in list(sys.modules.values()): + module_file = getattr(module, "__file__", None) + module_vars = getattr(module, "__dict__", None) + if not module_file or not isinstance(module_vars, dict): + continue + for value in list(module_vars.values()): + if isinstance(value, _Environment): + yield value, module_file + + +def _declaring_file(env: Environment, name: str) -> Path | None: + """A file whose fresh import re-declares *env*, else None. + + Candidate files hold the instance in their module globals, but a holder + may be a re-exporter (``from .env import env`` in a package + ``__init__``, a tasks file re-exporting its env): validate each by + loading it fresh — a declarer yields a *new* instance under *name*, a + re-exporter yields the same live one (or fails to import standalone). + ``__init__.py`` holders are tried last. + """ + from hud.environment import load_environment + + candidates = dict.fromkeys(Path(file) for live, file in _live_envs() if live is env) + for file in sorted(candidates, key=lambda f: f.name == "__init__.py"): + try: + probe = load_environment(file, name=name) + except Exception as e: + logger.debug("candidate %s does not rebuild env %r: %s", file, name, e) + continue + if probe is not env: + return file + return None + + +def _declared_env(name: str) -> Environment | None: + """The one live env named *name*, else None; two distinct ones raise. + + The same instance re-exported across modules is one match; distinct envs + claiming one name are ambiguous. + """ + matches = {id(env): env for env, _ in _live_envs() if env.name == name} + if len(matches) > 1: + files = sorted({file for env, file in _live_envs() if env.name == name}) + raise ValueError( + f"env name {name!r} is declared by multiple live environments " + f"({', '.join(files)}); pass runtime= explicitly — the exact " + "instance disambiguates: runtime=LocalRuntime(env)" + ) + return next(iter(matches.values()), None) + + +def _declared_names(source: Path) -> set[str]: + """Env names a ``.py`` source (file or directory) itself declares. + + A fresh execution of the source yields *new* instances for envs it + declares; an env it merely imports is the already-live one and does not + count — importing the source again could not rebuild it. + """ + from hud.environment.env import Environment as _Environment + from hud.utils.modules import iter_modules + + live = {id(env) for env, _ in _live_envs()} + return { + value.name + for module in iter_modules(source) + for value in vars(module).values() + if isinstance(value, _Environment) and id(value) not in live + } + + +class SubprocessRuntime: + """The child-process provider: serve the placed row's env from *path*. + + Each acquisition runs ``python -m hud.environment.server --env + name`` — the same serving entry point a container CMD runs — on an + ephemeral loopback port, yields its :class:`Runtime`, and terminates the + child on exit. *path* is a ``.py`` file or a directory of them. The served + env is the placed task's ``env`` name (so a mixed-env taskset works + against one source), unless *env* pins one explicitly; placing a row whose + env the source does not define fails loudly in the child. + + The child's working directory is the source's directory, so sibling + imports and relative data paths resolve; ``@env.initialize`` daemons start + in the child and die with it. Because the source is re-imported in the + child, a script spawning itself (``SubprocessRuntime(__file__)``) must keep + top-level run calls under ``if __name__ == "__main__":``. + """ + + def __init__( + self, + path: str | Path, + *, + env: str | None = None, + ready_timeout: float = 120.0, + ) -> None: + self.source = Path(path).resolve() + self.env = env + self.ready_timeout = ready_timeout + + @asynccontextmanager + async def __call__(self, task: Task) -> AsyncIterator[Runtime]: + if task.runtime_config is not None: + raise ValueError("SubprocessRuntime does not support task runtime_config") + if not self.source.exists(): + raise FileNotFoundError(f"SubprocessRuntime: source not found: {self.source}") + cmd = [sys.executable, "-m", "hud.environment.server", str(self.source)] + cmd += ["--env", self.env or task.env] + proc = await create_process_group_exec( + *cmd, + term_timeout=10.0, + stdout=asyncio.subprocess.PIPE, + # Capture stderr (don't inherit it): under concurrent rollouts an + # inherited fd interleaves every child's output unattributably, so a + # crash-before-serving leaves no traceable diagnostic. We keep a + # bounded tail and attach it to the failure below. + stderr=asyncio.subprocess.PIPE, + cwd=self.source if self.source.is_dir() else self.source.parent, + ) + assert proc.stderr is not None + # Drain stderr into a bounded tail from the start: it never blocks on a + # full pipe, and the last lines survive if the child dies early. + stderr_tail: deque[str] = deque(maxlen=50) + capture = asyncio.create_task(_capture(proc.stderr, stderr_tail)) + try: + assert proc.stdout is not None + port = await asyncio.wait_for(_read_port(proc.stdout), self.ready_timeout) + if port is None: + raise RuntimeError(await _exit_detail(proc, self.source, capture, stderr_tail)) + drain = asyncio.create_task(_drain(proc.stdout)) + try: + yield Runtime(f"tcp://127.0.0.1:{port}") + finally: + drain.cancel() + with contextlib.suppress(asyncio.CancelledError): + await drain + finally: + capture.cancel() + with contextlib.suppress(asyncio.CancelledError): + await capture + await proc.terminate() + + +@asynccontextmanager +async def _local(env: Environment, *, ready_timeout: float | None = None) -> AsyncIterator[Runtime]: + """Substrate-side serving: a live env owned by *this* process, as a runtime. + + One env lifecycle (start → serve → stop) around one bound control + channel; ``ready_timeout`` bounds ``env.start()`` (initialize + hooks/daemons). ``LocalRuntime`` enters this per acquisition with the + fresh env it built; test harnesses enter it directly with a live one. + """ + from hud.environment.server import _shutdown, bind + + # start() inside the try: a failed or timed-out initialize hook still gets + # its already-started daemons torn down by stop() (best-effort per hook). + try: + started = env.start() + await (asyncio.wait_for(started, ready_timeout) if ready_timeout is not None else started) + server = await bind(env, "127.0.0.1", 0) + host, port = server.sockets[0].getsockname()[:2] + serve_task = asyncio.create_task(server.serve_forever()) + try: + yield Runtime(f"tcp://{host}:{port}") + finally: + serve_task.cancel() + await _shutdown(server) + with contextlib.suppress(asyncio.CancelledError): + await serve_task + finally: + await env.stop() + + +async def _read_port(stdout: asyncio.StreamReader) -> int | None: + """Read the child's stdout until it announces its port; ``None`` if stdout + hits EOF first (the child exited before serving — caller builds the error).""" + # Imported lazily: a module-level import would pre-load hud.environment.server + # in every `python -m hud.environment.server` child, tripping runpy's + # found-in-sys.modules RuntimeWarning on each spawned rollout. + from hud.environment.server import PORT_ANNOUNCEMENT + + while True: + line = await stdout.readline() + if not line: + return None + text = line.decode("utf-8", "replace").strip() + if text.startswith(PORT_ANNOUNCEMENT): + return int(text.removeprefix(PORT_ANNOUNCEMENT)) + + +async def _exit_detail( + proc: ProcessGroup, + source: Path, + capture: asyncio.Task[None], + stderr_tail: deque[str], +) -> str: + """Message for a child that exited before serving, with its captured stderr + tail. The child is gone, so its stderr is at EOF — let the capture finish so + the traceback it wrote on the way out is included, not raced past.""" + code = await proc.wait() + with contextlib.suppress(TimeoutError): + await asyncio.wait_for(asyncio.shield(capture), 2.0) + tail = "\n".join(stderr_tail).strip() + detail = f":\n{tail}" if tail else " (no stderr captured)" + return f"spawned env exited with code {code} before serving (source: {source}){detail}" + + +async def _capture(stream: asyncio.StreamReader, sink: deque[str]) -> None: + """Drain a child stream into a bounded tail so it never blocks on a full pipe + and its last lines survive for diagnostics.""" + while line := await stream.readline(): + sink.append(line.decode("utf-8", "replace").rstrip()) + + +async def _drain(stream: asyncio.StreamReader) -> None: + """Keep consuming the child's stdout so it never blocks on a full pipe.""" + while await stream.read(65536): + pass diff --git a/hud/eval/runtime/daytona.py b/hud/eval/runtime/daytona.py new file mode 100644 index 000000000..33f880795 --- /dev/null +++ b/hud/eval/runtime/daytona.py @@ -0,0 +1,439 @@ +"""Daytona runtime provider.""" + +from __future__ import annotations + +import asyncio +import importlib +import logging +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from typing import TYPE_CHECKING, Any, Protocol, cast + +from .core import Runtime, RuntimeConfig + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Sequence + from pathlib import Path + + from hud.eval.task import Task + +logger = logging.getLogger("hud.eval.runtime") + + +class DaytonaContextEntry(Protocol): + @property + def source_path(self) -> str | Path: ... + + @property + def archive_path(self) -> str | Path: ... + + +class DaytonaImage(Protocol): + @property + def _context_list(self) -> Sequence[DaytonaContextEntry]: ... + + def dockerfile(self) -> str: ... + + +class _DaytonaBuildInfo(Protocol): + dockerfile_content: str + context_hashes: Sequence[str] | None + + +class DaytonaSnapshot(Protocol): + image_name: str + build_info: _DaytonaBuildInfo | None + + +class ObjectStorage(Protocol): + async def _compute_hash_for_path_md5( + self, + source_path: str | Path, + archive_path: str | Path, + ) -> str: ... + + +class ObjectStorageModule(Protocol): + AsyncObjectStorage: type[ObjectStorage] + + +class _DaytonaResources(Protocol): + cpu: int | None + memory: int | None + gpu: int | None + gpu_type: Sequence[object] | None + + +class _DaytonaSessionCommand(Protocol): + cmd_id: str + + +class _DaytonaSessionLogs(Protocol): + stderr: str | None + output: str | None + stdout: str | None + + +class _DaytonaProcess(Protocol): + async def create_session(self, session: str) -> object: ... + + async def execute_session_command( + self, + session: str, + request: object, + ) -> _DaytonaSessionCommand: ... + + async def get_session_command_logs( + self, + session: str, + command_id: str, + ) -> _DaytonaSessionLogs: ... + + +class _DaytonaSshAccess(Protocol): + token: str + + +class _DaytonaSandbox(Protocol): + id: str + process: _DaytonaProcess + + async def create_ssh_access(self, *, expires_in_minutes: int) -> _DaytonaSshAccess: ... + + +class _DaytonaSnapshotClient(Protocol): + async def get(self, name: str) -> DaytonaSnapshot: ... + + async def delete(self, snapshot: DaytonaSnapshot) -> object: ... + + async def create(self, params: object) -> object: ... + + +class _DaytonaCreate(Protocol): + def __call__( + self, + params: object, + *, + timeout: int, + ) -> Awaitable[_DaytonaSandbox]: ... + + +class _DaytonaClient(Protocol): + snapshot: _DaytonaSnapshotClient + create: _DaytonaCreate + + async def delete(self, sandbox: _DaytonaSandbox) -> object: ... + + +class _DaytonaFactory(Protocol): + def __call__(self) -> AbstractAsyncContextManager[_DaytonaClient]: ... + + +class _ObjectFactory(Protocol): + def __call__(self, *args: object, **kwargs: object) -> object: ... + + +class _ResourcesFactory(Protocol): + def __call__(self, *args: object, **kwargs: object) -> _DaytonaResources: ... + + +class _GpuTypeFactory(Protocol): + def __call__(self, value: str) -> object: ... + + +class _DaytonaImageFactory(Protocol): + def base(self, image: str) -> object: ... + + +class DaytonaModule(Protocol): + AsyncDaytona: _DaytonaFactory + CreateSandboxFromImageParams: _ObjectFactory + CreateSandboxFromSnapshotParams: _ObjectFactory + CreateSnapshotParams: _ObjectFactory + DaytonaNotFoundError: type[Exception] + GpuType: _GpuTypeFactory + Image: _DaytonaImageFactory + Resources: _ResourcesFactory + SessionExecuteRequest: _ObjectFactory + + +async def _snapshot_is_current( + snapshot: DaytonaSnapshot, + image: str | DaytonaImage, +) -> bool: + """Whether *snapshot* was built from *image* as it exists right now. + + Daytona records what a snapshot was built from — the registry ref, or for a + built ``Image`` its Dockerfile text plus the hashes of the context it + uploads (``build_info``). The hashes are recomputed here with the SDK's own + hasher so they are comparable to what ``snapshot.create`` would upload. + ``_context_list`` is the same private attribute ``snapshot.create`` reads. + """ + if isinstance(image, str): + return bool(snapshot.image_name == image) + build = snapshot.build_info + if build is None: + return False + object_storage = cast( + "ObjectStorageModule", + importlib.import_module("daytona._async.object_storage"), + ) + AsyncObjectStorage = object_storage.AsyncObjectStorage + + # The hasher is an instance method only for code organization; credentials + # are needed to upload, not to hash, so skip the credentialed __init__. + storage = AsyncObjectStorage.__new__(AsyncObjectStorage) + hashes = [ + await storage._compute_hash_for_path_md5(entry.source_path, entry.archive_path) + for entry in image._context_list + ] + return bool( + build.dockerfile_content == image.dockerfile() + and list(build.context_hashes or []) == hashes + ) + + +class DaytonaRuntime: + """The Daytona provider: each acquisition creates a fresh sandbox from a snapshot. + + The Daytona runtime boots a sandbox from a pre-built *snapshot* + (the durable handle, the snapshot equivalent of Modal's image name), starts the + env's control channel inside it, then reaches it over an SSH local-forward: + Daytona exposes services only as HTTPS previews, but :func:`hud.clients.connect` + dials ``tcp://``, so the raw control channel is tunneled over SSH to a local + port. Yields its :class:`Runtime`, deletes the sandbox on exit. + + Pass a snapshot name — ``DaytonaRuntime("hud-libero-env")`` — optionally with an + ``image`` (Dockerfile/registry ref) to build that snapshot if it is missing. + With *image*, an existing snapshot is compared against the image's recorded + build (Dockerfile plus context hashes) and rebuilt under the same name when + they differ, so editing the env never silently reuses the snapshot built + before the edit. + Resources (cpu/memory/gpu) live on the snapshot, not here. *workdir* defaults to + ``/app`` (the scaffolded ``Dockerfile.hud`` WORKDIR) since a Daytona session + starts in ``~``, not the image's WORKDIR; override only for a non-standard layout. + Requires the ``daytona`` extra and ``DAYTONA_API_KEY``. + """ + + def __init__( + self, + snapshot_name: str | None = None, + *, + image: str | DaytonaImage | None = None, + command: str | None = None, + workdir: str | None = "/app", + port: int = 8765, + ssh_host: str = "ssh.app.daytona.io", + ssh_expires_minutes: int = 24 * 60, + runtime_config: RuntimeConfig | dict[str, Any] | None = None, + ) -> None: + self.snapshot_name = snapshot_name + # Default command serves on *port*, so the SSH forward target always + # matches what's listening; override only for a non-default layout. + self.command = ( + command or f'PATH="$PWD/.venv/bin:$PATH" hud serve env.py --host 0.0.0.0 --port {port}' + ) + self.workdir = workdir + self.port = port + self.ssh_host = ssh_host + self.ssh_expires_minutes = ssh_expires_minutes + config = None + if runtime_config is not None: + config = RuntimeConfig.model_validate(runtime_config) + self.runtime_config = config + # Resolve each snapshot name against the image once; lock so concurrent + # first acquisitions resolve exactly once. + self._image = image + self._resolved: set[str] = set() + self._snapshot_lock = asyncio.Lock() + + @asynccontextmanager + async def __call__(self, task: Task) -> AsyncIterator[Runtime]: + import asyncssh + + daytona_sdk = cast("DaytonaModule", importlib.import_module("daytona")) + AsyncDaytona = daytona_sdk.AsyncDaytona + CreateSandboxFromImageParams = daytona_sdk.CreateSandboxFromImageParams + CreateSandboxFromSnapshotParams = daytona_sdk.CreateSandboxFromSnapshotParams + CreateSnapshotParams = daytona_sdk.CreateSnapshotParams + DaytonaNotFoundError = daytona_sdk.DaytonaNotFoundError + GpuType = daytona_sdk.GpuType + Image = daytona_sdk.Image + Resources = daytona_sdk.Resources + SessionExecuteRequest = daytona_sdk.SessionExecuteRequest + + async with AsyncDaytona() as daytona: + config = (self.runtime_config or RuntimeConfig()).with_overrides(task.runtime_config) + if config.compose is not None: + raise ValueError("DaytonaRuntime does not support runtime_config.compose") + if config.limits is not None and config.limits.run_timeout_s is not None: + raise ValueError("DaytonaRuntime does not support runtime_config.run_timeout_s") + + daytona_resources = None + resources = config.resources + if resources is not None: + resources._require_support( + "DaytonaRuntime", {"cpu", "memory_mb", "storage_mb", "gpu"} + ) + resource_kwargs: dict[str, Any] = {} + if resources.cpu is not None: + # Daytona allocates whole cores; truncating resizes silently. + if isinstance(resources.cpu, float) and not resources.cpu.is_integer(): + raise ValueError( + f"DaytonaRuntime needs a whole number of CPUs, got {resources.cpu}" + ) + resource_kwargs["cpu"] = int(resources.cpu) + if resources.memory_mb is not None: + resource_kwargs["memory"] = max( + 1, + (resources.memory_mb + 1023) // 1024, + ) + if resources.storage_mb is not None: + resource_kwargs["disk"] = max( + 1, + (resources.storage_mb + 1023) // 1024, + ) + if resources.gpu is not None: + resource_kwargs["gpu"] = resources.gpu.count + gpu_types = resources.gpu.acceptable_types + if gpu_types: + resource_kwargs["gpu_type"] = [GpuType(item) for item in gpu_types] + if resource_kwargs: + daytona_resources = Resources(**resource_kwargs) + + if config.image is not None: + sandbox_params = CreateSandboxFromImageParams( + image=Image.base(config.image), + ephemeral=True, + auto_stop_interval=0, + resources=daytona_resources, + ) + else: + snapshot_name = self.snapshot_name + snapshot_image = self._image + if snapshot_name is None: + raise ValueError( + "DaytonaRuntime requires snapshot_name or runtime_config.image" + ) + if daytona_resources is not None and snapshot_image is None: + raise ValueError( + "DaytonaRuntime cannot resize an already-built snapshot: resources " + "are fixed when it is built, so pass image= to build one" + ) + if snapshot_image is not None: + if daytona_resources is not None: + # Sizing is baked in at build time, so each sizing is its + # own snapshot under a readable suffix (env-4cpu-8gb). + sizing = [] + if daytona_resources.cpu: + sizing.append(f"{daytona_resources.cpu}cpu") + if daytona_resources.memory: + sizing.append(f"{daytona_resources.memory}gb") + if resources is not None and resources.storage_mb: + storage_gb = max( + 1, + (resources.storage_mb + 1023) // 1024, + ) + sizing.append(f"{storage_gb}gb-disk") + if daytona_resources.gpu: + sizing.append(f"{daytona_resources.gpu}gpu") + sizing.extend( + str(getattr(t, "value", t)).lower() + for t in daytona_resources.gpu_type or [] + ) + snapshot_name = "-".join([snapshot_name, *sizing]) + async with self._snapshot_lock: + if snapshot_name not in self._resolved: + try: + existing = await daytona.snapshot.get(snapshot_name) + except DaytonaNotFoundError: + existing = None + if existing is not None and not await _snapshot_is_current( + existing, snapshot_image + ): + logger.info( + "Daytona snapshot %s is stale; rebuilding", snapshot_name + ) + await daytona.snapshot.delete(existing) + # Deletion frees the name asynchronously (~10s + # observed); creating before it lands conflicts. + async with asyncio.timeout(120): + while True: + try: + await daytona.snapshot.get(snapshot_name) + except DaytonaNotFoundError: + break + await asyncio.sleep(0.5) + existing = None + if existing is None: + logger.info("building Daytona snapshot %s", snapshot_name) + await daytona.snapshot.create( + CreateSnapshotParams( + name=snapshot_name, + image=snapshot_image, + resources=daytona_resources, + ) + ) + self._resolved.add(snapshot_name) + sandbox_params = CreateSandboxFromSnapshotParams( + snapshot=snapshot_name, + ephemeral=True, + auto_stop_interval=0, + ) + + create_timeout = 120 + if config.limits is not None and config.limits.startup_timeout_s is not None: + create_timeout = config.limits.startup_timeout_s + # ephemeral: these sandboxes are per-rollout and deleted on exit anyway, + # and some regions only permit ephemeral sandboxes. + sandbox = await daytona.create( + sandbox_params, + timeout=create_timeout, + ) + try: + # Start the env server in a background session (the snapshot's CMD is + # not the sandbox's main process). connect() retries the handshake, + # so we don't poll for readiness here. + session: str = "hud-serve" + await sandbox.process.create_session(session) + cmd = f"cd {self.workdir} && {self.command}" if self.workdir else self.command + session_command = await sandbox.process.execute_session_command( + session, SessionExecuteRequest(command=cmd, run_async=True) + ) + ssh = await sandbox.create_ssh_access(expires_in_minutes=self.ssh_expires_minutes) + async with asyncssh.connect( + self.ssh_host, username=ssh.token, known_hosts=None + ) as conn: + listener = await conn.forward_local_port("127.0.0.1", 0, "127.0.0.1", self.port) + try: + yield Runtime( + f"tcp://127.0.0.1:{listener.get_port()}", + params={"provider": "daytona", "instance_id": sandbox.id}, + config=config if config.model_dump(exclude_none=True) else None, + ) + except (EOFError, OSError) as exc: + # Why it died only exists inside the sandbox, and the + # sandbox may already be gone. + try: + logs = await sandbox.process.get_session_command_logs( + session, session_command.cmd_id + ) + output = (logs.stderr or logs.output or logs.stdout or "").strip() + except Exception as log_exc: + exc.add_note(f"env output unavailable: {log_exc}") + else: + exc.add_note( + f"env output in sandbox {sandbox.id}:\n{output}" + if output + else "env printed nothing" + ) + raise + finally: + try: + await daytona.delete(sandbox) + except Exception: + # Swallowing this is how a billable sandbox outlives its process. + logger.warning( + "failed to delete Daytona sandbox %s; it may still be running", + sandbox.id, + exc_info=True, + ) diff --git a/hud/eval/runtime/docker.py b/hud/eval/runtime/docker.py new file mode 100644 index 000000000..eb0bef6c4 --- /dev/null +++ b/hud/eval/runtime/docker.py @@ -0,0 +1,323 @@ +"""Docker runtime provider.""" + +from __future__ import annotations + +import asyncio +import logging +import os +import tarfile +import tempfile +import uuid +from contextlib import asynccontextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit + +from hud.utils.docker import docker as _docker +from hud.utils.process import create_process_group_exec + +from .compose import ComposeConfig, ComposeProject +from .core import Runtime, RuntimeConfig + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Sequence + + from hud.eval.task import Task + +logger = logging.getLogger("hud.eval.runtime") + +#: DockerRuntime always serves HUD environments, so this is part of the +#: provider contract rather than a per-image option. This is intentionally a +#: default-allow compatibility profile: Workspace's bwrap sessions need the +#: namespace and mount syscalls, while unrelated kernel interfaces stay denied. +_DOCKER_SECCOMP_PROFILE = Path(__file__).parent.parent / "docker-seccomp.json" +_DOCKER_SECURITY_ARGS = ( + "--security-opt", + f"seccomp={_DOCKER_SECCOMP_PROFILE}", + # Docker exposes system-path masking only as an all-or-nothing option; + # bwrap replaces the container's proc and dev mounts while building a wall. + "--security-opt", + "systempaths=unconfined", + "--security-opt", + "apparmor=unconfined", +) + + +def _require_free_disk(output: str, storage_mb: int) -> None: + try: + available_kib = int(output.strip().splitlines()[-1].split()[-3]) + except (IndexError, ValueError): + raise RuntimeError("DockerRuntime could not measure the environment's free disk") from None + available_mb = available_kib // 1024 + if available_mb < storage_mb: + raise RuntimeError( + f"DockerRuntime requires {storage_mb} MB of free disk; " + f"the environment has {available_mb} MB" + ) + + +async def _prepare_compose_project(compose: Path) -> bool: + script = compose.parent / "build.sh" + if not script.is_file(): + return False + process = await create_process_group_exec( + "sh", + str(script), + cwd=str(compose.parent), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + result = await process.complete() + if result.returncode != 0: + detail = (result.stderr or result.stdout).decode("utf-8", "replace").strip() + raise RuntimeError(f"Compose project build failed: {detail}") + return True + + +class DockerRuntime: + """Start a HUD environment from an image or a Docker Compose file. + + An image is started with ``docker run``. A Compose file is started unchanged + except for a small provider override that publishes the ``main`` service's + control-channel port and applies HUD's nested-workspace security profile. + """ + + def __init__( + self, + image: str | None = None, + *, + port: int = 8765, + run_args: Sequence[str] = (), + compose_service_socket: str | Path | None = None, + runtime_config: RuntimeConfig | dict[str, Any] | None = None, + ) -> None: + self.port = port + self.run_args = tuple(run_args) + self.compose_service_socket = ( + str(Path(compose_service_socket)) if compose_service_socket is not None else None + ) + config = RuntimeConfig(image=image) if image is not None else RuntimeConfig() + if runtime_config is not None: + config = config.with_overrides(RuntimeConfig.model_validate(runtime_config)) + self.runtime_config = config if config.model_dump(exclude_none=True) else None + self._compose_preparation_locks: dict[Path, asyncio.Lock] = {} + + @asynccontextmanager + async def __call__(self, task: Task) -> AsyncIterator[Runtime]: + config = (self.runtime_config or RuntimeConfig()).with_overrides(task.runtime_config) + if config.limits is not None and config.limits.model_dump(exclude_none=True): + raise ValueError("DockerRuntime does not support runtime_config limits") + resources = config.resources + if resources is not None: + resources._require_support("DockerRuntime", {"cpu", "memory_mb", "storage_mb", "gpu"}) + compose_source = config.compose_source() + if compose_source is not None: + if self.run_args: + raise ValueError("DockerRuntime run_args apply only to image environments") + compose = compose_source.runnable_path("DockerRuntime") + port_service = ComposeConfig.from_file(compose).network_owner("main") + resources = config.resources + if ( + resources is not None + and resources.gpu is not None + and resources.gpu.type is not None + ): + raise ValueError("DockerRuntime cannot select Compose GPUs by type") + service_socket = None + if config.compose_service_access: + service_socket = self.compose_service_socket + if service_socket is None: + endpoint = os.environ.get("DOCKER_HOST") + if not endpoint: + endpoint, _ = await _docker( + "context", + "inspect", + "--format", + "{{.Endpoints.docker.Host}}", + ) + endpoint = endpoint.strip() + parsed = urlsplit(endpoint) + if parsed.scheme != "unix" or not parsed.path: + raise ValueError( + "DockerRuntime Compose service access through a remote daemon " + "requires compose_service_socket" + ) + service_socket = parsed.path + project_files = ComposeProject(compose) + project = f"hud-{uuid.uuid4().hex[:12]}" + lock = self._compose_preparation_locks.setdefault(compose, asyncio.Lock()) + async with lock: + prepared = await _prepare_compose_project(compose) + with project_files.stage( + f"127.0.0.1::{self.port}", + port_service=port_service, + seccomp=_DOCKER_SECCOMP_PROFILE, + service_socket=service_socket, + cpu=resources.cpu if resources is not None else None, + memory_mb=resources.memory_mb if resources is not None else None, + gpu_count=( + resources.gpu.count + if resources is not None and resources.gpu is not None + else None + ), + ) as files: + command = ( + "compose", + "--project-name", + project, + "--file", + str(files.compose), + "--file", + str(files.override), + "--file", + str(files.ports), + ) + try: + await _docker( + *command, + "up", + "--detach", + "--no-build" if prepared else "--build", + "--remove-orphans", + ) + if resources is not None and resources.storage_mb is not None: + free_disk, _ = await _docker( + *command, "exec", "-T", "main", "df", "-Pk", "/" + ) + _require_free_disk(free_disk, resources.storage_mb) + mapping, _ = await _docker(*command, "port", port_service, str(self.port)) + if not mapping.strip(): + logs_out, logs_err = await _docker( + *command, "logs", "--tail", "40", "main", check=False + ) + raise RuntimeError( + f"Compose main service exited before serving port {self.port}:\n" + f"{(logs_err or logs_out).strip()}" + ) + host_port = int(mapping.strip().splitlines()[0].rsplit(":", 1)[1]) + container, _ = await _docker(*command, "ps", "--quiet", "main") + handoff = _DockerHandoff(container.strip()) + await handoff.prepare() + yield Runtime( + f"tcp://127.0.0.1:{host_port}", + config=config if config.model_dump(exclude_none=True) else None, + handoff=handoff, + ) + finally: + await _docker( + *command, + "down", + "--volumes", + "--remove-orphans", + check=False, + ) + return + if config.image is None: + raise ValueError( + "DockerRuntime requires runtime_config.image or runtime_config.compose" + ) + + resource_args: list[str] = [] + resources = config.resources + if resources is not None: + if resources.cpu is not None: + cpu = ( + str(int(resources.cpu)) + if isinstance(resources.cpu, float) and resources.cpu.is_integer() + else str(resources.cpu) + ) + resource_args.extend(("--cpus", cpu)) + if resources.memory_mb is not None: + resource_args.extend(("--memory", f"{resources.memory_mb}m")) + if resources.gpu is not None: + if resources.gpu.type is not None: + raise ValueError("DockerRuntime cannot select GPUs by type") + resource_args.extend(("--gpus", str(resources.gpu.count))) + + out, _ = await _docker( + "run", + "--detach", + *self.run_args, + *resource_args, + *_DOCKER_SECURITY_ARGS, + "--publish", + f"127.0.0.1::{self.port}", + config.image, + ) + container = out.strip() + try: + if resources is not None and resources.storage_mb is not None: + free_disk, _ = await _docker("exec", container, "df", "-Pk", "/") + _require_free_disk(free_disk, resources.storage_mb) + mapping, _ = await _docker("port", container, str(self.port)) + if not mapping.strip(): + logs_out, logs_err = await _docker("logs", "--tail", "40", container, check=False) + raise RuntimeError( + f"container for image {config.image!r} exited before serving port " + f"{self.port}:\n{(logs_err or logs_out).strip()}", + ) + host_port = int(mapping.strip().splitlines()[0].rsplit(":", 1)[1]) + handoff = _DockerHandoff(container) + await handoff.prepare() + yield Runtime(f"tcp://127.0.0.1:{host_port}", config=config, handoff=handoff) + finally: + # check=False: teardown must not shadow the run's own error, and + # rm -f only fails when the daemon itself is broken. + await _docker("rm", "--force", container, check=False) + + +@dataclass(frozen=True, slots=True) +class _DockerHandoff: + container: str + + async def prepare(self) -> None: + await _docker("exec", self.container, "mkdir", "-p", "/media/hud/handoffs") + + async def export_to(self, destination: Path) -> None: + archive = f"/media/hud/handoff-export-{uuid.uuid4().hex}.tar.gz" + script = """ +import sys +import tarfile +from pathlib import Path + +root = Path("/media/hud/handoffs") +entries = list(root.rglob("*")) +if any(entry.is_symlink() for entry in entries): + raise ValueError("runtime handoff contains a symbolic link") +with tarfile.open(sys.argv[1], "w:gz") as output: + for entry in entries: + output.add(entry, arcname=entry.relative_to(root), recursive=False) +""" + try: + await _docker( + "exec", + "--user", + "0", + self.container, + "python3", + "-c", + script, + archive, + ) + await _docker("cp", f"{self.container}:{archive}", str(destination)) + finally: + await _docker("exec", "--user", "0", self.container, "rm", "-f", archive, check=False) + + async def import_from(self, source: Path) -> None: + with tempfile.TemporaryDirectory(prefix="hud-handoff-import-") as directory: + root = Path(directory) + _extract_handoff_archive(source, root) + await self.prepare() + await _docker( + "cp", + f"{root}/.", + f"{self.container}:/media/hud/handoffs", + ) + + +def _extract_handoff_archive(source: Path, destination: Path) -> None: + with tarfile.open(source, "r:gz") as archive: + if any(not (member.isfile() or member.isdir()) for member in archive.getmembers()): + raise ValueError("runtime handoff archive contains an unsupported entry") + archive.extractall(destination, filter="data") diff --git a/hud/eval/runtime/hosted.py b/hud/eval/runtime/hosted.py new file mode 100644 index 000000000..91aa0d9f7 --- /dev/null +++ b/hud/eval/runtime/hosted.py @@ -0,0 +1,192 @@ +"""Remote HUD-hosted rollout provider.""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from typing import TYPE_CHECKING, Any + +from hud.eval.run import Grade, Run +from hud.types import Step +from hud.utils.platform import PlatformClient + +if TYPE_CHECKING: + from hud.agents.base import Agent + from hud.eval.task import Task + +logger = logging.getLogger("hud.eval.runtime") + +_TERMINAL_TRACE_STATUSES = frozenset({"completed", "error", "cancelled"}) + + +class HostedRuntime: + """HUD-hosted placement: runs the rollout on a leased box and returns its ``Run``. + + The *client-elsewhere* placement. Where a :class:`Provider` yields a channel + this process drives, ``HostedRuntime`` runs the whole rollout off-box: the + platform leases an instance, brings the env's container up on it, and runs + the agent right next to it (the instance-side driver is just + :func:`hud.eval.run.rollout` over a ``DockerRuntime`` — co-location all the + way down). This process only submits the rollout and polls the trace to + completion, folding the result into a :class:`~hud.eval.run.Run`. Because + the agent runs remotely, its identity travels via :func:`_agent_spec`. + + ``run_timeout`` bounds one rollout end to end, including instance + provisioning (a cold EC2 boot plus image pull), queueing, and the agent + run itself. A local cancel (Ctrl-C) requests a platform-side cancel before + propagating, so abandoned rollouts do not hold instances open. + """ + + def __init__( + self, + *, + poll_interval: float = 5.0, + run_timeout: float = 3600.0, + ) -> None: + self.poll_interval = poll_interval + self.run_timeout = run_timeout + self._cancellations: set[asyncio.Task[None]] = set() + + async def run( + self, + task: Task, + agent: Agent, + *, + job_id: str, + group_id: str | None = None, + trace_id: str | None = None, + ) -> Run: + """Submit one rollout, await its terminal trace, and fold it into a ``Run``. + + The platform owns the trace lifecycle (the instance-side driver reports + enter/exit and streams telemetry), so this never double-reports. + Failures isolating one rollout from its batch (submit rejected, the + env/model unresolved) surface as :meth:`Run.failed`; a timeout or a + local cancel propagate, having first asked the platform to release the + lease. + """ + trace_id = trace_id or uuid.uuid4().hex + try: + if task.verifier is not None: + raise ValueError( + "HostedRuntime does not support verifier tasks until hosted rollouts " + "can keep both phases in one runtime scope" + ) + async with asyncio.timeout(self.run_timeout): + state = await self._submit_and_await( + task, agent, job_id=job_id, group_id=group_id, trace_id=trace_id + ) + except asyncio.CancelledError: + self._cancel_later(trace_id) + raise + except TimeoutError: + self._cancel_later(trace_id) + detail = f"hosted rollout {trace_id} did not finish within {self.run_timeout:g}s" + logger.warning(detail) + run = Run.failed(detail) + run.trace.stop_reason = "timeout" + except Exception as exc: + logger.warning("hosted rollout failed to launch: %s", exc) + run = Run.failed(str(exc)) + else: + run = self._fold(state, trace_id) + run.trace.trace_id = trace_id + run.job_id = job_id + run.group_id = group_id + return run + + async def _submit_and_await( + self, + task: Task, + agent: Agent, + *, + job_id: str, + group_id: str | None, + trace_id: str, + ) -> dict[str, Any]: + from hud.agents.tool_agent import ToolAgent + + if not isinstance(agent, ToolAgent): + raise ValueError( + f"hosted execution requires a gateway agent that can serialize its " + f"identity (Claude/OpenAI/Gemini/OpenAIChat); got {type(agent).__name__}" + ) + spec = agent.hosted_spec() + if task.agent_config: + spec = { + **spec, + "config": {**spec.get("config", {}), **task.agent_config}, + } + platform = PlatformClient.from_settings() + if not platform.api_key: + raise RuntimeError("HUD-hosted execution requires HUD_API_KEY") + payload: dict[str, Any] = { + # The SDK's hex ids travel as canonical UUID strings. + "trace_id": str(uuid.UUID(trace_id)), + "job_id": str(uuid.UUID(job_id)), + "env": task.env, + "task": task.id, + "slug": task.slug, + "args": task.args, + "agent": spec, + } + if group_id is not None: + payload["group_id"] = group_id + if task.runtime_config is not None: + runtime_config = task.runtime_config.request_payload() + if runtime_config: + payload["runtime_config"] = runtime_config + await platform.apost("/rollouts/submit", json=payload) + return await self._await_terminal(platform, payload["trace_id"]) + + @staticmethod + def _fold(state: dict[str, Any], trace_id: str) -> Run: + """Build the local view of a remotely-executed rollout from its trace state.""" + run = Run(None, "", {}) + # The poll loop only returns terminal states, so the status is one of + # the trace vocabulary; anything else would be a platform bug. + status = state.get("status") + run.trace.status = status if status in ("completed", "error", "cancelled") else "error" + error = state.get("error") + if error: + run.record(Step(source="system", error=str(error))) + reward = state.get("reward") + ungraded_failure = run.trace.status in ("error", "cancelled") and reward is None + grade_error = str(error) if error else None + if ungraded_failure and grade_error is None: + grade_error = ( + "rollout was cancelled before grading" + if run.trace.status == "cancelled" + else "rollout failed before grading" + ) + run.grade = Grade( + reward=float(reward) if reward is not None else 0.0, + is_error=ungraded_failure, + content=grade_error, + raw={"score": float(reward)} if reward is not None else {}, + ) + run._runtime = f"hud://trace/{trace_id}" + return run + + async def _await_terminal(self, platform: PlatformClient, trace_id: str) -> dict[str, Any]: + while True: + state: dict[str, Any] = await platform.aget(f"/trace/{trace_id}") + if state.get("status") in _TERMINAL_TRACE_STATUSES: + return state + await asyncio.sleep(self.poll_interval) + + async def _cancel(self, platform: PlatformClient, trace_id: str) -> None: + # The platform also bounds instances by max runtime; this just releases + # the lease promptly. Never shadow the caller's outcome. + try: + await platform.apost("/rollouts/cancel", json={"trace_id": trace_id}) + except Exception as exc: + logger.warning("hosted rollout %s cancel failed: %s", trace_id, exc) + + def _cancel_later(self, trace_id: str) -> None: + task = asyncio.create_task( + self._cancel(PlatformClient.from_settings(), str(uuid.UUID(trace_id))) + ) + self._cancellations.add(task) + task.add_done_callback(self._cancellations.discard) diff --git a/hud/eval/runtime/hud.py b/hud/eval/runtime/hud.py new file mode 100644 index 000000000..47110dffe --- /dev/null +++ b/hud/eval/runtime/hud.py @@ -0,0 +1,232 @@ +"""HUD runtime tunnel provider.""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import uuid +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit, urlunsplit + +import httpx + +from hud.eval.run import Run, rollout +from hud.telemetry.context import get_current_trace_id + +from .core import Runtime + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from hud.agents.base import Agent + from hud.eval.task import Task + +logger = logging.getLogger("hud.eval.runtime") + +_RUNTIME_READY_TIMEOUT = 300.0 + + +class HUDRuntime: + """HUD tunnel placement: local agent loop against a HUD-hosted environment. + + The SDK creates a runtime session by environment name, exposes the remote + control channel through a local TCP listener, and lets the normal rollout + atom drive it from this process. + """ + + def __init__(self, *, run_timeout: float = 3600.0, runtime_url: str | None = None) -> None: + self.run_timeout = run_timeout + self.runtime_url = runtime_url + self._warned_unsupported_config = False + + async def run( + self, + task: Task, + agent: Agent, + *, + job_id: str, + group_id: str | None = None, + trace_id: str | None = None, + ) -> Run: + return await rollout( + task, + agent, + runtime=self, + trace_id=trace_id, + job_id=job_id, + group_id=group_id, + rollout_timeout=self.run_timeout, + ) + + def __call__(self, task: Task) -> AbstractAsyncContextManager[Runtime]: + return self._runtime_session(task) + + @asynccontextmanager + async def _runtime_session(self, task: Task) -> AsyncIterator[Runtime]: + from hud.settings import settings as sdk_settings + + if task.runtime_config is not None: + # The lease resolves the env by name: a stamped image is + # provenance and rides along. Declared cpu/memory are best-effort + # on the platform's substrate (warned once, not fatal — loaders + # stamp them on every row), but a GPU or explicit limits change + # what the task *is*; running without them would grade a + # different environment than declared. + resources = task.runtime_config.resources + if ( + resources is not None + and ( + resources.gpu is not None + or resources.os is not None + or resources.tpu is not None + ) + ) or ( + task.runtime_config.limits is not None + and task.runtime_config.limits.model_dump(exclude_none=True) + ): + raise ValueError( + "HUDRuntime cannot honor this task's declared placement requirements or " + "limits on an " + "already-deployed env; run it on a placement that provisions them" + ) + softly_ignored = task.runtime_config.model_dump( + exclude_none=True, exclude={"image", "compose"} + ) + if softly_ignored and not self._warned_unsupported_config: + self._warned_unsupported_config = True + logger.warning( + "HUDRuntime cannot honor task runtime_config %s on an " + "already-deployed env; rollouts proceed on the platform's " + "defaults", + sorted(softly_ignored), + ) + api_key = sdk_settings.api_key + if not api_key: + raise RuntimeError("HUD runtime tunnel requires HUD_API_KEY") + runtime_url = (self.runtime_url or sdk_settings.hud_runtime_url).rstrip("/") + session_id = await self._create_runtime_session(runtime_url, api_key, task) + server: asyncio.Server | None = None + try: + server = await asyncio.start_server( + lambda reader, writer: self._forward_runtime_connection( + runtime_url, + api_key, + session_id, + reader, + writer, + ), + "127.0.0.1", + 0, + ) + port = server.sockets[0].getsockname()[1] + yield Runtime( + f"tcp://127.0.0.1:{port}", + params={ + "session_id": session_id, + "gateway_url": runtime_url, + "ready_timeout": min(self.run_timeout, _RUNTIME_READY_TIMEOUT), + }, + ) + finally: + if server is not None: + server.close() + await server.wait_closed() + await self._delete_runtime_session(runtime_url, api_key, session_id) + + async def _create_runtime_session(self, runtime_url: str, api_key: str, task: Task) -> str: + payload: dict[str, Any] = {"environment": task.env} + trace_id = get_current_trace_id() + if trace_id is not None: + with contextlib.suppress(ValueError): + payload["trace_id"] = str(uuid.UUID(trace_id)) + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.post( + f"{runtime_url}/runtime/sessions", + headers={"Authorization": f"Bearer {api_key}"}, + json=payload, + ) + resp.raise_for_status() + body = resp.json() + session_id = body.get("id") + if not isinstance(session_id, str): + raise RuntimeError("Runtime gateway did not return a session id") + return session_id + + async def _delete_runtime_session( + self, runtime_url: str, api_key: str, session_id: str + ) -> None: + async with httpx.AsyncClient(timeout=15.0) as client: + with contextlib.suppress(Exception): + await client.delete( + f"{runtime_url}/runtime/sessions/{session_id}", + headers={"Authorization": f"Bearer {api_key}"}, + ) + + async def _forward_runtime_connection( + self, + runtime_url: str, + api_key: str, + session_id: str, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + ) -> None: + import websockets + + ws_url = _runtime_tunnel_ws_url(runtime_url, session_id) + try: + async with websockets.connect( + ws_url, + additional_headers={"Authorization": f"Bearer {api_key}"}, + max_size=None, + ) as websocket: + await _splice_websocket(reader, writer, websocket) + finally: + if not writer.is_closing(): + writer.close() + with contextlib.suppress(Exception): + await writer.wait_closed() + + +def _runtime_tunnel_ws_url(runtime_url: str, session_id: str) -> str: + parts = urlsplit(runtime_url.rstrip("/")) + scheme = "wss" if parts.scheme == "https" else "ws" + path = f"{parts.path.rstrip('/')}/runtime/tunnels/{session_id}" + return urlunsplit((scheme, parts.netloc, path, "", "")) + + +async def _splice_websocket( + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + websocket: Any, +) -> None: + async def tcp_to_ws() -> None: + while data := await reader.read(65536): + await websocket.send(data) + + async def ws_to_tcp() -> None: + async for message in websocket: + data = message.encode("utf-8") if isinstance(message, str) else message + writer.write(data) + await writer.drain() + + tasks = [ + asyncio.create_task(tcp_to_ws()), + asyncio.create_task(ws_to_tcp()), + ] + try: + done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for task in pending: + task.cancel() + done_results = await asyncio.gather(*done, return_exceptions=True) + await asyncio.gather(*pending, return_exceptions=True) + finally: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + for result in done_results: + if isinstance(result, BaseException): + raise result diff --git a/hud/eval/runtime/modal.py b/hud/eval/runtime/modal.py new file mode 100644 index 000000000..4b08e7953 --- /dev/null +++ b/hud/eval/runtime/modal.py @@ -0,0 +1,398 @@ +"""Modal runtime provider.""" + +from __future__ import annotations + +import asyncio +import contextlib +import importlib +import logging +import shlex +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast + +from .compose import ComposeConfig, ComposeProject +from .core import Runtime, RuntimeConfig +from .docker import _DOCKER_SECCOMP_PROFILE + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Mapping, Sequence + from pathlib import Path + + from hud.eval.task import Task + +logger = logging.getLogger("hud.eval.runtime") + +T_co = TypeVar("T_co", covariant=True) + + +class AioMethod(Protocol[T_co]): + async def aio(self, *args: object, **kwargs: object) -> T_co: ... + + +class ModalImage(Protocol): + build: AioMethod[None] + + def env(self, variables: Mapping[str, str]) -> ModalImage: ... + + +class _ModalImageFactory(Protocol): + def from_id(self, image_id: str) -> ModalImage: ... + + def from_registry(self, image: str) -> ModalImage: ... + + def from_name(self, name: str) -> ModalImage: ... + + +class _ModalAppFactory(Protocol): + lookup: AioMethod[object] + + +class _ModalStream(Protocol): + read: AioMethod[str] + + +class _ModalProcess(Protocol): + wait: AioMethod[int] + stderr: _ModalStream + + +class _ModalFilesystem(Protocol): + copy_from_local: AioMethod[None] + copy_to_local: AioMethod[None] + + +class _ModalTunnel(Protocol): + tcp_socket: tuple[str, int] + + +class ModalSandbox(Protocol): + object_id: str + wait_until_ready: AioMethod[None] + filesystem: _ModalFilesystem + exec: AioMethod[_ModalProcess] + tunnels: AioMethod[dict[int, _ModalTunnel]] + terminate: AioMethod[None] + + +class _ModalSandboxFactory(Protocol): + create: AioMethod[ModalSandbox] + + +class _ModalProbeFactory(Protocol): + def with_tcp(self, port: int) -> object: ... + + +class ModalModule(Protocol): + Image: _ModalImageFactory + App: _ModalAppFactory + Sandbox: _ModalSandboxFactory + Probe: _ModalProbeFactory + + +_MODAL_COMPOSE_CPU = 4.0 +_MODAL_COMPOSE_MEMORY_MB = 8192 + + +def _modal_image_from_uri(modal: ModalModule, image_uri: str) -> ModalImage: + modal_uri_prefix = "modal://" + if image_uri.startswith(modal_uri_prefix): + return modal.Image.from_id(image_uri.removeprefix(modal_uri_prefix)) + return modal.Image.from_registry(image_uri) + + +@dataclass(frozen=True, slots=True) +class _ModalHandoff: + sandbox: ModalSandbox + compose: str | None + + def _container(self) -> str: + if self.compose is None: + return "" + compose = shlex.quote(self.compose) + return ( + "CONTAINER=$(docker compose --project-directory /hud/project " + f"--file /hud/project/{compose} --file /hud/override.json " + "--file /hud/ports.yaml ps --quiet main); " + ) + + async def _exec(self, command: str) -> None: + process = await self.sandbox.exec.aio("sh", "-c", command) + if await process.wait.aio() != 0: + raise RuntimeError((await process.stderr.read.aio()).strip()) + + async def prepare(self) -> None: + if self.compose is None: + await self._exec("mkdir -p /media/hud/handoffs") + else: + await self._exec( + self._container() + 'test -n "$CONTAINER" && docker exec "$CONTAINER" ' + "mkdir -p /media/hud/handoffs" + ) + + async def export_to(self, destination: Path) -> None: + if self.compose is None: + root = "/media/hud/handoffs" + command = "" + else: + root = "/media/hud/handoff-export" + command = ( + self._container() + + f"rm -rf {root} && mkdir -p {root} && " + + f'docker cp "$CONTAINER":/media/hud/handoffs/. {root} && ' + ) + command += ( + f"if find {root} -mindepth 1 ! -type f ! -type d -print -quit | grep -q .; " + "then echo 'runtime handoff contains an unsupported entry' >&2; exit 1; fi; " + f"tar -czf /media/hud/handoff.tar.gz -C {root} ." + ) + await self._exec(command) + await self.sandbox.filesystem.copy_to_local.aio("/media/hud/handoff.tar.gz", destination) + + async def import_from(self, source: Path) -> None: + await self.sandbox.filesystem.copy_from_local.aio(source, "/media/hud/handoff.tar.gz") + if self.compose is None: + command = ( + "mkdir -p /media/hud/handoffs && " + "tar -xzf /media/hud/handoff.tar.gz -C /media/hud/handoffs" + ) + else: + command = ( + self._container() + + "rm -rf /tmp/hud-handoff && mkdir -p /tmp/hud-handoff && " + + "tar -xzf /media/hud/handoff.tar.gz -C /tmp/hud-handoff && " + + 'docker exec "$CONTAINER" mkdir -p /media/hud/handoffs && ' + + 'docker cp /tmp/hud-handoff/. "$CONTAINER":/media/hud/handoffs' + ) + await self._exec(command) + + +class ModalRuntime: + """The Modal provider: each acquisition ``Sandbox.create``s a fresh container. + + The cloud :class:`DockerRuntime` — boots a sandbox from a pre-built image, + exposes the env's control channel as a raw-TCP tunnel (``unencrypted_ports``, + the only kind :func:`hud.clients.connect` dials), yields its :class:`Runtime`, + terminates on exit. Acquisitions are independent, so a batch fans out into + isolated containers (one ``sb-…`` id each). + + The image resolves once (so concurrent rollouts can't race a build): pass a + published name — ``ModalRuntime("hud-libero-env")``, the preferred durable + handle — or, as an escape hatch, an ``Image`` to build lazily on first use. + Requires the ``modal`` extra and a configured token. + """ + + def __init__( + self, + image_name: str | None = None, + *, + image: ModalImage | None = None, + command: Sequence[str] | None = None, + app_name: str = "hud-envs", + workdir: str | None = None, + port: int = 8765, + runtime_config: RuntimeConfig | dict[str, Any] | None = None, + env_vars: Mapping[str, str] | None = None, + ) -> None: + self.image_name = image_name + self.port = port + self.env_vars = dict(env_vars or {}) + self.workdir = workdir + # Default CMD mirrors the scaffolded Dockerfile.hud entrypoint. Leave + # workdir unset by default so Modal preserves the image WORKDIR. + self.command = ( + tuple(command) + if command is not None + else ( + "hud", + "serve", + "env.py", + "--host", + "0.0.0.0", # noqa: S104 - serving inside the sandbox; the tunnel is the only ingress + "--port", + str(port), + ) + ) + self.app_name = app_name + config = None + if runtime_config is not None: + config = RuntimeConfig.model_validate(runtime_config) + self.runtime_config = config + # Resolved (named) or built-once (from Dockerfile) image, behind a lock so + # concurrent first acquisitions build/look up exactly once. + self._image = image + self._resolved: ModalImage | None = None + self._image_lock = asyncio.Lock() + + @asynccontextmanager + async def __call__(self, task: Task) -> AsyncIterator[Runtime]: + config = (self.runtime_config or RuntimeConfig()).with_overrides(task.runtime_config) + resources = config.resources + if resources is not None: + resources._require_support("ModalRuntime", {"cpu", "memory_mb", "gpu"}) + compose_source = config.compose_source() + compose = ( + compose_source.runnable_path("ModalRuntime") if compose_source is not None else None + ) + port_service = ComposeConfig.from_file(compose).network_owner("main") if compose else "main" + modal = cast("ModalModule", importlib.import_module("modal")) + + app = None + if compose is not None: + image = modal.Image.from_registry("docker:28.3.3-dind") + elif config.image is not None: + image = _modal_image_from_uri(modal, config.image) + elif self.image_name is not None: + image = modal.Image.from_name(self.image_name) + elif self._image is None: + raise ValueError( + "ModalRuntime requires image=, image_name=, runtime_config.image, " + "or runtime_config.compose" + ) + else: + if self._resolved is None: + async with self._image_lock: + if self._resolved is None: + app = await modal.App.lookup.aio( + self.app_name, + create_if_missing=True, + ) + await self._image.build.aio(app=app) + self._resolved = self._image + image = self._resolved + if app is None: + app = await modal.App.lookup.aio(self.app_name, create_if_missing=True) + + sandbox_kwargs: dict[str, Any] = {} + if compose is not None: + sandbox_kwargs["cpu"] = max( + resources.cpu if resources is not None and resources.cpu is not None else 0, + _MODAL_COMPOSE_CPU, + ) + sandbox_kwargs["memory"] = max( + ( + resources.memory_mb + if resources is not None and resources.memory_mb is not None + else 0 + ), + _MODAL_COMPOSE_MEMORY_MB, + ) + else: + if resources is not None and resources.cpu is not None: + sandbox_kwargs["cpu"] = resources.cpu + if resources is not None and resources.memory_mb is not None: + sandbox_kwargs["memory"] = resources.memory_mb + if self.env_vars: + sandbox_kwargs["env"] = self.env_vars + if resources is not None and resources.gpu is not None: + gpu_types = resources.gpu.acceptable_types + gpu_type = gpu_types[0] if gpu_types else "any" + gpu = gpu_type if resources.gpu.count == 1 else f"{gpu_type}:{resources.gpu.count}" + sandbox_kwargs["gpu"] = gpu + + run_timeout = 3600 + ready_timeout = 600 + if config.limits is not None: + run_timeout = config.limits.run_timeout_s or run_timeout + ready_timeout = config.limits.startup_timeout_s or ready_timeout + + sb = await modal.Sandbox.create.aio( + *(() if compose is not None else self.command), + app=app, + image=image, + workdir=None if compose is not None else self.workdir, + unencrypted_ports=[self.port], + readiness_probe=(None if compose is not None else modal.Probe.with_tcp(self.port)), + # Modal types both timeouts as int seconds; floats raise at proto encode. + timeout=run_timeout, + **({"experimental_options": {"vm_runtime": True}} if compose is not None else {}), + **sandbox_kwargs, + ) + try: + if compose is None: + await sb.wait_until_ready.aio(timeout=ready_timeout) + else: + project = ComposeProject(compose) + with project.stage( + f"{self.port}:{self.port}", + port_service=port_service, + seccomp="/hud/docker-seccomp.json", + service_socket=( + "/var/run/docker.sock" if config.compose_service_access else None + ), + env_vars=self.env_vars, + cpu=resources.cpu if resources is not None else None, + memory_mb=resources.memory_mb if resources is not None else None, + gpu_count=( + resources.gpu.count + if resources is not None and resources.gpu is not None + else None + ), + archive=True, + ) as files: + assert files.archive is not None + await sb.filesystem.copy_from_local.aio(files.archive, "/hud/project.tar.gz") + await sb.filesystem.copy_from_local.aio(files.override, "/hud/override.json") + await sb.filesystem.copy_from_local.aio(files.ports, "/hud/ports.yaml") + await sb.filesystem.copy_from_local.aio( + _DOCKER_SECCOMP_PROFILE, "/hud/docker-seccomp.json" + ) + command = ( + "mkdir -p /hud/project && " + "tar -xzf /hud/project.tar.gz -C /hud/project && " + "until docker info >/dev/null 2>&1; do sleep 1; done && " + "BUILD_FLAG=--build && " + "if [ -f /hud/project/build.sh ]; then " + "sh /hud/project/build.sh && BUILD_FLAG=--no-build; fi && " + "docker compose --project-directory /hud/project " + f"--file /hud/project/{shlex.quote(compose.name)} " + "--file /hud/override.json --file /hud/ports.yaml " + 'up --detach "$BUILD_FLAG" --remove-orphans' + ) + try: + async with asyncio.timeout(ready_timeout): + process = await sb.exec.aio("sh", "-c", command, timeout=ready_timeout) + returncode = await process.wait.aio() + except TimeoutError: + raise TimeoutError( + f"Modal Compose startup timed out after {ready_timeout} seconds" + ) from None + if returncode != 0: + error = (await process.stderr.read.aio()).strip() + raise RuntimeError(f"Modal Compose startup failed: {error}") + host, port = (await sb.tunnels.aio())[self.port].tcp_socket + handoff = _ModalHandoff(sb, compose.name if compose is not None else None) + await handoff.prepare() + yield Runtime( + f"tcp://{host}:{port}", + params={ + "provider": "modal", + "instance_id": sb.object_id, + **({"ready_timeout": ready_timeout} if compose is not None else {}), + }, + config=config if config.model_dump(exclude_none=True) else None, + handoff=handoff, + ) + finally: + # check-free teardown: never shadow the run's own error. + if compose is not None: + with contextlib.suppress(Exception): + process = await sb.exec.aio( + "docker", + "compose", + "--project-directory", + "/hud/project", + "--file", + f"/hud/project/{compose.name}", + "--file", + "/hud/override.json", + "--file", + "/hud/ports.yaml", + "down", + "--volumes", + "--remove-orphans", + timeout=30, + ) + await process.wait.aio() + with contextlib.suppress(Exception): + await sb.terminate.aio() diff --git a/hud/eval/task.py b/hud/eval/task.py index 9ddd08db3..a063eefaf 100644 --- a/hud/eval/task.py +++ b/hud/eval/task.py @@ -71,6 +71,9 @@ class Task(BaseModel): #: Optional row-level runtime construction input. Runtime adapters apply the #: supported subset into their native launch shape or reject it. runtime_config: RuntimeConfig | None = None + #: The verifier consumes files produced by the actor acquisition. Providers + #: transfer the runtime handoff namespace when placement cannot be reused. + requires_handoff: bool | None = None #: Optional agent-less task whose evaluation is the grade of record. The #: rollout completes this task first, then starts and grades the verifier #: with the same answer. Placement may reuse the live substrate when both diff --git a/hud/eval/tests/test_docker_provider.py b/hud/eval/tests/test_docker_provider.py index dcefa6b75..c350281af 100644 --- a/hud/eval/tests/test_docker_provider.py +++ b/hud/eval/tests/test_docker_provider.py @@ -21,8 +21,7 @@ import pytest -import hud.eval.runtime as runtime_module -from hud.eval.compose import ComposeConfig, ComposeHealthcheck, ComposeService +import hud.eval.runtime.docker as runtime_module from hud.eval.runtime import ( DaytonaRuntime, DockerRuntime, @@ -32,6 +31,12 @@ RuntimeLimits, RuntimeResources, ) +from hud.eval.runtime.compose import ( + ComposeConfig, + ComposeHealthcheck, + ComposeProject, + ComposeService, +) from hud.eval.task import Task FAKE_DOCKER_SH = """\ @@ -40,6 +45,10 @@ case "$1" in run) echo cid-42 ;; port) {port_behavior} ;; + exec) if [ "$3" = "df" ]; then + printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n' + printf 'overlay 4194304 0 4194304 0%% /\n' + fi ;; logs) echo "ImportError: boom" ;; esac """ @@ -105,7 +114,7 @@ def _install_fake_docker( exe.write_text( FAKE_DOCKER_CMD.format(port_behavior=_port_behavior_for_windows(port_behavior)) ) - import hud.eval.runtime as runtime_module + import hud.eval.runtime.docker as runtime_module async def _docker(*args: str, check: bool = True) -> tuple[str, str]: return await _docker_via(exe, *args, check=check) @@ -150,7 +159,8 @@ def __init__(self, calls: dict[str, Any], port: int) -> None: self.tunnels = SimpleNamespace(aio=self._tunnels) self.terminate = SimpleNamespace(aio=self._terminate) self.filesystem = SimpleNamespace( - copy_from_local=SimpleNamespace(aio=self._copy_from_local) + copy_from_local=SimpleNamespace(aio=self._copy_from_local), + copy_to_local=SimpleNamespace(aio=self._copy_to_local), ) self.exec = SimpleNamespace(aio=self._exec) @@ -171,12 +181,20 @@ async def _copy_from_local(self, source: Path, target: str) -> None: content = await asyncio.to_thread(source.read_text, "utf-8") self._calls["compose_override"] = json.loads(content) + async def _copy_to_local(self, source: str, target: Path) -> None: + downloads = self._calls.setdefault("downloads", []) + assert isinstance(downloads, list) + downloads.append((source, target.name)) + async def _exec(self, *args: str, **kwargs: object) -> SimpleNamespace: commands = self._calls.setdefault("execs", []) assert isinstance(commands, list) commands.append((args, kwargs)) async def wait() -> int: + wait_event = self._calls.pop("exec_wait", None) + if isinstance(wait_event, asyncio.Event): + await wait_event.wait() return 0 async def read() -> str: @@ -266,6 +284,7 @@ class _DaytonaImage: class _DaytonaResources: cpu: float | None = None memory: int | None = None + disk: int | None = None gpu: int | None = None gpu_type: list[object] | None = None @@ -515,6 +534,32 @@ async def test_acquisition_publishes_ephemeral_port_and_removes_container( assert (await _docker_calls(docker_log))[-1] == "rm --force cid-42" +async def test_docker_handoff_archives_inside_the_container( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[tuple[str, ...], bool]] = [] + + async def fake_docker(*args: str, check: bool = True) -> tuple[str, str]: + calls.append((args, check)) + return "", "" + + monkeypatch.setattr(runtime_module, "_docker", fake_docker) + destination = tmp_path / "handoff.tar.gz" + + await runtime_module._DockerHandoff("cid-42").export_to(destination) + + export, copy, cleanup = calls + assert export[0][:6] == ("exec", "--user", "0", "cid-42", "python3", "-c") + assert "runtime handoff contains a symbolic link" in export[0][6] + archive = export[0][7] + assert copy == (("cp", f"cid-42:{archive}", str(destination)), True) + assert cleanup == ( + ("exec", "--user", "0", "cid-42", "rm", "-f", archive), + False, + ) + + async def test_runtime_config_supplies_image_and_resources( tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -569,7 +614,7 @@ def test_runtime_config_overrides_only_explicit_top_level_fields() -> None: resources=RuntimeResources( cpu=2, memory_mb=4096, - gpu=RuntimeGPU(type="A10G", count=2), + gpu=RuntimeGPU(type=["A10G", "L4"], count=2), ), limits=RuntimeLimits(startup_timeout_s=30, run_timeout_s=120), ) @@ -579,7 +624,7 @@ def test_runtime_config_overrides_only_explicit_top_level_fields() -> None: resources=RuntimeResources( cpu=2, memory_mb=4096, - gpu=RuntimeGPU(type="A10G", count=2), + gpu=RuntimeGPU(type=["A10G", "L4"], count=2), ), limits=RuntimeLimits(startup_timeout_s=30, run_timeout_s=120), ) @@ -688,6 +733,7 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: "security_opt": [ f"seccomp={runtime_module._DOCKER_SECCOMP_PROFILE}", "systempaths=unconfined", + "apparmor=unconfined", ], "cpus": 2.0, "mem_limit": "4096m", @@ -751,7 +797,7 @@ async def test_docker_runtime_rejects_remote_compose_service_access( compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") monkeypatch.setenv("DOCKER_HOST", "tcp://docker.example:2376") - with pytest.raises(ValueError, match="requires a local Unix Docker endpoint"): + with pytest.raises(ValueError, match="requires compose_service_socket"): async with DockerRuntime()( Task( env="any-env", @@ -765,6 +811,42 @@ async def test_docker_runtime_rejects_remote_compose_service_access( pass +async def test_docker_runtime_mounts_the_daemon_visible_socket( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + compose = tmp_path / "compose.yaml" + compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") + monkeypatch.setenv("DOCKER_HOST", "tcp://docker.example:2376") + rendered: dict[str, Any] = {} + + async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: + if args[-4:] == ("up", "--detach", "--build", "--remove-orphans"): + files = [Path(args[index + 1]) for index, value in enumerate(args) if value == "--file"] + rendered.update(json.loads(files[1].read_text("utf-8"))) + if args[-3:] == ("port", "main", "8765"): + return "127.0.0.1:43210\n", "" + return "", "" + + monkeypatch.setattr(runtime_module, "_docker", fake_docker) + task = Task( + env="any-env", + id="t", + runtime_config=RuntimeConfig(compose=compose, compose_service_access=True), + ) + + async with DockerRuntime(compose_service_socket="/vm/run/docker.sock")(task): + pass + + assert rendered["services"]["main"]["volumes"] == [ + { + "type": "bind", + "source": "/vm/run/docker.sock", + "target": "/media/hud/docker.sock", + } + ] + + def test_docker_runtime_accepts_only_one_environment_definition(tmp_path: Path) -> None: with pytest.raises(ValueError, match="either image or compose"): RuntimeConfig(image="img:tag", compose=tmp_path / "compose.yaml") @@ -807,7 +889,7 @@ async def test_modal_runtime_config_flows_into_modal_sdk( resources=RuntimeResources( cpu=2, memory_mb=4096, - gpu=RuntimeGPU(type="A10G", count=2), + gpu=RuntimeGPU(type=["A10G", "L4"], count=2), ), limits=RuntimeLimits(startup_timeout_s=30, run_timeout_s=120), ) @@ -854,6 +936,8 @@ async def test_modal_runtime_runs_compose_inside_a_dind_vm( )(_row()) as runtime: assert runtime.url == "tcp://modal.host:4567" assert runtime.params == {"provider": "modal", "instance_id": "sb-1", "ready_timeout": 600} + assert runtime.handoff is not None + await runtime.handoff.export_to(tmp_path / "handoff.tar.gz") assert calls["registry_image"] == "docker:28.3.3-dind" kwargs = calls["sandbox_kwargs"] @@ -884,8 +968,31 @@ async def test_modal_runtime_runs_compose_inside_a_dind_vm( assert "docker compose" in execs[0][0][-1] assert "sh /hud/project/build.sh" in execs[0][0][-1] assert 'up --detach "$BUILD_FLAG" --remove-orphans' in execs[0][0][-1] - assert "down" in execs[1][0] + assert any("runtime handoff contains an unsupported entry" in call[0][-1] for call in execs) + assert "down" in execs[-1][0] assert execs[0][1]["timeout"] == 600 + assert calls["downloads"] == [("/media/hud/handoff.tar.gz", "handoff.tar.gz")] + + +async def test_modal_runtime_bounds_compose_startup( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = _install_fake_modal(monkeypatch) + calls["exec_wait"] = asyncio.Event() + compose = tmp_path / "compose.yaml" + compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") + + with pytest.raises(TimeoutError, match="Modal Compose startup timed out after 1 seconds"): + async with ModalRuntime( + runtime_config=RuntimeConfig( + compose=compose, + limits=RuntimeLimits(startup_timeout_s=1), + ) + )(_row()): + pytest.fail("runtime should not become ready") + + assert calls["terminated"] is True async def test_modal_runtime_accepts_modal_image_uri( @@ -967,7 +1074,7 @@ async def test_modal_runtime_can_override_workdir( assert sandbox_kwargs["workdir"] == "/app" -async def test_modal_runtime_applies_env_vars_to_image( +async def test_modal_runtime_passes_env_vars_to_sandbox( monkeypatch: pytest.MonkeyPatch, ) -> None: calls = _install_fake_modal(monkeypatch) @@ -979,11 +1086,8 @@ async def test_modal_runtime_applies_env_vars_to_image( sandbox_kwargs = calls["sandbox_kwargs"] assert isinstance(sandbox_kwargs, dict) - assert sandbox_kwargs["image"] == _ModalImageRef( - "registry", - "img:tag", - {"TOKEN": "secret"}, - ) + assert sandbox_kwargs["image"] == _ModalImageRef("registry", "img:tag") + assert sandbox_kwargs["env"] == {"TOKEN": "secret"} async def test_daytona_runtime_config_flows_into_daytona_sdk( @@ -995,7 +1099,7 @@ async def test_daytona_runtime_config_flows_into_daytona_sdk( resources=RuntimeResources( cpu=2, memory_mb=4096, - gpu=RuntimeGPU(type="H100", count=2), + gpu=RuntimeGPU(type=["H100", "A100"], count=2), ), limits=RuntimeLimits(startup_timeout_s=45), ) @@ -1017,7 +1121,7 @@ async def test_daytona_runtime_config_flows_into_daytona_sdk( cpu=2, memory=4, gpu=2, - gpu_type=[_DaytonaGpuType("H100")], + gpu_type=[_DaytonaGpuType("H100"), _DaytonaGpuType("A100")], ), ) assert create_timeout == 45 @@ -1098,7 +1202,6 @@ def test_compose_config_normalizes_supported_short_syntax(tmp_path: Path) -> Non @pytest.mark.parametrize( "service", [ - "command: echo $VALUE", "extends: {file: base.yaml, service: base}", ], ) @@ -1110,6 +1213,55 @@ def test_compose_config_rejects_external_resolution(tmp_path: Path, service: str ComposeConfig.from_file(compose) +def test_compose_config_interpolates_only_artifact_supplied_values( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("HOST_ONLY", "secret") + (tmp_path / ".env").write_text("IMAGE=example:1\nEMPTY=\n", encoding="utf-8") + compose = tmp_path / "compose.yaml" + compose.write_text( + """ +services: + main: + image: ${IMAGE} + command: "${EMPTY:-serve} $$HOME ${MISSING-default} $? $" + environment: + LITERAL: '$HOST_ONLY' +""", + encoding="utf-8", + ) + + service = ComposeConfig.from_file(compose).services["main"] + + assert service.image == "example:1" + assert service.command == ["serve", "$$HOME", "default", "$?", "$"] + assert service.environment == {"LITERAL": "$HOST_ONLY"} + + +@pytest.mark.parametrize( + ("image", "message"), + [ + ("$HOST_ONLY", r"HOST_ONLY.*not set by the project \.env"), + ("${IMAGE:?set IMAGE in .env}", r"set IMAGE in \.env"), + ], +) +def test_compose_config_rejects_unbound_variables( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + image: str, + message: str, +) -> None: + monkeypatch.setenv("HOST_ONLY", "secret") + compose = tmp_path / "compose.yaml" + compose.write_text( + f"services:\n main:\n image: {image}\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match=message): + ComposeConfig.from_file(compose) + + def test_compose_config_relocates_project_paths() -> None: compose = ComposeConfig.model_validate( { @@ -1311,6 +1463,54 @@ async def test_daytona_rejects_a_fractional_cpu_request(monkeypatch: pytest.Monk pass +async def test_daytona_rounds_minimum_storage_up_to_gibibytes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + daytona = _install_fake_daytona(monkeypatch) + config = RuntimeConfig(image="img:tag", resources=RuntimeResources(storage_mb=1025)) + + async with DaytonaRuntime(runtime_config=config)(_row()): + pass + + assert daytona.created[-1].resources.disk == 2 + + +async def test_modal_accepts_best_effort_storage( + monkeypatch: pytest.MonkeyPatch, +) -> None: + config = RuntimeConfig(image="img:tag", resources=RuntimeResources(storage_mb=1024)) + calls = _install_fake_modal(monkeypatch) + + async with ModalRuntime(runtime_config=config)(_row()): + pass + + assert calls["sandbox_kwargs"] == { + "app": "app", + "image": _ModalImageRef("registry", "img:tag"), + "workdir": None, + "unencrypted_ports": [8765], + "readiness_probe": ("tcp", 8765), + "timeout": 3600, + } + + +async def test_docker_admits_minimum_free_disk( + tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_fake_docker( + tmp_path, + port_behavior="echo 127.0.0.1:43210", + monkeypatch=monkeypatch, + ) + config = RuntimeConfig(image="img:tag", resources=RuntimeResources(storage_mb=1024)) + + async with DockerRuntime(runtime_config=config)(_row()): + pass + + calls = await _docker_calls(docker_log) + assert "exec cid-42 df -Pk /" in calls + + async def test_daytona_names_a_sandbox_it_could_not_delete( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: @@ -1408,6 +1608,7 @@ async def test_container_that_dies_before_serving_fails_with_its_logs( def test_docker_profile_allows_workspace_namespace_syscalls() -> None: profile = json.loads(runtime_module._DOCKER_SECCOMP_PROFILE.read_text()) denied = {name for rule in profile["syscalls"] for name in rule["names"]} + ptrace = next(rule for rule in profile["syscalls"] if "ptrace" in rule["names"]) assert profile["defaultAction"] == "SCMP_ACT_ALLOW" assert { @@ -1425,6 +1626,7 @@ def test_docker_profile_allows_workspace_namespace_syscalls() -> None: "ptrace", "userfaultfd", } <= denied + assert ptrace["excludes"] == {"caps": ["CAP_SYS_PTRACE"]} async def test_docker_runtime_always_prepares_for_workspace_isolation( @@ -1439,3 +1641,68 @@ async def test_docker_runtime_always_prepares_for_workspace_isolation( assert f"seccomp={runtime_module._DOCKER_SECCOMP_PROFILE}" in calls[0] assert "seccomp=unconfined" not in calls[0] assert "systempaths=unconfined" in calls[0] + assert "apparmor=unconfined" in calls[0] + + +def test_compose_network_owner_follows_service_chains_and_stages_its_port( + tmp_path: Path, +) -> None: + compose = tmp_path / "compose.yaml" + compose.write_text( + "services:\n" + " main:\n image: hud:latest\n network_mode: service:relay\n" + " relay:\n image: relay:latest\n network_mode: service:gateway\n" + " gateway:\n image: gateway:latest\n", + encoding="utf-8", + ) + config = ComposeConfig.from_file(compose) + + assert config.network_owner("main") == "gateway" + with ComposeProject(compose).stage( + "127.0.0.1::8765", + port_service=config.network_owner("main"), + seccomp="profile.json", + ) as files: + assert " gateway:" in files.ports.read_text("utf-8") + + +@pytest.mark.parametrize( + ("services", "message"), + [ + ( + { + "main": {"network_mode": "service:relay"}, + "relay": {"network_mode": "service:main"}, + }, + "cycle", + ), + ({"main": {"network_mode": "service:missing"}}, "unknown service"), + ], +) +def test_compose_network_owner_rejects_invalid_graphs( + services: dict[str, dict[str, str]], + message: str, +) -> None: + config = ComposeConfig.model_validate({"services": services}) + + with pytest.raises(ValueError, match=message): + config.network_owner("main") + + +async def test_docker_rejects_insufficient_free_disk( + tmp_path: Path, + docker_log: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_fake_docker( + tmp_path, + port_behavior="echo 127.0.0.1:43210", + monkeypatch=monkeypatch, + ) + config = RuntimeConfig(image="img:tag", resources=RuntimeResources(storage_mb=8192)) + + with pytest.raises(RuntimeError, match=r"requires 8192 MB.*has 4096 MB"): + async with DockerRuntime(runtime_config=config)(_row()): + pass + + assert (await _docker_calls(docker_log))[-1] == "rm --force cid-42" diff --git a/hud/eval/tests/test_hosted.py b/hud/eval/tests/test_hosted.py index 4828cbfd9..965a3fb35 100644 --- a/hud/eval/tests/test_hosted.py +++ b/hud/eval/tests/test_hosted.py @@ -193,7 +193,7 @@ async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch ] ) monkeypatch.setattr( - "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform) + "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) ) hosted = HostedRuntime(poll_interval=0.0) @@ -246,7 +246,7 @@ async def test_run_preserves_runtime_config_null_override( ) -> None: platform = _FakePlatform([{"status": "completed", "reward": 0.5}]) monkeypatch.setattr( - "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform) + "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) ) await HostedRuntime(poll_interval=0.0).run( @@ -278,7 +278,7 @@ async def test_run_submits_compose_document( ) platform = _FakePlatform([{"status": "completed", "reward": 0.5}]) monkeypatch.setattr( - "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform) + "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) ) await HostedRuntime(poll_interval=0.0).run( @@ -302,7 +302,7 @@ async def test_run_submits_compose_document( async def test_run_timeout_requests_platform_cancel(monkeypatch: pytest.MonkeyPatch) -> None: platform = _FakePlatform([{"status": "running"}]) monkeypatch.setattr( - "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform) + "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) ) hosted = HostedRuntime(poll_interval=0.0, run_timeout=0.001) @@ -330,7 +330,7 @@ async def apost(self, path: str, *, json: Any | None = None) -> Any: platform = _StuckSubmitPlatform([]) monkeypatch.setattr( - "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform) + "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) ) run = await HostedRuntime(run_timeout=0.001).run( @@ -348,7 +348,7 @@ async def apost(self, path: str, *, json: Any | None = None) -> Any: async def test_run_folds_completed_receipt(monkeypatch: pytest.MonkeyPatch) -> None: platform = _FakePlatform([{"status": "completed", "reward": 1.0, "error": None}]) monkeypatch.setattr( - "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform) + "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) ) task = Task(env="sums", id="add", args={"a": 2, "b": 3}) @@ -367,7 +367,7 @@ async def test_run_folds_completed_receipt(monkeypatch: pytest.MonkeyPatch) -> N async def test_run_folds_error_receipt(monkeypatch: pytest.MonkeyPatch) -> None: platform = _FakePlatform([{"status": "error", "reward": None, "error": "env exploded"}]) monkeypatch.setattr( - "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform) + "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) ) task = Task(env="sums", id="add", args={}) @@ -384,7 +384,7 @@ async def test_run_keeps_a_grade_from_an_errored_hosted_trace( ) -> None: platform = _FakePlatform([{"status": "error", "reward": 0.75, "error": "agent timed out"}]) monkeypatch.setattr( - "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform) + "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) ) task = Task(env="sums", id="add", args={}) @@ -408,7 +408,7 @@ async def test_run_folds_ungraded_cancellation_as_an_error( ) -> None: platform = _FakePlatform([{"status": "cancelled", "reward": None, "error": None}]) monkeypatch.setattr( - "hud.eval.runtime.PlatformClient.from_settings", classmethod(lambda cls: platform) + "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) ) task = Task(env="sums", id="add", args={}) @@ -482,7 +482,7 @@ async def fake_rollout(task: Task, agent: Any, **kwargs: Any) -> Run: run.trace.status = "completed" return run - monkeypatch.setattr("hud.eval.runtime.rollout", fake_rollout) + monkeypatch.setattr("hud.eval.runtime.hud.rollout", fake_rollout) runtime = HUDRuntime(run_timeout=90.0) job_id = uuid.uuid4().hex @@ -502,6 +502,16 @@ async def fake_rollout(task: Task, agent: Any, **kwargs: Any) -> Run: assert seen["trace_id"] == trace_id assert seen["rollout_timeout"] == 90.0 + with pytest.raises(ValueError, match="placement requirements"): + async with runtime( + Task( + env="e", + id="x", + runtime_config=RuntimeConfig(resources=RuntimeResources(gpu=RuntimeGPU())), + ) + ): + pass + @pytest.mark.asyncio async def test_runtime_session_create_payload_omits_trace_id( @@ -530,7 +540,7 @@ async def post( posts.append({"path": path, "headers": headers, "json": json}) return _FakeResponse({"id": session_id}) - monkeypatch.setattr("hud.eval.runtime.httpx.AsyncClient", _RecordingAsyncClient) + monkeypatch.setattr("hud.eval.runtime.hud.httpx.AsyncClient", _RecordingAsyncClient) created = await HUDRuntime()._create_runtime_session( "https://mcp.hud.ai", @@ -576,7 +586,7 @@ async def post( posts.append({"path": path, "headers": headers, "json": json}) return _FakeResponse({"id": session_id}) - monkeypatch.setattr("hud.eval.runtime.httpx.AsyncClient", _RecordingAsyncClient) + monkeypatch.setattr("hud.eval.runtime.hud.httpx.AsyncClient", _RecordingAsyncClient) with set_trace_context(trace_id): created = await HUDRuntime()._create_runtime_session( @@ -644,7 +654,7 @@ async def fake_delete_runtime_session( deleted.append((runtime_url, api_key, session)) monkeypatch.setattr(settings, "api_key", "sk-hud-test") - monkeypatch.setattr("hud.eval.runtime.asyncio.start_server", fake_start_server) + monkeypatch.setattr("hud.eval.runtime.hud.asyncio.start_server", fake_start_server) monkeypatch.setattr(HUDRuntime, "_create_runtime_session", fake_create_runtime_session) monkeypatch.setattr(HUDRuntime, "_delete_runtime_session", fake_delete_runtime_session) diff --git a/hud/eval/tests/test_local_runtime.py b/hud/eval/tests/test_local_runtime.py index 67d901c26..39b404d9f 100644 --- a/hud/eval/tests/test_local_runtime.py +++ b/hud/eval/tests/test_local_runtime.py @@ -199,12 +199,12 @@ async def test_inferred_placement_includes_a_verifier_environment(tmp_path, requ @actor.template(id="solve") async def solve(): answer = yield "answer" - yield 0.25 + yield {{"score": 0.25, "answer": answer}} @verifier.template(id="verify") async def verify(): - answer = yield "" - yield 1.0 if answer == "secret" else 0.0 + result = yield "" + yield 1.0 if result["answer"] == "secret" else 0.0 """, encoding="utf-8", ) diff --git a/hud/eval/tests/test_rollout.py b/hud/eval/tests/test_rollout.py index 080f78407..4a06387e0 100644 --- a/hud/eval/tests/test_rollout.py +++ b/hud/eval/tests/test_rollout.py @@ -26,12 +26,13 @@ import mcp.types as mcp_types import pytest +from pydantic import BaseModel from hud.agents.base import Agent from hud.agents.openai_compatible import OpenAIChatAgent from hud.agents.types import OpenAIChatConfig -from hud.environment import Environment -from hud.eval import Job, SubprocessRuntime, Task, Taskset +from hud.environment import Answer, Environment +from hud.eval import Job, Runtime, SubprocessRuntime, Task, Taskset from hud.eval.run import Run, rollout from hud.eval.runtime import _local @@ -39,7 +40,6 @@ from collections.abc import AsyncIterator from pathlib import Path - from hud.eval.runtime import Runtime from hud.eval.task import Task as TaskRow _SUMS_ENV = """\ @@ -175,13 +175,13 @@ async def test_verifier_task_replaces_the_actor_grade_in_the_same_runtime() -> N async def solve(): answer = yield "answer secret" completed.append(f"actor:{answer}") - yield 0.25 + yield {"score": 0.25, "answer": answer} @env.template() async def verify(expected: str): - answer = yield "" - completed.append(f"verifier:{answer}") - yield 1.0 if answer == expected else 0.0 + result = yield "" + completed.append(f"verifier:{result['answer']}") + yield 1.0 if result["answer"] == expected else 0.0 task = Task( env="reviewed", @@ -200,6 +200,88 @@ async def verify(expected: str): assert evaluations == ["solve", "verify"] +async def test_actor_result_is_forwarded_to_the_verifier() -> None: + env = Environment("reviewed") + received: list[tuple[str, str]] = [] + + class ActorResult(BaseModel): + score: float + answer: str + handoff: str + + @env.template() + async def solve(): + answer = yield "answer secret" + yield {"score": 0.0, "answer": answer, "handoff": "token"} + + @env.template(returns=ActorResult) + async def verify(): + answer = yield "" + assert isinstance(answer, Answer) + assert isinstance(answer.content, ActorResult) + received.append((answer.content.answer, answer.content.handoff)) + yield 1.0 + + task = Task( + env="reviewed", + id="solve", + verifier=Task(env="reviewed", id="verify"), + ) + run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=lambda _row: _local(env)) + + assert run.reward == 1.0 + assert received == [("secret", "token")] + + +async def test_independent_verifier_receives_runtime_handoff() -> None: + actor_env = Environment("actor") + verifier_env = Environment("judge") + transfers: list[str] = [] + + @actor_env.template() + async def solve(): + yield "answer secret" + yield {"score": 0.0, "handoff": "token"} + + @verifier_env.template() + async def verify(): + result = yield "" + yield 1.0 if result["handoff"] == "token" else 0.0 + + class ActorHandoff: + async def export_to(self, destination: Path) -> None: + await asyncio.to_thread(destination.write_text, "files", encoding="utf-8") + transfers.append("export") + + async def import_from(self, source: Path) -> None: + raise AssertionError("actor imported a handoff") + + class VerifierHandoff: + async def export_to(self, destination: Path) -> None: + raise AssertionError("verifier exported a handoff") + + async def import_from(self, source: Path) -> None: + assert await asyncio.to_thread(source.read_text, "utf-8") == "files" + transfers.append("import") + + @asynccontextmanager + async def provider(row: TaskRow) -> AsyncIterator[Runtime]: + env = actor_env if row.env == "actor" else verifier_env + endpoint = ActorHandoff() if row.env == "actor" else VerifierHandoff() + async with _local(env) as runtime: + yield Runtime(runtime.url, handoff=endpoint) + + task = Task( + env="actor", + id="solve", + verifier=Task(env="judge", id="verify", requires_handoff=True), + ) + run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) + + assert run.reward == 1.0 + assert transfers == ["export", "import"] + + async def test_verifier_with_its_own_environment_is_placed_after_the_actor() -> None: actor_env = Environment("actor") verifier_env = Environment("judge") @@ -207,13 +289,13 @@ async def test_verifier_with_its_own_environment_is_placed_after_the_actor() -> @actor_env.template() async def solve(): - yield "answer secret" - yield 0.25 + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} @verifier_env.template() async def verify(): - answer = yield "" - yield 1.0 if answer == "secret" else 0.0 + result = yield "" + yield 1.0 if result["answer"] == "secret" else 0.0 @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: @@ -239,13 +321,13 @@ async def test_verifier_remains_authoritative_after_an_agent_error() -> None: @actor_env.template() async def solve(): - yield "answer secret" - yield 0.25 + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} @verifier_env.template() async def verify(): - answer = yield "" - yield 1.0 if answer == "secret" else 0.0 + result = yield "" + yield 1.0 if result["answer"] == "secret" else 0.0 @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: @@ -281,8 +363,8 @@ async def solve(): @verifier_env.template() async def verify(): - answer = yield "" - yield 1.0 if answer == "secret" else 0.0 + result = yield "" + yield 1.0 if result["answer"] == "secret" else 0.0 @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: @@ -315,13 +397,13 @@ async def test_verifier_remains_authoritative_when_actor_grade_is_scoreless( @actor_env.template() async def solve(): - yield "answer secret" - yield 0.25 + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} @verifier_env.template() async def verify(): - answer = yield "" - yield 1.0 if answer == "secret" else 0.0 + result = yield "" + yield 1.0 if result["answer"] == "secret" else 0.0 @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: @@ -360,8 +442,8 @@ async def test_verifier_provisioning_failure_leaves_the_run_ungraded() -> None: @actor_env.template() async def solve(): - yield "answer secret" - yield 0.25 + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: @@ -878,8 +960,8 @@ async def test_timeout_during_actor_cleanup_does_not_start_the_verifier() -> Non @actor_env.template() async def solve(): - yield "answer secret" - yield 0.25 + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: @@ -920,13 +1002,13 @@ async def test_timeout_does_not_cancel_verifier_provider_cleanup() -> None: @actor_env.template() async def solve(): - yield "answer secret" - yield 0.25 + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} @verifier_env.template() async def verify(): - answer = yield "" - yield 1.0 if answer == "secret" else 0.0 + result = yield "" + yield 1.0 if result["answer"] == "secret" else 0.0 @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: diff --git a/hud/eval/tests/test_task.py b/hud/eval/tests/test_task.py index cf1f78da5..cbec55c7e 100644 --- a/hud/eval/tests/test_task.py +++ b/hud/eval/tests/test_task.py @@ -21,10 +21,11 @@ RuntimeConfig, RuntimeGPU, RuntimeResources, + RuntimeTPU, Task, Taskset, ) -from hud.eval.compose import ComposeConfig, ComposeProjectRef +from hud.eval.runtime.compose import ComposeConfig, ComposeProjectRef if TYPE_CHECKING: from pathlib import Path @@ -122,12 +123,20 @@ def test_roundtrip_is_stable_through_plain_pydantic() -> None: def test_runtime_config_roundtrips_as_part_of_task_row() -> None: + resources = RuntimeResources( + cpu=2, + memory_mb=4096, + storage_mb=16384, + gpu=RuntimeGPU(type=["H100", "A100"]), + os="windows", + tpu=RuntimeTPU(type="v5", topology="2x2"), + ) original = Task( env="browser", id="checkout", runtime_config=RuntimeConfig( image="hud-browser:firefox", - resources=RuntimeResources(cpu=2, memory_mb=4096, gpu=RuntimeGPU()), + resources=resources, ), ).model_dump(exclude_none=True) @@ -135,7 +144,7 @@ def test_runtime_config_roundtrips_as_part_of_task_row() -> None: assert rebuilt.runtime_config == RuntimeConfig( image="hud-browser:firefox", - resources=RuntimeResources(cpu=2, memory_mb=4096, gpu=RuntimeGPU()), + resources=resources, ) assert rebuilt.model_dump(exclude_none=True) == original diff --git a/hud/integrations/harbor/Dockerfile b/hud/integrations/harbor/Dockerfile index ec2619695..fcdd22a18 100644 --- a/hud/integrations/harbor/Dockerfile +++ b/hud/integrations/harbor/Dockerfile @@ -9,6 +9,7 @@ USER root ARG HUD_REQUIREMENT=hud COPY --from=ghcr.io/astral-sh/uv:0.8.15 /uv /media/hud/bin/uv COPY env.py install.sh config.json image-config.json verifier-image-config.json /media/hud/ +COPY peer-image-configs /media/hud/peer-image-configs COPY packages /media/hud/packages RUN sh /media/hud/install.sh "${HUD_REQUIREMENT}" @@ -19,6 +20,8 @@ CMD ["/media/hud/venv/bin/hud", "serve", "/media/hud/env.py", "--host", "0.0.0.0 FROM runtime AS plain -FROM runtime AS verifier +FROM runtime AS service-access COPY --from=docker-cli /usr/local/bin/docker /media/hud/bin/docker + +FROM service-access AS verifier COPY --from=verifier-root / /media/hud/verifier diff --git a/hud/integrations/harbor/__init__.py b/hud/integrations/harbor/__init__.py index 668371a26..ac3cb5281 100644 --- a/hud/integrations/harbor/__init__.py +++ b/hud/integrations/harbor/__init__.py @@ -7,7 +7,7 @@ This API may change between minor releases while the integration is experimental. """ -from .adapt import adapt +from .adapt import AdaptFailure, AdaptFinding, AdaptResult, adapt from .export import export -__all__ = ["adapt", "export"] +__all__ = ["AdaptFailure", "AdaptFinding", "AdaptResult", "adapt", "export"] diff --git a/hud/integrations/harbor/adapt.py b/hud/integrations/harbor/adapt.py index 81b86bdf2..1c52923b2 100644 --- a/hud/integrations/harbor/adapt.py +++ b/hud/integrations/harbor/adapt.py @@ -5,6 +5,7 @@ import hashlib import json import logging +import math import os import re import shlex @@ -19,8 +20,13 @@ from hud.capabilities import Capability from hud.environment.egress import BRIDGE_PORT, VISITOR_PORT from hud.eval import Task, Taskset -from hud.eval.compose import ComposeConfig, ComposeHealthcheck, ComposeService -from hud.eval.runtime import RuntimeConfig, RuntimeGPU, RuntimeResources +from hud.eval.runtime import RuntimeConfig, RuntimeGPU, RuntimeLimits, RuntimeResources, RuntimeTPU +from hud.eval.runtime.compose import ( + ComposeConfig, + ComposeHealthcheck, + ComposeService, + ComposeUnboundVariableError, +) from hud.utils.naming import normalize_environment_name LOGGER = logging.getLogger(__name__) @@ -37,18 +43,16 @@ ) NetworkMode = Literal["public", "no-network", "allowlist"] MCPTransport = Literal["sse", "streamable-http", "stdio"] -COMPOSE_FILENAMES = ( - "compose.yaml", - "compose.yml", - "docker-compose.yaml", - "docker-compose.yml", -) +FindingKind = Literal["contract", "invalid"] +COMPOSE_FILENAME = "docker-compose.yaml" class Artifact(BaseModel): model_config = ConfigDict(extra="forbid") source: str = Field(pattern=r"^/") + destination: str | None = None + exclude: list[str] = Field(default_factory=list) service: str = Field(default="main", min_length=1) @model_validator(mode="before") @@ -64,6 +68,20 @@ def normalize_source(cls, value: str) -> str: raise ValueError("artifact source must name a path beneath /") return str(path) + @field_validator("destination") + @classmethod + def validate_destination(cls, value: str | None) -> str | None: + if not value: + return None + if "\\" in value: + raise ValueError("artifact destination must use forward slashes") + path = PurePosixPath(value) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ValueError("artifact destination must be a relative path") + if value.rstrip("/") == "manifest.json": + raise ValueError("artifact destination 'manifest.json' is reserved") + return value + class Collect(BaseModel): model_config = ConfigDict(extra="forbid") @@ -135,13 +153,14 @@ class EnvironmentConfig(BaseModel): model_config = ConfigDict(extra="allow") docker_image: str | None = None - os: str = "linux" + os: Literal["linux", "windows"] = "linux" cpus: float | None = Field(default=None, gt=0) memory_mb: int | None = Field(default=None, gt=0) storage_mb: int | None = Field(default=None, gt=0) + build_timeout_sec: float | None = Field(default=None, gt=0) gpus: int | None = Field(default=None, ge=0) gpu_types: list[str] = Field(default_factory=list) - tpu: dict[str, Any] | None = None + tpu: RuntimeTPU | None = None network_mode: NetworkMode = "public" allowed_hosts: list[str] = Field(default_factory=list) workdir: str | None = None @@ -160,7 +179,7 @@ class Phase(BaseModel): allowed_hosts: list[str] = Field(default_factory=list) env: dict[str, str] = Field(default_factory=dict) environment: EnvironmentConfig | None = None - environment_mode: Literal["separate"] | None = None + environment_mode: Literal["shared", "separate"] | None = None collect: list[Collect] = Field(default_factory=list) @property @@ -189,6 +208,30 @@ class TaskConfig(BaseModel): steps: list[dict[str, Any]] | None = None +class AdaptFinding(BaseModel): + """One independently detectable reason a Harbor task was not adapted.""" + + code: str + kind: FindingKind + message: str + + +class AdaptFailure(BaseModel): + """All findings for one Harbor task.""" + + task: str + path: Path + findings: tuple[AdaptFinding, ...] + + +@dataclass(frozen=True, slots=True) +class AdaptResult: + """Successful task rows and structured failures from one adaptation.""" + + taskset: Taskset + failures: tuple[AdaptFailure, ...] + + @dataclass(frozen=True, slots=True) class HarborTask: path: Path @@ -196,6 +239,9 @@ class HarborTask: instruction: str environment_hash: str compose: ComposeConfig | None + dockerfile: Path + base_image: str + resources: RuntimeResources | None def _tree_hash(root: Path) -> str: @@ -209,11 +255,306 @@ def _tree_hash(root: Path) -> str: return digest.hexdigest()[:16] +def _runtime_resources(environment: EnvironmentConfig) -> RuntimeResources | None: + resources = RuntimeResources( + cpu=environment.cpus, + memory_mb=environment.memory_mb, + storage_mb=environment.storage_mb, + gpu=( + RuntimeGPU( + count=environment.gpus, + type=( + environment.gpu_types[0] + if len(environment.gpu_types) == 1 + else environment.gpu_types or None + ), + ) + if environment.gpus + else None + ), + os=environment.os if environment.os != "linux" else None, + tpu=environment.tpu, + ) + return resources if resources.model_dump(exclude_none=True) else None + + +def _runtime_limits(environment: EnvironmentConfig) -> RuntimeLimits | None: + if environment.build_timeout_sec is None: + return None + return RuntimeLimits(startup_timeout_s=math.ceil(environment.build_timeout_sec)) + + +def _dockerfile_stages(lines: list[str]) -> list[tuple[int, str | None]]: + escape = "\\" + for line in lines: + stripped = line.strip() + if not stripped: + continue + directive = re.fullmatch(r"#\s*escape\s*=\s*([\\`])", stripped, re.IGNORECASE) + if directive is not None: + escape = directive.group(1) + if not stripped.startswith("#"): + break + + stages: list[tuple[int, str | None]] = [] + heredoc_pattern = re.compile( + r"<<(?P-?)[ \t]*(?P['\"]?)" + r"(?P[A-Za-z_][\w.-]*)(?P=quote)" + ) + index = 0 + pattern = re.compile( + r"^\s*FROM\s+(?:--platform=\S+\s+)?\S+" + r"(?:\s+AS\s+(?P[A-Za-z0-9_.-]+))?" + r"\s*(?:#.*)?$", + re.IGNORECASE, + ) + while index < len(lines): + parts: list[str] = [] + while index < len(lines): + content = lines[index].rstrip("\r\n") + stripped = content.rstrip(" \t") + continued = stripped.endswith(escape) + parts.append(stripped[:-1] if continued else content) + index += 1 + if not continued: + break + instruction = " ".join(parts) + if re.match(r"^\s*FROM\b", instruction, re.IGNORECASE): + match = pattern.fullmatch(instruction) + if match is None: + raise ValueError("unsupported FROM instruction") + stages.append((index - 1, match.group("name"))) + for heredoc in heredoc_pattern.finditer(instruction): + delimiter = heredoc.group("name") + strip_tabs = bool(heredoc.group("strip")) + while index < len(lines): + terminator = lines[index].rstrip("\r\n") + index += 1 + if strip_tabs: + terminator = terminator.lstrip("\t") + if terminator == delimiter: + break + return stages + + +def _inspect_task(task_dir: Path) -> tuple[HarborTask | None, tuple[AdaptFinding, ...]]: + findings: list[AdaptFinding] = [] + + def add(code: str, message: str) -> None: + kind: FindingKind = "contract" if ".unsupported." in code else "invalid" + findings.append(AdaptFinding(code=code, kind=kind, message=message)) + + try: + raw_config = tomllib.loads((task_dir / "task.toml").read_text("utf-8")) + except OSError as error: + return None, ( + AdaptFinding(code="harbor.invalid.task_config_io", kind="invalid", message=str(error)), + ) + except tomllib.TOMLDecodeError as error: + return None, ( + AdaptFinding( + code="harbor.invalid.task_config_toml", kind="invalid", message=str(error) + ), + ) + + try: + config = TaskConfig.model_validate(raw_config) + except ValidationError as error: + return None, tuple( + AdaptFinding( + code="harbor.invalid.task_config", + kind="invalid", + message=f"{'.'.join(str(part) for part in detail['loc'])}: {detail['msg']}", + ) + for detail in error.errors(include_url=False) + ) + + environment = config.environment + resources = _runtime_resources(environment) + if any(server.transport == "stdio" for server in environment.mcp_servers): + add("harbor.unsupported.mcp_stdio", "stdio MCP servers are not supported") + if environment.skills_dir: + add( + "harbor.unsupported.skills_dir", + "per-task agent skills are not supported", + ) + + if config.steps: + add("harbor.unsupported.multi_step", "multi-step tasks are not supported") + + server_names = [server.name for server in environment.mcp_servers] + if len(server_names) != len(set(server_names)): + add("harbor.invalid.duplicate_mcp_name", "MCP server names must be unique") + for name in sorted({"shell", "filetracking"} & set(server_names)): + add( + "harbor.invalid.reserved_mcp_name", + f"MCP server name {name!r} is reserved by the workspace", + ) + for server in environment.mcp_servers: + if server.transport != "stdio" and server.url is None: + add( + "harbor.invalid.mcp_url", + f"MCP server {server.name!r} requires a URL", + ) + + environment_dir = task_dir / "environment" + compose_path = environment_dir / COMPOSE_FILENAME + authored_compose = compose_path if compose_path.is_file() else None + compose = None + dockerfile = environment_dir / "Dockerfile" + base_image: str | None = None + if authored_compose is not None: + try: + compose = ComposeConfig.from_file(authored_compose) + except ComposeUnboundVariableError as error: + add("harbor.unsupported.host_compose_variable", str(error)) + except (OSError, ValueError, ValidationError) as error: + add("harbor.invalid.compose", str(error)) + else: + compose.services.setdefault("main", ComposeService()) + compose.name = None + try: + compose.with_project_directory("./environment") + except ValueError as error: + add("harbor.invalid.compose_project_path", str(error)) + + if authored_compose is None or compose is not None: + compose_main = compose.services["main"] if compose is not None else ComposeService() + base_image = environment.docker_image or compose_main.image + if compose is not None: + build = compose_main.build + if build is not None: + build_config = {"context": build} if isinstance(build, str) else build + build_context = build_config.get("context", ".") + build_dockerfile = build_config.get("dockerfile", "Dockerfile") + if not isinstance(build_context, str) or not isinstance(build_dockerfile, str): + add( + "harbor.invalid.compose_main_build_path", + "Compose main build paths must be strings", + ) + else: + dockerfile = (environment_dir / build_context / build_dockerfile).resolve() + try: + dockerfile.relative_to(environment_dir.resolve()) + except ValueError: + add( + "harbor.invalid.compose_main_build_escape", + "Compose main build escapes environment", + ) + if dockerfile.is_file(): + base_image = f"hud-harbor-base:{_tree_hash(environment_dir)}" + elif build is not None: + add( + "harbor.invalid.missing_compose_main_dockerfile", + "Compose main Dockerfile does not exist", + ) + elif base_image is None: + add( + "harbor.invalid.compose_main_recipe", + "Compose main has neither image nor build", + ) + elif dockerfile.is_file(): + base_image = f"hud-harbor-base:{_tree_hash(environment_dir)}" + elif base_image is None: + add( + "harbor.invalid.environment_recipe", + "task has neither environment/Dockerfile nor docker_image", + ) + + if not config.steps: + if config.verifier.separate and not (task_dir / "tests" / "Dockerfile").is_file(): + add( + "harbor.invalid.missing_verifier_dockerfile", + "separate verifier requires tests/Dockerfile", + ) + elif not (task_dir / "tests").is_dir(): + add( + "harbor.invalid.missing_tests", + "task requires a tests directory", + ) + + if compose is not None: + if {"hud-base", "hud-verifier"} & compose.services.keys(): + add( + "harbor.invalid.reserved_compose_service", + "Compose service names 'hud-base' and 'hud-verifier' are reserved", + ) + for service_name, service in compose.services.items(): + if service_name == "main": + continue + if service.build is None and service.image is None: + add( + "harbor.invalid.sidecar_recipe", + f"Compose service {service_name!r} has neither image nor build", + ) + workdir = environment.workdir or compose_main.working_dir + if workdir is not None and Path(workdir).is_relative_to(HUD_ROOT): + add( + "harbor.invalid.reserved_workdir", + f"Harbor workdir {workdir!r} is inside reserved path {HUD_ROOT}", + ) + for port in sorted(compose_main.tcp_ports & {BRIDGE_PORT, VISITOR_PORT, 8765}): + add( + "harbor.invalid.reserved_main_port", + f"Harbor main service port {port} conflicts with a HUD reserved port", + ) + if environment.healthcheck is None and compose_main.healthcheck is not None: + try: + HealthcheckConfig.from_compose(compose_main.healthcheck) + except ValueError as error: + add("harbor.invalid.healthcheck", str(error)) + + if compose is None and dockerfile.is_file(): + try: + lines = dockerfile.read_text("utf-8").splitlines(keepends=True) + stages = _dockerfile_stages(lines) + if not stages: + raise ValueError("environment/Dockerfile has no FROM stage") + except (OSError, UnicodeError, ValueError) as error: + add("harbor.invalid.dockerfile", str(error)) + else: + stage_names = { + stage_name.lower() for _, stage_name in stages if stage_name is not None + } + reserved_names = {"hud-base", "hud-runtime"} + if config.verifier.separate: + reserved_names.update({"hud-docker-cli", "hud-verifier", "hud-verifier-root"}) + for stage in sorted(reserved_names & stage_names): + add( + "harbor.invalid.reserved_dockerfile_stage", + f"environment/Dockerfile uses reserved stage {stage!r}", + ) + + instruction = task_dir / "instruction.md" + if not config.steps and not instruction.is_file(): + add( + "harbor.invalid.missing_instruction", + f"{task_dir.name} has no instruction.md", + ) + if findings: + return None, tuple(findings) + + assert base_image is not None + return ( + HarborTask( + path=task_dir, + config=config, + instruction=instruction.read_text("utf-8"), + environment_hash=_tree_hash(environment_dir) if environment_dir.exists() else "missing", + compose=compose, + dockerfile=dockerfile, + base_image=base_image, + resources=resources, + ), + (), + ) + + def adapt( path: str | Path, *, hud_requirement: str = "hud", -) -> Taskset: +) -> AdaptResult: """Package Harbor tasks as buildable Compose projects.""" root = Path(path).resolve() if (root / "task.toml").is_file(): @@ -228,95 +569,14 @@ def adapt( if not task_dirs: raise ValueError(f"no Harbor tasks found in {path}") - tasks = [] + tasks: list[HarborTask] = [] + failures: list[AdaptFailure] = [] for task_dir in task_dirs: - try: - config = TaskConfig.model_validate( - tomllib.loads((task_dir / "task.toml").read_text("utf-8")) - ) - except (OSError, tomllib.TOMLDecodeError, ValidationError) as error: - raise ValueError( - f"{task_dir.name}/task.toml is not a valid Harbor task: {error}" - ) from error - unsupported = [] - if config.environment.os != "linux": - unsupported.append(f"os={config.environment.os!r}") - if config.environment.tpu: - unsupported.append("TPUs") - if len(config.environment.gpu_types) > 1: - unsupported.append("multiple GPU types") - elif config.environment.gpu_types and not config.environment.gpus: - unsupported.append("GPU types without GPUs") - if any(server.transport == "stdio" for server in config.environment.mcp_servers): - unsupported.append("stdio MCP servers") - if config.environment.skills_dir: - unsupported.append("skills_dir") - verifier_environment = config.verifier.environment - if verifier_environment is not None: - if verifier_environment.os != "linux": - unsupported.append(f"verifier os={verifier_environment.os!r}") - if verifier_environment.tpu: - unsupported.append("verifier TPUs") - if len(verifier_environment.gpu_types) > 1: - unsupported.append("multiple verifier GPU types") - elif verifier_environment.gpu_types and not verifier_environment.gpus: - unsupported.append("verifier GPU types without GPUs") - gpu_types = { - *config.environment.gpu_types, - *verifier_environment.gpu_types, - } - if len(gpu_types) > 1: - unsupported.append("different agent and verifier GPU types") - if config.steps: - unsupported.append("multi-step tasks") - if unsupported: - raise NotImplementedError( - f"Harbor task {task_dir.name!r} uses unsupported features: " - + ", ".join(unsupported) - ) - - server_names = [server.name for server in config.environment.mcp_servers] - if len(server_names) != len(set(server_names)): - raise ValueError("MCP server names must be unique") - if reserved := {"shell", "filetracking"} & set(server_names): - raise ValueError(f"MCP server name {min(reserved)!r} is reserved by the workspace") - for server in config.environment.mcp_servers: - if server.url is None: - raise ValueError(f"MCP server {server.name!r} requires a URL") - - environment_dir = task_dir / "environment" - authored_compose = next( - ( - environment_dir / filename - for filename in COMPOSE_FILENAMES - if (environment_dir / filename).is_file() - ), - None, - ) - compose = None - if authored_compose is not None: - try: - compose = ComposeConfig.from_file(authored_compose) - compose.services["main"] - except (ValidationError, KeyError) as error: - raise ValueError(f"{task_dir.name} did not resolve to a Compose project") from error - compose.name = None - - instruction = task_dir / "instruction.md" - if not instruction.is_file(): - raise FileNotFoundError(f"{task_dir.name} has no instruction.md") - - tasks.append( - HarborTask( - path=task_dir, - config=config, - instruction=instruction.read_text("utf-8"), - environment_hash=_tree_hash(environment_dir) - if environment_dir.exists() - else "missing", - compose=compose, - ) - ) + task, findings = _inspect_task(task_dir) + if task is not None: + tasks.append(task) + else: + failures.append(AdaptFailure(task=task_dir.name, path=task_dir, findings=findings)) grouped: dict[tuple[str, str, str], list[HarborTask]] = {} for task in tasks: @@ -365,82 +625,56 @@ def adapt( update={"image": f"hud-harbor-sidecar:{sidecar_tag}"} ) compose_main = compose.services["main"] if compose is not None else ComposeService() - dockerfile = source.path / "environment" / "Dockerfile" - base_image = environment.docker_image or compose_main.image - if compose is not None: - build = compose_main.build - if build is not None: - build_config = {"context": build} if isinstance(build, str) else build - build_context = build_config.get("context", ".") - build_dockerfile = build_config.get("dockerfile", "Dockerfile") - if not isinstance(build_context, str) or not isinstance(build_dockerfile, str): - raise ValueError("Compose main build paths must be strings") - dockerfile = ( - source.path / "environment" / build_context / build_dockerfile - ).resolve() - try: - dockerfile.relative_to((source.path / "environment").resolve()) - except ValueError: - raise ValueError("Compose main build escapes environment") from None - if dockerfile.is_file(): - base_image = f"hud-harbor-base:{source.environment_hash}" - elif build is not None: - raise FileNotFoundError( - f"{source.path.name} Compose main Dockerfile does not exist" - ) - elif base_image is None: - raise FileNotFoundError( - f"{source.path.name} Compose main has neither image nor build" - ) - elif dockerfile.is_file(): - base_image = f"hud-harbor-base:{source.environment_hash}" - elif base_image is None: - raise FileNotFoundError( - f"{source.path.name} has neither environment/Dockerfile nor docker_image" - ) + dockerfile = source.dockerfile + base_image = source.base_image separate = source.config.verifier.separate verifier_environment = source.config.verifier.environment or EnvironmentConfig() verifier_image = base_image if separate: verifier_dockerfile = source.path / "tests" / "Dockerfile" - if not verifier_dockerfile.is_file(): - raise FileNotFoundError( - f"{source.path.name} uses a separate verifier but has no tests/Dockerfile" - ) verifier_image = f"hud-harbor-verifier:{name}-{_tree_hash(verifier_dockerfile.parent)}" peers = [] + healthy_services = [] + peer_image_configs: dict[str, str] = {} if compose is not None: - for service_name, service in compose.services.items(): - if service_name == "main": + completed_services: set[str] = set() + for service in compose.services.values(): + depends_on = (service.model_extra or {}).get("depends_on") + if not isinstance(depends_on, dict): continue - ports = { - int(value) - for exposed in service.expose - if (value := str(exposed).partition("/")[0]).isdigit() - } - ports.update( - published.target for published in service.ports if published.protocol == "tcp" + completed_services.update( + name + for name, dependency in depends_on.items() + if isinstance(name, str) + and isinstance(dependency, dict) + and dependency.get("condition") == "service_completed_successfully" ) - if service.build is None and service.image is None: - raise ValueError( - f"Compose service {service_name!r} has neither image nor build" - ) - if len(ports) > 1: - raise NotImplementedError( - f"Compose service {service_name!r} exposes multiple ports; " - "Peer names one endpoint" + for service_name, service in compose.services.items(): + if service_name == "main" or service_name in completed_services: + continue + healthcheck = service.healthcheck + if ( + healthcheck is not None + and healthcheck.disable is not True + and healthcheck.test != ["NONE"] + ): + healthy_services.append(service_name) + if service.tcp_ports: + peers.extend( + {"name": service_name, "port": port} for port in sorted(service.tcp_ports) ) - if not ports: - raise ValueError(f"Compose service {service_name!r} declares no TCP port") - peers.append({"name": service_name, "port": next(iter(ports))}) + else: + peer_image_configs[service_name] = f"peer-image-configs/{service_name}.json" context = dataset / ".hud-adapt" / name if context.exists(): shutil.rmtree(context) project = context / "compose-project" payload = project / ("main" if compose is not None else "hud") (payload / "packages").mkdir(parents=True) + if compose is not None: + (payload / "peer-image-configs").mkdir() shutil.copy2(ASSETS / "install.sh", payload / "install.sh") if compose is not None: shutil.copy2(ASSETS / "Dockerfile", payload / "Dockerfile") @@ -456,20 +690,7 @@ def adapt( target.write_text(served, encoding="utf-8", newline="\n") workdir = environment.workdir or compose_main.working_dir - if workdir is not None and Path(workdir).is_relative_to(HUD_ROOT): - raise ValueError(f"Harbor workdir {workdir!r} is inside reserved path {HUD_ROOT}") - ports = { - int(port) - for exposed in compose_main.expose - if (port := str(exposed).partition("/")[0]).isdigit() - } - ports.update( - published.target for published in compose_main.ports if published.protocol == "tcp" - ) - if conflict := ports & {BRIDGE_PORT, VISITOR_PORT, 8765}: - raise ValueError( - f"Harbor main service port {min(conflict)} conflicts with a HUD reserved port" - ) + ports = compose_main.tcp_ports healthcheck = environment.healthcheck if healthcheck is None and compose_main.healthcheck is not None: healthcheck = HealthcheckConfig.from_compose(compose_main.healthcheck) @@ -521,6 +742,8 @@ def adapt( ], "local_aliases": ["main"], "peers": peers, + "healthy_services": sorted(healthy_services), + "peer_image_configs": peer_image_configs, } (payload / "config.json").write_text( json.dumps(manifest, indent=2, sort_keys=True) + "\n", @@ -535,6 +758,11 @@ def adapt( tag = _tree_hash(payload) image = f"hud-harbor:{name}-{tag}" + group_service_access = bool(healthy_services) or any( + item.service != "main" + for task in group + for item in (*task.config.verifier.collect, *task.config.artifacts) + ) runtime_command = [ "/media/hud/venv/bin/hud", "serve", @@ -560,46 +788,17 @@ def adapt( ) lines = dockerfile_source.splitlines(keepends=True) - stages: list[tuple[int, re.Match[str]]] = [] - from_pattern = re.compile( - r"^(?P\s*FROM\s+(?:--platform=\S+\s+)?\S+)" - r"(?P\s+AS\s+(?P[A-Za-z0-9_.-]+))?" - r"(?P\s*(?:#.*)?)$", - re.IGNORECASE, - ) - for index, raw_line in enumerate(lines): - line = raw_line.rstrip("\r\n") - if not re.match(r"^\s*FROM\b", line, re.IGNORECASE): - continue - match = from_pattern.fullmatch(line) - if match is None: - raise ValueError( - f"{source.path.name} environment/Dockerfile has an unsupported " - "multi-line FROM instruction" - ) - stages.append((index, match)) - if not stages: - raise ValueError(f"{source.path.name} environment/Dockerfile has no FROM stage") - stage_names = { - match.group("name").lower() - for _, match in stages - if match.group("name") is not None - } - reserved_names = {"hud-base", "hud-runtime"} - if separate: - reserved_names.update({"hud-docker-cli", "hud-verifier", "hud-verifier-root"}) - reserved = reserved_names & stage_names if dockerfile.is_file() else set() - if reserved: - raise ValueError( - f"{source.path.name} environment/Dockerfile uses reserved stage " - f"{min(reserved)!r}" - ) - final_index, final = stages[-1] - base_stage = final.group("name") + stages = _dockerfile_stages(lines) + assert stages + final_index, base_stage = stages[-1] if base_stage is None: - ending = lines[final_index][len(lines[final_index].rstrip("\r\n")) :] + line = lines[final_index] + content = line.rstrip("\r\n") + ending = line[len(content) :] + suffix = re.search(r"\s*(?:#.*)?$", content) + assert suffix is not None lines[final_index] = ( - f"{final.group('from')} AS hud-base{final.group('suffix')}{ending}" + f"{content[: suffix.start()]} AS hud-base{content[suffix.start() :]}{ending}" ) base_stage = "hud-base" base_target = base_stage @@ -661,6 +860,16 @@ def adapt( if compose is not None: assert compose_project is not None + for service_name, service in compose_project.services.items(): + depends_on = (service.model_extra or {}).get("depends_on") + if service_name == "main" or not isinstance(depends_on, dict): + continue + main_dependency = depends_on.get("main") + if ( + isinstance(main_dependency, dict) + and main_dependency.get("condition") == "service_healthy" + ): + main_dependency["condition"] = "service_started" authored_main = compose_project.services["main"] main = authored_main.model_copy( update={ @@ -683,8 +892,6 @@ def adapt( source_environment = source.path / "environment" project_environment = project / "environment" shutil.copytree(source_environment, project_environment, symlinks=True) - if "hud-base" in compose_project.services or "hud-verifier" in compose_project.services: - raise ValueError("Compose service names 'hud-base' and 'hud-verifier' are reserved") base_build = authored_main.build if base_build is None and dockerfile.is_file(): base_build = {"context": "./environment"} @@ -695,8 +902,6 @@ def adapt( image=base_image, build=base_build, ).model_copy(update={"scale": 0}) - elif base_image is None: - raise ValueError("Compose main service requires an image or build") additional_contexts: dict[str, str] = {} if base_build is not None: @@ -711,7 +916,13 @@ def adapt( wrapper_build: dict[str, Any] = { "context": "./main", - "target": "verifier" if separate else "plain", + "target": ( + "verifier" + if separate + else "service-access" + if group_service_access + else "plain" + ), "args": { "BASE_IMAGE": "hud-base" if base_build is not None else base_image, "VERIFIER_IMAGE": "hud-verifier" if separate else base_image, @@ -766,6 +977,20 @@ def adapt( prepare_base = f"docker pull {shlex.quote(base_image)}" image_config_path = f'"$PROJECT/{payload.name}/image-config.json"' verifier_config_path = f'"$PROJECT/{payload.name}/verifier-image-config.json"' + peer_config_paths = { + service: f'"$PROJECT/{payload.name}/{path}"' + for service, path in peer_image_configs.items() + } + prepare_peer_lines = [] + for service, path in peer_config_paths.items(): + peer = compose_project.services[service] + assert peer.image is not None + operation = "build" if peer.build is not None else "pull" + prepare_peer_lines.append( + f"{compose_command} {operation} {shlex.quote(service)}\n" + f"inspect_peer {shlex.quote(peer.image)} {shlex.quote(service)} > {path}" + ) + prepare_peers = "\n".join(prepare_peer_lines) prepare_verifier = ( f"{compose_command} build hud-verifier" if separate @@ -780,16 +1005,26 @@ def adapt( set -eu PROJECT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) cleanup() {{ - rm -f {image_config_path} {verifier_config_path} + rm -f {image_config_path} {verifier_config_path} {" ".join(peer_config_paths.values())} }} trap cleanup EXIT HUP INT TERM inspect_image() {{ docker image inspect --format '{{{{json .Config}}}}' "$1" }} +inspect_peer() {{ + if ! docker image inspect --format \ + '{{{{range $port, $_ := .Config.ExposedPorts}}}}{{{{println $port}}}}{{{{end}}}}' "$1" \ + | grep -Eq '^[0-9]+/tcp$'; then + echo "Compose service '$2' declares no TCP ports in Compose or its image" >&2 + return 1 + fi + inspect_image "$1" +}} {prepare_base} inspect_image {shlex.quote(base_image)} > {image_config_path} {prepare_verifier} {inspect_verifier} +{prepare_peers} {compose_command} build if [ "$#" -gt 0 ]; then docker tag "$({compose_command} images -q main)" "$1" @@ -825,27 +1060,23 @@ def adapt( "verifier_timeout": config.verifier.timeout_sec or 600.0, "separate_verifier": task_separate, "collect": [hook.model_dump() for hook in config.verifier.collect], - "artifacts": [artifact.model_dump() for artifact in config.artifacts], - } - phase_environment = config.verifier.environment or EnvironmentConfig() - gpu_count = max(config.environment.gpus or 0, phase_environment.gpus or 0) - gpu_types = config.environment.gpu_types or phase_environment.gpu_types - resources = RuntimeResources( - cpu=max(config.environment.cpus or 0, phase_environment.cpus or 0) or None, - memory_mb=max( - config.environment.memory_mb or 0, - phase_environment.memory_mb or 0, - ) - or None, - gpu=( - RuntimeGPU( - count=gpu_count, - type=next(iter(filter(None, gpu_types)), None), + "artifacts": [ + artifact.model_dump( + exclude_none=True, + exclude={"exclude"} if not artifact.exclude else None, ) - if gpu_count - else None - ), + for artifact in config.artifacts + ], + } + verifier_resources = ( + _runtime_resources(config.verifier.environment) + if config.verifier.environment is not None + else None ) + verifier_limits = _runtime_limits(config.verifier.environment or config.environment) + needs_service_access = any( + item.service != "main" for item in (*config.verifier.collect, *config.artifacts) + ) or bool(healthy_services) columns = dict(config.metadata) if config.task.keywords: columns.setdefault("keywords", config.task.keywords) @@ -866,8 +1097,9 @@ def adapt( runtime_config=RuntimeConfig( compose=context / "compose-project" / "compose.json", compose_project=context, - compose_service_access=(True if task_separate else None), - resources=resources if resources.model_dump(exclude_none=True) else None, + compose_service_access=(True if needs_service_access else None), + resources=task.resources, + limits=_runtime_limits(config.environment), ), verifier=( Task( @@ -875,6 +1107,17 @@ def adapt( id="verify", args={"task": task_config}, slug=f"{task.path.name}:verify", + requires_handoff=True, + runtime_config=( + RuntimeConfig( + compose=context / "compose-project" / "compose.json", + compose_project=context, + resources=verifier_resources, + limits=verifier_limits, + ) + if verifier_resources is not None or verifier_limits is not None + else None + ), ) if task_separate else None @@ -885,4 +1128,7 @@ def adapt( Taskset(dataset.name, group_rows).to_file(context / "tasks.json") LOGGER.info("adapted %d Harbor project(s)", len({task.env for task in rows})) - return Taskset(dataset.name, rows, origin=f"harbor:{dataset}") + return AdaptResult( + taskset=Taskset(dataset.name, rows, origin=f"harbor:{dataset}"), + failures=tuple(failures), + ) diff --git a/hud/integrations/harbor/env.py b/hud/integrations/harbor/env.py index f16b5207a..e822b2b6a 100644 --- a/hud/integrations/harbor/env.py +++ b/hud/integrations/harbor/env.py @@ -4,20 +4,25 @@ import asyncio import contextlib +import fnmatch import grp import json import math import os import pwd +import shlex import shutil import socket import tempfile +import uuid from collections.abc import AsyncGenerator, Iterator # noqa: TC003 from pathlib import Path from typing import TYPE_CHECKING, Any +from pydantic import BaseModel + from hud.capabilities import Capability -from hud.environment import Environment, Mount, Peer, Workspace +from hud.environment import Answer, Environment, Mount, Peer, Workspace from hud.environment.egress import ANY_HOST, BRIDGE_PORT, VISITOR_PORT from hud.graders import EvaluationResult from hud.utils.process import ProcessResult, create_process_group_exec @@ -30,7 +35,9 @@ LOGS = Path("/logs") VERIFIER_LOGS = LOGS / "verifier" AGENT_ANSWER = LOGS / "agent_answer.txt" -ARTIFACTS = ROOT / "artifacts" +HANDOFFS = ROOT / "handoffs" +HANDOFF_ANSWER = "agent-answer.txt" +HANDOFF_ERROR = "error.txt" DOCKER_SOCKET = ROOT / "docker.sock" DOCKER = ROOT / "bin" / "docker" CONFIG = json.loads((ROOT / "config.json").read_text("utf-8")) @@ -73,6 +80,24 @@ def image_environment(config: dict[str, Any]) -> dict[str, str]: *(int(port) for value in exposed if (port := str(value).partition("/")[0]).isdigit()), } ) +for service, config_path in CONFIG["peer_image_configs"].items(): + peer_config = load_image_config(config_path) + peer_exposed = peer_config.get("ExposedPorts") or {} + if not isinstance(peer_exposed, dict): + raise ValueError( + f"OCI image ExposedPorts for Compose service {service!r} must be an object" + ) + peer_ports = sorted( + int(port) + for value in peer_exposed + if (port := str(value).partition("/")[0]).isdigit() + and str(value).partition("/")[2] in {"", "tcp"} + ) + if not peer_ports: + raise ValueError( + f"Compose service {service!r} declares no TCP ports in Compose or its image" + ) + CONFIG["peers"].extend({"name": service, "port": port} for port in peer_ports) if conflict := set(CONFIG["ports"]) & {BRIDGE_PORT, VISITOR_PORT, 8765}: raise ValueError(f"Harbor main service port {min(conflict)} conflicts with a HUD reserved port") verifier_image = CONFIG["verifier_image"] @@ -275,45 +300,75 @@ async def start_entrypoint() -> NamespaceProcess | None: async def wait_until_healthy(entrypoint: NamespaceProcess | None) -> None: healthcheck = CONFIG["environment"]["healthcheck"] - if healthcheck is None: - return - - loop = asyncio.get_running_loop() - start_period = healthcheck["start_period_sec"] - start_period_end = loop.time() + start_period - delay = healthcheck["start_interval_sec"] if start_period > 0 else healthcheck["interval_sec"] - failures = 0 - while True: - await asyncio.sleep(delay) - in_start_period = loop.time() < start_period_end - if entrypoint is not None and entrypoint.returncode is not None: - raise RuntimeError( - f"Harbor environment entrypoint exited with status {entrypoint.returncode}" - ) - result = await workspace.run( - ["sh", "-c", healthcheck["command"]], - env=CONFIG["environment"]["env"], - identity=image_identity, - inherit_workspace_env=False, - allowed_hosts=None if environment_hosts == agent_hosts else environment_hosts, - no_new_privs=False, - max_wait=healthcheck["timeout_sec"], - scope="environment", + if healthcheck is not None: + loop = asyncio.get_running_loop() + start_period = healthcheck["start_period_sec"] + start_period_end = loop.time() + start_period + delay = ( + healthcheck["start_interval_sec"] if start_period > 0 else healthcheck["interval_sec"] ) - if result.returncode == 0 and not result.timed_out: - return + failures = 0 + while True: + await asyncio.sleep(delay) + in_start_period = loop.time() < start_period_end + if entrypoint is not None and entrypoint.returncode is not None: + raise RuntimeError( + f"Harbor environment entrypoint exited with status {entrypoint.returncode}" + ) + result = await workspace.run( + ["sh", "-c", healthcheck["command"]], + env=CONFIG["environment"]["env"], + identity=image_identity, + inherit_workspace_env=False, + allowed_hosts=None if environment_hosts == agent_hosts else environment_hosts, + no_new_privs=False, + max_wait=healthcheck["timeout_sec"], + scope="environment", + ) + if result.returncode == 0 and not result.timed_out: + break + + if in_start_period: + delay = healthcheck["start_interval_sec"] + else: + failures += 1 + if failures >= healthcheck["retries"]: + detail = result.stderr.decode("utf-8", "replace").strip() + raise RuntimeError( + f"Harbor environment healthcheck failed after {failures} attempts" + + (f": {detail}" if detail else "") + ) + delay = healthcheck["interval_sec"] - if in_start_period: - delay = healthcheck["start_interval_sec"] - else: - failures += 1 - if failures >= healthcheck["retries"]: - detail = result.stderr.decode("utf-8", "replace").strip() + pending = set(CONFIG["healthy_services"]) + if not pending: + return + services = await compose_containers() + if missing := pending - services.keys(): + raise RuntimeError(f"Compose service {min(missing)!r} is not running") + while pending: + for service in sorted(pending): + result = await docker( + "inspect", + "--format", + "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}", + services[service], + ) + state, _, health = result.stdout.decode().strip().partition(" ") + if health == "healthy": + pending.remove(service) + elif health == "unhealthy": + raise RuntimeError(f"Compose service {service!r} is unhealthy") + elif state not in {"running", "restarting"}: + raise RuntimeError(f"Compose service {service!r} is {state}") + elif not health and state == "running": + raise RuntimeError(f"Compose service {service!r} has no health status") + if pending: + if entrypoint is not None and entrypoint.returncode is not None: raise RuntimeError( - f"Harbor environment healthcheck failed after {failures} attempts" - + (f": {detail}" if detail else "") + f"Harbor environment entrypoint exited with status {entrypoint.returncode}" ) - delay = healthcheck["interval_sec"] + await asyncio.sleep(1.0) async def docker(*args: str, max_wait: float = 60.0, check: bool = True) -> ProcessResult: @@ -335,7 +390,56 @@ async def docker(*args: str, max_wait: float = 60.0, check: bool = True) -> Proc return result -def copy_artifact(source: Path, target: Path) -> None: +async def compose_containers() -> dict[str, str]: + if not DOCKER_SOCKET.exists(): + raise RuntimeError("Compose service access is unavailable in this runtime") + project = await docker( + "inspect", + "--format", + '{{ index .Config.Labels "com.docker.compose.project" }}', + socket.gethostname(), + ) + project_name = project.stdout.decode().strip() + if not project_name: + raise RuntimeError("the Harbor container has no Compose project label") + listed = await docker( + "ps", + "--filter", + f"label=com.docker.compose.project={project_name}", + "--format", + '{{.ID}} {{.Label "com.docker.compose.service"}}', + ) + return { + service: container_id + for line in listed.stdout.decode().splitlines() + for container_id, service in (line.split(maxsplit=1),) + } + + +def exclude_artifact_paths(root: Path, patterns: list[str]) -> None: + if not patterns or not root.is_dir() or root.is_symlink(): + return + for entry in sorted(root.rglob("*"), key=lambda path: len(path.parts), reverse=True): + relative = entry.relative_to(root).as_posix() + parts = Path(relative).parts + candidates = ( + relative, + f"./{relative}", + *("/".join(parts[index:]) for index in range(1, len(parts))), + ) + if not any( + fnmatch.fnmatchcase(candidate, pattern) + for candidate in candidates + for pattern in patterns + ): + continue + if entry.is_dir() and not entry.is_symlink(): + shutil.rmtree(entry) + else: + entry.unlink() + + +def copy_artifact(source: Path, target: Path, exclude: list[str]) -> None: if source.is_symlink(): raise RuntimeError(f"artifact {source} is a symbolic link") if source.resolve(strict=False) != source.absolute(): @@ -348,44 +452,31 @@ def copy_artifact(source: Path, target: Path) -> None: if entry.is_symlink(): raise RuntimeError(f"artifact {source} contains symbolic link {entry}") shutil.copytree(source, target) + exclude_artifact_paths(target, exclude) elif source.exists() or source.is_symlink(): shutil.copy2(source, target, follow_symlinks=False) -async def collect(task: dict[str, Any]) -> None: - clear(ARTIFACTS) +class ActorHandoff(BaseModel): + handoff: str + + +def artifact_path(artifact: dict[str, Any], artifacts: Path) -> Path: + relative = artifact.get("destination") or artifact["source"].lstrip("/").rstrip("/") + return artifacts / relative + + +async def collect(task: dict[str, Any], artifacts: Path) -> None: + clear(artifacts) services: dict[str, str] = {} async def container(service: str) -> str: - service = "main" if service == "workspace" else service if service == "main": return "" if service in services: return services[service] - if not DOCKER_SOCKET.exists(): - raise RuntimeError( - f"collecting from Compose service {service!r} requires runtime service access" - ) if not services: - project = await docker( - "inspect", - "--format", - '{{ index .Config.Labels "com.docker.compose.project" }}', - socket.gethostname(), - ) - project_name = project.stdout.decode().strip() - if not project_name: - raise RuntimeError("the Harbor container has no Compose project label") - listed = await docker( - "ps", - "--filter", - f"label=com.docker.compose.project={project_name}", - "--format", - '{{.ID}} {{.Label "com.docker.compose.service"}}', - ) - for line in listed.stdout.decode().splitlines(): - container_id, service_name = line.split(maxsplit=1) - services[service_name] = container_id + services.update(await compose_containers()) try: return services[service] except KeyError as error: @@ -424,7 +515,8 @@ async def container(service: str) -> str: for artifact in task["artifacts"]: source = artifact["source"] - target = ARTIFACTS / source.lstrip("/").rstrip("/") + target = artifact_path(artifact, artifacts) + exclude = artifact.get("exclude", []) service = artifact["service"] container_id = await container(service) if container_id: @@ -438,8 +530,9 @@ async def container(service: str) -> str: ) if copied.returncode != 0: continue + exclude_artifact_paths(target, exclude) else: - copy_artifact(Path(source), target) + copy_artifact(Path(source), target, exclude) if target.is_symlink() or any(path.is_symlink() for path in target.rglob("*")): raise RuntimeError(f"artifact {source} contains a symbolic link") @@ -459,9 +552,29 @@ async def run(instruction: str, task: dict[str, Any]) -> AsyncGenerator[Any, Any f"Harbor environment entrypoint exited with status {entrypoint.returncode}" ) if task["separate_verifier"]: + token = uuid.uuid4().hex + handoff = HANDOFFS / token + artifacts = handoff / "artifacts" + handoff.mkdir(parents=True) + (handoff / HANDOFF_ANSWER).write_text( + "" if answer is None else str(answer), + encoding="utf-8", + ) + try: + await collect(task, artifacts) + except Exception as error: + detail = str(error) + (handoff / HANDOFF_ERROR).write_text(detail, encoding="utf-8") + result = { + "score": 0.0, + "handoff": token, + "content": detail, + "isError": True, + } + else: + result = {"score": 0.0, "handoff": token} await workspace.terminate_sessions() - await collect(task) - yield 0.0 + yield result else: yield await grade(task["id"], task["verifier_timeout"], answer) finally: @@ -474,14 +587,24 @@ async def run(instruction: str, task: dict[str, Any]) -> AsyncGenerator[Any, Any if CONFIG["verifier_root"] is not None: - @env.template(id="verify", description="Verify a Harbor task") + @env.template(id="verify", description="Verify a Harbor task", returns=ActorHandoff) async def verify(task: dict[str, Any]) -> AsyncGenerator[Any, Any]: - answer = yield "" + received = yield "" + if not isinstance(received, Answer) or not isinstance(received.content, ActorHandoff): + raise ValueError("Harbor verifier requires an actor handoff") + token = received.content.handoff + if len(token) != 32 or not token.isalnum(): + raise ValueError("Harbor verifier received an invalid actor handoff") + handoff = HANDOFFS / token + if not handoff.is_dir(): + raise ValueError("Harbor actor handoff is unavailable in this runtime") try: - yield await grade_separate(task, answer) + if (error := handoff / HANDOFF_ERROR).is_file(): + raise RuntimeError(error.read_text("utf-8")) + yield await grade_separate(task, handoff) finally: clear_grading_files() - shutil.rmtree(ARTIFACTS, ignore_errors=True) + shutil.rmtree(handoff, ignore_errors=True) def clear(path: Path) -> None: @@ -508,6 +631,17 @@ def clear_grading_files() -> None: AGENT_ANSWER.unlink() +def verifier_command(script: Path, path: str | None = None) -> list[str]: + target = path or str(script) + for line in script.read_text("utf-8").splitlines(): + stripped = line.strip() + if stripped.startswith("#!"): + return [*shlex.split(stripped[2:]), target] + if stripped and not stripped.startswith("#"): + break + return ["/bin/sh", target] + + async def grade(task_id: str, timeout_sec: float, answer: Any) -> EvaluationResult: clear(TESTS) shutil.copytree(ROOT / "tests" / task_id, TESTS, symlinks=True, dirs_exist_ok=True) @@ -531,7 +665,7 @@ async def grade(task_id: str, timeout_sec: float, answer: Any) -> EvaluationResu verifier_hosts = network(verifier)[1] execution = await workspace.run( - [str(test_script)], + verifier_command(test_script), mounts=harness_mounts, env=verifier_env, identity=verifier_identity, @@ -539,62 +673,109 @@ async def grade(task_id: str, timeout_sec: float, answer: Any) -> EvaluationResu allowed_hosts=verifier_hosts, no_new_privs=False, max_wait=timeout_sec, + writable_hosts=True, ) return evaluation(execution, timeout_sec) +def remove_path(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink(missing_ok=True) + + +def copy_path(source: Path, target: Path) -> None: + target.parent.mkdir(parents=True, exist_ok=True) + if source.is_symlink(): + target.symlink_to(os.readlink(source)) + elif source.is_dir(): + shutil.copytree(source, target, symlinks=True) + else: + shutil.copy2(source, target, follow_symlinks=False) + for path in (source, *source.rglob("*")): + relative = path.relative_to(source) if path != source else Path() + metadata = path.lstat() + os.lchown(target / relative, metadata.st_uid, metadata.st_gid) + + @contextlib.contextmanager -def artifact_mounts(task: dict[str, Any], verifier_root: Path) -> Iterator[list[Mount]]: - mounts: list[Mount] = [ +def materialized_artifacts( + task: dict[str, Any], + verifier_root: Path, + artifacts: Path, + verifier_identity: tuple[int, int] | None, +) -> Iterator[list[Mount]]: + mounts = [ Mount("dev", dst="/dev"), Mount("proc", dst="/proc"), + Mount("rw", src=str(LOGS), dst="/logs"), ] - bindings: list[tuple[Path | None, str]] = [(LOGS, "/logs")] - for artifact in task["artifacts"]: - source = artifact["source"].rstrip("/") or "/" - staged = ARTIFACTS / source.lstrip("/") - bindings.append((staged if staged.exists() or staged.is_symlink() else None, source)) - with tempfile.TemporaryDirectory(prefix="verifier-backup-", dir=ROOT) as directory: backup_root = Path(directory) replacements: list[tuple[Path, Path | None]] = [] + modes: dict[Path, int] = {} + entries: dict[Path, set[str]] = {} + created: list[Path] = [] try: - for staged, destination in bindings: - target = verifier_root / destination.lstrip("/") - compatible = ( - staged is not None - and target.exists() - and not target.is_symlink() - and staged.is_dir() == target.is_dir() - ) - if not compatible and (target.exists() or target.is_symlink()): - backup = backup_root / destination.lstrip("/") - backup.parent.mkdir(parents=True, exist_ok=True) - target.rename(backup) - replacements.append((target, backup)) - elif staged is not None and not compatible: - replacements.append((target, None)) - if staged is None: + for artifact in task["artifacts"]: + staged = artifact_path(artifact, artifacts) + if not staged.exists() and not staged.is_symlink(): continue + destination = artifact["source"].rstrip("/") or "/" + target = verifier_root / destination.lstrip("/") + if target == verifier_root: + raise ValueError("the verifier root cannot be replaced by an artifact") + + missing: list[Path] = [] + parent = target.parent + while parent != verifier_root: + if not parent.exists(): + missing.append(parent) + parent = parent.parent target.parent.mkdir(parents=True, exist_ok=True) - if staged.is_dir(): - target.mkdir(exist_ok=True) - else: - target.touch(exist_ok=True) - mounts.append(Mount("rw", src=str(staged), dst=destination)) + created.extend(reversed(missing)) + entries.setdefault(target.parent, {path.name for path in target.parent.iterdir()}) + + parent = target.parent + while parent != verifier_root: + mode = parent.stat().st_mode & 0o7777 + modes.setdefault(parent, mode) + required = 0o003 if parent == target.parent else 0o001 + parent.chmod(mode | required) + parent = parent.parent + + backup = None + if target.exists() or target.is_symlink(): + backup = backup_root / destination.lstrip("/") + copy_path(target, backup) + remove_path(target) + replacements.append((target, backup)) + copy_path(staged, target) + if verifier_identity is not None: + for path in (target, *target.rglob("*")): + os.lchown(path, *verifier_identity) yield mounts finally: for target, backup in reversed(replacements): - if target.is_dir() and not target.is_symlink(): - shutil.rmtree(target) - else: - target.unlink(missing_ok=True) + remove_path(target) if backup is not None: - target.parent.mkdir(parents=True, exist_ok=True) - backup.rename(target) - - -async def grade_separate(task: dict[str, Any], answer: Any) -> EvaluationResult: + copy_path(backup, target) + for parent, names in entries.items(): + for path in parent.iterdir(): + if path.name not in names: + remove_path(path) + for path, mode in modes.items(): + path.chmod(mode) + for path in reversed(created): + with contextlib.suppress(OSError): + path.rmdir() + + +async def grade_separate( + task: dict[str, Any], + handoff: Path, +) -> EvaluationResult: async with verifier_lock: verifier_root = Path(CONFIG["verifier_root"]) test_script = verifier_root / "tests/test.sh" @@ -604,21 +785,32 @@ async def grade_separate(task: dict[str, Any], answer: Any) -> EvaluationResult: await asyncio.to_thread(VERIFIER_LOGS.chmod, 0o777) await asyncio.to_thread(LOGS.mkdir, parents=True, exist_ok=True) await asyncio.to_thread( - AGENT_ANSWER.write_text, - "" if answer is None else str(answer), - encoding="utf-8", + AGENT_ANSWER.write_bytes, + (handoff / HANDOFF_ANSWER).read_bytes(), ) try: - with artifact_mounts(task, verifier_root) as mounts: - verifier = CONFIG["verifier"] - verifier_network, verifier_hosts = network(verifier) - image = CONFIG["verifier_image"] - verifier_identity = identity( - verifier, - image_user=image["user"], - root=verifier_root, - ) + verifier = CONFIG["verifier"] + verifier_network, verifier_hosts = network(verifier) + verifier_mode = verifier["network_mode"] or CONFIG["environment"]["network_mode"] + verifier_access = None if verifier_mode == "public" else verifier_hosts + image = CONFIG["verifier_image"] + verifier_identity = identity( + verifier, + image_user=image["user"], + root=verifier_root, + ) + with materialized_artifacts( + task, + verifier_root, + handoff / "artifacts", + verifier_identity, + ) as mounts: + verifier_mounts = tuple(mounts) + if verifier_mode == "public": + verifier_mounts += ( + Mount("ro", src="/etc/resolv.conf", dst="/etc/resolv.conf"), + ) verifier_uid = verifier_identity[0] if verifier_identity is not None else None verifier_env = { **CONFIG["environment"]["env"], @@ -631,10 +823,10 @@ async def grade_separate(task: dict[str, Any], answer: Any) -> EvaluationResult: verifier_root, guest_path="/", system_mounts=(), - mounts=mounts, + mounts=verifier_mounts, env=verifier_env, network=verifier_network, - allowed_hosts=verifier_hosts, + allowed_hosts=verifier_access, credentials_dir=ROOT / "verifier-keys", hand_over_root=False, require_isolation=True, @@ -642,14 +834,15 @@ async def grade_separate(task: dict[str, Any], answer: Any) -> EvaluationResult: try: await isolated.start() execution = await isolated.run( - ["/tests/test.sh"], + verifier_command(test_script, "/tests/test.sh"), env=verifier_env, cwd=image["workdir"], identity=verifier_identity, inherit_workspace_env=False, - allowed_hosts=verifier_hosts, + allowed_hosts=verifier_access, no_new_privs=False, max_wait=task["verifier_timeout"], + writable_hosts=True, ) finally: await isolated.stop() diff --git a/hud/integrations/harbor/install.sh b/hud/integrations/harbor/install.sh index 5b25ea6cf..08c6a3599 100644 --- a/hud/integrations/harbor/install.sh +++ b/hud/integrations/harbor/install.sh @@ -17,8 +17,11 @@ if command -v apt-get >/dev/null 2>&1; then rm -rf /var/lib/apt/lists/* elif command -v apk >/dev/null 2>&1; then apk add --no-cache bash bubblewrap util-linux python3 py3-pip git curl ca-certificates +elif command -v dnf >/dev/null 2>&1; then + dnf install -y bubblewrap util-linux python3 python3-pip git curl ca-certificates + dnf clean all else - echo "hud: Harbor environments require an apt- or apk-based image" >&2 + echo "hud: Harbor environments require an apt-, apk-, or dnf-based image" >&2 exit 1 fi diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/environment/compose.yaml b/hud/integrations/harbor/tests/tasks/hello-mcp/environment/docker-compose.yaml similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/environment/compose.yaml rename to hud/integrations/harbor/tests/tasks/hello-mcp/environment/docker-compose.yaml diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/compose.yaml b/hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/compose.yaml deleted file mode 100644 index 98eaa6924..000000000 --- a/hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/compose.yaml +++ /dev/null @@ -1,11 +0,0 @@ -services: - main: - healthcheck: - test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] - interval: 100ms - timeout: 2s - retries: 50 - web: - image: python:3.11-alpine - command: ["python", "-m", "http.server", "5678"] - expose: ["5678"] diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/docker-compose.yaml b/hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/docker-compose.yaml new file mode 100644 index 000000000..3829482e6 --- /dev/null +++ b/hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + main: + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] + interval: 100ms + timeout: 2s + retries: 50 + workspace: + image: ${SIDECAR_IMAGE:-python:3.11-alpine} + command: ["sh", "-c", "python -m http.server 5678 & exec python -m http.server 5679"] + expose: ["5678", "5679"] diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh b/hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh index f127af2ca..b4062bee9 100644 --- a/hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh +++ b/hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh @@ -2,7 +2,15 @@ set -eu [ -c /dev/null ] printf discarded > /dev/null -curl -fsS --max-time 10 http://web:5678/ > /app/sidecar.html +curl -fsS --max-time 10 http://workspace:5678/ > /app/sidecar.html +curl -fsS --max-time 10 http://workspace:5679/ >/dev/null +mkdir -p /app/results +printf keep > /app/results/keep.txt +printf drop > /app/results/drop.tmp +printf original > /app/binary +mkdir -p /opt/result +printf replacement > /opt/result/new.txt +printf private > /root/agent-output.txt case ",${NO_PROXY:-}," in *,main,*) ;; *) echo "main is absent from NO_PROXY" >&2; exit 1 ;; @@ -10,17 +18,20 @@ esac curl -fsS --max-time 10 http://main:8080/ > /app/main.html protected=/media/hud/session-"keys" [ ! -e "$protected" ] -processes=$(ps -ef) -if ! printf '%s\n' "$processes" | grep -F "python3 -m http.server 8080 --directory /app" >/dev/null; then +entrypoint_visible=false +for pid in $(pgrep -x python3); do + if tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null \ + | grep -F "python3 -m http.server 8080 --directory /app" >/dev/null; then + entrypoint_visible=true + break + fi +done +if [ "$entrypoint_visible" != true ]; then echo "the main entrypoint process is absent from the agent process namespace" >&2 exit 1 fi +processes=$(ps -ef) if printf '%s\n' "$processes" | grep -F "$protected"; then echo "protected bridge path is visible in the process list" >&2 exit 1 fi -( - while :; do - [ ! -e /tmp/main.txt ] || echo agent-race > /tmp/main.txt - done -) >/dev/null 2>&1 & diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml b/hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml index 64de0fa9c..785823638 100644 --- a/hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml +++ b/hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml @@ -1,8 +1,12 @@ artifacts = [ { source = "/app/sidecar.html", service = "main" }, { source = "/app/main.html", service = "main" }, + { source = "/app/results", destination = "results", exclude = ["*.tmp"] }, + { source = "/app/binary", service = "main" }, + { source = "/opt/result", service = "main" }, + { source = "/root/agent-output.txt", service = "main" }, { source = "/tmp/main.txt", service = "main" }, - { source = "/tmp/sidecar.txt", service = "web" }, + { source = "/tmp/sidecar.txt", service = "workspace" }, ] [task] @@ -17,10 +21,10 @@ timeout_sec = 30 [[verifier.collect]] service = "main" -command = "head -c 1 /dev/urandom >/dev/null && echo collected-from-main > /tmp/main.txt && sleep 0.2" +command = "curl -fsS --max-time 10 http://127.0.0.1:8080/ >/dev/null && head -c 1 /dev/urandom >/dev/null && echo collected-from-main > /tmp/main.txt && sleep 0.2" timeout_sec = 10 [[verifier.collect]] -service = "web" +service = "workspace" command = "echo collected-from-sidecar > /tmp/sidecar.txt" timeout_sec = 10 diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/Dockerfile b/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/Dockerfile index ae216f759..c80f07c43 100644 --- a/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/Dockerfile +++ b/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/Dockerfile @@ -2,9 +2,11 @@ FROM python:3.12-alpine ENV VERIFIER_PRECEDENCE=verifier-image COPY . /tests -RUN addgroup -g 1001 verifier \ +RUN apk add --no-cache bash \ + && addgroup -g 1001 verifier \ && adduser -D -u 1001 -G verifier verifier \ - && mkdir -p /app /logs/verifier \ + && mkdir -p /app /logs/verifier /opt/result \ + && echo base > /opt/result/base.txt \ && touch /home/verifier/owned \ && chown verifier:verifier /home/verifier/owned diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/test.sh b/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/test.sh index 8511a7fcf..7dc9d82ec 100644 --- a/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/test.sh +++ b/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/test.sh @@ -1,12 +1,25 @@ -#!/bin/sh -set -u +# generated task metadata +#!/bin/bash +set -uo pipefail mkdir -p /logs/verifier +mv /app/binary /app/binary.original +printf regenerated > /app/binary +mv /opt/result /opt/result.moved +env -i PATH=/usr/bin:/bin getent hosts pypi.org >/dev/null \ + || { echo "the public verifier could not resolve an external host"; exit 1; } if grep -q "Directory listing" /app/sidecar.html 2>/dev/null \ && grep -q "Directory listing" /app/main.html 2>/dev/null \ && [ -c /dev/null ] \ && [ "$(cat /tmp/main.txt 2>/dev/null)" = "collected-from-main" ] \ && [ "$(cat /tmp/sidecar.txt 2>/dev/null)" = "collected-from-sidecar" ] \ + && [ "$(cat /app/results/keep.txt 2>/dev/null)" = "keep" ] \ + && [ ! -e /app/results/drop.tmp ] \ + && [ "$(cat /app/binary.original 2>/dev/null)" = "original" ] \ + && [ "$(cat /app/binary 2>/dev/null)" = "regenerated" ] \ + && [ "$(cat /opt/result.moved/new.txt 2>/dev/null)" = "replacement" ] \ + && [ ! -e /opt/result.moved/base.txt ] \ + && [ "$(cat /root/agent-output.txt 2>/dev/null)" = "private" ] \ && [ "$(id -u)" = "1001" ] \ && [ "$(stat -c %u /home/verifier/owned)" = "1001" ] \ && [ "$PWD" = "/home/verifier" ] \ diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/tests/test.sh b/hud/integrations/harbor/tests/tasks/verifier-lifecycle/tests/test.sh index 2233c39a5..770f628ab 100644 --- a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/tests/test.sh +++ b/hud/integrations/harbor/tests/tasks/verifier-lifecycle/tests/test.sh @@ -3,6 +3,10 @@ set -u mkdir -p /logs/verifier fail() { echo "$1"; echo 0 > /logs/verifier/reward.txt; exit 0; } +printf '127.0.0.1 verifier-added\n' >> /etc/hosts \ + || fail "the verifier could not update its hosts file" +grep -q 'verifier-added' /etc/hosts \ + || fail "the verifier hosts update was not visible" chown -R 1000:2000 /app/data || fail "the verifier could not chown the graded tree" tar -cf /tmp/data.tar -C /app data || fail "the verifier could not archive the graded tree" [ "$(cat /app/data/payload.txt 2>/dev/null)" = "hello" ] \ diff --git a/hud/integrations/harbor/tests/test_contract.py b/hud/integrations/harbor/tests/test_contract.py index 84fb6a14f..634ca13e2 100644 --- a/hud/integrations/harbor/tests/test_contract.py +++ b/hud/integrations/harbor/tests/test_contract.py @@ -10,12 +10,25 @@ import pytest -from hud.eval import Task +from hud.eval import RuntimeGPU, RuntimeLimits, RuntimeResources, RuntimeTPU, Taskset from hud.integrations import harbor from .conftest import make_harbor_task, make_multi_step_task +def _adapt(path: Path, *, hud_requirement: str = "hud") -> Taskset: + result = harbor.adapt(path, hud_requirement=hud_requirement) + assert result.failures == () + return result.taskset + + +def _failure(path: Path) -> harbor.AdaptFailure: + result = harbor.adapt(path) + assert list(result.taskset) == [] + assert len(result.failures) == 1 + return result.failures[0] + + def _tree_snapshot(root: Path) -> dict[str, tuple[str, bytes | str]]: snapshot: dict[str, tuple[str, bytes | str]] = {} for entry in sorted(root.rglob("*")): @@ -85,7 +98,7 @@ def test_adapt_packages_an_image_task_as_a_compose_project(tmp_path: Path) -> No task_dir = make_harbor_task(tmp_path, "task-a") authored_environment = _tree_snapshot(task_dir / "environment") - taskset = harbor.adapt(tmp_path) + taskset = _adapt(tmp_path) (task,) = list(taskset) assert task.id == "run" @@ -158,7 +171,7 @@ def test_adapt_packages_an_image_task_as_a_compose_project(tmp_path: Path) -> No def test_task_content_changes_do_not_rebuild_the_environment(tmp_path: Path) -> None: task_dir = make_harbor_task(tmp_path, "task-a", instruction="First instruction") - (before,) = list(harbor.adapt(tmp_path)) + (before,) = list(_adapt(tmp_path)) assert before.runtime_config is not None assert isinstance(before.runtime_config.compose, Path) before_compose = json.loads(before.runtime_config.compose.read_text("utf-8")) @@ -166,7 +179,7 @@ def test_task_content_changes_do_not_rebuild_the_environment(tmp_path: Path) -> (task_dir / "instruction.md").write_text("Second instruction", encoding="utf-8") (task_dir / "tests" / "test.sh").write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") - (after,) = list(harbor.adapt(tmp_path)) + (after,) = list(_adapt(tmp_path)) assert after.runtime_config is not None assert isinstance(after.runtime_config.compose, Path) after_compose = json.loads(after.runtime_config.compose.read_text("utf-8")) @@ -178,11 +191,25 @@ def test_task_content_changes_do_not_rebuild_the_environment(tmp_path: Path) -> ) == "#!/bin/sh\nexit 1\n" +def test_image_task_keeps_non_recipe_compose_names_as_context_files(tmp_path: Path) -> None: + task = make_harbor_task(tmp_path, "task-a") + content = '{"api_gateway": {"interval": "30s"}}\n' + (task / "environment" / "docker-compose.yml").write_text(content, encoding="utf-8") + + _adapt(tmp_path) + + (context,) = (tmp_path / ".hud-adapt").iterdir() + environment = context / "compose-project" / "environment" + assert (environment / "docker-compose.yml").read_text("utf-8") == content + project = json.loads((context / "compose-project" / "compose.json").read_text("utf-8")) + assert set(project["services"]) == {"main"} + + def test_image_task_preserves_a_named_final_stage_verbatim(tmp_path: Path) -> None: dockerfile = 'FROM alpine AS build\r\nRUN true\r\nFROM alpine AS final\r\nCMD ["sh"]\r\n' make_harbor_task(tmp_path, "task-a", dockerfile=dockerfile) - harbor.adapt(tmp_path) + _adapt(tmp_path) (context,) = (tmp_path / ".hud-adapt").iterdir() environment = context / "compose-project" / "environment" @@ -191,6 +218,31 @@ def test_image_task_preserves_a_named_final_stage_verbatim(tmp_path: Path) -> No assert combined.startswith(dockerfile + "\nFROM final AS hud-runtime\n") +def test_image_task_names_an_unnamed_multiline_final_stage(tmp_path: Path) -> None: + dockerfile = "FROM --platform=linux/amd64 \\\n python:3.12-slim\nRUN true\n" + make_harbor_task(tmp_path, "task-a", dockerfile=dockerfile) + + _adapt(tmp_path) + + (context,) = (tmp_path / ".hud-adapt").iterdir() + combined = (context / "compose-project" / "Dockerfile").read_text("utf-8") + assert combined.startswith( + "FROM --platform=linux/amd64 \\\n python:3.12-slim AS hud-base\n" + "RUN true\n\nFROM hud-base AS hud-runtime\n" + ) + + +@pytest.mark.parametrize("delimiter", ["<<'PY'", "<< 'PY'"]) +def test_image_task_ignores_from_inside_dockerfile_heredoc(tmp_path: Path, delimiter: str) -> None: + make_harbor_task( + tmp_path, + "task-a", + dockerfile=f"FROM python:3.12\nRUN python - {delimiter}\nfrom pathlib import Path\nPY\n", + ) + + _adapt(tmp_path) + + @pytest.mark.parametrize("stage", ["hud-base", "HUD-RUNTIME"]) def test_image_task_rejects_reserved_user_stage_names( tmp_path: Path, @@ -198,8 +250,11 @@ def test_image_task_rejects_reserved_user_stage_names( ) -> None: make_harbor_task(tmp_path, "task-a", dockerfile=f"FROM alpine AS {stage}\n") - with pytest.raises(ValueError, match="reserved stage"): - harbor.adapt(tmp_path) + failure = _failure(tmp_path) + + assert [finding.code for finding in failure.findings] == [ + "harbor.invalid.reserved_dockerfile_stage" + ] def test_image_task_preserves_environment_ignored_paths_verbatim( @@ -211,7 +266,7 @@ def test_image_task_preserves_environment_ignored_paths_verbatim( (environment / "ignored.txt").write_bytes(b"unchanged\x00payload") authored = _tree_snapshot(environment) - harbor.adapt(tmp_path) + _adapt(tmp_path) (context,) = (tmp_path / ".hud-adapt").iterdir() project = context / "compose-project" @@ -226,7 +281,7 @@ def test_adapt_honors_compose_main_build_settings( task = make_harbor_task(tmp_path, "task-a", dockerfile=None) environment = task / "environment" environment.mkdir() - (environment / "compose.yaml").write_text( + (environment / "docker-compose.yaml").write_text( """\ services: main: @@ -240,7 +295,7 @@ def test_adapt_honors_compose_main_build_settings( ) (environment / "Containerfile").write_text("FROM python:3.12\n", encoding="utf-8") - (row,) = list(harbor.adapt(tmp_path)) + (row,) = list(_adapt(tmp_path)) assert row.runtime_config is not None compose_path = row.runtime_config.compose @@ -256,7 +311,7 @@ def test_adapt_emits_compose_project_and_peers( tmp_path: Path, ) -> None: task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "compose.yaml").write_text( + (task / "environment" / "docker-compose.yaml").write_text( """\ services: main: @@ -270,6 +325,10 @@ def test_adapt_emits_compose_project_and_peers( start_period: 1s redis: image: redis:7-alpine + depends_on: + main: + condition: service_healthy + restart: true command: [redis-server, --save, ""] environment: {SIDE: car} expose: [6379] @@ -283,10 +342,11 @@ def test_adapt_emits_compose_project_and_peers( encoding="utf-8", ) - (row,) = list(harbor.adapt(tmp_path)) + (row,) = list(_adapt(tmp_path)) assert row.runtime_config is not None assert row.runtime_config.image is None + assert row.runtime_config.compose_service_access is True (context,) = (tmp_path / ".hud-adapt").iterdir() assert row.runtime_config.compose == context / "compose-project" / "compose.json" assert row.runtime_config.compose_project == context @@ -297,6 +357,7 @@ def test_adapt_emits_compose_project_and_peers( assert project["services"]["redis"]["image"] == "redis:7-alpine" assert "build" not in project["services"]["redis"] assert project["services"]["main"]["build"]["context"] == "./main" + assert project["services"]["main"]["build"]["target"] == "service-access" assert project["services"]["main"]["build"]["additional_contexts"] == { "hud-base": "service:hud-base" } @@ -312,6 +373,7 @@ def test_adapt_emits_compose_project_and_peers( assert redis["image"] == "redis:7-alpine" assert redis["environment"] == {"SIDE": "car"} assert redis["command"] == ["redis-server", "--save", ""] + assert redis["depends_on"] == {"main": {"condition": "service_started", "restart": True}} assert redis["expose"] == ["6379"] assert redis["healthcheck"]["test"] == ["CMD", "redis-cli", "ping"] assert "build" not in redis @@ -329,6 +391,7 @@ def test_adapt_emits_compose_project_and_peers( assert manifest["ports"] == [8080] assert manifest["capabilities"] == [] assert manifest["peers"] == [{"name": "redis", "port": 6379}] + assert manifest["healthy_services"] == ["redis"] assert project["services"]["main"]["command"] == [ "/media/hud/venv/bin/hud", "serve", @@ -355,7 +418,7 @@ def test_compose_adapt_retains_builds_without_local_docker( (database / "Dockerfile").write_text("FROM postgres:16\n", encoding="utf-8") (database / "db.env").write_text("POSTGRES_DB=test\n", encoding="utf-8") (database / "data").mkdir() - (task / "environment" / "compose.yaml").write_text( + (task / "environment" / "docker-compose.yaml").write_text( """\ services: main: @@ -376,7 +439,7 @@ def test_compose_adapt_retains_builds_without_local_docker( (task / "environment" / "main.env").write_text("MAIN=true\n", encoding="utf-8") (task / "environment" / "main-data").mkdir() - (row,) = list(harbor.adapt(tmp_path)) + (row,) = list(_adapt(tmp_path)) assert row.runtime_config is not None compose_path = row.runtime_config.compose @@ -400,7 +463,7 @@ def test_adapt_moves_compose_main_process_settings_into_the_workspace( tmp_path: Path, ) -> None: task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "compose.yaml").write_text( + (task / "environment" / "docker-compose.yaml").write_text( """\ services: main: @@ -411,7 +474,7 @@ def test_adapt_moves_compose_main_process_settings_into_the_workspace( encoding="utf-8", ) - harbor.adapt(tmp_path) + _adapt(tmp_path) (context,) = (tmp_path / ".hud-adapt").iterdir() manifest = _environment_config(context) @@ -427,7 +490,7 @@ def test_adapt_moves_compose_main_healthcheck_into_the_workspace( tmp_path: Path, ) -> None: task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "compose.yaml").write_text( + (task / "environment" / "docker-compose.yaml").write_text( """\ services: main: @@ -441,7 +504,7 @@ def test_adapt_moves_compose_main_healthcheck_into_the_workspace( encoding="utf-8", ) - harbor.adapt(tmp_path) + _adapt(tmp_path) (context,) = (tmp_path / ".hud-adapt").iterdir() manifest = _environment_config(context) @@ -459,12 +522,12 @@ def test_adapt_moves_compose_main_healthcheck_into_the_workspace( def test_adapt_uses_compose_healthcheck_defaults(tmp_path: Path) -> None: task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "compose.yaml").write_text( + (task / "environment" / "docker-compose.yaml").write_text( "services:\n main:\n healthcheck:\n test: [CMD, 'true']\n", encoding="utf-8", ) - harbor.adapt(tmp_path) + _adapt(tmp_path) (context,) = (tmp_path / ".hud-adapt").iterdir() manifest = _environment_config(context) @@ -478,37 +541,26 @@ def test_adapt_uses_compose_healthcheck_defaults(tmp_path: Path) -> None: } -def test_adapt_requires_peer_port_in_the_compose_project( - tmp_path: Path, -) -> None: +def test_adapt_merges_implicit_main_into_authored_compose(tmp_path: Path) -> None: task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "compose.yaml").write_text( - "services:\n main: {}\n redis:\n image: redis:7-alpine\n", + (task / "environment" / "docker-compose.yaml").write_text( + "services:\n default:\n image: sidecar:latest\n", encoding="utf-8", ) - with pytest.raises(ValueError, match="declares no TCP port"): - harbor.adapt(tmp_path) + _adapt(tmp_path) - -def test_adapt_rejects_sidecar_without_a_tcp_port( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "compose.yaml").write_text( - "services:\n main: {}\n worker:\n image: no-ports:latest\n", - encoding="utf-8", - ) - - with pytest.raises(ValueError, match="declares no TCP port"): - harbor.adapt(tmp_path) + (context,) = (tmp_path / ".hud-adapt").iterdir() + compose = json.loads((context / "compose-project" / "compose.json").read_text("utf-8")) + assert {"main", "default"} <= compose["services"].keys() + assert _environment_config(context)["peers"] == [] def test_network_mcp_servers_become_named_capabilities( tmp_path: Path, ) -> None: task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "compose.yaml").write_text( + (task / "environment" / "docker-compose.yaml").write_text( "services:\n main: {}\n redis:\n image: redis:7-alpine\n expose: [6379]\n", encoding="utf-8", ) @@ -523,7 +575,7 @@ def test_network_mcp_servers_become_named_capabilities( encoding="utf-8", ) - harbor.adapt(tmp_path) + _adapt(tmp_path) (context,) = (tmp_path / ".hud-adapt").iterdir() manifest = _environment_config(context) @@ -553,8 +605,10 @@ def test_mcp_server_names_cannot_shadow_workspace_capabilities( encoding="utf-8", ) - with pytest.raises(ValueError, match=f"MCP server name {name!r} is reserved"): - harbor.adapt(tmp_path) + failure = _failure(tmp_path) + + assert [finding.code for finding in failure.findings] == ["harbor.invalid.reserved_mcp_name"] + assert name in failure.findings[0].message def test_adapt_groups_identical_images_and_keeps_row_metadata( @@ -577,7 +631,7 @@ def test_adapt_groups_identical_images_and_keeps_row_metadata( """, encoding="utf-8", ) - taskset = harbor.adapt(dataset_same_env) + taskset = _adapt(dataset_same_env) assert len(taskset) == 3 assert len(taskset.environment_names()) == 1 @@ -602,7 +656,7 @@ def test_adapt_groups_identical_images_and_keeps_row_metadata( def test_distinct_environments_build_distinct_images( dataset_multi_env: Path, ) -> None: - taskset = harbor.adapt(dataset_multi_env) + taskset = _adapt(dataset_multi_env) assert len(taskset.environment_names()) == 2 assert all(task.runtime_config is not None for task in taskset) @@ -619,13 +673,14 @@ def test_adapt_maps_resources_onto_the_compose_runtime(tmp_path: Path) -> None: [environment] cpus = 4 memory_mb = 8192 +storage_mb = 32768 gpus = 2 gpu_types = ["H100"] """, encoding="utf-8", ) - (row,) = list(harbor.adapt(tmp_path)) + (row,) = list(_adapt(tmp_path)) assert row.columns == {"difficulty": "hard"} assert row.runtime_config is not None @@ -634,6 +689,7 @@ def test_adapt_maps_resources_onto_the_compose_runtime(tmp_path: Path) -> None: assert row.runtime_config.resources is not None assert row.runtime_config.resources.cpu == 4 assert row.runtime_config.resources.memory_mb == 8192 + assert row.runtime_config.resources.storage_mb == 32768 assert row.runtime_config.resources.gpu is not None assert row.runtime_config.resources.gpu.count == 2 assert row.runtime_config.resources.gpu.type == "H100" @@ -648,7 +704,7 @@ def test_prebuilt_harbor_image_is_inspected_by_the_project_build( encoding="utf-8", ) - harbor.adapt(tmp_path) + _adapt(tmp_path) (context,) = (tmp_path / ".hud-adapt").iterdir() project = context / "compose-project" @@ -668,7 +724,7 @@ def test_zero_gpus_is_a_valid_harbor_resource_declaration( task = make_harbor_task(tmp_path, "cpu-only") (task / "task.toml").write_text("[environment]\ngpus = 0\n", encoding="utf-8") - (row,) = list(harbor.adapt(tmp_path)) + (row,) = list(_adapt(tmp_path)) assert row.runtime_config is not None assert row.runtime_config.resources is None @@ -712,7 +768,7 @@ def test_runtime_configuration_is_data_not_dockerfile_codegen( encoding="utf-8", ) - harbor.adapt(tmp_path) + _adapt(tmp_path) (context,) = (tmp_path / ".hud-adapt").iterdir() manifest = _environment_config(context) @@ -756,7 +812,7 @@ def test_image_entrypoint_is_preserved_as_runtime_data( encoding="utf-8", ) - harbor.adapt(tmp_path) + _adapt(tmp_path) (context,) = (tmp_path / ".hud-adapt").iterdir() manifest = _environment_config(context) @@ -766,32 +822,59 @@ def test_image_entrypoint_is_preserved_as_runtime_data( assert "docker image inspect" in script -@pytest.mark.parametrize( - ("declaration", "expected"), - [ - ('[environment]\nos = "windows"\n', "os="), - ('[environment]\ntpu = {type = "v5", topology = "2x2"}\n', "TPUs"), - ( - '[environment]\ngpus = 1\ngpu_types = ["H100", "A100"]\n', - "multiple GPU types", - ), - ('[environment]\ngpu_types = ["H100"]\n', "GPU types without GPUs"), - ( - '[[environment.mcp_servers]]\nname = "db"\ntransport = "stdio"\ncommand = "db-mcp"\n', - "stdio MCP servers", - ), - ], -) -def test_unsupported_harbor_behaviour_fails_before_building( +def test_dataset_adaptation_returns_successes_and_all_detectable_findings( tmp_path: Path, - declaration: str, - expected: str, ) -> None: + make_harbor_task(tmp_path, "supported") + unsupported = make_harbor_task(tmp_path, "unsupported") + (unsupported / "instruction.md").unlink() + (unsupported / "task.toml").write_text( + """\ +[environment] +os = "windows" +skills_dir = "skills" + +[[environment.mcp_servers]] +name = "shell" +transport = "streamable-http" + +[[environment.mcp_servers]] +name = "db" +transport = "stdio" +command = "db-mcp" +""", + encoding="utf-8", + ) + + result = harbor.adapt(tmp_path) + + assert [task.slug for task in result.taskset] == ["supported"] + assert len(result.failures) == 1 + failure = result.failures[0] + assert failure.task == "unsupported" + assert {finding.code for finding in failure.findings} == { + "harbor.unsupported.skills_dir", + "harbor.unsupported.mcp_stdio", + "harbor.invalid.reserved_mcp_name", + "harbor.invalid.mcp_url", + "harbor.invalid.missing_instruction", + } + assert {finding.kind for finding in failure.findings} == {"contract", "invalid"} + + +def test_unbound_compose_variables_are_deliberate_contract_refusals(tmp_path: Path) -> None: task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text(declaration, encoding="utf-8") + (task / "environment" / "docker-compose.yaml").write_text( + "services:\n main:\n image: ${MAIN_IMAGE}\n", + encoding="utf-8", + ) - with pytest.raises(NotImplementedError, match=expected): - harbor.adapt(tmp_path) + failure = _failure(tmp_path) + + assert [finding.code for finding in failure.findings] == [ + "harbor.unsupported.host_compose_variable" + ] + assert failure.findings[0].kind == "contract" @pytest.mark.parametrize("port", [3128, 3129, 8765]) @@ -800,20 +883,36 @@ def test_adapt_rejects_main_ports_reserved_by_hud( port: int, ) -> None: task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "compose.yaml").write_text( + (task / "environment" / "docker-compose.yaml").write_text( f"services:\n main:\n expose: [{port}]\n", encoding="utf-8", ) - with pytest.raises(ValueError, match=f"port {port} conflicts with a HUD reserved port"): - harbor.adapt(tmp_path) + failure = _failure(tmp_path) + + assert [finding.code for finding in failure.findings] == ["harbor.invalid.reserved_main_port"] + assert str(port) in failure.findings[0].message + + +def test_adapt_accepts_explicit_shared_verifier_mode(tmp_path: Path) -> None: + task = make_harbor_task(tmp_path, "shared") + (task / "task.toml").write_text( + '[verifier]\nenvironment_mode = "shared"\n', + encoding="utf-8", + ) + + (row,) = list(_adapt(tmp_path)) + + assert row.verifier is None + assert row.runtime_config is not None + assert row.runtime_config.compose_service_access is None -def test_adapt_builds_a_separate_verifier_and_reuses_the_runtime( +def test_adapt_builds_a_separate_verifier_with_its_own_placement( tmp_path: Path, ) -> None: task = make_harbor_task(tmp_path, "separate") - (task / "environment" / "compose.yaml").write_text( + (task / "environment" / "docker-compose.yaml").write_text( "services:\n main: {}\n redis:\n image: redis:7-alpine\n expose: [6379]\n", encoding="utf-8", ) @@ -828,6 +927,11 @@ def test_adapt_builds_a_separate_verifier_and_reuses_the_runtime( [environment] cpus = 2 memory_mb = 2048 +build_timeout_sec = 600.5 +gpus = 1 +gpu_types = ["H100", "A100"] +os = "windows" +tpu = {type = "v5", topology = "2x2"} [verifier] environment_mode = "separate" @@ -836,6 +940,9 @@ def test_adapt_builds_a_separate_verifier_and_reuses_the_runtime( [verifier.environment] cpus = 4 memory_mb = 1024 +build_timeout_sec = 1200 +gpus = 1 +gpu_types = ["T4"] workdir = "/judge" network_mode = "allowlist" allowed_hosts = ["verifier.example"] @@ -855,21 +962,30 @@ def test_adapt_builds_a_separate_verifier_and_reuses_the_runtime( encoding="utf-8", ) - (row,) = list(harbor.adapt(tmp_path)) + (row,) = list(_adapt(tmp_path)) assert row.id == "run" assert row.slug == "separate" - assert row.verifier == Task( - env=row.env, - id="verify", - args={"task": row.args["task"]}, - slug="separate:verify", - ) assert row.runtime_config is not None - assert row.runtime_config.resources is not None - assert row.runtime_config.resources.cpu == 4 - assert row.runtime_config.resources.memory_mb == 2048 + assert row.runtime_config.resources == RuntimeResources( + cpu=2, + memory_mb=2048, + gpu=RuntimeGPU(type=["H100", "A100"]), + os="windows", + tpu=RuntimeTPU(type="v5", topology="2x2"), + ) + assert row.runtime_config.limits == RuntimeLimits(startup_timeout_s=601) assert row.runtime_config.compose_service_access is True + assert row.verifier is not None + assert row.verifier.requires_handoff is True + assert row.verifier.runtime_config is not None + assert row.verifier.runtime_config.compose == row.runtime_config.compose + assert row.verifier.runtime_config.resources == RuntimeResources( + cpu=4, + memory_mb=1024, + gpu=RuntimeGPU(type="T4"), + ) + assert row.verifier.runtime_config.limits == RuntimeLimits(startup_timeout_s=1200) (context,) = (tmp_path / ".hud-adapt").iterdir() manifest = _environment_config(context) @@ -915,10 +1031,10 @@ def test_image_task_keeps_only_the_verifier_as_a_build_service( encoding="utf-8", ) - (row,) = list(harbor.adapt(tmp_path)) + (row,) = list(_adapt(tmp_path)) assert row.runtime_config is not None - assert row.runtime_config.compose_service_access is True + assert row.runtime_config.compose_service_access is None assert isinstance(row.runtime_config.compose, Path) project = _assert_stock_compose_complete(row.runtime_config.compose) assert set(project["services"]) == {"main", "hud-verifier"} @@ -945,10 +1061,10 @@ def test_separate_verifier_groups_have_distinct_environment_names( for name in ("task-a", "task-b"): task = make_harbor_task(tmp_path, name) (task / "task.toml").write_text(declaration, encoding="utf-8") - (task / "environment" / "compose.yaml").write_text(compose, encoding="utf-8") + (task / "environment" / "docker-compose.yaml").write_text(compose, encoding="utf-8") (task / "tests" / "Dockerfile").write_text(verifier, encoding="utf-8") - rows = list(harbor.adapt(tmp_path)) + rows = list(_adapt(tmp_path)) assert len({row.env for row in rows}) == 2 compose_paths = { @@ -969,30 +1085,33 @@ def test_separate_verifier_groups_have_distinct_environment_names( def test_multi_step_tasks_are_refused_directly(tmp_path: Path) -> None: make_multi_step_task(tmp_path, "multi") - with pytest.raises(NotImplementedError, match="multi-step"): - harbor.adapt(tmp_path) - - -def test_invalid_task_config_is_not_silently_defaulted( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text("[environment]\ncpus = 'many'\n", encoding="utf-8") + failure = _failure(tmp_path) - with pytest.raises(ValueError, match="not a valid Harbor task"): - harbor.adapt(tmp_path) + assert [finding.code for finding in failure.findings] == ["harbor.unsupported.multi_step"] -@pytest.mark.parametrize("source", ["/", "//", "/workspace/../secret"]) -def test_artifacts_must_name_normalized_paths_beneath_root( - tmp_path: Path, - source: str, -) -> None: +@pytest.mark.parametrize( + "artifact", + [ + '"/"', + '"//"', + '"/workspace/../secret"', + '{ source = "/output", destination = "/tmp/out" }', + '{ source = "/output", destination = "../out" }', + '{ source = "/output", destination = "a\\\\b" }', + '{ source = "/output", destination = "manifest.json" }', + ], +) +def test_artifact_paths_stay_beneath_their_roots(tmp_path: Path, artifact: str) -> None: task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text(f'artifacts = ["{source}"]\n', encoding="utf-8") + (task / "task.toml").write_text( + f"artifacts = [{artifact}]\n", + encoding="utf-8", + ) + + failure = _failure(tmp_path) - with pytest.raises(ValueError, match="artifact source must name a path beneath /"): - harbor.adapt(tmp_path) + assert [finding.code for finding in failure.findings] == ["harbor.invalid.task_config"] def test_agent_timeout_becomes_per_task_agent_policy( @@ -1001,7 +1120,7 @@ def test_agent_timeout_becomes_per_task_agent_policy( task = make_harbor_task(tmp_path, "task-a") (task / "task.toml").write_text("[agent]\ntimeout_sec = 60\n", encoding="utf-8") - taskset = harbor.adapt(tmp_path) + taskset = _adapt(tmp_path) (row,) = list(taskset) assert row.agent_config == {"timeout_seconds": 60.0} @@ -1015,7 +1134,7 @@ def test_task_symlinks_are_copied_without_reading_host_files( task = make_harbor_task(tmp_path / "dataset", "task-a") (task / "tests" / "link").symlink_to(outside) - harbor.adapt(task.parent) + _adapt(task.parent) (context,) = (task.parent / ".hud-adapt").iterdir() copied = context / "compose-project" / "tests" / "task-a" / "link" @@ -1031,9 +1150,9 @@ def test_adapt_hashes_links_not_their_targets( task = make_harbor_task(tmp_path / "dataset", "task-a") (task / "environment" / "link").symlink_to(outside) - (before,) = list(harbor.adapt(task.parent)) + (before,) = list(_adapt(task.parent)) outside.write_text("changed", encoding="utf-8") - (after,) = list(harbor.adapt(task.parent)) + (after,) = list(_adapt(task.parent)) assert before.runtime_config == after.runtime_config @@ -1047,6 +1166,7 @@ def test_authored_runtime_assets_are_valid_source() -> None: assert 'uv python install "$python_version"' in installer assert 'python="$root/bin/python$python_version"' in installer assert 'uv venv "$root/venv" --python "$python"' in installer + assert "dnf install -y bubblewrap" in installer result = subprocess.run( ["sh", "-n", integration / "install.sh"], check=False, @@ -1055,5 +1175,61 @@ def test_authored_runtime_assets_are_valid_source() -> None: assert result.returncode == 0, result.stderr.decode() -def test_public_surface_is_only_the_two_real_operations() -> None: - assert harbor.__all__ == ["adapt", "export"] +def test_public_surface_exposes_results_and_the_two_real_operations() -> None: + assert harbor.__all__ == [ + "AdaptFailure", + "AdaptFinding", + "AdaptResult", + "adapt", + "export", + ] + + +def test_portless_sidecar_ports_are_resolved_from_its_built_image(tmp_path: Path) -> None: + task = make_harbor_task(tmp_path, "task-a") + (task / "environment" / "docker-compose.yaml").write_text( + "services:\n main: {}\n redis:\n image: redis:7-alpine\n", + encoding="utf-8", + ) + + (row,) = list(_adapt(tmp_path)) + assert row.runtime_config is not None + context = row.runtime_config.compose_project + assert isinstance(context, Path) + manifest = _environment_config(context) + assert manifest["peers"] == [] + assert manifest["peer_image_configs"] == {"redis": "peer-image-configs/redis.json"} + script = (context / "compose-project" / "build.sh").read_text("utf-8") + assert "pull redis" in script + assert "inspect_peer redis:7-alpine redis" in script + assert "declares no TCP ports in Compose or its image" in script + assert "peer-image-configs/redis.json" in script + + +def test_completed_compose_dependencies_are_not_routed_as_peers(tmp_path: Path) -> None: + task = make_harbor_task(tmp_path, "task-a") + (task / "environment" / "docker-compose.yaml").write_text( + "services:\n" + " main:\n" + " depends_on:\n" + " seed:\n" + " condition: service_completed_successfully\n" + " seed:\n" + " build: ./seed\n", + encoding="utf-8", + ) + (task / "environment" / "seed").mkdir() + (task / "environment" / "seed" / "Dockerfile").write_text( + 'FROM busybox:1.37\nCMD ["true"]\n', + encoding="utf-8", + ) + + (row,) = list(_adapt(tmp_path)) + assert row.runtime_config is not None + context = row.runtime_config.compose_project + assert isinstance(context, Path) + manifest = _environment_config(context) + assert manifest["peers"] == [] + assert manifest["peer_image_configs"] == {} + script = (context / "compose-project" / "build.sh").read_text("utf-8") + assert script.count("inspect_peer") == 1 diff --git a/hud/integrations/harbor/tests/test_integration.py b/hud/integrations/harbor/tests/test_integration.py index 862ce0312..1aa31cc49 100644 --- a/hud/integrations/harbor/tests/test_integration.py +++ b/hud/integrations/harbor/tests/test_integration.py @@ -17,7 +17,7 @@ import pytest from hud.agents.base import Agent -from hud.eval import DockerRuntime, Shared +from hud.eval import DockerRuntime, Shared, Taskset from hud.integrations import harbor if TYPE_CHECKING: @@ -33,6 +33,12 @@ ] +def _adapt(path: Path, *, hud_requirement: str = "hud") -> Taskset: + result = harbor.adapt(path, hud_requirement=hud_requirement) + assert result.failures == () + return result.taskset + + @pytest.fixture(scope="module", autouse=True) def docker_daemon() -> None: if shutil.which("docker") is None: @@ -149,7 +155,7 @@ def load_solutions() -> dict[str, str]: } solutions = await asyncio.to_thread(load_solutions) - taskset = harbor.adapt(dataset, hud_requirement=str(wheel)) + taskset = _adapt(dataset, hud_requirement=str(wheel)) job = await taskset.run( Oracle(solutions), runtime=DockerRuntime(), @@ -184,7 +190,13 @@ def separately_graded(tmp_path_factory: pytest.TempPathFactory, wheel: Path) -> """Adapt and grade the separate-verifier fixture.""" dataset = tmp_path_factory.mktemp("harbor-separate") / "harbor-harness" dataset.mkdir() - shutil.copytree(TASKS / "sidecar-reachability", dataset / "sidecar-reachability") + task = dataset / "sidecar-reachability" + shutil.copytree(TASKS / "sidecar-reachability", task) + declaration = task / "task.toml" + declaration.write_text( + declaration.read_text("utf-8") + "\n[verifier.environment]\ncpus = 1\n", + encoding="utf-8", + ) return asyncio.run(_grade_every_task(dataset, wheel)) @@ -221,6 +233,112 @@ def test_separate_verifier_phase_behavior(separately_graded: dict[str, Run]) -> test_harbor_phase_behavior(separately_graded, "sidecar-reachability") +def test_sidecar_ports_are_discovered_from_the_built_image( + tmp_path_factory: pytest.TempPathFactory, + wheel: Path, +) -> None: + dataset = tmp_path_factory.mktemp("harbor-sidecar-ports") / "harbor-harness" + task = dataset / "sidecar-reachability" + shutil.copytree(TASKS / "sidecar-reachability", task) + compose = task / "environment/docker-compose.yaml" + authored = compose.read_text("utf-8") + inferred = authored.replace( + " image: ${SIDECAR_IMAGE:-python:3.11-alpine}\n", + " build: ./workspace\n", + ).replace(' expose: ["5678", "5679"]\n', "") + assert inferred != authored + compose.write_text(inferred, encoding="utf-8") + sidecar = task / "environment/workspace" + sidecar.mkdir() + (sidecar / "Dockerfile").write_text( + "FROM python:3.11-alpine\nEXPOSE 5678 5679\n", + encoding="utf-8", + ) + + runs = asyncio.run(_grade_every_task(dataset, wheel)) + + test_harbor_phase_behavior(runs, "sidecar-reachability") + + +def test_separate_verifier_artifact_materialization_is_repeatable( + tmp_path_factory: pytest.TempPathFactory, wheel: Path +) -> None: + dataset = tmp_path_factory.mktemp("harbor-verifier-artifacts") / "harbor-harness" + task = dataset / "sidecar-reachability" + shutil.copytree(TASKS / "sidecar-reachability", task) + + async def grade_twice() -> list[Run]: + taskset = _adapt(dataset, hud_requirement=str(wheel)) + job = await taskset.run( + Oracle({"sidecar-reachability": (task / "solution/solve.sh").read_text("utf-8")}), + runtime=Shared(DockerRuntime(), width=1), + group=2, + max_concurrent=1, + ) + return job.runs + + runs = asyncio.run(grade_twice()) + + assert len(runs) == 2 + assert all(run.reward == 1.0 for run in runs) + + +def test_separate_verifier_artifacts_are_accessible_to_child_user( + tmp_path_factory: pytest.TempPathFactory, wheel: Path +) -> None: + dataset = tmp_path_factory.mktemp("harbor-child-user") / "harbor-harness" + task = dataset / "sidecar-reachability" + shutil.copytree(TASKS / "sidecar-reachability", task) + dockerfile = task / "tests/Dockerfile" + dockerfile.write_text( + dockerfile.read_text("utf-8").replace("USER verifier\n", "USER root\n"), + encoding="utf-8", + ) + (task / "tests/test.sh").write_text( + "#!/bin/sh\n" + "set -eu\n" + "mkdir -p /logs/verifier\n" + "if su verifier -s /bin/sh -c 'test \"$(cat /root/agent-output.txt)\" = private'; then\n" + " echo 1 > /logs/verifier/reward.txt\n" + "else\n" + " echo 0 > /logs/verifier/reward.txt\n" + "fi\n", + encoding="utf-8", + ) + + run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] + + assert run.reward == 1.0 + + +def test_fedora_environment_bootstrap( + tmp_path_factory: pytest.TempPathFactory, wheel: Path +) -> None: + dataset = tmp_path_factory.mktemp("harbor-fedora") / "harbor-harness" + task = dataset / "fedora-bootstrap" + (task / "environment").mkdir(parents=True) + (task / "tests").mkdir() + (task / "solution").mkdir() + (task / "task.toml").write_text( + '[task]\nname = "fedora-bootstrap"\n\n[verifier]\ntimeout_sec = 30\n', + encoding="utf-8", + ) + (task / "instruction.md").write_text("Verify Fedora bootstrap.\n", encoding="utf-8") + (task / "environment/Dockerfile").write_text( + "FROM fedora:42\nWORKDIR /app\n", + encoding="utf-8", + ) + (task / "tests/test.sh").write_text( + "#!/bin/bash\necho 1 > /logs/verifier/reward.txt\n", + encoding="utf-8", + ) + (task / "solution/solve.sh").write_text("true\n", encoding="utf-8") + + run = asyncio.run(_grade_every_task(dataset, wheel))["fedora-bootstrap"] + + assert run.reward == 1.0 + + def test_separate_verifier_rejects_artifact_symlinks( tmp_path_factory: pytest.TempPathFactory, wheel: Path ) -> None: @@ -295,7 +413,7 @@ def test_separate_verifier_restores_image_paths_between_shared_rollouts( (task / "solution" / "solve.sh").write_text("true\n", encoding="utf-8") async def grade_twice() -> list[Run]: - taskset = harbor.adapt(dataset, hud_requirement=str(wheel)) + taskset = _adapt(dataset, hud_requirement=str(wheel)) job = await taskset.run( Oracle({"sidecar-reachability": "true"}), runtime=Shared(DockerRuntime(), width=1), diff --git a/hud/tests/test_init.py b/hud/tests/test_init.py index ab70c48b6..67ac56186 100644 --- a/hud/tests/test_init.py +++ b/hud/tests/test_init.py @@ -32,6 +32,7 @@ def test_all_exports_available(self): "RuntimeGPU", "RuntimeLimits", "RuntimeResources", + "RuntimeTPU", "LocalRuntime", "SubprocessRuntime", "SyncPlan", diff --git a/hud/tests/test_init_module.py b/hud/tests/test_init_module.py index d39cf78f5..c96cbd5ab 100644 --- a/hud/tests/test_init_module.py +++ b/hud/tests/test_init_module.py @@ -31,6 +31,7 @@ def test_all_exports(self): "RuntimeGPU", "RuntimeLimits", "RuntimeResources", + "RuntimeTPU", "LocalRuntime", "SubprocessRuntime", "SyncPlan", diff --git a/pyproject.toml b/pyproject.toml index 6d2e62d5b..a5cd908ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "packaging>=21.0", "pydantic>=2.10,<3", "pydantic-settings>=2.2,<3", + "python-dotenv>=1.0,<2", # MCP dependencies "mcp>=1.24.0,<2.0", "fastmcp==3.2.0", From 27bbc0352aa20f7f7ecc974be7a3d85a5621c14e Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Thu, 13 Aug 2026 18:18:20 +0530 Subject: [PATCH 02/10] accept Harbor artifact destination and exclude Harbor ArtifactConfig allows destination (host placement, no verifier-side effect) and exclude (tar --exclude patterns applied when downloading directory artifacts). The adapter rejected both with extra_forbidden, so valid Harbor tasks failed to adapt. Accept destination with Harbor's own validation, and prune excluded entries when staging directory artifacts so the verifier sees what Harbor's verifier would see. --- hud/integrations/harbor/adapt.py | 17 ++++++ hud/integrations/harbor/env.py | 28 +++++++++ .../harbor/tests/test_contract.py | 52 ++++++++++++++++- .../harbor/tests/test_integration.py | 57 +++++++++++++++++++ 4 files changed, 152 insertions(+), 2 deletions(-) diff --git a/hud/integrations/harbor/adapt.py b/hud/integrations/harbor/adapt.py index 81b86bdf2..7a5896faa 100644 --- a/hud/integrations/harbor/adapt.py +++ b/hud/integrations/harbor/adapt.py @@ -49,6 +49,8 @@ class Artifact(BaseModel): model_config = ConfigDict(extra="forbid") source: str = Field(pattern=r"^/") + destination: str | None = None + exclude: list[str] = Field(default_factory=list) service: str = Field(default="main", min_length=1) @model_validator(mode="before") @@ -64,6 +66,21 @@ def normalize_source(cls, value: str) -> str: raise ValueError("artifact source must name a path beneath /") return str(path) + @field_validator("destination") + @classmethod + def normalize_destination(cls, value: str | None) -> str | None: + """Harbor host placement: a relative path beneath the trial's artifacts dir.""" + if not value: + return None + if "\\" in value: + raise ValueError("artifact destination must use forward slashes") + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts: + raise ValueError("artifact destination must be a relative path without '..'") + if value.rstrip("/") == "manifest.json": + raise ValueError("artifact destination 'manifest.json' is reserved by Harbor") + return value + class Collect(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/hud/integrations/harbor/env.py b/hud/integrations/harbor/env.py index f16b5207a..3aed4f1c5 100644 --- a/hud/integrations/harbor/env.py +++ b/hud/integrations/harbor/env.py @@ -4,6 +4,7 @@ import asyncio import contextlib +import fnmatch import grp import json import math @@ -352,6 +353,31 @@ def copy_artifact(source: Path, target: Path) -> None: shutil.copy2(source, target, follow_symlinks=False) +def prune_excluded(target: Path, patterns: list[str]) -> None: + """Drop staged entries matching Harbor exclude patterns (GNU tar semantics). + + A pattern excludes an entry when it matches any run of trailing path + components; matching a directory prunes its whole subtree. + """ + + def excluded(entry: Path) -> bool: + parts = entry.relative_to(target).as_posix().split("/") + return any( + fnmatch.fnmatch("/".join(parts[start:]), pattern) + for pattern in patterns + for start in range(len(parts)) + ) + + for root, directories, files in os.walk(target, topdown=True): + for name in list(directories): + if excluded(Path(root, name)): + shutil.rmtree(Path(root, name)) + directories.remove(name) + for name in files: + if excluded(Path(root, name)): + Path(root, name).unlink() + + async def collect(task: dict[str, Any]) -> None: clear(ARTIFACTS) services: dict[str, str] = {} @@ -442,6 +468,8 @@ async def container(service: str) -> str: copy_artifact(Path(source), target) if target.is_symlink() or any(path.is_symlink() for path in target.rglob("*")): raise RuntimeError(f"artifact {source} contains a symbolic link") + if artifact["exclude"] and target.is_dir(): + prune_excluded(target, artifact["exclude"]) @env.template(id="run", description="Run a Harbor task") diff --git a/hud/integrations/harbor/tests/test_contract.py b/hud/integrations/harbor/tests/test_contract.py index 84fb6a14f..154b5b763 100644 --- a/hud/integrations/harbor/tests/test_contract.py +++ b/hud/integrations/harbor/tests/test_contract.py @@ -595,7 +595,7 @@ def test_adapt_groups_identical_images_and_keeps_row_metadata( assert taskset["build-pmars"].agent_config == {"timeout_seconds": 45.0} assert taskset["build-pmars"].args["task"]["verifier_timeout"] == 30.0 assert taskset["build-pmars"].args["task"]["artifacts"] == [ - {"service": "main", "source": "/tmp/result"} + {"service": "main", "source": "/tmp/result", "destination": None, "exclude": []} ] @@ -883,7 +883,9 @@ def test_adapt_builds_a_separate_verifier_and_reuses_the_runtime( } assert "tasks" not in manifest assert row.args["task"] == { - "artifacts": [{"service": "main", "source": "/tmp/agent.patch"}], + "artifacts": [ + {"service": "main", "source": "/tmp/agent.patch", "destination": None, "exclude": []} + ], "collect": [{"command": "redis-cli save", "service": "redis", "timeout_sec": 10.0}], "description": "", "id": "separate", @@ -995,6 +997,52 @@ def test_artifacts_must_name_normalized_paths_beneath_root( harbor.adapt(tmp_path) +def test_artifacts_keep_harbor_destination_and_exclude(tmp_path: Path) -> None: + task = make_harbor_task(tmp_path, "task-a") + (task / "task.toml").write_text( + """\ +[[artifacts]] +source = "/app/outputs" +destination = "results/outputs" +exclude = ["*.tmp", "cache"] +""", + encoding="utf-8", + ) + + taskset = harbor.adapt(tmp_path) + + assert taskset["task-a"].args["task"]["artifacts"] == [ + { + "service": "main", + "source": "/app/outputs", + "destination": "results/outputs", + "exclude": ["*.tmp", "cache"], + } + ] + + +@pytest.mark.parametrize( + "destination", + ["/absolute/path", "results/../../escape", "results\\\\windows", "manifest.json"], +) +def test_artifact_destinations_harbor_rejects_fail_adaptation( + tmp_path: Path, + destination: str, +) -> None: + task = make_harbor_task(tmp_path, "task-a") + (task / "task.toml").write_text( + f"""\ +[[artifacts]] +source = "/app/out.json" +destination = "{destination}" +""", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="not a valid Harbor task"): + harbor.adapt(tmp_path) + + def test_agent_timeout_becomes_per_task_agent_policy( tmp_path: Path, ) -> None: diff --git a/hud/integrations/harbor/tests/test_integration.py b/hud/integrations/harbor/tests/test_integration.py index 862ce0312..c96e61eaf 100644 --- a/hud/integrations/harbor/tests/test_integration.py +++ b/hud/integrations/harbor/tests/test_integration.py @@ -241,6 +241,63 @@ def test_separate_verifier_rejects_artifact_symlinks( assert "artifact /app/main.html is a symbolic link" in (run.trace.error or "") +def test_separate_verifier_sees_directory_artifacts_without_excluded_entries( + tmp_path_factory: pytest.TempPathFactory, wheel: Path +) -> None: + dataset = tmp_path_factory.mktemp("harbor-artifact-exclude") / "harbor-harness" + task = dataset / "sidecar-reachability" + shutil.copytree(TASKS / "sidecar-reachability", task) + (task / "task.toml").write_text( + """\ +artifacts = [ + { source = "/app/outputs", destination = "results", exclude = ["*.tmp", "cache"] }, +] + +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 +""", + encoding="utf-8", + ) + (task / "solution" / "solve.sh").write_text( + """\ +#!/bin/sh +set -eu +mkdir -p /app/outputs/cache/nested /app/outputs/logs +echo keep > /app/outputs/keep.txt +echo junk > /app/outputs/junk.tmp +echo junk > /app/outputs/logs/nested.tmp +echo junk > /app/outputs/cache/nested/blob +""", + encoding="utf-8", + ) + (task / "tests" / "test.sh").write_text( + """\ +#!/bin/sh +set -u +mkdir -p /logs/verifier +if [ "$(cat /app/outputs/keep.txt 2>/dev/null)" = "keep" ] \\ + && [ -d /app/outputs/logs ] \\ + && [ ! -e /app/outputs/junk.tmp ] \\ + && [ ! -e /app/outputs/logs/nested.tmp ] \\ + && [ ! -e /app/outputs/cache ]; then + echo 1 > /logs/verifier/reward.txt +else + echo "excluded artifact entries leaked into the verifier" >&2 + echo 0 > /logs/verifier/reward.txt +fi +""", + encoding="utf-8", + ) + + run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] + + assert run.reward == 1.0, run.trace.error + + def test_separate_verifier_rejects_artifacts_beneath_symlinks( tmp_path_factory: pytest.TempPathFactory, wheel: Path ) -> None: From 2f9b01b77f5d161f58c67d5ff5e8731d10c5bfb4 Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Thu, 13 Aug 2026 19:07:35 +0530 Subject: [PATCH 03/10] apply artifact exclude before symlink rejection --- hud/integrations/harbor/env.py | 19 +++++++++---------- .../harbor/tests/test_integration.py | 3 +++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/hud/integrations/harbor/env.py b/hud/integrations/harbor/env.py index 3aed4f1c5..e659794bf 100644 --- a/hud/integrations/harbor/env.py +++ b/hud/integrations/harbor/env.py @@ -343,12 +343,7 @@ def copy_artifact(source: Path, target: Path) -> None: raise RuntimeError(f"artifact {source} has a symbolic link in its path") target.parent.mkdir(parents=True, exist_ok=True) if source.is_dir(): - for root, directories, files in os.walk(source, followlinks=False): - for name in (*directories, *files): - entry = Path(root, name) - if entry.is_symlink(): - raise RuntimeError(f"artifact {source} contains symbolic link {entry}") - shutil.copytree(source, target) + shutil.copytree(source, target, symlinks=True) elif source.exists() or source.is_symlink(): shutil.copy2(source, target, follow_symlinks=False) @@ -370,8 +365,12 @@ def excluded(entry: Path) -> bool: for root, directories, files in os.walk(target, topdown=True): for name in list(directories): - if excluded(Path(root, name)): - shutil.rmtree(Path(root, name)) + entry = Path(root, name) + if excluded(entry): + if entry.is_symlink(): + entry.unlink() + else: + shutil.rmtree(entry) directories.remove(name) for name in files: if excluded(Path(root, name)): @@ -466,10 +465,10 @@ async def container(service: str) -> str: continue else: copy_artifact(Path(source), target) - if target.is_symlink() or any(path.is_symlink() for path in target.rglob("*")): - raise RuntimeError(f"artifact {source} contains a symbolic link") if artifact["exclude"] and target.is_dir(): prune_excluded(target, artifact["exclude"]) + if target.is_symlink() or any(path.is_symlink() for path in target.rglob("*")): + raise RuntimeError(f"artifact {source} contains a symbolic link") @env.template(id="run", description="Run a Harbor task") diff --git a/hud/integrations/harbor/tests/test_integration.py b/hud/integrations/harbor/tests/test_integration.py index c96e61eaf..578b25673 100644 --- a/hud/integrations/harbor/tests/test_integration.py +++ b/hud/integrations/harbor/tests/test_integration.py @@ -271,6 +271,8 @@ def test_separate_verifier_sees_directory_artifacts_without_excluded_entries( echo junk > /app/outputs/junk.tmp echo junk > /app/outputs/logs/nested.tmp echo junk > /app/outputs/cache/nested/blob +ln -s /etc/passwd /app/outputs/cache/link +ln -s missing /app/outputs/logs/dangling.tmp """, encoding="utf-8", ) @@ -283,6 +285,7 @@ def test_separate_verifier_sees_directory_artifacts_without_excluded_entries( && [ -d /app/outputs/logs ] \\ && [ ! -e /app/outputs/junk.tmp ] \\ && [ ! -e /app/outputs/logs/nested.tmp ] \\ + && [ ! -L /app/outputs/logs/dangling.tmp ] \\ && [ ! -e /app/outputs/cache ]; then echo 1 > /logs/verifier/reward.txt else From a537e065d48175161cc223f19d604c0017ed692a Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Thu, 13 Aug 2026 19:25:11 +0530 Subject: [PATCH 04/10] reject symlink artifact roots before pruning excludes --- hud/integrations/harbor/env.py | 4 ++- .../harbor/tests/test_integration.py | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/hud/integrations/harbor/env.py b/hud/integrations/harbor/env.py index e659794bf..2c3d436c0 100644 --- a/hud/integrations/harbor/env.py +++ b/hud/integrations/harbor/env.py @@ -465,9 +465,11 @@ async def container(service: str) -> str: continue else: copy_artifact(Path(source), target) + if target.is_symlink(): + raise RuntimeError(f"artifact {source} contains a symbolic link") if artifact["exclude"] and target.is_dir(): prune_excluded(target, artifact["exclude"]) - if target.is_symlink() or any(path.is_symlink() for path in target.rglob("*")): + if any(path.is_symlink() for path in target.rglob("*")): raise RuntimeError(f"artifact {source} contains a symbolic link") diff --git a/hud/integrations/harbor/tests/test_integration.py b/hud/integrations/harbor/tests/test_integration.py index 578b25673..dfb786d68 100644 --- a/hud/integrations/harbor/tests/test_integration.py +++ b/hud/integrations/harbor/tests/test_integration.py @@ -241,6 +241,38 @@ def test_separate_verifier_rejects_artifact_symlinks( assert "artifact /app/main.html is a symbolic link" in (run.trace.error or "") +def test_separate_verifier_rejects_sidecar_symlink_artifact_roots( + tmp_path_factory: pytest.TempPathFactory, wheel: Path +) -> None: + dataset = tmp_path_factory.mktemp("harbor-sidecar-symlink") / "harbor-harness" + task = dataset / "sidecar-reachability" + shutil.copytree(TASKS / "sidecar-reachability", task) + (task / "task.toml").write_text( + """\ +artifacts = [{ source = "/link", service = "web", exclude = ["main.html"] }] + +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +service = "web" +command = "ln -sfn /app /link" +timeout_sec = 10 +""", + encoding="utf-8", + ) + (task / "solution" / "solve.sh").write_text("true\n", encoding="utf-8") + + run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] + + assert run.reward == 0.0 + assert "artifact /link contains a symbolic link" in (run.trace.error or "") + + def test_separate_verifier_sees_directory_artifacts_without_excluded_entries( tmp_path_factory: pytest.TempPathFactory, wheel: Path ) -> None: From fe14dc0fc3cfaaacb34f023d963d4e69bd9884af Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Thu, 13 Aug 2026 21:54:02 +0530 Subject: [PATCH 05/10] Add env_vars to DockerRuntime matching ModalRuntime Compose environments stage the values into the launch-time override; image environments pass them as --env arguments. Without this there is no way to hand a local Compose run a host value at all (run_args is rejected for Compose). --- hud/eval/runtime.py | 7 +++++ hud/eval/tests/test_docker_provider.py | 43 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py index ec893f448..731fed280 100644 --- a/hud/eval/runtime.py +++ b/hud/eval/runtime.py @@ -584,9 +584,11 @@ def __init__( port: int = 8765, run_args: Sequence[str] = (), runtime_config: RuntimeConfig | dict[str, Any] | None = None, + env_vars: Mapping[str, str] | None = None, ) -> None: self.port = port self.run_args = tuple(run_args) + self.env_vars = dict(env_vars or {}) config = RuntimeConfig(image=image) if image is not None else RuntimeConfig() if runtime_config is not None: config = config.with_overrides(RuntimeConfig.model_validate(runtime_config)) @@ -637,6 +639,7 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: f"127.0.0.1::{self.port}", seccomp=_DOCKER_SECCOMP_PROFILE, service_socket=service_socket, + env_vars=self.env_vars, cpu=resources.cpu if resources is not None else None, memory_mb=resources.memory_mb if resources is not None else None, gpu_count=( @@ -709,10 +712,14 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: raise ValueError("DockerRuntime cannot select GPUs by type") resource_args.extend(("--gpus", str(resources.gpu.count))) + env_args: list[str] = [] + for key, value in self.env_vars.items(): + env_args.extend(("--env", f"{key}={value}")) out, _ = await _docker( "run", "--detach", *self.run_args, + *env_args, *resource_args, *_DOCKER_SECURITY_ARGS, "--publish", diff --git a/hud/eval/tests/test_docker_provider.py b/hud/eval/tests/test_docker_provider.py index dcefa6b75..f39f380ad 100644 --- a/hud/eval/tests/test_docker_provider.py +++ b/hud/eval/tests/test_docker_provider.py @@ -707,6 +707,49 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: assert calls[-1][-3:] == ("down", "--volumes", "--remove-orphans") +async def test_docker_runtime_passes_env_vars_to_docker_run( + tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch) + + provider = DockerRuntime("img:tag", env_vars={"OPENAI_API_KEY": "sk-test"}) + async with provider(_row()) as runtime: + assert runtime.url == "tcp://127.0.0.1:43210" + + calls = await _docker_calls(docker_log) + assert calls[0] == ( + f"run --detach --env OPENAI_API_KEY=sk-test {_docker_security_args()} " + "--publish 127.0.0.1::8765 img:tag" + ) + + +async def test_docker_runtime_stages_env_vars_into_the_compose_override( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + rendered: dict[str, Any] = {} + compose = tmp_path / "compose.yaml" + compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") + + async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: + if "up" in args: + files = [Path(args[index + 1]) for index, value in enumerate(args) if value == "--file"] + _, override, _ = files + rendered.update(json.loads(override.read_text("utf-8"))) + if args[-3:] == ("port", "main", "8765"): + return "127.0.0.1:43210\n", "" + return "", "" + + monkeypatch.setattr(runtime_module, "_docker", fake_docker) + task = Task(env="any-env", id="t", runtime_config=RuntimeConfig(compose=compose)) + provider = DockerRuntime(env_vars={"OPENAI_API_KEY": "sk-test"}) + + async with provider(task): + pass + + assert rendered["services"]["main"]["environment"] == {"OPENAI_API_KEY": "sk-test"} + + async def test_docker_runtime_serializes_shared_compose_preparation( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 285aa2497db81deb761291be175ae3a0631545fc Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Thu, 13 Aug 2026 21:54:12 +0530 Subject: [PATCH 06/10] Resolve Harbor env templates from the runtime environment Harbor task env values that are exactly ${VAR} or ${VAR:-default} resolve from the host environment when a trial starts. The adapter passed them through verbatim, so agents and verifiers saw the literal template string and LLM-judge verifiers silently failed auth. Resolution happens in env.py at startup with Harbor's fullmatch semantics, sourced from the container process env the runtime's env_vars populate. Templates stay verbatim in the content-hashed manifest and persisted task rows, so host secrets never enter an image or a task file; a missing required variable aborts startup naming the variable. --- hud/integrations/harbor/env.py | 31 +++++ .../harbor/tests/test_contract.py | 35 +++++ .../harbor/tests/test_integration.py | 120 ++++++++++++++++++ 3 files changed, 186 insertions(+) diff --git a/hud/integrations/harbor/env.py b/hud/integrations/harbor/env.py index f16b5207a..261f2df35 100644 --- a/hud/integrations/harbor/env.py +++ b/hud/integrations/harbor/env.py @@ -9,6 +9,7 @@ import math import os import pwd +import re import shutil import socket import tempfile @@ -82,6 +83,36 @@ def image_environment(config: dict[str, Any]) -> dict[str, str]: verifier_image["workdir"] = VERIFIER_IMAGE_CONFIG.get("WorkingDir") or "/" verifier_image["env"] = image_environment(VERIFIER_IMAGE_CONFIG) +# Harbor's host-side env template contract (harbor/utils/env.py): a value that +# is exactly ``${VAR}`` or ``${VAR:-default}`` resolves from the environment at +# startup; anything else, including embedded templates, stays literal. Here the +# source is this process's environment, which the runtime provider populates +# from the host via ``env_vars`` (secrets never enter the content-hashed image). +ENV_TEMPLATE = re.compile(r"\$\{([^}:]+)(?::-(.*))?\}") + + +def resolve_env_templates(env: dict[str, str]) -> dict[str, str]: + resolved: dict[str, str] = {} + for key, value in env.items(): + match = ENV_TEMPLATE.fullmatch(value) + if match is None: + resolved[key] = value + continue + name, default = match.group(1), match.group(2) + if name in os.environ: + resolved[key] = os.environ[name] + elif default is not None: + resolved[key] = default + else: + raise ValueError( + f"Harbor env template for {key!r} needs {name!r}; " + "pass it through the runtime's env_vars" + ) + return resolved + + +for policy in (CONFIG["environment"], CONFIG["agent"], CONFIG["verifier"]): + policy["env"] = resolve_env_templates(policy["env"]) os.environ.update(CONFIG["environment"]["env"]) WORKDIR = Path(CONFIG["workdir"]) os.chdir(WORKDIR) diff --git a/hud/integrations/harbor/tests/test_contract.py b/hud/integrations/harbor/tests/test_contract.py index 84fb6a14f..c733035d3 100644 --- a/hud/integrations/harbor/tests/test_contract.py +++ b/hud/integrations/harbor/tests/test_contract.py @@ -639,6 +639,41 @@ def test_adapt_maps_resources_onto_the_compose_runtime(tmp_path: Path) -> None: assert row.runtime_config.resources.gpu.type == "H100" +def test_env_templates_are_persisted_verbatim_not_resolved( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Host values must never be baked into the content-hashed image payload + or the persisted task rows; ``${VAR}`` templates resolve at runtime.""" + monkeypatch.setenv("HARBOR_JUDGE_KEY", "sk-live-secret") + task = make_harbor_task(tmp_path, "templated") + (task / "task.toml").write_text( + """ +[environment.env] +JUDGE_KEY = "${HARBOR_JUDGE_KEY}" +MODEL = "${HARBOR_JUDGE_MODEL:-gpt-4o}" + +[verifier] +timeout_sec = 60 + +[verifier.env] +VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" +""", + encoding="utf-8", + ) + + harbor.adapt(tmp_path) + + (context,) = (tmp_path / ".hud-adapt").iterdir() + manifest = json.loads((context / "compose-project" / "hud" / "config.json").read_text("utf-8")) + assert manifest["environment"]["env"] == { + "JUDGE_KEY": "${HARBOR_JUDGE_KEY}", + "MODEL": "${HARBOR_JUDGE_MODEL:-gpt-4o}", + } + assert manifest["verifier"]["env"] == {"VERIFIER_KEY": "${HARBOR_JUDGE_KEY}"} + for persisted in sorted(path for path in context.rglob("*") if path.is_file()): + assert b"sk-live-secret" not in persisted.read_bytes(), persisted + + def test_prebuilt_harbor_image_is_inspected_by_the_project_build( tmp_path: Path, ) -> None: diff --git a/hud/integrations/harbor/tests/test_integration.py b/hud/integrations/harbor/tests/test_integration.py index 862ce0312..563cb7da6 100644 --- a/hud/integrations/harbor/tests/test_integration.py +++ b/hud/integrations/harbor/tests/test_integration.py @@ -20,6 +20,8 @@ from hud.eval import DockerRuntime, Shared from hud.integrations import harbor +from .conftest import make_harbor_task + if TYPE_CHECKING: from hud.capabilities import MCPClient, SSHClient from hud.eval.run import Run @@ -347,3 +349,121 @@ def test_separate_verifier_honors_the_test_script_shebang( run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] assert run.reward == 1.0 + + +def test_env_templates_resolve_from_runtime_env_vars( + tmp_path_factory: pytest.TempPathFactory, wheel: Path +) -> None: + """``${VAR}``/``${VAR:-default}`` env values resolve exactly like Harbor's + host-side resolution, sourced from the runtime's ``env_vars``.""" + dataset = tmp_path_factory.mktemp("harbor-env-templates") / "harbor-harness" + task = make_harbor_task( + dataset, + "env-templates", + task_toml="""\ +[metadata] +category = "systems" + +[environment.env] +GREETING = "${HARBOR_GREETING:-hello}" +JUDGE_KEY = "${HARBOR_JUDGE_KEY}" +EMBEDDED = "Bearer ${HARBOR_JUDGE_KEY}" + +[verifier] +timeout_sec = 120 + +[verifier.env] +VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" +EMPTY_DEFAULT = "${HARBOR_UNSET:-}" +""", + ) + (task / "tests" / "test.sh").write_text( + """\ +#!/bin/bash +fail() { echo "unexpected $1"; echo "0.0" > /logs/verifier/reward.txt; exit 0; } +[ "$GREETING" = "hello" ] || fail "GREETING=$GREETING" +[ "$JUDGE_KEY" = "judge-secret" ] || fail "JUDGE_KEY=$JUDGE_KEY" +[ "$EMBEDDED" = 'Bearer ${HARBOR_JUDGE_KEY}' ] || fail "EMBEDDED=$EMBEDDED" +[ "$VERIFIER_KEY" = "judge-secret" ] || fail "VERIFIER_KEY=$VERIFIER_KEY" +[ "${EMPTY_DEFAULT-unset}" = "" ] || fail "EMPTY_DEFAULT=${EMPTY_DEFAULT-unset}" +echo "1.0" > /logs/verifier/reward.txt +""", + encoding="utf-8", + ) + solution = '[ "$JUDGE_KEY" = "judge-secret" ] && [ "$GREETING" = "hello" ]' + + async def grade() -> Run: + taskset = harbor.adapt(dataset, hud_requirement=str(wheel)) + job = await taskset.run( + Oracle({"env-templates": solution}), + runtime=DockerRuntime(env_vars={"HARBOR_JUDGE_KEY": "judge-secret"}), + max_concurrent=1, + ) + (run,) = job.runs + return run + + run = asyncio.run(grade()) + + evaluation = run.evaluation + info = evaluation.get("info") or {} + detail = "\n".join( + filter( + None, + ( + run.trace.content, + run.trace.error, + evaluation.get("content") or "", + info.get("stdout"), + info.get("stderr"), + ), + ) + ) + assert run.reward == 1.0, f"env-templates scored {run.reward}; the verifier reported:\n{detail}" + + +def test_missing_env_template_aborts_startup( + tmp_path_factory: pytest.TempPathFactory, wheel: Path +) -> None: + """A required ``${VAR}`` with no default and no runtime value must abort + environment startup with an error naming the variable, like Harbor.""" + dataset = tmp_path_factory.mktemp("harbor-env-missing") / "harbor-harness" + make_harbor_task( + dataset, + "env-missing", + task_toml="""\ +[metadata] +category = "systems" + +[environment.env] +API_KEY = "${HARBOR_MISSING_KEY}" + +[verifier] +timeout_sec = 120 +""", + ) + + taskset = harbor.adapt(dataset, hud_requirement=str(wheel)) + task = next(iter(taskset)) + assert task.runtime_config is not None + source = task.runtime_config.compose_source() + assert source is not None + compose = source.runnable_path("test") + # Providers yield as soon as the published port exists, before the serve + # process proves itself, so an early abort is only observable from the + # adapted artifact: run its main service in the foreground. + subprocess.run( + ["sh", "build.sh"], cwd=compose.parent, check=True, capture_output=True, timeout=600 + ) + command = ["docker", "compose", "--file", str(compose), "run", "--rm", "main"] + try: + serve = subprocess.run(command, capture_output=True, text=True, timeout=60) + except subprocess.TimeoutExpired: + pytest.fail("main service kept serving despite an unresolvable env template") + finally: + subprocess.run( + ["docker", "compose", "--file", str(compose), "down", "--volumes", "--remove-orphans"], + capture_output=True, + check=False, + ) + assert serve.returncode != 0 + assert "HARBOR_MISSING_KEY" in serve.stdout + serve.stderr From ab63fcafbe2caf510c1c686e142efaca731b92a1 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:56:45 -0700 Subject: [PATCH 07/10] fix(hosted): preserve shared verifier tasks --- .../v6/experimental/verifier-environments.mdx | 5 ++-- docs/v6/reference/runtime.mdx | 4 ++- hud/eval/runtime/hosted.py | 10 +++++-- hud/eval/runtime/modal.py | 5 ++++ hud/eval/sync.py | 4 +-- hud/eval/task.py | 13 ++++++++ hud/eval/tests/test_docker_provider.py | 16 ++++++++++ hud/eval/tests/test_hosted.py | 30 +++++++++++++++++-- hud/eval/tests/test_sync.py | 16 +++++++--- hud/integrations/harbor/adapt.py | 6 +++- hud/integrations/harbor/env.py | 18 +++++++++++ .../harbor/tests/test_contract.py | 4 +++ 12 files changed, 115 insertions(+), 16 deletions(-) diff --git a/docs/v6/experimental/verifier-environments.mdx b/docs/v6/experimental/verifier-environments.mdx index 131171571..3cd16ec74 100644 --- a/docs/v6/experimental/verifier-environments.mdx +++ b/docs/v6/experimental/verifier-environments.mdx @@ -92,8 +92,9 @@ engine keeps the actor connection and substrate alive and starts the verifier ta channel immediately after the actor task completes. A different environment name or verifier runtime configuration forces actor cleanup followed by a fresh provider acquisition. -`HostedRuntime` does not accept verifier task rows; verifier environments run under a -client-driven provider such as `LocalRuntime`, `DockerRuntime`, or a custom provider. +`HostedRuntime` supports the same-environment form without a verifier `runtime_config`, keeping +both phases inside one hosted acquisition. Verifiers that require another runtime remain +client-driven through `LocalRuntime`, `DockerRuntime`, or a custom provider. ## What runs where diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index 8f3335615..ee88d6b3c 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -159,7 +159,9 @@ ModalRuntime(image_name=None, *, image=None, command=None, app_name="hud-envs", For Compose input, `ModalRuntime` runs the project in a Docker-in-Docker sandbox in your Modal account and merges `env_vars` into `main` at acquisition -time. See [Compose environments](/v6/experimental/compose#runtime-behavior). +time. GPU requests require a plain or platform-materialized image; Modal Compose +rejects them because its nested Docker daemon cannot receive the sandbox GPU. +See [Compose environments](/v6/experimental/compose#runtime-behavior). Requires the `modal` extra and a configured token. diff --git a/hud/eval/runtime/hosted.py b/hud/eval/runtime/hosted.py index 91aa0d9f7..925890087 100644 --- a/hud/eval/runtime/hosted.py +++ b/hud/eval/runtime/hosted.py @@ -68,10 +68,12 @@ async def run( """ trace_id = trace_id or uuid.uuid4().hex try: - if task.verifier is not None: + if task.verifier is not None and ( + task.verifier.env != task.env or task.verifier.runtime_config is not None + ): raise ValueError( - "HostedRuntime does not support verifier tasks until hosted rollouts " - "can keep both phases in one runtime scope" + "hosted verifier tasks must reuse the actor runtime: verifier.env must " + "match task.env and verifier.runtime_config must be omitted" ) async with asyncio.timeout(self.run_timeout): state = await self._submit_and_await( @@ -137,6 +139,8 @@ async def _submit_and_await( runtime_config = task.runtime_config.request_payload() if runtime_config: payload["runtime_config"] = runtime_config + if task.verifier is not None: + payload["verifier"] = task.verifier.wire_payload() await platform.apost("/rollouts/submit", json=payload) return await self._await_terminal(platform, payload["trace_id"]) diff --git a/hud/eval/runtime/modal.py b/hud/eval/runtime/modal.py index 4b08e7953..c543e4ba7 100644 --- a/hud/eval/runtime/modal.py +++ b/hud/eval/runtime/modal.py @@ -234,6 +234,11 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: compose = ( compose_source.runnable_path("ModalRuntime") if compose_source is not None else None ) + if compose is not None and resources is not None and resources.gpu is not None: + raise ValueError( + "ModalRuntime cannot attach GPUs to services inside Docker-in-Docker; " + "use a materialized image or omit runtime_config.compose" + ) port_service = ComposeConfig.from_file(compose).network_owner("main") if compose else "main" modal = cast("ModalModule", importlib.import_module("modal")) diff --git a/hud/eval/sync.py b/hud/eval/sync.py index aa37e433a..cf50032b6 100644 --- a/hud/eval/sync.py +++ b/hud/eval/sync.py @@ -166,7 +166,7 @@ def task_upload_payload(task: Task) -> dict[str, Any]: if task.runtime_config is not None: payload["runtime_config"] = task.runtime_config.request_payload() if task.verifier is not None: - payload["verifier"] = task.verifier.model_dump(mode="json", exclude_none=True) + payload["verifier"] = task.verifier.wire_payload() return payload @@ -181,7 +181,7 @@ def _task_signature(task: Task) -> str: if task.runtime_config is not None: sig_data["runtime_config"] = task.runtime_config.request_payload() if task.verifier is not None: - sig_data["verifier"] = task.verifier.model_dump(mode="json", exclude_none=True) + sig_data["verifier"] = task.verifier.wire_payload() return f"{task.id}|" + json.dumps( sig_data, sort_keys=True, diff --git a/hud/eval/task.py b/hud/eval/task.py index a063eefaf..b760b826f 100644 --- a/hud/eval/task.py +++ b/hud/eval/task.py @@ -87,6 +87,19 @@ def _reject_nested_verifier(cls, verifier: Task | None) -> Task | None: raise ValueError("nested verifier tasks are not supported") return verifier + def wire_payload(self) -> dict[str, Any]: + """Serialize the task for platform transport.""" + payload = self.model_dump( + mode="json", + exclude_none=True, + exclude={"runtime_config", "verifier"}, + ) + if self.runtime_config is not None: + payload["runtime_config"] = self.runtime_config.request_payload() + if self.verifier is not None: + payload["verifier"] = self.verifier.wire_payload() + return payload + # ─── execution ──────────────────────────────────────────────────── async def run( diff --git a/hud/eval/tests/test_docker_provider.py b/hud/eval/tests/test_docker_provider.py index c350281af..a19de6144 100644 --- a/hud/eval/tests/test_docker_provider.py +++ b/hud/eval/tests/test_docker_provider.py @@ -995,6 +995,22 @@ async def test_modal_runtime_bounds_compose_startup( assert calls["terminated"] is True +async def test_modal_runtime_rejects_gpu_inside_compose_dind(tmp_path: Path) -> None: + compose = tmp_path / "compose.yaml" + compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") + + provider = ModalRuntime( + runtime_config=RuntimeConfig( + compose=compose, + resources=RuntimeResources(gpu=RuntimeGPU(type="H100")), + ) + ) + + with pytest.raises(ValueError, match=r"cannot attach GPUs.*Docker-in-Docker"): + async with provider(_row()): + pass + + async def test_modal_runtime_accepts_modal_image_uri( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/hud/eval/tests/test_hosted.py b/hud/eval/tests/test_hosted.py index 965a3fb35..81ddaae9d 100644 --- a/hud/eval/tests/test_hosted.py +++ b/hud/eval/tests/test_hosted.py @@ -168,19 +168,30 @@ async def test_run_rejects_non_gateway_agent() -> None: @pytest.mark.asyncio -async def test_run_rejects_verifier_tasks_until_hosted_supports_both_phases() -> None: +@pytest.mark.parametrize( + "verifier", + [ + Task(env="judge", id="verify"), + Task( + env="actor", + id="verify", + runtime_config=RuntimeConfig(image="judge:latest"), + ), + ], +) +async def test_run_rejects_verifier_that_requires_another_runtime(verifier: Task) -> None: run = await HostedRuntime(poll_interval=0.0).run( Task( env="actor", id="solve", - verifier=Task(env="judge", id="verify"), + verifier=verifier, ), _agent(), job_id="j", ) assert run.trace.is_error - assert "does not support verifier tasks" in (run.trace.error or "") + assert "must reuse the actor runtime" in (run.trace.error or "") @pytest.mark.asyncio @@ -210,6 +221,12 @@ async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch resources=RuntimeResources(cpu=2, gpu=RuntimeGPU(type="L4", count=1)), limits=RuntimeLimits(startup_timeout_s=120, run_timeout_s=900), ), + verifier=Task( + env="sums", + id="verify", + args={"expected": 3}, + requires_handoff=True, + ), ) run = await hosted.run(task, _agent(), job_id=job_id, group_id="g1", trace_id=trace_id) @@ -234,6 +251,13 @@ async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch "resources": {"cpu": 2.0, "gpu": {"type": "L4", "count": 1}}, "limits": {"startup_timeout_s": 120, "run_timeout_s": 900}, } + assert payload["verifier"] == { + "env": "sums", + "id": "verify", + "args": {"expected": 3}, + "slug": "verify-5579a3e5", + "requires_handoff": True, + } assert payload["group_id"] == "g1" assert payload["agent"]["type"] == "openai_compatible" assert payload["agent"]["config"]["model"] == "test-model" diff --git a/hud/eval/tests/test_sync.py b/hud/eval/tests/test_sync.py index 71c89eb48..09252d75f 100644 --- a/hud/eval/tests/test_sync.py +++ b/hud/eval/tests/test_sync.py @@ -193,23 +193,31 @@ def test_task_upload_payload_preserves_runtime_config_null_override() -> None: assert payload["runtime_config"] == {"resources": None} -def test_task_upload_payload_includes_verifier_task() -> None: +def test_task_upload_payload_includes_verifier_task(tmp_path: Path) -> None: + compose = tmp_path / "compose.json" + compose.write_text( + json.dumps({"services": {"main": {"image": "judge:latest"}}}), + encoding="utf-8", + ) task = Task( env="actor", id="solve", verifier=Task( env="judge", id="verify", - runtime_config=RuntimeConfig(image="judge:latest"), + runtime_config=RuntimeConfig(compose=compose), ), ) payload = task_upload_payload(task) - assert payload["verifier"] == { + verifier = payload["verifier"] + runtime_config = verifier.pop("runtime_config") + assert verifier == { "env": "judge", "id": "verify", "args": {}, "slug": "verify", - "runtime_config": {"image": "judge:latest"}, } + assert runtime_config["compose"]["services"]["main"]["image"] == "judge:latest" + assert str(compose) not in json.dumps(runtime_config) diff --git a/hud/integrations/harbor/adapt.py b/hud/integrations/harbor/adapt.py index 1c52923b2..6dfec6822 100644 --- a/hud/integrations/harbor/adapt.py +++ b/hud/integrations/harbor/adapt.py @@ -1073,7 +1073,11 @@ def adapt( if config.verifier.environment is not None else None ) - verifier_limits = _runtime_limits(config.verifier.environment or config.environment) + verifier_limits = ( + _runtime_limits(config.verifier.environment) + if config.verifier.environment is not None + else None + ) needs_service_access = any( item.service != "main" for item in (*config.verifier.collect, *config.artifacts) ) or bool(healthy_services) diff --git a/hud/integrations/harbor/env.py b/hud/integrations/harbor/env.py index e822b2b6a..da7c2b426 100644 --- a/hud/integrations/harbor/env.py +++ b/hud/integrations/harbor/env.py @@ -706,10 +706,28 @@ def materialized_artifacts( artifacts: Path, verifier_identity: tuple[int, int] | None, ) -> Iterator[list[Mount]]: + driver_files = ( + { + path + for root in (Path("/usr/lib"), Path("/usr/lib64")) + if root.is_dir() + for path in root.rglob("*.so*") + if path.name.startswith(("libcuda.so", "libnvidia-")) + and (path.is_file() or path.is_symlink()) + } + | { + path + for path in Path("/usr/bin").glob("nvidia-*") + if path.is_file() or path.is_symlink() + } + if Path("/dev/nvidiactl").exists() + else set() + ) mounts = [ Mount("dev", dst="/dev"), Mount("proc", dst="/proc"), Mount("rw", src=str(LOGS), dst="/logs"), + *(Mount("ro", src=str(path), dst=str(path)) for path in sorted(driver_files)), ] with tempfile.TemporaryDirectory(prefix="verifier-backup-", dir=ROOT) as directory: backup_root = Path(directory) diff --git a/hud/integrations/harbor/tests/test_contract.py b/hud/integrations/harbor/tests/test_contract.py index 634ca13e2..3942a0daa 100644 --- a/hud/integrations/harbor/tests/test_contract.py +++ b/hud/integrations/harbor/tests/test_contract.py @@ -1027,6 +1027,7 @@ def test_image_task_keeps_only_the_verifier_as_a_build_service( encoding="utf-8", ) (task / "task.toml").write_text( + '[environment]\nbuild_timeout_sec = 300\n\n' '[verifier]\nenvironment_mode = "separate"\n', encoding="utf-8", ) @@ -1034,7 +1035,10 @@ def test_image_task_keeps_only_the_verifier_as_a_build_service( (row,) = list(_adapt(tmp_path)) assert row.runtime_config is not None + assert row.runtime_config.limits == RuntimeLimits(startup_timeout_s=300) assert row.runtime_config.compose_service_access is None + assert row.verifier is not None + assert row.verifier.runtime_config is None assert isinstance(row.runtime_config.compose, Path) project = _assert_stock_compose_complete(row.runtime_config.compose) assert set(project["services"]) == {"main", "hud-verifier"} From bdc2c8f816c207e8889779798b14fa1e9cfd6e11 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:41:27 -0700 Subject: [PATCH 08/10] fix(runtime): honor Docker startup timeouts --- hud/eval/runtime/docker.py | 17 ++++++++++++++--- hud/eval/tests/test_docker_provider.py | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/hud/eval/runtime/docker.py b/hud/eval/runtime/docker.py index eb0bef6c4..c6cd6a117 100644 --- a/hud/eval/runtime/docker.py +++ b/hud/eval/runtime/docker.py @@ -106,8 +106,13 @@ def __init__( @asynccontextmanager async def __call__(self, task: Task) -> AsyncIterator[Runtime]: config = (self.runtime_config or RuntimeConfig()).with_overrides(task.runtime_config) - if config.limits is not None and config.limits.model_dump(exclude_none=True): - raise ValueError("DockerRuntime does not support runtime_config limits") + if config.limits is not None and config.limits.run_timeout_s is not None: + raise ValueError("DockerRuntime does not support runtime_config.limits.run_timeout_s") + params = ( + {"ready_timeout": config.limits.startup_timeout_s} + if config.limits is not None and config.limits.startup_timeout_s is not None + else {} + ) resources = config.resources if resources is not None: resources._require_support("DockerRuntime", {"cpu", "memory_mb", "storage_mb", "gpu"}) @@ -201,6 +206,7 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: await handoff.prepare() yield Runtime( f"tcp://127.0.0.1:{host_port}", + params=params, config=config if config.model_dump(exclude_none=True) else None, handoff=handoff, ) @@ -260,7 +266,12 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: host_port = int(mapping.strip().splitlines()[0].rsplit(":", 1)[1]) handoff = _DockerHandoff(container) await handoff.prepare() - yield Runtime(f"tcp://127.0.0.1:{host_port}", config=config, handoff=handoff) + yield Runtime( + f"tcp://127.0.0.1:{host_port}", + params=params, + config=config, + handoff=handoff, + ) finally: # check=False: teardown must not shadow the run's own error, and # rm -f only fails when the daemon itself is broken. diff --git a/hud/eval/tests/test_docker_provider.py b/hud/eval/tests/test_docker_provider.py index a19de6144..33d12f8d1 100644 --- a/hud/eval/tests/test_docker_provider.py +++ b/hud/eval/tests/test_docker_provider.py @@ -1527,6 +1527,31 @@ async def test_docker_admits_minimum_free_disk( assert "exec cid-42 df -Pk /" in calls +async def test_docker_honors_startup_timeout( + tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_fake_docker( + tmp_path, + port_behavior="echo 127.0.0.1:43210", + monkeypatch=monkeypatch, + ) + config = RuntimeConfig(image="img:tag", limits=RuntimeLimits(startup_timeout_s=300)) + + async with DockerRuntime(runtime_config=config)(_row()) as runtime: + assert runtime.params == {"ready_timeout": 300} + + +async def test_docker_rejects_run_timeout( + tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch) + config = RuntimeConfig(image="img:tag", limits=RuntimeLimits(run_timeout_s=300)) + + with pytest.raises(ValueError, match=r"runtime_config\.limits\.run_timeout_s"): + async with DockerRuntime(runtime_config=config)(_row()): + pass + + async def test_daytona_names_a_sandbox_it_could_not_delete( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: From 58694880db13916ccf13f57f6cc689be73c59d94 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Mon, 17 Aug 2026 18:57:16 -0700 Subject: [PATCH 09/10] refactor(runtime): simplify environment composition --- .github/workflows/ci.yml | 4 +- cookbooks/fireworks-rl-training/train.py | 8 +- docs/skill.md | 3 +- docs/v6/experimental/compose.mdx | 21 +- .../v6/experimental/verifier-environments.mdx | 6 +- docs/v6/reference/runtime.mdx | 32 +- hud/__init__.py | 2 + hud/cli/deploy.py | 25 +- hud/cli/sync.py | 5 +- hud/cli/tests/test_deploy.py | 14 +- hud/environment/env.py | 11 +- hud/environment/tests/conftest.py | 10 +- .../tests/test_capability_backing.py | 4 +- hud/environment/tests/test_sessions.py | 30 +- hud/eval/__init__.py | 2 + hud/eval/run.py | 182 +++---- hud/eval/runtime/__init__.py | 19 +- hud/eval/runtime/compose.py | 162 ++++-- hud/eval/runtime/core.py | 461 +++--------------- hud/eval/runtime/daytona.py | 176 +------ hud/eval/runtime/docker.py | 123 +++-- hud/eval/runtime/hosted.py | 4 +- hud/eval/runtime/local.py | 217 +++++++++ hud/eval/runtime/modal.py | 245 ++++------ hud/eval/sync.py | 38 +- hud/eval/task.py | 99 ++-- hud/eval/taskset.py | 101 ++-- hud/eval/tests/test_docker_provider.py | 195 ++++++-- hud/eval/tests/test_hosted.py | 13 +- hud/eval/tests/test_local_runtime.py | 319 +++--------- hud/eval/tests/test_rollout.py | 163 ++++--- hud/eval/tests/test_shared.py | 78 ++- hud/eval/tests/test_sync.py | 15 +- hud/eval/tests/test_task.py | 106 ++-- hud/integrations/harbor/adapt.py | 18 +- hud/integrations/harbor/env.py | 66 ++- .../sidecar-reachability/solution/solve.sh | 6 +- .../tasks/sidecar-reachability/task.toml | 2 +- .../harbor/tests/test_contract.py | 72 +-- .../harbor/tests/test_integration.py | 6 +- hud/tests/test_init_module.py | 1 + hud/tests/test_robot.py | 10 +- 42 files changed, 1457 insertions(+), 1617 deletions(-) create mode 100644 hud/eval/runtime/local.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cb017e5b..52c974ec4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,4 +46,6 @@ jobs: uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - name: Run ty - run: uv run --extra dev --extra train ty check --error-on-warning + run: >- + uv run --extra dev --extra train --extra modal --extra daytona + ty check --error-on-warning diff --git a/cookbooks/fireworks-rl-training/train.py b/cookbooks/fireworks-rl-training/train.py index 1dbad31d5..93a5836b1 100644 --- a/cookbooks/fireworks-rl-training/train.py +++ b/cookbooks/fireworks-rl-training/train.py @@ -140,11 +140,15 @@ def split_taskset( ) random.Random(seed).shuffle(tasks) - train = Taskset(f"{source.name}-train", tasks[:train_count], origin=source.origin) + train = Taskset( + f"{source.name}-train", + tasks[:train_count], + taskset_id=source.taskset_id, + ) evaluation = Taskset( f"{source.name}-eval", tasks[train_count:required], - origin=source.origin, + taskset_id=source.taskset_id, ) return train, evaluation diff --git a/docs/skill.md b/docs/skill.md index 4589ddbbb..bce5fcf52 100644 --- a/docs/skill.md +++ b/docs/skill.md @@ -322,7 +322,8 @@ the user judges a task by its *average* reward. rollout in the group is equal, the advantage is zero and **no gradient is produced** — the task teaches nothing, however good the average looks. The unit of trainability is *within-group spread*, not the mean. Run a group -(`await Taskset("name", tasks).run(agent, group=16)`) and confirm a non-degenerate spread. +(`await Taskset("name", tasks).run(agent, runtime=runtime, group=16)`) and confirm a +non-degenerate spread. All-one (saturated) is wasted surface; all-zero at small group sizes may still be learnable at training scale, but investigate it. diff --git a/docs/v6/experimental/compose.mdx b/docs/v6/experimental/compose.mdx index ad694959d..d307c5f8e 100644 --- a/docs/v6/experimental/compose.mdx +++ b/docs/v6/experimental/compose.mdx @@ -16,8 +16,8 @@ supervised startup order - are the container boundaries and `depends_on` conditi | Concept | What it is | | --- | --- | | **`main`** | The service that runs `hud serve`; the only service whose control port HUD publishes. | -| **Compose document** | The parsed recipe (`hud/eval/compose.py`); `RuntimeConfig.compose` selects it. | -| **Project root** | The directory serialized and uploaded when the project leaves the machine; `RuntimeConfig.compose_project` names it. | +| **Compose document** | The parsed recipe; `RuntimeConfig.compose.document` selects it. | +| **Project root** | The directory serialized and uploaded when the project leaves the machine; `RuntimeConfig.compose.root` names it. | | **Build-only service** | A service with `scale: 0` that exists as a build input, never a container. | | **Named context** | A `build.additional_contexts` entry - `service:` for a build-only image, a path for a local payload. | @@ -28,8 +28,8 @@ supervised startup order - are the container boundaries and `depends_on` conditi
1 · The document is the contract
-A Compose row selects its document with `RuntimeConfig.compose`; `compose_project` names the root -that travels when the project is serialized and uploaded. Everything the build needs - contexts, +A Compose row selects its document and project root with `RuntimeConfig.compose`. The project root +travels when the project is serialized and uploaded. Everything the build needs - contexts, bind mounts, `env_file`, configs, secrets - must live under that root. Interpolation resolves from the `.env` beside the Compose file and from defaults in the document; HUD does not read values from the process environment, so an unbound variable is rejected. `include` and `extends` are not part @@ -47,14 +47,13 @@ images before the runtime starts the project. ```python from pathlib import Path -from hud import DockerRuntime, RuntimeConfig, Task +from hud import ComposeProject, DockerRuntime, RuntimeConfig, Task task = Task( env="full-stack", id="repair", runtime_config=RuntimeConfig( - compose=Path("compose.yaml"), - compose_project=Path("."), + compose=ComposeProject(document=Path("compose.yaml"), root=Path(".")), ), ) job = await task.run(agent, runtime=DockerRuntime()) @@ -152,7 +151,7 @@ Two consequences do most of the work: ## Runtime behavior -**Local `DockerRuntime`** (`hud/eval/runtime.py`) runs `build.sh` when present, stages override +**Local `DockerRuntime`** (`hud/eval/runtime/docker.py`) runs `build.sh` when present, stages override files for `main`'s security options, resources, and an ephemeral host port, starts the project with `docker compose up` (`--no-build` after a successful preparation hook, otherwise `--build`), returns the published control-channel address, and removes the project and its volumes when the @@ -162,9 +161,9 @@ rollout ends. project root, the platform builds every service privately from it, and hosted rollouts run the stack. Declared `RuntimeResources` apply. -`compose_service_access=True` mounts a Docker socket at `/media/hud/docker.sock` in `main`. It is -an explicit capability for environment code that must inspect or copy from sibling service -containers, and what it grants differs by placement: +`RuntimeConfig.compose.service_access=True` mounts a Docker socket at `/media/hud/docker.sock` in +`main`. It is an explicit capability for environment code that must inspect or copy from sibling +service containers, and what it grants differs by placement: Under local `DockerRuntime`, the mounted socket is the **host's own Docker daemon** (remote Docker diff --git a/docs/v6/experimental/verifier-environments.mdx b/docs/v6/experimental/verifier-environments.mdx index 3cd16ec74..a7eab4fbe 100644 --- a/docs/v6/experimental/verifier-environments.mdx +++ b/docs/v6/experimental/verifier-environments.mdx @@ -136,9 +136,9 @@ runtime, the phases remain isolated: mode, allowlist, and credentials directory; and - verifier output is read from `/logs/verifier/reward.json` or `/logs/verifier/reward.txt`. -`compose_service_access=True` gives the generated `main` service access to the runtime's Docker -socket solely for declared collection from sibling services. The local and hosted socket behavior -is described in [Compose environments](/v6/experimental/compose#runtime-behavior). +`RuntimeConfig.compose.service_access=True` gives the generated `main` service access to the +runtime's Docker socket solely for declared collection from sibling services. The local and hosted +socket behavior is described in [Compose environments](/v6/experimental/compose#runtime-behavior). ## See also diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index ee88d6b3c..8c544e14b 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -32,17 +32,13 @@ Most runtimes are on the top-level package (`from hud import LocalRuntime, Docke HostedRuntime, Runtime`); `ModalRuntime` and `DaytonaRuntime` import from `hud.eval`. -**You can usually omit `runtime=`.** A run without one uses what HUD already knows: +**You can usually omit `runtime=`.** A run without one uses its known placement: -- a taskset loaded from Python source (`Taskset.from_file` / `from_module`) runs against that source - a platform taskset (`Taskset.from_api`) runs on the platform -- otherwise, if the envs your tasks name are defined in files you've imported, each rollout gets a - fresh env served from its file — so `my_task().run(agent)` just works in the project that defines - the env +- a task created by `@env.template` runs against that environment When none of these apply, `run` raises and lists the runtimes you can pass — it never silently picks -one. If two imported files define an env with the same name, that's also an error; disambiguate by -passing the instance you mean: `runtime=LocalRuntime(env)`. +one. For example, a task loaded from JSON needs `runtime=LocalRuntime(env)` or another runtime. To deploy an environment to the platform and run against it, see @@ -57,7 +53,7 @@ Compose project, hardware, and timeouts. Set it on the runtime (`runtime_config= supports. ```python -from hud.eval import RuntimeConfig, RuntimeGPU, RuntimeLimits, RuntimeResources, RuntimeTPU +from hud.eval import ComposeProject, RuntimeConfig, RuntimeGPU, RuntimeLimits, RuntimeResources, RuntimeTPU RuntimeConfig( image="my-env", @@ -70,16 +66,14 @@ RuntimeConfig( limits=RuntimeLimits(startup_timeout_s=300, run_timeout_s=1800), ) -RuntimeConfig(compose="./compose.yaml") +RuntimeConfig(compose=ComposeProject(document="./compose.yaml")) RuntimeConfig(resources=RuntimeResources(tpu=RuntimeTPU(type="v5", topology="2x2"))) ``` | Field | Description | |-------|-------------| | `image` | Image to run. | -| `compose` | Local path to a Compose file, or its serialized `ComposeConfig`. Mutually exclusive with `image`. | -| `compose_project` | Local project root for upload, or a serialized `ComposeProjectRef`. Requires `compose`. | -| `compose_service_access` | Mount the runtime's Docker socket into Compose `main` at `/media/hud/docker.sock`. Requires `compose`. | +| `compose` | `ComposeProject(document, root, service_access)`. Mutually exclusive with `image`. | | `resources` | Placement requests: CPU, memory, disk, acceptable GPU types and count, OS, or TPU slice. | | `limits` | `RuntimeLimits(startup_timeout_s, run_timeout_s)`. | @@ -100,22 +94,18 @@ The constructor for each built-in runtime: LocalRuntime(source, *, env=None, ready_timeout=120.0) ``` -Serves a fresh env per rollout, in this process, over the same control channel as every placement. -`source` is any pointer to the env: +Runs an environment in this process. `source` can be: - **a `.py` file or directory** that declares it, imported fresh per rollout (sibling imports resolve). **`env`** pins one name when the source declares several; it defaults to the placed task's env. -- **a live `Environment`** declared at module level - its declaring module's file is the recipe; - the instance itself is never served, so every rollout is still fresh. +- **a live `Environment`** - served directly. - **a `(task) -> Environment` constructor** for envs built in code (integrations, parameterized envs), called fresh per rollout with the placed row. -`ready_timeout` bounds `@env.initialize` startup. The freshness boundary is the env's own source: -it is re-imported per rollout, while modules it imports follow normal Python import caching and are -shared process-wide - state kept in helper modules persists across rollouts. Env hooks run in this -process and share its event loop - keep envs async, or use `SubprocessRuntime` / `DockerRuntime` -when rollouts need whole-process isolation. +`ready_timeout` bounds `@env.initialize` startup. Paths and constructors create a fresh environment +per rollout. Env hooks share this process and event loop; use `SubprocessRuntime` or +`DockerRuntime` when rollouts need process isolation. ### `SubprocessRuntime` diff --git a/hud/__init__.py b/hud/__init__.py index 675324e3d..fdcdc7b0f 100644 --- a/hud/__init__.py +++ b/hud/__init__.py @@ -11,6 +11,7 @@ from .environment import Environment from .eval import ( Chat, + ComposeProject, DockerRuntime, Grade, HostedRuntime, @@ -36,6 +37,7 @@ __all__ = [ "Chat", + "ComposeProject", "DockerRuntime", "Environment", "Grade", diff --git a/hud/cli/deploy.py b/hud/cli/deploy.py index 815d6e42e..dc9e26d6c 100644 --- a/hud/cli/deploy.py +++ b/hud/cli/deploy.py @@ -21,7 +21,7 @@ from hud.cli.utils.context import create_build_context_tarball, format_size from hud.cli.utils.registry import get_registry_environment from hud.cli.utils.source import EnvironmentSource -from hud.eval.runtime import RuntimeConfig +from hud.eval.runtime import ComposeProject, RuntimeConfig from hud.utils.exceptions import HudRequestError from hud.utils.hud_console import HUDConsole from hud.utils.naming import normalize_environment_name @@ -109,11 +109,15 @@ def _load_runtime_config(path: str | None, console: HUDConsole) -> RuntimeConfig raw = json.loads(config_path.read_text(encoding="utf-8")) if isinstance(raw, dict): raw_config = cast("dict[str, Any]", raw) - for field in ("compose", "compose_project"): - value = raw_config.get(field) - if isinstance(value, str): + compose = raw_config.get("compose") + if isinstance(compose, dict): + compose_config = cast("dict[str, Any]", compose) + for field in ("document", "root"): + value = compose_config.get(field) + if not isinstance(value, str): + continue candidate = Path(value).expanduser() - raw_config[field] = str( + compose_config[field] = str( candidate if candidate.is_absolute() else (config_path.parent / candidate).resolve() @@ -415,9 +419,7 @@ def _prepare_deploy_plan( recipe = _compose_recipe(env_dir) if recipe is not None: if loaded_runtime_config is not None and ( - loaded_runtime_config.image is not None - or loaded_runtime_config.compose is not None - or loaded_runtime_config.compose_project is not None + loaded_runtime_config.image is not None or loaded_runtime_config.compose is not None ): console.error("--runtime-config cannot set image or Compose for a Compose context") raise typer.Exit(1) @@ -428,8 +430,7 @@ def _prepare_deploy_plan( if loaded_runtime_config is not None else {} ), - "compose": recipe, - "compose_project": env_dir, + "compose": ComposeProject(document=recipe, root=env_dir), } ) @@ -557,7 +558,9 @@ async def _trigger_build( ("runtime_provider", plan.runtime), ( "runtime_config", - plan.runtime_config.request_payload() if plan.runtime_config else None, + plan.runtime_config.model_dump(mode="json", exclude_unset=True) + if plan.runtime_config + else None, ), ("environment_variables", plan.env_vars), ("build_args", plan.build_args), diff --git a/hud/cli/sync.py b/hud/cli/sync.py index c0aaf9275..fab060487 100644 --- a/hud/cli/sync.py +++ b/hud/cli/sync.py @@ -94,7 +94,10 @@ def _export_taskset( out = Path(output_path) if out.suffix.lower() == ".csv": out.parent.mkdir(parents=True, exist_ok=True) - _write_csv(out, [task.model_dump(exclude_none=True) for task in remote_taskset]) + _write_csv( + out, + [task.model_dump(mode="json", exclude_none=True) for task in remote_taskset], + ) else: out = remote_taskset.to_file(out) except (HudException, ValueError) as e: diff --git a/hud/cli/tests/test_deploy.py b/hud/cli/tests/test_deploy.py index 1533bdc08..e915092fe 100644 --- a/hud/cli/tests/test_deploy.py +++ b/hud/cli/tests/test_deploy.py @@ -242,9 +242,9 @@ def test_prepare_deploy_uses_context_recipe( ) assert plan.runtime_config is not None - payload = plan.runtime_config.request_payload() - assert payload["compose_project"] == {"compose_path": filename} - assert set(payload["compose"]["services"]) == {"main", "redis"} + payload = plan.runtime_config.model_dump(mode="json", exclude_unset=True) + assert payload["compose"]["root"] == {"compose_path": filename} + assert set(payload["compose"]["document"]["services"]) == {"main", "redis"} def test_load_runtime_config_uses_sdk_shape(self, tmp_path: Path) -> None: from hud.cli.deploy import _load_runtime_config @@ -263,7 +263,7 @@ def test_load_runtime_config_uses_sdk_shape(self, tmp_path: Path) -> None: config = _load_runtime_config(str(config_path), HUDConsole()) assert config is not None - assert config.request_payload() == { + assert config.model_dump(mode="json", exclude_unset=True) == { "resources": {"gpu": {"type": "A10G", "count": 2}}, "limits": {"startup_timeout_s": 300}, } @@ -277,7 +277,7 @@ def test_load_runtime_config_preserves_null_override(self, tmp_path: Path) -> No config = _load_runtime_config(str(config_path), HUDConsole()) assert config is not None - assert config.request_payload() == {"resources": None} + assert config.model_dump(mode="json", exclude_unset=True) == {"resources": None} def test_load_runtime_config_resolves_compose_project_from_config_directory( self, @@ -290,12 +290,12 @@ def test_load_runtime_config_resolves_compose_project_from_config_directory( compose = project / "compose.json" compose.write_text('{"services":{"main":{"image":"postgres:16"}}}') config_path = tmp_path / "runtime.json" - config_path.write_text('{"compose":"project/compose.json","compose_project":"."}') + config_path.write_text('{"compose":{"document":"project/compose.json","root":"."}}') config = _load_runtime_config(str(config_path), HUDConsole()) assert config is not None - assert config.request_payload()["compose_project"] == { + assert config.model_dump(mode="json", exclude_unset=True)["compose"]["root"] == { "compose_path": "project/compose.json" } diff --git a/hud/environment/env.py b/hud/environment/env.py index 934ef942d..86023f3da 100644 --- a/hud/environment/env.py +++ b/hud/environment/env.py @@ -86,7 +86,7 @@ class _TaskFactory(Generic[P]): binds a runnable :class:`~hud.eval.Task`:: task = fix_bug(difficulty=3) # -> Task - job = await task.run(agent, runtime=LocalRuntime("env.py")) + job = await task.run(agent) """ def __init__( @@ -123,14 +123,13 @@ def manifest_entry(self) -> dict[str, Any]: return entry def __call__(self, *args: P.args, **kwargs: P.kwargs) -> EvalTask: - # The one sanctioned upward import: eval sits above environment and - # agents and imports both; neither imports eval. Calling a declaration - # is where env hands the row to eval, and the import stays local to - # break the load-time cycle. Don't add more edges like this. + # Avoid the environment -> eval import cycle. from hud.eval.task import Task bound = self.sig.bind(*args, **kwargs) - return Task(env=self.env.name, id=self.id, args=dict(bound.arguments)) + task = Task(env=self.env.name, id=self.id, args=dict(bound.arguments)) + task._env = self.env + return task class Environment: diff --git a/hud/environment/tests/conftest.py b/hud/environment/tests/conftest.py index 2862d10c6..bb52f5b22 100644 --- a/hud/environment/tests/conftest.py +++ b/hud/environment/tests/conftest.py @@ -1,9 +1,8 @@ """Harnesses for protocol-level environment tests. Inline-defined envs have no source file to ``spawn``, so :func:`served` drives -the connect path against a loopback substrate served by this process (the -same ``_local`` serving ``AgentTool`` adapts inside a placed substrate). This -is a test harness, not an engine placement. +the connect path against a loopback substrate served by this process. This is +a test harness, not an engine placement. """ from __future__ import annotations @@ -12,7 +11,7 @@ from typing import TYPE_CHECKING from hud.clients import connect -from hud.eval.runtime import _local +from hud.eval import LocalRuntime, Task if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -24,5 +23,6 @@ @asynccontextmanager async def served(env: Environment) -> AsyncIterator[HudClient]: """Serve *env* on a loopback substrate and yield a connected client.""" - async with _local(env) as runtime, connect(runtime) as client: + task = Task(env=env.name, id="test") + async with LocalRuntime(env)(task) as runtime, connect(runtime) as client: yield client diff --git a/hud/environment/tests/test_capability_backing.py b/hud/environment/tests/test_capability_backing.py index 22701a7f8..94621a61c 100644 --- a/hud/environment/tests/test_capability_backing.py +++ b/hud/environment/tests/test_capability_backing.py @@ -103,14 +103,14 @@ async def test_workspace_file_tracking_can_be_opted_out( async def test_reconnecting_reuses_the_same_workspace(tmp_path: Path) -> None: from hud.clients import connect - from hud.eval.runtime import _local + from hud.eval import LocalRuntime, Task env = Environment("ws-env") env.workspace(tmp_path / "root") # Client-side urls are per-connection (forwarded); the daemon's identity # is its host key, which only stays stable if the workspace is reused. - async with _local(env) as runtime: + async with LocalRuntime(env)(Task(env=env.name, id="test")) as runtime: async with connect(runtime) as client: first = client.binding("shell").params["host_pubkey"] async with connect(runtime) as client: diff --git a/hud/environment/tests/test_sessions.py b/hud/environment/tests/test_sessions.py index 1145b462e..e924ac35b 100644 --- a/hud/environment/tests/test_sessions.py +++ b/hud/environment/tests/test_sessions.py @@ -13,7 +13,9 @@ from hud.clients import HudProtocolError, connect from hud.environment import Environment -from hud.eval.runtime import _local +from hud.eval import LocalRuntime, Task + +_SESSION = Task(env="sessions", id="echo") def _env() -> Environment: @@ -28,7 +30,11 @@ async def echo(tag: str): async def test_concurrent_sessions_grade_their_own_tasks() -> None: - async with _local(_env()) as runtime, connect(runtime) as a, connect(runtime) as b: + async with ( + LocalRuntime(_env())(_SESSION) as runtime, + connect(runtime) as a, + connect(runtime) as b, + ): await a.start_task("echo", {"tag": "a"}) await b.start_task("echo", {"tag": "b"}) # must not disturb a's task assert (await a.grade({"answer": "x"}))["tag"] == "a" @@ -36,7 +42,11 @@ async def test_concurrent_sessions_grade_their_own_tasks() -> None: async def test_restart_replaces_only_the_sessions_own_task() -> None: - async with _local(_env()) as runtime, connect(runtime) as a, connect(runtime) as b: + async with ( + LocalRuntime(_env())(_SESSION) as runtime, + connect(runtime) as a, + connect(runtime) as b, + ): await b.start_task("echo", {"tag": "b"}) await a.start_task("echo", {"tag": "first"}) await a.start_task("echo", {"tag": "second"}) @@ -45,7 +55,7 @@ async def test_restart_replaces_only_the_sessions_own_task() -> None: async def test_disconnect_parks_the_task_for_a_later_connection() -> None: - async with _local(_env()) as runtime: + async with LocalRuntime(_env())(_SESSION) as runtime: async with connect(runtime) as first: await first.start_task("echo", {"tag": "parked"}) async with connect(runtime) as later: @@ -53,7 +63,7 @@ async def test_disconnect_parks_the_task_for_a_later_connection() -> None: async def test_grade_with_multiple_parked_sessions_errors_loudly() -> None: - async with _local(_env()) as runtime: + async with LocalRuntime(_env())(_SESSION) as runtime: for tag in ("one", "two"): async with connect(runtime) as client: await client.start_task("echo", {"tag": tag}) @@ -63,7 +73,7 @@ async def test_grade_with_multiple_parked_sessions_errors_loudly() -> None: async def test_hello_resumes_a_parked_session_by_id() -> None: - async with _local(_env()) as runtime: + async with LocalRuntime(_env())(_SESSION) as runtime: ids: dict[str, str] = {} for tag in ("one", "two"): async with connect(runtime) as client: @@ -76,13 +86,17 @@ async def test_hello_resumes_a_parked_session_by_id() -> None: async def test_hello_with_an_unknown_session_id_errors() -> None: - async with _local(_env()) as runtime, connect(runtime) as client: + async with LocalRuntime(_env())(_SESSION) as runtime, connect(runtime) as client: with pytest.raises(HudProtocolError, match="unknown session"): await client.hello(session_id="sess-nope") async def test_hello_cannot_resume_a_live_session() -> None: - async with _local(_env()) as runtime, connect(runtime) as a, connect(runtime) as b: + async with ( + LocalRuntime(_env())(_SESSION) as runtime, + connect(runtime) as a, + connect(runtime) as b, + ): assert a.manifest is not None await a.start_task("echo", {"tag": "a"}) with pytest.raises(HudProtocolError, match="live connection"): diff --git a/hud/eval/__init__.py b/hud/eval/__init__.py index f1fd9bdc2..0bce06d61 100644 --- a/hud/eval/__init__.py +++ b/hud/eval/__init__.py @@ -34,6 +34,7 @@ from .job import Job from .run import Grade, Run, rollout from .runtime import ( + ComposeProject, DaytonaRuntime, DockerRuntime, HostedRuntime, @@ -56,6 +57,7 @@ __all__ = [ "Chat", + "ComposeProject", "DaytonaRuntime", "DockerRuntime", "Grade", diff --git a/hud/eval/run.py b/hud/eval/run.py index dca443403..3603f37a7 100644 --- a/hud/eval/run.py +++ b/hud/eval/run.py @@ -24,11 +24,9 @@ import asyncio import contextlib import logging -import tempfile import traceback import uuid from dataclasses import dataclass, field -from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, Self, cast import mcp.types as mcp_types @@ -49,6 +47,7 @@ from hud.clients.client import HudClient from .runtime import Provider + from .runtime.core import RuntimeSession from .task import Task logger = logging.getLogger("hud.eval.run") @@ -106,6 +105,14 @@ class Grade: info: dict[str, Any] = field(default_factory=dict) is_error: bool = False raw: dict[str, Any] = field(default_factory=dict) + evaluation: dict[str, Any] = field(init=False, repr=False) + + def __post_init__(self) -> None: + self.evaluation = dict(self.raw) + if isinstance(subscores := self.evaluation.get("subscores"), list): + self.evaluation["subscores"] = [ + SubScore.model_validate(subscore).to_summary() for subscore in subscores + ] @classmethod def from_dict(cls, data: dict[str, Any]) -> Grade: @@ -193,12 +200,7 @@ def reward(self) -> float: @property def evaluation(self) -> dict[str, Any]: """A persistence-safe view of the task's evaluation result.""" - evaluation = dict(self.grade.raw) - if isinstance(subscores := evaluation.get("subscores"), list): - evaluation["subscores"] = [ - SubScore.model_validate(subscore).to_summary() for subscore in subscores - ] - return evaluation + return dict(self.grade.evaluation) @property def trace_id(self) -> str | None: @@ -451,91 +453,105 @@ async def rollout( async def _drive() -> None: nonlocal client, run, _phase actor_result: dict[str, Any] = {} + actor_session: RuntimeSession | None = None verifier = task.verifier shared_verifier = ( verifier is not None and verifier.env == task.env and verifier.runtime_config is None ) - transfer_handoff = verifier is not None and verifier.requires_handoff - with tempfile.TemporaryDirectory(prefix="hud-handoff-") as directory: - handoff = Path(directory) / "handoff.tar.gz" - async with runtime(task) as addr: - if transfer_handoff and not shared_verifier and addr.handoff is None: - raise ValueError("the actor runtime cannot transfer verifier handoff files") - _phase = "starting task" - async with connect(addr) as actor_client: - client = actor_client - live = Run( - actor_client, - task.id, - task.args, - best_effort_grade=task.verifier is not None, - ) - live._runtime = addr.url # the placement record for the receipt - async with live: # start on enter; complete on exit - run = live # bound only once live: an earlier failure synthesizes - _phase = "agent loop" - try: - async with file_tracking_observer(actor_client): - if agent_timeout is None: - await agent(run) - else: - deadline = asyncio.timeout(agent_timeout) - try: - async with deadline: - await agent(run) - except TimeoutError: - if not deadline.expired(): - raise - detail = f"agent timed out after {agent_timeout:g}s" - logger.warning(detail) - run.trace.status = "error" - run.trace.stop_reason = "timeout" - run.record(Step(source="system", error=detail)) - except Exception as exc: - if task.verifier is None: - raise - detail = "".join(traceback.format_exception_only(exc)).strip() - logger.warning("rollout failed mid-run (%s): %s", _phase, detail) - run.trace.status = "error" - run.record(Step(source="system", error=f"[{_phase}] {detail}")) - _phase = "grading" - - if verifier is not None: - actor_result = live.grade.raw - # The verifier is authoritative. Once its phase begins, - # an actor-side grade must not survive a verifier failure. - live.grade = Grade() - if shared_verifier: - assert verifier is not None - _phase = "verifying" - await _verify(live, actor_client, verifier, actor_result) - _phase = "cleanup" - return - - _phase = "actor cleanup" - if transfer_handoff: - assert addr.handoff is not None - await addr.handoff.export_to(handoff) + async with contextlib.AsyncExitStack() as scope: + actor = contextlib.AsyncExitStack() + await actor.__aenter__() + + async def close_actor() -> None: + cleanup = asyncio.create_task(actor.aclose()) + cleanup.add_done_callback(_consume_task_result) + await asyncio.shield(cleanup) + + scope.push_async_callback(close_actor) + addr = await actor.enter_async_context(runtime(task)) + _phase = "starting task" + async with connect(addr) as actor_client: + client = actor_client + live = Run( + actor_client, + task.id, + task.args, + best_effort_grade=task.verifier is not None, + ) + live._runtime = addr.url # the placement record for the receipt + async with live: # start on enter; complete on exit + run = live # bound only once live: an earlier failure synthesizes + _phase = "agent loop" + try: + async with file_tracking_observer(actor_client): + if agent_timeout is None: + await agent(run) + else: + deadline = asyncio.timeout(agent_timeout) + try: + async with deadline: + await agent(run) + except TimeoutError: + if not deadline.expired(): + raise + detail = f"agent timed out after {agent_timeout:g}s" + logger.warning(detail) + run.trace.status = "error" + run.trace.stop_reason = "timeout" + run.record(Step(source="system", error=detail)) + except Exception as exc: + if task.verifier is None: + raise + detail = "".join(traceback.format_exception_only(exc)).strip() + logger.warning("rollout failed mid-run (%s): %s", _phase, detail) + run.trace.status = "error" + run.record(Step(source="system", error=f"[{_phase}] {detail}")) + _phase = "grading" + + if verifier is not None: + actor_result = live.grade.raw + assert actor_client.manifest is not None + actor_session = addr.session(actor_client.manifest.session_id) + # The verifier is authoritative. Once its phase begins, + # an actor-side grade must not survive a verifier failure. + live.grade = Grade() + if shared_verifier: + assert verifier is not None + _phase = "verifying" + await _verify(live, actor_client, verifier, actor_result) + _phase = "cleanup" + return if rollout_expired: return - if verifier is not None: + if verifier is None: + _phase = "actor cleanup" + await actor.aclose() + _phase = "cleanup" + return + + assert actor_session is not None + _phase = "snapshotting actor" + async with actor_session.snapshot() as archive: + _phase = "actor cleanup" + await actor.aclose() + client = None + if rollout_expired: + return _phase = "provisioning verifier" - async with runtime(verifier) as verifier_addr: - if transfer_handoff and verifier_addr.handoff is None: - raise ValueError( - "the verifier runtime cannot receive actor handoff files" - ) - if transfer_handoff: - assert verifier_addr.handoff is not None - await verifier_addr.handoff.import_from(handoff) - async with connect(verifier_addr) as verifier_client: - client = verifier_client - _phase = "verifying" - await _verify(live, verifier_client, verifier, actor_result) - _phase = "cleanup" + verifier_addr = await scope.enter_async_context(runtime(verifier)) + verifier_client = await scope.enter_async_context(connect(verifier_addr)) + if archive is not None: + assert verifier_client.manifest is not None + await verifier_addr.session(verifier_client.manifest.session_id).restore( + archive + ) + client = verifier_client + _phase = "verifying" + await _verify(live, verifier_client, verifier, actor_result) + _phase = "cleanup" driver = asyncio.create_task(_drive()) try: diff --git a/hud/eval/runtime/__init__.py b/hud/eval/runtime/__init__.py index 3ec438a2d..35a86ef66 100644 --- a/hud/eval/runtime/__init__.py +++ b/hud/eval/runtime/__init__.py @@ -1,36 +1,26 @@ """Runtime placement and provider configuration.""" +from .compose import ComposeProject from .core import ( - LocalRuntime, Provider, Runtime, RuntimeConfig, RuntimeGPU, RuntimeLimits, RuntimeResources, + RuntimeSession, RuntimeTPU, Shared, - SubprocessRuntime, -) -from .core import ( - _declared_env as _declared_env, -) -from .core import ( - _declared_names as _declared_names, -) -from .core import ( - _local as _local, ) from .daytona import DaytonaRuntime from .docker import DockerRuntime from .hosted import HostedRuntime from .hud import HUDRuntime -from .hud import ( - _splice_websocket as _splice_websocket, -) +from .local import LocalRuntime, SubprocessRuntime from .modal import ModalRuntime __all__ = [ + "ComposeProject", "DaytonaRuntime", "DockerRuntime", "HUDRuntime", @@ -43,6 +33,7 @@ "RuntimeGPU", "RuntimeLimits", "RuntimeResources", + "RuntimeSession", "RuntimeTPU", "Shared", "SubprocessRuntime", diff --git a/hud/eval/runtime/compose.py b/hud/eval/runtime/compose.py index 4950fe14d..3ce96b443 100644 --- a/hud/eval/runtime/compose.py +++ b/hud/eval/runtime/compose.py @@ -4,6 +4,7 @@ import contextlib import json +import os import posixpath import re import shlex @@ -15,7 +16,16 @@ import yaml from dotenv import dotenv_values -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SerializationInfo, + ValidationInfo, + field_serializer, + field_validator, + model_validator, +) from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode if TYPE_CHECKING: @@ -39,7 +49,7 @@ def _interpolate_compose_value(value: str, environment: Mapping[str, str]) -> st break result.append(value[index:marker]) if marker + 1 >= len(value): - result.append("$") + result.append("$$") break following = value[marker + 1] if following == "$": @@ -67,7 +77,7 @@ def _interpolate_compose_value(value: str, environment: Mapping[str, str]) -> st continue match = _COMPOSE_VARIABLE.match(value, marker + 1) if match is None: - result.append("$") + result.append("$$") index = marker + 1 continue name = match.group() @@ -75,7 +85,7 @@ def _interpolate_compose_value(value: str, environment: Mapping[str, str]) -> st raise ComposeUnboundVariableError( f"Compose variable {name!r} is not set by the project .env" ) - result.append(environment[name]) + result.append(environment[name].replace("$", "$$")) index = match.end() return "".join(result) @@ -87,12 +97,13 @@ def _resolve_compose_variable(expression: str, environment: Mapping[str, str]) - name = match.group() suffix = expression[match.end() :] value = environment.get(name) + escaped = value.replace("$", "$$") if value is not None else "" if not suffix: if value is None: raise ComposeUnboundVariableError( f"Compose variable {name!r} is not set by the project .env" ) - return value + return escaped operator = next( (item for item in (":-", ":?", ":+", "-", "?", "+") if suffix.startswith(item)), None @@ -103,9 +114,9 @@ def _resolve_compose_variable(expression: str, environment: Mapping[str, str]) - is_set = value is not None is_nonempty = is_set and value != "" if operator == ":-": - return value if is_nonempty else _interpolate_compose_value(operand, environment) + return escaped if is_nonempty else _interpolate_compose_value(operand, environment) if operator == "-": - return value if is_set else _interpolate_compose_value(operand, environment) + return escaped if is_set else _interpolate_compose_value(operand, environment) if operator == ":+": return _interpolate_compose_value(operand, environment) if is_nonempty else "" if operator == "+": @@ -113,8 +124,7 @@ def _resolve_compose_variable(expression: str, environment: Mapping[str, str]) - if (operator == ":?" and not is_nonempty) or (operator == "?" and not is_set): detail = operand or f"Compose variable {name!r} is required" raise ComposeUnboundVariableError(detail) - assert value is not None - return value + return escaped def _interpolate_compose_node( @@ -131,8 +141,12 @@ def _interpolate_compose_node( elif isinstance(node, SequenceNode): for value in node.value: _interpolate_compose_node(value, environment, seen) - elif isinstance(node, ScalarNode) and node.tag == "tag:yaml.org,2002:str" and node.style != "'": - node.value = _interpolate_compose_value(node.value, environment) + elif isinstance(node, ScalarNode) and node.tag == "tag:yaml.org,2002:str": + node.value = ( + node.value.replace("$", "$$") + if node.style == "'" + else _interpolate_compose_value(node.value, environment) + ) class ComposeHealthcheck(BaseModel): @@ -409,54 +423,75 @@ class ComposeProjectRef(BaseModel): compose_path: str -@dataclass(frozen=True, slots=True) -class ComposeSource: - """One authored or platform-wire Compose runtime source.""" - - document: Path | ComposeConfig - project: Path | ComposeProjectRef | None = None - - def request_payload(self) -> dict[str, Any]: - if isinstance(self.document, Path): - document = ComposeConfig.from_file(self.document) - else: - document = self.document - payload: dict[str, Any] = { - "compose": document.model_dump(mode="json", exclude_none=True), - } - if isinstance(self.project, Path): - if not isinstance(self.document, Path): - raise ValueError("compose_project as a path requires compose as a path") - try: - compose_path = ( - self.document.resolve().relative_to(self.project.resolve()).as_posix() - ) - except ValueError: - raise ValueError("runtime_config.compose must be inside compose_project") from None - payload["compose_project"] = {"compose_path": compose_path} - elif self.project is not None: - payload["compose_project"] = self.project.model_dump(mode="json") - return payload - - def runnable_path(self, provider: str) -> Path: - if not isinstance(self.document, Path): - raise ValueError(f"{provider} requires runtime_config.compose as a local file path") - return self.document.resolve() - - @dataclass(frozen=True, slots=True) class ComposeLaunchFiles: compose: Path + project_directory: Path override: Path ports: Path archive: Path | None -@dataclass(frozen=True, slots=True) -class ComposeProject: - """A local Compose project staged with HUD's main-service overrides.""" +class ComposeProject(BaseModel): + """A Compose recipe and the project data it may need at runtime.""" - compose: Path + model_config = ConfigDict(extra="forbid", frozen=True) + + document: Path | ComposeConfig + root: Path | ComposeProjectRef | None = None + service_access: bool | None = None + + @field_validator("document", "root", mode="before") + @classmethod + def resolve_local_path(cls, value: Any, info: ValidationInfo) -> Any: + base = (info.context or {}).get("base_path") + if isinstance(value, str) and isinstance(base, Path): + return (base / value).resolve() + return value + + @model_validator(mode="after") + def validate_source(self) -> ComposeProject: + if self.root is None: + return self + if isinstance(self.root, Path) != isinstance(self.document, Path): + raise ValueError("Compose source and project root must use the same form") + if isinstance(self.root, Path): + assert isinstance(self.document, Path) + try: + self.document.resolve().relative_to(self.root.resolve()) + except ValueError: + raise ValueError("Compose file must be inside its project root") from None + return self + + @field_serializer("document", when_used="json") + def serialize_document( + self, + document: Path | ComposeConfig, + info: SerializationInfo, + ) -> dict[str, Any] | str: + base = (info.context or {}).get("base_path") + if isinstance(document, Path) and isinstance(base, Path): + return os.path.relpath(document.resolve(), base) + config = ComposeConfig.from_file(document) if isinstance(document, Path) else document + return config.model_dump(mode="json", exclude_none=True) + + @field_serializer("root", when_used="json") + def serialize_root( + self, + root: Path | ComposeProjectRef | None, + info: SerializationInfo, + ) -> dict[str, str] | str | None: + if root is None: + return None + if isinstance(root, ComposeProjectRef): + return root.model_dump(mode="json") + base = (info.context or {}).get("base_path") + if isinstance(base, Path): + return os.path.relpath(root.resolve(), base) + assert isinstance(self.document, Path) + return { + "compose_path": self.document.resolve().relative_to(root.resolve()).as_posix(), + } @contextlib.contextmanager def stage( @@ -472,6 +507,9 @@ def stage( gpu_count: int | None = None, archive: bool = False, ) -> Iterator[ComposeLaunchFiles]: + if not isinstance(self.document, Path): + raise ValueError("Compose project is not available on the local filesystem") + compose = self.document.resolve() main: dict[str, Any] = { "security_opt": [ f"seccomp={seccomp}", @@ -498,6 +536,13 @@ def stage( with tempfile.TemporaryDirectory(prefix="hud-compose-") as directory: root = Path(directory) + normalized = root / "compose.json" + normalized.write_text( + json.dumps( + ComposeConfig.from_file(compose).model_dump(mode="json", exclude_none=True) + ), + encoding="utf-8", + ) override = root / "override.json" override.write_text( json.dumps({"services": {"main": main}}), @@ -511,11 +556,21 @@ def stage( archive_path = None if archive: archive_path = root / "project.tar.gz" + project_root = ( + self.root.resolve() if isinstance(self.root, Path) else compose.parent + ) + compose_path = compose.relative_to(project_root).as_posix() + + def omit_authored_compose(info: tarfile.TarInfo) -> tarfile.TarInfo | None: + return None if info.name == compose_path else info + with tarfile.open(archive_path, "w:gz") as tar: - for entry in self.compose.parent.iterdir(): - tar.add(entry, arcname=entry.name) + for entry in project_root.iterdir(): + tar.add(entry, arcname=entry.name, filter=omit_authored_compose) + tar.add(normalized, arcname=compose_path) yield ComposeLaunchFiles( - compose=self.compose, + compose=normalized, + project_directory=compose.parent, override=override, ports=ports, archive=archive_path, @@ -529,5 +584,4 @@ def stage( "ComposeProject", "ComposeProjectRef", "ComposeService", - "ComposeSource", ] diff --git a/hud/eval/runtime/core.py b/hud/eval/runtime/core.py index e95c89a9a..0302c5bd2 100644 --- a/hud/eval/runtime/core.py +++ b/hud/eval/runtime/core.py @@ -1,31 +1,24 @@ -"""Runtime configuration, addresses, sharing, and local placement.""" +"""Shared runtime configuration and placement contracts.""" from __future__ import annotations import asyncio import contextlib -import logging -import sys -from collections import deque +import json from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext from dataclasses import dataclass, field -from pathlib import Path from typing import TYPE_CHECKING, Any, Protocol, Self from pydantic import BaseModel, ConfigDict, Field, model_validator -from hud.utils.process import ProcessGroup, create_process_group_exec - -from .compose import ComposeConfig, ComposeProjectRef, ComposeSource +from .compose import ComposeProject if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Iterator + from collections.abc import AsyncIterator + from pathlib import Path - from hud.environment.env import Environment from hud.eval.task import Task -logger = logging.getLogger("hud.eval.runtime") - class RuntimeGPU(BaseModel): """Requested GPU resources, provider-neutral where possible.""" @@ -86,18 +79,14 @@ class RuntimeConfig(BaseModel): ``Task.runtime_config`` is requested construction input. ``Runtime.config`` is the effective config used to construct a runtime. - ``compose`` and ``compose_project`` are authored as local paths; platform - task records carry them as the serialized compose document and a - :class:`ComposeProjectRef`. Both forms validate; only the path form is - runnable by local providers. + A Compose project uses local paths while authored and serialized project + data in platform records. """ model_config = ConfigDict(extra="forbid") image: str | None = Field(default=None, min_length=1) - compose: Path | ComposeConfig | None = None - compose_project: Path | ComposeProjectRef | None = None - compose_service_access: bool | None = None + compose: ComposeProject | None = None resources: RuntimeResources | None = None limits: RuntimeLimits | None = None @@ -105,10 +94,6 @@ class RuntimeConfig(BaseModel): def validate_source(self) -> Self: if self.image is not None and self.compose is not None: raise ValueError("runtime_config accepts either image or compose, not both") - if self.compose_project is not None and self.compose is None: - raise ValueError("compose_project requires runtime_config.compose") - if self.compose_service_access and self.compose is None: - raise ValueError("compose_service_access requires runtime_config.compose") return self def with_overrides(self, override: RuntimeConfig | None) -> RuntimeConfig: @@ -118,26 +103,10 @@ def with_overrides(self, override: RuntimeConfig | None) -> RuntimeConfig: changes = override.model_dump(exclude_unset=True) if override.image is not None: config["compose"] = None - config["compose_project"] = None - config["compose_service_access"] = None elif override.compose is not None: config["image"] = None - config["compose_project"] = None return RuntimeConfig.model_validate(config | changes) - def request_payload(self) -> dict[str, Any]: - payload = self.model_dump(mode="json", exclude_unset=True) - source = self.compose_source() - if source is not None: - payload.update(source.request_payload()) - return payload - - def compose_source(self) -> ComposeSource | None: - """The authored or wire-form Compose source, when configured.""" - if self.compose is None: - return None - return ComposeSource(self.compose, self.compose_project) - class Provider(Protocol): """Server placement: called with the task row being placed, acquire one @@ -153,12 +122,29 @@ class Provider(Protocol): def __call__(self, task: Task, /) -> AbstractAsyncContextManager[Runtime]: ... -class HandoffEndpoint(Protocol): - """Provider-owned transfer of the runtime handoff namespace.""" +@dataclass(frozen=True) +class RuntimeSession: + """One control session in a provisioned runtime.""" - async def export_to(self, destination: Path) -> None: ... + session_id: str - async def import_from(self, source: Path) -> None: ... + def __post_init__(self) -> None: + if ( + not self.session_id + or self.session_id in {".", ".."} + or "/" in self.session_id + or "\\" in self.session_id + ): + raise ValueError("runtime session id must be a single path component") + + @asynccontextmanager + async def snapshot(self) -> AsyncIterator[Path | None]: + """Yield a portable archive of this session's files when present.""" + yield None + + async def restore(self, source: Path) -> None: + """Restore a portable session archive into this session.""" + return @dataclass(frozen=True) @@ -177,18 +163,20 @@ class Runtime: url: str params: dict[str, Any] = field(default_factory=dict) config: RuntimeConfig | None = None - handoff: HandoffEndpoint | None = field(default=None, repr=False, compare=False) def __call__(self, task: Task) -> AbstractAsyncContextManager[Runtime]: return nullcontext(self) + def session(self, session_id: str) -> RuntimeSession: + """Bind a negotiated control session to this runtime.""" + return RuntimeSession(session_id) + class Shared: - """Lease provider: at most ``width`` concurrent rollouts share one substrate. + """Lease provider: at most ``width`` rollouts share each task placement. - The substrate boots lazily on the first lease and lives for the enclosing - ``async with`` scope — one boot however many rollouts flow through, torn - down deterministically at scope exit. ``width`` is the substrate's real + Each environment and runtime configuration boots lazily on its first lease + and lives for the enclosing ``async with`` scope. ``width`` is a substrate's capacity (e.g. a vectorized sim's slot count): lease ``width + 1`` waits for a slot instead of erroring, so the scheduler needs no pairing — ``group`` and ``max_concurrent`` keep their ordinary meanings. @@ -206,9 +194,9 @@ def __init__(self, inner: Provider, *, width: int) -> None: raise ValueError("Shared width must be >= 1") self.inner = inner self.width = width - self._sem = asyncio.Semaphore(width) self._boot = asyncio.Lock() - self._addr: Runtime | None = None + self._semaphores: dict[tuple[str, str], asyncio.Semaphore] = {} + self._addresses: dict[tuple[str, str], Runtime] = {} self._stack: contextlib.AsyncExitStack | None = None self._opens = 0 @@ -219,7 +207,9 @@ async def __aenter__(self) -> Self: async def __aexit__(self, *exc: object) -> None: self._opens -= 1 if self._opens == 0 and self._stack is not None: - stack, self._stack, self._addr = self._stack, None, None + stack, self._stack = self._stack, None + self._addresses.clear() + self._semaphores.clear() await stack.aclose() @asynccontextmanager @@ -229,359 +219,24 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: "Shared substrates outlive single rollouts; lease inside the scope " "(Taskset.run opens it for you, or wrap calls in `async with Shared(...)`)" ) - async with self._sem: + config = ( + json.dumps( + task.runtime_config.model_dump(mode="python", exclude_unset=True), + sort_keys=True, + default=str, + ) + if task.runtime_config is not None + else "" + ) + key = (task.env, config) + semaphore = self._semaphores.setdefault(key, asyncio.Semaphore(self.width)) + async with semaphore: async with self._boot: - if self._addr is None: + if key not in self._addresses: # First leaseholder boots. A failed boot fails only its own # rollout (nothing entered the stack); the next lease retries. - stack = contextlib.AsyncExitStack() - self._addr = await stack.enter_async_context(self.inner(task)) - self._stack = stack - addr = self._addr + if self._stack is None: + self._stack = contextlib.AsyncExitStack() + self._addresses[key] = await self._stack.enter_async_context(self.inner(task)) + addr = self._addresses[key] yield addr - - -class LocalRuntime: - """The local provider: serve a fresh env per rollout, in this process. - - *source* points at the env in whatever form you have: - - - a ``.py`` file or directory — imported fresh per acquisition (sibling - imports resolve); *env* pins one name when several are declared, - defaulting to the placed task's env - - a live :class:`~hud.environment.Environment` — shorthand for its - declaring file; the instance itself is never served - - a ``(task) -> Environment`` callable — called per acquisition with the - placed row - - :: - - runtime = LocalRuntime("env.py") - runtime = LocalRuntime(env) - runtime = LocalRuntime(lambda task: build_env(task.env)) - - ``ready_timeout`` bounds ``@env.initialize`` startup. Freshness covers - the env's own source; modules it imports are cached as usual and shared - across rollouts. Hooks share this process's event loop, so blocking env - code stalls concurrent rollouts — use :class:`SubprocessRuntime` or - :class:`DockerRuntime` for process isolation, and ``Runtime(url)`` to - attach to a substrate served elsewhere. - """ - - def __init__( - self, - source: str | Path | Environment | Callable[[Task], Environment], - *, - env: str | None = None, - ready_timeout: float = 120.0, - ) -> None: - from hud.environment.env import Environment as _Environment - - self.ready_timeout = ready_timeout - # A live instance may have been mutated since its module was imported; - # verify the fresh copy still declares its templates, so drift fails - # at acquisition with the cause named instead of "unknown task" later. - expected_templates: frozenset[str] = frozenset() - if isinstance(source, _Environment): - file = _declaring_file(source, env or source.name) - if file is None: - raise TypeError( - f"LocalRuntime: env {source.name!r} is not rebuilt by importing " - "any file this process has loaded (constructed in a function or " - "notebook cell, or declared inside a package using relative " - "imports); pass its constructor instead: " - "LocalRuntime(lambda task: )" - ) - expected_templates = frozenset(source.tasks) - source, env = file, env or source.name - self._source_dir: Path | None = None - if isinstance(source, (str, Path)): - path, pinned = Path(source).resolve(), env - self._source_dir = path if path.is_dir() else path.parent - from hud.environment import load_environment - - def _load(task: Task) -> _Environment: - loaded = load_environment(path, name=pinned or task.env) - missing = expected_templates - loaded.tasks.keys() - if missing: - raise ValueError( - f"env {loaded.name!r} loaded from {path} lacks template(s) " - f"{sorted(missing)} present on the live instance — it was " - "modified after import; pass a constructor instead: " - "LocalRuntime(lambda task: )" - ) - return loaded - - self._build: Callable[[Task], _Environment] = _load - elif callable(source): - if env is not None: - raise TypeError("LocalRuntime: env= applies only to source paths") - self._build = source - else: - raise TypeError( - f"LocalRuntime: expected a source path, a live Environment, or a " - f"(task) -> Environment constructor; got {source!r}" - ) - - @asynccontextmanager - async def __call__(self, task: Task) -> AsyncIterator[Runtime]: - from hud.environment.env import Environment as _Environment - - if task.runtime_config is not None: - raise ValueError("LocalRuntime does not support task runtime_config") - # The source dir stays importable for the whole acquisition, not just - # the initial import, so a template can lazily import a sibling - # module at run time (as it could under the child-process runtime). - # Always insert-and-remove one entry: balanced under concurrency. - if self._source_dir is not None: - sys.path.insert(0, str(self._source_dir)) - try: - try: - env = self._build(task) - except RuntimeError as e: - # The source ran an event loop at import — usually an unguarded - # top-level run call; name the actual mistake. - if "running event loop" not in str(e): - raise - raise RuntimeError( - "the env source ran async code while being imported to place a " - 'rollout — guard top-level run calls with `if __name__ == "__main__":`' - ) from e - if not isinstance(env, _Environment): - raise TypeError(f"LocalRuntime: constructor returned {env!r}, not an Environment") - async with _local(env, ready_timeout=self.ready_timeout) as runtime: - yield runtime - finally: - if self._source_dir is not None: - with contextlib.suppress(ValueError): - sys.path.remove(str(self._source_dir)) - - -def _live_envs() -> Iterator[tuple[Environment, str]]: - """Envs declared in loaded, file-backed modules' globals, with their files. - - The in-memory counterpart of scanning ``.py`` sources on disk - (:func:`~hud.environment.load_environment`): an env found here can be - served fresh by re-importing its file. Envs in modules without a file - (a notebook ``__main__``) are not yielded — re-import could not - reconstruct them. - """ - from hud.environment.env import Environment as _Environment - - for module in list(sys.modules.values()): - module_file = getattr(module, "__file__", None) - module_vars = getattr(module, "__dict__", None) - if not module_file or not isinstance(module_vars, dict): - continue - for value in list(module_vars.values()): - if isinstance(value, _Environment): - yield value, module_file - - -def _declaring_file(env: Environment, name: str) -> Path | None: - """A file whose fresh import re-declares *env*, else None. - - Candidate files hold the instance in their module globals, but a holder - may be a re-exporter (``from .env import env`` in a package - ``__init__``, a tasks file re-exporting its env): validate each by - loading it fresh — a declarer yields a *new* instance under *name*, a - re-exporter yields the same live one (or fails to import standalone). - ``__init__.py`` holders are tried last. - """ - from hud.environment import load_environment - - candidates = dict.fromkeys(Path(file) for live, file in _live_envs() if live is env) - for file in sorted(candidates, key=lambda f: f.name == "__init__.py"): - try: - probe = load_environment(file, name=name) - except Exception as e: - logger.debug("candidate %s does not rebuild env %r: %s", file, name, e) - continue - if probe is not env: - return file - return None - - -def _declared_env(name: str) -> Environment | None: - """The one live env named *name*, else None; two distinct ones raise. - - The same instance re-exported across modules is one match; distinct envs - claiming one name are ambiguous. - """ - matches = {id(env): env for env, _ in _live_envs() if env.name == name} - if len(matches) > 1: - files = sorted({file for env, file in _live_envs() if env.name == name}) - raise ValueError( - f"env name {name!r} is declared by multiple live environments " - f"({', '.join(files)}); pass runtime= explicitly — the exact " - "instance disambiguates: runtime=LocalRuntime(env)" - ) - return next(iter(matches.values()), None) - - -def _declared_names(source: Path) -> set[str]: - """Env names a ``.py`` source (file or directory) itself declares. - - A fresh execution of the source yields *new* instances for envs it - declares; an env it merely imports is the already-live one and does not - count — importing the source again could not rebuild it. - """ - from hud.environment.env import Environment as _Environment - from hud.utils.modules import iter_modules - - live = {id(env) for env, _ in _live_envs()} - return { - value.name - for module in iter_modules(source) - for value in vars(module).values() - if isinstance(value, _Environment) and id(value) not in live - } - - -class SubprocessRuntime: - """The child-process provider: serve the placed row's env from *path*. - - Each acquisition runs ``python -m hud.environment.server --env - name`` — the same serving entry point a container CMD runs — on an - ephemeral loopback port, yields its :class:`Runtime`, and terminates the - child on exit. *path* is a ``.py`` file or a directory of them. The served - env is the placed task's ``env`` name (so a mixed-env taskset works - against one source), unless *env* pins one explicitly; placing a row whose - env the source does not define fails loudly in the child. - - The child's working directory is the source's directory, so sibling - imports and relative data paths resolve; ``@env.initialize`` daemons start - in the child and die with it. Because the source is re-imported in the - child, a script spawning itself (``SubprocessRuntime(__file__)``) must keep - top-level run calls under ``if __name__ == "__main__":``. - """ - - def __init__( - self, - path: str | Path, - *, - env: str | None = None, - ready_timeout: float = 120.0, - ) -> None: - self.source = Path(path).resolve() - self.env = env - self.ready_timeout = ready_timeout - - @asynccontextmanager - async def __call__(self, task: Task) -> AsyncIterator[Runtime]: - if task.runtime_config is not None: - raise ValueError("SubprocessRuntime does not support task runtime_config") - if not self.source.exists(): - raise FileNotFoundError(f"SubprocessRuntime: source not found: {self.source}") - cmd = [sys.executable, "-m", "hud.environment.server", str(self.source)] - cmd += ["--env", self.env or task.env] - proc = await create_process_group_exec( - *cmd, - term_timeout=10.0, - stdout=asyncio.subprocess.PIPE, - # Capture stderr (don't inherit it): under concurrent rollouts an - # inherited fd interleaves every child's output unattributably, so a - # crash-before-serving leaves no traceable diagnostic. We keep a - # bounded tail and attach it to the failure below. - stderr=asyncio.subprocess.PIPE, - cwd=self.source if self.source.is_dir() else self.source.parent, - ) - assert proc.stderr is not None - # Drain stderr into a bounded tail from the start: it never blocks on a - # full pipe, and the last lines survive if the child dies early. - stderr_tail: deque[str] = deque(maxlen=50) - capture = asyncio.create_task(_capture(proc.stderr, stderr_tail)) - try: - assert proc.stdout is not None - port = await asyncio.wait_for(_read_port(proc.stdout), self.ready_timeout) - if port is None: - raise RuntimeError(await _exit_detail(proc, self.source, capture, stderr_tail)) - drain = asyncio.create_task(_drain(proc.stdout)) - try: - yield Runtime(f"tcp://127.0.0.1:{port}") - finally: - drain.cancel() - with contextlib.suppress(asyncio.CancelledError): - await drain - finally: - capture.cancel() - with contextlib.suppress(asyncio.CancelledError): - await capture - await proc.terminate() - - -@asynccontextmanager -async def _local(env: Environment, *, ready_timeout: float | None = None) -> AsyncIterator[Runtime]: - """Substrate-side serving: a live env owned by *this* process, as a runtime. - - One env lifecycle (start → serve → stop) around one bound control - channel; ``ready_timeout`` bounds ``env.start()`` (initialize - hooks/daemons). ``LocalRuntime`` enters this per acquisition with the - fresh env it built; test harnesses enter it directly with a live one. - """ - from hud.environment.server import _shutdown, bind - - # start() inside the try: a failed or timed-out initialize hook still gets - # its already-started daemons torn down by stop() (best-effort per hook). - try: - started = env.start() - await (asyncio.wait_for(started, ready_timeout) if ready_timeout is not None else started) - server = await bind(env, "127.0.0.1", 0) - host, port = server.sockets[0].getsockname()[:2] - serve_task = asyncio.create_task(server.serve_forever()) - try: - yield Runtime(f"tcp://{host}:{port}") - finally: - serve_task.cancel() - await _shutdown(server) - with contextlib.suppress(asyncio.CancelledError): - await serve_task - finally: - await env.stop() - - -async def _read_port(stdout: asyncio.StreamReader) -> int | None: - """Read the child's stdout until it announces its port; ``None`` if stdout - hits EOF first (the child exited before serving — caller builds the error).""" - # Imported lazily: a module-level import would pre-load hud.environment.server - # in every `python -m hud.environment.server` child, tripping runpy's - # found-in-sys.modules RuntimeWarning on each spawned rollout. - from hud.environment.server import PORT_ANNOUNCEMENT - - while True: - line = await stdout.readline() - if not line: - return None - text = line.decode("utf-8", "replace").strip() - if text.startswith(PORT_ANNOUNCEMENT): - return int(text.removeprefix(PORT_ANNOUNCEMENT)) - - -async def _exit_detail( - proc: ProcessGroup, - source: Path, - capture: asyncio.Task[None], - stderr_tail: deque[str], -) -> str: - """Message for a child that exited before serving, with its captured stderr - tail. The child is gone, so its stderr is at EOF — let the capture finish so - the traceback it wrote on the way out is included, not raced past.""" - code = await proc.wait() - with contextlib.suppress(TimeoutError): - await asyncio.wait_for(asyncio.shield(capture), 2.0) - tail = "\n".join(stderr_tail).strip() - detail = f":\n{tail}" if tail else " (no stderr captured)" - return f"spawned env exited with code {code} before serving (source: {source}){detail}" - - -async def _capture(stream: asyncio.StreamReader, sink: deque[str]) -> None: - """Drain a child stream into a bounded tail so it never blocks on a full pipe - and its last lines survive for diagnostics.""" - while line := await stream.readline(): - sink.append(line.decode("utf-8", "replace").rstrip()) - - -async def _drain(stream: asyncio.StreamReader) -> None: - """Keep consuming the child's stdout so it never blocks on a full pipe.""" - while await stream.read(65536): - pass diff --git a/hud/eval/runtime/daytona.py b/hud/eval/runtime/daytona.py index 33f880795..4f277eca1 100644 --- a/hud/eval/runtime/daytona.py +++ b/hud/eval/runtime/daytona.py @@ -3,159 +3,23 @@ from __future__ import annotations import asyncio -import importlib import logging -from contextlib import AbstractAsyncContextManager, asynccontextmanager -from typing import TYPE_CHECKING, Any, Protocol, cast +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any from .core import Runtime, RuntimeConfig if TYPE_CHECKING: - from collections.abc import AsyncIterator, Awaitable, Sequence - from pathlib import Path + from collections.abc import AsyncIterator + + from daytona import Image as DaytonaImage + from daytona.common.snapshot import Snapshot as DaytonaSnapshot from hud.eval.task import Task logger = logging.getLogger("hud.eval.runtime") -class DaytonaContextEntry(Protocol): - @property - def source_path(self) -> str | Path: ... - - @property - def archive_path(self) -> str | Path: ... - - -class DaytonaImage(Protocol): - @property - def _context_list(self) -> Sequence[DaytonaContextEntry]: ... - - def dockerfile(self) -> str: ... - - -class _DaytonaBuildInfo(Protocol): - dockerfile_content: str - context_hashes: Sequence[str] | None - - -class DaytonaSnapshot(Protocol): - image_name: str - build_info: _DaytonaBuildInfo | None - - -class ObjectStorage(Protocol): - async def _compute_hash_for_path_md5( - self, - source_path: str | Path, - archive_path: str | Path, - ) -> str: ... - - -class ObjectStorageModule(Protocol): - AsyncObjectStorage: type[ObjectStorage] - - -class _DaytonaResources(Protocol): - cpu: int | None - memory: int | None - gpu: int | None - gpu_type: Sequence[object] | None - - -class _DaytonaSessionCommand(Protocol): - cmd_id: str - - -class _DaytonaSessionLogs(Protocol): - stderr: str | None - output: str | None - stdout: str | None - - -class _DaytonaProcess(Protocol): - async def create_session(self, session: str) -> object: ... - - async def execute_session_command( - self, - session: str, - request: object, - ) -> _DaytonaSessionCommand: ... - - async def get_session_command_logs( - self, - session: str, - command_id: str, - ) -> _DaytonaSessionLogs: ... - - -class _DaytonaSshAccess(Protocol): - token: str - - -class _DaytonaSandbox(Protocol): - id: str - process: _DaytonaProcess - - async def create_ssh_access(self, *, expires_in_minutes: int) -> _DaytonaSshAccess: ... - - -class _DaytonaSnapshotClient(Protocol): - async def get(self, name: str) -> DaytonaSnapshot: ... - - async def delete(self, snapshot: DaytonaSnapshot) -> object: ... - - async def create(self, params: object) -> object: ... - - -class _DaytonaCreate(Protocol): - def __call__( - self, - params: object, - *, - timeout: int, - ) -> Awaitable[_DaytonaSandbox]: ... - - -class _DaytonaClient(Protocol): - snapshot: _DaytonaSnapshotClient - create: _DaytonaCreate - - async def delete(self, sandbox: _DaytonaSandbox) -> object: ... - - -class _DaytonaFactory(Protocol): - def __call__(self) -> AbstractAsyncContextManager[_DaytonaClient]: ... - - -class _ObjectFactory(Protocol): - def __call__(self, *args: object, **kwargs: object) -> object: ... - - -class _ResourcesFactory(Protocol): - def __call__(self, *args: object, **kwargs: object) -> _DaytonaResources: ... - - -class _GpuTypeFactory(Protocol): - def __call__(self, value: str) -> object: ... - - -class _DaytonaImageFactory(Protocol): - def base(self, image: str) -> object: ... - - -class DaytonaModule(Protocol): - AsyncDaytona: _DaytonaFactory - CreateSandboxFromImageParams: _ObjectFactory - CreateSandboxFromSnapshotParams: _ObjectFactory - CreateSnapshotParams: _ObjectFactory - DaytonaNotFoundError: type[Exception] - GpuType: _GpuTypeFactory - Image: _DaytonaImageFactory - Resources: _ResourcesFactory - SessionExecuteRequest: _ObjectFactory - - async def _snapshot_is_current( snapshot: DaytonaSnapshot, image: str | DaytonaImage, @@ -173,11 +37,7 @@ async def _snapshot_is_current( build = snapshot.build_info if build is None: return False - object_storage = cast( - "ObjectStorageModule", - importlib.import_module("daytona._async.object_storage"), - ) - AsyncObjectStorage = object_storage.AsyncObjectStorage + from daytona._async.object_storage import AsyncObjectStorage # The hasher is an instance method only for code organization; credentials # are needed to upload, not to hash, so skip the credentialed __init__. @@ -249,17 +109,17 @@ def __init__( @asynccontextmanager async def __call__(self, task: Task) -> AsyncIterator[Runtime]: import asyncssh - - daytona_sdk = cast("DaytonaModule", importlib.import_module("daytona")) - AsyncDaytona = daytona_sdk.AsyncDaytona - CreateSandboxFromImageParams = daytona_sdk.CreateSandboxFromImageParams - CreateSandboxFromSnapshotParams = daytona_sdk.CreateSandboxFromSnapshotParams - CreateSnapshotParams = daytona_sdk.CreateSnapshotParams - DaytonaNotFoundError = daytona_sdk.DaytonaNotFoundError - GpuType = daytona_sdk.GpuType - Image = daytona_sdk.Image - Resources = daytona_sdk.Resources - SessionExecuteRequest = daytona_sdk.SessionExecuteRequest + from daytona import ( + AsyncDaytona, + CreateSandboxFromImageParams, + CreateSandboxFromSnapshotParams, + CreateSnapshotParams, + DaytonaNotFoundError, + GpuType, + Image, + Resources, + SessionExecuteRequest, + ) async with AsyncDaytona() as daytona: config = (self.runtime_config or RuntimeConfig()).with_overrides(task.runtime_config) diff --git a/hud/eval/runtime/docker.py b/hud/eval/runtime/docker.py index ea6ca10d4..493588dd7 100644 --- a/hud/eval/runtime/docker.py +++ b/hud/eval/runtime/docker.py @@ -17,8 +17,8 @@ from hud.utils.docker import docker as _docker from hud.utils.process import create_process_group_exec -from .compose import ComposeConfig, ComposeProject -from .core import Runtime, RuntimeConfig +from .compose import ComposeConfig +from .core import Runtime, RuntimeConfig, RuntimeSession if TYPE_CHECKING: from collections.abc import AsyncIterator, Mapping, Sequence @@ -118,11 +118,14 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: resources = config.resources if resources is not None: resources._require_support("DockerRuntime", {"cpu", "memory_mb", "storage_mb", "gpu"}) - compose_source = config.compose_source() - if compose_source is not None: + compose_project = config.compose + if compose_project is not None: + compose = compose_project.document + if not isinstance(compose, Path): + raise ValueError("DockerRuntime requires compose as a local file path") + compose = compose.resolve() if self.run_args: raise ValueError("DockerRuntime run_args apply only to image environments") - compose = compose_source.runnable_path("DockerRuntime") port_service = ComposeConfig.from_file(compose).network_owner("main") resources = config.resources if ( @@ -132,7 +135,7 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: ): raise ValueError("DockerRuntime cannot select Compose GPUs by type") service_socket = None - if config.compose_service_access: + if compose_project.service_access: service_socket = self.compose_service_socket if service_socket is None: endpoint = os.environ.get("DOCKER_HOST") @@ -151,12 +154,11 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: "requires compose_service_socket" ) service_socket = parsed.path - project_files = ComposeProject(compose) project = f"hud-{uuid.uuid4().hex[:12]}" lock = self._compose_preparation_locks.setdefault(compose, asyncio.Lock()) async with lock: prepared = await _prepare_compose_project(compose) - with project_files.stage( + with compose_project.stage( f"127.0.0.1::{self.port}", port_service=port_service, seccomp=_DOCKER_SECCOMP_PROFILE, @@ -174,6 +176,8 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: "compose", "--project-name", project, + "--project-directory", + str(files.project_directory), "--file", str(files.compose), "--file", @@ -205,13 +209,11 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: ) host_port = int(mapping.strip().splitlines()[0].rsplit(":", 1)[1]) container, _ = await _docker(*command, "ps", "--quiet", "main") - handoff = _DockerHandoff(container.strip()) - await handoff.prepare() - yield Runtime( + yield _DockerEndpoint( f"tcp://127.0.0.1:{host_port}", params=params, config=config if config.model_dump(exclude_none=True) else None, - handoff=handoff, + container=container.strip(), ) finally: await _docker( @@ -271,13 +273,11 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: f"{self.port}:\n{(logs_err or logs_out).strip()}", ) host_port = int(mapping.strip().splitlines()[0].rsplit(":", 1)[1]) - handoff = _DockerHandoff(container) - await handoff.prepare() - yield Runtime( + yield _DockerEndpoint( f"tcp://127.0.0.1:{host_port}", params=params, config=config, - handoff=handoff, + container=container, ) finally: # check=False: teardown must not shadow the run's own error, and @@ -285,57 +285,86 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: await _docker("rm", "--force", container, check=False) -@dataclass(frozen=True, slots=True) -class _DockerHandoff: +@dataclass(frozen=True, slots=True, kw_only=True) +class _DockerEndpoint(Runtime): container: str - async def prepare(self) -> None: - await _docker("exec", self.container, "mkdir", "-p", "/media/hud/handoffs") + def session(self, session_id: str) -> RuntimeSession: + return _DockerSession(session_id=session_id, container=self.container) - async def export_to(self, destination: Path) -> None: - archive = f"/media/hud/handoff-export-{uuid.uuid4().hex}.tar.gz" - script = """ + +@dataclass(frozen=True, slots=True, kw_only=True) +class _DockerSession(RuntimeSession): + container: str + + @asynccontextmanager + async def snapshot(self) -> AsyncIterator[Path | None]: + with tempfile.TemporaryDirectory(prefix="hud-session-") as directory: + destination = Path(directory) / "session.tar.gz" + root = f"/media/hud/sessions/{self.session_id}" + exists, _ = await _docker( + "exec", + self.container, + "sh", + "-c", + 'if [ -d "$1" ]; then printf 1; fi', + "hud-session", + root, + ) + if not exists: + yield None + return + archive = f"/media/hud/session-export-{uuid.uuid4().hex}.tar.gz" + script = """ import sys import tarfile from pathlib import Path -root = Path("/media/hud/handoffs") +root = Path(sys.argv[1]) entries = list(root.rglob("*")) if any(entry.is_symlink() for entry in entries): - raise ValueError("runtime handoff contains a symbolic link") -with tarfile.open(sys.argv[1], "w:gz") as output: + raise ValueError("runtime session contains a symbolic link") +if any(not (entry.is_file() or entry.is_dir()) for entry in entries): + raise ValueError("runtime session contains an unsupported entry") +with tarfile.open(sys.argv[2], "w:gz") as output: for entry in entries: output.add(entry, arcname=entry.relative_to(root), recursive=False) """ - try: - await _docker( - "exec", - "--user", - "0", - self.container, - "python3", - "-c", - script, - archive, - ) - await _docker("cp", f"{self.container}:{archive}", str(destination)) - finally: - await _docker("exec", "--user", "0", self.container, "rm", "-f", archive, check=False) + try: + await _docker( + "exec", + "--user", + "0", + self.container, + "python3", + "-c", + script, + root, + archive, + ) + await _docker("cp", f"{self.container}:{archive}", str(destination)) + finally: + await _docker( + "exec", "--user", "0", self.container, "rm", "-f", archive, check=False + ) + yield destination - async def import_from(self, source: Path) -> None: - with tempfile.TemporaryDirectory(prefix="hud-handoff-import-") as directory: + async def restore(self, source: Path) -> None: + target = f"/media/hud/sessions/{self.session_id}" + with tempfile.TemporaryDirectory(prefix="hud-session-import-") as directory: root = Path(directory) - _extract_handoff_archive(source, root) - await self.prepare() + _extract_session_archive(source, root) + await _docker("exec", "--user", "0", self.container, "rm", "-rf", target) + await _docker("exec", "--user", "0", self.container, "mkdir", "-p", target) await _docker( "cp", f"{root}/.", - f"{self.container}:/media/hud/handoffs", + f"{self.container}:{target}", ) -def _extract_handoff_archive(source: Path, destination: Path) -> None: +def _extract_session_archive(source: Path, destination: Path) -> None: with tarfile.open(source, "r:gz") as archive: if any(not (member.isfile() or member.isdir()) for member in archive.getmembers()): - raise ValueError("runtime handoff archive contains an unsupported entry") + raise ValueError("runtime session archive contains an unsupported entry") archive.extractall(destination, filter="data") diff --git a/hud/eval/runtime/hosted.py b/hud/eval/runtime/hosted.py index 925890087..2ebdb8658 100644 --- a/hud/eval/runtime/hosted.py +++ b/hud/eval/runtime/hosted.py @@ -136,11 +136,11 @@ async def _submit_and_await( if group_id is not None: payload["group_id"] = group_id if task.runtime_config is not None: - runtime_config = task.runtime_config.request_payload() + runtime_config = task.runtime_config.model_dump(mode="json", exclude_unset=True) if runtime_config: payload["runtime_config"] = runtime_config if task.verifier is not None: - payload["verifier"] = task.verifier.wire_payload() + payload["verifier"] = task.verifier.model_dump(mode="json", exclude_none=True) await platform.apost("/rollouts/submit", json=payload) return await self._await_terminal(platform, payload["trace_id"]) diff --git a/hud/eval/runtime/local.py b/hud/eval/runtime/local.py new file mode 100644 index 000000000..3ed959e49 --- /dev/null +++ b/hud/eval/runtime/local.py @@ -0,0 +1,217 @@ +"""Local in-process and subprocess runtime providers.""" + +from __future__ import annotations + +import asyncio +import contextlib +import sys +from collections import deque +from contextlib import asynccontextmanager +from pathlib import Path +from typing import TYPE_CHECKING + +from hud.utils.process import create_process_group_exec + +from .core import Runtime + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable + + from hud.environment.env import Environment + from hud.eval.task import Task + + +class LocalRuntime: + """The local provider: serve a fresh env per rollout, in this process. + + *source* points at the env in whatever form you have: + + - a ``.py`` file or directory — imported fresh per acquisition (sibling + imports resolve); *env* pins one name when several are declared, + defaulting to the placed task's env + - a live :class:`~hud.environment.Environment` — served directly, one + acquisition at a time + - a ``(task) -> Environment`` callable — called per acquisition with the + placed row + + :: + + runtime = LocalRuntime("env.py") + runtime = LocalRuntime(env) + runtime = LocalRuntime(lambda task: build_env(task.env)) + + ``ready_timeout`` bounds ``@env.initialize`` startup. Source paths and + constructors create a fresh environment per acquisition. Hooks share this + process's event loop, so blocking env code stalls concurrent rollouts — + use :class:`SubprocessRuntime` or :class:`DockerRuntime` for process + isolation, and ``Runtime(url)`` to attach to a substrate served elsewhere. + """ + + def __init__( + self, + source: str | Path | Environment | Callable[[Task], Environment], + *, + env: str | None = None, + ready_timeout: float = 120.0, + ) -> None: + from hud.environment.env import Environment as _Environment + + self.ready_timeout = ready_timeout + self._source_dir: Path | None = None + self._live_lock: asyncio.Lock | None = None + if isinstance(source, _Environment): + if env is not None: + raise TypeError("LocalRuntime: env= applies only to source paths") + self._build: Callable[[Task], Environment] = lambda _task: source + self._live_lock = asyncio.Lock() + elif isinstance(source, (str, Path)): + path, pinned = Path(source).resolve(), env + self._source_dir = path if path.is_dir() else path.parent + from hud.environment import load_environment + + def _load(task: Task) -> Environment: + return load_environment(path, name=pinned or task.env) + + self._build: Callable[[Task], Environment] = _load + elif callable(source): + if env is not None: + raise TypeError("LocalRuntime: env= applies only to source paths") + self._build = source + else: + raise TypeError( + f"LocalRuntime: expected a source path, an Environment, or a " + f"(task) -> Environment constructor; got {source!r}" + ) + + @asynccontextmanager + async def __call__(self, task: Task) -> AsyncIterator[Runtime]: + from hud.environment.env import Environment as _Environment + + if task.runtime_config is not None: + raise ValueError("LocalRuntime does not support task runtime_config") + # The source dir stays importable for the whole acquisition, not just + # the initial import, so a template can lazily import a sibling + # module at run time (as it could under the child-process runtime). + # Always insert-and-remove one entry: balanced under concurrency. + if self._live_lock is not None: + await self._live_lock.acquire() + if self._source_dir is not None: + sys.path.insert(0, str(self._source_dir)) + try: + try: + env = self._build(task) + except RuntimeError as e: + # The source ran an event loop at import — usually an unguarded + # top-level run call; name the actual mistake. + if "running event loop" not in str(e): + raise + raise RuntimeError( + "the env source ran async code while being imported to place a " + 'rollout — guard top-level run calls with `if __name__ == "__main__":`' + ) from e + if not isinstance(env, _Environment): + raise TypeError(f"LocalRuntime: constructor returned {env!r}, not an Environment") + from hud.environment.server import _shutdown, bind + + try: + await asyncio.wait_for(env.start(), self.ready_timeout) + server = await bind(env, "127.0.0.1", 0) + host, port = server.sockets[0].getsockname()[:2] + serve_task = asyncio.create_task(server.serve_forever()) + try: + yield Runtime(f"tcp://{host}:{port}") + finally: + serve_task.cancel() + await _shutdown(server) + with contextlib.suppress(asyncio.CancelledError): + await serve_task + finally: + await env.stop() + finally: + if self._source_dir is not None: + with contextlib.suppress(ValueError): + sys.path.remove(str(self._source_dir)) + if self._live_lock is not None: + self._live_lock.release() + + +class SubprocessRuntime: + """The child-process provider: serve the placed row's env from *path*. + + Each acquisition runs ``python -m hud.environment.server --env + name`` — the same serving entry point a container CMD runs — on an + ephemeral loopback port, yields its :class:`Runtime`, and terminates the + child on exit. *path* is a ``.py`` file or a directory of them. The served + env is the placed task's ``env`` name (so a mixed-env taskset works + against one source), unless *env* pins one explicitly; placing a row whose + env the source does not define fails loudly in the child. + + The child's working directory is the source's directory, so sibling + imports and relative data paths resolve; ``@env.initialize`` daemons start + in the child and die with it. Because the source is re-imported in the + child, a script spawning itself (``SubprocessRuntime(__file__)``) must keep + top-level run calls under ``if __name__ == "__main__":``. + """ + + def __init__( + self, + path: str | Path, + *, + env: str | None = None, + ready_timeout: float = 120.0, + ) -> None: + self.source = Path(path).resolve() + self.env = env + self.ready_timeout = ready_timeout + + @asynccontextmanager + async def __call__(self, task: Task) -> AsyncIterator[Runtime]: + if task.runtime_config is not None: + raise ValueError("SubprocessRuntime does not support task runtime_config") + if not self.source.exists(): + raise FileNotFoundError(f"SubprocessRuntime: source not found: {self.source}") + cmd = [sys.executable, "-m", "hud.environment.server", str(self.source)] + cmd += ["--env", self.env or task.env] + proc = await create_process_group_exec( + *cmd, + term_timeout=10.0, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + cwd=self.source if self.source.is_dir() else self.source.parent, + ) + assert proc.stdout is not None + output = proc.stdout + output_tail: deque[str] = deque(maxlen=50) + try: + from hud.environment.server import PORT_ANNOUNCEMENT + + port = None + async with asyncio.timeout(self.ready_timeout): + while line := await output.readline(): + text = line.decode("utf-8", "replace").strip() + if text.startswith(PORT_ANNOUNCEMENT): + port = int(text.removeprefix(PORT_ANNOUNCEMENT)) + break + output_tail.append(text) + if port is None: + code = await proc.wait() + tail = "\n".join(output_tail).strip() + detail = f":\n{tail}" if tail else " (no output captured)" + raise RuntimeError( + f"spawned env exited with code {code} before serving " + f"(source: {self.source}){detail}" + ) + + async def discard_output() -> None: + while await output.read(65536): + pass + + drain = asyncio.create_task(discard_output()) + try: + yield Runtime(f"tcp://127.0.0.1:{port}") + finally: + drain.cancel() + with contextlib.suppress(asyncio.CancelledError): + await drain + finally: + await proc.terminate() diff --git a/hud/eval/runtime/modal.py b/hud/eval/runtime/modal.py index c543e4ba7..b11579fb1 100644 --- a/hud/eval/runtime/modal.py +++ b/hud/eval/runtime/modal.py @@ -4,165 +4,117 @@ import asyncio import contextlib -import importlib import logging import shlex +import tempfile from contextlib import asynccontextmanager from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Any -from .compose import ComposeConfig, ComposeProject -from .core import Runtime, RuntimeConfig +from .compose import ComposeConfig +from .core import Runtime, RuntimeConfig, RuntimeSession from .docker import _DOCKER_SECCOMP_PROFILE if TYPE_CHECKING: from collections.abc import AsyncIterator, Mapping, Sequence - from pathlib import Path + + import modal from hud.eval.task import Task logger = logging.getLogger("hud.eval.runtime") -T_co = TypeVar("T_co", covariant=True) - - -class AioMethod(Protocol[T_co]): - async def aio(self, *args: object, **kwargs: object) -> T_co: ... - - -class ModalImage(Protocol): - build: AioMethod[None] - - def env(self, variables: Mapping[str, str]) -> ModalImage: ... - - -class _ModalImageFactory(Protocol): - def from_id(self, image_id: str) -> ModalImage: ... - - def from_registry(self, image: str) -> ModalImage: ... - - def from_name(self, name: str) -> ModalImage: ... - - -class _ModalAppFactory(Protocol): - lookup: AioMethod[object] - - -class _ModalStream(Protocol): - read: AioMethod[str] - - -class _ModalProcess(Protocol): - wait: AioMethod[int] - stderr: _ModalStream - - -class _ModalFilesystem(Protocol): - copy_from_local: AioMethod[None] - copy_to_local: AioMethod[None] - - -class _ModalTunnel(Protocol): - tcp_socket: tuple[str, int] - - -class ModalSandbox(Protocol): - object_id: str - wait_until_ready: AioMethod[None] - filesystem: _ModalFilesystem - exec: AioMethod[_ModalProcess] - tunnels: AioMethod[dict[int, _ModalTunnel]] - terminate: AioMethod[None] - - -class _ModalSandboxFactory(Protocol): - create: AioMethod[ModalSandbox] - - -class _ModalProbeFactory(Protocol): - def with_tcp(self, port: int) -> object: ... - - -class ModalModule(Protocol): - Image: _ModalImageFactory - App: _ModalAppFactory - Sandbox: _ModalSandboxFactory - Probe: _ModalProbeFactory - - _MODAL_COMPOSE_CPU = 4.0 _MODAL_COMPOSE_MEMORY_MB = 8192 -def _modal_image_from_uri(modal: ModalModule, image_uri: str) -> ModalImage: - modal_uri_prefix = "modal://" - if image_uri.startswith(modal_uri_prefix): - return modal.Image.from_id(image_uri.removeprefix(modal_uri_prefix)) - return modal.Image.from_registry(image_uri) +@dataclass(frozen=True, slots=True, kw_only=True) +class _ModalEndpoint(Runtime): + sandbox: modal.Sandbox + compose: str | None + + def session(self, session_id: str) -> RuntimeSession: + return _ModalSession( + session_id=session_id, + sandbox=self.sandbox, + compose=self.compose, + ) -@dataclass(frozen=True, slots=True) -class _ModalHandoff: - sandbox: ModalSandbox +@dataclass(frozen=True, slots=True, kw_only=True) +class _ModalSession(RuntimeSession): + sandbox: modal.Sandbox compose: str | None def _container(self) -> str: if self.compose is None: return "" compose = shlex.quote(self.compose) + project_directory = shlex.quote(str(PurePosixPath(self.compose).parent)) return ( - "CONTAINER=$(docker compose --project-directory /hud/project " - f"--file /hud/project/{compose} --file /hud/override.json " + f"CONTAINER=$(docker compose --project-directory {project_directory} " + f"--file {compose} --file /hud/override.json " "--file /hud/ports.yaml ps --quiet main); " ) - async def _exec(self, command: str) -> None: + async def _exec(self, command: str) -> str: process = await self.sandbox.exec.aio("sh", "-c", command) - if await process.wait.aio() != 0: + returncode = await process.wait.aio() + stdout = await process.stdout.read.aio() + if returncode != 0: raise RuntimeError((await process.stderr.read.aio()).strip()) + return stdout - async def prepare(self) -> None: - if self.compose is None: - await self._exec("mkdir -p /media/hud/handoffs") - else: - await self._exec( - self._container() + 'test -n "$CONTAINER" && docker exec "$CONTAINER" ' - "mkdir -p /media/hud/handoffs" + @asynccontextmanager + async def snapshot(self) -> AsyncIterator[Path | None]: + with tempfile.TemporaryDirectory(prefix="hud-session-") as directory: + destination = Path(directory) / "session.tar.gz" + session = f"/media/hud/sessions/{self.session_id}" + if self.compose is None: + root = session + command = "" + check = f"if [ -d {root} ]; then printf 1; fi" + else: + root = "/media/hud/session-export" + command = ( + self._container() + + f"rm -rf {root} && mkdir -p {root} && " + + f'docker cp "$CONTAINER":{session}/. {root} && ' + ) + check = ( + self._container() + + f'docker exec "$CONTAINER" test -d {session} && printf 1 || true' + ) + if not (await self._exec(check)).strip(): + yield None + return + command += ( + f"if find {root} -mindepth 1 ! -type f ! -type d -print -quit | grep -q .; " + "then echo 'runtime session contains an unsupported entry' >&2; exit 1; fi; " + f"tar -czf /media/hud/session.tar.gz -C {root} ." ) - - async def export_to(self, destination: Path) -> None: - if self.compose is None: - root = "/media/hud/handoffs" - command = "" - else: - root = "/media/hud/handoff-export" - command = ( - self._container() - + f"rm -rf {root} && mkdir -p {root} && " - + f'docker cp "$CONTAINER":/media/hud/handoffs/. {root} && ' + await self._exec(command) + await self.sandbox.filesystem.copy_to_local.aio( + "/media/hud/session.tar.gz", destination ) - command += ( - f"if find {root} -mindepth 1 ! -type f ! -type d -print -quit | grep -q .; " - "then echo 'runtime handoff contains an unsupported entry' >&2; exit 1; fi; " - f"tar -czf /media/hud/handoff.tar.gz -C {root} ." - ) - await self._exec(command) - await self.sandbox.filesystem.copy_to_local.aio("/media/hud/handoff.tar.gz", destination) + yield destination - async def import_from(self, source: Path) -> None: - await self.sandbox.filesystem.copy_from_local.aio(source, "/media/hud/handoff.tar.gz") + async def restore(self, source: Path) -> None: + session = f"/media/hud/sessions/{self.session_id}" + await self.sandbox.filesystem.copy_from_local.aio(source, "/media/hud/session.tar.gz") if self.compose is None: command = ( - "mkdir -p /media/hud/handoffs && " - "tar -xzf /media/hud/handoff.tar.gz -C /media/hud/handoffs" + f"rm -rf {session} && mkdir -p {session} && " + f"tar -xzf /media/hud/session.tar.gz -C {session}" ) else: command = ( self._container() - + "rm -rf /tmp/hud-handoff && mkdir -p /tmp/hud-handoff && " - + "tar -xzf /media/hud/handoff.tar.gz -C /tmp/hud-handoff && " - + 'docker exec "$CONTAINER" mkdir -p /media/hud/handoffs && ' - + 'docker cp /tmp/hud-handoff/. "$CONTAINER":/media/hud/handoffs' + + "rm -rf /tmp/hud-session && mkdir -p /tmp/hud-session && " + + "tar -xzf /media/hud/session.tar.gz -C /tmp/hud-session && " + + f'docker exec "$CONTAINER" sh -c "rm -rf {session} && mkdir -p {session}" && ' + + f'docker cp /tmp/hud-session/. "$CONTAINER":{session}' ) await self._exec(command) @@ -186,7 +138,7 @@ def __init__( self, image_name: str | None = None, *, - image: ModalImage | None = None, + image: modal.Image | None = None, command: Sequence[str] | None = None, app_name: str = "hud-envs", workdir: str | None = None, @@ -221,32 +173,39 @@ def __init__( # Resolved (named) or built-once (from Dockerfile) image, behind a lock so # concurrent first acquisitions build/look up exactly once. self._image = image - self._resolved: ModalImage | None = None + self._resolved: modal.Image | None = None self._image_lock = asyncio.Lock() @asynccontextmanager async def __call__(self, task: Task) -> AsyncIterator[Runtime]: + import modal + config = (self.runtime_config or RuntimeConfig()).with_overrides(task.runtime_config) resources = config.resources if resources is not None: resources._require_support("ModalRuntime", {"cpu", "memory_mb", "gpu"}) - compose_source = config.compose_source() - compose = ( - compose_source.runnable_path("ModalRuntime") if compose_source is not None else None - ) + project = config.compose + compose = None + if project is not None: + compose = project.document + if not isinstance(compose, Path): + raise ValueError("ModalRuntime requires compose as a local file path") + compose = compose.resolve() if compose is not None and resources is not None and resources.gpu is not None: raise ValueError( "ModalRuntime cannot attach GPUs to services inside Docker-in-Docker; " "use a materialized image or omit runtime_config.compose" ) port_service = ComposeConfig.from_file(compose).network_owner("main") if compose else "main" - modal = cast("ModalModule", importlib.import_module("modal")) - app = None if compose is not None: image = modal.Image.from_registry("docker:28.3.3-dind") elif config.image is not None: - image = _modal_image_from_uri(modal, config.image) + image = ( + modal.Image.from_id(config.image.removeprefix("modal://")) + if config.image.startswith("modal://") + else modal.Image.from_registry(config.image) + ) elif self.image_name is not None: image = modal.Image.from_name(self.image_name) elif self._image is None: @@ -313,18 +272,23 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: **({"experimental_options": {"vm_runtime": True}} if compose is not None else {}), **sandbox_kwargs, ) + compose_path: str | None = None try: - if compose is None: + if project is None: await sb.wait_until_ready.aio(timeout=ready_timeout) else: - project = ComposeProject(compose) + assert compose is not None + project_root = project.root if isinstance(project.root, Path) else compose.parent + compose_path = str( + PurePosixPath("/hud/project") + / compose.relative_to(project_root.resolve()).as_posix() + ) + project_directory = str(PurePosixPath(compose_path).parent) with project.stage( f"{self.port}:{self.port}", port_service=port_service, seccomp="/hud/docker-seccomp.json", - service_socket=( - "/var/run/docker.sock" if config.compose_service_access else None - ), + service_socket=("/var/run/docker.sock" if project.service_access else None), env_vars=self.env_vars, cpu=resources.cpu if resources is not None else None, memory_mb=resources.memory_mb if resources is not None else None, @@ -349,8 +313,8 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: "BUILD_FLAG=--build && " "if [ -f /hud/project/build.sh ]; then " "sh /hud/project/build.sh && BUILD_FLAG=--no-build; fi && " - "docker compose --project-directory /hud/project " - f"--file /hud/project/{shlex.quote(compose.name)} " + f"docker compose --project-directory {shlex.quote(project_directory)} " + f"--file {shlex.quote(compose_path)} " "--file /hud/override.json --file /hud/ports.yaml " 'up --detach "$BUILD_FLAG" --remove-orphans' ) @@ -366,29 +330,28 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: error = (await process.stderr.read.aio()).strip() raise RuntimeError(f"Modal Compose startup failed: {error}") host, port = (await sb.tunnels.aio())[self.port].tcp_socket - handoff = _ModalHandoff(sb, compose.name if compose is not None else None) - await handoff.prepare() - yield Runtime( - f"tcp://{host}:{port}", + yield _ModalEndpoint( + url=f"tcp://{host}:{port}", params={ "provider": "modal", "instance_id": sb.object_id, **({"ready_timeout": ready_timeout} if compose is not None else {}), }, config=config if config.model_dump(exclude_none=True) else None, - handoff=handoff, + sandbox=sb, + compose=compose_path, ) finally: # check-free teardown: never shadow the run's own error. - if compose is not None: + if compose_path is not None: with contextlib.suppress(Exception): process = await sb.exec.aio( "docker", "compose", "--project-directory", - "/hud/project", + str(PurePosixPath(compose_path).parent), "--file", - f"/hud/project/{compose.name}", + compose_path, "--file", "/hud/override.json", "--file", diff --git a/hud/eval/sync.py b/hud/eval/sync.py index cf50032b6..67a588334 100644 --- a/hud/eval/sync.py +++ b/hud/eval/sync.py @@ -151,37 +151,21 @@ def task_upload_payload(task: Task) -> dict[str, Any]: The platform resolves `(env, task_id)` against the env's latest build manifest and validates `args` against the task's schema. """ - payload: dict[str, Any] = { - "name": task.slug, - "env": {"name": task.env}, - "task_id": task.id, - "args": task.args, + row = task.model_dump(mode="json", exclude_none=True) + return { + "name": row.pop("slug"), + "env": {"name": row.pop("env")}, + "task_id": row.pop("id"), + **row, } - if task.validation is not None: - payload["validation"] = task.validation - if task.agent_config: - payload["agent_config"] = task.agent_config - if task.columns: - payload["columns"] = task.columns - if task.runtime_config is not None: - payload["runtime_config"] = task.runtime_config.request_payload() - if task.verifier is not None: - payload["verifier"] = task.verifier.wire_payload() - return payload def _task_signature(task: Task) -> str: - sig_data: dict[str, Any] = {"args": task.args or {}} - if task.validation is not None: - sig_data["validation"] = task.validation - if task.agent_config: - sig_data["agent_config"] = task.agent_config - if task.columns: - sig_data["columns"] = task.columns - if task.runtime_config is not None: - sig_data["runtime_config"] = task.runtime_config.request_payload() - if task.verifier is not None: - sig_data["verifier"] = task.verifier.wire_payload() + sig_data = task.model_dump( + mode="json", + exclude_none=True, + exclude={"env", "id", "slug"}, + ) return f"{task.id}|" + json.dumps( sig_data, sort_keys=True, diff --git a/hud/eval/task.py b/hud/eval/task.py index b760b826f..304cc40c6 100644 --- a/hud/eval/task.py +++ b/hud/eval/task.py @@ -1,22 +1,4 @@ -"""Task: one task row — an env name, a task id, bound args, and metadata. - -``foo(x, y)`` (an ``@env.template`` factory call) returns one of these. ``env`` -is the environment's *name*: the join key between the data plane (rows) and -whatever placement can bring that environment up. Running a task never needs -a live env — the prompt and grading arrive over the wire from the substrate -the placement brought up — so the row holds the reference explicitly instead -of wrapping it in an :class:`~hud.environment.Environment` object. - -The model *is* the row: field names are the wire keys, so plain pydantic -(``Task.model_validate(entry)`` / ``task.model_dump()``) is the whole codec — -there is no bespoke serialization layer. - -Placement is ``runtime: Provider | HostedRuntime | None`` (see :mod:`.runtime`). -Execution lives entirely in :mod:`.rollout` and scheduling in -:mod:`.taskset` — :meth:`Task.run` is the single-task form of -``Taskset.run``, so the row is always an argument to the engine, never a -participant in it. Platform sync lives in :mod:`hud.eval.sync`. -""" +"""Portable task rows and single-task execution.""" from __future__ import annotations @@ -24,7 +6,17 @@ import json from typing import TYPE_CHECKING, Any -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PrivateAttr, + SerializationInfo, + field_serializer, + field_validator, +) + +from hud.environment.env import Environment from .runtime import RuntimeConfig @@ -35,34 +27,35 @@ from .runtime import HostedRuntime, Provider -def _default_slug(data: dict[str, Any]) -> str: - task_id = data.get("id") - if not isinstance(task_id, str): - return "" - args = data.get("args") - if not isinstance(args, dict) or not args: - return task_id - digest = hashlib.sha1( # noqa: S324 - non-crypto, stable disambiguator - json.dumps(args, sort_keys=True, default=str).encode("utf-8"), - ).hexdigest()[:8] - return f"{task_id}-{digest}" - - class Task(BaseModel): """One concrete task: an env name plus data (id, args, metadata). - Pure data — holds no execution state, so one ``Task`` can drive many - concurrent rollouts. ``run`` it for a graded :class:`~hud.eval.job.Job`; - placement comes from ``runtime=`` (a provider), else the HUD runtime - tunnel by ``env`` name. + Its fields are pure data, so one ``Task`` can drive many concurrent + rollouts. ``run`` it for a graded :class:`~hud.eval.job.Job`; placement + comes from ``runtime=`` or the environment that created it. """ model_config = ConfigDict(validate_assignment=True) + _env: Environment | None = PrivateAttr(default=None) + env: str = Field(min_length=1) id: str = Field(min_length=1) args: dict[str, Any] = Field(default_factory=dict) - slug: str = Field(default_factory=_default_slug, min_length=1) + slug: str = Field( + default_factory=lambda data: ( + str(data.get("id", "")) + + ( + "-" + + hashlib.sha1( # noqa: S324 - stable non-cryptographic suffix + json.dumps(args, sort_keys=True, default=str).encode("utf-8") + ).hexdigest()[:8] + if (args := data.get("args")) + else "" + ) + ), + min_length=1, + ) validation: list[dict[str, Any]] | None = None agent_config: dict[str, Any] | None = None #: Arbitrary metadata fields surfaced as filterable columns / leaderboard @@ -71,9 +64,6 @@ class Task(BaseModel): #: Optional row-level runtime construction input. Runtime adapters apply the #: supported subset into their native launch shape or reject it. runtime_config: RuntimeConfig | None = None - #: The verifier consumes files produced by the actor acquisition. Providers - #: transfer the runtime handoff namespace when placement cannot be reused. - requires_handoff: bool | None = None #: Optional agent-less task whose evaluation is the grade of record. The #: rollout completes this task first, then starts and grades the verifier #: with the same answer. Placement may reuse the live substrate when both @@ -87,18 +77,17 @@ def _reject_nested_verifier(cls, verifier: Task | None) -> Task | None: raise ValueError("nested verifier tasks are not supported") return verifier - def wire_payload(self) -> dict[str, Any]: - """Serialize the task for platform transport.""" - payload = self.model_dump( - mode="json", - exclude_none=True, - exclude={"runtime_config", "verifier"}, + @field_serializer("runtime_config") + def _serialize_runtime_config( + self, + config: RuntimeConfig | None, + info: SerializationInfo, + ) -> dict[str, Any] | None: + return ( + config.model_dump(mode="json", exclude_unset=True, context=info.context) + if config is not None + else None ) - if self.runtime_config is not None: - payload["runtime_config"] = self.runtime_config.request_payload() - if self.verifier is not None: - payload["verifier"] = self.verifier.wire_payload() - return payload # ─── execution ──────────────────────────────────────────────────── @@ -117,9 +106,9 @@ async def run( Identical scheduling semantics — one HUD job as the receipt (or an open ``job`` from :meth:`Job.start` to accumulate into), ``group`` repeats sharing a group_id, ``max_concurrent`` capping parallelism — - over a taskset of one. ``runtime`` is the placement; left unset it - falls back to the HUD runtime tunnel by ``env`` name. For a local - run, pass one explicitly (``runtime=LocalRuntime("env.py")``). + over a taskset of one. A task created by ``@env.template`` runs against + that environment by default. Other rows require an explicit placement, + such as ``runtime=LocalRuntime("env.py")``. """ from .taskset import Taskset # circular: taskset -> sync -> task diff --git a/hud/eval/taskset.py b/hud/eval/taskset.py index cfe127e16..4c45fa1f5 100644 --- a/hud/eval/taskset.py +++ b/hud/eval/taskset.py @@ -28,8 +28,6 @@ HostedRuntime, HUDRuntime, LocalRuntime, - _declared_env, - _declared_names, ) from .sync import fetch_taskset_tasks, resolve_taskset_id @@ -60,23 +58,12 @@ def __init__( name: str | None = None, tasks: Iterable[Task] = (), *, - origin: str | None = None, + taskset_id: str | None = None, ) -> None: self.name = name or "taskset" - self.origin = origin + self.taskset_id = taskset_id self.tasks: dict[str, Task] = self._index_by_slug(list(tasks)) - @property - def api_id(self) -> str | None: - """The platform taskset id when loaded via :meth:`from_api`, else None. - - Threaded into the job so a remote run of a synced taskset links to it; - ad-hoc/file/module tasksets have none and create no taskset. - """ - if self.origin and self.origin.startswith("api:"): - return self.origin[len("api:") :] - return None - @classmethod def from_file(cls, path: str | Path) -> Taskset: """Load a taskset from ``.py`` source, a directory, or JSON/JSONL data. @@ -86,7 +73,7 @@ def from_file(cls, path: str | Path) -> Taskset: """ source = Path(path) if source.suffix in {".json", ".jsonl"}: - return cls(source.stem, cls._load_tasks_json(source), origin=f"file:{source}") + return cls(source.stem, cls._load_tasks_json(source)) if source.suffix == ".py" or source.is_dir(): return cls.from_module(source) raise ValueError(f"unsupported taskset source: {source}") @@ -97,11 +84,7 @@ def from_module(cls, source: str | Path) -> Taskset: path = Path(source).resolve() found = [task for module in iter_modules(path) for task in cls._scan_tasks(module)] - return cls( - path.stem if path.is_file() else path.name, - found, - origin=f"module:{path}", - ) + return cls(path.stem if path.is_file() else path.name, found) @classmethod def from_api(cls, name: str) -> Taskset: @@ -111,7 +94,7 @@ def from_api(cls, name: str) -> Taskset: if not taskset_id: raise ValueError(f"taskset not found: {name}") fetched_display, tasks = fetch_taskset_tasks(platform, taskset_id) - return cls(fetched_display or display, tasks, origin=f"api:{taskset_id}") + return cls(fetched_display or display, tasks, taskset_id=taskset_id) def to_file(self, path: str | Path) -> Path: """Write this taskset's portable rows to JSON or JSONL.""" @@ -119,7 +102,8 @@ def to_file(self, path: str | Path) -> Path: target.parent.mkdir(parents=True, exist_ok=True) suffix = target.suffix.lower() # Compact rows: unset metadata is omitted (defaults restore it on load). - data = [task.model_dump(exclude_none=True) for task in self] + context = {"base_path": target.parent.resolve()} + data = [task.model_dump(mode="json", exclude_none=True, context=context) for task in self] if suffix == ".json": target.write_text(json.dumps(data, indent=2, default=str) + "\n", encoding="utf-8") @@ -167,7 +151,7 @@ def _load_tasks_json(path: Path) -> list[Task]: for entry in entries: if not isinstance(entry, dict): raise ValueError(f"{path}: each task entry must be an object") - tasks.append(Task.model_validate(entry)) + tasks.append(Task.model_validate(entry, context={"base_path": path.parent.resolve()})) return tasks @staticmethod @@ -200,7 +184,7 @@ def filter(self, slugs: Iterable[str]) -> Taskset: return Taskset( self.name, (task for slug, task in self.tasks.items() if slug in selected), - origin=self.origin, + taskset_id=self.taskset_id, ) def exclude(self, slugs: Iterable[str]) -> Taskset: @@ -208,7 +192,7 @@ def exclude(self, slugs: Iterable[str]) -> Taskset: return Taskset( self.name, (task for slug, task in self.tasks.items() if slug not in excluded), - origin=self.origin, + taskset_id=self.taskset_id, ) def environment_names(self) -> set[str]: @@ -217,35 +201,24 @@ def environment_names(self) -> set[str]: task.verifier.env for task in self if task.verifier is not None } - def _resolve_placement(self) -> Provider | HUDRuntime: - if self.origin and self.origin.startswith("module:"): - # The origin claims the rows only if it actually declares their - # envs (a tasks-only module importing its envs from elsewhere - # does not) — and it serves as the exact path, so a same-named - # variant in a sibling file is never dragged in. - source = Path(self.origin[len("module:") :]) - if self.environment_names() <= _declared_names(source): - return LocalRuntime(source) - if self.origin and self.origin.startswith("api:"): + def _resolve_placement(self) -> Provider: + if self.taskset_id is not None: return HUDRuntime() - declared = {name: _declared_env(name) for name in self.environment_names()} - if declared and all(declared.values()): - providers = { - name: LocalRuntime(env) for name, env in declared.items() if env is not None - } - logger.info( - "no runtime given: serving %s fresh from their declaring modules", - ", ".join(sorted(providers)), - ) - return lambda task: providers[task.env](task) - missing = sorted(name for name, env in declared.items() if env is None) + rows = list(self) + rows.extend(task.verifier for task in self if task.verifier is not None) + if rows and all(task._env is not None for task in rows): + providers: dict[int, LocalRuntime] = {} + for task in rows: + env = task._env + assert env is not None + if id(env) not in providers: + providers[id(env)] = LocalRuntime(env) + return lambda task: providers[id(task._env)](task) raise ValueError( - f"no placement for env(s) {', '.join(missing) or ''}: pass runtime= — " + "no placement: pass runtime= — " 'LocalRuntime("env.py") (a source file), LocalRuntime(env) (a live env), ' "LocalRuntime(build) (a (task) -> Environment constructor), Runtime(url) " - "(a served substrate), or HUDRuntime() (your deployed env). A row taken " - "from a loaded taskset keeps its placement when run through it: " - 'taskset.filter(["slug"]).run(...)' + "(a served substrate), or HUDRuntime() (your deployed env)" ) async def run( @@ -264,14 +237,12 @@ async def run( placement: a :class:`~hud.eval.runtime.Provider` (the env served somewhere, the agent loop driven here by :func:`~hud.eval.run.rollout`), or :class:`~hud.eval.runtime.HostedRuntime` to run each rollout remotely - on the platform. Left unset, what is already known decides: a - taskset loaded from local ``.py`` source serves that source's - directory; a platform taskset runs on the platform; rows naming envs - declared in imported modules serve each fresh from its file; anything - else raises, naming the forms to pass. One provider serves a - mixed-env - taskset and can size each substrate per row. Registers one HUD job as - the platform receipt and reports each run's trace under it — or, given + on the platform. Left unset, a platform taskset runs on the platform, + tasks created by a live environment run against that environment, and + portable rows require an explicit placement. One provider serves a + mixed-env taskset and can size each substrate per row. + Registers one HUD job as the platform receipt and reports each run's + trace under it — or, given an open ``job`` (:meth:`Job.start`), accumulates this batch into it instead, so a longer arc (a training session) spans many calls under one id. Returned ``job.runs`` preserves expansion order (task-major, @@ -292,14 +263,6 @@ async def run( raise ValueError("max_concurrent must be >= 1") task_list = list(self) - # Placement is chosen once for the batch: HostedRuntime delegates the - # whole rollout to the platform, anything else is a Provider driven - # locally by rollout(). No runtime: what the taskset or this process - # already knows decides (rows never carry placement) — a loaded - # taskset runs where it came from; rows naming envs declared in - # imported modules serve each fresh from its file; anything else is - # an error naming the forms to pass. - # An empty taskset schedules nothing, so it needs no placement. placement = runtime if runtime is not None or not task_list else self._resolve_placement() group = group or (job.group if job else 1) if group < 1: @@ -317,9 +280,9 @@ async def run( id=uuid.uuid4().hex, name=_job_name(self.name, task_list, group), group=group, - taskset_id=self.api_id, + taskset_id=self.taskset_id, ) - await job_enter(job.id, name=job.name, group=group, taskset_id=self.api_id) + await job_enter(job.id, name=job.name, group=group, taskset_id=self.taskset_id) job_id = job.id sem = asyncio.Semaphore(max_concurrent) if max_concurrent else None timeout = ( diff --git a/hud/eval/tests/test_docker_provider.py b/hud/eval/tests/test_docker_provider.py index a4f087aba..eb8a68930 100644 --- a/hud/eval/tests/test_docker_provider.py +++ b/hud/eval/tests/test_docker_provider.py @@ -14,10 +14,11 @@ import logging import os import sys +import tarfile from dataclasses import dataclass from pathlib import Path from types import ModuleType, SimpleNamespace -from typing import Any +from typing import TYPE_CHECKING, Any, cast import pytest @@ -39,6 +40,9 @@ ) from hud.eval.task import Task +if TYPE_CHECKING: + from daytona import Image as DaytonaImage + FAKE_DOCKER_SH = """\ #!/bin/sh echo "$@" >> "$DOCKER_LOG" @@ -197,12 +201,22 @@ async def wait() -> int: await wait_event.wait() return 0 - async def read() -> str: + async def read_stderr() -> str: return "" + async def read_stdout() -> str: + command = args[-1] + return ( + "1" + if "sessions/sess-actor" in command + and ("test -d" in command or "if [ -d" in command) + else "" + ) + return SimpleNamespace( wait=SimpleNamespace(aio=wait), - stderr=SimpleNamespace(read=SimpleNamespace(aio=read)), + stdout=SimpleNamespace(read=SimpleNamespace(aio=read_stdout)), + stderr=SimpleNamespace(read=SimpleNamespace(aio=read_stderr)), ) @@ -534,25 +548,30 @@ async def test_acquisition_publishes_ephemeral_port_and_removes_container( assert (await _docker_calls(docker_log))[-1] == "rm --force cid-42" -async def test_docker_handoff_archives_inside_the_container( - tmp_path: Path, +async def test_docker_session_archives_inside_the_container( monkeypatch: pytest.MonkeyPatch, ) -> None: calls: list[tuple[tuple[str, ...], bool]] = [] async def fake_docker(*args: str, check: bool = True) -> tuple[str, str]: calls.append((args, check)) + if args[:3] == ("exec", "cid-42", "sh"): + return "1", "" return "", "" monkeypatch.setattr(runtime_module, "_docker", fake_docker) - destination = tmp_path / "handoff.tar.gz" - - await runtime_module._DockerHandoff("cid-42").export_to(destination) - - export, copy, cleanup = calls + async with runtime_module._DockerSession( + session_id="sess-actor", + container="cid-42", + ).snapshot() as destination: + assert destination is not None + + probe, export, copy, cleanup = calls + assert probe[0][:3] == ("exec", "cid-42", "sh") assert export[0][:6] == ("exec", "--user", "0", "cid-42", "python3", "-c") - assert "runtime handoff contains a symbolic link" in export[0][6] - archive = export[0][7] + assert "runtime session contains a symbolic link" in export[0][6] + assert export[0][7] == "/media/hud/sessions/sess-actor" + archive = export[0][8] assert copy == (("cp", f"cid-42:{archive}", str(destination)), True) assert cleanup == ( ("exec", "--user", "0", "cid-42", "rm", "-f", archive), @@ -642,14 +661,14 @@ def test_runtime_config_source_override_is_mutually_exclusive(tmp_path: Path) -> replacement = tmp_path / "replacement.yaml" assert RuntimeConfig(image="img:default").with_overrides( - RuntimeConfig(compose=compose) - ) == RuntimeConfig(compose=compose) - assert RuntimeConfig(compose=compose).with_overrides( + RuntimeConfig(compose=ComposeProject(document=compose)) + ) == RuntimeConfig(compose=ComposeProject(document=compose)) + assert RuntimeConfig(compose=ComposeProject(document=compose)).with_overrides( RuntimeConfig(image="img:task") ) == RuntimeConfig(image="img:task") - assert RuntimeConfig(compose=compose, compose_project=tmp_path).with_overrides( - RuntimeConfig(compose=replacement) - ) == RuntimeConfig(compose=replacement) + assert RuntimeConfig(compose=ComposeProject(document=compose, root=tmp_path)).with_overrides( + RuntimeConfig(compose=ComposeProject(document=replacement)) + ) == RuntimeConfig(compose=ComposeProject(document=replacement)) async def test_runtime_config_rejects_unsupported_docker_fields() -> None: @@ -686,6 +705,7 @@ async def test_docker_runtime_starts_compose_with_a_main_service_override( ) -> None: monkeypatch.delenv("DOCKER_HOST", raising=False) calls: list[tuple[str, ...]] = [] + recipe: dict[str, Any] = {} rendered: dict[str, Any] = {} port_override = "" compose = tmp_path / "compose.yaml" @@ -706,7 +726,8 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: return "unix:///Users/test/.docker/run/docker.sock\n", "" if args[-4:] == ("up", "--detach", "--no-build", "--remove-orphans"): files = [Path(args[index + 1]) for index, value in enumerate(args) if value == "--file"] - _, override, ports = files + compose_file, override, ports = files + recipe.update(json.loads(compose_file.read_text("utf-8"))) rendered.update(json.loads(override.read_text("utf-8"))) port_override = ports.read_text("utf-8") if args[-3:] == ("port", "main", "8765"): @@ -718,8 +739,7 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: env="any-env", id="t", runtime_config=RuntimeConfig( - compose=compose, - compose_service_access=True, + compose=ComposeProject(document=compose, service_access=True), resources=RuntimeResources(cpu=2, memory_mb=4096), ), ) @@ -729,6 +749,7 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: assert runtime.config == task.runtime_config assert marker.read_text("utf-8") == "prepared" + assert recipe["services"]["main"]["image"] == "hud-env:one" assert rendered["services"]["main"] == { "security_opt": [ f"seccomp={runtime_module._DOCKER_SECCOMP_PROFILE}", @@ -749,7 +770,8 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: up = next( call for call in calls if call[-4:] == ("up", "--detach", "--no-build", "--remove-orphans") ) - assert str(compose.resolve()) in up + assert up[up.index("--project-directory") + 1] == str(tmp_path) + assert str(compose) not in up assert calls[-1][-3:] == ("down", "--volumes", "--remove-orphans") @@ -787,7 +809,11 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: return "", "" monkeypatch.setattr(runtime_module, "_docker", fake_docker) - task = Task(env="any-env", id="t", runtime_config=RuntimeConfig(compose=compose)) + task = Task( + env="any-env", + id="t", + runtime_config=RuntimeConfig(compose=ComposeProject(document=compose)), + ) provider = DockerRuntime(env_vars={"OPENAI_API_KEY": "sk-test"}) async with provider(task): @@ -820,7 +846,11 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: monkeypatch.setattr(runtime_module, "_prepare_compose_project", prepare) monkeypatch.setattr(runtime_module, "_docker", fake_docker) - task = Task(env="any-env", id="t", runtime_config=RuntimeConfig(compose=compose)) + task = Task( + env="any-env", + id="t", + runtime_config=RuntimeConfig(compose=ComposeProject(document=compose)), + ) provider = DockerRuntime() async def acquire() -> None: @@ -846,8 +876,7 @@ async def test_docker_runtime_rejects_remote_compose_service_access( env="any-env", id="t", runtime_config=RuntimeConfig( - compose=compose, - compose_service_access=True, + compose=ComposeProject(document=compose, service_access=True), ), ) ): @@ -875,7 +904,7 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: task = Task( env="any-env", id="t", - runtime_config=RuntimeConfig(compose=compose, compose_service_access=True), + runtime_config=RuntimeConfig(compose=ComposeProject(document=compose, service_access=True)), ) async with DockerRuntime(compose_service_socket="/vm/run/docker.sock")(task): @@ -892,9 +921,10 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: def test_docker_runtime_accepts_only_one_environment_definition(tmp_path: Path) -> None: with pytest.raises(ValueError, match="either image or compose"): - RuntimeConfig(image="img:tag", compose=tmp_path / "compose.yaml") - with pytest.raises(ValueError, match="compose_service_access requires"): - RuntimeConfig(image="img:tag", compose_service_access=True) + RuntimeConfig( + image="img:tag", + compose=ComposeProject(document=tmp_path / "compose.yaml"), + ) def test_docker_runtime_accepts_runtime_config_defaults() -> None: @@ -966,21 +996,23 @@ async def test_modal_runtime_runs_compose_inside_a_dind_vm( monkeypatch: pytest.MonkeyPatch, ) -> None: calls = _install_fake_modal(monkeypatch) - compose = tmp_path / "compose.yaml" + project = tmp_path / "artifact" + compose = project / "compose-project" / "compose.yaml" + compose.parent.mkdir(parents=True) compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") async with ModalRuntime( runtime_config=RuntimeConfig( - compose=compose, - compose_service_access=True, + compose=ComposeProject(document=compose, root=project, service_access=True), limits=RuntimeLimits(startup_timeout_s=600), ), env_vars={"HUD_API_KEY": "secret"}, )(_row()) as runtime: assert runtime.url == "tcp://modal.host:4567" assert runtime.params == {"provider": "modal", "instance_id": "sb-1", "ready_timeout": 600} - assert runtime.handoff is not None - await runtime.handoff.export_to(tmp_path / "handoff.tar.gz") + async with runtime.session("sess-actor").snapshot() as archive: + assert archive is not None + await runtime.session("sess-verifier").restore(archive) assert calls["registry_image"] == "docker:28.3.3-dind" kwargs = calls["sandbox_kwargs"] @@ -995,6 +1027,7 @@ async def test_modal_runtime_runs_compose_inside_a_dind_vm( ("override.json", "/hud/override.json"), ("ports.yaml", "/hud/ports.yaml"), ("docker-seccomp.json", "/hud/docker-seccomp.json"), + ("session.tar.gz", "/media/hud/session.tar.gz"), ] override = calls["compose_override"] assert isinstance(override, dict) @@ -1008,13 +1041,30 @@ async def test_modal_runtime_runs_compose_inside_a_dind_vm( assert override["services"]["main"]["environment"] == {"HUD_API_KEY": "secret"} execs = calls["execs"] assert isinstance(execs, list) - assert "docker compose" in execs[0][0][-1] - assert "sh /hud/project/build.sh" in execs[0][0][-1] - assert 'up --detach "$BUILD_FLAG" --remove-orphans' in execs[0][0][-1] - assert any("runtime handoff contains an unsupported entry" in call[0][-1] for call in execs) - assert "down" in execs[-1][0] + startup = execs[0][0][-1] + assert "docker compose" in startup + assert "--project-directory /hud/project/compose-project" in startup + assert "--file /hud/project/compose-project/compose.yaml" in startup + assert "sh /hud/project/build.sh" in startup + assert 'up --detach "$BUILD_FLAG" --remove-orphans' in startup + session_commands = [ + call[0][-1] + for call in execs[1:-1] + if call[0][:2] == ("sh", "-c") and "docker compose" in call[0][-1] + ] + assert session_commands + assert all( + "--project-directory /hud/project/compose-project" in command + and "--file /hud/project/compose-project/compose.yaml" in command + for command in session_commands + ) + assert any("runtime session contains an unsupported entry" in call[0][-1] for call in execs) + teardown = execs[-1][0] + assert teardown[teardown.index("--project-directory") + 1] == ("/hud/project/compose-project") + assert teardown[teardown.index("--file") + 1] == ("/hud/project/compose-project/compose.yaml") + assert "down" in teardown assert execs[0][1]["timeout"] == 600 - assert calls["downloads"] == [("/media/hud/handoff.tar.gz", "handoff.tar.gz")] + assert calls["downloads"] == [("/media/hud/session.tar.gz", "session.tar.gz")] async def test_modal_runtime_bounds_compose_startup( @@ -1029,7 +1079,7 @@ async def test_modal_runtime_bounds_compose_startup( with pytest.raises(TimeoutError, match="Modal Compose startup timed out after 1 seconds"): async with ModalRuntime( runtime_config=RuntimeConfig( - compose=compose, + compose=ComposeProject(document=compose), limits=RuntimeLimits(startup_timeout_s=1), ) )(_row()): @@ -1044,7 +1094,7 @@ async def test_modal_runtime_rejects_gpu_inside_compose_dind(tmp_path: Path) -> provider = ModalRuntime( runtime_config=RuntimeConfig( - compose=compose, + compose=ComposeProject(document=compose), resources=RuntimeResources(gpu=RuntimeGPU(type="H100")), ) ) @@ -1212,7 +1262,9 @@ async def test_daytona_runtime_rejects_compose( compose.write_text("services: {}\n", encoding="utf-8") with pytest.raises(ValueError, match=r"does not support runtime_config\.compose"): - async with DaytonaRuntime(runtime_config=RuntimeConfig(compose=compose))(_row()): + async with DaytonaRuntime( + runtime_config=RuntimeConfig(compose=ComposeProject(document=compose)) + )(_row()): pass assert client.created == [] @@ -1276,7 +1328,10 @@ def test_compose_config_interpolates_only_artifact_supplied_values( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("HOST_ONLY", "secret") - (tmp_path / ".env").write_text("IMAGE=example:1\nEMPTY=\n", encoding="utf-8") + (tmp_path / ".env").write_text( + "IMAGE=example:1\nEMPTY=\nTOKEN=cost$5\n", + encoding="utf-8", + ) compose = tmp_path / "compose.yaml" compose.write_text( """ @@ -1286,15 +1341,21 @@ def test_compose_config_interpolates_only_artifact_supplied_values( command: "${EMPTY:-serve} $$HOME ${MISSING-default} $? $" environment: LITERAL: '$HOST_ONLY' + TOKEN: ${TOKEN} """, encoding="utf-8", ) - service = ComposeConfig.from_file(compose).services["main"] + config = ComposeConfig.from_file(compose) + service = config.services["main"] assert service.image == "example:1" - assert service.command == ["serve", "$$HOME", "default", "$?", "$"] - assert service.environment == {"LITERAL": "$HOST_ONLY"} + assert service.command == ["serve", "$$HOME", "default", "$$?", "$$"] + assert service.environment == {"LITERAL": "$$HOST_ONLY", "TOKEN": "cost$$5"} + + normalized = tmp_path / "normalized.json" + normalized.write_text(json.dumps(config.model_dump(mode="json")), encoding="utf-8") + assert ComposeConfig.from_file(normalized) == config @pytest.mark.parametrize( @@ -1462,7 +1523,7 @@ def _build_image(context: Path) -> _BuildImage: async def _boot_snapshot(context: Path, daytona: _FakeDaytonaClient) -> str: """Acquire once through a fresh provider; return the snapshot it booted.""" - async with DaytonaRuntime("env", image=_build_image(context))(_row()): + async with DaytonaRuntime("env", image=cast("DaytonaImage", _build_image(context)))(_row()): pass return daytona.created[-1].snapshot @@ -1616,7 +1677,7 @@ async def test_daytona_sizes_each_row_from_its_own_runtime_config( # different sizes must boot distinct snapshots, not the first row's. daytona = _install_fake_daytona(monkeypatch) (tmp_path / "env.py").write_text("REWARD = 1.0\n") - provider = DaytonaRuntime("env", image=_build_image(tmp_path)) + provider = DaytonaRuntime("env", image=cast("DaytonaImage", _build_image(tmp_path))) for cpu in (2, 4): task = Task( @@ -1742,7 +1803,7 @@ def test_compose_network_owner_follows_service_chains_and_stages_its_port( config = ComposeConfig.from_file(compose) assert config.network_owner("main") == "gateway" - with ComposeProject(compose).stage( + with ComposeProject(document=compose).stage( "127.0.0.1::8765", port_service=config.network_owner("main"), seccomp="profile.json", @@ -1750,6 +1811,38 @@ def test_compose_network_owner_follows_service_chains_and_stages_its_port( assert " gateway:" in files.ports.read_text("utf-8") +def test_compose_stage_archives_the_normalized_document(tmp_path: Path) -> None: + project = tmp_path / "project" + recipe = project / "recipe" / "compose.yaml" + recipe.parent.mkdir(parents=True) + (recipe.parent / ".env").write_text("IMAGE=example:1\n", encoding="utf-8") + recipe.write_text( + "services:\n main:\n image: ${IMAGE}\n environment:\n HOME: '$HOME'\n", + encoding="utf-8", + ) + + with ComposeProject(document=recipe, root=project).stage( + "127.0.0.1::8765", + seccomp="profile.json", + archive=True, + ) as files: + assert files.archive is not None + assert files.project_directory == recipe.parent.resolve() + assert json.loads(files.compose.read_text("utf-8"))["services"]["main"] == { + "image": "example:1", + "environment": {"HOME": "$$HOME"}, + "expose": [], + "ports": [], + "volumes": [], + } + with tarfile.open(files.archive, "r:gz") as archive: + archived = archive.extractfile("recipe/compose.yaml") + assert archived is not None + assert json.load(archived)["services"]["main"]["image"] == "example:1" + + assert "${IMAGE}" in recipe.read_text("utf-8") + + @pytest.mark.parametrize( ("services", "message"), [ diff --git a/hud/eval/tests/test_hosted.py b/hud/eval/tests/test_hosted.py index 81ddaae9d..a6e38e5e8 100644 --- a/hud/eval/tests/test_hosted.py +++ b/hud/eval/tests/test_hosted.py @@ -23,6 +23,7 @@ from hud.eval.job import Job from hud.eval.run import Run from hud.eval.runtime import ( + ComposeProject, HostedRuntime, HUDRuntime, Runtime, @@ -30,8 +31,8 @@ RuntimeGPU, RuntimeLimits, RuntimeResources, - _splice_websocket, ) +from hud.eval.runtime.hud import _splice_websocket from hud.eval.task import Task if TYPE_CHECKING: @@ -225,7 +226,6 @@ async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch env="sums", id="verify", args={"expected": 3}, - requires_handoff=True, ), ) @@ -256,7 +256,6 @@ async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch "id": "verify", "args": {"expected": 3}, "slug": "verify-5579a3e5", - "requires_handoff": True, } assert payload["group_id"] == "g1" assert payload["agent"]["type"] == "openai_compatible" @@ -309,7 +308,9 @@ async def test_run_submits_compose_document( Task( env="harbor", id="solve", - runtime_config=RuntimeConfig(compose=compose, compose_service_access=True), + runtime_config=RuntimeConfig( + compose=ComposeProject(document=compose, service_access=True) + ), ), _agent(), job_id=uuid.uuid4().hex, @@ -317,8 +318,8 @@ async def test_run_submits_compose_document( ) runtime_config = platform.posts[0][1]["runtime_config"] - assert runtime_config["compose"]["services"]["database"]["image"] == "postgres:16" - assert runtime_config["compose_service_access"] is True + assert runtime_config["compose"]["document"]["services"]["database"]["image"] == "postgres:16" + assert runtime_config["compose"]["service_access"] is True assert str(compose) not in json.dumps(runtime_config) diff --git a/hud/eval/tests/test_local_runtime.py b/hud/eval/tests/test_local_runtime.py index 39b404d9f..08b54ec95 100644 --- a/hud/eval/tests/test_local_runtime.py +++ b/hud/eval/tests/test_local_runtime.py @@ -1,16 +1,8 @@ -"""Local placement: LocalRuntime and the no-runtime resolution ladder. - -LocalRuntime serves a fresh env per rollout from any pointer to it — a source -path (throwaway import), a live module-level env (its declaring file is the -recipe), or a ``(task) -> Environment`` constructor. With no runtime, a run -uses what is already known: taskset origin, then envs declared in imported -modules, else a loud error. Everything crosses the real control channel — -these tests drive the rollout engine end to end. -""" +"""LocalRuntime serves environments from live instances and source recipes.""" from __future__ import annotations -import importlib.util +import asyncio import sys from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime from typing import Any, cast @@ -20,7 +12,6 @@ from hud.agents.base import Agent from hud.environment import Environment from hud.eval import LocalRuntime, Task, Taskset -from hud.eval.run import rollout _SUMS_ENV = """\ from hud import Environment @@ -35,25 +26,6 @@ async def add(a: int, b: int): """ -@pytest.fixture -def imported_env(tmp_path, request): - """Write an env module, import it for real, and clean it up after. - - The module stays in ``sys.modules`` for the test's duration — the state a - user's ``from env import add`` leaves behind. - """ - module_name = f"_sums_mod_{request.node.name}" - file = tmp_path / f"{module_name}.py" - file.write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") - spec = importlib.util.spec_from_file_location(module_name, file) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - yield module - del sys.modules[module_name] - - def _sums_env(name: str = "sums") -> Environment: env = Environment(name) @@ -80,7 +52,7 @@ def _solve_add(prompt: str) -> str: return str(int(a) + int(b)) -# ─── LocalRuntime: the three pointer forms ───────────────────────────── +# ─── LocalRuntime sources ────────────────────────────────────────────── async def test_source_path_serves_a_fresh_env_per_rollout(tmp_path) -> None: @@ -111,21 +83,6 @@ def _solve(prompt: str) -> str: assert [run.reward for run in job.runs] == [1.0, 1.0] -async def test_live_env_pointer_resolves_to_its_declaring_file(imported_env) -> None: - run = await rollout( - Task(env="sums", id="add", args={"a": 2, "b": 3}), - _FnAgent(_solve_add), - runtime=LocalRuntime(imported_env.env), - ) - - assert run.reward == 1.0 - - -def test_live_env_without_a_declaring_file_is_rejected() -> None: - with pytest.raises(TypeError, match="constructor instead"): - LocalRuntime(_sums_env()) - - async def test_constructor_builds_fresh_per_rollout_from_the_row() -> None: built: list[str] = [] @@ -144,6 +101,52 @@ def env_for(task: Task) -> Environment: assert built == ["sums", "sums", "sums"] +async def test_live_environment_is_served_serially() -> None: + env = Environment("sums") + active = 0 + + @env.template(id="add") + async def add(a: int, b: int): + answer = yield f"add:{a}:{b}" + yield 1.0 if answer == str(a + b) else 0.0 + + @env.initialize + async def _start() -> None: + nonlocal active + active += 1 + assert active == 1 + await asyncio.sleep(0) + + @env.shutdown + async def _stop() -> None: + nonlocal active + active -= 1 + + job = await add(a=2, b=3).run( + _FnAgent(_solve_add), + group=2, + max_concurrent=2, + ) + + assert [run.reward for run in job.runs] == [1.0, 1.0] + assert active == 0 + + +async def test_serialized_task_does_not_retain_local_placement() -> None: + env = Environment("sums") + + @env.template(id="add") + async def add(a: int, b: int): + yield f"add:{a}:{b}" + yield 1.0 + + task = add(a=2, b=3) + portable = Task.model_validate(task.model_dump()) + + with pytest.raises(ValueError, match="no placement: pass runtime="): + await portable.run(_FnAgent(_solve_add)) + + async def test_source_missing_env_name_fails_loudly(tmp_path) -> None: (tmp_path / "env.py").write_text( 'from hud import Environment\n\nenv = Environment("sums")\n', @@ -156,96 +159,45 @@ async def test_source_missing_env_name_fails_loudly(tmp_path) -> None: pass -# ─── the no-runtime resolution ladder ────────────────────────────────── - - -async def test_module_loaded_taskset_serves_its_source_by_default(tmp_path, request) -> None: - (tmp_path / "env.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") - (tmp_path / "tasks.py").write_text( - "from env import add\n\ntasks = [add(a=2, b=3), add(a=4, b=5)]\n", +async def test_module_taskset_uses_factory_environments(tmp_path) -> None: + source = tmp_path / "tasks.py" + source.write_text( + _SUMS_ENV.format(name="sums") + "\ntasks = [add(a=2, b=3), add(a=4, b=5)]\n", encoding="utf-8", ) - # tasks.py's `from env import add` imports env.py normally, so it outlives - # the throwaway tasks module. - request.addfinalizer(lambda: sys.modules.pop("env", None)) - taskset = Taskset.from_module(tmp_path / "tasks.py") - job = await taskset.run(_FnAgent(_solve_add)) + job = await Taskset.from_module(source).run(_FnAgent(_solve_add)) - assert len(job.runs) == 2 - assert all(run.reward == 1.0 for run in job.runs) + assert [run.reward for run in job.runs] == [1.0, 1.0] -async def test_minted_tasks_resolve_a_declared_env_by_name(imported_env) -> None: - job = await Taskset("sums", [imported_env.add(a=2, b=3), imported_env.add(a=4, b=5)]).run( - _FnAgent(_solve_add) +async def test_tasks_module_uses_factory_environment(tmp_path, request) -> None: + module_name = f"sums_env_{request.node.name}" + (tmp_path / f"{module_name}.py").write_text( + _SUMS_ENV.format(name="sums"), + encoding="utf-8", + ) + tasks = tmp_path / "tasks.py" + tasks.write_text( + f"from {module_name} import add\n\ntasks = [add(a=2, b=3)]\n", + encoding="utf-8", ) + request.addfinalizer(lambda: sys.modules.pop(module_name, None)) - assert len(job.runs) == 2 - assert all(run.reward == 1.0 for run in job.runs) + job = await Taskset.from_module(tasks).run(_FnAgent(_solve_add)) + assert job.reward == 1.0 -async def test_inferred_placement_includes_a_verifier_environment(tmp_path, request) -> None: - actor_name = f"actor-{request.node.name}" - verifier_name = f"verifier-{request.node.name}" - module_name = f"_verifier_placement_{request.node.name}" - source = tmp_path / f"{module_name}.py" - source.write_text( - f"""from hud import Environment -actor = Environment("{actor_name}") -verifier = Environment("{verifier_name}") +async def test_ad_hoc_taskset_requires_explicit_placement() -> None: + with pytest.raises(ValueError, match="no placement: pass runtime="): + await Taskset("sums", [Task(env="sums", id="add")]).run(_FnAgent(_solve_add)) -@actor.template(id="solve") -async def solve(): - answer = yield "answer" - yield {{"score": 0.25, "answer": answer}} -@verifier.template(id="verify") -async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "secret" else 0.0 -""", - encoding="utf-8", - ) - spec = importlib.util.spec_from_file_location(module_name, source) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - - try: - task = Task( - env=actor_name, - id="solve", - verifier=Task(env=verifier_name, id="verify"), - ) - job = await task.run(_FnAgent(lambda _: "secret")) - finally: - del sys.modules[module_name] - - assert job.runs[0].reward == 1.0 - - -async def test_no_placement_fails_with_the_forms_to_pass() -> None: - with pytest.raises(ValueError, match="no placement for env"): - await Task(env="ghost", id="add").run(_FnAgent(_solve_add)) - - -async def test_ambiguous_env_name_fails_loudly(imported_env, tmp_path, request) -> None: - module_name = f"_sums_rival_{request.node.name}" - file = tmp_path / f"{module_name}.py" - file.write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") - spec = importlib.util.spec_from_file_location(module_name, file) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - try: - spec.loader.exec_module(module) - with pytest.raises(ValueError, match="LocalRuntime\\(env\\)"): - await Task(env="sums", id="add").run(_FnAgent(_solve_add)) - finally: - del sys.modules[module_name] +async def test_empty_taskset_needs_no_placement() -> None: + job = await Taskset("empty", []).run(_FnAgent(_solve_add)) + + assert job.runs == [] def test_rejects_a_non_pointer_argument() -> None: @@ -254,8 +206,6 @@ def test_rejects_a_non_pointer_argument() -> None: async def test_failed_startup_still_runs_shutdown_hooks() -> None: - from hud.eval.runtime import _local - env = _sums_env() lifecycle: list[str] = [] @@ -272,116 +222,12 @@ async def _boom() -> None: raise RuntimeError("daemon failed to start") with pytest.raises(RuntimeError, match="daemon failed to start"): - async with _local(env): + async with LocalRuntime(env)(Task(env="sums", id="add")): pass assert lifecycle == ["up", "down"] -async def test_tasks_only_module_resolves_envs_imported_from_elsewhere( - tmp_path, monkeypatch, request -) -> None: - # The env lives in a separate importable package, not next to tasks.py: - # the origin declares no envs, so resolution falls through to the live - # env the tasks module imported. - env_dir = tmp_path / "pkg" - env_dir.mkdir() - (env_dir / "sums_envmod.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") - tasks_dir = tmp_path / "tasks" - tasks_dir.mkdir() - (tasks_dir / "tasks.py").write_text( - "from sums_envmod import add\n\ntasks = [add(a=2, b=3)]\n", - encoding="utf-8", - ) - monkeypatch.syspath_prepend(str(env_dir)) - request.addfinalizer(lambda: sys.modules.pop("sums_envmod", None)) - - taskset = Taskset.from_module(tasks_dir / "tasks.py") - job = await taskset.run(_FnAgent(_solve_add)) - - assert [run.reward for run in job.runs] == [1.0] - - -async def test_single_file_taskset_never_drags_in_a_same_named_sibling(tmp_path) -> None: - (tmp_path / "env_a.py").write_text( - _SUMS_ENV.format(name="sums") + "\ntasks = [add(a=2, b=3)]\n", encoding="utf-8" - ) - (tmp_path / "env_b.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") - - taskset = Taskset.from_module(tmp_path / "env_a.py") - job = await taskset.run(_FnAgent(_solve_add)) - - assert [run.reward for run in job.runs] == [1.0] - - -async def test_empty_taskset_runs_without_a_placement() -> None: - job = await Taskset("empty", []).run(_FnAgent(_solve_add)) - - assert job.runs == [] - - -async def test_reexporting_tasks_module_does_not_claim_the_env( - tmp_path, monkeypatch, request -) -> None: - # tasks.py re-exports the env object alongside the factory: the origin - # must not claim it (re-import of tasks.py would reuse the cached env - # module) — each rollout rebuilds the env from its real file instead. - env_dir = tmp_path / "pkg" - env_dir.mkdir() - (env_dir / "sums_reexp_envmod.py").write_text( - "from hud import Environment\n\n" - "LOADS = []\n" - 'env = Environment("sums")\n\n\n' - '@env.template(id="add")\nasync def add(a: int, b: int):\n' - " LOADS.append(1)\n" - ' answer = yield f"add:{a}:{b}:{len(LOADS)}"\n' - " yield 1.0 if answer == str(a + b) else 0.0\n", - encoding="utf-8", - ) - tasks_dir = tmp_path / "tasks" - tasks_dir.mkdir() - (tasks_dir / "tasks.py").write_text( - "from sums_reexp_envmod import add, env\n\ntasks = [add(a=2, b=3)]\n", - encoding="utf-8", - ) - monkeypatch.syspath_prepend(str(env_dir)) - request.addfinalizer(lambda: sys.modules.pop("sums_reexp_envmod", None)) - - def _solve(prompt: str) -> str: - _, a, b, loads = prompt.split(":") - assert loads == "1" # a fresh env module per rollout, not the cached one - return str(int(a) + int(b)) - - taskset = Taskset.from_module(tasks_dir / "tasks.py") - job = await taskset.run(_FnAgent(_solve), group=2) - - assert [run.reward for run in job.runs] == [1.0, 1.0] - - -async def test_package_reexported_env_pointer_uses_the_declaring_submodule( - tmp_path, monkeypatch, request -) -> None: - pkg = tmp_path / "sums_pkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("from .env_mod import env\n", encoding="utf-8") - (pkg / "env_mod.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") - monkeypatch.syspath_prepend(str(tmp_path)) - request.addfinalizer( - lambda: [sys.modules.pop(m, None) for m in ("sums_pkg", "sums_pkg.env_mod")] - ) - import importlib - - sums_pkg = importlib.import_module("sums_pkg") - - run = await rollout( - Task(env="sums", id="add", args={"a": 2, "b": 3}), - _FnAgent(_solve_add), - runtime=LocalRuntime(sums_pkg.env), - ) - - assert run.reward == 1.0 - - async def test_template_can_lazily_import_a_sibling_module(tmp_path) -> None: (tmp_path / "lazy_helper.py").write_text("ANSWER_SUFFIX = ':ok'\n", encoding="utf-8") (tmp_path / "env.py").write_text( @@ -425,16 +271,3 @@ async def test_unguarded_run_call_in_source_names_the_mistake(tmp_path) -> None: with pytest.raises(RuntimeError, match='if __name__ == "__main__"'): async with provider(Task(env="sums", id="add")): pass - - -async def test_live_env_mutated_after_import_fails_with_the_cause(imported_env) -> None: - @imported_env.env.template(id="patched") - async def patched() -> Any: - answer = yield "noop" - yield 1.0 if answer else 0.0 - - provider = LocalRuntime(imported_env.env) - - with pytest.raises(ValueError, match="modified after import"): - async with provider(Task(env="sums", id="patched")): - pass diff --git a/hud/eval/tests/test_rollout.py b/hud/eval/tests/test_rollout.py index 90755a4d8..5420ef345 100644 --- a/hud/eval/tests/test_rollout.py +++ b/hud/eval/tests/test_rollout.py @@ -32,9 +32,9 @@ from hud.agents.openai_compatible import OpenAIChatAgent from hud.agents.types import OpenAIChatConfig from hud.environment import Answer, Environment -from hud.eval import Job, Runtime, SubprocessRuntime, Task, Taskset +from hud.eval import Job, LocalRuntime, Runtime, SubprocessRuntime, Task, Taskset from hud.eval.run import Run, rollout -from hud.eval.runtime import _local +from hud.eval.runtime import RuntimeSession if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -188,7 +188,7 @@ async def verify(expected: str): id="solve", verifier=Task(env="reviewed", id="verify", args={"expected": "secret"}), ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=lambda _row: _local(env)) + run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) assert run.reward == 1.0 assert completed == ["actor:secret", "verifier:secret"] @@ -202,24 +202,23 @@ async def verify(expected: str): async def test_actor_result_is_forwarded_to_the_verifier() -> None: env = Environment("reviewed") - received: list[tuple[str, str]] = [] + received: list[str] = [] class ActorResult(BaseModel): score: float answer: str - handoff: str @env.template() async def solve(): answer = yield "answer secret" - yield {"score": 0.0, "answer": answer, "handoff": "token"} + yield {"score": 0.0, "answer": answer} @env.template(returns=ActorResult) async def verify(): answer = yield "" assert isinstance(answer, Answer) assert isinstance(answer.content, ActorResult) - received.append((answer.content.answer, answer.content.handoff)) + received.append(answer.content.answer) yield 1.0 task = Task( @@ -227,59 +226,106 @@ async def verify(): id="solve", verifier=Task(env="reviewed", id="verify"), ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=lambda _row: _local(env)) + run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) assert run.reward == 1.0 - assert received == [("secret", "token")] + assert received == ["secret"] -async def test_independent_verifier_receives_runtime_handoff() -> None: +async def test_malformed_subscores_fail_inside_the_rollout_boundary( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from hud.clients import HudClient + + env = Environment("malformed-grade") + + @env.template() + async def solve(): + yield "answer" + yield 1.0 + + async def malformed_grade( + _client: HudClient, + _payload: dict[str, Any], + ) -> dict[str, Any]: + return {"score": 1.0, "subscores": [{"name": "missing-value"}]} + + reported: list[dict[str, Any]] = [] + + async def report(run: Run) -> None: + reported.append(run.evaluation) + + monkeypatch.setattr(HudClient, "grade", malformed_grade) + monkeypatch.setattr("hud.eval.run.trace_exit", report) + + run = await rollout( + Task(env="malformed-grade", id="solve"), + _FnAgent(lambda _prompt: "done"), + runtime=LocalRuntime(env), + ) + + assert run.trace.is_error + assert "value" in (run.trace.error or "") + assert reported == [{}] + + +async def test_independent_verifier_receives_runtime_session_files(tmp_path: Path) -> None: actor_env = Environment("actor") verifier_env = Environment("judge") - transfers: list[str] = [] + transfers: list[tuple[str, str]] = [] @actor_env.template() async def solve(): yield "answer secret" - yield {"score": 0.0, "handoff": "token"} + yield {"score": 0.0} @verifier_env.template() async def verify(): - result = yield "" - yield 1.0 if result["handoff"] == "token" else 0.0 + yield "" + yield 1.0 - class ActorHandoff: - async def export_to(self, destination: Path) -> None: - await asyncio.to_thread(destination.write_text, "files", encoding="utf-8") - transfers.append("export") + class ActorSession(RuntimeSession): + @asynccontextmanager + async def snapshot(self) -> AsyncIterator[Path | None]: + destination = tmp_path / "session.tar.gz" + await asyncio.to_thread(destination.write_text, self.session_id, encoding="utf-8") + transfers.append(("actor", self.session_id)) + yield destination - async def import_from(self, source: Path) -> None: - raise AssertionError("actor imported a handoff") + class VerifierSession(RuntimeSession): + async def restore(self, source: Path) -> None: + content = await asyncio.to_thread(source.read_text, encoding="utf-8") + assert content.startswith("sess-") + transfers.append(("verifier", self.session_id)) - class VerifierHandoff: - async def export_to(self, destination: Path) -> None: - raise AssertionError("verifier exported a handoff") + class ActorRuntime(Runtime): + def session(self, session_id: str) -> RuntimeSession: + return ActorSession(session_id) - async def import_from(self, source: Path) -> None: - assert await asyncio.to_thread(source.read_text, "utf-8") == "files" - transfers.append("import") + class VerifierRuntime(Runtime): + def session(self, session_id: str) -> RuntimeSession: + return VerifierSession(session_id) @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: env = actor_env if row.env == "actor" else verifier_env - endpoint = ActorHandoff() if row.env == "actor" else VerifierHandoff() - async with _local(env) as runtime: - yield Runtime(runtime.url, handoff=endpoint) + async with LocalRuntime(env)(row) as runtime: + yield ( + ActorRuntime(runtime.url) if row.env == "actor" else VerifierRuntime(runtime.url) + ) task = Task( env="actor", id="solve", - verifier=Task(env="judge", id="verify", requires_handoff=True), + verifier=Task(env="judge", id="verify"), ) run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) assert run.reward == 1.0 - assert transfers == ["export", "import"] + assert [runtime for runtime, _ in transfers] == ["actor", "verifier"] + assert transfers[0][1].startswith("sess-") + assert transfers[1][1].startswith("sess-") + assert transfers[0][1] != transfers[1][1] async def test_verifier_with_its_own_environment_is_placed_after_the_actor() -> None: @@ -299,9 +345,14 @@ async def verify(): @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.env) - async with _local(actor_env if row.env == "actor" else verifier_env) as runtime: - yield runtime + placements.append(f"start:{row.env}") + try: + async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)( + row + ) as runtime: + yield runtime + finally: + placements.append(f"stop:{row.env}") task = Task( env="actor", @@ -311,7 +362,7 @@ async def provider(row: TaskRow) -> AsyncIterator[Runtime]: run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) assert run.reward == 1.0 - assert placements == ["actor", "judge"] + assert placements == ["start:actor", "stop:actor", "start:judge", "stop:judge"] async def test_verifier_remains_authoritative_after_an_agent_error() -> None: @@ -332,7 +383,7 @@ async def verify(): @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: placements.append(row.env) - async with _local(actor_env if row.env == "actor" else verifier_env) as runtime: + async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)(row) as runtime: yield runtime task = Task( @@ -369,7 +420,7 @@ async def verify(): @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: placements.append(row.env) - async with _local(actor_env if row.env == "actor" else verifier_env) as runtime: + async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)(row) as runtime: yield runtime task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) @@ -408,7 +459,7 @@ async def verify(): @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: placements.append(row.env) - async with _local(actor_env if row.env == "actor" else verifier_env) as runtime: + async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)(row) as runtime: yield runtime original_grade = HudClient.grade @@ -449,7 +500,7 @@ async def solve(): async def provider(row: TaskRow) -> AsyncIterator[Runtime]: if row.env == "judge": raise RuntimeError("verifier unavailable") - async with _local(actor_env) as runtime: + async with LocalRuntime(actor_env)(row) as runtime: yield runtime task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) @@ -495,7 +546,7 @@ async def fail_verifier_grade(self: HudClient, payload: dict[str, Any]) -> dict[ verifier=Task(env="reviewed", id="verify"), ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=lambda _row: _local(env)) + run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) job = Job(id="verify-failed", name="verify-failed", runs=[run]) assert run.trace.is_error @@ -538,7 +589,7 @@ async def return_scoreless_frame(self: HudClient, payload: dict[str, Any]) -> di verifier=Task(env="reviewed", id="verify"), ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=lambda _row: _local(env)) + run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) job = Job(id="verify-scoreless", name="verify-scoreless", runs=[run]) assert run.trace.is_error @@ -568,7 +619,7 @@ async def verify(): run = await rollout( task, _AnswerThenBoomAgent(lambda _prompt: "secret"), - runtime=lambda _row: _local(env), + runtime=LocalRuntime(env), ) job = Job(id="verify-zero", name="verify-zero", runs=[run]) @@ -598,9 +649,7 @@ async def __call__(self, run: Any) -> None: seen.update(run.bindings) env = _bindings_env({"robot": {"token": "slot-2"}}) - run = await rollout( - Task(env="slots", id="claim"), _Claiming(), runtime=lambda _task: _local(env) - ) + run = await rollout(Task(env="slots", id="claim"), _Claiming(), runtime=LocalRuntime(env)) # Episode-scoped connection data reaches the agent by capability name, # without reading it back off the recorded setup step. @@ -611,7 +660,7 @@ async def __call__(self, run: Any) -> None: async def test_a_malformed_bindings_frame_fails_the_rollout_loudly() -> None: env = _bindings_env({"robot": "slot-2"}) # capability data must be an object run = await rollout( - Task(env="slots", id="claim"), _FnAgent(lambda _p: ""), runtime=lambda _task: _local(env) + Task(env="slots", id="claim"), _FnAgent(lambda _p: ""), runtime=LocalRuntime(env) ) assert run.trace.status == "error" @@ -650,7 +699,7 @@ async def write_report(): run = await rollout( Task(env="opencode_report", id="write_report"), agent, - runtime=lambda _task: _local(env), + runtime=LocalRuntime(env), ) assert run.reward == 1.0 @@ -711,7 +760,7 @@ async def wait_for_cleanup(): run = await rollout( Task(env="timeout_cleanup", id="wait_for_cleanup"), agent, - runtime=lambda _task: _local(env), + runtime=LocalRuntime(env), ) assert run.trace.status == "error" @@ -839,7 +888,7 @@ async def add(a: int, b: int): run = await rollout( _add_task(2, 3), agent, - runtime=lambda _row: _local(env), + runtime=LocalRuntime(env), rollout_timeout=0.2, ) @@ -862,7 +911,7 @@ async def add(a: int, b: int): agent = _SlowAgent(_solve_add) task = _add_task(2, 3).model_copy(update={"agent_config": {"timeout_seconds": 0.05}}) - run = await rollout(task, agent, runtime=lambda _row: _local(env)) + run = await rollout(task, agent, runtime=LocalRuntime(env)) assert run.reward == 1.0 assert run.trace.status == "error" @@ -891,7 +940,7 @@ async def __call__(self, run: Any) -> None: task = _add_task(2, 3).model_copy( update={"agent_config": {"timeout_seconds": agent_timeout} if agent_timeout else None} ) - run = await rollout(task, TimeoutAgent(), runtime=lambda _row: _local(env)) + run = await rollout(task, TimeoutAgent(), runtime=LocalRuntime(env)) assert run.reward == 1.0 assert run.trace.status == "error" @@ -917,7 +966,7 @@ async def add(a: int, b: int): run = await rollout( _add_task(2, 3), _FnAgent(_solve_add), - runtime=lambda _row: _local(env), + runtime=LocalRuntime(env), rollout_timeout=0.2, ) @@ -956,7 +1005,7 @@ def track_abort(self: HudClient) -> None: run = await rollout( _add_task(2, 3), _SlowAgent(_solve_add), - runtime=lambda _row: _local(env), + runtime=LocalRuntime(env), rollout_timeout=0.2, ) elapsed = loop.time() - started @@ -982,7 +1031,7 @@ async def add(a: int, b: int): @asynccontextmanager async def provider(_task: TaskRow) -> AsyncIterator[Runtime]: try: - async with _local(env) as runtime: + async with LocalRuntime(env)(_task) as runtime: yield runtime finally: cleanup_started.set() @@ -1024,7 +1073,7 @@ async def provider(row: TaskRow) -> AsyncIterator[Runtime]: if row.env == "judge": raise AssertionError("verifier started after the rollout returned") try: - async with _local(actor_env) as runtime: + async with LocalRuntime(actor_env)(row) as runtime: yield runtime finally: cleanup_started.set() @@ -1068,7 +1117,9 @@ async def verify(): @asynccontextmanager async def provider(row: TaskRow) -> AsyncIterator[Runtime]: try: - async with _local(actor_env if row.env == "actor" else verifier_env) as runtime: + async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)( + row + ) as runtime: yield runtime finally: if row.env == "judge": diff --git a/hud/eval/tests/test_shared.py b/hud/eval/tests/test_shared.py index a3a52fbe1..71d744347 100644 --- a/hud/eval/tests/test_shared.py +++ b/hud/eval/tests/test_shared.py @@ -18,8 +18,8 @@ from hud.agents.base import Agent from hud.environment import Environment -from hud.eval import Shared, Task, Taskset -from hud.eval.runtime import Runtime, _local +from hud.eval import LocalRuntime, RuntimeConfig, Shared, Task, Taskset +from hud.eval.runtime import Runtime if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -28,10 +28,10 @@ class _CountingProvider: - """Wraps ``_local`` around one env, counting provisions and teardowns.""" + """Wraps one local env, counting provisions and teardowns.""" def __init__(self, env: Environment) -> None: - self.env = env + self.runtime = LocalRuntime(env) self.provisions = 0 self.teardowns = 0 @@ -39,7 +39,7 @@ def __init__(self, env: Environment) -> None: async def __call__(self, task: TaskRow) -> AsyncIterator[Runtime]: self.provisions += 1 try: - async with _local(self.env) as runtime: + async with self.runtime(task) as runtime: yield runtime finally: self.teardowns += 1 @@ -88,6 +88,72 @@ async def test_an_open_scope_keeps_the_substrate_warm_across_calls() -> None: assert inner.teardowns == 1 +async def test_distinct_task_placements_get_distinct_substrates() -> None: + provisions: list[str] = [] + teardowns: list[str] = [] + + @asynccontextmanager + async def provider(task: Task) -> AsyncIterator[Runtime]: + provisions.append(task.env) + try: + yield Runtime(f"tcp://{task.env}") + finally: + teardowns.append(task.env) + + async with Shared(provider, width=1) as shared: + async with shared(Task(env="actor", id="solve")) as actor: + assert actor.url == "tcp://actor" + async with shared(Task(env="verifier", id="verify")) as verifier: + assert verifier.url == "tcp://verifier" + + assert provisions == ["actor", "verifier"] + assert teardowns == ["verifier", "actor"] + + +async def test_width_one_releases_the_actor_lease_before_verifying() -> None: + env = Environment("reviewed") + + @env.template() + async def solve(): + answer = yield "answer" + yield {"score": 0.0, "answer": answer} + + @env.template() + async def verify(): + result = yield "" + yield 1.0 if result["answer"] == "ok" else 0.0 + + provisions = 0 + + @asynccontextmanager + async def provider(task: Task) -> AsyncIterator[Runtime]: + nonlocal provisions + provisions += 1 + async with LocalRuntime(env)(task.model_copy(update={"runtime_config": None})) as runtime: + yield runtime + + config = RuntimeConfig() + task = Task( + env="reviewed", + id="solve", + runtime_config=config, + verifier=Task( + env="reviewed", + id="verify", + runtime_config=config, + ), + ) + + job = await task.run( + _OkAgent(), + runtime=Shared(provider, width=1), + rollout_timeout=1, + ) + + assert job.reward == 1.0 + assert provisions == 1 + + async def test_width_bounds_occupancy_by_waiting_not_erroring() -> None: live = 0 peak = 0 @@ -119,7 +185,7 @@ async def flaky(task: TaskRow) -> AsyncIterator[Runtime]: attempts += 1 if attempts == 1: raise RuntimeError("boot boom") - async with _local(env) as runtime: + async with LocalRuntime(env)(task) as runtime: yield runtime job = await Taskset("pool", [Task(env="pool", id="echo")]).run( diff --git a/hud/eval/tests/test_sync.py b/hud/eval/tests/test_sync.py index 09252d75f..0b43e4a9c 100644 --- a/hud/eval/tests/test_sync.py +++ b/hud/eval/tests/test_sync.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any from hud.eval import Task, Taskset -from hud.eval.runtime import RuntimeConfig +from hud.eval.runtime import ComposeProject, RuntimeConfig from hud.eval.sync import ( diff, fetch_taskset_tasks, @@ -171,13 +171,16 @@ def test_task_upload_payload_embeds_compose_document(tmp_path: Path) -> None: task = Task( env="e", id="solve", - runtime_config=RuntimeConfig(compose=compose, compose_service_access=True), + runtime_config=RuntimeConfig(compose=ComposeProject(document=compose, service_access=True)), ) payload = task_upload_payload(task) - assert payload["runtime_config"]["compose"]["services"]["database"]["image"] == "postgres:16" - assert payload["runtime_config"]["compose_service_access"] is True + assert ( + payload["runtime_config"]["compose"]["document"]["services"]["database"]["image"] + == "postgres:16" + ) + assert payload["runtime_config"]["compose"]["service_access"] is True assert str(compose) not in json.dumps(payload) @@ -205,7 +208,7 @@ def test_task_upload_payload_includes_verifier_task(tmp_path: Path) -> None: verifier=Task( env="judge", id="verify", - runtime_config=RuntimeConfig(compose=compose), + runtime_config=RuntimeConfig(compose=ComposeProject(document=compose)), ), ) @@ -219,5 +222,5 @@ def test_task_upload_payload_includes_verifier_task(tmp_path: Path) -> None: "args": {}, "slug": "verify", } - assert runtime_config["compose"]["services"]["main"]["image"] == "judge:latest" + assert runtime_config["compose"]["document"]["services"]["main"]["image"] == "judge:latest" assert str(compose) not in json.dumps(runtime_config) diff --git a/hud/eval/tests/test_task.py b/hud/eval/tests/test_task.py index cbec55c7e..a9f4de6eb 100644 --- a/hud/eval/tests/test_task.py +++ b/hud/eval/tests/test_task.py @@ -3,8 +3,8 @@ The model is the row: plain pydantic (``model_validate``/``model_dump``) is the whole codec for ``hud sync`` and the JSON/JSONL taskset path. ``env`` is carried as its name, the join key to whatever placement can bring that environment up. -Placement is never part of the row — without an ``runtime=`` provider, execution -defaults to the HUD runtime tunnel by env name. +Placement is never part of the row. Factory-created tasks can run locally; +portable rows require a provider or source placement. """ from __future__ import annotations @@ -25,7 +25,7 @@ Task, Taskset, ) -from hud.eval.runtime.compose import ComposeConfig, ComposeProjectRef +from hud.eval.runtime.compose import ComposeConfig, ComposeProject, ComposeProjectRef if TYPE_CHECKING: from pathlib import Path @@ -209,26 +209,32 @@ def test_compose_runtime_config_serializes_as_task_data(tmp_path: Path) -> None: json.dumps({"services": {"main": {"image": "hud-env:latest"}}}), encoding="utf-8", ) - config = RuntimeConfig(compose=compose) + config = RuntimeConfig(compose=ComposeProject(document=compose)) task = Task(env="database", id="cutover", runtime_config=config) - rebuilt = Task.model_validate(task.model_dump()) + payload = task.model_dump(exclude_none=True) + rebuilt = Task.model_validate(payload) - assert rebuilt.runtime_config == config - assert config.request_payload() == { + assert payload["runtime_config"] == { "compose": { - "services": { - "main": { - "image": "hud-env:latest", - "environment": {}, - "expose": [], - "ports": [], - "volumes": [], - } + "document": { + "services": { + "main": { + "image": "hud-env:latest", + "environment": {}, + "expose": [], + "ports": [], + "volumes": [], + } + }, + "networks": {}, }, - "networks": {}, } } + assert rebuilt.runtime_config is not None + assert rebuilt.runtime_config.compose is not None + assert isinstance(rebuilt.runtime_config.compose.document, ComposeConfig) + assert str(compose) not in json.dumps(payload) def test_compose_project_serializes_document_location_not_author_path(tmp_path: Path) -> None: @@ -250,28 +256,34 @@ def test_compose_project_serializes_document_location_not_author_path(tmp_path: encoding="utf-8", ) - payload = RuntimeConfig(compose=compose, compose_project=project).request_payload() + payload = RuntimeConfig(compose=ComposeProject(document=compose, root=project)).model_dump( + mode="json", + exclude_unset=True, + ) - assert payload["compose_project"] == {"compose_path": "compose-project/compose.json"} + assert payload["compose"]["root"] == {"compose_path": "compose-project/compose.json"} assert str(tmp_path) not in json.dumps(payload) def test_compose_runtime_config_round_trips_platform_records() -> None: record = { "compose": { - "services": {"main": {"image": "hud-harbor:local"}}, - "networks": {}, + "document": { + "services": {"main": {"image": "hud-harbor:local"}}, + "networks": {}, + }, + "root": {"compose_path": "compose-project/compose.json"}, }, - "compose_project": {"compose_path": "compose-project/compose.json"}, } config = RuntimeConfig.model_validate(record) - assert isinstance(config.compose, ComposeConfig) - assert isinstance(config.compose_project, ComposeProjectRef) - payload = config.request_payload() - assert payload["compose"]["services"]["main"]["image"] == "hud-harbor:local" - assert payload["compose_project"] == {"compose_path": "compose-project/compose.json"} + assert config.compose is not None + assert isinstance(config.compose.document, ComposeConfig) + assert isinstance(config.compose.root, ComposeProjectRef) + payload = config.model_dump(mode="json", exclude_unset=True) + assert payload["compose"]["document"]["services"]["main"]["image"] == "hud-harbor:local" + assert payload["compose"]["root"] == {"compose_path": "compose-project/compose.json"} assert RuntimeConfig.model_validate(payload) == config @@ -304,7 +316,7 @@ async def fake_rollout(task: Task, agent: Agent, **kwargs: object) -> Run: monkeypatch.setattr(taskset_mod, "rollout", fake_rollout) task = Task(env="hosted-env", id="solve", args={"n": 1}) - taskset = taskset_mod.Taskset("hosted", [task], origin="api:ts_123") + taskset = taskset_mod.Taskset("hosted", [task], taskset_id="ts_123") job = await taskset.run(cast("Agent", object())) (run,) = job.runs @@ -326,12 +338,16 @@ def test_taskset_is_ordered_and_keyed_by_slug() -> None: verifier=Task(env="judge", id="verify"), ) - tasks = Taskset("demo", [first, second]) + tasks = Taskset("demo", [first, second], taskset_id="ts_123") assert list(tasks) == [first, second] assert tasks["first"] is first - assert list(tasks.filter(["second"])) == [second] - assert list(tasks.exclude(["first"])) == [second] + filtered = tasks.filter(["second"]) + excluded = tasks.exclude(["first"]) + assert list(filtered) == [second] + assert list(excluded) == [second] + assert filtered.taskset_id == "ts_123" + assert excluded.taskset_id == "ts_123" assert list(tasks.items()) == [("first", first), ("second", second)] assert tasks.environment_names() == {"e", "judge"} @@ -365,6 +381,35 @@ def test_file_roundtrip_keeps_rows_and_env_names(tmp_path) -> None: assert list(loaded) == authored # rows survive the file intact (value equality) +def test_taskset_file_preserves_local_compose_project(tmp_path: Path) -> None: + compose = tmp_path / "compose.yaml" + compose.write_text("services:\n main:\n image: alpine:3.21\n", encoding="utf-8") + output = Taskset( + "compose", + [ + Task( + env="compose", + id="solve", + runtime_config=RuntimeConfig( + compose=ComposeProject(document=compose, root=tmp_path), + ), + ), + ], + ).to_file(tmp_path / "tasks.json") + + row = json.loads(output.read_text(encoding="utf-8"))[0] + assert row["runtime_config"]["compose"] == { + "document": "compose.yaml", + "root": ".", + } + loaded = next(iter(Taskset.from_file(output))) + assert loaded.runtime_config is not None + assert loaded.runtime_config.compose == ComposeProject( + document=compose.resolve(), + root=tmp_path.resolve(), + ) + + def test_taskset_to_file_writes_json_and_jsonl(tmp_path) -> None: taskset = Taskset( "demo", @@ -427,6 +472,7 @@ def fake_request(method: str, url: str, **kwargs: object) -> dict[str, object]: taskset = Taskset.from_api("demo") assert taskset.name == "Demo" + assert taskset.taskset_id == "ts_123" assert taskset["one"].id == "solve" assert taskset["one"].env == "e" assert taskset["one"].args == {"n": 1} diff --git a/hud/integrations/harbor/adapt.py b/hud/integrations/harbor/adapt.py index 6dfec6822..cba12e9c7 100644 --- a/hud/integrations/harbor/adapt.py +++ b/hud/integrations/harbor/adapt.py @@ -24,6 +24,7 @@ from hud.eval.runtime.compose import ( ComposeConfig, ComposeHealthcheck, + ComposeProject, ComposeService, ComposeUnboundVariableError, ) @@ -1099,9 +1100,11 @@ def adapt( ), columns=columns or None, runtime_config=RuntimeConfig( - compose=context / "compose-project" / "compose.json", - compose_project=context, - compose_service_access=(True if needs_service_access else None), + compose=ComposeProject( + document=context / "compose-project" / "compose.json", + root=context, + service_access=(True if needs_service_access else None), + ), resources=task.resources, limits=_runtime_limits(config.environment), ), @@ -1111,11 +1114,12 @@ def adapt( id="verify", args={"task": task_config}, slug=f"{task.path.name}:verify", - requires_handoff=True, runtime_config=( RuntimeConfig( - compose=context / "compose-project" / "compose.json", - compose_project=context, + compose=ComposeProject( + document=context / "compose-project" / "compose.json", + root=context, + ), resources=verifier_resources, limits=verifier_limits, ) @@ -1133,6 +1137,6 @@ def adapt( LOGGER.info("adapted %d Harbor project(s)", len({task.env for task in rows})) return AdaptResult( - taskset=Taskset(dataset.name, rows, origin=f"harbor:{dataset}"), + taskset=Taskset(dataset.name, rows), failures=tuple(failures), ) diff --git a/hud/integrations/harbor/env.py b/hud/integrations/harbor/env.py index 09467222b..955987fba 100644 --- a/hud/integrations/harbor/env.py +++ b/hud/integrations/harbor/env.py @@ -15,16 +15,14 @@ import shutil import socket import tempfile -import uuid from collections.abc import AsyncGenerator, Iterator # noqa: TC003 from pathlib import Path from typing import TYPE_CHECKING, Any -from pydantic import BaseModel - from hud.capabilities import Capability -from hud.environment import Answer, Environment, Mount, Peer, Workspace +from hud.environment import Environment, Mount, Peer, Workspace from hud.environment.egress import ANY_HOST, BRIDGE_PORT, VISITOR_PORT +from hud.environment.env import current_session_id from hud.graders import EvaluationResult from hud.utils.process import ProcessResult, create_process_group_exec @@ -36,9 +34,9 @@ LOGS = Path("/logs") VERIFIER_LOGS = LOGS / "verifier" AGENT_ANSWER = LOGS / "agent_answer.txt" -HANDOFFS = ROOT / "handoffs" -HANDOFF_ANSWER = "agent-answer.txt" -HANDOFF_ERROR = "error.txt" +SESSIONS = ROOT / "sessions" +SESSION_ANSWER = "agent-answer.txt" +SESSION_ERROR = "error.txt" DOCKER_SOCKET = ROOT / "docker.sock" DOCKER = ROOT / "bin" / "docker" CONFIG = json.loads((ROOT / "config.json").read_text("utf-8")) @@ -317,6 +315,7 @@ async def start_entrypoint() -> NamespaceProcess | None: inherit_workspace_env=False, no_new_privs=False, persistent=True, + scope="environment", ) await asyncio.sleep(0) if process.returncode is not None: @@ -478,10 +477,6 @@ def copy_artifact(source: Path, target: Path, exclude: list[str]) -> None: shutil.copy2(source, target, follow_symlinks=False) -class ActorHandoff(BaseModel): - handoff: str - - def artifact_path(artifact: dict[str, Any], artifacts: Path) -> Path: relative = artifact.get("destination") or artifact["source"].lstrip("/").rstrip("/") return artifacts / relative @@ -573,28 +568,29 @@ async def run(instruction: str, task: dict[str, Any]) -> AsyncGenerator[Any, Any f"Harbor environment entrypoint exited with status {entrypoint.returncode}" ) if task["separate_verifier"]: - token = uuid.uuid4().hex - handoff = HANDOFFS / token - artifacts = handoff / "artifacts" - handoff.mkdir(parents=True) - (handoff / HANDOFF_ANSWER).write_text( + session_id = current_session_id.get() + if session_id is None: + raise RuntimeError("Harbor actor is not running in an environment session") + session = SESSIONS / session_id + clear(session) + artifacts = session / "artifacts" + (session / SESSION_ANSWER).write_text( "" if answer is None else str(answer), encoding="utf-8", ) + await workspace.terminate_sessions() try: await collect(task, artifacts) except Exception as error: detail = str(error) - (handoff / HANDOFF_ERROR).write_text(detail, encoding="utf-8") + (session / SESSION_ERROR).write_text(detail, encoding="utf-8") result = { "score": 0.0, - "handoff": token, "content": detail, "isError": True, } else: - result = {"score": 0.0, "handoff": token} - await workspace.terminate_sessions() + result = {"score": 0.0} yield result else: yield await grade(task["id"], task["verifier_timeout"], answer) @@ -608,24 +604,22 @@ async def run(instruction: str, task: dict[str, Any]) -> AsyncGenerator[Any, Any if CONFIG["verifier_root"] is not None: - @env.template(id="verify", description="Verify a Harbor task", returns=ActorHandoff) + @env.template(id="verify", description="Verify a Harbor task") async def verify(task: dict[str, Any]) -> AsyncGenerator[Any, Any]: - received = yield "" - if not isinstance(received, Answer) or not isinstance(received.content, ActorHandoff): - raise ValueError("Harbor verifier requires an actor handoff") - token = received.content.handoff - if len(token) != 32 or not token.isalnum(): - raise ValueError("Harbor verifier received an invalid actor handoff") - handoff = HANDOFFS / token - if not handoff.is_dir(): - raise ValueError("Harbor actor handoff is unavailable in this runtime") + yield "" + session_id = current_session_id.get() + if session_id is None: + raise RuntimeError("Harbor verifier is not running in an environment session") + session = SESSIONS / session_id + if not session.is_dir(): + raise ValueError("Harbor actor session files are unavailable in this runtime") try: - if (error := handoff / HANDOFF_ERROR).is_file(): + if (error := session / SESSION_ERROR).is_file(): raise RuntimeError(error.read_text("utf-8")) - yield await grade_separate(task, handoff) + yield await grade_separate(task, session) finally: clear_grading_files() - shutil.rmtree(handoff, ignore_errors=True) + shutil.rmtree(session, ignore_errors=True) def clear(path: Path) -> None: @@ -813,7 +807,7 @@ def materialized_artifacts( async def grade_separate( task: dict[str, Any], - handoff: Path, + session: Path, ) -> EvaluationResult: async with verifier_lock: verifier_root = Path(CONFIG["verifier_root"]) @@ -825,7 +819,7 @@ async def grade_separate( await asyncio.to_thread(LOGS.mkdir, parents=True, exist_ok=True) await asyncio.to_thread( AGENT_ANSWER.write_bytes, - (handoff / HANDOFF_ANSWER).read_bytes(), + (session / SESSION_ANSWER).read_bytes(), ) try: @@ -842,7 +836,7 @@ async def grade_separate( with materialized_artifacts( task, verifier_root, - handoff / "artifacts", + session / "artifacts", verifier_identity, ) as mounts: verifier_mounts = tuple(mounts) diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh b/hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh index b4062bee9..5ece78a3b 100644 --- a/hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh +++ b/hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh @@ -18,6 +18,8 @@ esac curl -fsS --max-time 10 http://main:8080/ > /app/main.html protected=/media/hud/session-"keys" [ ! -e "$protected" ] +(while :; do sleep 1; done) >/tmp/actor-background.log 2>&1 & +printf '%s\n' "$!" > /app/actor.pid entrypoint_visible=false for pid in $(pgrep -x python3); do if tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null \ @@ -26,8 +28,8 @@ for pid in $(pgrep -x python3); do break fi done -if [ "$entrypoint_visible" != true ]; then - echo "the main entrypoint process is absent from the agent process namespace" >&2 +if [ "$entrypoint_visible" = true ]; then + echo "the environment entrypoint process is visible in the agent process namespace" >&2 exit 1 fi processes=$(ps -ef) diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml b/hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml index 785823638..136c73399 100644 --- a/hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml +++ b/hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml @@ -21,7 +21,7 @@ timeout_sec = 30 [[verifier.collect]] service = "main" -command = "curl -fsS --max-time 10 http://127.0.0.1:8080/ >/dev/null && head -c 1 /dev/urandom >/dev/null && echo collected-from-main > /tmp/main.txt && sleep 0.2" +command = "test -s /app/actor.pid && ! kill -0 \"$(cat /app/actor.pid)\" 2>/dev/null && curl -fsS --max-time 10 http://127.0.0.1:8080/ >/dev/null && head -c 1 /dev/urandom >/dev/null && echo collected-from-main > /tmp/main.txt && sleep 0.2" timeout_sec = 10 [[verifier.collect]] diff --git a/hud/integrations/harbor/tests/test_contract.py b/hud/integrations/harbor/tests/test_contract.py index 6580103b4..31450a8f2 100644 --- a/hud/integrations/harbor/tests/test_contract.py +++ b/hud/integrations/harbor/tests/test_contract.py @@ -148,8 +148,9 @@ def test_adapt_packages_an_image_task_as_a_compose_project(tmp_path: Path) -> No assert (project_root / "tests" / "task-a" / "test.sh").is_file() assert not any(path.name in {"tasks", "tasks.json"} for path in payload.rglob("*")) assert not (context / "compose.json").exists() - assert task.runtime_config.compose == context / "compose-project" / "compose.json" - assert task.runtime_config.compose_project == context + assert task.runtime_config.compose is not None + assert task.runtime_config.compose.document == context / "compose-project" / "compose.json" + assert task.runtime_config.compose.root == context compose_path = context / "compose-project" / "compose.json" project = _assert_stock_compose_complete(compose_path) assert set(project["services"]) == {"main"} @@ -173,22 +174,24 @@ def test_task_content_changes_do_not_rebuild_the_environment(tmp_path: Path) -> (before,) = list(_adapt(tmp_path)) assert before.runtime_config is not None - assert isinstance(before.runtime_config.compose, Path) - before_compose = json.loads(before.runtime_config.compose.read_text("utf-8")) + assert before.runtime_config.compose is not None + assert isinstance(before.runtime_config.compose.document, Path) + before_compose = json.loads(before.runtime_config.compose.document.read_text("utf-8")) before_image = before_compose["services"]["main"]["image"] (task_dir / "instruction.md").write_text("Second instruction", encoding="utf-8") (task_dir / "tests" / "test.sh").write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") (after,) = list(_adapt(tmp_path)) assert after.runtime_config is not None - assert isinstance(after.runtime_config.compose, Path) - after_compose = json.loads(after.runtime_config.compose.read_text("utf-8")) + assert after.runtime_config.compose is not None + assert isinstance(after.runtime_config.compose.document, Path) + after_compose = json.loads(after.runtime_config.compose.document.read_text("utf-8")) assert after_compose["services"]["main"]["image"] == before_image assert after.args["instruction"] == "Second instruction" - assert (after.runtime_config.compose.parent / "tests" / "task-a" / "test.sh").read_text( - "utf-8" - ) == "#!/bin/sh\nexit 1\n" + assert ( + after.runtime_config.compose.document.parent / "tests" / "task-a" / "test.sh" + ).read_text("utf-8") == "#!/bin/sh\nexit 1\n" def test_image_task_keeps_non_recipe_compose_names_as_context_files(tmp_path: Path) -> None: @@ -298,7 +301,8 @@ def test_adapt_honors_compose_main_build_settings( (row,) = list(_adapt(tmp_path)) assert row.runtime_config is not None - compose_path = row.runtime_config.compose + assert row.runtime_config.compose is not None + compose_path = row.runtime_config.compose.document assert isinstance(compose_path, Path) project = _assert_stock_compose_complete(compose_path) base = project["services"]["hud-base"] @@ -346,12 +350,13 @@ def test_adapt_emits_compose_project_and_peers( assert row.runtime_config is not None assert row.runtime_config.image is None - assert row.runtime_config.compose_service_access is True + assert row.runtime_config.compose is not None + assert row.runtime_config.compose.service_access is True (context,) = (tmp_path / ".hud-adapt").iterdir() - assert row.runtime_config.compose == context / "compose-project" / "compose.json" - assert row.runtime_config.compose_project == context + assert row.runtime_config.compose.document == context / "compose-project" / "compose.json" + assert row.runtime_config.compose.root == context assert not (context / "compose.json").exists() - compose_path = row.runtime_config.compose + compose_path = row.runtime_config.compose.document assert isinstance(compose_path, Path) project = _assert_stock_compose_complete(compose_path) assert project["services"]["redis"]["image"] == "redis:7-alpine" @@ -442,7 +447,8 @@ def test_compose_adapt_retains_builds_without_local_docker( (row,) = list(_adapt(tmp_path)) assert row.runtime_config is not None - compose_path = row.runtime_config.compose + assert row.runtime_config.compose is not None + compose_path = row.runtime_config.compose.document assert isinstance(compose_path, Path) project = json.loads(compose_path.read_text("utf-8")) assert project["services"]["database"]["build"]["context"] == ("./environment/database") @@ -685,7 +691,8 @@ def test_adapt_maps_resources_onto_the_compose_runtime(tmp_path: Path) -> None: assert row.columns == {"difficulty": "hard"} assert row.runtime_config is not None assert row.runtime_config.image is None - assert isinstance(row.runtime_config.compose, Path) + assert row.runtime_config.compose is not None + assert isinstance(row.runtime_config.compose.document, Path) assert row.runtime_config.resources is not None assert row.runtime_config.resources.cpu == 4 assert row.runtime_config.resources.memory_mb == 8192 @@ -940,7 +947,8 @@ def test_adapt_accepts_explicit_shared_verifier_mode(tmp_path: Path) -> None: assert row.verifier is None assert row.runtime_config is not None - assert row.runtime_config.compose_service_access is None + assert row.runtime_config.compose is not None + assert row.runtime_config.compose.service_access is None def test_adapt_builds_a_separate_verifier_with_its_own_placement( @@ -1010,11 +1018,13 @@ def test_adapt_builds_a_separate_verifier_with_its_own_placement( tpu=RuntimeTPU(type="v5", topology="2x2"), ) assert row.runtime_config.limits == RuntimeLimits(startup_timeout_s=601) - assert row.runtime_config.compose_service_access is True + assert row.runtime_config.compose is not None + assert row.runtime_config.compose.service_access is True assert row.verifier is not None - assert row.verifier.requires_handoff is True assert row.verifier.runtime_config is not None - assert row.verifier.runtime_config.compose == row.runtime_config.compose + assert row.verifier.runtime_config.compose is not None + assert row.verifier.runtime_config.compose.document == row.runtime_config.compose.document + assert row.verifier.runtime_config.compose.root == row.runtime_config.compose.root assert row.verifier.runtime_config.resources == RuntimeResources( cpu=4, memory_mb=1024, @@ -1062,8 +1072,7 @@ def test_image_task_keeps_only_the_verifier_as_a_build_service( encoding="utf-8", ) (task / "task.toml").write_text( - '[environment]\nbuild_timeout_sec = 300\n\n' - '[verifier]\nenvironment_mode = "separate"\n', + '[environment]\nbuild_timeout_sec = 300\n\n[verifier]\nenvironment_mode = "separate"\n', encoding="utf-8", ) @@ -1071,11 +1080,12 @@ def test_image_task_keeps_only_the_verifier_as_a_build_service( assert row.runtime_config is not None assert row.runtime_config.limits == RuntimeLimits(startup_timeout_s=300) - assert row.runtime_config.compose_service_access is None + assert row.runtime_config.compose is not None + assert row.runtime_config.compose.service_access is None assert row.verifier is not None assert row.verifier.runtime_config is None - assert isinstance(row.runtime_config.compose, Path) - project = _assert_stock_compose_complete(row.runtime_config.compose) + assert isinstance(row.runtime_config.compose.document, Path) + project = _assert_stock_compose_complete(row.runtime_config.compose.document) assert set(project["services"]) == {"main", "hud-verifier"} assert project["services"]["main"]["build"] == { "additional_contexts": { @@ -1086,7 +1096,7 @@ def test_image_task_keeps_only_the_verifier_as_a_build_service( "dockerfile": "../Dockerfile", } assert project["services"]["hud-verifier"]["scale"] == 0 - combined = (row.runtime_config.compose.parent / "Dockerfile").read_text("utf-8") + combined = (row.runtime_config.compose.document.parent / "Dockerfile").read_text("utf-8") assert "FROM hud-verifier AS hud-verifier-root" in combined assert "COPY --from=hud-verifier-root / /media/hud/verifier" in combined @@ -1110,7 +1120,8 @@ def test_separate_verifier_groups_have_distinct_environment_names( compose for row in rows if row.runtime_config is not None - and isinstance((compose := row.runtime_config.compose), Path) + and row.runtime_config.compose is not None + and isinstance((compose := row.runtime_config.compose.document), Path) } assert len(compose_paths) == 2 assert all(path.is_file() for path in compose_paths) @@ -1176,6 +1187,7 @@ def test_artifacts_keep_harbor_destination_and_exclude(tmp_path: Path) -> None: } ] + def test_agent_timeout_becomes_per_task_agent_policy( tmp_path: Path, ) -> None: @@ -1256,7 +1268,8 @@ def test_portless_sidecar_ports_are_resolved_from_its_built_image(tmp_path: Path (row,) = list(_adapt(tmp_path)) assert row.runtime_config is not None - context = row.runtime_config.compose_project + assert row.runtime_config.compose is not None + context = row.runtime_config.compose.root assert isinstance(context, Path) manifest = _environment_config(context) assert manifest["peers"] == [] @@ -1288,7 +1301,8 @@ def test_completed_compose_dependencies_are_not_routed_as_peers(tmp_path: Path) (row,) = list(_adapt(tmp_path)) assert row.runtime_config is not None - context = row.runtime_config.compose_project + assert row.runtime_config.compose is not None + context = row.runtime_config.compose.root assert isinstance(context, Path) manifest = _environment_config(context) assert manifest["peers"] == [] diff --git a/hud/integrations/harbor/tests/test_integration.py b/hud/integrations/harbor/tests/test_integration.py index f3178faa9..cc761d898 100644 --- a/hud/integrations/harbor/tests/test_integration.py +++ b/hud/integrations/harbor/tests/test_integration.py @@ -655,9 +655,9 @@ def test_missing_env_template_aborts_startup( taskset = harbor.adapt(dataset, hud_requirement=str(wheel)).taskset task = next(iter(taskset)) assert task.runtime_config is not None - source = task.runtime_config.compose_source() - assert source is not None - compose = source.runnable_path("test") + assert task.runtime_config.compose is not None + compose = task.runtime_config.compose.document + assert isinstance(compose, Path) # Providers yield as soon as the published port exists, before the serve # process proves itself, so an early abort is only observable from the # adapted artifact: run its main service in the foreground. diff --git a/hud/tests/test_init_module.py b/hud/tests/test_init_module.py index c96cbd5ab..62bd04723 100644 --- a/hud/tests/test_init_module.py +++ b/hud/tests/test_init_module.py @@ -19,6 +19,7 @@ def test_all_exports(self): expected = [ "Chat", + "ComposeProject", "DockerRuntime", "Environment", "Grade", diff --git a/hud/tests/test_robot.py b/hud/tests/test_robot.py index 0a5238777..8057d1498 100644 --- a/hud/tests/test_robot.py +++ b/hud/tests/test_robot.py @@ -29,7 +29,6 @@ from hud.environment import Environment from hud.environment.robot import RobotBridge, RobotEndpoint from hud.eval import LocalRuntime, Shared, Task, Taskset, rollout -from hud.eval.runtime import _local if TYPE_CHECKING: from numpy.typing import NDArray @@ -39,7 +38,6 @@ from pathlib import Path from hud.eval import Provider - from hud.eval.runtime import Runtime #: Ticks a stub sim runs before terminating — every gate here is a whole rollout. EPISODE_TICKS = 3 @@ -134,13 +132,7 @@ async def _down() -> None: def _serving(env: Environment) -> Provider: """Placement that serves this one env — the substrate a vectorized sim shares.""" - - @asynccontextmanager - async def provider(task: Task) -> AsyncIterator[Runtime]: - async with _local(env) as runtime: - yield runtime - - return provider + return LocalRuntime(env) # ─── the agent side ──────────────────────────────────────────────────── From 4df6f559a0cb7509727df951a7dcbfa9262354a8 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:20:52 -0700 Subject: [PATCH 10/10] ci: test all optional dependencies --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52c974ec4..5ee201f56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: run: uv python install ${{ matrix.python-version }} - name: Run tests - run: uv run --python ${{ matrix.python-version }} --with=".[dev]" pytest --cov --cov-report='' + run: uv run --python ${{ matrix.python-version }} --all-extras pytest --cov --cov-report='' lint-ruff: runs-on: ubuntu-latest