diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py index ec893f448..731fed280 100644 --- a/hud/eval/runtime.py +++ b/hud/eval/runtime.py @@ -584,9 +584,11 @@ def __init__( port: int = 8765, run_args: Sequence[str] = (), runtime_config: RuntimeConfig | dict[str, Any] | None = None, + env_vars: Mapping[str, str] | None = None, ) -> None: self.port = port self.run_args = tuple(run_args) + self.env_vars = dict(env_vars or {}) config = RuntimeConfig(image=image) if image is not None else RuntimeConfig() if runtime_config is not None: config = config.with_overrides(RuntimeConfig.model_validate(runtime_config)) @@ -637,6 +639,7 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: f"127.0.0.1::{self.port}", seccomp=_DOCKER_SECCOMP_PROFILE, service_socket=service_socket, + env_vars=self.env_vars, cpu=resources.cpu if resources is not None else None, memory_mb=resources.memory_mb if resources is not None else None, gpu_count=( @@ -709,10 +712,14 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: raise ValueError("DockerRuntime cannot select GPUs by type") resource_args.extend(("--gpus", str(resources.gpu.count))) + env_args: list[str] = [] + for key, value in self.env_vars.items(): + env_args.extend(("--env", f"{key}={value}")) out, _ = await _docker( "run", "--detach", *self.run_args, + *env_args, *resource_args, *_DOCKER_SECURITY_ARGS, "--publish", diff --git a/hud/eval/tests/test_docker_provider.py b/hud/eval/tests/test_docker_provider.py index dcefa6b75..f39f380ad 100644 --- a/hud/eval/tests/test_docker_provider.py +++ b/hud/eval/tests/test_docker_provider.py @@ -707,6 +707,49 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: assert calls[-1][-3:] == ("down", "--volumes", "--remove-orphans") +async def test_docker_runtime_passes_env_vars_to_docker_run( + tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch) + + provider = DockerRuntime("img:tag", env_vars={"OPENAI_API_KEY": "sk-test"}) + async with provider(_row()) as runtime: + assert runtime.url == "tcp://127.0.0.1:43210" + + calls = await _docker_calls(docker_log) + assert calls[0] == ( + f"run --detach --env OPENAI_API_KEY=sk-test {_docker_security_args()} " + "--publish 127.0.0.1::8765 img:tag" + ) + + +async def test_docker_runtime_stages_env_vars_into_the_compose_override( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + rendered: dict[str, Any] = {} + compose = tmp_path / "compose.yaml" + compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") + + async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: + if "up" in args: + files = [Path(args[index + 1]) for index, value in enumerate(args) if value == "--file"] + _, override, _ = files + rendered.update(json.loads(override.read_text("utf-8"))) + if args[-3:] == ("port", "main", "8765"): + return "127.0.0.1:43210\n", "" + return "", "" + + monkeypatch.setattr(runtime_module, "_docker", fake_docker) + task = Task(env="any-env", id="t", runtime_config=RuntimeConfig(compose=compose)) + provider = DockerRuntime(env_vars={"OPENAI_API_KEY": "sk-test"}) + + async with provider(task): + pass + + assert rendered["services"]["main"]["environment"] == {"OPENAI_API_KEY": "sk-test"} + + async def test_docker_runtime_serializes_shared_compose_preparation( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/hud/integrations/harbor/env.py b/hud/integrations/harbor/env.py index f16b5207a..261f2df35 100644 --- a/hud/integrations/harbor/env.py +++ b/hud/integrations/harbor/env.py @@ -9,6 +9,7 @@ import math import os import pwd +import re import shutil import socket import tempfile @@ -82,6 +83,36 @@ def image_environment(config: dict[str, Any]) -> dict[str, str]: verifier_image["workdir"] = VERIFIER_IMAGE_CONFIG.get("WorkingDir") or "/" verifier_image["env"] = image_environment(VERIFIER_IMAGE_CONFIG) +# Harbor's host-side env template contract (harbor/utils/env.py): a value that +# is exactly ``${VAR}`` or ``${VAR:-default}`` resolves from the environment at +# startup; anything else, including embedded templates, stays literal. Here the +# source is this process's environment, which the runtime provider populates +# from the host via ``env_vars`` (secrets never enter the content-hashed image). +ENV_TEMPLATE = re.compile(r"\$\{([^}:]+)(?::-(.*))?\}") + + +def resolve_env_templates(env: dict[str, str]) -> dict[str, str]: + resolved: dict[str, str] = {} + for key, value in env.items(): + match = ENV_TEMPLATE.fullmatch(value) + if match is None: + resolved[key] = value + continue + name, default = match.group(1), match.group(2) + if name in os.environ: + resolved[key] = os.environ[name] + elif default is not None: + resolved[key] = default + else: + raise ValueError( + f"Harbor env template for {key!r} needs {name!r}; " + "pass it through the runtime's env_vars" + ) + return resolved + + +for policy in (CONFIG["environment"], CONFIG["agent"], CONFIG["verifier"]): + policy["env"] = resolve_env_templates(policy["env"]) os.environ.update(CONFIG["environment"]["env"]) WORKDIR = Path(CONFIG["workdir"]) os.chdir(WORKDIR) diff --git a/hud/integrations/harbor/tests/test_contract.py b/hud/integrations/harbor/tests/test_contract.py index 84fb6a14f..c733035d3 100644 --- a/hud/integrations/harbor/tests/test_contract.py +++ b/hud/integrations/harbor/tests/test_contract.py @@ -639,6 +639,41 @@ def test_adapt_maps_resources_onto_the_compose_runtime(tmp_path: Path) -> None: assert row.runtime_config.resources.gpu.type == "H100" +def test_env_templates_are_persisted_verbatim_not_resolved( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Host values must never be baked into the content-hashed image payload + or the persisted task rows; ``${VAR}`` templates resolve at runtime.""" + monkeypatch.setenv("HARBOR_JUDGE_KEY", "sk-live-secret") + task = make_harbor_task(tmp_path, "templated") + (task / "task.toml").write_text( + """ +[environment.env] +JUDGE_KEY = "${HARBOR_JUDGE_KEY}" +MODEL = "${HARBOR_JUDGE_MODEL:-gpt-4o}" + +[verifier] +timeout_sec = 60 + +[verifier.env] +VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" +""", + encoding="utf-8", + ) + + harbor.adapt(tmp_path) + + (context,) = (tmp_path / ".hud-adapt").iterdir() + manifest = json.loads((context / "compose-project" / "hud" / "config.json").read_text("utf-8")) + assert manifest["environment"]["env"] == { + "JUDGE_KEY": "${HARBOR_JUDGE_KEY}", + "MODEL": "${HARBOR_JUDGE_MODEL:-gpt-4o}", + } + assert manifest["verifier"]["env"] == {"VERIFIER_KEY": "${HARBOR_JUDGE_KEY}"} + for persisted in sorted(path for path in context.rglob("*") if path.is_file()): + assert b"sk-live-secret" not in persisted.read_bytes(), persisted + + def test_prebuilt_harbor_image_is_inspected_by_the_project_build( tmp_path: Path, ) -> None: diff --git a/hud/integrations/harbor/tests/test_integration.py b/hud/integrations/harbor/tests/test_integration.py index 862ce0312..563cb7da6 100644 --- a/hud/integrations/harbor/tests/test_integration.py +++ b/hud/integrations/harbor/tests/test_integration.py @@ -20,6 +20,8 @@ from hud.eval import DockerRuntime, Shared from hud.integrations import harbor +from .conftest import make_harbor_task + if TYPE_CHECKING: from hud.capabilities import MCPClient, SSHClient from hud.eval.run import Run @@ -347,3 +349,121 @@ def test_separate_verifier_honors_the_test_script_shebang( run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] assert run.reward == 1.0 + + +def test_env_templates_resolve_from_runtime_env_vars( + tmp_path_factory: pytest.TempPathFactory, wheel: Path +) -> None: + """``${VAR}``/``${VAR:-default}`` env values resolve exactly like Harbor's + host-side resolution, sourced from the runtime's ``env_vars``.""" + dataset = tmp_path_factory.mktemp("harbor-env-templates") / "harbor-harness" + task = make_harbor_task( + dataset, + "env-templates", + task_toml="""\ +[metadata] +category = "systems" + +[environment.env] +GREETING = "${HARBOR_GREETING:-hello}" +JUDGE_KEY = "${HARBOR_JUDGE_KEY}" +EMBEDDED = "Bearer ${HARBOR_JUDGE_KEY}" + +[verifier] +timeout_sec = 120 + +[verifier.env] +VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" +EMPTY_DEFAULT = "${HARBOR_UNSET:-}" +""", + ) + (task / "tests" / "test.sh").write_text( + """\ +#!/bin/bash +fail() { echo "unexpected $1"; echo "0.0" > /logs/verifier/reward.txt; exit 0; } +[ "$GREETING" = "hello" ] || fail "GREETING=$GREETING" +[ "$JUDGE_KEY" = "judge-secret" ] || fail "JUDGE_KEY=$JUDGE_KEY" +[ "$EMBEDDED" = 'Bearer ${HARBOR_JUDGE_KEY}' ] || fail "EMBEDDED=$EMBEDDED" +[ "$VERIFIER_KEY" = "judge-secret" ] || fail "VERIFIER_KEY=$VERIFIER_KEY" +[ "${EMPTY_DEFAULT-unset}" = "" ] || fail "EMPTY_DEFAULT=${EMPTY_DEFAULT-unset}" +echo "1.0" > /logs/verifier/reward.txt +""", + encoding="utf-8", + ) + solution = '[ "$JUDGE_KEY" = "judge-secret" ] && [ "$GREETING" = "hello" ]' + + async def grade() -> Run: + taskset = harbor.adapt(dataset, hud_requirement=str(wheel)) + job = await taskset.run( + Oracle({"env-templates": solution}), + runtime=DockerRuntime(env_vars={"HARBOR_JUDGE_KEY": "judge-secret"}), + max_concurrent=1, + ) + (run,) = job.runs + return run + + run = asyncio.run(grade()) + + evaluation = run.evaluation + info = evaluation.get("info") or {} + detail = "\n".join( + filter( + None, + ( + run.trace.content, + run.trace.error, + evaluation.get("content") or "", + info.get("stdout"), + info.get("stderr"), + ), + ) + ) + assert run.reward == 1.0, f"env-templates scored {run.reward}; the verifier reported:\n{detail}" + + +def test_missing_env_template_aborts_startup( + tmp_path_factory: pytest.TempPathFactory, wheel: Path +) -> None: + """A required ``${VAR}`` with no default and no runtime value must abort + environment startup with an error naming the variable, like Harbor.""" + dataset = tmp_path_factory.mktemp("harbor-env-missing") / "harbor-harness" + make_harbor_task( + dataset, + "env-missing", + task_toml="""\ +[metadata] +category = "systems" + +[environment.env] +API_KEY = "${HARBOR_MISSING_KEY}" + +[verifier] +timeout_sec = 120 +""", + ) + + taskset = harbor.adapt(dataset, hud_requirement=str(wheel)) + task = next(iter(taskset)) + assert task.runtime_config is not None + source = task.runtime_config.compose_source() + assert source is not None + compose = source.runnable_path("test") + # Providers yield as soon as the published port exists, before the serve + # process proves itself, so an early abort is only observable from the + # adapted artifact: run its main service in the foreground. + subprocess.run( + ["sh", "build.sh"], cwd=compose.parent, check=True, capture_output=True, timeout=600 + ) + command = ["docker", "compose", "--file", str(compose), "run", "--rm", "main"] + try: + serve = subprocess.run(command, capture_output=True, text=True, timeout=60) + except subprocess.TimeoutExpired: + pytest.fail("main service kept serving despite an unresolvable env template") + finally: + subprocess.run( + ["docker", "compose", "--file", str(compose), "down", "--volumes", "--remove-orphans"], + capture_output=True, + check=False, + ) + assert serve.returncode != 0 + assert "HARBOR_MISSING_KEY" in serve.stdout + serve.stderr