Resolve Harbor ${VAR} env templates from the runtime environment

hud/eval/runtime.py

@@ -584,9 +584,11 @@ def __init__(
         port: int = 8765,
         run_args: Sequence[str] = (),
         runtime_config: RuntimeConfig | dict[str, Any] | None = None,
+        env_vars: Mapping[str, str] | None = None,
     ) -> None:
         self.port = port
         self.run_args = tuple(run_args)
+        self.env_vars = dict(env_vars or {})
         config = RuntimeConfig(image=image) if image is not None else RuntimeConfig()
         if runtime_config is not None:
             config = config.with_overrides(RuntimeConfig.model_validate(runtime_config))
@@ -637,6 +639,7 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]:
                 f"127.0.0.1::{self.port}",
                 seccomp=_DOCKER_SECCOMP_PROFILE,
                 service_socket=service_socket,
+                env_vars=self.env_vars,
                 cpu=resources.cpu if resources is not None else None,
                 memory_mb=resources.memory_mb if resources is not None else None,
                 gpu_count=(
@@ -709,10 +712,14 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]:
                     raise ValueError("DockerRuntime cannot select GPUs by type")
                 resource_args.extend(("--gpus", str(resources.gpu.count)))
 
+        env_args: list[str] = []
+        for key, value in self.env_vars.items():
+            env_args.extend(("--env", f"{key}={value}"))
         out, _ = await _docker(
             "run",
             "--detach",
             *self.run_args,
+            *env_args,
             *resource_args,
             *_DOCKER_SECURITY_ARGS,
             "--publish",

hud/eval/tests/test_docker_provider.py

@@ -707,6 +707,49 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]:
     assert calls[-1][-3:] == ("down", "--volumes", "--remove-orphans")
 
 
+async def test_docker_runtime_passes_env_vars_to_docker_run(
+    tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch)
+
+    provider = DockerRuntime("img:tag", env_vars={"OPENAI_API_KEY": "sk-test"})
+    async with provider(_row()) as runtime:
+        assert runtime.url == "tcp://127.0.0.1:43210"
+
+    calls = await _docker_calls(docker_log)
+    assert calls[0] == (
+        f"run --detach --env OPENAI_API_KEY=sk-test {_docker_security_args()} "
+        "--publish 127.0.0.1::8765 img:tag"
+    )
+
+
+async def test_docker_runtime_stages_env_vars_into_the_compose_override(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    rendered: dict[str, Any] = {}
+    compose = tmp_path / "compose.yaml"
+    compose.write_text("services:\n  main:\n    image: hud-env:one\n", encoding="utf-8")
+
+    async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]:
+        if "up" in args:
+            files = [Path(args[index + 1]) for index, value in enumerate(args) if value == "--file"]
+            _, override, _ = files
+            rendered.update(json.loads(override.read_text("utf-8")))
+        if args[-3:] == ("port", "main", "8765"):
+            return "127.0.0.1:43210\n", ""
+        return "", ""
+
+    monkeypatch.setattr(runtime_module, "_docker", fake_docker)
+    task = Task(env="any-env", id="t", runtime_config=RuntimeConfig(compose=compose))
+    provider = DockerRuntime(env_vars={"OPENAI_API_KEY": "sk-test"})
+
+    async with provider(task):
+        pass
+
+    assert rendered["services"]["main"]["environment"] == {"OPENAI_API_KEY": "sk-test"}
+
+
 async def test_docker_runtime_serializes_shared_compose_preparation(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,

hud/integrations/harbor/env.py

@@ -9,6 +9,7 @@
 import math
 import os
 import pwd
+import re
 import shutil
 import socket
 import tempfile
@@ -82,6 +83,36 @@ def image_environment(config: dict[str, Any]) -> dict[str, str]:
     verifier_image["workdir"] = VERIFIER_IMAGE_CONFIG.get("WorkingDir") or "/"
 verifier_image["env"] = image_environment(VERIFIER_IMAGE_CONFIG)
 
+# Harbor's host-side env template contract (harbor/utils/env.py): a value that
+# is exactly ``${VAR}`` or ``${VAR:-default}`` resolves from the environment at
+# startup; anything else, including embedded templates, stays literal. Here the
+# source is this process's environment, which the runtime provider populates
+# from the host via ``env_vars`` (secrets never enter the content-hashed image).
+ENV_TEMPLATE = re.compile(r"\$\{([^}:]+)(?::-(.*))?\}")
+
+
+def resolve_env_templates(env: dict[str, str]) -> dict[str, str]:
+    resolved: dict[str, str] = {}
+    for key, value in env.items():
+        match = ENV_TEMPLATE.fullmatch(value)
+        if match is None:
+            resolved[key] = value
+            continue
+        name, default = match.group(1), match.group(2)
+        if name in os.environ:
+            resolved[key] = os.environ[name]
+        elif default is not None:
+            resolved[key] = default
+        else:
+            raise ValueError(
+                f"Harbor env template for {key!r} needs {name!r}; "
+                "pass it through the runtime's env_vars"
+            )
+    return resolved
+
+
+for policy in (CONFIG["environment"], CONFIG["agent"], CONFIG["verifier"]):
+    policy["env"] = resolve_env_templates(policy["env"])
 os.environ.update(CONFIG["environment"]["env"])
 WORKDIR = Path(CONFIG["workdir"])
 os.chdir(WORKDIR)

hud/integrations/harbor/tests/test_contract.py

@@ -639,6 +639,41 @@ def test_adapt_maps_resources_onto_the_compose_runtime(tmp_path: Path) -> None:
     assert row.runtime_config.resources.gpu.type == "H100"
 
 
+def test_env_templates_are_persisted_verbatim_not_resolved(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Host values must never be baked into the content-hashed image payload
+    or the persisted task rows; ``${VAR}`` templates resolve at runtime."""
+    monkeypatch.setenv("HARBOR_JUDGE_KEY", "sk-live-secret")
+    task = make_harbor_task(tmp_path, "templated")
+    (task / "task.toml").write_text(
+        """
+[environment.env]
+JUDGE_KEY = "${HARBOR_JUDGE_KEY}"
+MODEL = "${HARBOR_JUDGE_MODEL:-gpt-4o}"
+
+[verifier]
+timeout_sec = 60
+
+[verifier.env]
+VERIFIER_KEY = "${HARBOR_JUDGE_KEY}"
+""",
+        encoding="utf-8",
+    )
+
+    harbor.adapt(tmp_path)
+
+    (context,) = (tmp_path / ".hud-adapt").iterdir()
+    manifest = json.loads((context / "compose-project" / "hud" / "config.json").read_text("utf-8"))
+    assert manifest["environment"]["env"] == {
+        "JUDGE_KEY": "${HARBOR_JUDGE_KEY}",
+        "MODEL": "${HARBOR_JUDGE_MODEL:-gpt-4o}",
+    }
+    assert manifest["verifier"]["env"] == {"VERIFIER_KEY": "${HARBOR_JUDGE_KEY}"}
+    for persisted in sorted(path for path in context.rglob("*") if path.is_file()):
+        assert b"sk-live-secret" not in persisted.read_bytes(), persisted
+
+
 def test_prebuilt_harbor_image_is_inspected_by_the_project_build(
     tmp_path: Path,
 ) -> None:

hud/integrations/harbor/tests/test_integration.py

@@ -20,6 +20,8 @@
 from hud.eval import DockerRuntime, Shared
 from hud.integrations import harbor
 
+from .conftest import make_harbor_task
+
 if TYPE_CHECKING:
     from hud.capabilities import MCPClient, SSHClient
     from hud.eval.run import Run
@@ -347,3 +349,121 @@ def test_separate_verifier_honors_the_test_script_shebang(
     run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"]
 
     assert run.reward == 1.0
+
+
+def test_env_templates_resolve_from_runtime_env_vars(
+    tmp_path_factory: pytest.TempPathFactory, wheel: Path
+) -> None:
+    """``${VAR}``/``${VAR:-default}`` env values resolve exactly like Harbor's
+    host-side resolution, sourced from the runtime's ``env_vars``."""
+    dataset = tmp_path_factory.mktemp("harbor-env-templates") / "harbor-harness"
+    task = make_harbor_task(
+        dataset,
+        "env-templates",
+        task_toml="""\
+[metadata]
+category = "systems"
+
+[environment.env]
+GREETING = "${HARBOR_GREETING:-hello}"
+JUDGE_KEY = "${HARBOR_JUDGE_KEY}"
+EMBEDDED = "Bearer ${HARBOR_JUDGE_KEY}"
+
+[verifier]
+timeout_sec = 120
+
+[verifier.env]
+VERIFIER_KEY = "${HARBOR_JUDGE_KEY}"
+EMPTY_DEFAULT = "${HARBOR_UNSET:-}"
+""",
+    )
+    (task / "tests" / "test.sh").write_text(
+        """\
+#!/bin/bash
+fail() { echo "unexpected $1"; echo "0.0" > /logs/verifier/reward.txt; exit 0; }
+[ "$GREETING" = "hello" ] || fail "GREETING=$GREETING"
+[ "$JUDGE_KEY" = "judge-secret" ] || fail "JUDGE_KEY=$JUDGE_KEY"
+[ "$EMBEDDED" = 'Bearer ${HARBOR_JUDGE_KEY}' ] || fail "EMBEDDED=$EMBEDDED"
+[ "$VERIFIER_KEY" = "judge-secret" ] || fail "VERIFIER_KEY=$VERIFIER_KEY"
+[ "${EMPTY_DEFAULT-unset}" = "" ] || fail "EMPTY_DEFAULT=${EMPTY_DEFAULT-unset}"
+echo "1.0" > /logs/verifier/reward.txt
+""",
+        encoding="utf-8",
+    )
+    solution = '[ "$JUDGE_KEY" = "judge-secret" ] && [ "$GREETING" = "hello" ]'
+
+    async def grade() -> Run:
+        taskset = harbor.adapt(dataset, hud_requirement=str(wheel))
+        job = await taskset.run(
+            Oracle({"env-templates": solution}),
+            runtime=DockerRuntime(env_vars={"HARBOR_JUDGE_KEY": "judge-secret"}),
+            max_concurrent=1,
+        )
+        (run,) = job.runs
+        return run
+
+    run = asyncio.run(grade())
+
+    evaluation = run.evaluation
+    info = evaluation.get("info") or {}
+    detail = "\n".join(
+        filter(
+            None,
+            (
+                run.trace.content,
+                run.trace.error,
+                evaluation.get("content") or "",
+                info.get("stdout"),
+                info.get("stderr"),
+            ),
+        )
+    )
+    assert run.reward == 1.0, f"env-templates scored {run.reward}; the verifier reported:\n{detail}"
+
+
+def test_missing_env_template_aborts_startup(
+    tmp_path_factory: pytest.TempPathFactory, wheel: Path
+) -> None:
+    """A required ``${VAR}`` with no default and no runtime value must abort
+    environment startup with an error naming the variable, like Harbor."""
+    dataset = tmp_path_factory.mktemp("harbor-env-missing") / "harbor-harness"
+    make_harbor_task(
+        dataset,
+        "env-missing",
+        task_toml="""\
+[metadata]
+category = "systems"
+
+[environment.env]
+API_KEY = "${HARBOR_MISSING_KEY}"
+
+[verifier]
+timeout_sec = 120
+""",
+    )
+
+    taskset = harbor.adapt(dataset, hud_requirement=str(wheel))
+    task = next(iter(taskset))
+    assert task.runtime_config is not None
+    source = task.runtime_config.compose_source()
+    assert source is not None
+    compose = source.runnable_path("test")
+    # Providers yield as soon as the published port exists, before the serve
+    # process proves itself, so an early abort is only observable from the
+    # adapted artifact: run its main service in the foreground.
+    subprocess.run(
+        ["sh", "build.sh"], cwd=compose.parent, check=True, capture_output=True, timeout=600
+    )
+    command = ["docker", "compose", "--file", str(compose), "run", "--rm", "main"]
+    try:
+        serve = subprocess.run(command, capture_output=True, text=True, timeout=60)
+    except subprocess.TimeoutExpired:
+        pytest.fail("main service kept serving despite an unresolvable env template")
+    finally:
+        subprocess.run(
+            ["docker", "compose", "--file", str(compose), "down", "--volumes", "--remove-orphans"],
+            capture_output=True,
+            check=False,
+        )
+    assert serve.returncode != 0
+    assert "HARBOR_MISSING_KEY" in serve.stdout + serve.stderr