diff --git a/hud/cli/eval.py b/hud/cli/eval.py index 8b3f4ccd1..1530720be 100644 --- a/hud/cli/eval.py +++ b/hud/cli/eval.py @@ -301,6 +301,15 @@ def _parse_agent_type(cls, v: Any) -> AgentType | None: try: return AgentType(v) except ValueError: + if v == "integration_test": + raise ValueError( + "integration_test is the platform's authoring agent: " + "it pre-stages the golden solution (Task.validation) " + "and runs the graders expecting Reward 1.0. Run it " + "against a deployed environment on the platform — " + "`hud eval integration_test --task-ids " + " -y` — it is not available with --runtime local." + ) from None valid = [e.value for e in AgentType] raise ValueError( f"Invalid agent: {v}. Must be one of: {', '.join(valid)}" diff --git a/hud/cli/tests/test_eval_config.py b/hud/cli/tests/test_eval_config.py index fad19d88f..62a7556ee 100644 --- a/hud/cli/tests/test_eval_config.py +++ b/hud/cli/tests/test_eval_config.py @@ -25,6 +25,11 @@ def test_is_bedrock_arn() -> None: assert _is_bedrock_arn(None) is False +def test_parse_agent_type_points_integration_test_to_the_platform() -> None: + with pytest.raises(ValueError, match="integration_test is the platform"): + EvalConfig(agent_type="integration_test") + + def test_parse_agent_type_accepts_known_value() -> None: cfg = EvalConfig(agent_type="openai") assert cfg.agent_type is not None diff --git a/hud/integrations/harbor/adapt.py b/hud/integrations/harbor/adapt.py index 81b86bdf2..eae0ec3c7 100644 --- a/hud/integrations/harbor/adapt.py +++ b/hud/integrations/harbor/adapt.py @@ -189,6 +189,13 @@ class TaskConfig(BaseModel): steps: list[dict[str, Any]] | None = None +# The Harbor task schema this build adapts. export() stamps the same value +# into generated task.toml files so round-tripped tasks validate; a task that +# declares any other schema_version was authored against a different adapter +# and must fail loudly rather than adapt with silently wrong semantics. +HARBOR_SCHEMA_VERSION = "1.0" + + @dataclass(frozen=True, slots=True) class HarborTask: path: Path @@ -238,6 +245,12 @@ def adapt( raise ValueError( f"{task_dir.name}/task.toml is not a valid Harbor task: {error}" ) from error + if config.schema_version is not None and config.schema_version != HARBOR_SCHEMA_VERSION: + raise ValueError( + f"{task_dir.name}/task.toml declares unsupported Harbor schema " + f"{config.schema_version!r} — this HUD build adapts schema " + f"{HARBOR_SCHEMA_VERSION!r}" + ) unsupported = [] if config.environment.os != "linux": unsupported.append(f"os={config.environment.os!r}") diff --git a/hud/integrations/harbor/export.py b/hud/integrations/harbor/export.py index 4eb4bbe7c..b304a6fd6 100644 --- a/hud/integrations/harbor/export.py +++ b/hud/integrations/harbor/export.py @@ -12,6 +12,7 @@ from hud.environment import Environment, load_environment from hud.environment.server import TaskRunner from hud.eval import Taskset +from hud.integrations.harbor.adapt import HARBOR_SCHEMA_VERSION from hud.utils.naming import normalize_environment_name ALLOWED_PROTOCOLS = ("ssh", "mcp") @@ -162,6 +163,7 @@ def ignore_export(dirpath: str, names: list[str]) -> set[str]: args_json = json.dumps(task.args) (task_dir / "task.toml").write_text( 'version = "1.0"\n' + f'schema_version = "{HARBOR_SCHEMA_VERSION}"\n' f"name = {json.dumps(slug)}\n" "\n[metadata]\n" f"hud_task = {json.dumps(task.id)}\n" diff --git a/hud/integrations/harbor/tests/test_contract.py b/hud/integrations/harbor/tests/test_contract.py index 84fb6a14f..98704d999 100644 --- a/hud/integrations/harbor/tests/test_contract.py +++ b/hud/integrations/harbor/tests/test_contract.py @@ -12,6 +12,7 @@ from hud.eval import Task from hud.integrations import harbor +from hud.integrations.harbor.adapt import HARBOR_SCHEMA_VERSION from .conftest import make_harbor_task, make_multi_step_task @@ -1057,3 +1058,27 @@ def test_authored_runtime_assets_are_valid_source() -> None: def test_public_surface_is_only_the_two_real_operations() -> None: assert harbor.__all__ == ["adapt", "export"] + + +def test_unknown_schema_version_fails_loudly(tmp_path: Path) -> None: + task = make_harbor_task(tmp_path, "task-a") + (task / "task.toml").write_text('schema_version = "99.9"\n', encoding="utf-8") + + with pytest.raises(ValueError, match="unsupported Harbor schema"): + harbor.adapt(tmp_path) + + +def test_supported_schema_version_adapts(tmp_path: Path) -> None: + task = make_harbor_task(tmp_path, "task-a") + (task / "task.toml").write_text( + f'schema_version = "{HARBOR_SCHEMA_VERSION}"\n', encoding="utf-8" + ) + + taskset = harbor.adapt(tmp_path) + assert len(list(taskset)) == 1 + + +def test_absent_schema_version_still_adapts(tmp_path: Path) -> None: + """Unversioned tasks (the historical export shape) keep adapting.""" + make_harbor_task(tmp_path, "task-a") + assert len(list(harbor.adapt(tmp_path))) == 1