diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 60e4a3a6..09ecf98b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,13 +35,6 @@ jobs: name: vero runs-on: ubuntu-latest timeout-minutes: 25 - env: - # The task compiler refuses to emit a task whose declared credentials are - # absent ("declared task credentials are missing", compiler.py), so the - # build tests need these present. Placeholders only: nothing dials them, - # and port 1 fails fast instead of hanging if anything ever tries. - OPENAI_API_KEY: ci-not-a-real-key - OPENAI_BASE_URL: http://127.0.0.1:1/v1 steps: - uses: actions/checkout@v4 diff --git a/harness-opt-bench/CONFIGURATION.md b/harness-opt-bench/CONFIGURATION.md index 447695cf..7f78b000 100644 --- a/harness-opt-bench/CONFIGURATION.md +++ b/harness-opt-bench/CONFIGURATION.md @@ -208,8 +208,8 @@ repeat their meaning. It carries the extra for the evaluation environment and can be overridden with `--param harbor_requirement=`. - `secrets` lists the environment variable names the run needs for its execution - and telemetry services. Every name must be present in the env file; the - compiler refuses to compile otherwise. Edit the list and the env file together. + and telemetry services. `vero harbor run` refuses to launch while any is + unset. Edit the list and the env file together. - `wandb` configures telemetry and is optional. Remove the block to run without it. - `extra_harbor_args` passes options to the evaluation sandboxes. The defaults group them under one application and reclaim idle ones; remove them for an diff --git a/harness-opt-bench/README.md b/harness-opt-bench/README.md index 84fa0277..5b58cccd 100644 --- a/harness-opt-bench/README.md +++ b/harness-opt-bench/README.md @@ -100,14 +100,14 @@ inspect the generated task: ```bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench//baseline/build.yaml \ --param inner_env= \ --output ``` -The compiler otherwise requires every credential the benchmark declares to be -present in the environment; the variable skips that check for a compile-only run. +Compiling needs no credentials. `vero harbor run` checks that every credential +the benchmark declares is set before it launches anything. See [`CONFIGURATION.md`](CONFIGURATION.md) for the shared evaluation protocol, per-benchmark values, and rules for changing a benchmark. diff --git a/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md b/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md index 83fa2468..e506f93d 100644 --- a/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md +++ b/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md @@ -21,15 +21,13 @@ From the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/archive/swe-atlas-qna/baseline/build.yaml \ --param inner_env= \ --output ~~~ -Use `build.gpt54mini.yaml` to compile the alternate target-model build. The -`VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. +Use `build.gpt54mini.yaml` to compile the alternate target-model build. Dataset and split details are in the [SWE-Atlas-QnA overview](../README.md). For a real optimization run, follow the diff --git a/harness-opt-bench/archive/swe-bench-pro/README.md b/harness-opt-bench/archive/swe-bench-pro/README.md index 19534844..cae91773 100644 --- a/harness-opt-bench/archive/swe-bench-pro/README.md +++ b/harness-opt-bench/archive/swe-bench-pro/README.md @@ -51,12 +51,11 @@ From the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/archive/swe-bench-pro/baseline/build.yaml \ --param inner_env= \ --output ~~~ -The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. For a real optimization run, follow the shared +For a real optimization run, follow the shared [`run-benchmark` guide](../../skills/run-benchmark/SKILL.md). diff --git a/harness-opt-bench/archive/tau3/baseline/README.md b/harness-opt-bench/archive/tau3/baseline/README.md index 757ac392..5c0f8239 100644 --- a/harness-opt-bench/archive/tau3/baseline/README.md +++ b/harness-opt-bench/archive/tau3/baseline/README.md @@ -14,13 +14,12 @@ From the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/archive/tau3/baseline/build.yaml \ --param inner_env= \ --output ~~~ -The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. Dataset and split details are in the +Dataset and split details are in the [tau3 overview](../README.md). For a real optimization run, follow the shared [`run-benchmark` guide](../../../skills/run-benchmark/SKILL.md). diff --git a/harness-opt-bench/browsecomp-plus/baseline/README.md b/harness-opt-bench/browsecomp-plus/baseline/README.md index 9915d6b5..0bb33a34 100644 --- a/harness-opt-bench/browsecomp-plus/baseline/README.md +++ b/harness-opt-bench/browsecomp-plus/baseline/README.md @@ -15,12 +15,11 @@ Prepare the generated tasks as described in the ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/browsecomp-plus/baseline/build.yaml \ --param inner_env= \ --output ~~~ -The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. For a real optimization run, use a local credential file and follow +For a real optimization run, use a local credential file and follow the shared [`run-benchmark` guide](../../skills/run-benchmark/SKILL.md). diff --git a/harness-opt-bench/gaia/baseline/README.md b/harness-opt-bench/gaia/baseline/README.md index 5f91ae38..d3e86e3a 100644 --- a/harness-opt-bench/gaia/baseline/README.md +++ b/harness-opt-bench/gaia/baseline/README.md @@ -28,15 +28,13 @@ From the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/gaia/baseline/build.yaml \ --param inner_env= \ --output ~~~ Use `build.shell.yaml` in place of `build.yaml` to compile the shell variant. -The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. For a real optimization run, follow the shared [`run-benchmark` guide](../../skills/run-benchmark/SKILL.md). Dataset and split diff --git a/harness-opt-bench/officeqa/README.md b/harness-opt-bench/officeqa/README.md index 30545c55..1214fc6d 100644 --- a/harness-opt-bench/officeqa/README.md +++ b/harness-opt-bench/officeqa/README.md @@ -49,14 +49,13 @@ From the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/officeqa/baseline/build.yaml \ --param inner_env= \ --output ~~~ -The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. For a real optimization run, follow the shared +For a real optimization run, follow the shared [`run-benchmark` guide](../skills/run-benchmark/SKILL.md). ## Where to look diff --git a/harness-opt-bench/skills/run-benchmark/SKILL.md b/harness-opt-bench/skills/run-benchmark/SKILL.md index d03792ba..2f98cdd3 100644 --- a/harness-opt-bench/skills/run-benchmark/SKILL.md +++ b/harness-opt-bench/skills/run-benchmark/SKILL.md @@ -67,7 +67,7 @@ large run. Compile before launching: ```bash -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench//baseline/build.yaml \ --param inner_env= \ --output /tmp/harness-opt-preflight diff --git a/harness-opt-bench/terminal-bench/README.md b/harness-opt-bench/terminal-bench/README.md index 7835101a..d29bb858 100644 --- a/harness-opt-bench/terminal-bench/README.md +++ b/harness-opt-bench/terminal-bench/README.md @@ -74,7 +74,7 @@ Compile from the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/terminal-bench/baseline/build.yaml \ --param inner_env= \ --output diff --git a/vero/src/vero/harbor/build/__init__.py b/vero/src/vero/harbor/build/__init__.py index ceb851c9..55f9689c 100644 --- a/vero/src/vero/harbor/build/__init__.py +++ b/vero/src/vero/harbor/build/__init__.py @@ -6,7 +6,7 @@ YAML into one, and compiler.py lowers it into a task directory. """ -from vero.harbor.build.compiler import compile_harbor_task +from vero.harbor.build.compiler import compile_harbor_task, declared_credentials from vero.harbor.build.config import HarborBuildConfig from vero.harbor.build.loader import load_harbor_build_config from vero.harbor.build.specs import ( diff --git a/vero/src/vero/harbor/build/compiler.py b/vero/src/vero/harbor/build/compiler.py index 2a459a8d..f4d7b5f1 100644 --- a/vero/src/vero/harbor/build/compiler.py +++ b/vero/src/vero/harbor/build/compiler.py @@ -584,6 +584,22 @@ def _render( ) +def declared_credentials(config) -> list[str]: + """Environment variable names a run of this build must have set. + + The build's `secrets` plus the gateway's upstream key and base URL sources. + Compiling needs none of their values, only their names, so the compiler + does not check for them; `vero harbor run` does, before it spends anything. + """ + names = list(getattr(config, "secrets", []) or []) + gateway = getattr(config, "inference_gateway", None) + if gateway is not None: + names.append(gateway.upstream_api_key_env) + if gateway.upstream_base_url_env is not None: + names.append(gateway.upstream_base_url_env) + return list(dict.fromkeys(names)) + + def compile_harbor_task( config: HarborBuildConfig, output_dir: Path | str, @@ -644,13 +660,6 @@ def compile_harbor_task( ) if name not in GATEWAY_ROUTED_CREDENTIALS ] - if os.environ.get("VERO_SKIP_SECRET_CHECK") is None: - required_sources = list(dict.fromkeys([*config.secrets, *credential_sources])) - missing = [name for name in required_sources if not os.environ.get(name)] - if missing: - raise ValueError( - "declared task credentials are missing: " + ", ".join(missing) - ) if output.exists(): shutil.rmtree(output) environment_dir = output / "environment" diff --git a/vero/src/vero/harbor/build/config.py b/vero/src/vero/harbor/build/config.py index 23b61b2b..e209aef8 100644 --- a/vero/src/vero/harbor/build/config.py +++ b/vero/src/vero/harbor/build/config.py @@ -468,9 +468,9 @@ class _TaskEnvironmentFields(StrictModel): """Credentials, model access, and reporting for the containers. Attributes: - secrets: Environment variable names routed into the task. Their presence - on the build host is checked at compile time unless - VERO_SKIP_SECRET_CHECK is set. + secrets: Environment variable names routed into the task. `vero harbor + run` refuses to launch while any is unset; compiling needs only the + names. inference_gateway: Gateway credential source and per-scope policies; see InferenceGatewaySpec. Omit for no metered model access. wandb: Trusted-side Weights & Biases reporting from the evaluation diff --git a/vero/src/vero/harbor/cli.py b/vero/src/vero/harbor/cli.py index 663ccc43..f03e41ea 100644 --- a/vero/src/vero/harbor/cli.py +++ b/vero/src/vero/harbor/cli.py @@ -939,7 +939,11 @@ def _preflight_models(config) -> None: @click.argument("extra", nargs=-1, type=click.UNPROCESSED) def run_command(config_path, agent, model, environment, params, env_file, extra): """Compile to a temporary directory and invoke `harbor run`.""" - from vero.harbor.build import compile_harbor_task, load_harbor_build_config + from vero.harbor.build import ( + compile_harbor_task, + declared_credentials, + load_harbor_build_config, + ) uvx = shutil.which("uvx") if uvx is None: @@ -947,9 +951,9 @@ def run_command(config_path, agent, model, environment, params, env_file, extra) overrides = _load_env_file(env_file) if env_file else None if overrides: click.echo(f"Loaded {len(overrides)} secret(s) from {env_file}") - # Apply before compiling: the build's declared-credential check and the - # ${NAME} param resolution both read os.environ, so the env-file must - # satisfy them too — not just the launched subprocess. + # Apply before loading the build: ${NAME} param resolution reads + # os.environ, so the env-file must satisfy it too, not just the launched + # subprocess. os.environ.update(overrides) resolved = _parse_build_params(params) # The optimizer model is both codex's -m and the producer scope's allow-list; @@ -958,6 +962,14 @@ def run_command(config_path, agent, model, environment, params, env_file, extra) if model is not None: resolved.setdefault("optimizer_model", model) config = load_harbor_build_config(config_path, params=resolved) + # A missing credential fails deep inside a container an hour from now, so + # refuse here, before compiling or spending anything. Only `run` checks: + # compiling needs the names, never the values. + missing = [name for name in declared_credentials(config) if not os.environ.get(name)] + if missing: + raise click.ClickException( + "declared task credentials are missing: " + ", ".join(missing) + ) _preflight_models(config) with tempfile.TemporaryDirectory(prefix="vero-harbor-") as temporary: task = compile_harbor_task( diff --git a/vero/tests/test_v05_cli.py b/vero/tests/test_v05_cli.py index 9113d275..9970c8b4 100644 --- a/vero/tests/test_v05_cli.py +++ b/vero/tests/test_v05_cli.py @@ -905,3 +905,32 @@ def test_outer_sandbox_args_come_from_the_build_and_yield_to_the_caller(): ] cfg.optimizer_sandbox_idle_timeout_seconds = None assert _outer_sandbox_args("modal", cfg, ()) == ["--ek", "sandbox_timeout_secs=86400"] + + +def test_harbor_run_refuses_when_a_declared_credential_is_unset(tmp_path, monkeypatch): + """The check moved from the compiler to the launcher: compiling needs names, running needs values.""" + from types import SimpleNamespace + + from vero.harbor import build as harbor_build + from vero.harbor import cli as harbor_cli + + config_path = tmp_path / "build.yaml" + config_path.write_text("name: org/task\n", encoding="utf-8") + config = SimpleNamespace( + secrets=["DEFINITELY_UNSET_CREDENTIAL"], inference_gateway=None, + harbor_requirement="harbor==0.20.0", agent_env={}, optimizer_harbor_args=[], + optimizer_sandbox_timeout_seconds=86400, optimizer_sandbox_idle_timeout_seconds=None, + name="vero/stub", + ) + monkeypatch.delenv("DEFINITELY_UNSET_CREDENTIAL", raising=False) + monkeypatch.setattr(harbor_build, "load_harbor_build_config", lambda *a, **k: config) + compiled = [] + monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda c, o: compiled.append(o) or o) + monkeypatch.setattr(harbor_cli.shutil, "which", lambda name: "/usr/bin/uvx") + + result = CliRunner().invoke( + main, ["harbor", "run", "--config", str(config_path), "--agent", "codex", "--yes"] + ) + assert result.exit_code != 0 + assert "declared task credentials are missing: DEFINITELY_UNSET_CREDENTIAL" in result.output + assert compiled == [] # refused before compiling, so before any cost diff --git a/vero/tests/test_v05_harbor_build.py b/vero/tests/test_v05_harbor_build.py index 1c3282ec..84592b61 100644 --- a/vero/tests/test_v05_harbor_build.py +++ b/vero/tests/test_v05_harbor_build.py @@ -1055,32 +1055,27 @@ def test_compiler_emits_isolated_canonical_harbor_task(tmp_path): assert "vero harbor export-session" in test_script.read_text() -def test_compiler_checks_secrets_before_writing_and_rejects_source_overlap( +def test_compiler_needs_credential_names_not_values_and_rejects_source_overlap( tmp_path, monkeypatch, ): + """Compiling writes `${NAME}` placeholders, so an unset value is not its concern. + + The presence check lives in `vero harbor run`, the one place the values are + consumed (see test_v05_cli). `declared_credentials` is how it learns the names. + """ + from vero.harbor.build import declared_credentials + config = _config(tmp_path, secrets=["MISSING_TEST_SECRET"]) output = tmp_path / "compiled" monkeypatch.delenv("MISSING_TEST_SECRET", raising=False) - with pytest.raises(ValueError, match="MISSING_TEST_SECRET"): - compile_harbor_task( - config, - output, - vero_root=Path(__file__).parents[1], - ) - assert not output.exists() - - monkeypatch.setenv("MISSING_TEST_SECRET", "configured") - compile_harbor_task( - config, - output, - vero_root=Path(__file__).parents[1], - ) + compile_harbor_task(config, output, vero_root=Path(__file__).parents[1]) task = tomllib.loads((output / "task.toml").read_text(encoding="utf-8")) assert task["environment"]["env"] == { "MISSING_TEST_SECRET": "${MISSING_TEST_SECRET}" } + assert declared_credentials(config) == ["MISSING_TEST_SECRET"] safe = config.model_copy(update={"secrets": []}) with pytest.raises(ValueError, match="overlaps protected source"):