From 3636560f28e431d3d3b72d300e2d02b01759d6d6 Mon Sep 17 00:00:00 2001 From: Varun Ursekar Date: Sat, 12 Sep 2026 21:47:24 -0700 Subject: [PATCH] Check declared credentials at launch, not at compile Compiling writes only the variable names into task.toml and compose, so the compiler's insistence that every value be set forced VERO_SKIP_SECRET_CHECK onto every compile-only use and placeholder keys onto CI. The check moves to vero harbor run, after the env file is loaded and before anything is compiled or spent. The flag is gone from the compiler, eleven documents and the CI workflow. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/tests.yml | 7 ----- harness-opt-bench/CONFIGURATION.md | 4 +-- harness-opt-bench/README.md | 6 ++-- .../archive/swe-atlas-qna/baseline/README.md | 6 ++-- .../archive/swe-bench-pro/README.md | 5 ++-- .../archive/tau3/baseline/README.md | 5 ++-- .../browsecomp-plus/baseline/README.md | 5 ++-- harness-opt-bench/gaia/baseline/README.md | 4 +-- harness-opt-bench/officeqa/README.md | 5 ++-- .../skills/run-benchmark/SKILL.md | 2 +- harness-opt-bench/terminal-bench/README.md | 2 +- vero/src/vero/harbor/build/__init__.py | 2 +- vero/src/vero/harbor/build/compiler.py | 23 ++++++++++----- vero/src/vero/harbor/build/config.py | 6 ++-- vero/src/vero/harbor/cli.py | 20 ++++++++++--- vero/tests/test_v05_cli.py | 29 +++++++++++++++++++ vero/tests/test_v05_harbor_build.py | 25 +++++++--------- 17 files changed, 93 insertions(+), 63 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 60e4a3a6..09ecf98b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,13 +35,6 @@ jobs: name: vero runs-on: ubuntu-latest timeout-minutes: 25 - env: - # The task compiler refuses to emit a task whose declared credentials are - # absent ("declared task credentials are missing", compiler.py), so the - # build tests need these present. Placeholders only: nothing dials them, - # and port 1 fails fast instead of hanging if anything ever tries. - OPENAI_API_KEY: ci-not-a-real-key - OPENAI_BASE_URL: http://127.0.0.1:1/v1 steps: - uses: actions/checkout@v4 diff --git a/harness-opt-bench/CONFIGURATION.md b/harness-opt-bench/CONFIGURATION.md index 447695cf..7f78b000 100644 --- a/harness-opt-bench/CONFIGURATION.md +++ b/harness-opt-bench/CONFIGURATION.md @@ -208,8 +208,8 @@ repeat their meaning. It carries the extra for the evaluation environment and can be overridden with `--param harbor_requirement=`. - `secrets` lists the environment variable names the run needs for its execution - and telemetry services. Every name must be present in the env file; the - compiler refuses to compile otherwise. Edit the list and the env file together. + and telemetry services. `vero harbor run` refuses to launch while any is + unset. Edit the list and the env file together. - `wandb` configures telemetry and is optional. Remove the block to run without it. - `extra_harbor_args` passes options to the evaluation sandboxes. The defaults group them under one application and reclaim idle ones; remove them for an diff --git a/harness-opt-bench/README.md b/harness-opt-bench/README.md index 84fa0277..5b58cccd 100644 --- a/harness-opt-bench/README.md +++ b/harness-opt-bench/README.md @@ -100,14 +100,14 @@ inspect the generated task: ```bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench//baseline/build.yaml \ --param inner_env= \ --output ``` -The compiler otherwise requires every credential the benchmark declares to be -present in the environment; the variable skips that check for a compile-only run. +Compiling needs no credentials. `vero harbor run` checks that every credential +the benchmark declares is set before it launches anything. See [`CONFIGURATION.md`](CONFIGURATION.md) for the shared evaluation protocol, per-benchmark values, and rules for changing a benchmark. diff --git a/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md b/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md index 83fa2468..e506f93d 100644 --- a/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md +++ b/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md @@ -21,15 +21,13 @@ From the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/archive/swe-atlas-qna/baseline/build.yaml \ --param inner_env= \ --output ~~~ -Use `build.gpt54mini.yaml` to compile the alternate target-model build. The -`VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. +Use `build.gpt54mini.yaml` to compile the alternate target-model build. Dataset and split details are in the [SWE-Atlas-QnA overview](../README.md). For a real optimization run, follow the diff --git a/harness-opt-bench/archive/swe-bench-pro/README.md b/harness-opt-bench/archive/swe-bench-pro/README.md index 19534844..cae91773 100644 --- a/harness-opt-bench/archive/swe-bench-pro/README.md +++ b/harness-opt-bench/archive/swe-bench-pro/README.md @@ -51,12 +51,11 @@ From the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/archive/swe-bench-pro/baseline/build.yaml \ --param inner_env= \ --output ~~~ -The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. For a real optimization run, follow the shared +For a real optimization run, follow the shared [`run-benchmark` guide](../../skills/run-benchmark/SKILL.md). diff --git a/harness-opt-bench/archive/tau3/baseline/README.md b/harness-opt-bench/archive/tau3/baseline/README.md index 757ac392..5c0f8239 100644 --- a/harness-opt-bench/archive/tau3/baseline/README.md +++ b/harness-opt-bench/archive/tau3/baseline/README.md @@ -14,13 +14,12 @@ From the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/archive/tau3/baseline/build.yaml \ --param inner_env= \ --output ~~~ -The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. Dataset and split details are in the +Dataset and split details are in the [tau3 overview](../README.md). For a real optimization run, follow the shared [`run-benchmark` guide](../../../skills/run-benchmark/SKILL.md). diff --git a/harness-opt-bench/browsecomp-plus/baseline/README.md b/harness-opt-bench/browsecomp-plus/baseline/README.md index 9915d6b5..0bb33a34 100644 --- a/harness-opt-bench/browsecomp-plus/baseline/README.md +++ b/harness-opt-bench/browsecomp-plus/baseline/README.md @@ -15,12 +15,11 @@ Prepare the generated tasks as described in the ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/browsecomp-plus/baseline/build.yaml \ --param inner_env= \ --output ~~~ -The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. For a real optimization run, use a local credential file and follow +For a real optimization run, use a local credential file and follow the shared [`run-benchmark` guide](../../skills/run-benchmark/SKILL.md). diff --git a/harness-opt-bench/gaia/baseline/README.md b/harness-opt-bench/gaia/baseline/README.md index 5f91ae38..d3e86e3a 100644 --- a/harness-opt-bench/gaia/baseline/README.md +++ b/harness-opt-bench/gaia/baseline/README.md @@ -28,15 +28,13 @@ From the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/gaia/baseline/build.yaml \ --param inner_env= \ --output ~~~ Use `build.shell.yaml` in place of `build.yaml` to compile the shell variant. -The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. For a real optimization run, follow the shared [`run-benchmark` guide](../../skills/run-benchmark/SKILL.md). Dataset and split diff --git a/harness-opt-bench/officeqa/README.md b/harness-opt-bench/officeqa/README.md index 30545c55..1214fc6d 100644 --- a/harness-opt-bench/officeqa/README.md +++ b/harness-opt-bench/officeqa/README.md @@ -49,14 +49,13 @@ From the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/officeqa/baseline/build.yaml \ --param inner_env= \ --output ~~~ -The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time -validation. For a real optimization run, follow the shared +For a real optimization run, follow the shared [`run-benchmark` guide](../skills/run-benchmark/SKILL.md). ## Where to look diff --git a/harness-opt-bench/skills/run-benchmark/SKILL.md b/harness-opt-bench/skills/run-benchmark/SKILL.md index d03792ba..2f98cdd3 100644 --- a/harness-opt-bench/skills/run-benchmark/SKILL.md +++ b/harness-opt-bench/skills/run-benchmark/SKILL.md @@ -67,7 +67,7 @@ large run. Compile before launching: ```bash -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench//baseline/build.yaml \ --param inner_env= \ --output /tmp/harness-opt-preflight diff --git a/harness-opt-bench/terminal-bench/README.md b/harness-opt-bench/terminal-bench/README.md index 7835101a..d29bb858 100644 --- a/harness-opt-bench/terminal-bench/README.md +++ b/harness-opt-bench/terminal-bench/README.md @@ -74,7 +74,7 @@ Compile from the repository root: ~~~bash cd vero -VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ +uv run vero harbor build \ --config ../harness-opt-bench/terminal-bench/baseline/build.yaml \ --param inner_env= \ --output diff --git a/vero/src/vero/harbor/build/__init__.py b/vero/src/vero/harbor/build/__init__.py index ceb851c9..55f9689c 100644 --- a/vero/src/vero/harbor/build/__init__.py +++ b/vero/src/vero/harbor/build/__init__.py @@ -6,7 +6,7 @@ YAML into one, and compiler.py lowers it into a task directory. """ -from vero.harbor.build.compiler import compile_harbor_task +from vero.harbor.build.compiler import compile_harbor_task, declared_credentials from vero.harbor.build.config import HarborBuildConfig from vero.harbor.build.loader import load_harbor_build_config from vero.harbor.build.specs import ( diff --git a/vero/src/vero/harbor/build/compiler.py b/vero/src/vero/harbor/build/compiler.py index 2a459a8d..f4d7b5f1 100644 --- a/vero/src/vero/harbor/build/compiler.py +++ b/vero/src/vero/harbor/build/compiler.py @@ -584,6 +584,22 @@ def _render( ) +def declared_credentials(config) -> list[str]: + """Environment variable names a run of this build must have set. + + The build's `secrets` plus the gateway's upstream key and base URL sources. + Compiling needs none of their values, only their names, so the compiler + does not check for them; `vero harbor run` does, before it spends anything. + """ + names = list(getattr(config, "secrets", []) or []) + gateway = getattr(config, "inference_gateway", None) + if gateway is not None: + names.append(gateway.upstream_api_key_env) + if gateway.upstream_base_url_env is not None: + names.append(gateway.upstream_base_url_env) + return list(dict.fromkeys(names)) + + def compile_harbor_task( config: HarborBuildConfig, output_dir: Path | str, @@ -644,13 +660,6 @@ def compile_harbor_task( ) if name not in GATEWAY_ROUTED_CREDENTIALS ] - if os.environ.get("VERO_SKIP_SECRET_CHECK") is None: - required_sources = list(dict.fromkeys([*config.secrets, *credential_sources])) - missing = [name for name in required_sources if not os.environ.get(name)] - if missing: - raise ValueError( - "declared task credentials are missing: " + ", ".join(missing) - ) if output.exists(): shutil.rmtree(output) environment_dir = output / "environment" diff --git a/vero/src/vero/harbor/build/config.py b/vero/src/vero/harbor/build/config.py index 23b61b2b..e209aef8 100644 --- a/vero/src/vero/harbor/build/config.py +++ b/vero/src/vero/harbor/build/config.py @@ -468,9 +468,9 @@ class _TaskEnvironmentFields(StrictModel): """Credentials, model access, and reporting for the containers. Attributes: - secrets: Environment variable names routed into the task. Their presence - on the build host is checked at compile time unless - VERO_SKIP_SECRET_CHECK is set. + secrets: Environment variable names routed into the task. `vero harbor + run` refuses to launch while any is unset; compiling needs only the + names. inference_gateway: Gateway credential source and per-scope policies; see InferenceGatewaySpec. Omit for no metered model access. wandb: Trusted-side Weights & Biases reporting from the evaluation diff --git a/vero/src/vero/harbor/cli.py b/vero/src/vero/harbor/cli.py index 663ccc43..f03e41ea 100644 --- a/vero/src/vero/harbor/cli.py +++ b/vero/src/vero/harbor/cli.py @@ -939,7 +939,11 @@ def _preflight_models(config) -> None: @click.argument("extra", nargs=-1, type=click.UNPROCESSED) def run_command(config_path, agent, model, environment, params, env_file, extra): """Compile to a temporary directory and invoke `harbor run`.""" - from vero.harbor.build import compile_harbor_task, load_harbor_build_config + from vero.harbor.build import ( + compile_harbor_task, + declared_credentials, + load_harbor_build_config, + ) uvx = shutil.which("uvx") if uvx is None: @@ -947,9 +951,9 @@ def run_command(config_path, agent, model, environment, params, env_file, extra) overrides = _load_env_file(env_file) if env_file else None if overrides: click.echo(f"Loaded {len(overrides)} secret(s) from {env_file}") - # Apply before compiling: the build's declared-credential check and the - # ${NAME} param resolution both read os.environ, so the env-file must - # satisfy them too — not just the launched subprocess. + # Apply before loading the build: ${NAME} param resolution reads + # os.environ, so the env-file must satisfy it too, not just the launched + # subprocess. os.environ.update(overrides) resolved = _parse_build_params(params) # The optimizer model is both codex's -m and the producer scope's allow-list; @@ -958,6 +962,14 @@ def run_command(config_path, agent, model, environment, params, env_file, extra) if model is not None: resolved.setdefault("optimizer_model", model) config = load_harbor_build_config(config_path, params=resolved) + # A missing credential fails deep inside a container an hour from now, so + # refuse here, before compiling or spending anything. Only `run` checks: + # compiling needs the names, never the values. + missing = [name for name in declared_credentials(config) if not os.environ.get(name)] + if missing: + raise click.ClickException( + "declared task credentials are missing: " + ", ".join(missing) + ) _preflight_models(config) with tempfile.TemporaryDirectory(prefix="vero-harbor-") as temporary: task = compile_harbor_task( diff --git a/vero/tests/test_v05_cli.py b/vero/tests/test_v05_cli.py index 9113d275..9970c8b4 100644 --- a/vero/tests/test_v05_cli.py +++ b/vero/tests/test_v05_cli.py @@ -905,3 +905,32 @@ def test_outer_sandbox_args_come_from_the_build_and_yield_to_the_caller(): ] cfg.optimizer_sandbox_idle_timeout_seconds = None assert _outer_sandbox_args("modal", cfg, ()) == ["--ek", "sandbox_timeout_secs=86400"] + + +def test_harbor_run_refuses_when_a_declared_credential_is_unset(tmp_path, monkeypatch): + """The check moved from the compiler to the launcher: compiling needs names, running needs values.""" + from types import SimpleNamespace + + from vero.harbor import build as harbor_build + from vero.harbor import cli as harbor_cli + + config_path = tmp_path / "build.yaml" + config_path.write_text("name: org/task\n", encoding="utf-8") + config = SimpleNamespace( + secrets=["DEFINITELY_UNSET_CREDENTIAL"], inference_gateway=None, + harbor_requirement="harbor==0.20.0", agent_env={}, optimizer_harbor_args=[], + optimizer_sandbox_timeout_seconds=86400, optimizer_sandbox_idle_timeout_seconds=None, + name="vero/stub", + ) + monkeypatch.delenv("DEFINITELY_UNSET_CREDENTIAL", raising=False) + monkeypatch.setattr(harbor_build, "load_harbor_build_config", lambda *a, **k: config) + compiled = [] + monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda c, o: compiled.append(o) or o) + monkeypatch.setattr(harbor_cli.shutil, "which", lambda name: "/usr/bin/uvx") + + result = CliRunner().invoke( + main, ["harbor", "run", "--config", str(config_path), "--agent", "codex", "--yes"] + ) + assert result.exit_code != 0 + assert "declared task credentials are missing: DEFINITELY_UNSET_CREDENTIAL" in result.output + assert compiled == [] # refused before compiling, so before any cost diff --git a/vero/tests/test_v05_harbor_build.py b/vero/tests/test_v05_harbor_build.py index 1c3282ec..84592b61 100644 --- a/vero/tests/test_v05_harbor_build.py +++ b/vero/tests/test_v05_harbor_build.py @@ -1055,32 +1055,27 @@ def test_compiler_emits_isolated_canonical_harbor_task(tmp_path): assert "vero harbor export-session" in test_script.read_text() -def test_compiler_checks_secrets_before_writing_and_rejects_source_overlap( +def test_compiler_needs_credential_names_not_values_and_rejects_source_overlap( tmp_path, monkeypatch, ): + """Compiling writes `${NAME}` placeholders, so an unset value is not its concern. + + The presence check lives in `vero harbor run`, the one place the values are + consumed (see test_v05_cli). `declared_credentials` is how it learns the names. + """ + from vero.harbor.build import declared_credentials + config = _config(tmp_path, secrets=["MISSING_TEST_SECRET"]) output = tmp_path / "compiled" monkeypatch.delenv("MISSING_TEST_SECRET", raising=False) - with pytest.raises(ValueError, match="MISSING_TEST_SECRET"): - compile_harbor_task( - config, - output, - vero_root=Path(__file__).parents[1], - ) - assert not output.exists() - - monkeypatch.setenv("MISSING_TEST_SECRET", "configured") - compile_harbor_task( - config, - output, - vero_root=Path(__file__).parents[1], - ) + compile_harbor_task(config, output, vero_root=Path(__file__).parents[1]) task = tomllib.loads((output / "task.toml").read_text(encoding="utf-8")) assert task["environment"]["env"] == { "MISSING_TEST_SECRET": "${MISSING_TEST_SECRET}" } + assert declared_credentials(config) == ["MISSING_TEST_SECRET"] safe = config.model_copy(update={"secrets": []}) with pytest.raises(ValueError, match="overlaps protected source"):