From 17e280159b108f710d4591cea624b4f0c575d90b Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Sat, 1 Aug 2026 20:47:24 +0300 Subject: [PATCH] feat: let a relaunch resume a Harbor run from the session it left behind PR #78 hardened a resume path that nothing could enter. This opens one door into it, explicitly, and reports honestly on the two it cannot open. Where the session directory actually lives, verified rather than assumed: a compiled task's trusted state sits at /state/admin/session on the `admin_state` compose volume (docker-compose.yaml.j2), inside the DinD stack, inside a per-trial Modal sandbox, and `vero harbor run` itself compiles into a tempfile.TemporaryDirectory. Nothing on the launching host holds it, so no stable identity could ever have made a relaunch find it again. What does survive is the archive harbor collects to /artifacts/session-rescue.tar.gz on every terminal outcome (#77), and the verifier/session.tar.gz a finished run leaves. The resume is therefore archive transport, not a naming scheme. `vero harbor run --resume-from ` copies the archive into the sidecar build context; the Dockerfile COPYs it to /opt/session-seed.tar.gz and chmods it 600, because it carries database.json, whose per-case records disclose held-out membership and scores, so it gets the same lock as the case lists. serve.json names it as `session_seed_archive`, and build_harbor_components restores it into the session directory before the candidate repository, the evaluation database, the budget ledger and the agent's disclosure ledger are constructed. Each of those reopens what is on disk, so seeding the directory is the whole of the restore. A bad archive is refused on the host in the second before the compile, by reading its manifest without unpacking it, rather than an hour later inside a sandbox nobody is tailing. A restore into a session that already carries its manifest is a no-op, so a sidecar restart inside a live run is not rolled back to an older attempt's state. Why this is not the deterministic default session id #78 backed out. That change was unsafe because it changed a *default*: a second `vero optimize` over one target silently produced zero new evaluations and adopted the first attempt's unusable baseline record, sticky until `vero session clear`. Nothing here changes a default. A launch resumes only when the operator names an archive, and the uuid4 fallback in `_session_identity` is untouched. `vero init` additionally emits a commented `[session]` block, documenting the local-path opt-in that has always worked and that no template, doc string or build config ever mentioned, together with the baseline hazard that is the reason to keep it opt-in. What this does not do. #74's outer-trial retry still restarts from zero and cannot do otherwise. harbor/trial/queue.py::_execute_trial_with_retries rmtrees the trial directory and calls Trial.create again, and a Trial builds its own environment (harbor/trial/trial.py:825), so the retried attempt gets a new sandbox with new volumes. It also runs inside one `vero harbor run` process that compiled its task before the first attempt began, so there is no archive of the failed attempt to bake in and no recompile to bake it into. Resuming is a second launch, by hand, after the first has exited. The optimizer's own context is not restored. It re-reads its prior evaluations from the restored .evals directory, so it is informed, but it starts from an empty transcript and /work/agent is re-seeded from the seed repo rather than from the last candidate. The baseline-reuse guards #78 named are still absent. `_run_configured` still passes skip_baseline_evaluation=manifest_path.exists(), so a local resume whose first attempt died during its baseline still adopts that unusable record. That is documented in the emitted template, not fixed, because fixing it changes what gets measured. A restored finalization.json and reward.json come back with everything else. Harmless today, since test.sh POSTs finalize unconditionally and the writes are atomic overwrites, but nothing asserts the restored copies are ignored. An infrastructure failure recorded as a scoreable zero is restored as one, exactly as #78 left it. The budget ledger resumes spent, which is intended, and means a resumed run cannot be given a fresh agent budget without editing budgets.json. Archive size is unmeasured. It becomes a docker layer, so a large candidates/repository.git is paid for in image build and sandbox pull time on every resumed launch, with no cap and no warning. `vero harbor build` did not get the flag, and launch_cell.sh has no passthrough for it. None of this has been executed against Modal. Six new tests cover the compile output, the archive checks, the restore semantics and the deployment factory, including that a resume into a drifted build fails on the restored manifest instead of silently mixing two incomparable runs. That a restored directory is enough for a live sidecar to behave as a continuation is tested at the factory level, not in a sandbox. Three existing tests had their compile_harbor_task stubs widened to accept keyword arguments; no existing assertion changed. Full suite: 552 passed, 16 skipped, 0 failures (546 and 16 before). Co-Authored-By: Claude Opus 5 (1M context) --- vero/docs/harbor-architecture.md | 33 +- vero/src/vero/cli.py | 13 + vero/src/vero/harbor/build/compiler.py | 39 +- .../build/templates/Dockerfile.sidecar.j2 | 9 + vero/src/vero/harbor/cli.py | 19 +- vero/src/vero/harbor/deployment.py | 24 +- vero/src/vero/layout.py | 5 + vero/src/vero/sidecar/session.py | 106 +++- vero/tests/test_resume_across_relaunch.py | 465 ++++++++++++++++++ vero/tests/test_v05_cli.py | 4 +- vero/tests/test_v05_harbor_build.py | 2 +- vero/tests/test_v05_harbor_http.py | 4 +- 12 files changed, 698 insertions(+), 25 deletions(-) create mode 100644 vero/tests/test_resume_across_relaunch.py diff --git a/vero/docs/harbor-architecture.md b/vero/docs/harbor-architecture.md index 0bb3ea69..04a70953 100644 --- a/vero/docs/harbor-architecture.md +++ b/vero/docs/harbor-architecture.md @@ -102,10 +102,35 @@ against a benchmark's pinned baseline is `harness-engineering-bench/scripts/rescore_candidate.py --session `, which lives out of tree because it needs that benchmark's `build.yaml`. -A true *resume* is not available and is not the goal here: the optimizer's -working tree, its harness process, and its agent context all live in the Modal -sandbox, which is torn down. What survives is every candidate the optimizer -committed and every score it measured. +The optimizer's working tree and its harness process are not recoverable: they +live in the Modal sandbox, which is torn down. What survives is every candidate +the optimizer committed and every score it measured. + +### Relaunching from what survived + +`vero harbor run --resume-from ` takes either archive and bakes it into +the new stack's sidecar image, and the sidecar restores it into the session +directory on first boot, before the candidate repository, the evaluation +database, the budget ledger or the agent's disclosure ledger are opened. Each of +those reuses what is on disk, so seeding the directory *is* the resume. + +Baking it in is not an aesthetic choice. The session directory is a compose +volume inside a per-trial sandbox, so nothing a relaunch could mount survives +the trial that wrote it, and the archive sitting on the launching host is the +only copy left. Two consequences follow. The archive is added to the sidecar +image, so a large `candidates/repository.git` is paid for in build time and +image size. And harbor's own `--max-retries` restart (`harbor/trial/queue.py`, +`_execute_trial_with_retries`) does **not** benefit: it `rmtree`s the trial +directory and calls `Trial.create` again against a fresh sandbox, within one +`vero harbor run` process that has long since compiled its task, so no archive +from the failed attempt exists yet, let alone one baked into an image. Resuming +is a second `vero harbor run`, by hand, after the first has exited. + +What comes back is what the archive holds: the candidate commits, the evaluation +records and their scores, and the spent agent budget. What does not is the +optimizer's own reasoning. It restarts with an empty context and re-reads its +prior evaluations from the restored `.evals` directory, so it is informed, not +mid-thought. ## The evaluation core diff --git a/vero/src/vero/cli.py b/vero/src/vero/cli.py index 792fd90f..6ab44ed5 100644 --- a/vero/src/vero/cli.py +++ b/vero/src/vero/cli.py @@ -103,6 +103,19 @@ def _default_home() -> Path: [optimizer] kind = "claude" instruction = "Improve the program without changing its intended behavior" + +# [session] +# Uncommented, `id` turns every later `vero run` over this file into a relaunch +# of one logical run: the session directory becomes $VERO_HOME/sessions/ +# instead of a fresh uuid4 per invocation, and the candidates, scores and budget +# already on disk are picked up rather than remade. Commented out because the +# resume is not free: a relaunch skips the baseline evaluation whenever a +# manifest exists, so if the first attempt died *during* its baseline it hands +# the rerun that attempt's unusable baseline record (no objective, no cases) and +# every later comparison is against it, until `vero session clear`. Opt in per +# run, with an id that names the run, and clear the session when the identity of +# what you are optimizing changes. +# id = "my-run-2026-08-01" ''' diff --git a/vero/src/vero/harbor/build/compiler.py b/vero/src/vero/harbor/build/compiler.py index 58df9f38..62475fd0 100644 --- a/vero/src/vero/harbor/build/compiler.py +++ b/vero/src/vero/harbor/build/compiler.py @@ -25,6 +25,7 @@ from vero.harbor.build.config import HarborBuildConfig from vero.harbor.build.specs import WorkspaceOverlaySpec from vero.layout import LAYOUT +from vero.sidecar.session import read_harbor_session_archive_manifest logger = logging.getLogger(__name__) @@ -66,6 +67,7 @@ AGENT_VOLUME = LAYOUT.agent_volume ADMIN_VOLUME = LAYOUT.admin_volume SESSION_DIR = LAYOUT.session_dir +SESSION_SEED_ARCHIVE = LAYOUT.session_seed_archive TOKEN_PATH = LAYOUT.token_path INFERENCE_STATE = LAYOUT.inference_state INFERENCE_REQUEST_LOG_DIR = LAYOUT.inference_request_log_dir @@ -277,6 +279,7 @@ def _deployment_config( local_task_source: bool, evaluation_inference_token: str | None, finalization_inference_token: str | None, + session_seed_archive: bool = False, ) -> dict: task_source = TASK_SOURCE_DIR if local_task_source else config.task_source backends = {} @@ -459,6 +462,9 @@ def _deployment_config( "agent_repo_path": AGENT_REPO, "session_dir": SESSION_DIR, "session_id": SESSION_ID, + "session_seed_archive": ( + SESSION_SEED_ARCHIVE if session_seed_archive else None + ), "backends": backends, "access_policies": policies, "budgets": budgets, @@ -588,9 +594,29 @@ def compile_harbor_task( output_dir: Path | str, *, vero_root: Path | None = None, + session_seed_archive: Path | str | None = None, ) -> Path: - """Emit a self-contained Harbor task directory from validated config.""" + """Emit a self-contained Harbor task directory from validated config. + + ``session_seed_archive`` is a previously exported session (the + ``session-rescue.tar.gz`` a dead trial leaves in its artifacts, or the + ``verifier/session.tar.gz`` a finished one leaves) to restore into the new + stack's session directory on first boot. Baking it into the sidecar image is + the only transport available: the session directory lives on a compose volume + inside a per-trial Modal sandbox, so nothing a relaunch could mount survives, + and the archive on the launching host is the only copy that does. + """ output = Path(output_dir).expanduser().resolve() + seed_archive = ( + Path(session_seed_archive).expanduser().resolve() + if session_seed_archive is not None + else None + ) + if seed_archive is not None: + # Fail on the host, now, rather than after an image build and a stack + # bring-up inside a sandbox whose logs nobody is tailing. Reads the + # manifest only; the bytes are not unpacked here. + read_harbor_session_archive_manifest(seed_archive) source_root = (vero_root or Path(__file__).parents[4]).resolve() use_local_vero = _is_vero_source(source_root) if vero_root is not None and not use_local_vero: @@ -715,6 +741,13 @@ def compile_harbor_task( Path(config.command_backend.harness_source), sidecar_dir / "harness", ) + if seed_archive is not None: + # Into the sidecar's build context, beside the case lists it is as + # sensitive as: the archive carries database.json, whose per-case records + # name held-out tasks and their scores. The Dockerfile chmods it 600 on + # the way in, matching serve.json. + sidecar_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(seed_archive, sidecar_dir / "session-seed.tar.gz") local_task_source = ( config.task_source is not None and Path(config.task_source).exists() ) @@ -754,6 +787,7 @@ def compile_harbor_task( local_task_source=local_task_source, evaluation_inference_token=evaluation_inference_token, finalization_inference_token=finalization_inference_token, + session_seed_archive=seed_archive is not None, ) (sidecar_dir / "serve.json").write_text( json.dumps(deployment, ensure_ascii=False, indent=2) + "\n", @@ -871,6 +905,9 @@ def compile_harbor_task( "sidecar_factory": FACTORY_PATH, "producer_base_url": PRODUCER_BASE_URL, "command_harness": config.command_backend is not None, + "session_seed_archive": ( + SESSION_SEED_ARCHIVE if seed_archive is not None else None + ), # The Harbor backend hard-rejects request.seed, so only advertise the # flag when the build evaluates through a command backend. "seed_supported": config.command_backend is not None, diff --git a/vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2 b/vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2 index 0b30e2f5..3f3f89ba 100644 --- a/vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2 +++ b/vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2 @@ -16,6 +16,9 @@ RUN uv pip install --system "{{ harbor_requirement }}" COPY agent-baseline {{ layout.trusted_repo }} COPY sidecar/cases {{ layout.cases }} COPY sidecar/serve.json {{ layout.serve_config }} +{% if session_seed_archive %} +COPY sidecar/session-seed.tar.gz {{ session_seed_archive }} +{% endif %} {% if local_task_source %} COPY sidecar/task-source {{ layout.task_source }} {% endif %} @@ -49,5 +52,11 @@ RUN mkdir -p /home/harness/.cache \ # {{ layout.task_source }}, which harbor must read to grade host-side, remain readable # and are addressed by the deeper in-container isolation work.) RUN chmod 700 {{ layout.cases }} && chmod 600 {{ layout.serve_config }} +{% if session_seed_archive %} +{# The restored session's database.json carries held-out case membership and + per-case scores, the same secret the case lists above hold, so the seed + archive is locked to root on the same grounds. #} +RUN chmod 600 {{ session_seed_archive }} +{% endif %} WORKDIR /opt diff --git a/vero/src/vero/harbor/cli.py b/vero/src/vero/harbor/cli.py index 9be14ae5..2b40ec45 100644 --- a/vero/src/vero/harbor/cli.py +++ b/vero/src/vero/harbor/cli.py @@ -775,6 +775,18 @@ def _preflight_models(config) -> None: @click.option("--agent", required=True, help="Harbor optimizer agent.") @click.option("--model", help="Model used by the optimizer agent.") @click.option("--environment", default="modal", show_default=True) +@click.option( + "--resume-from", + "resume_from", + type=click.Path(path_type=Path, exists=True, dir_okay=False), + help=( + "Restore a previous run's exported session into this one, so the " + "relaunch keeps its candidates, its scores and its spent budget instead " + "of starting over. Takes the artifacts/session-rescue.tar.gz a dead " + "trial leaves behind, or the verifier/session.tar.gz a finished one " + "does. Opt-in: without it a relaunch is a fresh run." + ), +) @click.option( "--env-file", "env_file", @@ -787,7 +799,9 @@ def _preflight_models(config) -> None: ) @_PARAM_OPTION @click.argument("extra", nargs=-1, type=click.UNPROCESSED) -def run_command(config_path, agent, model, environment, params, env_file, extra): +def run_command( + config_path, agent, model, environment, resume_from, params, env_file, extra +): """Compile to a temporary directory and invoke `harbor run`.""" from vero.harbor.build import compile_harbor_task, load_harbor_build_config @@ -813,7 +827,10 @@ def run_command(config_path, agent, model, environment, params, env_file, extra) task = compile_harbor_task( config, Path(temporary) / "task", + session_seed_archive=resume_from, ) + if resume_from is not None: + click.echo(f"Resuming from session archive {resume_from}") command = [ uvx, "--python", diff --git a/vero/src/vero/harbor/deployment.py b/vero/src/vero/harbor/deployment.py index 0bf66710..0eea3820 100644 --- a/vero/src/vero/harbor/deployment.py +++ b/vero/src/vero/harbor/deployment.py @@ -28,7 +28,10 @@ from vero.runtime.artifacts import ArtifactStore from vero.sandbox import LocalSandbox from vero.sidecar.serve import SidecarComponents -from vero.sidecar.session import initialize_harbor_session_manifest +from vero.sidecar.session import ( + initialize_harbor_session_manifest, + restore_harbor_session_archive, +) from vero.sidecar.sidecar import EvaluationSidecar, SidecarEvaluationPolicy from vero.sidecar.transport import GitCandidateTransport from vero.sidecar.verifier import ( @@ -123,6 +126,10 @@ class HarborDeploymentConfig(StrictModel): agent_repo_path: str session_dir: str session_id: str = "trial" + # A previous run's exported session, baked into the sidecar image, restored + # into session_dir on first boot. Absent by default: a relaunch starts clean + # unless the operator named an archive on the command line. + session_seed_archive: str | None = None backends: dict[str, DeploymentBackendConfig] access_policies: list[SidecarEvaluationPolicy] budgets: list[EvaluationBudget] = Field(default_factory=list) @@ -176,7 +183,11 @@ def validate_absolute_path(cls, value: str) -> str: raise ValueError("deployment paths must be absolute") return value - @field_validator("inference_usage_path", "inference_request_log_dir") + @field_validator( + "inference_usage_path", + "inference_request_log_dir", + "session_seed_archive", + ) @classmethod def validate_optional_file_path(cls, value: str | None) -> str | None: if value is not None: @@ -271,6 +282,15 @@ async def build_harbor_components(config: dict) -> SidecarComponents: parsed = HarborDeploymentConfig.model_validate(config) session_dir = Path(parsed.session_dir) session_dir.mkdir(parents=True, exist_ok=True) + # Before anything reads the session dir. Every consumer below it -- the + # candidate repository, the evaluation database, the budget ledger, the + # agent's disclosure ledger -- opens what is on disk and reuses it if it is + # there, so seeding here is the whole of the restore: no consumer needs to + # know a resume happened. Missing archive is fatal rather than ignored: the + # operator asked for a resume, and silently starting a fresh multi-hour run + # instead is the failure this exists to prevent. + if parsed.session_seed_archive is not None: + restore_harbor_session_archive(parsed.session_seed_archive, session_dir) # The session dir holds the trusted state — held-out evaluation records and # scores, the budget ledger, and every candidate's code. Lock it to the # owning (trusted) user so the unprivileged harness that executes candidate diff --git a/vero/src/vero/layout.py b/vero/src/vero/layout.py index 741df26c..23da2542 100644 --- a/vero/src/vero/layout.py +++ b/vero/src/vero/layout.py @@ -38,6 +38,10 @@ class TaskLayout: harness: The scoring program, for a command backend. overlay: Host files baked into the optimizer's workspace. serve_config: The trusted deployment config, root-only. + session_seed_archive: A previously exported session, baked in so a + relaunch can restore it. Root-only for the same reason as + serve_config and the case lists: it carries database.json, whose + per-case records disclose held-out membership and scores. seed_script: Script that seeds the target repo on first boot. inference_config: The gateway's scope config. agent_volume: The optimizer's context directory, written by the sidecar. @@ -63,6 +67,7 @@ class TaskLayout: harness: str = "/opt/harness" overlay: str = "/opt/overlay" serve_config: str = "/opt/serve.json" + session_seed_archive: str = "/opt/session-seed.tar.gz" seed_script: str = "/opt/seed.sh" inference_config: str = "/opt/inference.json" agent_volume: str = "/state/agent-context" diff --git a/vero/src/vero/sidecar/session.py b/vero/src/vero/sidecar/session.py index 1c69151d..f6928b52 100644 --- a/vero/src/vero/sidecar/session.py +++ b/vero/src/vero/sidecar/session.py @@ -7,6 +7,7 @@ import json import logging import os +import shutil import tarfile import tempfile from collections import deque @@ -227,6 +228,32 @@ def _text_member(archive: tarfile.TarFile, arcname: str, payload: bytes) -> None return output +def _validated_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]: + """Every member of a session archive, or a refusal naming the first bad one. + + Factored out so the read-only inspection below applies exactly the rules the + extractor applies. Duplicating them would let the two drift, and the drift + that matters is the one where a launch-time check passes an archive the + sidecar then refuses an hour into the run, inside a sandbox nobody is + watching. + """ + + members = archive.getmembers() + for member in members: + path = PurePosixPath(member.name) + if ( + path.is_absolute() + or not path.parts + or path.parts[0] != "session" + or ".." in path.parts + or member.issym() + or member.islnk() + or member.isdev() + ): + raise ValueError(f"unsafe Harbor session archive member: {member.name}") + return members + + def extract_harbor_session_archive( archive_path: Path | str, destination: Path | str, @@ -237,19 +264,7 @@ def extract_harbor_session_archive( destination = Path(destination).expanduser().resolve() destination.mkdir(parents=True, exist_ok=True) with tarfile.open(archive_path, "r:gz") as archive: - members = archive.getmembers() - for member in members: - path = PurePosixPath(member.name) - if ( - path.is_absolute() - or not path.parts - or path.parts[0] != "session" - or ".." in path.parts - or member.issym() - or member.islnk() - or member.isdev() - ): - raise ValueError(f"unsafe Harbor session archive member: {member.name}") + members = _validated_members(archive) archive.extractall(destination, members=members, filter="data") session = destination / "session" HarborSessionManifest.model_validate_json( @@ -258,6 +273,71 @@ def extract_harbor_session_archive( return session +def read_harbor_session_archive_manifest( + archive_path: Path | str, +) -> HarborSessionManifest: + """Read a session archive's identity without unpacking it. + + Exists so `vero harbor run --resume-from` can reject a wrong or corrupt + archive on the host, in the second before the compile, rather than baking it + into a sidecar image and discovering it inside a Modal sandbox after the + image build, the stack bring-up and the seed have already been paid for. + """ + + with tarfile.open(Path(archive_path).expanduser().resolve(), "r:gz") as archive: + _validated_members(archive) + try: + manifest = archive.extractfile("session/harbor-session.json") + except KeyError: + manifest = None + if manifest is None: + raise ValueError("Harbor session archive has no session manifest") + return HarborSessionManifest.model_validate_json(manifest.read()) + + +def restore_harbor_session_archive( + archive_path: Path | str, + session_dir: Path | str, +) -> bool: + """Seed an empty session directory from a previous run's export. + + Returns whether anything was restored. A session that already carries its + manifest is left untouched, because the two callers are indistinguishable + from here: a *relaunch* boots against a fresh volume and has to be seeded, + while a sidecar *restart* inside a live run boots against the volume it has + been writing all along, and overwriting that with the state of an older run + would undo everything since. Refusing on the manifest and not on emptiness + is deliberate: a first boot that died between mkdir and the manifest write + leaves a non-empty directory that still holds nothing worth keeping. + + Staged into a sibling directory and moved in entry by entry, so a restore + killed half way leaves the session dir either untouched or with whole files, + never with a truncated database.json that reads as a complete one. + """ + + session_dir = Path(session_dir).expanduser().resolve() + if (session_dir / "harbor-session.json").is_file(): + return False + session_dir.mkdir(parents=True, exist_ok=True) + staging = Path( + tempfile.mkdtemp(dir=session_dir.parent, prefix=f".{session_dir.name}-restore-") + ) + try: + source = extract_harbor_session_archive(archive_path, staging) + for entry in sorted(source.iterdir()): + target = session_dir / entry.name + if target.is_dir() and not target.is_symlink(): + shutil.rmtree(target) + elif target.exists() or target.is_symlink(): + target.unlink() + shutil.move(str(entry), str(target)) + finally: + shutil.rmtree(staging, ignore_errors=True) + _fsync_path(session_dir) + logger.info("Restored Harbor session into %s from %s", session_dir, archive_path) + return True + + def file_sha256(path: Path | str) -> str: digest = hashlib.sha256() with Path(path).open("rb") as file: diff --git a/vero/tests/test_resume_across_relaunch.py b/vero/tests/test_resume_across_relaunch.py new file mode 100644 index 00000000..2a9fefd1 --- /dev/null +++ b/vero/tests/test_resume_across_relaunch.py @@ -0,0 +1,465 @@ +"""A second launch of the same run has to be able to pick up the first one's work. + +PR #78 hardened a resume path, and every one of those guards sits inside a +session directory that a relaunch never reaches: `vero harbor run` compiles into +a throwaway directory, the session lives on the `admin_state` compose volume +inside a per-trial Modal sandbox, and the sandbox goes away with the trial. The +one copy that outlives it is the archive on the launching host, so the tests +here are about that archive: refusing a bad one before an image is built, +carrying a good one into the next stack's sidecar without widening who can read +it, restoring it before anything opens the session directory, and not restoring +it over a session that is currently being written. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tarfile +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from vero.evaluation import ( + DisclosureLevel, + EvaluationAccessPolicy, + EvaluationBudget, + EvaluationCost, + EvaluationSet, + MetricSelector, + ObjectiveSpec, +) +from vero.harbor import ( + AgentAccessSpec, + HarborBackendConfig, + HarborBuildConfig, + VerificationTargetSpec, + build_harbor_components, + compile_harbor_task, +) +from vero.layout import LAYOUT +from vero.sidecar import SidecarEvaluationPolicy, VerificationTarget +from vero.sidecar.session import ( + HarborSessionManifest, + create_harbor_session_archive, + read_harbor_session_archive_manifest, + restore_harbor_session_archive, +) +from vero.sidecar.verifier import VerificationSelection + +_VERO_ROOT = Path(__file__).parents[1] + + +def _git(path: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", *arguments], + cwd=path, + check=True, + text=True, + capture_output=True, + ) + return result.stdout.strip() + + +def _repo(path: Path) -> str: + path.mkdir(parents=True) + _git(path, "init", "-q") + _git(path, "config", "user.name", "VeRO Test") + _git(path, "config", "user.email", "vero@example.test") + (path / "program.py").write_text("VALUE = 1\n", encoding="utf-8") + (path / "pyproject.toml").write_text( + '[project]\nname="target"\nversion="0.1.0"\n', encoding="utf-8" + ) + _git(path, "add", ".") + _git(path, "commit", "-q", "-m", "baseline") + return _git(path, "rev-parse", "HEAD") + + +def _session_archive(root: Path, *, payload: str = "restored") -> Path: + """The smallest thing that satisfies the archive contract, plus a marker. + + A real archive is whatever a run left behind; what every consumer here keys + off is the manifest at the root, so this stands in for one wherever the test + is about transport rather than about the state itself. + """ + + session = root / "session" + session.mkdir(parents=True) + manifest = HarborSessionManifest( + id="trial", + task_name="org/task", + created_at=datetime.now(UTC), + backends={}, + selection=VerificationSelection(mode="submit"), + targets=[], + ) + (session / "harbor-session.json").write_text( + manifest.model_dump_json(), encoding="utf-8" + ) + (session / "marker.txt").write_text(payload, encoding="utf-8") + return create_harbor_session_archive(session, root / "session.tar.gz") + + +def _build_config(root: Path) -> HarborBuildConfig: + target = root / "target" + _repo(target) + tasks = root / "tasks" + for name in ("task-a", "task-hidden"): + (tasks / name).mkdir(parents=True) + (tasks / name / "task.toml").write_text( + f'[task]\nname="org/{name}"\n', encoding="utf-8" + ) + return HarborBuildConfig( + name="org/optimize-program", + description="Improve the program", + agent_repo=str(target), + task_source=str(tasks), + agent_import_path="target.agent:Agent", + harbor_requirement="harbor==0.1.17", + partitions={"validation": ["task-a"], "test": ["task-hidden"]}, + agent_access=[AgentAccessSpec(partition="validation", total_runs=5)], + selection_partition="validation", + targets=[VerificationTargetSpec(partition="test")], + ) + + +def test_a_compile_carries_the_session_archive_only_when_a_launch_asked_for_one( + tmp_path, +): + """The archive has to travel in the sidecar image, and only on request. + + Baking it in is the only transport there is: the session directory is a + compose volume inside the sandbox, so a relaunch has nothing to mount. That + also means the default must stay clean, because every launch that is not a + resume would otherwise pay the image size and, worse, adopt a previous run's + scores without anyone having asked for it. + """ + + config = _build_config(tmp_path / "build") + archive = _session_archive(tmp_path / "previous") + + plain = compile_harbor_task( + config, tmp_path / "plain", vero_root=_VERO_ROOT + ) + sidecar = plain / "environment/sidecar" + assert not (sidecar / "session-seed.tar.gz").exists() + assert json.loads((sidecar / "serve.json").read_text())[ + "session_seed_archive" + ] is None + assert "session-seed.tar.gz" not in (sidecar / "Dockerfile").read_text() + + resumed = compile_harbor_task( + config, + tmp_path / "resumed", + vero_root=_VERO_ROOT, + session_seed_archive=archive, + ) + sidecar = resumed / "environment/sidecar" + assert (sidecar / "session-seed.tar.gz").read_bytes() == archive.read_bytes() + assert ( + json.loads((sidecar / "serve.json").read_text())["session_seed_archive"] + == LAYOUT.session_seed_archive + ) + dockerfile = (sidecar / "Dockerfile").read_text() + copy = f"COPY sidecar/session-seed.tar.gz {LAYOUT.session_seed_archive}" + assert copy in dockerfile + # The archive carries database.json, so it discloses held-out membership and + # per-case scores exactly as the case lists do. It gets the same lock, or the + # unprivileged harness that runs candidate code can read the answers. + assert f"chmod 600 {LAYOUT.session_seed_archive}" in dockerfile + + +def test_a_launch_refuses_an_archive_it_could_not_have_restored(tmp_path): + """Reject on the host, before an image build, not inside the sandbox. + + The failure this prevents is silent and expensive: a wrong path compiles + fine, builds three images, brings the stack up, and only then does the + sidecar refuse, tens of minutes into a run whose logs nobody is tailing. + """ + + config = _build_config(tmp_path / "build") + plain = tmp_path / "not-an-archive.tar.gz" + with tarfile.open(plain, "w:gz") as archive: + source = tmp_path / "loose.txt" + source.write_text("no manifest here\n", encoding="utf-8") + archive.add(source, arcname="session/loose.txt") + + with pytest.raises(ValueError, match="no session manifest"): + compile_harbor_task( + config, + tmp_path / "out", + vero_root=_VERO_ROOT, + session_seed_archive=plain, + ) + assert not (tmp_path / "out").exists() + + escaping = tmp_path / "escaping.tar.gz" + with tarfile.open(escaping, "w:gz") as archive: + archive.add(tmp_path / "loose.txt", arcname="../escape.txt") + with pytest.raises(ValueError, match="unsafe Harbor session archive member"): + read_harbor_session_archive_manifest(escaping) + + +def test_a_restore_seeds_an_empty_session_and_spares_a_live_one(tmp_path): + """The two boots that reach this code are indistinguishable from inside it. + + A relaunch boots against a fresh volume and must be seeded. A sidecar restart + inside a live run boots against the volume it has been writing all along, and + seeding that would roll the run back to an older attempt's state. The + manifest is what tells them apart. + """ + + archive = _session_archive(tmp_path / "previous") + + fresh = tmp_path / "state/session" + assert restore_harbor_session_archive(archive, fresh) is True + assert (fresh / "marker.txt").read_text() == "restored" + assert json.loads((fresh / "harbor-session.json").read_text())["id"] == "trial" + # Staged into a sibling and moved in, so nothing is left behind to be picked + # up as session state on a later boot. + assert sorted(path.name for path in (tmp_path / "state").iterdir()) == ["session"] + + live = tmp_path / "live/session" + live.mkdir(parents=True) + (live / "harbor-session.json").write_text( + (fresh / "harbor-session.json").read_text(), encoding="utf-8" + ) + (live / "marker.txt").write_text("work since the restart", encoding="utf-8") + assert restore_harbor_session_archive(archive, live) is False + assert (live / "marker.txt").read_text() == "work since the restart" + + +@pytest.mark.asyncio +async def test_a_relaunch_resumes_the_previous_run_s_ledger_and_candidates(tmp_path): + """The end of the line: a fresh volume that behaves like the old one. + + Everything the sidecar builds below the session directory reopens what is on + disk, so restoring the directory before any of them is constructed is the + whole of the resume. The budget ledger is the proof that carries the least + ceremony and the most meaning: a relaunch that did not resume hands the + optimizer its agent budget back, and a run that has already spent hours can + then spend it a second time. + """ + + trusted = tmp_path / "trusted" + agent = tmp_path / "agent" + _repo(trusted) + _repo(agent) + cases = tmp_path / "cases.jsonl" + cases.write_text( + json.dumps({"id": "task", "task_name": "org/task"}) + "\n", encoding="utf-8" + ) + evaluation_set = EvaluationSet(name="benchmark", partition="validation") + objective = ObjectiveSpec( + selector=MetricSelector(metric="score"), direction="maximize" + ) + backend = HarborBackendConfig( + task_source="org/benchmark@1.0", + agent_import_path="program:Agent", + cases_path=str(cases), + harbor_requirement="harbor==0.1.17", + evaluation_set_name="benchmark", + partition="validation", + uv_executable=sys.executable, + ) + + def _config(session_dir: Path, seed: Path | None = None) -> dict: + return { + "repo_path": str(trusted), + "agent_repo_path": str(agent), + "session_dir": str(session_dir), + "session_id": "trial", + **({"session_seed_archive": str(seed)} if seed is not None else {}), + "backends": {"validation": backend.model_dump(mode="json")}, + "access_policies": [ + SidecarEvaluationPolicy( + backend_id="validation", + evaluation_set_name="benchmark", + partition="validation", + objective=objective, + access=EvaluationAccessPolicy( + disclosure=DisclosureLevel.AGGREGATE + ), + ).model_dump(mode="json") + ], + "budgets": [ + EvaluationBudget( + backend_id="validation", + evaluation_set_key=evaluation_set.budget_key("validation"), + total_runs=4, + ).model_dump(mode="json") + ], + "selection": { + "mode": "auto_best", + "backend_id": "validation", + "evaluation_set": evaluation_set.model_dump(mode="json"), + "objective": objective.model_dump(mode="json"), + "baseline_version": "HEAD", + }, + "targets": [ + VerificationTarget( + reward_key="reward", + backend_id="validation", + evaluation_set=evaluation_set, + objective=objective, + ).model_dump(mode="json") + ], + "agent_volume": str(session_dir.parent / "agent"), + "admin_volume": str(session_dir.parent), + } + + first = tmp_path / "attempt-one/session" + components = await build_harbor_components(_config(first)) + ledger = components.sidecar.engine.budget_ledger + assert ledger is not None + await ledger.reserve("validation", evaluation_set, EvaluationCost(runs=3)) + archive = create_harbor_session_archive(first, tmp_path / "rescue.tar.gz") + + # The relaunch: a session directory that has never existed, exactly what a + # new sandbox presents. Without the archive it is a brand new run. + clean = await build_harbor_components(_config(tmp_path / "attempt-two/session")) + assert clean.sidecar.status().evaluation_access[0].budget.remaining_runs == 4 + + resumed = await build_harbor_components( + _config(tmp_path / "attempt-three/session", seed=archive) + ) + assert resumed.sidecar.status().evaluation_access[0].budget.remaining_runs == 1 + third = tmp_path / "attempt-three/session" + assert (third / "candidates/repository.git").is_dir() + assert ( + json.loads((third / "harbor-session.json").read_text())["created_at"] + == json.loads((first / "harbor-session.json").read_text())["created_at"] + ) + # Restored before the lockdown, not after: the trusted state is still closed + # to the unprivileged harness on a resumed boot. + assert (third.stat().st_mode & 0o777) == 0o700 + + # A resume into a build that is no longer the one that produced the archive + # has to fail, and loudly. The scores in a restored database were measured + # against a particular backend and objective, and silently continuing under + # a different one would mix two incomparable runs into a single ranking. + # `initialize_harbor_session_manifest` is what catches it, on the restored + # manifest, which only exists here because the restore ran. + drifted = _config(tmp_path / "attempt-four/session", seed=archive) + drifted["selection"]["objective"] = ObjectiveSpec( + selector=MetricSelector(metric="score"), direction="minimize" + ).model_dump(mode="json") + with pytest.raises(ValueError, match="incompatible with deployment"): + await build_harbor_components(drifted) + + +def test_the_config_starter_documents_the_local_resume_opt_in(tmp_path): + """`vero init` never mentioned the one knob that makes a rerun resume. + + `[session] id` has always worked, and no template, doc string or build config + emitted it, so every config-driven run took the `uuid4` fallback in + `_session_identity` and reached none of the resume machinery. Emitting it + commented out keeps the default untouched and makes the opt-in findable. + """ + + from click.testing import CliRunner + + from vero.cli import _CONFIG_TEMPLATE, main + from vero.config import _session_identity, load_config + + result = CliRunner().invoke(main, ["init", str(tmp_path / "starter")]) + assert result.exit_code == 0, result.output + emitted = (tmp_path / "starter/vero.toml").read_text() + assert "# [session]" in emitted + assert '# id = "my-run-2026-08-01"' in emitted + assert emitted == _CONFIG_TEMPLATE + + # As emitted, two launches are two runs: the fallback mints a fresh uuid4. + (tmp_path / "starter/target").mkdir() + (tmp_path / "starter/harness").mkdir() + config_path = tmp_path / "starter/vero.toml" + first = _session_identity(load_config(config_path))[1] + assert _session_identity(load_config(config_path))[1] != first + + # Uncommented, they are one run over one directory, which is what every + # guard in the resume path needs in order to ever run. + config_path.write_text( + emitted.replace("# [session]", "[session]").replace( + '# id = "my-run-2026-08-01"', 'id = "my-run-2026-08-01"' + ), + encoding="utf-8", + ) + identity, directory = _session_identity(load_config(config_path)) + assert identity == "my-run-2026-08-01" + assert directory == _session_identity(load_config(config_path))[1] + assert directory.name == "my-run-2026-08-01" + + +def test_harbor_run_hands_the_resume_archive_to_the_compile(tmp_path, monkeypatch): + """The flag is only worth anything if it reaches the compiler. + + Asserted rather than assumed because the wiring is invisible at run time: the + launch prints the same command line either way, and a dropped argument shows + up as a silently fresh run hours later. + """ + + from vero.harbor import build as harbor_build + from vero.harbor import cli as harbor_cli + + config_path = tmp_path / "build.yaml" + config_path.write_text("name: org/task\n", encoding="utf-8") + archive = _session_archive(tmp_path / "previous") + + class _Config: + harbor_requirement = "harbor[modal]==0.20.0" + agent_env: dict[str, str] = {} + optimizer_harbor_args: list[str] = [] + extra_harbor_args: list[str] = [] + name = "vero/stub-benchmark" + + seen: dict[str, object] = {} + + def _compile(config, output, **keywords): + seen.update(keywords) + output.mkdir(parents=True) + return output + + monkeypatch.setattr( + harbor_build, "load_harbor_build_config", lambda *a, **k: _Config() + ) + monkeypatch.setattr(harbor_build, "compile_harbor_task", _compile) + monkeypatch.setattr(harbor_cli.shutil, "which", lambda name: "/usr/bin/uvx") + monkeypatch.setattr( + harbor_cli, "_compiled_run_environment", lambda task, overrides: {} + ) + monkeypatch.setattr( + harbor_cli.subprocess, + "run", + lambda command, env=None: subprocess.CompletedProcess(command, 0), + ) + + from click.testing import CliRunner + + from vero.cli import main + + result = CliRunner().invoke( + main, + [ + "harbor", + "run", + "--config", + str(config_path), + "--agent", + "codex", + "--resume-from", + str(archive), + ], + ) + assert result.exit_code == 0, result.output + assert seen["session_seed_archive"] == archive + assert str(archive) in result.output + + seen.clear() + result = CliRunner().invoke( + main, + ["harbor", "run", "--config", str(config_path), "--agent", "codex"], + ) + assert result.exit_code == 0, result.output + assert seen["session_seed_archive"] is None diff --git a/vero/tests/test_v05_cli.py b/vero/tests/test_v05_cli.py index 335a52a0..0ec43035 100644 --- a/vero/tests/test_v05_cli.py +++ b/vero/tests/test_v05_cli.py @@ -646,7 +646,9 @@ class _Config: name = "vero/stub-benchmark" monkeypatch.setattr(harbor_build, "load_harbor_build_config", lambda *a, **k: _Config()) - monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda config, output: output) + monkeypatch.setattr( + harbor_build, "compile_harbor_task", lambda config, output, **_: output + ) monkeypatch.setattr(harbor_cli.shutil, "which", lambda name: "/usr/bin/uvx") monkeypatch.setattr( harbor_cli, "_compiled_run_environment", lambda task, overrides: {} diff --git a/vero/tests/test_v05_harbor_build.py b/vero/tests/test_v05_harbor_build.py index 77e074bc..4736cef6 100644 --- a/vero/tests/test_v05_harbor_build.py +++ b/vero/tests/test_v05_harbor_build.py @@ -1270,7 +1270,7 @@ def fake_run(command, *args, **kwargs): monkeypatch.setattr( harbor_build, "load_harbor_build_config", lambda *a, **k: config ) - monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda cfg, out: out) + monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda cfg, out, **_: out) result = CliRunner().invoke( harbor_cli.harbor, diff --git a/vero/tests/test_v05_harbor_http.py b/vero/tests/test_v05_harbor_http.py index 6c9e14dc..510b95e7 100644 --- a/vero/tests/test_v05_harbor_http.py +++ b/vero/tests/test_v05_harbor_http.py @@ -270,7 +270,7 @@ def test_harbor_run_uses_current_python_and_pinned_harbor_extra(tmp_path, monkey ) observed = {} - def compile_task(_config, output): + def compile_task(_config, output, **_): output.mkdir(parents=True) return output @@ -330,7 +330,7 @@ def test_harbor_run_env_file_secrets_reach_subprocess_not_command_line( ) observed = {} - def compile_task(_config, output): + def compile_task(_config, output, **_): # The build's declared-credential check reads os.environ at compile time, # so the env-file must already be applied here (not just at subprocess launch). observed["modal_at_compile"] = os.environ.get("MODAL_TOKEN_ID")