diff --git a/vero/docs/harbor-architecture.md b/vero/docs/harbor-architecture.md index 0bb3ea69..04a70953 100644 --- a/vero/docs/harbor-architecture.md +++ b/vero/docs/harbor-architecture.md @@ -102,10 +102,35 @@ against a benchmark's pinned baseline is `harness-engineering-bench/scripts/rescore_candidate.py --session `, which lives out of tree because it needs that benchmark's `build.yaml`. -A true *resume* is not available and is not the goal here: the optimizer's -working tree, its harness process, and its agent context all live in the Modal -sandbox, which is torn down. What survives is every candidate the optimizer -committed and every score it measured. +The optimizer's working tree and its harness process are not recoverable: they +live in the Modal sandbox, which is torn down. What survives is every candidate +the optimizer committed and every score it measured. + +### Relaunching from what survived + +`vero harbor run --resume-from ` takes either archive and bakes it into +the new stack's sidecar image, and the sidecar restores it into the session +directory on first boot, before the candidate repository, the evaluation +database, the budget ledger or the agent's disclosure ledger are opened. Each of +those reuses what is on disk, so seeding the directory *is* the resume. + +Baking it in is not an aesthetic choice. The session directory is a compose +volume inside a per-trial sandbox, so nothing a relaunch could mount survives +the trial that wrote it, and the archive sitting on the launching host is the +only copy left. Two consequences follow. The archive is added to the sidecar +image, so a large `candidates/repository.git` is paid for in build time and +image size. And harbor's own `--max-retries` restart (`harbor/trial/queue.py`, +`_execute_trial_with_retries`) does **not** benefit: it `rmtree`s the trial +directory and calls `Trial.create` again against a fresh sandbox, within one +`vero harbor run` process that has long since compiled its task, so no archive +from the failed attempt exists yet, let alone one baked into an image. Resuming +is a second `vero harbor run`, by hand, after the first has exited. + +What comes back is what the archive holds: the candidate commits, the evaluation +records and their scores, and the spent agent budget. What does not is the +optimizer's own reasoning. It restarts with an empty context and re-reads its +prior evaluations from the restored `.evals` directory, so it is informed, not +mid-thought. ## The evaluation core diff --git a/vero/src/vero/cli.py b/vero/src/vero/cli.py index 792fd90f..6ab44ed5 100644 --- a/vero/src/vero/cli.py +++ b/vero/src/vero/cli.py @@ -103,6 +103,19 @@ def _default_home() -> Path: [optimizer] kind = "claude" instruction = "Improve the program without changing its intended behavior" + +# [session] +# Uncommented, `id` turns every later `vero run` over this file into a relaunch +# of one logical run: the session directory becomes $VERO_HOME/sessions/ +# instead of a fresh uuid4 per invocation, and the candidates, scores and budget +# already on disk are picked up rather than remade. Commented out because the +# resume is not free: a relaunch skips the baseline evaluation whenever a +# manifest exists, so if the first attempt died *during* its baseline it hands +# the rerun that attempt's unusable baseline record (no objective, no cases) and +# every later comparison is against it, until `vero session clear`. Opt in per +# run, with an id that names the run, and clear the session when the identity of +# what you are optimizing changes. +# id = "my-run-2026-08-01" ''' diff --git a/vero/src/vero/harbor/build/compiler.py b/vero/src/vero/harbor/build/compiler.py index 58df9f38..62475fd0 100644 --- a/vero/src/vero/harbor/build/compiler.py +++ b/vero/src/vero/harbor/build/compiler.py @@ -25,6 +25,7 @@ from vero.harbor.build.config import HarborBuildConfig from vero.harbor.build.specs import WorkspaceOverlaySpec from vero.layout import LAYOUT +from vero.sidecar.session import read_harbor_session_archive_manifest logger = logging.getLogger(__name__) @@ -66,6 +67,7 @@ AGENT_VOLUME = LAYOUT.agent_volume ADMIN_VOLUME = LAYOUT.admin_volume SESSION_DIR = LAYOUT.session_dir +SESSION_SEED_ARCHIVE = LAYOUT.session_seed_archive TOKEN_PATH = LAYOUT.token_path INFERENCE_STATE = LAYOUT.inference_state INFERENCE_REQUEST_LOG_DIR = LAYOUT.inference_request_log_dir @@ -277,6 +279,7 @@ def _deployment_config( local_task_source: bool, evaluation_inference_token: str | None, finalization_inference_token: str | None, + session_seed_archive: bool = False, ) -> dict: task_source = TASK_SOURCE_DIR if local_task_source else config.task_source backends = {} @@ -459,6 +462,9 @@ def _deployment_config( "agent_repo_path": AGENT_REPO, "session_dir": SESSION_DIR, "session_id": SESSION_ID, + "session_seed_archive": ( + SESSION_SEED_ARCHIVE if session_seed_archive else None + ), "backends": backends, "access_policies": policies, "budgets": budgets, @@ -588,9 +594,29 @@ def compile_harbor_task( output_dir: Path | str, *, vero_root: Path | None = None, + session_seed_archive: Path | str | None = None, ) -> Path: - """Emit a self-contained Harbor task directory from validated config.""" + """Emit a self-contained Harbor task directory from validated config. + + ``session_seed_archive`` is a previously exported session (the + ``session-rescue.tar.gz`` a dead trial leaves in its artifacts, or the + ``verifier/session.tar.gz`` a finished one leaves) to restore into the new + stack's session directory on first boot. Baking it into the sidecar image is + the only transport available: the session directory lives on a compose volume + inside a per-trial Modal sandbox, so nothing a relaunch could mount survives, + and the archive on the launching host is the only copy that does. + """ output = Path(output_dir).expanduser().resolve() + seed_archive = ( + Path(session_seed_archive).expanduser().resolve() + if session_seed_archive is not None + else None + ) + if seed_archive is not None: + # Fail on the host, now, rather than after an image build and a stack + # bring-up inside a sandbox whose logs nobody is tailing. Reads the + # manifest only; the bytes are not unpacked here. + read_harbor_session_archive_manifest(seed_archive) source_root = (vero_root or Path(__file__).parents[4]).resolve() use_local_vero = _is_vero_source(source_root) if vero_root is not None and not use_local_vero: @@ -715,6 +741,13 @@ def compile_harbor_task( Path(config.command_backend.harness_source), sidecar_dir / "harness", ) + if seed_archive is not None: + # Into the sidecar's build context, beside the case lists it is as + # sensitive as: the archive carries database.json, whose per-case records + # name held-out tasks and their scores. The Dockerfile chmods it 600 on + # the way in, matching serve.json. + sidecar_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(seed_archive, sidecar_dir / "session-seed.tar.gz") local_task_source = ( config.task_source is not None and Path(config.task_source).exists() ) @@ -754,6 +787,7 @@ def compile_harbor_task( local_task_source=local_task_source, evaluation_inference_token=evaluation_inference_token, finalization_inference_token=finalization_inference_token, + session_seed_archive=seed_archive is not None, ) (sidecar_dir / "serve.json").write_text( json.dumps(deployment, ensure_ascii=False, indent=2) + "\n", @@ -871,6 +905,9 @@ def compile_harbor_task( "sidecar_factory": FACTORY_PATH, "producer_base_url": PRODUCER_BASE_URL, "command_harness": config.command_backend is not None, + "session_seed_archive": ( + SESSION_SEED_ARCHIVE if seed_archive is not None else None + ), # The Harbor backend hard-rejects request.seed, so only advertise the # flag when the build evaluates through a command backend. "seed_supported": config.command_backend is not None, diff --git a/vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2 b/vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2 index 0b30e2f5..3f3f89ba 100644 --- a/vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2 +++ b/vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2 @@ -16,6 +16,9 @@ RUN uv pip install --system "{{ harbor_requirement }}" COPY agent-baseline {{ layout.trusted_repo }} COPY sidecar/cases {{ layout.cases }} COPY sidecar/serve.json {{ layout.serve_config }} +{% if session_seed_archive %} +COPY sidecar/session-seed.tar.gz {{ session_seed_archive }} +{% endif %} {% if local_task_source %} COPY sidecar/task-source {{ layout.task_source }} {% endif %} @@ -49,5 +52,11 @@ RUN mkdir -p /home/harness/.cache \ # {{ layout.task_source }}, which harbor must read to grade host-side, remain readable # and are addressed by the deeper in-container isolation work.) RUN chmod 700 {{ layout.cases }} && chmod 600 {{ layout.serve_config }} +{% if session_seed_archive %} +{# The restored session's database.json carries held-out case membership and + per-case scores, the same secret the case lists above hold, so the seed + archive is locked to root on the same grounds. #} +RUN chmod 600 {{ session_seed_archive }} +{% endif %} WORKDIR /opt diff --git a/vero/src/vero/harbor/cli.py b/vero/src/vero/harbor/cli.py index 9be14ae5..2b40ec45 100644 --- a/vero/src/vero/harbor/cli.py +++ b/vero/src/vero/harbor/cli.py @@ -775,6 +775,18 @@ def _preflight_models(config) -> None: @click.option("--agent", required=True, help="Harbor optimizer agent.") @click.option("--model", help="Model used by the optimizer agent.") @click.option("--environment", default="modal", show_default=True) +@click.option( + "--resume-from", + "resume_from", + type=click.Path(path_type=Path, exists=True, dir_okay=False), + help=( + "Restore a previous run's exported session into this one, so the " + "relaunch keeps its candidates, its scores and its spent budget instead " + "of starting over. Takes the artifacts/session-rescue.tar.gz a dead " + "trial leaves behind, or the verifier/session.tar.gz a finished one " + "does. Opt-in: without it a relaunch is a fresh run." + ), +) @click.option( "--env-file", "env_file", @@ -787,7 +799,9 @@ def _preflight_models(config) -> None: ) @_PARAM_OPTION @click.argument("extra", nargs=-1, type=click.UNPROCESSED) -def run_command(config_path, agent, model, environment, params, env_file, extra): +def run_command( + config_path, agent, model, environment, resume_from, params, env_file, extra +): """Compile to a temporary directory and invoke `harbor run`.""" from vero.harbor.build import compile_harbor_task, load_harbor_build_config @@ -813,7 +827,10 @@ def run_command(config_path, agent, model, environment, params, env_file, extra) task = compile_harbor_task( config, Path(temporary) / "task", + session_seed_archive=resume_from, ) + if resume_from is not None: + click.echo(f"Resuming from session archive {resume_from}") command = [ uvx, "--python", diff --git a/vero/src/vero/harbor/deployment.py b/vero/src/vero/harbor/deployment.py index 0bf66710..0eea3820 100644 --- a/vero/src/vero/harbor/deployment.py +++ b/vero/src/vero/harbor/deployment.py @@ -28,7 +28,10 @@ from vero.runtime.artifacts import ArtifactStore from vero.sandbox import LocalSandbox from vero.sidecar.serve import SidecarComponents -from vero.sidecar.session import initialize_harbor_session_manifest +from vero.sidecar.session import ( + initialize_harbor_session_manifest, + restore_harbor_session_archive, +) from vero.sidecar.sidecar import EvaluationSidecar, SidecarEvaluationPolicy from vero.sidecar.transport import GitCandidateTransport from vero.sidecar.verifier import ( @@ -123,6 +126,10 @@ class HarborDeploymentConfig(StrictModel): agent_repo_path: str session_dir: str session_id: str = "trial" + # A previous run's exported session, baked into the sidecar image, restored + # into session_dir on first boot. Absent by default: a relaunch starts clean + # unless the operator named an archive on the command line. + session_seed_archive: str | None = None backends: dict[str, DeploymentBackendConfig] access_policies: list[SidecarEvaluationPolicy] budgets: list[EvaluationBudget] = Field(default_factory=list) @@ -176,7 +183,11 @@ def validate_absolute_path(cls, value: str) -> str: raise ValueError("deployment paths must be absolute") return value - @field_validator("inference_usage_path", "inference_request_log_dir") + @field_validator( + "inference_usage_path", + "inference_request_log_dir", + "session_seed_archive", + ) @classmethod def validate_optional_file_path(cls, value: str | None) -> str | None: if value is not None: @@ -271,6 +282,15 @@ async def build_harbor_components(config: dict) -> SidecarComponents: parsed = HarborDeploymentConfig.model_validate(config) session_dir = Path(parsed.session_dir) session_dir.mkdir(parents=True, exist_ok=True) + # Before anything reads the session dir. Every consumer below it -- the + # candidate repository, the evaluation database, the budget ledger, the + # agent's disclosure ledger -- opens what is on disk and reuses it if it is + # there, so seeding here is the whole of the restore: no consumer needs to + # know a resume happened. Missing archive is fatal rather than ignored: the + # operator asked for a resume, and silently starting a fresh multi-hour run + # instead is the failure this exists to prevent. + if parsed.session_seed_archive is not None: + restore_harbor_session_archive(parsed.session_seed_archive, session_dir) # The session dir holds the trusted state — held-out evaluation records and # scores, the budget ledger, and every candidate's code. Lock it to the # owning (trusted) user so the unprivileged harness that executes candidate diff --git a/vero/src/vero/layout.py b/vero/src/vero/layout.py index 741df26c..23da2542 100644 --- a/vero/src/vero/layout.py +++ b/vero/src/vero/layout.py @@ -38,6 +38,10 @@ class TaskLayout: harness: The scoring program, for a command backend. overlay: Host files baked into the optimizer's workspace. serve_config: The trusted deployment config, root-only. + session_seed_archive: A previously exported session, baked in so a + relaunch can restore it. Root-only for the same reason as + serve_config and the case lists: it carries database.json, whose + per-case records disclose held-out membership and scores. seed_script: Script that seeds the target repo on first boot. inference_config: The gateway's scope config. agent_volume: The optimizer's context directory, written by the sidecar. @@ -63,6 +67,7 @@ class TaskLayout: harness: str = "/opt/harness" overlay: str = "/opt/overlay" serve_config: str = "/opt/serve.json" + session_seed_archive: str = "/opt/session-seed.tar.gz" seed_script: str = "/opt/seed.sh" inference_config: str = "/opt/inference.json" agent_volume: str = "/state/agent-context" diff --git a/vero/src/vero/sidecar/session.py b/vero/src/vero/sidecar/session.py index 1c69151d..f6928b52 100644 --- a/vero/src/vero/sidecar/session.py +++ b/vero/src/vero/sidecar/session.py @@ -7,6 +7,7 @@ import json import logging import os +import shutil import tarfile import tempfile from collections import deque @@ -227,6 +228,32 @@ def _text_member(archive: tarfile.TarFile, arcname: str, payload: bytes) -> None return output +def _validated_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]: + """Every member of a session archive, or a refusal naming the first bad one. + + Factored out so the read-only inspection below applies exactly the rules the + extractor applies. Duplicating them would let the two drift, and the drift + that matters is the one where a launch-time check passes an archive the + sidecar then refuses an hour into the run, inside a sandbox nobody is + watching. + """ + + members = archive.getmembers() + for member in members: + path = PurePosixPath(member.name) + if ( + path.is_absolute() + or not path.parts + or path.parts[0] != "session" + or ".." in path.parts + or member.issym() + or member.islnk() + or member.isdev() + ): + raise ValueError(f"unsafe Harbor session archive member: {member.name}") + return members + + def extract_harbor_session_archive( archive_path: Path | str, destination: Path | str, @@ -237,19 +264,7 @@ def extract_harbor_session_archive( destination = Path(destination).expanduser().resolve() destination.mkdir(parents=True, exist_ok=True) with tarfile.open(archive_path, "r:gz") as archive: - members = archive.getmembers() - for member in members: - path = PurePosixPath(member.name) - if ( - path.is_absolute() - or not path.parts - or path.parts[0] != "session" - or ".." in path.parts - or member.issym() - or member.islnk() - or member.isdev() - ): - raise ValueError(f"unsafe Harbor session archive member: {member.name}") + members = _validated_members(archive) archive.extractall(destination, members=members, filter="data") session = destination / "session" HarborSessionManifest.model_validate_json( @@ -258,6 +273,71 @@ def extract_harbor_session_archive( return session +def read_harbor_session_archive_manifest( + archive_path: Path | str, +) -> HarborSessionManifest: + """Read a session archive's identity without unpacking it. + + Exists so `vero harbor run --resume-from` can reject a wrong or corrupt + archive on the host, in the second before the compile, rather than baking it + into a sidecar image and discovering it inside a Modal sandbox after the + image build, the stack bring-up and the seed have already been paid for. + """ + + with tarfile.open(Path(archive_path).expanduser().resolve(), "r:gz") as archive: + _validated_members(archive) + try: + manifest = archive.extractfile("session/harbor-session.json") + except KeyError: + manifest = None + if manifest is None: + raise ValueError("Harbor session archive has no session manifest") + return HarborSessionManifest.model_validate_json(manifest.read()) + + +def restore_harbor_session_archive( + archive_path: Path | str, + session_dir: Path | str, +) -> bool: + """Seed an empty session directory from a previous run's export. + + Returns whether anything was restored. A session that already carries its + manifest is left untouched, because the two callers are indistinguishable + from here: a *relaunch* boots against a fresh volume and has to be seeded, + while a sidecar *restart* inside a live run boots against the volume it has + been writing all along, and overwriting that with the state of an older run + would undo everything since. Refusing on the manifest and not on emptiness + is deliberate: a first boot that died between mkdir and the manifest write + leaves a non-empty directory that still holds nothing worth keeping. + + Staged into a sibling directory and moved in entry by entry, so a restore + killed half way leaves the session dir either untouched or with whole files, + never with a truncated database.json that reads as a complete one. + """ + + session_dir = Path(session_dir).expanduser().resolve() + if (session_dir / "harbor-session.json").is_file(): + return False + session_dir.mkdir(parents=True, exist_ok=True) + staging = Path( + tempfile.mkdtemp(dir=session_dir.parent, prefix=f".{session_dir.name}-restore-") + ) + try: + source = extract_harbor_session_archive(archive_path, staging) + for entry in sorted(source.iterdir()): + target = session_dir / entry.name + if target.is_dir() and not target.is_symlink(): + shutil.rmtree(target) + elif target.exists() or target.is_symlink(): + target.unlink() + shutil.move(str(entry), str(target)) + finally: + shutil.rmtree(staging, ignore_errors=True) + _fsync_path(session_dir) + logger.info("Restored Harbor session into %s from %s", session_dir, archive_path) + return True + + def file_sha256(path: Path | str) -> str: digest = hashlib.sha256() with Path(path).open("rb") as file: diff --git a/vero/tests/test_resume_across_relaunch.py b/vero/tests/test_resume_across_relaunch.py new file mode 100644 index 00000000..2a9fefd1 --- /dev/null +++ b/vero/tests/test_resume_across_relaunch.py @@ -0,0 +1,465 @@ +"""A second launch of the same run has to be able to pick up the first one's work. + +PR #78 hardened a resume path, and every one of those guards sits inside a +session directory that a relaunch never reaches: `vero harbor run` compiles into +a throwaway directory, the session lives on the `admin_state` compose volume +inside a per-trial Modal sandbox, and the sandbox goes away with the trial. The +one copy that outlives it is the archive on the launching host, so the tests +here are about that archive: refusing a bad one before an image is built, +carrying a good one into the next stack's sidecar without widening who can read +it, restoring it before anything opens the session directory, and not restoring +it over a session that is currently being written. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tarfile +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from vero.evaluation import ( + DisclosureLevel, + EvaluationAccessPolicy, + EvaluationBudget, + EvaluationCost, + EvaluationSet, + MetricSelector, + ObjectiveSpec, +) +from vero.harbor import ( + AgentAccessSpec, + HarborBackendConfig, + HarborBuildConfig, + VerificationTargetSpec, + build_harbor_components, + compile_harbor_task, +) +from vero.layout import LAYOUT +from vero.sidecar import SidecarEvaluationPolicy, VerificationTarget +from vero.sidecar.session import ( + HarborSessionManifest, + create_harbor_session_archive, + read_harbor_session_archive_manifest, + restore_harbor_session_archive, +) +from vero.sidecar.verifier import VerificationSelection + +_VERO_ROOT = Path(__file__).parents[1] + + +def _git(path: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", *arguments], + cwd=path, + check=True, + text=True, + capture_output=True, + ) + return result.stdout.strip() + + +def _repo(path: Path) -> str: + path.mkdir(parents=True) + _git(path, "init", "-q") + _git(path, "config", "user.name", "VeRO Test") + _git(path, "config", "user.email", "vero@example.test") + (path / "program.py").write_text("VALUE = 1\n", encoding="utf-8") + (path / "pyproject.toml").write_text( + '[project]\nname="target"\nversion="0.1.0"\n', encoding="utf-8" + ) + _git(path, "add", ".") + _git(path, "commit", "-q", "-m", "baseline") + return _git(path, "rev-parse", "HEAD") + + +def _session_archive(root: Path, *, payload: str = "restored") -> Path: + """The smallest thing that satisfies the archive contract, plus a marker. + + A real archive is whatever a run left behind; what every consumer here keys + off is the manifest at the root, so this stands in for one wherever the test + is about transport rather than about the state itself. + """ + + session = root / "session" + session.mkdir(parents=True) + manifest = HarborSessionManifest( + id="trial", + task_name="org/task", + created_at=datetime.now(UTC), + backends={}, + selection=VerificationSelection(mode="submit"), + targets=[], + ) + (session / "harbor-session.json").write_text( + manifest.model_dump_json(), encoding="utf-8" + ) + (session / "marker.txt").write_text(payload, encoding="utf-8") + return create_harbor_session_archive(session, root / "session.tar.gz") + + +def _build_config(root: Path) -> HarborBuildConfig: + target = root / "target" + _repo(target) + tasks = root / "tasks" + for name in ("task-a", "task-hidden"): + (tasks / name).mkdir(parents=True) + (tasks / name / "task.toml").write_text( + f'[task]\nname="org/{name}"\n', encoding="utf-8" + ) + return HarborBuildConfig( + name="org/optimize-program", + description="Improve the program", + agent_repo=str(target), + task_source=str(tasks), + agent_import_path="target.agent:Agent", + harbor_requirement="harbor==0.1.17", + partitions={"validation": ["task-a"], "test": ["task-hidden"]}, + agent_access=[AgentAccessSpec(partition="validation", total_runs=5)], + selection_partition="validation", + targets=[VerificationTargetSpec(partition="test")], + ) + + +def test_a_compile_carries_the_session_archive_only_when_a_launch_asked_for_one( + tmp_path, +): + """The archive has to travel in the sidecar image, and only on request. + + Baking it in is the only transport there is: the session directory is a + compose volume inside the sandbox, so a relaunch has nothing to mount. That + also means the default must stay clean, because every launch that is not a + resume would otherwise pay the image size and, worse, adopt a previous run's + scores without anyone having asked for it. + """ + + config = _build_config(tmp_path / "build") + archive = _session_archive(tmp_path / "previous") + + plain = compile_harbor_task( + config, tmp_path / "plain", vero_root=_VERO_ROOT + ) + sidecar = plain / "environment/sidecar" + assert not (sidecar / "session-seed.tar.gz").exists() + assert json.loads((sidecar / "serve.json").read_text())[ + "session_seed_archive" + ] is None + assert "session-seed.tar.gz" not in (sidecar / "Dockerfile").read_text() + + resumed = compile_harbor_task( + config, + tmp_path / "resumed", + vero_root=_VERO_ROOT, + session_seed_archive=archive, + ) + sidecar = resumed / "environment/sidecar" + assert (sidecar / "session-seed.tar.gz").read_bytes() == archive.read_bytes() + assert ( + json.loads((sidecar / "serve.json").read_text())["session_seed_archive"] + == LAYOUT.session_seed_archive + ) + dockerfile = (sidecar / "Dockerfile").read_text() + copy = f"COPY sidecar/session-seed.tar.gz {LAYOUT.session_seed_archive}" + assert copy in dockerfile + # The archive carries database.json, so it discloses held-out membership and + # per-case scores exactly as the case lists do. It gets the same lock, or the + # unprivileged harness that runs candidate code can read the answers. + assert f"chmod 600 {LAYOUT.session_seed_archive}" in dockerfile + + +def test_a_launch_refuses_an_archive_it_could_not_have_restored(tmp_path): + """Reject on the host, before an image build, not inside the sandbox. + + The failure this prevents is silent and expensive: a wrong path compiles + fine, builds three images, brings the stack up, and only then does the + sidecar refuse, tens of minutes into a run whose logs nobody is tailing. + """ + + config = _build_config(tmp_path / "build") + plain = tmp_path / "not-an-archive.tar.gz" + with tarfile.open(plain, "w:gz") as archive: + source = tmp_path / "loose.txt" + source.write_text("no manifest here\n", encoding="utf-8") + archive.add(source, arcname="session/loose.txt") + + with pytest.raises(ValueError, match="no session manifest"): + compile_harbor_task( + config, + tmp_path / "out", + vero_root=_VERO_ROOT, + session_seed_archive=plain, + ) + assert not (tmp_path / "out").exists() + + escaping = tmp_path / "escaping.tar.gz" + with tarfile.open(escaping, "w:gz") as archive: + archive.add(tmp_path / "loose.txt", arcname="../escape.txt") + with pytest.raises(ValueError, match="unsafe Harbor session archive member"): + read_harbor_session_archive_manifest(escaping) + + +def test_a_restore_seeds_an_empty_session_and_spares_a_live_one(tmp_path): + """The two boots that reach this code are indistinguishable from inside it. + + A relaunch boots against a fresh volume and must be seeded. A sidecar restart + inside a live run boots against the volume it has been writing all along, and + seeding that would roll the run back to an older attempt's state. The + manifest is what tells them apart. + """ + + archive = _session_archive(tmp_path / "previous") + + fresh = tmp_path / "state/session" + assert restore_harbor_session_archive(archive, fresh) is True + assert (fresh / "marker.txt").read_text() == "restored" + assert json.loads((fresh / "harbor-session.json").read_text())["id"] == "trial" + # Staged into a sibling and moved in, so nothing is left behind to be picked + # up as session state on a later boot. + assert sorted(path.name for path in (tmp_path / "state").iterdir()) == ["session"] + + live = tmp_path / "live/session" + live.mkdir(parents=True) + (live / "harbor-session.json").write_text( + (fresh / "harbor-session.json").read_text(), encoding="utf-8" + ) + (live / "marker.txt").write_text("work since the restart", encoding="utf-8") + assert restore_harbor_session_archive(archive, live) is False + assert (live / "marker.txt").read_text() == "work since the restart" + + +@pytest.mark.asyncio +async def test_a_relaunch_resumes_the_previous_run_s_ledger_and_candidates(tmp_path): + """The end of the line: a fresh volume that behaves like the old one. + + Everything the sidecar builds below the session directory reopens what is on + disk, so restoring the directory before any of them is constructed is the + whole of the resume. The budget ledger is the proof that carries the least + ceremony and the most meaning: a relaunch that did not resume hands the + optimizer its agent budget back, and a run that has already spent hours can + then spend it a second time. + """ + + trusted = tmp_path / "trusted" + agent = tmp_path / "agent" + _repo(trusted) + _repo(agent) + cases = tmp_path / "cases.jsonl" + cases.write_text( + json.dumps({"id": "task", "task_name": "org/task"}) + "\n", encoding="utf-8" + ) + evaluation_set = EvaluationSet(name="benchmark", partition="validation") + objective = ObjectiveSpec( + selector=MetricSelector(metric="score"), direction="maximize" + ) + backend = HarborBackendConfig( + task_source="org/benchmark@1.0", + agent_import_path="program:Agent", + cases_path=str(cases), + harbor_requirement="harbor==0.1.17", + evaluation_set_name="benchmark", + partition="validation", + uv_executable=sys.executable, + ) + + def _config(session_dir: Path, seed: Path | None = None) -> dict: + return { + "repo_path": str(trusted), + "agent_repo_path": str(agent), + "session_dir": str(session_dir), + "session_id": "trial", + **({"session_seed_archive": str(seed)} if seed is not None else {}), + "backends": {"validation": backend.model_dump(mode="json")}, + "access_policies": [ + SidecarEvaluationPolicy( + backend_id="validation", + evaluation_set_name="benchmark", + partition="validation", + objective=objective, + access=EvaluationAccessPolicy( + disclosure=DisclosureLevel.AGGREGATE + ), + ).model_dump(mode="json") + ], + "budgets": [ + EvaluationBudget( + backend_id="validation", + evaluation_set_key=evaluation_set.budget_key("validation"), + total_runs=4, + ).model_dump(mode="json") + ], + "selection": { + "mode": "auto_best", + "backend_id": "validation", + "evaluation_set": evaluation_set.model_dump(mode="json"), + "objective": objective.model_dump(mode="json"), + "baseline_version": "HEAD", + }, + "targets": [ + VerificationTarget( + reward_key="reward", + backend_id="validation", + evaluation_set=evaluation_set, + objective=objective, + ).model_dump(mode="json") + ], + "agent_volume": str(session_dir.parent / "agent"), + "admin_volume": str(session_dir.parent), + } + + first = tmp_path / "attempt-one/session" + components = await build_harbor_components(_config(first)) + ledger = components.sidecar.engine.budget_ledger + assert ledger is not None + await ledger.reserve("validation", evaluation_set, EvaluationCost(runs=3)) + archive = create_harbor_session_archive(first, tmp_path / "rescue.tar.gz") + + # The relaunch: a session directory that has never existed, exactly what a + # new sandbox presents. Without the archive it is a brand new run. + clean = await build_harbor_components(_config(tmp_path / "attempt-two/session")) + assert clean.sidecar.status().evaluation_access[0].budget.remaining_runs == 4 + + resumed = await build_harbor_components( + _config(tmp_path / "attempt-three/session", seed=archive) + ) + assert resumed.sidecar.status().evaluation_access[0].budget.remaining_runs == 1 + third = tmp_path / "attempt-three/session" + assert (third / "candidates/repository.git").is_dir() + assert ( + json.loads((third / "harbor-session.json").read_text())["created_at"] + == json.loads((first / "harbor-session.json").read_text())["created_at"] + ) + # Restored before the lockdown, not after: the trusted state is still closed + # to the unprivileged harness on a resumed boot. + assert (third.stat().st_mode & 0o777) == 0o700 + + # A resume into a build that is no longer the one that produced the archive + # has to fail, and loudly. The scores in a restored database were measured + # against a particular backend and objective, and silently continuing under + # a different one would mix two incomparable runs into a single ranking. + # `initialize_harbor_session_manifest` is what catches it, on the restored + # manifest, which only exists here because the restore ran. + drifted = _config(tmp_path / "attempt-four/session", seed=archive) + drifted["selection"]["objective"] = ObjectiveSpec( + selector=MetricSelector(metric="score"), direction="minimize" + ).model_dump(mode="json") + with pytest.raises(ValueError, match="incompatible with deployment"): + await build_harbor_components(drifted) + + +def test_the_config_starter_documents_the_local_resume_opt_in(tmp_path): + """`vero init` never mentioned the one knob that makes a rerun resume. + + `[session] id` has always worked, and no template, doc string or build config + emitted it, so every config-driven run took the `uuid4` fallback in + `_session_identity` and reached none of the resume machinery. Emitting it + commented out keeps the default untouched and makes the opt-in findable. + """ + + from click.testing import CliRunner + + from vero.cli import _CONFIG_TEMPLATE, main + from vero.config import _session_identity, load_config + + result = CliRunner().invoke(main, ["init", str(tmp_path / "starter")]) + assert result.exit_code == 0, result.output + emitted = (tmp_path / "starter/vero.toml").read_text() + assert "# [session]" in emitted + assert '# id = "my-run-2026-08-01"' in emitted + assert emitted == _CONFIG_TEMPLATE + + # As emitted, two launches are two runs: the fallback mints a fresh uuid4. + (tmp_path / "starter/target").mkdir() + (tmp_path / "starter/harness").mkdir() + config_path = tmp_path / "starter/vero.toml" + first = _session_identity(load_config(config_path))[1] + assert _session_identity(load_config(config_path))[1] != first + + # Uncommented, they are one run over one directory, which is what every + # guard in the resume path needs in order to ever run. + config_path.write_text( + emitted.replace("# [session]", "[session]").replace( + '# id = "my-run-2026-08-01"', 'id = "my-run-2026-08-01"' + ), + encoding="utf-8", + ) + identity, directory = _session_identity(load_config(config_path)) + assert identity == "my-run-2026-08-01" + assert directory == _session_identity(load_config(config_path))[1] + assert directory.name == "my-run-2026-08-01" + + +def test_harbor_run_hands_the_resume_archive_to_the_compile(tmp_path, monkeypatch): + """The flag is only worth anything if it reaches the compiler. + + Asserted rather than assumed because the wiring is invisible at run time: the + launch prints the same command line either way, and a dropped argument shows + up as a silently fresh run hours later. + """ + + from vero.harbor import build as harbor_build + from vero.harbor import cli as harbor_cli + + config_path = tmp_path / "build.yaml" + config_path.write_text("name: org/task\n", encoding="utf-8") + archive = _session_archive(tmp_path / "previous") + + class _Config: + harbor_requirement = "harbor[modal]==0.20.0" + agent_env: dict[str, str] = {} + optimizer_harbor_args: list[str] = [] + extra_harbor_args: list[str] = [] + name = "vero/stub-benchmark" + + seen: dict[str, object] = {} + + def _compile(config, output, **keywords): + seen.update(keywords) + output.mkdir(parents=True) + return output + + monkeypatch.setattr( + harbor_build, "load_harbor_build_config", lambda *a, **k: _Config() + ) + monkeypatch.setattr(harbor_build, "compile_harbor_task", _compile) + monkeypatch.setattr(harbor_cli.shutil, "which", lambda name: "/usr/bin/uvx") + monkeypatch.setattr( + harbor_cli, "_compiled_run_environment", lambda task, overrides: {} + ) + monkeypatch.setattr( + harbor_cli.subprocess, + "run", + lambda command, env=None: subprocess.CompletedProcess(command, 0), + ) + + from click.testing import CliRunner + + from vero.cli import main + + result = CliRunner().invoke( + main, + [ + "harbor", + "run", + "--config", + str(config_path), + "--agent", + "codex", + "--resume-from", + str(archive), + ], + ) + assert result.exit_code == 0, result.output + assert seen["session_seed_archive"] == archive + assert str(archive) in result.output + + seen.clear() + result = CliRunner().invoke( + main, + ["harbor", "run", "--config", str(config_path), "--agent", "codex"], + ) + assert result.exit_code == 0, result.output + assert seen["session_seed_archive"] is None diff --git a/vero/tests/test_v05_cli.py b/vero/tests/test_v05_cli.py index 335a52a0..0ec43035 100644 --- a/vero/tests/test_v05_cli.py +++ b/vero/tests/test_v05_cli.py @@ -646,7 +646,9 @@ class _Config: name = "vero/stub-benchmark" monkeypatch.setattr(harbor_build, "load_harbor_build_config", lambda *a, **k: _Config()) - monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda config, output: output) + monkeypatch.setattr( + harbor_build, "compile_harbor_task", lambda config, output, **_: output + ) monkeypatch.setattr(harbor_cli.shutil, "which", lambda name: "/usr/bin/uvx") monkeypatch.setattr( harbor_cli, "_compiled_run_environment", lambda task, overrides: {} diff --git a/vero/tests/test_v05_harbor_build.py b/vero/tests/test_v05_harbor_build.py index 77e074bc..4736cef6 100644 --- a/vero/tests/test_v05_harbor_build.py +++ b/vero/tests/test_v05_harbor_build.py @@ -1270,7 +1270,7 @@ def fake_run(command, *args, **kwargs): monkeypatch.setattr( harbor_build, "load_harbor_build_config", lambda *a, **k: config ) - monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda cfg, out: out) + monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda cfg, out, **_: out) result = CliRunner().invoke( harbor_cli.harbor, diff --git a/vero/tests/test_v05_harbor_http.py b/vero/tests/test_v05_harbor_http.py index 6c9e14dc..510b95e7 100644 --- a/vero/tests/test_v05_harbor_http.py +++ b/vero/tests/test_v05_harbor_http.py @@ -270,7 +270,7 @@ def test_harbor_run_uses_current_python_and_pinned_harbor_extra(tmp_path, monkey ) observed = {} - def compile_task(_config, output): + def compile_task(_config, output, **_): output.mkdir(parents=True) return output @@ -330,7 +330,7 @@ def test_harbor_run_env_file_secrets_reach_subprocess_not_command_line( ) observed = {} - def compile_task(_config, output): + def compile_task(_config, output, **_): # The build's declared-credential check reads os.environ at compile time, # so the env-file must already be applied here (not just at subprocess launch). observed["modal_at_compile"] = os.environ.get("MODAL_TOKEN_ID")