feat: let a relaunch resume a Harbor run from the session it left behind

vero/docs/harbor-architecture.md

@@ -102,10 +102,35 @@ against a benchmark's pinned baseline is
 `harness-engineering-bench/scripts/rescore_candidate.py --session <archive>`,
 which lives out of tree because it needs that benchmark's `build.yaml`.
 
-A true *resume* is not available and is not the goal here: the optimizer's
-working tree, its harness process, and its agent context all live in the Modal
-sandbox, which is torn down. What survives is every candidate the optimizer
-committed and every score it measured.
+The optimizer's working tree and its harness process are not recoverable: they
+live in the Modal sandbox, which is torn down. What survives is every candidate
+the optimizer committed and every score it measured.
+
+### Relaunching from what survived
+
+`vero harbor run --resume-from <archive>` takes either archive and bakes it into
+the new stack's sidecar image, and the sidecar restores it into the session
+directory on first boot, before the candidate repository, the evaluation
+database, the budget ledger or the agent's disclosure ledger are opened. Each of
+those reuses what is on disk, so seeding the directory *is* the resume.
+
+Baking it in is not an aesthetic choice. The session directory is a compose
+volume inside a per-trial sandbox, so nothing a relaunch could mount survives
+the trial that wrote it, and the archive sitting on the launching host is the
+only copy left. Two consequences follow. The archive is added to the sidecar
+image, so a large `candidates/repository.git` is paid for in build time and
+image size. And harbor's own `--max-retries` restart (`harbor/trial/queue.py`,
+`_execute_trial_with_retries`) does **not** benefit: it `rmtree`s the trial
+directory and calls `Trial.create` again against a fresh sandbox, within one
+`vero harbor run` process that has long since compiled its task, so no archive
+from the failed attempt exists yet, let alone one baked into an image. Resuming
+is a second `vero harbor run`, by hand, after the first has exited.
+
+What comes back is what the archive holds: the candidate commits, the evaluation
+records and their scores, and the spent agent budget. What does not is the
+optimizer's own reasoning. It restarts with an empty context and re-reads its
+prior evaluations from the restored `.evals` directory, so it is informed, not
+mid-thought.
 
 ## The evaluation core
 

vero/src/vero/cli.py

@@ -103,6 +103,19 @@ def _default_home() -> Path:
 [optimizer]
 kind = "claude"
 instruction = "Improve the program without changing its intended behavior"
+
+# [session]
+# Uncommented, `id` turns every later `vero run` over this file into a relaunch
+# of one logical run: the session directory becomes $VERO_HOME/sessions/<id>
+# instead of a fresh uuid4 per invocation, and the candidates, scores and budget
+# already on disk are picked up rather than remade. Commented out because the
+# resume is not free: a relaunch skips the baseline evaluation whenever a
+# manifest exists, so if the first attempt died *during* its baseline it hands
+# the rerun that attempt's unusable baseline record (no objective, no cases) and
+# every later comparison is against it, until `vero session clear`. Opt in per
+# run, with an id that names the run, and clear the session when the identity of
+# what you are optimizing changes.
+# id = "my-run-2026-08-01"
 '''
 
 

vero/src/vero/harbor/build/compiler.py

@@ -25,6 +25,7 @@
 from vero.harbor.build.config import HarborBuildConfig
 from vero.harbor.build.specs import WorkspaceOverlaySpec
 from vero.layout import LAYOUT
+from vero.sidecar.session import read_harbor_session_archive_manifest
 
 logger = logging.getLogger(__name__)
 
@@ -66,6 +67,7 @@
 AGENT_VOLUME = LAYOUT.agent_volume
 ADMIN_VOLUME = LAYOUT.admin_volume
 SESSION_DIR = LAYOUT.session_dir
+SESSION_SEED_ARCHIVE = LAYOUT.session_seed_archive
 TOKEN_PATH = LAYOUT.token_path
 INFERENCE_STATE = LAYOUT.inference_state
 INFERENCE_REQUEST_LOG_DIR = LAYOUT.inference_request_log_dir
@@ -277,6 +279,7 @@ def _deployment_config(
     local_task_source: bool,
     evaluation_inference_token: str | None,
     finalization_inference_token: str | None,
+    session_seed_archive: bool = False,
 ) -> dict:
     task_source = TASK_SOURCE_DIR if local_task_source else config.task_source
     backends = {}
@@ -459,6 +462,9 @@ def _deployment_config(
         "agent_repo_path": AGENT_REPO,
         "session_dir": SESSION_DIR,
         "session_id": SESSION_ID,
+        "session_seed_archive": (
+            SESSION_SEED_ARCHIVE if session_seed_archive else None
+        ),
         "backends": backends,
         "access_policies": policies,
         "budgets": budgets,
@@ -588,9 +594,29 @@ def compile_harbor_task(
     output_dir: Path | str,
     *,
     vero_root: Path | None = None,
+    session_seed_archive: Path | str | None = None,
 ) -> Path:
-    """Emit a self-contained Harbor task directory from validated config."""
+    """Emit a self-contained Harbor task directory from validated config.
+
+    ``session_seed_archive`` is a previously exported session (the
+    ``session-rescue.tar.gz`` a dead trial leaves in its artifacts, or the
+    ``verifier/session.tar.gz`` a finished one leaves) to restore into the new
+    stack's session directory on first boot. Baking it into the sidecar image is
+    the only transport available: the session directory lives on a compose volume
+    inside a per-trial Modal sandbox, so nothing a relaunch could mount survives,
+    and the archive on the launching host is the only copy that does.
+    """
     output = Path(output_dir).expanduser().resolve()
+    seed_archive = (
+        Path(session_seed_archive).expanduser().resolve()
+        if session_seed_archive is not None
+        else None
+    )
+    if seed_archive is not None:
+        # Fail on the host, now, rather than after an image build and a stack
+        # bring-up inside a sandbox whose logs nobody is tailing. Reads the
+        # manifest only; the bytes are not unpacked here.
+        read_harbor_session_archive_manifest(seed_archive)
     source_root = (vero_root or Path(__file__).parents[4]).resolve()
     use_local_vero = _is_vero_source(source_root)
     if vero_root is not None and not use_local_vero:
@@ -715,6 +741,13 @@ def compile_harbor_task(
             Path(config.command_backend.harness_source),
             sidecar_dir / "harness",
         )
+    if seed_archive is not None:
+        # Into the sidecar's build context, beside the case lists it is as
+        # sensitive as: the archive carries database.json, whose per-case records
+        # name held-out tasks and their scores. The Dockerfile chmods it 600 on
+        # the way in, matching serve.json.
+        sidecar_dir.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(seed_archive, sidecar_dir / "session-seed.tar.gz")
     local_task_source = (
         config.task_source is not None and Path(config.task_source).exists()
     )
@@ -754,6 +787,7 @@ def compile_harbor_task(
         local_task_source=local_task_source,
         evaluation_inference_token=evaluation_inference_token,
         finalization_inference_token=finalization_inference_token,
+        session_seed_archive=seed_archive is not None,
     )
     (sidecar_dir / "serve.json").write_text(
         json.dumps(deployment, ensure_ascii=False, indent=2) + "\n",
@@ -871,6 +905,9 @@ def compile_harbor_task(
         "sidecar_factory": FACTORY_PATH,
         "producer_base_url": PRODUCER_BASE_URL,
         "command_harness": config.command_backend is not None,
+        "session_seed_archive": (
+            SESSION_SEED_ARCHIVE if seed_archive is not None else None
+        ),
         # The Harbor backend hard-rejects request.seed, so only advertise the
         # flag when the build evaluates through a command backend.
         "seed_supported": config.command_backend is not None,

vero/src/vero/harbor/build/templates/Dockerfile.sidecar.j2

@@ -16,6 +16,9 @@ RUN uv pip install --system "{{ harbor_requirement }}"
 COPY agent-baseline {{ layout.trusted_repo }}
 COPY sidecar/cases {{ layout.cases }}
 COPY sidecar/serve.json {{ layout.serve_config }}
+{% if session_seed_archive %}
+COPY sidecar/session-seed.tar.gz {{ session_seed_archive }}
+{% endif %}
 {% if local_task_source %}
 COPY sidecar/task-source {{ layout.task_source }}
 {% endif %}
@@ -49,5 +52,11 @@ RUN mkdir -p /home/harness/.cache \
 # {{ layout.task_source }}, which harbor must read to grade host-side, remain readable
 # and are addressed by the deeper in-container isolation work.)
 RUN chmod 700 {{ layout.cases }} && chmod 600 {{ layout.serve_config }}
+{% if session_seed_archive %}
+{# The restored session's database.json carries held-out case membership and
+   per-case scores, the same secret the case lists above hold, so the seed
+   archive is locked to root on the same grounds. #}
+RUN chmod 600 {{ session_seed_archive }}
+{% endif %}
 
 WORKDIR /opt

vero/src/vero/harbor/cli.py

@@ -775,6 +775,18 @@ def _preflight_models(config) -> None:
 @click.option("--agent", required=True, help="Harbor optimizer agent.")
 @click.option("--model", help="Model used by the optimizer agent.")
 @click.option("--environment", default="modal", show_default=True)
+@click.option(
+    "--resume-from",
+    "resume_from",
+    type=click.Path(path_type=Path, exists=True, dir_okay=False),
+    help=(
+        "Restore a previous run's exported session into this one, so the "
+        "relaunch keeps its candidates, its scores and its spent budget instead "
+        "of starting over. Takes the artifacts/session-rescue.tar.gz a dead "
+        "trial leaves behind, or the verifier/session.tar.gz a finished one "
+        "does. Opt-in: without it a relaunch is a fresh run."
+    ),
+)
 @click.option(
     "--env-file",
     "env_file",
@@ -787,7 +799,9 @@ def _preflight_models(config) -> None:
 )
 @_PARAM_OPTION
 @click.argument("extra", nargs=-1, type=click.UNPROCESSED)
-def run_command(config_path, agent, model, environment, params, env_file, extra):
+def run_command(
+    config_path, agent, model, environment, resume_from, params, env_file, extra
+):
     """Compile to a temporary directory and invoke `harbor run`."""
     from vero.harbor.build import compile_harbor_task, load_harbor_build_config
 
@@ -813,7 +827,10 @@ def run_command(config_path, agent, model, environment, params, env_file, extra)
         task = compile_harbor_task(
             config,
             Path(temporary) / "task",
+            session_seed_archive=resume_from,
         )
+        if resume_from is not None:
+            click.echo(f"Resuming from session archive {resume_from}")
         command = [
             uvx,
             "--python",

vero/src/vero/harbor/deployment.py

@@ -28,7 +28,10 @@
 from vero.runtime.artifacts import ArtifactStore
 from vero.sandbox import LocalSandbox
 from vero.sidecar.serve import SidecarComponents
-from vero.sidecar.session import initialize_harbor_session_manifest
+from vero.sidecar.session import (
+    initialize_harbor_session_manifest,
+    restore_harbor_session_archive,
+)
 from vero.sidecar.sidecar import EvaluationSidecar, SidecarEvaluationPolicy
 from vero.sidecar.transport import GitCandidateTransport
 from vero.sidecar.verifier import (
@@ -123,6 +126,10 @@ class HarborDeploymentConfig(StrictModel):
     agent_repo_path: str
     session_dir: str
     session_id: str = "trial"
+    # A previous run's exported session, baked into the sidecar image, restored
+    # into session_dir on first boot. Absent by default: a relaunch starts clean
+    # unless the operator named an archive on the command line.
+    session_seed_archive: str | None = None
     backends: dict[str, DeploymentBackendConfig]
     access_policies: list[SidecarEvaluationPolicy]
     budgets: list[EvaluationBudget] = Field(default_factory=list)
@@ -176,7 +183,11 @@ def validate_absolute_path(cls, value: str) -> str:
             raise ValueError("deployment paths must be absolute")
         return value
 
-    @field_validator("inference_usage_path", "inference_request_log_dir")
+    @field_validator(
+        "inference_usage_path",
+        "inference_request_log_dir",
+        "session_seed_archive",
+    )
     @classmethod
     def validate_optional_file_path(cls, value: str | None) -> str | None:
         if value is not None:
@@ -271,6 +282,15 @@ async def build_harbor_components(config: dict) -> SidecarComponents:
     parsed = HarborDeploymentConfig.model_validate(config)
     session_dir = Path(parsed.session_dir)
     session_dir.mkdir(parents=True, exist_ok=True)
+    # Before anything reads the session dir. Every consumer below it -- the
+    # candidate repository, the evaluation database, the budget ledger, the
+    # agent's disclosure ledger -- opens what is on disk and reuses it if it is
+    # there, so seeding here is the whole of the restore: no consumer needs to
+    # know a resume happened. Missing archive is fatal rather than ignored: the
+    # operator asked for a resume, and silently starting a fresh multi-hour run
+    # instead is the failure this exists to prevent.
+    if parsed.session_seed_archive is not None:
+        restore_harbor_session_archive(parsed.session_seed_archive, session_dir)
     # The session dir holds the trusted state — held-out evaluation records and
     # scores, the budget ledger, and every candidate's code. Lock it to the
     # owning (trusted) user so the unprivileged harness that executes candidate

vero/src/vero/layout.py

@@ -38,6 +38,10 @@ class TaskLayout:
         harness: The scoring program, for a command backend.
         overlay: Host files baked into the optimizer's workspace.
         serve_config: The trusted deployment config, root-only.
+        session_seed_archive: A previously exported session, baked in so a
+            relaunch can restore it. Root-only for the same reason as
+            serve_config and the case lists: it carries database.json, whose
+            per-case records disclose held-out membership and scores.
         seed_script: Script that seeds the target repo on first boot.
         inference_config: The gateway's scope config.
         agent_volume: The optimizer's context directory, written by the sidecar.
@@ -63,6 +67,7 @@ class TaskLayout:
     harness: str = "/opt/harness"
     overlay: str = "/opt/overlay"
     serve_config: str = "/opt/serve.json"
+    session_seed_archive: str = "/opt/session-seed.tar.gz"
     seed_script: str = "/opt/seed.sh"
     inference_config: str = "/opt/inference.json"
     agent_volume: str = "/state/agent-context"