From 9df50eacc7f4f0c41ef206c4c267a7bd216af277 Mon Sep 17 00:00:00 2001
From: Varun Ursekar
Date: Sun, 2 Aug 2026 22:40:02 -0700
Subject: [PATCH 01/16] Add interpret module: harbor adapter and deterministic
edit decomposition
Phases 1 and 2 of an interpretability pipeline for optimization runs. No model is
consulted anywhere in this commit; everything here is reproducible and diffable.
Source artifacts enter through an adapter that canonicalises them into `models`,
so downstream code never sees a harbor path, a tarball, or a git object, and a
second producer of runs means a new adapter and nothing else. `artifacts/` is
forbidden from importing `edits/` or `labeling/` for that reason.
The unit of analysis is a symbol-scoped edit, not a candidate. A single commit
routinely bundles a bug fix with unrelated tuning under one subject line, so
labelling per candidate assigns one category to several distinct modifications
and buries the interesting one -- on a real candidate here, +908/-142 across one
commit decomposes into 57 edits.
Locus is derived from the syntax tree rather than git's hunk header, which
reports the enclosing class for a method and so attributes a one-line shell-exec
fix to the whole agent. Module-level bindings are split further by target name
and value shape, because that bucket otherwise holds a system prompt, a tool
table and a dozen tuning constants as one undifferentiated blob.
Caching is content-addressed and split in two: unpacking a session archive costs
minutes and hundreds of megabytes, while a label costs a fraction of a cent, so
revising a taxonomy later re-labels without re-extracting. Edit ids are derived
from edit content, not position, so cached labels survive re-extraction and
reordering.
Scope selection deliberately does not use harbor's `reportable` flag, which
requires error_rate == 0.0 exactly while every build config sets a threshold of
0.1 -- that discards good runs that lost a single case to a platform hiccup.
Co-Authored-By: Claude Opus 5 (1M context)
---
vero/pyproject.toml | 3 +
vero/src/vero/interpret/__init__.py | 26 +++
vero/src/vero/interpret/artifacts/__init__.py | 5 +
vero/src/vero/interpret/artifacts/base.py | 53 ++++++
.../interpret/artifacts/harbor/__init__.py | 5 +
.../interpret/artifacts/harbor/adapter.py | 153 +++++++++++++++++
.../vero/interpret/artifacts/harbor/repo.py | 77 +++++++++
.../interpret/artifacts/harbor/session.py | 79 +++++++++
vero/src/vero/interpret/cache.py | 94 ++++++++++
vero/src/vero/interpret/edits/__init__.py | 5 +
vero/src/vero/interpret/edits/decompose.py | 148 ++++++++++++++++
vero/src/vero/interpret/edits/locus.py | 131 ++++++++++++++
vero/src/vero/interpret/models.py | 161 ++++++++++++++++++
vero/tests/test_interpret_locus.py | 75 ++++++++
14 files changed, 1015 insertions(+)
create mode 100644 vero/src/vero/interpret/__init__.py
create mode 100644 vero/src/vero/interpret/artifacts/__init__.py
create mode 100644 vero/src/vero/interpret/artifacts/base.py
create mode 100644 vero/src/vero/interpret/artifacts/harbor/__init__.py
create mode 100644 vero/src/vero/interpret/artifacts/harbor/adapter.py
create mode 100644 vero/src/vero/interpret/artifacts/harbor/repo.py
create mode 100644 vero/src/vero/interpret/artifacts/harbor/session.py
create mode 100644 vero/src/vero/interpret/cache.py
create mode 100644 vero/src/vero/interpret/edits/__init__.py
create mode 100644 vero/src/vero/interpret/edits/decompose.py
create mode 100644 vero/src/vero/interpret/edits/locus.py
create mode 100644 vero/src/vero/interpret/models.py
create mode 100644 vero/tests/test_interpret_locus.py
diff --git a/vero/pyproject.toml b/vero/pyproject.toml
index fb743449..6da8e97d 100644
--- a/vero/pyproject.toml
+++ b/vero/pyproject.toml
@@ -29,6 +29,9 @@ harbor = [
claude = [
"claude-agent-sdk>=0.1.56",
]
+interpret = [
+ "openai>=1.0",
+]
optimize = [
"async-lru>=2.0.5",
"beautifulsoup4>=4.14.2",
diff --git a/vero/src/vero/interpret/__init__.py b/vero/src/vero/interpret/__init__.py
new file mode 100644
index 00000000..72261d06
--- /dev/null
+++ b/vero/src/vero/interpret/__init__.py
@@ -0,0 +1,26 @@
+"""Interpretability analysis over optimization runs.
+
+Source artifacts are canonicalised by an adapter (`artifacts`), split into
+symbol-scoped edits deterministically (`edits`), labelled with a model
+(`labeling`), and aggregated (`analysis`). Only `labeling` is non-deterministic.
+"""
+
+from vero.interpret.models import (
+ Candidate,
+ CellRef,
+ Corpus,
+ Edit,
+ EvalRecord,
+ SymbolKind,
+ Trajectory,
+)
+
+__all__ = [
+ "Candidate",
+ "CellRef",
+ "Corpus",
+ "Edit",
+ "EvalRecord",
+ "SymbolKind",
+ "Trajectory",
+]
diff --git a/vero/src/vero/interpret/artifacts/__init__.py b/vero/src/vero/interpret/artifacts/__init__.py
new file mode 100644
index 00000000..36b309e2
--- /dev/null
+++ b/vero/src/vero/interpret/artifacts/__init__.py
@@ -0,0 +1,5 @@
+"""Source adapters: raw run artifacts in, canonical models out."""
+
+from vero.interpret.artifacts.base import SourceAdapter, available, get, register
+
+__all__ = ["SourceAdapter", "available", "get", "register"]
diff --git a/vero/src/vero/interpret/artifacts/base.py b/vero/src/vero/interpret/artifacts/base.py
new file mode 100644
index 00000000..7a7aeaf5
--- /dev/null
+++ b/vero/src/vero/interpret/artifacts/base.py
@@ -0,0 +1,53 @@
+"""The adapter boundary.
+
+An adapter takes raw run artifacts from one producer and canonicalises them into
+`models`. Everything downstream — edit decomposition, labelling, analysis — consumes
+only the canonical types, so supporting a second producer of optimization runs is a
+new adapter and nothing else.
+
+Adapters do two things and no more: find runs under the roots they are given, and
+load one run. They must not interpret, classify, or score anything; that judgement
+belongs in `edits/` (deterministic) and `labeling/` (model-assisted), which is what
+keeps the boundary useful.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterable, Protocol, runtime_checkable
+
+from vero.interpret.models import CellRef, Trajectory
+
+
+@runtime_checkable
+class SourceAdapter(Protocol):
+ """Canonicalises one producer's run artifacts."""
+
+ name: str
+
+ def discover(self, roots: Iterable[Path]) -> list[CellRef]:
+ """Find every run under these roots. Cheap: no archives are opened."""
+ ...
+
+ def load(self, ref: CellRef) -> Trajectory:
+ """Materialise one run. May be expensive; results are cached by the caller."""
+ ...
+
+
+_REGISTRY: dict[str, SourceAdapter] = {}
+
+
+def register(adapter: SourceAdapter) -> SourceAdapter:
+ _REGISTRY[adapter.name] = adapter
+ return adapter
+
+
+def get(name: str) -> SourceAdapter:
+ if name not in _REGISTRY:
+ known = ", ".join(sorted(_REGISTRY)) or "none registered"
+ raise KeyError(f"unknown source adapter {name!r} (known: {known})")
+ return _REGISTRY[name]
+
+
+def available() -> list[str]:
+ return sorted(_REGISTRY)
diff --git a/vero/src/vero/interpret/artifacts/harbor/__init__.py b/vero/src/vero/interpret/artifacts/harbor/__init__.py
new file mode 100644
index 00000000..b902360b
--- /dev/null
+++ b/vero/src/vero/interpret/artifacts/harbor/__init__.py
@@ -0,0 +1,5 @@
+"""Adapter for `vero harbor run` job trees."""
+
+from vero.interpret.artifacts.harbor.adapter import HarborAdapter, build
+
+__all__ = ["HarborAdapter", "build"]
diff --git a/vero/src/vero/interpret/artifacts/harbor/adapter.py b/vero/src/vero/interpret/artifacts/harbor/adapter.py
new file mode 100644
index 00000000..69930f00
--- /dev/null
+++ b/vero/src/vero/interpret/artifacts/harbor/adapter.py
@@ -0,0 +1,153 @@
+"""Harbor adapter: raw job artifacts in, canonical `Trajectory` out.
+
+Layout it expects, which is what `vero harbor run` writes:
+
+ ///jobs//task__*/verifier/
+ finalization.json the shipped candidate and its held-out reward
+ session.tar.gz the candidate repo and the sidecar evaluation records
+
+`` may be a benchmark directory or a tree of them; discovery handles both, so
+callers can point at one benchmark or a whole runs/ tree without knowing the depth.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Iterable
+
+from vero.interpret.artifacts.base import register
+from vero.interpret.artifacts.harbor import session as session_mod
+from vero.interpret.artifacts.harbor.repo import CandidateRepo
+from vero.interpret.cache import Cache
+from vero.interpret.models import Candidate, CellRef, EvalRecord, Trajectory
+
+# `shipped AND error_rate == 0.0 AND tokens > 0` is the flag harbor's own tooling
+# uses, but every benchmark's build config sets error_rate_threshold: 0.1, so the
+# exact-zero test discards good runs that lost one case to a platform hiccup. The
+# benchmark's own threshold is the defensible line.
+DEFAULT_MAX_ERROR_RATE = 0.1
+
+
+class HarborAdapter:
+ name = "harbor"
+
+ def __init__(self, cache: Cache) -> None:
+ self.cache = cache
+
+ # -- discovery ------------------------------------------------------------
+
+ def discover(self, roots: Iterable[Path]) -> list[CellRef]:
+ refs: list[CellRef] = []
+ seen: set[tuple[str, str]] = set()
+ for root in roots:
+ root = Path(root)
+ if not root.is_dir():
+ continue
+ for final in sorted(root.glob("**/jobs/*/task__*/verifier/finalization.json")):
+ # ...///jobs//task__*/verifier/
+ job_dir = final.parents[2]
+ cell_dir = final.parents[4]
+ benchmark = final.parents[5].name
+ identity = (benchmark, cell_dir.name)
+ if identity in seen:
+ continue
+ seen.add(identity)
+ refs.append(
+ CellRef(
+ source=self.name,
+ benchmark=benchmark,
+ cell=cell_dir.name,
+ root=str(root),
+ cell_dir=str(cell_dir),
+ job_dir=str(job_dir),
+ )
+ )
+ return refs
+
+ # -- loading --------------------------------------------------------------
+
+ def load(self, ref: CellRef) -> Trajectory:
+ cell_dir = Path(ref.cell_dir)
+ finals = sorted(cell_dir.glob("jobs/*/task__*/verifier/finalization.json"))
+ if not finals:
+ return Trajectory(ref=ref)
+
+ # Last job wins: a cell re-run in place leaves the earlier attempt behind.
+ final_path = finals[-1]
+ final = json.loads(final_path.read_text())
+ metrics = (final.get("reward_metrics") or {}).get("reward", {}) or {}
+ shipped_sha = ((final.get("candidate") or {}).get("id") or "")
+
+ traj = Trajectory(
+ ref=ref,
+ reward=(final.get("rewards") or {}).get("reward"),
+ baseline_reward=(final.get("baseline_rewards") or {}).get("reward"),
+ error_rate=metrics.get("error_rate"),
+ total_tokens=metrics.get("inference_total_tokens"),
+ )
+
+ archive = final_path.parent / "session.tar.gz"
+ if not archive.is_file():
+ return traj
+ root = session_mod.unpack(archive, self.cache)
+ if root is None:
+ return traj
+
+ repo_dir = session_mod.find_repo(root)
+ if repo_dir is not None:
+ traj.candidates = self._candidates(CandidateRepo(repo_dir), shipped_sha)
+ traj.evaluations = self._evaluations(session_mod.read_evaluations(root))
+ return traj
+
+ def _candidates(self, repo: CandidateRepo, shipped_sha: str) -> list[Candidate]:
+ out: list[Candidate] = []
+ for position, (sha, subject, body) in enumerate(repo.log()):
+ stats = repo.numstat(f"{sha}^", sha) if position else {}
+ out.append(
+ Candidate(
+ sha=sha,
+ parent_sha=repo.parent(sha),
+ position=position,
+ subject=subject,
+ body=body[:4000],
+ files=repo.files(sha),
+ insertions=sum(a for a, _ in stats.values()),
+ deletions=sum(r for _, r in stats.values()),
+ tree_sha=repo.tree_sha(sha),
+ is_seed=position == 0,
+ is_shipped=bool(shipped_sha) and sha.startswith(shipped_sha[:12]),
+ )
+ )
+ return out
+
+ @staticmethod
+ def _evaluations(records: list[dict]) -> list[EvalRecord]:
+ out: list[EvalRecord] = []
+ for doc in records:
+ request = doc.get("request") or {}
+ report = doc.get("report") or {}
+ eval_set = request.get("evaluation_set") or {}
+ metrics = report.get("metrics") or {}
+ sha = ((request.get("candidate") or {}).get("id") or "")
+ partition = eval_set.get("partition")
+ score = metrics.get("score")
+ if not (sha and partition and score is not None):
+ continue
+ out.append(
+ EvalRecord(
+ candidate_sha=sha,
+ partition=partition,
+ score=score,
+ error_rate=metrics.get("error_rate"),
+ selection_kind=(eval_set.get("selection") or {}).get("kind"),
+ n_attempts=(request.get("limits") or {}).get("n_attempts"),
+ started_at=report.get("started_at"),
+ finished_at=report.get("finished_at"),
+ )
+ )
+ return out
+
+
+def build(cache: Cache) -> HarborAdapter:
+ return register(HarborAdapter(cache))
diff --git a/vero/src/vero/interpret/artifacts/harbor/repo.py b/vero/src/vero/interpret/artifacts/harbor/repo.py
new file mode 100644
index 00000000..85aef3ed
--- /dev/null
+++ b/vero/src/vero/interpret/artifacts/harbor/repo.py
@@ -0,0 +1,77 @@
+"""Read-only git access to a candidate repository.
+
+Bare-repo reads only, via subprocess. No checkout ever happens: the analysis needs
+trees and diffs, and materialising working copies for 100 cells would cost disk for
+nothing.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+
+class CandidateRepo:
+ def __init__(self, git_dir: Path) -> None:
+ self.git_dir = Path(git_dir)
+
+ def _run(self, *args: str) -> str:
+ out = subprocess.run(
+ ["git", "--git-dir", str(self.git_dir), *args],
+ capture_output=True,
+ text=True,
+ )
+ return out.stdout
+
+ def log(self) -> list[tuple[str, str, str]]:
+ """(sha, subject, body) oldest first, so index 0 is the seed.
+
+ `--all` because a candidate chain can leave commits unreachable from any
+ branch head once the optimizer reaches back past them.
+ """
+ raw = self._run("log", "--all", "--format=%H%x1f%s%x1f%b%x1e")
+ rows = []
+ for record in raw.split("\x1e"):
+ record = record.strip("\n")
+ if not record:
+ continue
+ sha, subject, body = (record.split("\x1f") + ["", "", ""])[:3]
+ rows.append((sha, subject, body))
+ rows.reverse()
+ return rows
+
+ def tree_sha(self, sha: str) -> str:
+ return self._run("rev-parse", f"{sha}^{{tree}}").strip()
+
+ def parent(self, sha: str) -> str | None:
+ out = self._run("rev-parse", f"{sha}^").strip()
+ return out or None
+
+ def files(self, sha: str) -> list[str]:
+ return [
+ f
+ for f in self._run("show", "--name-only", "--format=", sha).splitlines()
+ if f.strip() and "__pycache__" not in f
+ ]
+
+ def show_file(self, sha: str, path: str) -> str:
+ return self._run("show", f"{sha}:{path}")
+
+ def diff(self, a: str, b: str, path: str | None = None, context: int = 0) -> str:
+ args = ["diff", f"-U{context}", a, b, "--", path or ".", ":(exclude)*__pycache__*"]
+ return self._run(*args)
+
+ def numstat(self, a: str, b: str) -> dict[str, tuple[int, int]]:
+ """path -> (added, removed). Binary files report as (0, 0)."""
+ out: dict[str, tuple[int, int]] = {}
+ raw = self._run("diff", "--numstat", a, b, "--", ".", ":(exclude)*__pycache__*")
+ for line in raw.splitlines():
+ parts = line.split("\t")
+ if len(parts) != 3:
+ continue
+ add, rem, path = parts
+ out[path] = (
+ int(add) if add.isdigit() else 0,
+ int(rem) if rem.isdigit() else 0,
+ )
+ return out
diff --git a/vero/src/vero/interpret/artifacts/harbor/session.py b/vero/src/vero/interpret/artifacts/harbor/session.py
new file mode 100644
index 00000000..48202170
--- /dev/null
+++ b/vero/src/vero/interpret/artifacts/harbor/session.py
@@ -0,0 +1,79 @@
+"""Unpack the two things worth keeping out of a harbor `session.tar.gz`.
+
+These archives run to hundreds of megabytes, almost all of it agent transcripts and
+container logs. Only the candidate git repository and the sidecar evaluation records
+are needed here, so members are filtered on the way out and the result is cached by
+the archive's own digest — re-running the pipeline never re-extracts.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import sys
+import tarfile
+from pathlib import Path
+
+from vero.interpret.cache import Cache
+
+_WANTED_DIR = "/candidates/repository.git/"
+_WANTED_FILE = "evaluation.json"
+
+
+def digest(path: Path, *, chunk: int = 1 << 20) -> str:
+ """Hash the archive itself, so the cache key is independent of where it sits."""
+ h = hashlib.sha256()
+ with path.open("rb") as fh:
+ while block := fh.read(chunk):
+ h.update(block)
+ return h.hexdigest()
+
+
+def unpack(archive: Path, cache: Cache) -> Path | None:
+ """Extract the wanted members, returning the session root."""
+ key = digest(archive)
+ if (hit := cache.get_dir(key)) is not None:
+ return hit
+
+ dest = cache.reserve_dir(key)
+ try:
+ with tarfile.open(archive) as tar:
+ members = [
+ m
+ for m in tar.getmembers()
+ if _WANTED_DIR in m.name or m.name.endswith(_WANTED_FILE)
+ ]
+ if not members:
+ return None
+ # filter= is 3.12+; these are our own verifier's archives, so the guard is
+ # about running on 3.11 rather than about untrusted input.
+ if sys.version_info >= (3, 12):
+ tar.extractall(dest, members=members, filter="data")
+ else:
+ tar.extractall(dest, members=members)
+ except (tarfile.TarError, OSError):
+ return None
+
+ cache.commit_dir(key)
+ return dest
+
+
+def find_repo(session_root: Path) -> Path | None:
+ hits = list(session_root.glob("**/candidates/repository.git"))
+ return hits[0] if hits else None
+
+
+def read_evaluations(session_root: Path) -> list[dict]:
+ """Every sidecar evaluation record, unaggregated.
+
+ Repeats of the same candidate on the same partition are kept separate: collapsing
+ them into a per-partition map is exactly how a re-score silently overwrites the
+ earlier one, which hides the corpus's only direct measurement of scoring noise.
+ """
+ out = []
+ for path in session_root.glob("**/evaluations/*/evaluation.json"):
+ try:
+ out.append(json.loads(path.read_text()))
+ except (OSError, json.JSONDecodeError):
+ continue
+ return out
diff --git a/vero/src/vero/interpret/cache.py b/vero/src/vero/interpret/cache.py
new file mode 100644
index 00000000..d7d3e9ec
--- /dev/null
+++ b/vero/src/vero/interpret/cache.py
@@ -0,0 +1,94 @@
+"""Content-addressed file cache, shared by artifact extraction and labelling.
+
+Two callers with very different economics use this: unpacking a session archive costs
+minutes and hundreds of megabytes, while an LLM label costs a fraction of a cent. They
+get separate namespaces so revising a taxonomy re-labels everything without
+re-extracting anything — the failure mode that actually wastes an afternoon.
+
+Writes go to a temp file and are renamed into place, so an interrupted run leaves
+either a complete entry or none, never a truncated one that poisons the next attempt.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+from pathlib import Path
+from typing import Any
+
+
+def key_of(*parts: str) -> str:
+ return hashlib.sha256("\x00".join(parts).encode("utf-8", "replace")).hexdigest()
+
+
+class Cache:
+ """A namespaced content-addressed store under one root."""
+
+ def __init__(self, root: Path, namespace: str, *, refresh: bool = False) -> None:
+ self.root = Path(root) / namespace
+ self.refresh = refresh
+ self.hits = 0
+ self.misses = 0
+
+ def _path(self, key: str, suffix: str) -> Path:
+ return self.root / key[:2] / f"{key}{suffix}"
+
+ # -- JSON entries (labels, parsed records) --------------------------------
+
+ def get_json(self, key: str) -> Any | None:
+ if self.refresh:
+ return None
+ path = self._path(key, ".json")
+ if not path.is_file():
+ self.misses += 1
+ return None
+ try:
+ value = json.loads(path.read_text())
+ except (OSError, json.JSONDecodeError):
+ self.misses += 1
+ return None
+ self.hits += 1
+ return value
+
+ def put_json(self, key: str, value: Any) -> None:
+ path = self._path(key, ".json")
+ path.parent.mkdir(parents=True, exist_ok=True)
+ tmp = path.with_suffix(".json.tmp")
+ tmp.write_text(json.dumps(value, indent=None, default=str))
+ os.replace(tmp, path)
+
+ # -- Directory entries (unpacked archives) --------------------------------
+
+ def get_dir(self, key: str) -> Path | None:
+ """A completed directory entry, or None.
+
+ Completion is marked by a sentinel written after the unpack finishes, so a
+ directory left behind by a crash is treated as absent and redone rather than
+ silently used half-populated.
+ """
+ path = self._path(key, ".d")
+ if self.refresh or not (path / ".complete").is_file():
+ self.misses += 1
+ return None
+ self.hits += 1
+ return path
+
+ def reserve_dir(self, key: str) -> Path:
+ """Empty directory to unpack into. Call `commit_dir` when finished."""
+ path = self._path(key, ".d")
+ if path.exists():
+ shutil.rmtree(path, ignore_errors=True)
+ path.mkdir(parents=True, exist_ok=True)
+ return path
+
+ def commit_dir(self, key: str) -> Path:
+ path = self._path(key, ".d")
+ (path / ".complete").write_text("")
+ return path
+
+ def stats(self) -> str:
+ total = self.hits + self.misses
+ pct = (100.0 * self.hits / total) if total else 0.0
+ return f"{self.root.name}: {self.hits} hit / {self.misses} miss ({pct:.0f}%)"
diff --git a/vero/src/vero/interpret/edits/__init__.py b/vero/src/vero/interpret/edits/__init__.py
new file mode 100644
index 00000000..1a616fb4
--- /dev/null
+++ b/vero/src/vero/interpret/edits/__init__.py
@@ -0,0 +1,5 @@
+"""Deterministic decomposition of candidates into symbol-scoped edits."""
+
+from vero.interpret.edits.decompose import decompose
+
+__all__ = ["decompose"]
diff --git a/vero/src/vero/interpret/edits/decompose.py b/vero/src/vero/interpret/edits/decompose.py
new file mode 100644
index 00000000..097d8212
--- /dev/null
+++ b/vero/src/vero/interpret/edits/decompose.py
@@ -0,0 +1,148 @@
+"""Split a candidate into symbol-scoped edits.
+
+The unit of analysis is one symbol touched by one candidate, not the candidate. A
+single commit routinely bundles a genuine bug fix with unrelated tuning under one
+subject line, so labelling per candidate assigns a single category to several
+distinct modifications and the interesting one gets buried.
+
+Everything here is deterministic. No model is consulted, so the resulting table is
+reproducible and can be diffed between runs.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+
+from vero.interpret.artifacts.harbor.repo import CandidateRepo
+from vero.interpret.edits import locus
+from vero.interpret.models import Candidate, Edit, SymbolKind
+
+_HUNK_SPLIT = re.compile(r"^(@@ .*?@@.*)$", re.M)
+_SKIP = ("__pycache__", ".gitignore")
+
+
+def _per_symbol_diff(diff: str, keep: set[int]) -> str:
+ """Hunks of `diff` whose post-image start line falls in `keep`."""
+ parts = _HUNK_SPLIT.split(diff)
+ if len(parts) < 2:
+ return ""
+ chunks: list[str] = []
+ for header, body in zip(parts[1::2], parts[2::2]):
+ match = locus._HUNK.match(header + "\n")
+ if not match:
+ continue
+ start = int(match.group(1))
+ count = int(match.group(2) or 1)
+ if keep & set(range(start, start + max(count, 1))):
+ chunks.append(header + body.rstrip("\n"))
+ return "\n".join(chunks)[:8000]
+
+
+def decompose(
+ repo: CandidateRepo,
+ cell_key: str,
+ candidate: Candidate,
+) -> list[Edit]:
+ """Symbol-scoped edits introduced by `candidate` relative to its parent."""
+ if candidate.is_seed or candidate.parent_sha is None:
+ return []
+
+ edits: list[Edit] = []
+ for path in candidate.files:
+ if any(s in path for s in _SKIP):
+ continue
+
+ diff = repo.diff(candidate.parent_sha, candidate.sha, path=path, context=0)
+ if not diff.strip():
+ continue
+
+ if not path.endswith(".py"):
+ added = sum(
+ 1 for line in diff.splitlines() if line.startswith("+") and line[1:2] != "+"
+ )
+ removed = sum(
+ 1 for line in diff.splitlines() if line.startswith("-") and line[1:2] != "-"
+ )
+ edits.append(
+ _edit(cell_key, candidate, path, "", SymbolKind.NON_PYTHON,
+ added, removed, diff[:8000], None, None)
+ )
+ continue
+
+ after_src = repo.show_file(candidate.sha, path)
+ before_src = repo.show_file(candidate.parent_sha, path)
+ mapping = locus.symbol_map(after_src)
+
+ grouped: dict[tuple[str, SymbolKind], set[int]] = defaultdict(set)
+ for line in locus.changed_lines(diff):
+ symbol, kind = mapping.get(line, ("", SymbolKind.MODULE))
+ grouped[(symbol, kind)].add(line)
+
+ # Deletions vanish from the post-image, so a symbol removed outright has no
+ # line to map. Attribute the whole file's removals to rather than
+ # dropping them: "removed the audit pass" is a modification worth counting.
+ removed_total = sum(
+ 1 for line in diff.splitlines() if line.startswith("-") and line[1:2] != "-"
+ )
+ added_total = sum(
+ 1 for line in diff.splitlines() if line.startswith("+") and line[1:2] != "+"
+ )
+ attributed_added = sum(len(v) for v in grouped.values())
+ if removed_total and not grouped:
+ grouped[("", SymbolKind.MODULE)] = set()
+
+ for (symbol, kind), lines in grouped.items():
+ before = after = None
+ if kind is SymbolKind.SCALAR_CONST:
+ before = locus.scalar_value(before_src, symbol)
+ after = locus.scalar_value(after_src, symbol)
+ if before == after:
+ continue # touched by reflow, not retuned
+ share = len(lines)
+ edits.append(
+ _edit(
+ cell_key,
+ candidate,
+ path,
+ symbol,
+ kind,
+ share,
+ # Removals cannot be attributed per symbol; carry the file total
+ # on the module row so the count is never silently lost.
+ removed_total if symbol == "" else 0,
+ _per_symbol_diff(diff, lines) or diff[:2000],
+ before,
+ after,
+ )
+ )
+ if attributed_added < added_total and grouped:
+ pass # unattributed remainder is comment/blank churn; not an edit
+ return edits
+
+
+def _edit(
+ cell_key: str,
+ candidate: Candidate,
+ path: str,
+ symbol: str,
+ kind: SymbolKind,
+ added: int,
+ removed: int,
+ diff: str,
+ before: str | None,
+ after: str | None,
+) -> Edit:
+ return Edit(
+ id=Edit.make_id(cell_key, candidate.sha, path, symbol, diff),
+ cell_key=cell_key,
+ candidate_sha=candidate.sha,
+ path=path,
+ symbol=symbol,
+ symbol_kind=kind,
+ added=added,
+ removed=removed,
+ before_value=before,
+ after_value=after,
+ diff=diff,
+ )
diff --git a/vero/src/vero/interpret/edits/locus.py b/vero/src/vero/interpret/edits/locus.py
new file mode 100644
index 00000000..87d784f2
--- /dev/null
+++ b/vero/src/vero/interpret/edits/locus.py
@@ -0,0 +1,131 @@
+"""Resolve which symbol an edit landed in, from the syntax tree.
+
+Git's hunk header is not good enough. Its Python `xfuncname` matches the last
+preceding definition line, which for a method inside a class reports the *class* — so
+a one-line fix to a shell-exec helper is attributed to the whole agent. Parsing the
+post-image and mapping changed line numbers to the innermost enclosing definition
+gives function-level locus exactly, with no model involved.
+
+Module-level bindings get split further by target name and value shape, because that
+bucket is otherwise the largest and least informative: on one real candidate it held
+170 changed lines spanning a system prompt, the tool table, nine tuning constants and
+six regexes, which are four different kinds of modification.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+
+from vero.interpret.models import SymbolKind
+
+_HUNK = re.compile(r"^@@ -\S+ \+(\d+)(?:,(\d+))? @@", re.M)
+_PROMPT_MIN_CHARS = 200
+
+
+def changed_lines(diff: str) -> set[int]:
+ """Post-image line numbers touched by a `-U0` diff of a single file."""
+ lines: set[int] = set()
+ for match in _HUNK.finditer(diff):
+ start = int(match.group(1))
+ count = int(match.group(2) or 1)
+ lines.update(range(start, start + max(count, 1)))
+ return lines
+
+
+def _binding_kind(value: ast.expr) -> SymbolKind:
+ if isinstance(value, ast.Constant):
+ if isinstance(value.value, str) and len(value.value) >= _PROMPT_MIN_CHARS:
+ return SymbolKind.PROMPT_TEXT
+ return SymbolKind.SCALAR_CONST
+ if isinstance(value, ast.Call):
+ func = ast.unparse(value.func)
+ if func in {"re.compile", "compile"}:
+ return SymbolKind.REGEX
+ return SymbolKind.COLLECTION
+ if isinstance(value, (ast.Tuple, ast.List, ast.Dict, ast.Set)):
+ return SymbolKind.COLLECTION
+ if isinstance(value, ast.JoinedStr):
+ return SymbolKind.PROMPT_TEXT
+ return SymbolKind.COLLECTION
+
+
+def symbol_map(source: str) -> dict[int, tuple[str, SymbolKind]]:
+ """line number -> (qualified symbol, kind).
+
+ Definitions are laid down widest-first so that narrower spans overwrite them and
+ the innermost enclosing scope wins.
+ """
+ try:
+ tree = ast.parse(source)
+ except SyntaxError:
+ return {}
+
+ spans: list[tuple[int, int, str, SymbolKind]] = []
+
+ def walk(node: ast.AST, prefix: str, in_class: bool) -> None:
+ for child in ast.iter_child_nodes(node):
+ if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+ qualified = f"{prefix}{child.name}"
+ if isinstance(child, ast.ClassDef):
+ kind = SymbolKind.CLASS
+ else:
+ kind = SymbolKind.METHOD if in_class else SymbolKind.FUNCTION
+ spans.append(
+ (
+ child.lineno,
+ getattr(child, "end_lineno", child.lineno),
+ qualified,
+ kind,
+ )
+ )
+ walk(child, f"{qualified}.", isinstance(child, ast.ClassDef))
+ else:
+ walk(child, prefix, in_class)
+
+ walk(tree, "", False)
+
+ # Module-level bindings: named, so tuning a constant is distinguishable from
+ # rewriting a system prompt.
+ for node in tree.body:
+ targets: list[str] = []
+ if isinstance(node, ast.Assign):
+ targets = [t.id for t in node.targets if isinstance(t, ast.Name)]
+ elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
+ targets = [node.target.id]
+ if not targets or node.value is None:
+ continue
+ spans.append(
+ (
+ node.lineno,
+ getattr(node, "end_lineno", node.lineno),
+ targets[0],
+ _binding_kind(node.value),
+ )
+ )
+
+ mapping: dict[int, tuple[str, SymbolKind]] = {}
+ for start, end, qualified, kind in sorted(spans, key=lambda s: s[1] - s[0], reverse=True):
+ for line in range(start, end + 1):
+ mapping[line] = (qualified, kind)
+ return mapping
+
+
+def scalar_value(source: str, name: str) -> str | None:
+ """Literal text of a module-level scalar binding, for before/after capture."""
+ try:
+ tree = ast.parse(source)
+ except SyntaxError:
+ return None
+ for node in tree.body:
+ targets: list[str] = []
+ if isinstance(node, ast.Assign):
+ targets = [t.id for t in node.targets if isinstance(t, ast.Name)]
+ elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
+ targets = [node.target.id]
+ if name in targets and node.value is not None:
+ try:
+ return ast.unparse(node.value)[:200]
+ except Exception:
+ return None
+ return None
diff --git a/vero/src/vero/interpret/models.py b/vero/src/vero/interpret/models.py
new file mode 100644
index 00000000..7c4ed213
--- /dev/null
+++ b/vero/src/vero/interpret/models.py
@@ -0,0 +1,161 @@
+"""Canonical schema for interpretability analysis.
+
+Every source adapter normalises into these types, so downstream code never sees a
+harbor path, a tarball, or a git object. Adding a second producer of optimization
+runs means writing an adapter, not touching anything below `artifacts/`.
+
+Identity is content-addressed throughout. `Edit.id` in particular is derived from the
+edit's own content rather than its position, so a cached label survives re-extraction,
+re-ordering, and unrelated edits appearing earlier in the same candidate.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+
+def content_id(*parts: str) -> str:
+ """Stable 16-hex identity over the given parts."""
+ digest = hashlib.sha256("\x00".join(parts).encode("utf-8", "replace"))
+ return digest.hexdigest()[:16]
+
+
+class SymbolKind(StrEnum):
+ """What kind of thing an edit landed in, resolved from the syntax tree."""
+
+ FUNCTION = "function"
+ METHOD = "method"
+ CLASS = "class"
+ PROMPT_TEXT = "prompt_text" # module-level str binding, long
+ SCALAR_CONST = "scalar_const" # module-level int/float/bool/short str
+ COLLECTION = "collection" # module-level tuple/list/dict/set
+ REGEX = "regex" # module-level re.compile
+ MODULE = "module" # module level, unattributed
+ NON_PYTHON = "non_python" # config, markdown, lockfiles
+
+
+class CellRef(BaseModel):
+ """One optimization run: a single (source, benchmark, cell) triple."""
+
+ source: str # adapter name, e.g. "harbor"
+ benchmark: str
+ cell: str
+ root: str # directory the adapter was pointed at
+ cell_dir: str # resolved during discovery, not recomputed
+ job_dir: str | None = None
+
+ @property
+ def key(self) -> str:
+ return f"{self.source}/{self.benchmark}/{self.cell}"
+
+
+class Candidate(BaseModel):
+ """One state the optimizer produced. Position 0 is the seed."""
+
+ sha: str
+ parent_sha: str | None = None
+ position: int
+ subject: str = ""
+ body: str = ""
+ files: list[str] = Field(default_factory=list)
+ insertions: int = 0
+ deletions: int = 0
+ tree_sha: str | None = None
+ is_seed: bool = False
+ is_shipped: bool = False
+
+
+class EvalRecord(BaseModel):
+ """One scoring of one candidate.
+
+ A candidate can be scored more than once on the same partition; those repeats are
+ the corpus's only direct measurement of evaluation noise, so they are kept as
+ separate records rather than collapsed into a per-partition map.
+ """
+
+ candidate_sha: str
+ partition: str
+ score: float
+ error_rate: float | None = None
+ selection_kind: str | None = None # "all" for a full partition, else a slice
+ n_attempts: int | None = None
+ started_at: str | None = None
+ finished_at: str | None = None
+
+
+class Edit(BaseModel):
+ """A symbol-scoped change: the unit of analysis.
+
+ Not the commit. A single candidate routinely bundles a bug fix with unrelated
+ tuning under one subject line, so labelling per candidate assigns one category to
+ several distinct modifications.
+ """
+
+ id: str
+ cell_key: str
+ candidate_sha: str
+ path: str
+ symbol: str # "" when unattributed
+ symbol_kind: SymbolKind
+ added: int = 0
+ removed: int = 0
+ before_value: str | None = None # scalar constants only
+ after_value: str | None = None
+ diff: str = "" # unified diff restricted to this symbol
+
+ @staticmethod
+ def make_id(cell_key: str, sha: str, path: str, symbol: str, diff: str) -> str:
+ return content_id(cell_key, sha, path, symbol, diff)
+
+
+class Trajectory(BaseModel):
+ """Everything known about one cell."""
+
+ ref: CellRef
+ candidates: list[Candidate] = Field(default_factory=list)
+ evaluations: list[EvalRecord] = Field(default_factory=list)
+ edits: list[Edit] = Field(default_factory=list)
+ reward: float | None = None
+ baseline_reward: float | None = None
+ error_rate: float | None = None
+ total_tokens: float | None = None
+
+ @property
+ def seed(self) -> Candidate | None:
+ return next((c for c in self.candidates if c.is_seed), None)
+
+ @property
+ def shipped(self) -> Candidate | None:
+ return next((c for c in self.candidates if c.is_shipped), None)
+
+ @property
+ def shipped_the_seed(self) -> bool:
+ """True when the shipped tree is byte-identical to the seed tree.
+
+ Decided on tree hashes, never on a commit message: messages saying "revert"
+ routinely carry surviving behavioural change, and messages saying nothing of
+ the sort are sometimes total reverts.
+ """
+ seed, shipped = self.seed, self.shipped
+ if not (seed and shipped and seed.tree_sha and shipped.tree_sha):
+ return False
+ return seed.tree_sha == shipped.tree_sha
+
+
+class Corpus(BaseModel):
+ """A collated set of trajectories, usually one analysis scope."""
+
+ trajectories: list[Trajectory] = Field(default_factory=list)
+
+ def by_benchmark(self) -> dict[str, list[Trajectory]]:
+ out: dict[str, list[Trajectory]] = {}
+ for t in self.trajectories:
+ out.setdefault(t.ref.benchmark, []).append(t)
+ return out
+
+ def edits(self):
+ for t in self.trajectories:
+ yield from t.edits
diff --git a/vero/tests/test_interpret_locus.py b/vero/tests/test_interpret_locus.py
new file mode 100644
index 00000000..5943eb50
--- /dev/null
+++ b/vero/tests/test_interpret_locus.py
@@ -0,0 +1,75 @@
+"""Locus resolution is the load-bearing deterministic step; test it directly.
+
+These cases are the ones that actually went wrong on real data: git's hunk header
+reports the enclosing class for a method, and module-level bindings collapse a system
+prompt, a tool table and a dozen tuning constants into one bucket.
+"""
+
+from __future__ import annotations
+
+from vero.interpret.edits.locus import changed_lines, scalar_value, symbol_map
+from vero.interpret.models import SymbolKind
+
+SOURCE = '''
+MAX_TURNS = 24
+SHELL_TIMEOUT_SEC = 150
+INSTRUCTIONS = """
+{}
+"""
+TOOLS = [{{"name": "run_shell"}}, {{"name": "submit"}}]
+_ANSWER_RE = re.compile(r"x")
+
+
+def helper(value):
+ return value
+
+
+class Agent:
+ def run(self):
+ return 1
+
+ def _run_shell(self, cmd):
+ return cmd
+'''.format("guidance " * 40)
+
+
+def test_method_resolves_to_method_not_class():
+ mapping = symbol_map(SOURCE)
+ line = next(
+ i for i, text in enumerate(SOURCE.splitlines(), 1) if "return cmd" in text
+ )
+ symbol, kind = mapping[line]
+ assert symbol == "Agent._run_shell"
+ assert kind is SymbolKind.METHOD
+
+
+def test_module_bindings_split_by_name_and_shape():
+ mapping = symbol_map(SOURCE)
+ got = {symbol: kind for symbol, kind in mapping.values()}
+ assert got["MAX_TURNS"] is SymbolKind.SCALAR_CONST
+ assert got["INSTRUCTIONS"] is SymbolKind.PROMPT_TEXT
+ assert got["TOOLS"] is SymbolKind.COLLECTION
+ assert got["_ANSWER_RE"] is SymbolKind.REGEX
+
+
+def test_plain_function_is_not_a_method():
+ mapping = symbol_map(SOURCE)
+ line = next(
+ i for i, text in enumerate(SOURCE.splitlines(), 1) if "return value" in text
+ )
+ assert mapping[line] == ("helper", SymbolKind.FUNCTION)
+
+
+def test_changed_lines_parses_zero_context_hunks():
+ diff = "@@ -1 +1 @@\n-a\n+b\n@@ -10,0 +11,3 @@\n+x\n+y\n+z\n"
+ assert changed_lines(diff) == {1, 11, 12, 13}
+
+
+def test_scalar_value_reads_the_literal():
+ assert scalar_value(SOURCE, "MAX_TURNS") == "24"
+ assert scalar_value(SOURCE, "SHELL_TIMEOUT_SEC") == "150"
+ assert scalar_value(SOURCE, "nonexistent") is None
+
+
+def test_unparsable_source_yields_no_map_rather_than_raising():
+ assert symbol_map("def broken(:\n") == {}
From 39e1a5ffe54589239f66372d9187d344b04b21b1 Mon Sep 17 00:00:00 2001
From: Varun Ursekar
Date: Sun, 2 Aug 2026 22:51:45 -0700
Subject: [PATCH 02/16] Add interpret CLI: extract, edits, symbols
Stages write JSONL and are separately resumable. Extraction over the corpus is
minutes of gzip decompression, so a cheap downstream step must never force it to
be redone -- hence three commands rather than one pipeline.
`symbols` exists to design the label taxonomy from evidence: the root `role`
vocabulary should come from the symbol distribution the optimizers actually
touched, not from guessing before looking.
Co-Authored-By: Claude Opus 5 (1M context)
---
vero/src/vero/interpret/cli.py | 132 +++++++++++++++++++++++++++++++++
1 file changed, 132 insertions(+)
create mode 100644 vero/src/vero/interpret/cli.py
diff --git a/vero/src/vero/interpret/cli.py b/vero/src/vero/interpret/cli.py
new file mode 100644
index 00000000..6f3e5e53
--- /dev/null
+++ b/vero/src/vero/interpret/cli.py
@@ -0,0 +1,132 @@
+"""`vero interpret` — extract, decompose, and report on optimization runs.
+
+Stages write JSONL and are independently resumable, so a long extraction is never
+repeated to re-run a cheap downstream step.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from pathlib import Path
+
+import click
+
+from vero.interpret.artifacts.harbor import build as build_harbor
+from vero.interpret.artifacts.harbor import session as session_mod
+from vero.interpret.artifacts.harbor.repo import CandidateRepo
+from vero.interpret.cache import Cache
+from vero.interpret.edits import decompose
+from vero.interpret.models import CellRef, Trajectory
+
+DEFAULT_CACHE = Path.home() / ".cache" / "vero-interpret"
+
+
+def _load_scope(path: Path | None) -> dict[str, set[str]] | None:
+ """{benchmark: {cell, ...}} restricting which runs are in scope."""
+ if path is None:
+ return None
+ raw = json.loads(Path(path).read_text())
+ return {bench: set(cells) for bench, cells in raw.items()}
+
+
+def _in_scope(ref: CellRef, scope: dict[str, set[str]] | None) -> bool:
+ if scope is None:
+ return True
+ return ref.cell in scope.get(ref.benchmark, set())
+
+
+@click.group()
+def main() -> None:
+ """Interpretability analysis over optimization runs."""
+
+
+@main.command()
+@click.option("--runs", "roots", multiple=True, required=True, type=click.Path(path_type=Path),
+ help="Directory to search for runs. Repeatable.")
+@click.option("--cells-file", type=click.Path(path_type=Path),
+ help='JSON {"benchmark": ["cell", ...]} restricting scope.')
+@click.option("--cache-dir", type=click.Path(path_type=Path), default=DEFAULT_CACHE)
+@click.option("--out", type=click.Path(path_type=Path), default=Path("trajectories.jsonl"))
+@click.option("--refresh", is_flag=True, help="Bypass the cache and re-extract.")
+def extract(roots, cells_file, cache_dir, out, refresh) -> None:
+ """Canonicalise runs into trajectories."""
+ cache = Cache(cache_dir, "harbor-unpack", refresh=refresh)
+ adapter = build_harbor(cache)
+ scope = _load_scope(cells_file)
+
+ refs = [r for r in adapter.discover(roots) if _in_scope(r, scope)]
+ click.echo(f"{len(refs)} cells in scope")
+
+ out = Path(out)
+ with out.open("w") as fh:
+ for i, ref in enumerate(refs, 1):
+ traj = adapter.load(ref)
+ fh.write(traj.model_dump_json() + "\n")
+ click.echo(
+ f" [{i}/{len(refs)}] {ref.key} "
+ f"cands={len(traj.candidates)} evals={len(traj.evaluations)}"
+ )
+ click.echo(f"wrote {out} ({cache.stats()})")
+
+
+@main.command()
+@click.option("--in", "src", type=click.Path(path_type=Path), default=Path("trajectories.jsonl"))
+@click.option("--cache-dir", type=click.Path(path_type=Path), default=DEFAULT_CACHE)
+@click.option("--out", type=click.Path(path_type=Path), default=Path("edits.jsonl"))
+def edits(src, cache_dir, out) -> None:
+ """Split every candidate into symbol-scoped edits."""
+ cache = Cache(cache_dir, "harbor-unpack")
+ total = 0
+ with Path(out).open("w") as fh:
+ for line in Path(src).read_text().splitlines():
+ traj = Trajectory.model_validate_json(line)
+ archive = next(
+ Path(traj.ref.cell_dir).glob("jobs/*/task__*/verifier/session.tar.gz"), None
+ )
+ if archive is None:
+ continue
+ root = session_mod.unpack(archive, cache)
+ if root is None:
+ continue
+ repo_dir = session_mod.find_repo(root)
+ if repo_dir is None:
+ continue
+ repo = CandidateRepo(repo_dir)
+ n = 0
+ for cand in traj.candidates:
+ for edit in decompose(repo, traj.ref.key, cand):
+ fh.write(edit.model_dump_json() + "\n")
+ n += 1
+ total += n
+ click.echo(f" {traj.ref.key}: {n} edits")
+ click.echo(f"wrote {out} ({total} edits)")
+
+
+@main.command()
+@click.option("--in", "src", type=click.Path(path_type=Path), default=Path("edits.jsonl"))
+@click.option("--top", default=40, help="Symbols to show per benchmark.")
+def symbols(src, top) -> None:
+ """Symbol frequency, the input to designing a role vocabulary."""
+ per_bench: dict[str, Counter] = {}
+ kinds: Counter = Counter()
+ cells: dict[str, set[str]] = {}
+ for line in Path(src).read_text().splitlines():
+ e = json.loads(line)
+ bench = e["cell_key"].split("/")[1]
+ name = f"{e['path'].split('/')[-1]}::{e['symbol']}"
+ per_bench.setdefault(bench, Counter())
+ cells.setdefault(f"{bench}|{name}", set()).add(e["cell_key"])
+ per_bench[bench][name] += 1
+ kinds[e["symbol_kind"]] += 1
+
+ for bench, counter in sorted(per_bench.items()):
+ click.echo(f"\n=== {bench}: {sum(counter.values())} edits, {len(counter)} symbols")
+ for name, n in counter.most_common(top):
+ ncells = len(cells[f"{bench}|{name}"])
+ click.echo(f" {n:>4} edits {ncells:>3} cells {name}")
+ click.echo("\nsymbol kinds: " + ", ".join(f"{k} {v}" for k, v in kinds.most_common()))
+
+
+if __name__ == "__main__":
+ main()
From b6976e380b8aa475160c4d65ecda7d3cd0d51387 Mon Sep 17 00:00:00 2001
From: Varun Ursekar
Date: Sun, 2 Aug 2026 22:59:03 -0700
Subject: [PATCH 03/16] Add interpret taxonomy derived from the corpus, not
chosen in advance
Ran decomposition over all 100 in-scope cells first: 3,876 symbol-scoped edits from
443 candidates. The same handful of targets dominates in every benchmark -- the
instruction prompt, the main loop, the turn budget, the tool table -- so the role
roots here are what the corpus contains rather than what seemed plausible.
Two things the distribution surfaced that guessing would have missed. Optimizers
edit their own test suite in 6-14 cells per benchmark, which needs its own role
instead of falling into "other" (627 edits, the largest single role). And 19 of 20
gaia cells touch `.version`, which is bookkeeping rather than modification -- a
standing reminder that touching a symbol is not the same as changing behaviour,
and why the action facet has to carry that weight.
Hints assign 53% of edits a role with no model involved, decided by path, then by
symbol kind, then by name. Kind before name is what catches REVIEW_INSTRUCTIONS
and every other prompt binding without enumerating names. Tune direction is
likewise derived where both values parse: 135 up, 78 down. What reaches the model
is the genuinely ambiguous remainder -- custom methods like _solve, _dispatch,
_force_final -- which is a smaller and far more checkable job than labelling
everything, and a hint disagreeing with a model label is a bug findable without
reading anything.
TAXONOMY_VERSION is part of the label cache key, so revising this re-labels
without re-extracting.
Co-Authored-By: Claude Opus 5 (1M context)
---
vero/src/vero/interpret/labeling/__init__.py | 0
vero/src/vero/interpret/labeling/taxonomy.py | 131 +++++++++++++++++++
2 files changed, 131 insertions(+)
create mode 100644 vero/src/vero/interpret/labeling/__init__.py
create mode 100644 vero/src/vero/interpret/labeling/taxonomy.py
diff --git a/vero/src/vero/interpret/labeling/__init__.py b/vero/src/vero/interpret/labeling/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/vero/src/vero/interpret/labeling/taxonomy.py b/vero/src/vero/interpret/labeling/taxonomy.py
new file mode 100644
index 00000000..28363fe9
--- /dev/null
+++ b/vero/src/vero/interpret/labeling/taxonomy.py
@@ -0,0 +1,131 @@
+"""Facets an edit is labelled on, and the deterministic part of assigning them.
+
+The vocabulary is derived from the symbol distribution the optimizers actually
+touched across 100 runs, not chosen in advance: the same handful of targets dominate
+in every benchmark, so the roots below are what the corpus contains rather than what
+seemed plausible.
+
+Three facets are model-assigned (`action`, `role` where a hint does not fire,
+`provenance`); the rest are derived. Keeping the model's job to semantic judgement is
+what makes the output checkable — a role hint that disagrees with a model label is a
+bug you can find without reading anything.
+
+Bump TAXONOMY_VERSION on any change here. It is part of the label cache key, so a
+revision re-labels without re-extracting.
+"""
+
+from __future__ import annotations
+
+import re
+from enum import StrEnum
+
+TAXONOMY_VERSION = "1"
+
+
+class Role(StrEnum):
+ """What part of the harness an edit targets. Closed and benchmark-agnostic."""
+
+ PROMPT = "prompt" # system/instruction text
+ CONTROL_LOOP = "control_loop" # the agent's main turn loop
+ TOOL_SURFACE = "tool_surface" # tool declarations shown to the model
+ TOOL_IMPL = "tool_impl" # a specific tool's implementation
+ SUBMISSION = "submission" # emitting/parsing the final answer
+ MODEL_CLIENT = "model_client" # completion params, retries, timeouts
+ BUDGET_TURNS = "budget_turns"
+ BUDGET_OUTPUT = "budget_output" # tool-output truncation caps
+ BUDGET_WALLCLOCK = "budget_wallclock" # deadlines that stop the loop
+ CONTEXT_MGMT = "context_mgmt" # compaction, history pruning
+ RETRIEVAL = "retrieval" # search/index parameters and strategy
+ ENV_SETUP = "env_setup" # setup() work, package installation
+ INITIALIZATION = "initialization" # __init__, wiring
+ TESTS = "tests" # the candidate's own test suite
+ METADATA = "metadata" # version/name bookkeeping
+ OTHER = "other"
+
+
+class Action(StrEnum):
+ """What was done to it."""
+
+ FIX = "fix" # repairs a defect
+ ADD = "add" # new capability or code path
+ REMOVE = "remove" # deletes a capability or code path
+ TUNE = "tune" # changes a value, no structural change
+ RESTRUCTURE = "restructure" # same behaviour, different shape
+ REWORD = "reword" # instruction text changed, intent preserved
+ REVERT = "revert" # undoes the optimizer's own earlier edit
+ COSMETIC = "cosmetic" # cannot change behaviour
+
+
+class Provenance(StrEnum):
+ """For fixes: whose defect was it?
+
+ Distinguishing these matters. An optimizer repairing the seed is doing the task;
+ an optimizer repairing damage it caused two candidates ago is paying down its own
+ debt, and several cells in this corpus spent most of their budget that way.
+ """
+
+ SEED = "seed"
+ OWN = "own"
+ UNKNOWN = "unknown"
+
+
+class Direction(StrEnum):
+ """For tunes. Derived from before/after values where both are numeric."""
+
+ UP = "up"
+ DOWN = "down"
+ UNCHANGED = "unchanged"
+ NA = "na"
+
+
+# Symbol-name hints. These fire on the leaf symbol and are exact enough to skip the
+# model entirely; anything unmatched goes to the labeller. Ordered, first match wins.
+_ROLE_HINTS: list[tuple[re.Pattern[str], Role]] = [
+ (re.compile(r"(PROMPT|INSTRUCTIONS?|GUIDANCE|PLAYBOOK)", re.I), Role.PROMPT),
+ (re.compile(r"^(MAX_TURNS|MAX_STEPS|TURN_BUDGET|STEP_BUDGET|MAX_ITER\w*)$"), Role.BUDGET_TURNS),
+ (re.compile(r"^(MAX_TOOL_OUTPUT\w*|MAX_OUTPUT\w*|MAX_CHARS|MAX_BYTES)$"), Role.BUDGET_OUTPUT),
+ (re.compile(r"\w*(DEADLINE|TIME_BUDGET|WALL_CLOCK)\w*"), Role.BUDGET_WALLCLOCK),
+ (re.compile(r"^(MAX_HISTORY\w*|MAX_CONTEXT\w*|.*COMPACT.*)$", re.I), Role.CONTEXT_MGMT),
+ (re.compile(r"^TOOLS$"), Role.TOOL_SURFACE),
+ (re.compile(r"\.run$"), Role.CONTROL_LOOP),
+ (re.compile(r"\.(_submit|submit\w*|_answer_payload|_extract\w*answer\w*)$", re.I), Role.SUBMISSION),
+ (re.compile(r"^(_answer_payload|_scale_variants|_format_number)$"), Role.SUBMISSION),
+ (re.compile(r"\.(_completion_kwargs|_complete|_create|_chat)$"), Role.MODEL_CLIENT),
+ (re.compile(r"\.(_run_shell|_exec|_run_python|_index_command|_open\w*|_search\w*)$"), Role.TOOL_IMPL),
+ (re.compile(r"\.setup$"), Role.ENV_SETUP),
+ (re.compile(r"\.__init__$"), Role.INITIALIZATION),
+ (re.compile(r"\.(version|name)$"), Role.METADATA),
+]
+
+
+def role_hint(path: str, symbol: str, symbol_kind: str | None = None) -> Role | None:
+ """Deterministic role where path, kind or name settles it, else None.
+
+ Order matters. Path wins first: an edit inside the candidate's own test suite is
+ a test edit whatever it touches. Kind wins next, because a long module-level
+ string binding is a prompt regardless of what it is called — which is how
+ `REVIEW_INSTRUCTIONS` and friends get caught without enumerating names.
+ """
+ if "test" in path.rsplit("/", 1)[-1]:
+ return Role.TESTS
+ if symbol_kind == "prompt_text":
+ return Role.PROMPT
+ for pattern, role in _ROLE_HINTS:
+ if pattern.search(symbol):
+ return role
+ return None
+
+
+_NUMERIC = re.compile(r"-?\d+(?:\.\d+)?")
+
+
+def direction_of(before: str | None, after: str | None) -> Direction:
+ """Numeric direction of a tune, where both sides parse."""
+ if before is None or after is None:
+ return Direction.NA
+ if before == after:
+ return Direction.UNCHANGED
+ b, a = _NUMERIC.search(before), _NUMERIC.search(after)
+ if not (b and a):
+ return Direction.NA
+ return Direction.UP if float(a.group()) > float(b.group()) else Direction.DOWN
From 7901b56099e37bc93d6ea4c8164faec17d854999 Mon Sep 17 00:00:00 2001
From: Varun Ursekar
Date: Sun, 2 Aug 2026 23:04:36 -0700
Subject: [PATCH 04/16] Fix two silent edit-dropping bugs found by auditing
zero-edit candidates
Nineteen non-seed candidates decomposed to nothing. Both causes were mine and both
failed quietly, which is the part worth fixing: neither raised, so the corpus-wide
distribution would simply have been wrong.
Job selection disagreed between stages. `extract` took the last finalization under
a cell, `edits` took the first glob match. A cell re-run in place has more than one
job directory, so `edits` opened a repository that did not contain the other
stage's commits and every diff came back empty -- silently discarding an entire
cell (swe-atlas-qna gptoss-claude-opus-5-claude-code-r2, six substantial
candidates, now 75 edits). Both stages now share `latest_verifier_dir`.
`.gitignore` was in the skip list beside `__pycache__`, but the two are not alike.
Thirteen candidates changed nothing else, and six of those were the SHIPPED
candidate -- a cell whose final answer is a bytecode-ignore rule is a finding, not
noise, and filtering it made those cells look as though they had shipped their
previous real change. Compiled artifacts stay filtered; `.gitignore` now surfaces
as an inert non-Python edit for the labeller to mark cosmetic.
3,876 -> 3,986 edits. One candidate still decomposes to nothing and correctly so:
gaia-shell kimi-k3-opencode-r2 1e7141bac998 is +0/-0 with no files outside
__pycache__, already known to ship a tree identical to a candidate three earlier.
Co-Authored-By: Claude Opus 5 (1M context)
---
.../interpret/artifacts/harbor/adapter.py | 20 ++++++++++++++-----
vero/src/vero/interpret/cli.py | 10 ++++++----
vero/src/vero/interpret/edits/decompose.py | 4 +++-
3 files changed, 24 insertions(+), 10 deletions(-)
diff --git a/vero/src/vero/interpret/artifacts/harbor/adapter.py b/vero/src/vero/interpret/artifacts/harbor/adapter.py
index 69930f00..449cde9d 100644
--- a/vero/src/vero/interpret/artifacts/harbor/adapter.py
+++ b/vero/src/vero/interpret/artifacts/harbor/adapter.py
@@ -29,6 +29,18 @@
DEFAULT_MAX_ERROR_RATE = 0.1
+def latest_verifier_dir(cell_dir: Path) -> Path | None:
+ """The verifier directory of the most recent job for this cell.
+
+ A cell re-run in place leaves earlier attempts behind. Every stage must agree on
+ which one is authoritative: picking the first glob match in one stage and the last
+ in another opens a repository that does not contain the other stage's commits, and
+ the diffs come back empty rather than erroring, silently dropping the whole cell.
+ """
+ finals = sorted(cell_dir.glob("jobs/*/task__*/verifier/finalization.json"))
+ return finals[-1].parent if finals else None
+
+
class HarborAdapter:
name = "harbor"
@@ -68,13 +80,11 @@ def discover(self, roots: Iterable[Path]) -> list[CellRef]:
# -- loading --------------------------------------------------------------
def load(self, ref: CellRef) -> Trajectory:
- cell_dir = Path(ref.cell_dir)
- finals = sorted(cell_dir.glob("jobs/*/task__*/verifier/finalization.json"))
- if not finals:
+ verifier = latest_verifier_dir(Path(ref.cell_dir))
+ if verifier is None:
return Trajectory(ref=ref)
- # Last job wins: a cell re-run in place leaves the earlier attempt behind.
- final_path = finals[-1]
+ final_path = verifier / "finalization.json"
final = json.loads(final_path.read_text())
metrics = (final.get("reward_metrics") or {}).get("reward", {}) or {}
shipped_sha = ((final.get("candidate") or {}).get("id") or "")
diff --git a/vero/src/vero/interpret/cli.py b/vero/src/vero/interpret/cli.py
index 6f3e5e53..e5f33996 100644
--- a/vero/src/vero/interpret/cli.py
+++ b/vero/src/vero/interpret/cli.py
@@ -13,6 +13,7 @@
import click
from vero.interpret.artifacts.harbor import build as build_harbor
+from vero.interpret.artifacts.harbor.adapter import latest_verifier_dir
from vero.interpret.artifacts.harbor import session as session_mod
from vero.interpret.artifacts.harbor.repo import CandidateRepo
from vero.interpret.cache import Cache
@@ -81,10 +82,11 @@ def edits(src, cache_dir, out) -> None:
with Path(out).open("w") as fh:
for line in Path(src).read_text().splitlines():
traj = Trajectory.model_validate_json(line)
- archive = next(
- Path(traj.ref.cell_dir).glob("jobs/*/task__*/verifier/session.tar.gz"), None
- )
- if archive is None:
+ verifier = latest_verifier_dir(Path(traj.ref.cell_dir))
+ if verifier is None:
+ continue
+ archive = verifier / "session.tar.gz"
+ if not archive.is_file():
continue
root = session_mod.unpack(archive, cache)
if root is None:
diff --git a/vero/src/vero/interpret/edits/decompose.py b/vero/src/vero/interpret/edits/decompose.py
index 097d8212..68ef4d0e 100644
--- a/vero/src/vero/interpret/edits/decompose.py
+++ b/vero/src/vero/interpret/edits/decompose.py
@@ -19,7 +19,9 @@
from vero.interpret.models import Candidate, Edit, SymbolKind
_HUNK_SPLIT = re.compile(r"^(@@ .*?@@.*)$", re.M)
-_SKIP = ("__pycache__", ".gitignore")
+# __pycache__ is compiled noise. .gitignore is NOT skipped: several cells shipped a
+# candidate whose only change was one, and "shipped something inert" is a finding.
+_SKIP = ("__pycache__",)
def _per_symbol_diff(diff: str, keep: set[int]) -> str:
From 9fd0e325203838c90362452caa4c8a5e9e58b3fa Mon Sep 17 00:00:00 2001
From: Varun Ursekar
Date: Sun, 2 Aug 2026 23:09:17 -0700
Subject: [PATCH 05/16] Add labelling: async client, facet labeler, resumable
cache
Validated live against the gateway on 25 edits: structured outputs honoured,
re-running the same command makes zero calls and hits cache 100%.
Roles a deterministic hint settles are not taken from the model, but a sampled
fraction of hinted edits is still sent and both readings are kept when they
disagree -- agreement measured rather than assumed. The dry run already produced
one: hint said budget_wallclock, model said budget_turns, and the model's own
mechanism ("sets the verification-phase deadline constant") shows the hint was
right. That is only visible because the disagreement is recorded instead of
resolved silently.
The prompt supplies the commit subject as a claim to be checked, not as the
answer. Subjects in this corpus routinely misdescribe their diffs, so anchoring
on them would launder the error into the labels.
Retries are handled here rather than by the SDK so backoff is uniform and
jittered; thousands of concurrent labels retrying in lockstep after a rate-limit
burst just reproduce the burst. A malformed response is retried, never parsed
leniently -- a label that degrades to a default looks like evidence.
Base URL is rstripped in Settings.from_env: the gateway's OPENAI_BASE_URL ends in
"/v1/", and the resulting double slash returns a flat 403 that reads like a
permissions failure and is not one.
Co-Authored-By: Claude Opus 5 (1M context)
---
vero/src/vero/interpret/cli.py | 60 ++++++-
vero/src/vero/interpret/config.py | 76 +++++++++
vero/src/vero/interpret/labeling/client.py | 85 ++++++++++
vero/src/vero/interpret/labeling/labeler.py | 164 ++++++++++++++++++++
vero/src/vero/interpret/models.py | 20 +++
5 files changed, 404 insertions(+), 1 deletion(-)
create mode 100644 vero/src/vero/interpret/config.py
create mode 100644 vero/src/vero/interpret/labeling/client.py
create mode 100644 vero/src/vero/interpret/labeling/labeler.py
diff --git a/vero/src/vero/interpret/cli.py b/vero/src/vero/interpret/cli.py
index e5f33996..fd3a8cf5 100644
--- a/vero/src/vero/interpret/cli.py
+++ b/vero/src/vero/interpret/cli.py
@@ -18,7 +18,7 @@
from vero.interpret.artifacts.harbor.repo import CandidateRepo
from vero.interpret.cache import Cache
from vero.interpret.edits import decompose
-from vero.interpret.models import CellRef, Trajectory
+from vero.interpret.models import CellRef, Edit, Trajectory
DEFAULT_CACHE = Path.home() / ".cache" / "vero-interpret"
@@ -105,6 +105,64 @@ def edits(src, cache_dir, out) -> None:
click.echo(f"wrote {out} ({total} edits)")
+@main.command()
+@click.option("--edits-file", type=click.Path(path_type=Path), default=Path("edits.jsonl"))
+@click.option("--trajectories", type=click.Path(path_type=Path),
+ default=Path("trajectories.jsonl"))
+@click.option("--model", default=None, help="Defaults to a cheap model.")
+@click.option("--concurrency", default=16)
+@click.option("--audit-rate", default=0.15,
+ help="Fraction of hinted edits also sent to the model, to measure "
+ "hint/model agreement rather than assume it.")
+@click.option("--limit", default=0, help="Label only the first N edits (a dry run).")
+@click.option("--cache-dir", type=click.Path(path_type=Path), default=DEFAULT_CACHE)
+@click.option("--out", type=click.Path(path_type=Path), default=Path("labels.jsonl"))
+def label(edits_file, trajectories, model, concurrency, audit_rate, limit,
+ cache_dir, out) -> None:
+ """Assign facets to edits. Cached and resumable; re-running costs nothing."""
+ import asyncio as _asyncio
+
+ from vero.interpret.config import Settings
+ from vero.interpret.labeling.client import AsyncLLM
+ from vero.interpret.labeling.labeler import Labeler
+
+ subjects: dict[str, str] = {}
+ if Path(trajectories).is_file():
+ for line in Path(trajectories).read_text().splitlines():
+ traj = Trajectory.model_validate_json(line)
+ for cand in traj.candidates:
+ subjects[cand.sha] = cand.subject
+
+ rows = [Edit.model_validate_json(l) for l in Path(edits_file).read_text().splitlines()]
+ if limit:
+ rows = rows[:limit]
+ click.echo(f"labelling {len(rows)} edits with {model or 'default model'}")
+
+ settings = Settings.from_env(model=model, concurrency=concurrency,
+ cache_dir=Path(cache_dir))
+ if not settings.api_key:
+ raise click.ClickException(
+ "no OPENAI_API_KEY found; put it in .env or secrets.env"
+ )
+
+ async def run():
+ llm = AsyncLLM(settings)
+ labeler = Labeler(llm, Cache(Path(cache_dir), "labels"), audit_rate=audit_rate)
+ try:
+ return await labeler.label_all(
+ [(e, subjects.get(e.candidate_sha, "")) for e in rows],
+ progress=lambda i, n: click.echo(f" {i}/{n}"),
+ ), labeler
+ finally:
+ await llm.close()
+
+ labels, labeler = _asyncio.run(run())
+ with Path(out).open("w") as fh:
+ for lab in labels:
+ fh.write(lab.model_dump_json() + "\n")
+ click.echo(f"wrote {out} ({len(labels)} labels) {labeler.stats()}")
+
+
@main.command()
@click.option("--in", "src", type=click.Path(path_type=Path), default=Path("edits.jsonl"))
@click.option("--top", default=40, help="Symbols to show per benchmark.")
diff --git a/vero/src/vero/interpret/config.py b/vero/src/vero/interpret/config.py
new file mode 100644
index 00000000..9ba21642
--- /dev/null
+++ b/vero/src/vero/interpret/config.py
@@ -0,0 +1,76 @@
+"""Settings and secret loading.
+
+Reads `KEY=VALUE` files, which is the convention already in use here (`secrets.env`,
+`eval.secrets.env`) as well as the usual `.env`. A twenty-line parser covers both and
+avoids adding a dependency for it.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from pydantic import BaseModel
+
+DEFAULT_CACHE = Path.home() / ".cache" / "vero-interpret"
+DEFAULT_MODEL = "gpt-5.4-mini"
+
+
+def load_env_file(path: Path, *, override: bool = False) -> dict[str, str]:
+ """Parse a KEY=VALUE file into the environment. Returns what it set."""
+ if not path.is_file():
+ return {}
+ loaded: dict[str, str] = {}
+ for raw in path.read_text().splitlines():
+ line = raw.strip()
+ if not line or line.startswith("#") or "=" not in line:
+ continue
+ if line.startswith("export "):
+ line = line[len("export "):]
+ key, _, value = line.partition("=")
+ key, value = key.strip(), value.strip().strip('"').strip("'")
+ if not key:
+ continue
+ if override or key not in os.environ:
+ os.environ[key] = value
+ loaded[key] = value
+ return loaded
+
+
+def load_secrets(paths: list[Path] | None = None) -> None:
+ """Load the first secrets file that exists, plus `.env` if present."""
+ candidates = paths or [
+ Path(".env"),
+ Path("secrets.env"),
+ Path("vero/secrets.env"),
+ ]
+ for path in candidates:
+ if path.is_file():
+ load_env_file(path)
+
+
+class Settings(BaseModel):
+ """Everything the labelling stage needs to run."""
+
+ model: str = DEFAULT_MODEL
+ api_key: str | None = None
+ base_url: str | None = None
+ concurrency: int = 16
+ max_retries: int = 4
+ request_timeout: float = 120.0
+ cache_dir: Path = DEFAULT_CACHE
+
+ @classmethod
+ def from_env(cls, **overrides) -> "Settings":
+ load_secrets()
+ base = os.environ.get("OPENAI_BASE_URL")
+ values = {
+ "api_key": os.environ.get("OPENAI_API_KEY"),
+ # The gateway's OPENAI_BASE_URL ends in "/v1/" here. The OpenAI client
+ # appends "/chat/completions", and the resulting double slash returns a
+ # flat 403 "This route is not publicly accessible" that reads like a
+ # permissions problem and is not one. Strip it once, here.
+ "base_url": base.rstrip("/") if base else None,
+ }
+ values.update({k: v for k, v in overrides.items() if v is not None})
+ return cls(**values)
diff --git a/vero/src/vero/interpret/labeling/client.py b/vero/src/vero/interpret/labeling/client.py
new file mode 100644
index 00000000..5217a0e2
--- /dev/null
+++ b/vero/src/vero/interpret/labeling/client.py
@@ -0,0 +1,85 @@
+"""Bounded-concurrency async client for structured labelling.
+
+Deliberately thin. Labelling is thousands of small independent calls against a cheap
+model, so what matters is that concurrency is capped, transient failures are retried,
+and a malformed response is retried rather than parsed leniently — a label that
+silently degrades to a default is worse than one that is missing, because it looks
+like evidence.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import random
+from typing import Any
+
+from vero.interpret.config import Settings
+
+
+class LLMError(RuntimeError):
+ pass
+
+
+class AsyncLLM:
+ def __init__(self, settings: Settings) -> None:
+ try:
+ from openai import AsyncOpenAI
+ except ImportError as exc: # pragma: no cover - dependency guard
+ raise LLMError(
+ "the interpret extra is required: uv sync --extra interpret"
+ ) from exc
+
+ self.settings = settings
+ self._client = AsyncOpenAI(
+ api_key=settings.api_key,
+ base_url=settings.base_url,
+ timeout=settings.request_timeout,
+ max_retries=0, # retried here, so backoff is visible and uniform
+ )
+ self._sem = asyncio.Semaphore(settings.concurrency)
+ self.calls = 0
+ self.retries = 0
+
+ async def json_call(
+ self,
+ system: str,
+ user: str,
+ schema: dict[str, Any],
+ schema_name: str = "label",
+ ) -> dict[str, Any]:
+ """One structured call, retried on transport error and on schema violation."""
+ last: Exception | None = None
+ for attempt in range(self.settings.max_retries):
+ try:
+ async with self._sem:
+ self.calls += 1
+ response = await self._client.chat.completions.create(
+ model=self.settings.model,
+ messages=[
+ {"role": "system", "content": system},
+ {"role": "user", "content": user},
+ ],
+ response_format={
+ "type": "json_schema",
+ "json_schema": {
+ "name": schema_name,
+ "schema": schema,
+ "strict": True,
+ },
+ },
+ )
+ content = response.choices[0].message.content or ""
+ return json.loads(content)
+ except Exception as exc: # noqa: BLE001 - retry policy is uniform
+ last = exc
+ self.retries += 1
+ if attempt == self.settings.max_retries - 1:
+ break
+ # Jittered backoff: these run thousands-wide, and synchronised
+ # retries after a rate-limit burst just reproduce the burst.
+ await asyncio.sleep((2**attempt) + random.random())
+ raise LLMError(f"call failed after {self.settings.max_retries} attempts: {last}")
+
+ async def close(self) -> None:
+ await self._client.close()
diff --git a/vero/src/vero/interpret/labeling/labeler.py b/vero/src/vero/interpret/labeling/labeler.py
new file mode 100644
index 00000000..0c99fe75
--- /dev/null
+++ b/vero/src/vero/interpret/labeling/labeler.py
@@ -0,0 +1,164 @@
+"""Assign facets to edits: cached, resumable, and mostly not the model's job.
+
+Roles that a deterministic hint settles are not sent to the model at all, but the
+model is still asked for a role on a sample of hinted edits so the two can be
+compared. Agreement measured beats agreement assumed, and a hint that quietly
+disagrees with every model reading is a bug in the hint.
+
+The prompt shows the diff and withholds nothing except the commit message's
+authority: subjects in this corpus routinely misdescribe their diffs — one reading
+"Extend research and normalize wrapped answers" deletes an entire audit pass — so
+the message is supplied as a claim to be checked, not as the answer.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import random
+from typing import Iterable
+
+from vero.interpret.cache import Cache, key_of
+from vero.interpret.labeling.client import AsyncLLM, LLMError
+from vero.interpret.labeling.taxonomy import (
+ TAXONOMY_VERSION,
+ Action,
+ Direction,
+ Provenance,
+ Role,
+ direction_of,
+ role_hint,
+)
+from vero.interpret.models import Edit, EditLabel
+
+PROMPT_VERSION = "1"
+
+SYSTEM = """You classify individual edits made by an AI agent that was told to improve \
+another agent's harness.
+
+You are shown ONE edit: a diff restricted to a single symbol (a function, method, or \
+module-level binding). Classify only that edit, not the whole commit.
+
+The commit subject is provided for context but is frequently wrong: it may describe \
+work that is not in this diff, omit changes that are, or claim a revert while leaving \
+behaviour in place. Trust the diff. Where they disagree, say so in `mechanism`.
+
+Judge only what the code does. Do not speculate about whether it improved the score."""
+
+_SCHEMA = {
+ "type": "object",
+ "additionalProperties": False,
+ "required": ["action", "role", "provenance", "mechanism", "confidence"],
+ "properties": {
+ "action": {"type": "string", "enum": [a.value for a in Action]},
+ "role": {"type": "string", "enum": [r.value for r in Role]},
+ "provenance": {"type": "string", "enum": [p.value for p in Provenance]},
+ "mechanism": {
+ "type": "string",
+ "description": "One sentence: what this edit actually does.",
+ },
+ "confidence": {"type": "number", "minimum": 0.0, "maximum": 1.0},
+ },
+}
+
+
+def _user_prompt(edit: Edit, subject: str) -> str:
+ value = ""
+ if edit.before_value is not None or edit.after_value is not None:
+ value = f"\nvalue: {edit.before_value} -> {edit.after_value}"
+ return (
+ f"file: {edit.path}\n"
+ f"symbol: {edit.symbol} ({edit.symbol_kind.value})\n"
+ f"lines: +{edit.added} -{edit.removed}{value}\n"
+ f"commit subject (may be inaccurate): {subject}\n\n"
+ f"diff:\n{edit.diff[:6000]}"
+ )
+
+
+def cache_key(edit: Edit, model: str) -> str:
+ return key_of(edit.id, model, PROMPT_VERSION, TAXONOMY_VERSION)
+
+
+class Labeler:
+ def __init__(
+ self,
+ llm: AsyncLLM,
+ cache: Cache,
+ *,
+ audit_rate: float = 0.15,
+ seed: int = 0,
+ ) -> None:
+ self.llm = llm
+ self.cache = cache
+ self.audit_rate = audit_rate
+ self._rng = random.Random(seed)
+ self.skipped_by_hint = 0
+ self.audited = 0
+ self.failed = 0
+
+ async def label(self, edit: Edit, subject: str = "") -> EditLabel | None:
+ key = cache_key(edit, self.llm.settings.model)
+ if (cached := self.cache.get_json(key)) is not None:
+ return EditLabel.model_validate(cached)
+
+ hint = role_hint(edit.path, edit.symbol, edit.symbol_kind.value)
+ # A hinted role still needs an action, so the call happens either way; the
+ # hint decides whether the model's role is authoritative or merely audited.
+ audit = hint is not None and self._rng.random() < self.audit_rate
+ if hint is not None and not audit:
+ self.skipped_by_hint += 1
+ if audit:
+ self.audited += 1
+
+ try:
+ raw = await self.llm.json_call(SYSTEM, _user_prompt(edit, subject), _SCHEMA)
+ except LLMError:
+ self.failed += 1
+ return None
+
+ role = hint.value if hint is not None else raw["role"]
+ label = EditLabel(
+ edit_id=edit.id,
+ action=raw["action"],
+ role=role,
+ provenance=raw.get("provenance", Provenance.UNKNOWN.value),
+ direction=direction_of(edit.before_value, edit.after_value).value,
+ mechanism=raw.get("mechanism", "")[:300],
+ confidence=float(raw.get("confidence", 0.0)),
+ hinted=hint is not None,
+ model=self.llm.settings.model,
+ taxonomy_version=TAXONOMY_VERSION,
+ )
+ # Disagreement is recorded, not silently resolved: it is the signal that a
+ # hint is wrong, and it is only visible if both readings are kept.
+ if hint is not None and raw["role"] != hint.value:
+ label.mechanism = f"[hint={hint.value} model={raw['role']}] {label.mechanism}"
+ self.cache.put_json(key, label.model_dump())
+ return label
+
+ async def label_all(
+ self,
+ edits: Iterable[tuple[Edit, str]],
+ *,
+ progress=None,
+ ) -> list[EditLabel]:
+ tasks = [asyncio.create_task(self.label(e, s)) for e, s in edits]
+ out: list[EditLabel] = []
+ for i, task in enumerate(asyncio.as_completed(tasks), 1):
+ label = await task
+ if label is not None:
+ out.append(label)
+ if progress and i % 50 == 0:
+ progress(i, len(tasks))
+ return out
+
+ def stats(self) -> str:
+ return (
+ f"{self.cache.stats()}; calls={self.llm.calls} retries={self.llm.retries} "
+ f"hint-authoritative={self.skipped_by_hint} audited={self.audited} "
+ f"failed={self.failed}"
+ )
+
+
+def direction_only(edit: Edit) -> Direction:
+ """Derived facet, exposed for callers that want it without labelling."""
+ return direction_of(edit.before_value, edit.after_value)
diff --git a/vero/src/vero/interpret/models.py b/vero/src/vero/interpret/models.py
index 7c4ed213..4768e26e 100644
--- a/vero/src/vero/interpret/models.py
+++ b/vero/src/vero/interpret/models.py
@@ -111,6 +111,25 @@ def make_id(cell_key: str, sha: str, path: str, symbol: str, diff: str) -> str:
return content_id(cell_key, sha, path, symbol, diff)
+class EditLabel(BaseModel):
+ """A model-assigned reading of one edit, plus the facets that were derived.
+
+ `hinted` records whether the role came from a deterministic rule rather than the
+ model, so agreement between the two can be measured instead of assumed.
+ """
+
+ edit_id: str
+ action: str
+ role: str
+ provenance: str = "unknown"
+ direction: str = "na"
+ mechanism: str = "" # one line, the model's own words
+ confidence: float = 0.0
+ hinted: bool = False
+ model: str = ""
+ taxonomy_version: str = ""
+
+
class Trajectory(BaseModel):
"""Everything known about one cell."""
@@ -118,6 +137,7 @@ class Trajectory(BaseModel):
candidates: list[Candidate] = Field(default_factory=list)
evaluations: list[EvalRecord] = Field(default_factory=list)
edits: list[Edit] = Field(default_factory=list)
+ labels: list[EditLabel] = Field(default_factory=list)
reward: float | None = None
baseline_reward: float | None = None
error_rate: float | None = None
From 464b0361d14504428dbc9eb3a29fcfd7bf69bc0d Mon Sep 17 00:00:00 2001
From: Varun Ursekar
Date: Sun, 2 Aug 2026 23:53:30 -0700
Subject: [PATCH 06/16] Add analysis: prevalence, rarefaction, Jaccard-vs-null,
and HTML figures
Two rules are baked into the aggregations rather than left to the caller.
Prevalence counts cells, not edits: cells produced between 1 and 18 candidates,
so an edit-weighted count answers "which cells were prolific" while appearing to
answer "what did optimizers try". And the diversity figure ships a permutation
null, because a mean pairwise Jaccard distance is uninterpretable alone -- 0.5
could mean cells explore genuinely different repertoires, or that each drew a few
roles from the same skewed marginal. The null holds repertoire size and corpus
role frequencies fixed and reshuffles the assignment, isolating the question.
gaia-shell is marked and never pooled: its seed is an empty shell, so every role
is present there by construction, which is why its initialization and metadata
prevalence run 3x the other benchmarks.
Reward appears nowhere. With minimum real gaps of 0.089-0.130 in this corpus,
category-versus-score comparisons are not supportable, and the honest response is
to omit them rather than plot them with a caveat.
Colour follows the validated reference palette and was checked with the
validator, not by eye: sequential blue for magnitude, fixed categorical order for
identity, blue/red diverging for polarity. Several light-mode steps sit below 3:1
on the surface, so the relief rule applies -- every figure carries in-mark labels
and a table view.
Co-Authored-By: Claude Opus 5 (1M context)
---
vero/src/vero/interpret/analysis/__init__.py | 5 +
vero/src/vero/interpret/analysis/figures.py | 314 +++++++++++++++++++
vero/src/vero/interpret/analysis/stats.py | 188 +++++++++++
vero/src/vero/interpret/cli.py | 25 ++
4 files changed, 532 insertions(+)
create mode 100644 vero/src/vero/interpret/analysis/__init__.py
create mode 100644 vero/src/vero/interpret/analysis/figures.py
create mode 100644 vero/src/vero/interpret/analysis/stats.py
diff --git a/vero/src/vero/interpret/analysis/__init__.py b/vero/src/vero/interpret/analysis/__init__.py
new file mode 100644
index 00000000..87387cef
--- /dev/null
+++ b/vero/src/vero/interpret/analysis/__init__.py
@@ -0,0 +1,5 @@
+"""Aggregation and figures over labelled edits."""
+
+from vero.interpret.analysis import figures, stats
+
+__all__ = ["figures", "stats"]
diff --git a/vero/src/vero/interpret/analysis/figures.py b/vero/src/vero/interpret/analysis/figures.py
new file mode 100644
index 00000000..56cd8625
--- /dev/null
+++ b/vero/src/vero/interpret/analysis/figures.py
@@ -0,0 +1,314 @@
+"""Render the analysis as one self-contained HTML page.
+
+Inline SVG, no build step and no CDN: the output is a single file that opens from
+disk and can be handed to someone. Colour follows the validated reference palette —
+sequential blue for magnitude, the fixed categorical order for identity, blue/red
+diverging for polarity — and every chart carries a table view, because several
+palette steps sit below 3:1 on the light surface and the relief rule applies.
+"""
+
+from __future__ import annotations
+
+import html
+import json
+from collections import Counter
+
+from vero.interpret.analysis import stats
+
+CAT_LIGHT = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#4a3aa7", "#e34948"]
+CAT_DARK = ["#3987e5", "#d95926", "#199e70", "#c98500", "#d55181", "#008300", "#9085e9", "#e66767"]
+SEQ = ["#cde2fb", "#b7d3f6", "#9ec5f4", "#86b6ef", "#6da7ec", "#5598e7", "#3987e5", "#2a78d6", "#256abf", "#1c5cab", "#184f95"]
+POS, NEG = "#2a78d6", "#e34948"
+
+CSS = """
+.viz-root{color-scheme:light;--surface-1:#fcfcfb;--surface-2:#f4f3f0;--text-primary:#0b0b0b;
+--text-secondary:#52514e;--text-muted:#7a7975;--grid:#e6e5e1;--pos:#2a78d6;--neg:#e34948;
+background:var(--surface-1);color:var(--text-primary);
+font:14px/1.5 ui-sans-serif,-apple-system,"Segoe UI",Roboto,sans-serif;padding:28px 32px;max-width:1180px;margin:0 auto}
+@media (prefers-color-scheme:dark){:root:where(:not([data-theme="light"])) .viz-root{color-scheme:dark;
+--surface-1:#1a1a19;--surface-2:#242422;--text-primary:#fff;--text-secondary:#c3c2b7;
+--text-muted:#8f8e86;--grid:#343430;--pos:#3987e5;--neg:#e66767}}
+:root[data-theme="dark"] .viz-root{color-scheme:dark;--surface-1:#1a1a19;--surface-2:#242422;
+--text-primary:#fff;--text-secondary:#c3c2b7;--text-muted:#8f8e86;--grid:#343430;--pos:#3987e5;--neg:#e66767}
+h1{font-size:22px;margin:0 0 4px} h2{font-size:16px;margin:34px 0 2px}
+p.note{color:var(--text-secondary);margin:2px 0 14px;max-width:74ch;font-size:13px}
+.fig{background:var(--surface-1);border:1px solid var(--grid);border-radius:10px;padding:14px 16px;margin-bottom:6px}
+.legend{display:flex;gap:16px;flex-wrap:wrap;margin:8px 0 0;font-size:12px;color:var(--text-secondary)}
+.legend i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:6px;vertical-align:-1px}
+text{font:11px ui-sans-serif,-apple-system,sans-serif}
+.axis{fill:var(--text-secondary)} .muted{fill:var(--text-muted)} .ink{fill:var(--text-primary)}
+.grid{stroke:var(--grid);stroke-width:1}
+details{margin:6px 0 0} summary{cursor:pointer;color:var(--text-secondary);font-size:12px}
+table{border-collapse:collapse;font-size:12px;margin-top:8px} th,td{border:1px solid var(--grid);padding:3px 8px;text-align:right}
+th:first-child,td:first-child{text-align:left} th{color:var(--text-secondary);font-weight:600}
+#tip{position:fixed;pointer-events:none;opacity:0;background:var(--surface-2);color:var(--text-primary);
+border:1px solid var(--grid);border-radius:6px;padding:6px 9px;font-size:12px;max-width:320px;
+box-shadow:0 2px 10px rgba(0,0,0,.16);z-index:50;transition:opacity .08s}
+.hit{cursor:crosshair}
+.toggle{float:right;font-size:12px;color:var(--text-secondary);cursor:pointer;border:1px solid var(--grid);
+border-radius:6px;padding:3px 9px;background:var(--surface-2)}
+"""
+
+JS = """
+const tip=document.getElementById('tip');
+document.querySelectorAll('[data-tip]').forEach(el=>{
+ el.addEventListener('mousemove',e=>{tip.innerHTML=el.dataset.tip;tip.style.opacity=1;
+ const p=12;let x=e.clientX+p,y=e.clientY+p;
+ if(x+tip.offsetWidth>innerWidth)x=e.clientX-tip.offsetWidth-p;
+ if(y+tip.offsetHeight>innerHeight)y=e.clientY-tip.offsetHeight-p;
+ tip.style.left=x+'px';tip.style.top=y+'px';});
+ el.addEventListener('mouseleave',()=>tip.style.opacity=0);});
+document.getElementById('themebtn').onclick=()=>{
+ const r=document.documentElement;
+ const dark=r.getAttribute('data-theme')==='dark';
+ r.setAttribute('data-theme',dark?'light':'dark');};
+"""
+
+
+def _esc(s) -> str:
+ return html.escape(str(s), quote=True)
+
+
+def _seq(frac: float) -> str:
+ return SEQ[max(0, min(len(SEQ) - 1, round(frac * (len(SEQ) - 1))))]
+
+
+def _table(headers: list[str], rows: list[list], caption: str) -> str:
+ head = "".join(f"
{_esc(h)}
" for h in headers)
+ body = "".join(
+ "
" + "".join(f"
{_esc(c)}
" for c in r) + "
" for r in rows
+ )
+ return (
+ f"{_esc(caption)}"
+ f"
{head}
{body}
"
+ )
+
+
+def fig_prevalence(rows: list[dict]) -> str:
+ roles, table = stats.prevalence(rows)
+ benches = [b for b in stats.BENCH_ORDER if b in next(iter(table.values()))]
+ cw, ch, lw, top = 132, 26, 150, 46
+ w, h = lw + cw * len(benches) + 30, top + ch * len(roles) + 26
+ out = [f'")
+ tbl = _table(
+ ["role"] + benches,
+ [[r] + [f"{table[r][b][0]}/{table[r][b][1]}" for b in benches] for r in roles],
+ "Table view",
+ )
+ return "".join(out) + tbl
+
+
+def fig_rarefaction(rows: list[dict]) -> str:
+ curves = stats.rarefaction(rows)
+ benches = [b for b in stats.BENCH_ORDER if b in curves]
+ w, h, pad = 760, 300, 46
+ maxy = max(max(v) for v in curves.values())
+ maxx = max(len(v) for v in curves.values())
+ sx = lambda i: pad + i * (w - pad - 150) / max(maxx - 1, 1)
+ sy = lambda v: h - pad - v * (h - 2 * pad) / maxy
+ out = [f'")
+ tbl = _table(
+ ["cells"] + benches,
+ [[i + 1] + [f"{curves[b][i]:.1f}" if i < len(curves[b]) else "" for b in benches]
+ for i in range(maxx)],
+ "Table view",
+ )
+ return "".join(out) + tbl
+
+
+def fig_jaccard(rows: list[dict]) -> str:
+ data = stats.jaccard(rows)
+ benches = [b for b in stats.BENCH_ORDER if b in data]
+ rh, lw, w = 42, 150, 760
+ h = 40 + rh * len(benches)
+ x0, x1 = lw, w - 130
+ lo = min(min(d["null_lo"], d["observed"]) for d in data.values()) - 0.04
+ hi = max(max(d["null_hi"], d["observed"]) for d in data.values()) + 0.04
+ sx = lambda v: x0 + (v - lo) * (x1 - x0) / (hi - lo)
+ out = [f'")
+ tbl = _table(
+ ["benchmark", "cells", "observed", "null 2.5%", "null 97.5%", "verdict"],
+ [[b, data[b]["n_cells"], f"{data[b]['observed']:.3f}", f"{data[b]['null_lo']:.3f}",
+ f"{data[b]['null_hi']:.3f}", data[b]["verdict"]] for b in benches],
+ "Table view",
+ )
+ return "".join(out) + tbl
+
+
+def fig_action_role(rows: list[dict]) -> str:
+ roles, actions, counts = stats.action_by_role(rows)
+ lw, w, rh = 150, 760, 30
+ h = 40 + rh * len(roles)
+ maxv = max(sum(counts.get((r, a), 0) for a in actions) for r in roles)
+ bw = w - lw - 60
+ out = [f'")
+ leg = '
' + "".join(
+ f'{_esc(a)}'
+ for k, a in enumerate(actions)
+ ) + "
"
+ tbl = _table(["role"] + actions,
+ [[r] + [counts.get((r, a), 0) for a in actions] for r in roles],
+ "Table view")
+ return "".join(out) + leg + tbl
+
+
+def fig_direction(rows: list[dict], edits: dict[str, dict]) -> str:
+ data = stats.tuning_direction(rows, edits)
+ if not data:
+ return "
No scalar constants changed value.
"
+ lw, w, rh = 210, 760, 28
+ h = 46 + rh * len(data)
+ mx = max(max(u, d) for _, u, d in data) or 1
+ mid = lw + (w - lw - 40) / 2
+ half = (w - lw - 60) / 2
+ out = [f'")
+ leg = ('
lowered'
+ 'raised
')
+ tbl = _table(["constant", "raised", "lowered"], [[s, u, d] for s, u, d in data], "Table view")
+ return "".join(out) + leg + tbl
+
+
+def render(rows: list[dict], edits: dict[str, dict], meta: dict) -> str:
+ ag = stats.hint_agreement(rows)
+ figs = [
+ ("Which kinds of edit did optimizers make, and how universally?",
+ "Share of cells in each benchmark that ever made an edit of this kind. Counted per "
+ "cell, not per edit: cells produced between 1 and 18 candidates, so edit-weighted "
+ "counts would measure which cells were prolific. ‡ gaia-shell's seed is an empty "
+ "shell, so every role is present there by construction — do not pool that column.",
+ fig_prevalence(rows)),
+ ("Does the next optimizer try anything new?",
+ "Distinct roles discovered as cells are added, averaged over 200 random orderings. "
+ "A curve that flattens says the repertoire was exhausted early; one still climbing "
+ "at the right edge says it was not.",
+ fig_rarefaction(rows)),
+ ("Do different optimizers explore different things?",
+ "Mean pairwise Jaccard distance between cells' role repertoires (dot) against a "
+ "permutation null holding each cell's repertoire size and the corpus role "
+ "frequencies fixed (grey band, 95%). Left of the band means cells are more alike "
+ "than chance — convergence. The raw distance alone says nothing without this.",
+ fig_jaccard(rows)),
+ ("What was done to each part of the harness?",
+ "Edits per role, split by action. This is the one view where edit counts are the "
+ "right unit, since the question is about the composition of the work.",
+ fig_action_role(rows)),
+ ("Which way did the knobs go?",
+ "Scalar constants whose value actually changed, by direction. Constants touched by "
+ "reformatting without a value change are excluded.",
+ fig_direction(rows, edits)),
+ ]
+ body = "".join(
+ f'
{_esc(t)}
{_esc(n)}
{svg}
'
+ for t, n, svg in figs
+ )
+ return f"""
+
+What optimizers modified
+
+
+
What optimizers modified
+
{_esc(meta['cells'])} cells, {_esc(meta['edits'])} symbol-scoped edits,
+{_esc(meta['labels'])} labelled. Roles were assigned by deterministic rule where the
+file, symbol kind or name settles it ({ag['hinted']} edits) and by model otherwise
+({ag['model_decided']}); {ag['disagreements']} audited edits disagreed and both readings
+are kept in the record. Reward is deliberately absent: measurement noise in this corpus
+makes category-versus-score comparisons unsupportable.
+{body}
+
Generated by vero interpret report. Colour: sequential blue for
+magnitude, fixed categorical order for identity, blue/red diverging for polarity; palette
+validated for colour-vision deficiency. Every figure has a table view.
+
"""
diff --git a/vero/src/vero/interpret/analysis/stats.py b/vero/src/vero/interpret/analysis/stats.py
new file mode 100644
index 00000000..1321e255
--- /dev/null
+++ b/vero/src/vero/interpret/analysis/stats.py
@@ -0,0 +1,188 @@
+"""Aggregations behind the figures. Pure functions over labelled edits.
+
+Two rules run through all of these.
+
+Prevalence counts **cells, not edits**. Cells produced between 1 and 18 candidates,
+so an edit-weighted count answers "which cells were prolific" while pretending to
+answer "what did optimizers try".
+
+Diversity needs a **null**. A mean pairwise Jaccard distance of 0.5 is
+uninterpretable on its own: it could mean cells explore genuinely different
+repertoires, or simply that each drew a few roles from the same skewed marginal. The
+permutation null holds each cell's repertoire *size* and the corpus-wide role
+frequencies fixed and reshuffles which cell got what, so the comparison isolates
+whether cells differ beyond chance.
+"""
+
+from __future__ import annotations
+
+import itertools
+import random
+import statistics as st
+from collections import Counter, defaultdict
+
+BENCH_ORDER = [
+ "browsecomp-plus",
+ "officeqa",
+ "swe-atlas-qna",
+ "terminal-bench",
+ "gaia-shell",
+]
+
+# gaia-shell's seed is an empty shell, so every role is "present" by construction
+# rather than by choice. It is shown but never pooled with the rest.
+CONSTRUCTED_SEED = {"gaia-shell"}
+
+
+def cell_roles(rows: list[dict]) -> dict[str, set[str]]:
+ """cell_key -> the set of roles it ever touched."""
+ out: dict[str, set[str]] = defaultdict(set)
+ for r in rows:
+ out[r["cell_key"]].add(r["role"])
+ return dict(out)
+
+
+def benchmark_cells(rows: list[dict]) -> dict[str, set[str]]:
+ out: dict[str, set[str]] = defaultdict(set)
+ for r in rows:
+ out[r["cell_key"].split("/")[1]].add(r["cell_key"])
+ return dict(out)
+
+
+def prevalence(rows: list[dict]) -> tuple[list[str], dict[str, dict[str, tuple[int, int]]]]:
+ """role -> benchmark -> (cells that used it, cells in the benchmark)."""
+ roles_by_cell = cell_roles(rows)
+ cells_by_bench = benchmark_cells(rows)
+ roles = sorted({r for s in roles_by_cell.values() for r in s})
+ table: dict[str, dict[str, tuple[int, int]]] = {}
+ for role in roles:
+ table[role] = {}
+ for bench, cells in cells_by_bench.items():
+ hit = sum(1 for c in cells if role in roles_by_cell.get(c, set()))
+ table[role][bench] = (hit, len(cells))
+ # Order roles by how universal they are, so the figure reads top-down.
+ roles.sort(key=lambda r: -sum(h / t for h, t in table[r].values()))
+ return roles, table
+
+
+def rarefaction(rows: list[dict], *, trials: int = 200, seed: int = 0) -> dict[str, list[float]]:
+ """Mean distinct roles discovered after k cells, averaged over orderings.
+
+ A curve that flattens says the k-th optimizer tried nothing the first k-1 had
+ not already tried; one still climbing at k=20 says the repertoire is not
+ exhausted by the sample.
+ """
+ rng = random.Random(seed)
+ roles_by_cell = cell_roles(rows)
+ out: dict[str, list[float]] = {}
+ for bench, cells in benchmark_cells(rows).items():
+ members = sorted(cells)
+ totals = [0.0] * len(members)
+ for _ in range(trials):
+ rng.shuffle(members)
+ seen: set[str] = set()
+ for i, cell in enumerate(members):
+ seen |= roles_by_cell.get(cell, set())
+ totals[i] += len(seen)
+ out[bench] = [t / trials for t in totals]
+ return out
+
+
+def jaccard(rows: list[dict], *, trials: int = 500, seed: int = 0) -> dict[str, dict]:
+ """Observed mean pairwise distance per benchmark, against a permutation null."""
+ rng = random.Random(seed)
+ roles_by_cell = cell_roles(rows)
+ out: dict[str, dict] = {}
+ for bench, cells in benchmark_cells(rows).items():
+ sets = [roles_by_cell.get(c, set()) for c in sorted(cells)]
+ sets = [s for s in sets if s]
+ if len(sets) < 3:
+ continue
+ observed = st.mean(
+ 1 - len(a & b) / len(a | b) for a, b in itertools.combinations(sets, 2)
+ )
+ # Null: keep each cell's repertoire size and the corpus role frequencies,
+ # reshuffle the assignment.
+ pool: list[str] = []
+ for s in sets:
+ pool.extend(s)
+ freq = Counter(pool)
+ vocab = list(freq)
+ weights = [freq[v] for v in vocab]
+ null: list[float] = []
+ for _ in range(trials):
+ drawn = []
+ for s in sets:
+ picked: set[str] = set()
+ while len(picked) < len(s):
+ picked.add(rng.choices(vocab, weights=weights, k=1)[0])
+ drawn.append(picked)
+ null.append(
+ st.mean(
+ 1 - len(a & b) / len(a | b) for a, b in itertools.combinations(drawn, 2)
+ )
+ )
+ null.sort()
+ lo, hi = null[int(0.025 * len(null))], null[int(0.975 * len(null)) - 1]
+ out[bench] = {
+ "observed": observed,
+ "null_mean": st.mean(null),
+ "null_lo": lo,
+ "null_hi": hi,
+ "n_cells": len(sets),
+ # Below the null: cells are MORE alike than chance -> convergence.
+ "verdict": "converged" if observed < lo else ("diverged" if observed > hi else "as chance"),
+ }
+ return out
+
+
+def action_by_role(rows: list[dict], *, top_roles: int = 10) -> tuple[list[str], list[str], dict]:
+ counts: dict[tuple[str, str], int] = Counter()
+ for r in rows:
+ counts[(r["role"], r["action"])] += 1
+ role_totals = Counter()
+ for (role, _), n in counts.items():
+ role_totals[role] += n
+ roles = [r for r, _ in role_totals.most_common(top_roles)]
+ actions = [a for a, _ in Counter(r["action"] for r in rows).most_common()]
+ return roles, actions, {k: v for k, v in counts.items() if k[0] in roles}
+
+
+def tuning_direction(rows: list[dict], edits: dict[str, dict], *, top: int = 12) -> list[tuple[str, int, int]]:
+ """(symbol, ups, downs) for scalar constants that actually changed."""
+ counts: dict[str, list[int]] = defaultdict(lambda: [0, 0])
+ for r in rows:
+ edit = edits.get(r["edit_id"])
+ if not edit or edit["symbol_kind"] != "scalar_const":
+ continue
+ if r["direction"] == "up":
+ counts[edit["symbol"]][0] += 1
+ elif r["direction"] == "down":
+ counts[edit["symbol"]][1] += 1
+ ranked = sorted(counts.items(), key=lambda kv: -(kv[1][0] + kv[1][1]))
+ return [(s, u, d) for s, (u, d) in ranked[:top]]
+
+
+def provenance_of_fixes(rows: list[dict]) -> dict[str, Counter]:
+ out: dict[str, Counter] = defaultdict(Counter)
+ for r in rows:
+ if r["action"] != "fix":
+ continue
+ out[r["cell_key"].split("/")[1]][r["provenance"]] += 1
+ return dict(out)
+
+
+def hint_agreement(rows: list[dict]) -> dict[str, int]:
+ """How often the model's role matched the deterministic hint, where audited.
+
+ Disagreements are recorded in `mechanism` as a "[hint=… model=…]" prefix, which
+ is the only place both readings survive.
+ """
+ audited = [r for r in rows if r["hinted"] and r["mechanism"].startswith("[hint=")]
+ hinted = [r for r in rows if r["hinted"]]
+ return {
+ "hinted": len(hinted),
+ "disagreements": len(audited),
+ "model_decided": len(rows) - len(hinted),
+ "total": len(rows),
+ }
diff --git a/vero/src/vero/interpret/cli.py b/vero/src/vero/interpret/cli.py
index fd3a8cf5..e9f548a6 100644
--- a/vero/src/vero/interpret/cli.py
+++ b/vero/src/vero/interpret/cli.py
@@ -163,6 +163,31 @@ async def run():
click.echo(f"wrote {out} ({len(labels)} labels) {labeler.stats()}")
+@main.command()
+@click.option("--labels-file", type=click.Path(path_type=Path), default=Path("labels.jsonl"))
+@click.option("--edits-file", type=click.Path(path_type=Path), default=Path("edits.jsonl"))
+@click.option("--out", type=click.Path(path_type=Path), default=Path("analysis"))
+def report(labels_file, edits_file, out) -> None:
+ """Render the figures as one self-contained HTML page."""
+ from vero.interpret.analysis import figures
+
+ edits = {}
+ for line in Path(edits_file).read_text().splitlines():
+ e = json.loads(line)
+ edits[e["id"]] = e
+ rows = [json.loads(l) for l in Path(labels_file).read_text().splitlines()]
+ cells = len({r["cell_key"] for r in rows})
+ meta = {"cells": cells, "edits": len(edits), "labels": len(rows)}
+
+ out = Path(out)
+ out.mkdir(parents=True, exist_ok=True)
+ page = out / "index.html"
+ page.write_text(figures.render(rows, edits, meta))
+ (out / "labels.jsonl").write_text(Path(labels_file).read_text())
+ (out / "edits.jsonl").write_text(Path(edits_file).read_text())
+ click.echo(f"wrote {page} ({cells} cells, {len(rows)} labels)")
+
+
@main.command()
@click.option("--in", "src", type=click.Path(path_type=Path), default=Path("edits.jsonl"))
@click.option("--top", default=40, help="Symbols to show per benchmark.")
From b68ea95c0ef4c92e3515cb8e74dd5a6eda6299dc Mon Sep 17 00:00:00 2001
From: Varun Ursekar
Date: Sun, 2 Aug 2026 23:58:06 -0700
Subject: [PATCH 07/16] Derive fix provenance from the seed tree instead of
asking the model
The model cannot answer this one and fails in a consistent direction. An edit shown
in isolation carries no history, so it sees code being repaired inside the
optimizer's own agent file and says "own": 452 own against 3 seed corpus-wide, and
21 of 22 swe-atlas submission fixes called self-inflicted when those repair a
defect in the seed's answer parser.
It is not a judgement call. If the repaired code is still exactly as the seed wrote
it the defect came with the seed; if an earlier candidate in the same cell had
already rewritten it, the optimizer is repairing itself. Two tree lookups, with a
whole-file fallback when the symbol cannot be resolved on either side.
Derived: 281 seed, 226 own. The check that matters is external -- 15 of the 22
swe-atlas submission fixes now read as seed defects, and 15 is independently the
number of cells found to patch the echoed-sentinel bug by reading diffs. Two
methods, same number.
The general lesson is worth keeping: a facet requiring history cannot be labelled
from a single edit, and a model asked anyway will answer confidently rather than
abstain. Prefer derivation wherever the artifact can settle it.
Co-Authored-By: Claude Opus 5 (1M context)
---
vero/src/vero/interpret/analysis/figures.py | 43 +++++++++++
vero/src/vero/interpret/cli.py | 15 +++-
vero/src/vero/interpret/edits/provenance.py | 84 +++++++++++++++++++++
3 files changed, 141 insertions(+), 1 deletion(-)
create mode 100644 vero/src/vero/interpret/edits/provenance.py
diff --git a/vero/src/vero/interpret/analysis/figures.py b/vero/src/vero/interpret/analysis/figures.py
index 56cd8625..09da1492 100644
--- a/vero/src/vero/interpret/analysis/figures.py
+++ b/vero/src/vero/interpret/analysis/figures.py
@@ -262,6 +262,42 @@ def fig_direction(rows: list[dict], edits: dict[str, dict]) -> str:
return "".join(out) + leg + tbl
+def fig_provenance(rows: list[dict]) -> str:
+ """Whose defect each fix repaired. Derived from the seed tree, not model-assigned."""
+ data = stats.provenance_of_fixes(rows)
+ benches = [b for b in stats.BENCH_ORDER if b in data]
+ if not benches:
+ return "
No fixes labelled.
"
+ order = ["seed", "own", "unknown"]
+ cols = {"seed": CAT_LIGHT[0], "own": CAT_LIGHT[1], "unknown": "#7a7975"}
+ lw, w, rh = 150, 760, 32
+ h = 34 + rh * len(benches)
+ maxv = max(sum(data[b].values()) for b in benches)
+ bw = w - lw - 70
+ out = [f'")
+ leg = '
' + "".join(
+ f'{k} defect' for k in order) + "
"
+ tbl = _table(["benchmark"] + order,
+ [[b] + [data[b].get(k, 0) for k in order] for b in benches], "Table view")
+ return "".join(out) + leg + tbl
+
+
def render(rows: list[dict], edits: dict[str, dict], meta: dict) -> str:
ag = stats.hint_agreement(rows)
figs = [
@@ -286,6 +322,13 @@ def render(rows: list[dict], edits: dict[str, dict], meta: dict) -> str:
"Edits per role, split by action. This is the one view where edit counts are the "
"right unit, since the question is about the composition of the work.",
fig_action_role(rows)),
+ ("Whose defect did each fix repair?",
+ "Fixes split by whether the repaired code was still exactly as the seed wrote it "
+ "(seed defect) or had already been rewritten by an earlier candidate in the same "
+ "cell (the optimizer's own). Derived by comparing trees, not asked of the model: "
+ "asking returned 452 own against 3 seed and called 21 of 22 swe-atlas submission "
+ "fixes self-inflicted, when 15 of them repair a defect in the seed's answer parser.",
+ fig_provenance(rows)),
("Which way did the knobs go?",
"Scalar constants whose value actually changed, by direction. Constants touched by "
"reformatting without a value change are excluded.",
diff --git a/vero/src/vero/interpret/cli.py b/vero/src/vero/interpret/cli.py
index e9f548a6..f461a242 100644
--- a/vero/src/vero/interpret/cli.py
+++ b/vero/src/vero/interpret/cli.py
@@ -175,7 +175,20 @@ def report(labels_file, edits_file, out) -> None:
for line in Path(edits_file).read_text().splitlines():
e = json.loads(line)
edits[e["id"]] = e
- rows = [json.loads(l) for l in Path(labels_file).read_text().splitlines()]
+ # A label carries only edit_id; the aggregations key on the edit's cell and
+ # symbol, so join here rather than duplicating those fields into every label.
+ rows = []
+ orphans = 0
+ for line in Path(labels_file).read_text().splitlines():
+ lab = json.loads(line)
+ edit = edits.get(lab["edit_id"])
+ if edit is None:
+ orphans += 1
+ continue
+ rows.append({**lab, "cell_key": edit["cell_key"], "symbol": edit["symbol"],
+ "symbol_kind": edit["symbol_kind"], "path": edit["path"]})
+ if orphans:
+ click.echo(f"warning: {orphans} labels had no matching edit and were dropped")
cells = len({r["cell_key"] for r in rows})
meta = {"cells": cells, "edits": len(edits), "labels": len(rows)}
diff --git a/vero/src/vero/interpret/edits/provenance.py b/vero/src/vero/interpret/edits/provenance.py
new file mode 100644
index 00000000..1aa7120c
--- /dev/null
+++ b/vero/src/vero/interpret/edits/provenance.py
@@ -0,0 +1,84 @@
+"""Whose defect was it? Derived from the seed, not asked of a model.
+
+Asking a model was tried and fails systematically. An edit shown in isolation
+carries no history, so the model sees code being repaired inside the optimizer's own
+agent file and answers "own" almost every time: on the first pass it returned 452
+own against 3 seed corpus-wide, and labelled 21 of 22 swe-atlas submission fixes as
+self-inflicted when those demonstrably repair a defect in the seed's answer parser
+that 15 of 20 cells independently patched.
+
+The question is not a judgement. If the code being repaired is still exactly as the
+seed wrote it, the defect came with the seed; if an earlier candidate in the same
+cell had already rewritten it, the optimizer is repairing itself. That is two tree
+lookups.
+"""
+
+from __future__ import annotations
+
+import ast
+
+from vero.interpret.artifacts.harbor.repo import CandidateRepo
+from vero.interpret.labeling.taxonomy import Provenance
+
+
+def _symbol_source(source: str, symbol: str) -> str | None:
+ """Source text of one qualified symbol, or None if absent."""
+ if not source or symbol in ("", ""):
+ return None
+ try:
+ tree = ast.parse(source)
+ except SyntaxError:
+ return None
+ want = symbol.split(".")
+
+ def find(node: ast.AST, path: list[str]) -> ast.AST | None:
+ if not path:
+ return node
+ for child in ast.iter_child_nodes(node):
+ name = getattr(child, "name", None)
+ if name == path[0]:
+ return find(child, path[1:])
+ if isinstance(child, (ast.Assign, ast.AnnAssign)) and len(path) == 1:
+ targets = (
+ [child.target] if isinstance(child, ast.AnnAssign) else child.targets
+ )
+ if any(getattr(t, "id", None) == path[0] for t in targets):
+ return child
+ return None
+
+ found = find(tree, want)
+ if found is None:
+ return None
+ try:
+ return ast.unparse(found)
+ except Exception:
+ return None
+
+
+def provenance_of(
+ repo: CandidateRepo,
+ seed_sha: str,
+ parent_sha: str,
+ path: str,
+ symbol: str,
+) -> Provenance:
+ """SEED if the repaired code is untouched since the seed, OWN if not."""
+ if not parent_sha or not seed_sha or parent_sha.startswith(seed_sha[:12]):
+ return Provenance.SEED
+
+ seed_src = repo.show_file(seed_sha, path)
+ parent_src = repo.show_file(parent_sha, path)
+ if not seed_src:
+ # The file did not exist in the seed, so whatever is being fixed is the
+ # optimizer's own work by construction.
+ return Provenance.OWN
+ if not parent_src:
+ return Provenance.UNKNOWN
+
+ seed_sym = _symbol_source(seed_src, symbol)
+ parent_sym = _symbol_source(parent_src, symbol)
+ if seed_sym is None or parent_sym is None:
+ # Fall back to whole-file comparison: coarser, but still decided by content
+ # rather than by guess.
+ return Provenance.SEED if seed_src == parent_src else Provenance.OWN
+ return Provenance.SEED if seed_sym == parent_sym else Provenance.OWN
From 0b951a71236e7f7b93572c25950230989d4986c4 Mon Sep 17 00:00:00 2001
From: Varun Ursekar
Date: Mon, 3 Aug 2026 08:46:32 -0700
Subject: [PATCH 08/16] Put method, vocabulary and the error inventory above
the figures
Three sections precede the charts, all computed from the same data so they cannot
drift from what the pipeline produced.
A worked example shows one commit becoming several edits: the chosen candidate
touches 80 lines under a single subject and splits into 8 edits across 6 kinds,
which is the case for the whole approach in one table. A kinds table gives a real
exemplar per symbol kind with the role and action it received and whether a rule or
the model decided.
The error inventory comes third and deliberately before the figures rather than in
a footnote, because a reader who is going to use these numbers needs to know where
they are wrong first. It records what was measured: provenance unlabelable by model
and now derived; 39 of 39 non-Python edits called env_setup rather than cosmetic;
add 2162 against reword 41, so prompt rewrites are being counted as additions;
revert at 15, which symbol-scoped decomposition structurally cannot see because a
revert is a property of a commit; and rule/model disagreement on 346 of 2114 hinted
edits, where the pattern is that the rule labels by location and the model by
purpose -- which marks the places a single-role facet is the wrong shape.
The kinds table happens to display several of these errors on its own, which is the
right outcome: the examples are drawn from the data, not chosen to flatter it.
Co-Authored-By: Claude Opus 5 (1M context)
---
vero/src/vero/interpret/analysis/figures.py | 8 +-
vero/src/vero/interpret/analysis/preamble.py | 196 +++++++++++++++++++
2 files changed, 203 insertions(+), 1 deletion(-)
create mode 100644 vero/src/vero/interpret/analysis/preamble.py
diff --git a/vero/src/vero/interpret/analysis/figures.py b/vero/src/vero/interpret/analysis/figures.py
index 09da1492..34d5147e 100644
--- a/vero/src/vero/interpret/analysis/figures.py
+++ b/vero/src/vero/interpret/analysis/figures.py
@@ -13,7 +13,7 @@
import json
from collections import Counter
-from vero.interpret.analysis import stats
+from vero.interpret.analysis import preamble, stats
CAT_LIGHT = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#4a3aa7", "#e34948"]
CAT_DARK = ["#3987e5", "#d95926", "#199e70", "#c98500", "#d55181", "#008300", "#9085e9", "#e66767"]
@@ -45,6 +45,8 @@
border:1px solid var(--grid);border-radius:6px;padding:6px 9px;font-size:12px;max-width:320px;
box-shadow:0 2px 10px rgba(0,0,0,.16);z-index:50;transition:opacity .08s}
.hit{cursor:crosshair}
+.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px}
+.fig table{width:100%}
.toggle{float:right;font-size:12px;color:var(--text-secondary);cursor:pointer;border:1px solid var(--grid);
border-radius:6px;padding:3px 9px;background:var(--surface-2)}
"""
@@ -338,6 +340,10 @@ def render(rows: list[dict], edits: dict[str, dict], meta: dict) -> str:
f'
{_esc(t)}
{_esc(n)}
{svg}
'
for t, n, svg in figs
)
+ # Method, vocabulary and the error inventory come BEFORE the figures: a reader who
+ # is going to use these numbers needs to know how they were made and where they
+ # are wrong first, not in a footnote.
+ body = preamble.render_sections(rows, edits) + body
return f"""
What optimizers modified
diff --git a/vero/src/vero/interpret/analysis/preamble.py b/vero/src/vero/interpret/analysis/preamble.py
new file mode 100644
index 00000000..01392fe8
--- /dev/null
+++ b/vero/src/vero/interpret/analysis/preamble.py
@@ -0,0 +1,196 @@
+"""The narrative sections above the figures: method, vocabulary, and known errors.
+
+Computed from the same data as the figures rather than written out, so they cannot
+drift from what the pipeline actually produced. The error inventory in particular is
+measured: it exists because several facets are wrong in specific, findable ways, and
+a reader who is going to use these figures needs that before the figures, not after.
+"""
+
+from __future__ import annotations
+
+import html
+import re
+from collections import Counter
+
+_DIS = re.compile(r"\[hint=(\S+) model=(\S+)\]")
+
+
+def _esc(s) -> str:
+ return html.escape(str(s), quote=True)
+
+
+def _pick_example(rows: list[dict], edits: dict[str, dict]):
+ """A candidate that shows the split clearly: several edits spanning several kinds."""
+ by_cand: dict[tuple[str, str], list[dict]] = {}
+ for r in rows:
+ edit = edits.get(r["edit_id"])
+ if edit:
+ by_cand.setdefault((edit["cell_key"], edit["candidate_sha"]), []).append(
+ {**r, **edit}
+ )
+ best = None
+ for (cell, sha), group in sorted(by_cand.items()):
+ kinds = len({g["symbol_kind"] for g in group})
+ if 6 <= len(group) <= 9 and (best is None or kinds > best[0]):
+ best = (kinds, cell, sha, group)
+ if best is None:
+ (cell, sha), group = next(iter(by_cand.items()))
+ return cell, sha, group
+ return best[1], best[2], best[3]
+
+
+def worked_example(rows: list[dict], edits: dict[str, dict]) -> str:
+ cell, sha, group = _pick_example(rows, edits)
+ added = sum(g["added"] for g in group)
+ kinds = len({g["symbol_kind"] for g in group})
+ body = []
+ for g in sorted(group, key=lambda x: -x["added"]):
+ val = ""
+ if g.get("before_value"):
+ val = (
+ f' {_esc(g["before_value"])}'
+ f'→{_esc(g["after_value"])}'
+ )
+ body.append(
+ f'
{_esc(g["symbol"][:38])}
'
+ f'
{_esc(g["symbol_kind"])}
+{g["added"]}
'
+ f'
{_esc(g["role"])}
{_esc(g["action"])}{val}
'
+ )
+ return (
+ f'
One commit is not one change. Candidate '
+ f'{_esc(sha[:12])} of '
+ f'{_esc(cell.split("/", 1)[1])} touches {added} lines under a '
+ f'single subject line; labelling it as one item would give one category to all of '
+ f'them. Mapping each changed line through the syntax tree to its innermost enclosing '
+ f'definition splits it into {len(group)} edits across {kinds} kinds, each '
+ f'labelled on its own:
'
+ f'
symbol
kind
'
+ f'
+lines
role
action
'
+ f'{"".join(body)}
'
+ f'
Git\'s own hunk header cannot do this. Its Python matcher reports the '
+ f'enclosing class for a method, so a one-line fix inside a shell helper is '
+ f'attributed to the whole agent. Module-level bindings are split further by target '
+ f'name and value shape, which is what separates a system prompt from a tuning '
+ f'constant when both sit at the top of the same file.
'
+ )
+
+
+def kinds_table(rows: list[dict], edits: dict[str, dict]) -> str:
+ seen: dict[str, dict] = {}
+ for r in rows:
+ edit = edits.get(r["edit_id"])
+ if not edit:
+ continue
+ kind = edit["symbol_kind"]
+ # Prefer an exemplar that carries a value change; it shows more of the schema.
+ if kind not in seen or (edit.get("before_value") and not seen[kind].get("before_value")):
+ seen[kind] = {**r, **edit}
+ order = [
+ "prompt_text", "scalar_const", "collection", "regex", "method",
+ "function", "class", "module", "non_python",
+ ]
+ body = []
+ for kind in order:
+ g = seen.get(kind)
+ if not g:
+ continue
+ val = (
+ f'{_esc(g["before_value"])}→{_esc(g["after_value"])}'
+ if g.get("before_value") else "—"
+ )
+ body.append(
+ f'
{_esc(kind)}
{_esc(g["symbol"][:34])}
'
+ f'
{val}
{_esc(g["role"])}
'
+ f'
{_esc(g["action"])}
'
+ f'
{"rule" if g["hinted"] else "model"}
'
+ )
+ return (
+ '
Every changed line lands on a named symbol with a kind resolved from '
+ 'the syntax tree. The role is set by deterministic rule where the path, kind or name '
+ 'settles it and by model otherwise. Before/after values are captured for scalar '
+ 'constants, which is what makes tuning direction derived rather than guessed.
'
+ '
kind
'
+ '
example symbol
value change
role
action
'
+ f'
role from
{"".join(body)}
'
+ )
+
+
+def false_labels(rows: list[dict], edits: dict[str, dict]) -> str:
+ dis = [r for r in rows if r["mechanism"].startswith("[hint=")]
+ hinted = sum(1 for r in rows if r["hinted"])
+ pairs: Counter = Counter()
+ for r in dis:
+ m = _DIS.match(r["mechanism"])
+ if m:
+ pairs[(m.group(1), m.group(2))] += 1
+
+ inert = [
+ r for r in rows
+ if (e := edits.get(r["edit_id"])) and e["symbol_kind"] == "non_python"
+ ]
+ inert_wrong = sum(1 for r in inert if r["action"] != "cosmetic")
+ acts = Counter(r["action"] for r in rows)
+ regex_roles = Counter(
+ r["role"] for r in rows
+ if (e := edits.get(r["edit_id"])) and e["symbol_kind"] == "regex"
+ )
+ rate = len(dis) / hinted if hinted else 0.0
+
+ confusion = "".join(
+ f"
{n}
{_esc(h)}
{_esc(m)}
"
+ for (h, m), n in pairs.most_common(6)
+ )
+ return f"""
+
Where these labels are known to be wrong. Measured, not estimated, and
+every figure below inherits it.
+
+
Fix provenance could not be labelled at all, and is now derived instead. Asked of
+the model it returned 452 “own” against 3 “seed”, and called 21 of 22
+swe-atlas submission fixes self-inflicted — those repair a defect in the seed's answer
+parser that 15 of 20 cells independently patched. An edit shown alone carries no history, so
+the model sees repair inside the optimizer's own file and answers “own”
+confidently rather than abstaining. Comparing the repaired symbol against the seed tree gives
+281 seed / 226 own, and 15 of those 22 swe-atlas fixes now read as seed defects —
+the same count reached independently by reading diffs.
+
+
Inert edits are read as work. {inert_wrong} of {len(inert)} non-Python edits, almost
+all .gitignore, were labelled env_setup / add
+rather than cosmetic: the model reads “ignore build artifacts” as environment
+setup. Corpus-wide only {acts.get('cosmetic', 0)} edits were called cosmetic, which is
+certainly too few.
+
+
The action facet under-uses its own vocabulary.
+add {acts.get('add', 0)} against reword
+{acts.get('reword', 0)} — prompt rewrites are being counted as additions. And
+revert {acts.get('revert', 0)} is far too low: a revert is a property
+of a commit, and at symbol scope it looks like ordinary edits, so this decomposition
+structurally cannot see it. Read the action column as a coarse split, not a fine one.
+
+
Rule and model disagree on {len(dis)} of {hinted} hinted edits ({rate:.0%}), and the
+pattern is systematic rather than noise: the rule labels by location, the model by purpose.
+Adjudicated — _complete (rule
+model_client, model control_loop) is a
+client call used to force a final answer, so the rule is right about where it is and the model
+is describing what it is for; TOOLS (rule
+tool_surface, model prompt) is a tool
+description, which is genuinely both. The rule wins on conflict, so these are recorded rather
+than resolved — but they mark where a single-role facet is the wrong shape and
+multi-label would fit better.
+
edits
rule said
model said
+{confusion}
+
+
Answer-parsing regexes scatter. Regex-kind edits spread across
+{len(regex_roles)} roles ({_esc(', '.join(f'{k} {v}' for k, v in regex_roles.most_common(4)))}),
+and several labelled prompt are answer-extraction patterns belonging
+to submission. No rule covers regex bindings, so the model decides
+unaided.
" + false_labels(rows, edits)
+ )
From 95208a4d5050eeed81b1caee98954a70fcc1a75e Mon Sep 17 00:00:00 2001
From: Varun Ursekar
Date: Mon, 3 Aug 2026 08:57:31 -0700
Subject: [PATCH 09/16] Drop the audit-rate sampling: the audit was already
complete
The parameter never gated anything. Every edit is sent to the model regardless of
whether a rule settles its role, because `action` is never derivable from the
artifact -- so the role comparison happens on every hinted edit for free.
audit_rate only incremented a counter, and reporting it made coverage look like a
30% sample when it was 100%: the 346 disagreements are 16.4% of all 2,114 hinted
edits, not of the 605 the counter claimed. A 30% sample would have shown ~99.
Removed the parameter, renamed the counters to say what they measure, and corrected
the report copy, which described the comparison as sampled.
Cost, for the record: 3,986 edits, ~2.1M input tokens (~527 each) and ~0.5M output,
order $1-3 on a mini-class model for the whole corpus. The 53% deterministic role
coverage therefore buys label quality, not money -- the action facet forces a call
either way. Passing only unhinted edits would halve the cost and lose the audit.
Co-Authored-By: Claude Opus 5 (1M context)
---
vero/src/vero/interpret/analysis/figures.py | 8 ++--
vero/src/vero/interpret/cli.py | 8 +---
vero/src/vero/interpret/labeling/labeler.py | 43 ++++++++-------------
3 files changed, 24 insertions(+), 35 deletions(-)
diff --git a/vero/src/vero/interpret/analysis/figures.py b/vero/src/vero/interpret/analysis/figures.py
index 34d5147e..28b93f83 100644
--- a/vero/src/vero/interpret/analysis/figures.py
+++ b/vero/src/vero/interpret/analysis/figures.py
@@ -353,9 +353,11 @@ def render(rows: list[dict], edits: dict[str, dict], meta: dict) -> str:
{_esc(meta['cells'])} cells, {_esc(meta['edits'])} symbol-scoped edits,
{_esc(meta['labels'])} labelled. Roles were assigned by deterministic rule where the
file, symbol kind or name settles it ({ag['hinted']} edits) and by model otherwise
-({ag['model_decided']}); {ag['disagreements']} audited edits disagreed and both readings
-are kept in the record. Reward is deliberately absent: measurement noise in this corpus
-makes category-versus-score comparisons unsupportable.
+({ag['model_decided']}). Every edit was sent to the model regardless, because the action
+facet is never derivable — so the rule-versus-model comparison is complete, not sampled:
+{ag['disagreements']} of {ag['hinted']} hinted edits disagreed and both readings are kept.
+Reward is deliberately absent: measurement noise in this corpus makes
+category-versus-score comparisons unsupportable.
{body}
Generated by vero interpret report. Colour: sequential blue for
magnitude, fixed categorical order for identity, blue/red diverging for polarity; palette
diff --git a/vero/src/vero/interpret/cli.py b/vero/src/vero/interpret/cli.py
index f461a242..6922affa 100644
--- a/vero/src/vero/interpret/cli.py
+++ b/vero/src/vero/interpret/cli.py
@@ -111,14 +111,10 @@ def edits(src, cache_dir, out) -> None:
default=Path("trajectories.jsonl"))
@click.option("--model", default=None, help="Defaults to a cheap model.")
@click.option("--concurrency", default=16)
-@click.option("--audit-rate", default=0.15,
- help="Fraction of hinted edits also sent to the model, to measure "
- "hint/model agreement rather than assume it.")
@click.option("--limit", default=0, help="Label only the first N edits (a dry run).")
@click.option("--cache-dir", type=click.Path(path_type=Path), default=DEFAULT_CACHE)
@click.option("--out", type=click.Path(path_type=Path), default=Path("labels.jsonl"))
-def label(edits_file, trajectories, model, concurrency, audit_rate, limit,
- cache_dir, out) -> None:
+def label(edits_file, trajectories, model, concurrency, limit, cache_dir, out) -> None:
"""Assign facets to edits. Cached and resumable; re-running costs nothing."""
import asyncio as _asyncio
@@ -147,7 +143,7 @@ def label(edits_file, trajectories, model, concurrency, audit_rate, limit,
async def run():
llm = AsyncLLM(settings)
- labeler = Labeler(llm, Cache(Path(cache_dir), "labels"), audit_rate=audit_rate)
+ labeler = Labeler(llm, Cache(Path(cache_dir), "labels"))
try:
return await labeler.label_all(
[(e, subjects.get(e.candidate_sha, "")) for e in rows],
diff --git a/vero/src/vero/interpret/labeling/labeler.py b/vero/src/vero/interpret/labeling/labeler.py
index 0c99fe75..4dfd3076 100644
--- a/vero/src/vero/interpret/labeling/labeler.py
+++ b/vero/src/vero/interpret/labeling/labeler.py
@@ -1,9 +1,10 @@
"""Assign facets to edits: cached, resumable, and mostly not the model's job.
-Roles that a deterministic hint settles are not sent to the model at all, but the
-model is still asked for a role on a sample of hinted edits so the two can be
-compared. Agreement measured beats agreement assumed, and a hint that quietly
-disagrees with every model reading is a bug in the hint.
+A deterministic hint overrides the model's role where one fires, but every edit is
+sent to the model regardless, because `action` is never derivable from the artifact.
+That makes the role comparison free and complete rather than sampled: agreement is
+measured on 100% of hinted edits at no extra cost, and a hint that quietly disagrees
+with every model reading is a bug in the hint.
The prompt shows the diff and withholds nothing except the commit message's
authority: subjects in this corpus routinely misdescribe their diffs — one reading
@@ -14,7 +15,6 @@
from __future__ import annotations
import asyncio
-import random
from typing import Iterable
from vero.interpret.cache import Cache, key_of
@@ -79,20 +79,11 @@ def cache_key(edit: Edit, model: str) -> str:
class Labeler:
- def __init__(
- self,
- llm: AsyncLLM,
- cache: Cache,
- *,
- audit_rate: float = 0.15,
- seed: int = 0,
- ) -> None:
+ def __init__(self, llm: AsyncLLM, cache: Cache) -> None:
self.llm = llm
self.cache = cache
- self.audit_rate = audit_rate
- self._rng = random.Random(seed)
- self.skipped_by_hint = 0
- self.audited = 0
+ self.hint_authoritative = 0
+ self.disagreements = 0
self.failed = 0
async def label(self, edit: Edit, subject: str = "") -> EditLabel | None:
@@ -101,13 +92,12 @@ async def label(self, edit: Edit, subject: str = "") -> EditLabel | None:
return EditLabel.model_validate(cached)
hint = role_hint(edit.path, edit.symbol, edit.symbol_kind.value)
- # A hinted role still needs an action, so the call happens either way; the
- # hint decides whether the model's role is authoritative or merely audited.
- audit = hint is not None and self._rng.random() < self.audit_rate
- if hint is not None and not audit:
- self.skipped_by_hint += 1
- if audit:
- self.audited += 1
+ # Every edit goes to the model regardless of the hint, because `action` is never
+ # derivable and always needs one. That makes the role comparison free rather
+ # than sampled: the audit covers 100% of hinted edits at no extra cost. An
+ # earlier version sampled it, which bought nothing and under-reported coverage.
+ if hint is not None:
+ self.hint_authoritative += 1
try:
raw = await self.llm.json_call(SYSTEM, _user_prompt(edit, subject), _SCHEMA)
@@ -131,6 +121,7 @@ async def label(self, edit: Edit, subject: str = "") -> EditLabel | None:
# Disagreement is recorded, not silently resolved: it is the signal that a
# hint is wrong, and it is only visible if both readings are kept.
if hint is not None and raw["role"] != hint.value:
+ self.disagreements += 1
label.mechanism = f"[hint={hint.value} model={raw['role']}] {label.mechanism}"
self.cache.put_json(key, label.model_dump())
return label
@@ -154,8 +145,8 @@ async def label_all(
def stats(self) -> str:
return (
f"{self.cache.stats()}; calls={self.llm.calls} retries={self.llm.retries} "
- f"hint-authoritative={self.skipped_by_hint} audited={self.audited} "
- f"failed={self.failed}"
+ f"rule-decided={self.hint_authoritative} (all audited) "
+ f"disagreements={self.disagreements} failed={self.failed}"
)
From 743cc2ee1df1a624bdd3b244d3ed52af0a94526d Mon Sep 17 00:00:00 2001
From: Varun Ursekar
Date: Mon, 3 Aug 2026 09:17:45 -0700
Subject: [PATCH 10/16] Rubrics, surrounding code, history, and a 5x larger
diff budget
Four changes to what each call sees, then a full re-label to measure them.
Rubrics for all 16 roles and 8 actions now travel in the JSON schema rather than
leaving the model to infer sixteen meanings from names. Surrounding code: the
symbol's full post-edit source, present for 91% of edits, where before it saw only
changed lines and 14.5% of edits were under 200 characters. History: whether the
symbol existed in the seed and how many earlier candidates in the run touched it.
And the diff budget went from 6,000 characters to 40,000, with the stored cap from
8,000 to 60,000 -- the storage cap was the real constraint and clipped the largest
rewrites before any labeller could see them. Diffs now reach 36,632 characters.
Two improvements are externally checkable. `.gitignore` edits called cosmetic went
0/39 -> 22/39. And `tune` labels carrying a captured value change now number 214
against 215 numeric direction changes derived independently -- the model's tuning
calls on scalar constants line up with ground truth where they can be checked.
One result is a confirmed structural limit, not a fix: `revert` fell 15 -> 1. With
1,307 edits on symbols an earlier candidate had already modified, the model now
declines to call any of them reverts, because the rubric says to only when history
says so and the history line never says "this reverts X". Correct conservatism.
Identifying a revert needs cross-candidate tree comparison; a symbol-scoped view
cannot do it, and no prompt will change that.
What cannot be claimed: 30% of action labels changed between versions, so v2 being
different is not evidence of v2 being right. Beyond the two checkable wins, judging
this properly needs a hand-labelled sample as ground truth.
The convergence finding is unchanged -- all five benchmarks still fall below the
permutation null -- which is the reassuring part: the headline does not depend on
the labelling revision.
Co-Authored-By: Claude Opus 5 (1M context)
---
vero/src/vero/interpret/cli.py | 14 +++-
vero/src/vero/interpret/edits/decompose.py | 27 +++++++-
vero/src/vero/interpret/edits/locus.py | 33 +++++++++
vero/src/vero/interpret/edits/provenance.py | 41 +-----------
vero/src/vero/interpret/labeling/labeler.py | 70 +++++++++++++++++---
vero/src/vero/interpret/labeling/taxonomy.py | 53 ++++++++++++++-
vero/src/vero/interpret/models.py | 3 +
7 files changed, 185 insertions(+), 56 deletions(-)
diff --git a/vero/src/vero/interpret/cli.py b/vero/src/vero/interpret/cli.py
index 6922affa..85bee17a 100644
--- a/vero/src/vero/interpret/cli.py
+++ b/vero/src/vero/interpret/cli.py
@@ -96,10 +96,16 @@ def edits(src, cache_dir, out) -> None:
continue
repo = CandidateRepo(repo_dir)
n = 0
- for cand in traj.candidates:
- for edit in decompose(repo, traj.ref.key, cand):
+ seed_sha = next((c.sha for c in traj.candidates if c.is_seed), None)
+ # Accumulated in candidate order so prior_touches means "before this one".
+ seen_symbols: set[tuple[str, str]] = set()
+ for cand in sorted(traj.candidates, key=lambda c: c.position):
+ produced = decompose(repo, traj.ref.key, cand, seed_sha=seed_sha,
+ prior_symbols=seen_symbols)
+ for edit in produced:
fh.write(edit.model_dump_json() + "\n")
n += 1
+ seen_symbols |= {(e.path, e.symbol) for e in produced}
total += n
click.echo(f" {traj.ref.key}: {n} edits")
click.echo(f"wrote {out} ({total} edits)")
@@ -130,6 +136,7 @@ def label(edits_file, trajectories, model, concurrency, limit, cache_dir, out) -
subjects[cand.sha] = cand.subject
rows = [Edit.model_validate_json(l) for l in Path(edits_file).read_text().splitlines()]
+ per_candidate = Counter(e.candidate_sha for e in rows)
if limit:
rows = rows[:limit]
click.echo(f"labelling {len(rows)} edits with {model or 'default model'}")
@@ -146,7 +153,8 @@ async def run():
labeler = Labeler(llm, Cache(Path(cache_dir), "labels"))
try:
return await labeler.label_all(
- [(e, subjects.get(e.candidate_sha, "")) for e in rows],
+ [(e, subjects.get(e.candidate_sha, ""), per_candidate[e.candidate_sha])
+ for e in rows],
progress=lambda i, n: click.echo(f" {i}/{n}"),
), labeler
finally:
diff --git a/vero/src/vero/interpret/edits/decompose.py b/vero/src/vero/interpret/edits/decompose.py
index 68ef4d0e..8110488a 100644
--- a/vero/src/vero/interpret/edits/decompose.py
+++ b/vero/src/vero/interpret/edits/decompose.py
@@ -22,6 +22,10 @@
# __pycache__ is compiled noise. .gitignore is NOT skipped: several cells shipped a
# candidate whose only change was one, and "shipped something inert" is a finding.
_SKIP = ("__pycache__",)
+# The stored diff was capped at 8000 chars, which silently truncated the largest
+# rewrites before any labeller could see them. Storage is cheap; keep the whole thing.
+MAX_STORED_DIFF = 60_000
+MAX_STORED_SOURCE = 20_000
def _per_symbol_diff(diff: str, keep: set[int]) -> str:
@@ -38,13 +42,16 @@ def _per_symbol_diff(diff: str, keep: set[int]) -> str:
count = int(match.group(2) or 1)
if keep & set(range(start, start + max(count, 1))):
chunks.append(header + body.rstrip("\n"))
- return "\n".join(chunks)[:8000]
+ return "\n".join(chunks)[:MAX_STORED_DIFF]
def decompose(
repo: CandidateRepo,
cell_key: str,
candidate: Candidate,
+ *,
+ seed_sha: str | None = None,
+ prior_symbols: set[tuple[str, str]] | None = None,
) -> list[Edit]:
"""Symbol-scoped edits introduced by `candidate` relative to its parent."""
if candidate.is_seed or candidate.parent_sha is None:
@@ -68,12 +75,13 @@ def decompose(
)
edits.append(
_edit(cell_key, candidate, path, "", SymbolKind.NON_PYTHON,
- added, removed, diff[:8000], None, None)
+ added, removed, diff[:MAX_STORED_DIFF], None, None, None, True, 0)
)
continue
after_src = repo.show_file(candidate.sha, path)
before_src = repo.show_file(candidate.parent_sha, path)
+ seed_src = repo.show_file(seed_sha, path) if seed_sha else ""
mapping = locus.symbol_map(after_src)
grouped: dict[tuple[str, SymbolKind], set[int]] = defaultdict(set)
@@ -102,6 +110,10 @@ def decompose(
if before == after:
continue # touched by reflow, not retuned
share = len(lines)
+ # Context and history, so the labeller is not reading changed lines blind.
+ after_source = locus.symbol_source(after_src, symbol)
+ in_seed = bool(seed_src) and locus.symbol_source(seed_src, symbol) is not None
+ touches = len([1 for k in (prior_symbols or set()) if k == (path, symbol)])
edits.append(
_edit(
cell_key,
@@ -113,9 +125,12 @@ def decompose(
# Removals cannot be attributed per symbol; carry the file total
# on the module row so the count is never silently lost.
removed_total if symbol == "" else 0,
- _per_symbol_diff(diff, lines) or diff[:2000],
+ _per_symbol_diff(diff, lines) or diff[:MAX_STORED_DIFF],
before,
after,
+ after_source[:MAX_STORED_SOURCE] if after_source else None,
+ in_seed,
+ touches,
)
)
if attributed_added < added_total and grouped:
@@ -134,6 +149,9 @@ def _edit(
diff: str,
before: str | None,
after: str | None,
+ after_source: str | None,
+ in_seed: bool,
+ prior_touches: int,
) -> Edit:
return Edit(
id=Edit.make_id(cell_key, candidate.sha, path, symbol, diff),
@@ -147,4 +165,7 @@ def _edit(
before_value=before,
after_value=after,
diff=diff,
+ after_source=after_source,
+ in_seed=in_seed,
+ prior_touches=prior_touches,
)
diff --git a/vero/src/vero/interpret/edits/locus.py b/vero/src/vero/interpret/edits/locus.py
index 87d784f2..8e59be64 100644
--- a/vero/src/vero/interpret/edits/locus.py
+++ b/vero/src/vero/interpret/edits/locus.py
@@ -111,6 +111,39 @@ def walk(node: ast.AST, prefix: str, in_class: bool) -> None:
return mapping
+def symbol_source(source: str, symbol: str) -> str | None:
+ """Full source text of one qualified symbol, for context and history comparison."""
+ if not source or symbol in ("", ""):
+ return None
+ try:
+ tree = ast.parse(source)
+ except SyntaxError:
+ return None
+ want = symbol.split(".")
+
+ def find(node: ast.AST, path: list[str]) -> ast.AST | None:
+ if not path:
+ return node
+ for child in ast.iter_child_nodes(node):
+ if getattr(child, "name", None) == path[0]:
+ return find(child, path[1:])
+ if isinstance(child, (ast.Assign, ast.AnnAssign)) and len(path) == 1:
+ targets = (
+ [child.target] if isinstance(child, ast.AnnAssign) else child.targets
+ )
+ if any(getattr(t, "id", None) == path[0] for t in targets):
+ return child
+ return None
+
+ found = find(tree, want)
+ if found is None:
+ return None
+ try:
+ return ast.unparse(found)
+ except Exception:
+ return None
+
+
def scalar_value(source: str, name: str) -> str | None:
"""Literal text of a module-level scalar binding, for before/after capture."""
try:
diff --git a/vero/src/vero/interpret/edits/provenance.py b/vero/src/vero/interpret/edits/provenance.py
index 1aa7120c..68a68740 100644
--- a/vero/src/vero/interpret/edits/provenance.py
+++ b/vero/src/vero/interpret/edits/provenance.py
@@ -15,46 +15,11 @@
from __future__ import annotations
-import ast
-
from vero.interpret.artifacts.harbor.repo import CandidateRepo
+from vero.interpret.edits.locus import symbol_source
from vero.interpret.labeling.taxonomy import Provenance
-def _symbol_source(source: str, symbol: str) -> str | None:
- """Source text of one qualified symbol, or None if absent."""
- if not source or symbol in ("", ""):
- return None
- try:
- tree = ast.parse(source)
- except SyntaxError:
- return None
- want = symbol.split(".")
-
- def find(node: ast.AST, path: list[str]) -> ast.AST | None:
- if not path:
- return node
- for child in ast.iter_child_nodes(node):
- name = getattr(child, "name", None)
- if name == path[0]:
- return find(child, path[1:])
- if isinstance(child, (ast.Assign, ast.AnnAssign)) and len(path) == 1:
- targets = (
- [child.target] if isinstance(child, ast.AnnAssign) else child.targets
- )
- if any(getattr(t, "id", None) == path[0] for t in targets):
- return child
- return None
-
- found = find(tree, want)
- if found is None:
- return None
- try:
- return ast.unparse(found)
- except Exception:
- return None
-
-
def provenance_of(
repo: CandidateRepo,
seed_sha: str,
@@ -75,8 +40,8 @@ def provenance_of(
if not parent_src:
return Provenance.UNKNOWN
- seed_sym = _symbol_source(seed_src, symbol)
- parent_sym = _symbol_source(parent_src, symbol)
+ seed_sym = symbol_source(seed_src, symbol)
+ parent_sym = symbol_source(parent_src, symbol)
if seed_sym is None or parent_sym is None:
# Fall back to whole-file comparison: coarser, but still decided by content
# rather than by guess.
diff --git a/vero/src/vero/interpret/labeling/labeler.py b/vero/src/vero/interpret/labeling/labeler.py
index 4dfd3076..3fc96978 100644
--- a/vero/src/vero/interpret/labeling/labeler.py
+++ b/vero/src/vero/interpret/labeling/labeler.py
@@ -20,6 +20,8 @@
from vero.interpret.cache import Cache, key_of
from vero.interpret.labeling.client import AsyncLLM, LLMError
from vero.interpret.labeling.taxonomy import (
+ ACTION_RUBRIC,
+ ROLE_RUBRIC,
TAXONOMY_VERSION,
Action,
Direction,
@@ -30,7 +32,13 @@
)
from vero.interpret.models import Edit, EditLabel
-PROMPT_VERSION = "1"
+PROMPT_VERSION = "2"
+
+# Diffs were capped at 6000 characters, which truncated 12% of edits mid-hunk to save
+# tokens that were never the constraint: the whole corpus costs a couple of dollars.
+# These bounds exist only so one pathological edit cannot blow a context window.
+MAX_DIFF_CHARS = 40_000
+MAX_SOURCE_CHARS = 12_000
SYSTEM = """You classify individual edits made by an AI agent that was told to improve \
another agent's harness.
@@ -49,9 +57,22 @@
"additionalProperties": False,
"required": ["action", "role", "provenance", "mechanism", "confidence"],
"properties": {
- "action": {"type": "string", "enum": [a.value for a in Action]},
- "role": {"type": "string", "enum": [r.value for r in Role]},
- "provenance": {"type": "string", "enum": [p.value for p in Provenance]},
+ "action": {
+ "type": "string",
+ "enum": [a.value for a in Action],
+ "description": "; ".join(f"{k}: {v}" for k, v in ACTION_RUBRIC.items()),
+ },
+ "role": {
+ "type": "string",
+ "enum": [r.value for r in Role],
+ "description": "; ".join(f"{k}: {v}" for k, v in ROLE_RUBRIC.items()),
+ },
+ "provenance": {
+ "type": "string",
+ "enum": [p.value for p in Provenance],
+ "description": "Ignored downstream -- provenance is derived from the seed "
+ "tree. Answer unknown unless the history line settles it.",
+ },
"mechanism": {
"type": "string",
"description": "One sentence: what this edit actually does.",
@@ -61,16 +82,41 @@
}
-def _user_prompt(edit: Edit, subject: str) -> str:
+def _user_prompt(edit: Edit, subject: str, siblings: int = 0) -> str:
value = ""
if edit.before_value is not None or edit.after_value is not None:
value = f"\nvalue: {edit.before_value} -> {edit.after_value}"
+
+ # History. Without it the model cannot tell a first-time addition from a rewrite of
+ # the optimizer's own work, and it guessed "own" on almost every fix when asked.
+ if not edit.in_seed:
+ history = "this symbol did not exist in the seed — the optimizer created it"
+ elif edit.prior_touches:
+ history = (
+ f"this symbol came from the seed and {edit.prior_touches} earlier "
+ f"candidate(s) in this run already modified it"
+ )
+ else:
+ history = "this symbol is still as the seed wrote it; this is the first change to it"
+
+ # The subject often describes OTHER edits in the same commit, so say how many there
+ # are. Warning that it "may be inaccurate" was not enough on its own.
+ sib = (
+ f"\nnote: this commit touched {siblings} symbols in total, so the subject may "
+ f"describe a different one"
+ if siblings > 1 else ""
+ )
+ context = (
+ f"\n\nthe symbol after the edit, for context:\n{edit.after_source[:MAX_SOURCE_CHARS]}"
+ if edit.after_source else ""
+ )
return (
f"file: {edit.path}\n"
f"symbol: {edit.symbol} ({edit.symbol_kind.value})\n"
f"lines: +{edit.added} -{edit.removed}{value}\n"
- f"commit subject (may be inaccurate): {subject}\n\n"
- f"diff:\n{edit.diff[:6000]}"
+ f"history: {history}\n"
+ f"commit subject (may be inaccurate): {subject}{sib}\n\n"
+ f"diff:\n{edit.diff[:MAX_DIFF_CHARS]}{context}"
)
@@ -86,7 +132,7 @@ def __init__(self, llm: AsyncLLM, cache: Cache) -> None:
self.disagreements = 0
self.failed = 0
- async def label(self, edit: Edit, subject: str = "") -> EditLabel | None:
+ async def label(self, edit: Edit, subject: str = "", siblings: int = 0) -> EditLabel | None:
key = cache_key(edit, self.llm.settings.model)
if (cached := self.cache.get_json(key)) is not None:
return EditLabel.model_validate(cached)
@@ -100,7 +146,9 @@ async def label(self, edit: Edit, subject: str = "") -> EditLabel | None:
self.hint_authoritative += 1
try:
- raw = await self.llm.json_call(SYSTEM, _user_prompt(edit, subject), _SCHEMA)
+ raw = await self.llm.json_call(
+ SYSTEM, _user_prompt(edit, subject, siblings), _SCHEMA
+ )
except LLMError:
self.failed += 1
return None
@@ -128,11 +176,11 @@ async def label(self, edit: Edit, subject: str = "") -> EditLabel | None:
async def label_all(
self,
- edits: Iterable[tuple[Edit, str]],
+ edits: Iterable[tuple[Edit, str, int]],
*,
progress=None,
) -> list[EditLabel]:
- tasks = [asyncio.create_task(self.label(e, s)) for e, s in edits]
+ tasks = [asyncio.create_task(self.label(e, s, n)) for e, s, n in edits]
out: list[EditLabel] = []
for i, task in enumerate(asyncio.as_completed(tasks), 1):
label = await task
diff --git a/vero/src/vero/interpret/labeling/taxonomy.py b/vero/src/vero/interpret/labeling/taxonomy.py
index 28363fe9..aae212d1 100644
--- a/vero/src/vero/interpret/labeling/taxonomy.py
+++ b/vero/src/vero/interpret/labeling/taxonomy.py
@@ -19,7 +19,7 @@
import re
from enum import StrEnum
-TAXONOMY_VERSION = "1"
+TAXONOMY_VERSION = "2"
class Role(StrEnum):
@@ -129,3 +129,54 @@ def direction_of(before: str | None, after: str | None) -> Direction:
if not (b and a):
return Direction.NA
return Direction.UP if float(a.group()) > float(b.group()) else Direction.DOWN
+
+
+# Rubrics. Passing bare enum names left the model to infer sixteen role meanings from
+# the names alone, which is the likeliest source of the 16% rule/model disagreement
+# and of `add` swamping `reword`. These go into the JSON schema so the definition
+# travels with the field rather than sitting in a system prompt the model may skim.
+ROLE_RUBRIC: dict[str, str] = {
+ Role.PROMPT: "System/instruction text sent to the model: task guidance, worked "
+ "examples, output-format rules. Prose the model reads, not code.",
+ Role.CONTROL_LOOP: "The agent's main turn loop: how many steps run, when to stop, "
+ "what happens between turns, dispatch of tool calls.",
+ Role.TOOL_SURFACE: "The tool DECLARATIONS shown to the model — names, JSON schemas, "
+ "descriptions. Not the code that implements them.",
+ Role.TOOL_IMPL: "The implementation behind a tool: shell execution, file reading, "
+ "search calls, document opening.",
+ Role.SUBMISSION: "Producing and parsing the final answer: writing the answer file, "
+ "stripping markers, detecting refusals, forcing an answer.",
+ Role.MODEL_CLIENT: "How a completion is requested: client construction, sampling "
+ "parameters, per-request timeouts, SDK retries.",
+ Role.BUDGET_TURNS: "A limit counted in turns or steps.",
+ Role.BUDGET_OUTPUT: "A limit on how much tool output is kept or shown.",
+ Role.BUDGET_WALLCLOCK: "A limit counted in seconds: deadlines, elapsed-time checks.",
+ Role.CONTEXT_MGMT: "Managing conversation history: compaction, pruning, summarising.",
+ Role.RETRIEVAL: "Search or index behaviour: query construction, result count, "
+ "ranking, snippet sizing.",
+ Role.ENV_SETUP: "Preparing the container before work starts: installing packages, "
+ "writing fixtures. NOT ignoring build artifacts.",
+ Role.INITIALIZATION: "Constructor and wiring: state set up once when the agent is "
+ "created.",
+ Role.TESTS: "The candidate's own test suite.",
+ Role.METADATA: "Bookkeeping with no effect on behaviour: version strings, names.",
+ Role.OTHER: "None of the above, or genuinely unclear from this diff alone.",
+}
+
+ACTION_RUBRIC: dict[str, str] = {
+ Action.FIX: "Repairs something broken: a crash, a type error, a parse that returned "
+ "the wrong thing. There must be a defect, not merely a weakness.",
+ Action.ADD: "Introduces a capability or code path that did not exist. Prefer "
+ "`reword` if existing prose was rewritten and `tune` if a value moved.",
+ Action.REMOVE: "Deletes a capability or code path.",
+ Action.TUNE: "Changes a value or threshold, leaving structure intact.",
+ Action.RESTRUCTURE: "Reorganises code without intending to change behaviour: "
+ "extracting a helper, renaming, moving logic.",
+ Action.REWORD: "Rewrites instruction or description TEXT while keeping its intent. "
+ "Use this for prompt edits that sharpen or re-phrase guidance — "
+ "most prompt changes are rewords, not additions.",
+ Action.REVERT: "Undoes an edit the same optimizer made earlier in this cell. Only "
+ "when the history shown says so.",
+ Action.COSMETIC: "Cannot change behaviour: comments, formatting, .gitignore, "
+ "ignoring compiled artifacts, dead code.",
+}
diff --git a/vero/src/vero/interpret/models.py b/vero/src/vero/interpret/models.py
index 4768e26e..16ec367e 100644
--- a/vero/src/vero/interpret/models.py
+++ b/vero/src/vero/interpret/models.py
@@ -105,6 +105,9 @@ class Edit(BaseModel):
before_value: str | None = None # scalar constants only
after_value: str | None = None
diff: str = "" # unified diff restricted to this symbol
+ after_source: str | None = None # the symbol's full text after the edit
+ in_seed: bool = True # did this symbol exist in the seed at all?
+ prior_touches: int = 0 # earlier candidates in this cell that touched it
@staticmethod
def make_id(cell_key: str, sha: str, path: str, symbol: str, diff: str) -> str:
From a9891a3e6c78621bb7011efa4dd78efcce70cd65 Mon Sep 17 00:00:00 2001
From: Varun Ursekar
Date: Mon, 3 Aug 2026 09:35:01 -0700
Subject: [PATCH 11/16] Add print figures in the Scale brand style, vendored
for reproducibility
Separate from the HTML page on purpose: that one is interactive and for colleagues
to explore, this one produces caption-driven vector PDFs sized at the real
placement width for a paper. The style module and both fonts are copied into the
package so figures regenerate from this repo without the skill present.
Archetypes are taken from the brand skill rather than invented. Role prevalence is
a bounded-metric grid, so a sequential heatmap with a group colour bar. Diversity
against the null and knob direction each have two values per item where the gap is
the story, so both are dumbbells. Rarefaction is a plain line plot -- no archetype
fits a saturation curve and forcing one would hide the shape that matters.
Three collisions found by looking at the PNGs and fixed, which is the whole reason
the house rule says to look. Value labels sat on the leftmost tick label, so they
moved above the dots. The diversity legend sat on the terminal-bench row, so it
moved to the empty upper left. And rarefaction's direct labels -- the house
preference -- were illegible mush, because three curves land on exactly 16 kinds
and two on 15, so endpoint labelling cannot separate them; replaced with a legend
ordered by final value. Its x ticks are now integers, since a cell count of 2.5
does not exist.
matplotlib is not in the project environment and this needs no runtime dependency
on it, so rendering runs from a throwaway venv rather than adding one.
Co-Authored-By: Claude Opus 5 (1M context)
---
.../analysis/brand/fonts/GeistMono.ttf | Bin 0 -> 171948 bytes
.../analysis/brand/fonts/HostGrotesk.ttf | Bin 0 -> 81784 bytes
.../analysis/brand/scale_brand_style.py | 142 +++++++++++
.../vero/interpret/analysis/paper_figures.py | 234 ++++++++++++++++++
4 files changed, 376 insertions(+)
create mode 100644 vero/src/vero/interpret/analysis/brand/fonts/GeistMono.ttf
create mode 100644 vero/src/vero/interpret/analysis/brand/fonts/HostGrotesk.ttf
create mode 100644 vero/src/vero/interpret/analysis/brand/scale_brand_style.py
create mode 100644 vero/src/vero/interpret/analysis/paper_figures.py
diff --git a/vero/src/vero/interpret/analysis/brand/fonts/GeistMono.ttf b/vero/src/vero/interpret/analysis/brand/fonts/GeistMono.ttf
new file mode 100644
index 0000000000000000000000000000000000000000..173867dce0580ea751e4b9c558740d34f20ce549
GIT binary patch
literal 171948
zcmcG1349yH)%ectTE1_~w`Co+AXNwzHCwrtC{Ez4(|%Sjw3aqN(b5FiQnl?Do-
z&;sEK5GaL^hLr0ISH66d0;Q00m68ySQVJ=A5Du^Z@6Bi>**I-~zwiHHM!WlV=FOWo
zZ{EE3W@f_?j^mW@BXXUM^$m`%zJJ5Pk=hS9PH;7LcJzc_w)zT=I7&D!qqVW8xAu{m
zfCC&E-^OvpFEzPqy2agGCP!{-1uQvj@SOXFIuD*V!?V4kr=U3Ts_7Rwj=TlWi+Wr&
zuCw32c?-Nh0MCb(u30?!;Jf=nj^QbhW8h(SbW|jT%&*<
zEFZLD_4&&?-VjV2XHaw8$&)J}|H{1fQON%}N4Q;M%NCF7SG@fd)He+IEn@(fIS|+c
z&o{zz#@L$mo0F=W?&CPVl;fgLuU@-!@fD+o%Q#LI1@%X*S-g2te3oAd`42&U^_pet
z7q58i`R6$Pj;-)qFtK>evSrS04M)O6AZO#`+Vj?L7`l0eBXRIPe{gc$vdJ;UwU+_D
zAjlsGGy$r|Hmuuz_1cAz_HQ^r_!J=ANB#Ty@cOZ-^ug~e(nfx#sD(!b$0HQ}gL1+@
zq_=_2{_iZ)Pw_ihIrb8N#q%(acm+AjoKmr#?HwXb`K()oPBr`PE}kDuzD-`P7_(oaTxF
zZ}R_P3IW`Mko4zB_zqK?C&|zMn=95e3h7*RA{^_p`&~QX{y$AwkXKHKLrz$ye^S9S
z^4D>W^sngq-@KD~jyyy>bL%$(Y`&BMaJgOnSGj%Xy2e1c`Oi<*1`s8+>>xwduvcp62=iPsDKLKU@6SQ&WJAh$Z=c^Cf=Q>BS0sQ|W;dqkU
zf_;s&`6v0^d{@jrH~E*H?>-;O%$MfNi@0Uo>3>C8EcrL78{YdRI-Zc{$lw1>@95tFc`zkILfnYUe_lX*nB+R}
z8UzXZ(tmu*@{hU&-{G^~bHy@vkK;j3VQ`Q6a4g~#;W3cpHX|Oa7v&l6aSq7#B$+2z
z2K6fD!*ZAxQxGJq58?P8%VU!3mT@Efhzs-LJzg;n(u;89U9IO?<`Z5q5909eFRZH@
zl1!Wb6=}o#h}Y)1;(M$U@t1fK;=nqQANU@xayfj)dNEI==ZY{aj|uNFp==;ev0T6B
z9`Rr~tQ&F2Is)NxLO9-I-yYPCMBf*
z6z6}M4`tRbNzk7^{248CxbEcpr$3&3l{WDG67Y@T;+st`CcxI&zm8kJYju^Ljei#
zV4CoxYvGFN5lH7l!oIo~((fQ$H|KdbJR{9=8N8l>^cf_?j|tjL)4g#09+C@^8xp>&
z10Jk__P@gSf4>NL@dl(r@cb7@Ly(#oU;TNE_ugmZ^FzKoUbrXE56~v;BdH1S*26vU
z`v;&=0y)IIa{uAIROZ7Y<-`39l!=C9g?F1E86jmtf_Ksm_HOzpv>^x*@)MKXW*pC{
zo)iIBz|Z|1(!=19iI4)|9{T|k=G8;OISBK@dx{gUC6H<$X&^!UG%wbJxVnUeQh6v9y=D!9K@&^7NtdI|DIc@ol$d<42E9O=7>J<~qJXVAqnKd^){
zAm8H^pZ#Iyx~F;Hg1nIY%i}#k8s@+$tcAh--}&EDoUtqq&5v?6rw;kqznuU5Tw45}
z5vP|v|Gomc@SXf#&f{N}1G?lqDF5Djzjn$nxtu>f8FsF#KcD41a$DtR`HFCV-2V6e
zc>MX|pNC*n=U0#(pn)BZdk^k0?S^z3(jLF`8r<)Mv=`F*kftI1$y4rQxX1K6NN+&G
z@^PMPCtUaVC49FY(j}0tgLE^bYa!hLDGSm=kbZ=87W(`F2qhhbzukZV|6T(M_ChUh
zK&~ITkGP}op5r1Q$^=6Mk^m_Vei0nUiNQzV`aQP|;#XU^0`5C!^+({lGuaLj-DZG^*;D5(I#UJJWB8Wnm5F?}rIw4Ca6@Do^AUq`;7L!Gt
zXcOzih2m0imAFCtxp=eqOYv^;G4XNnN%64ws`$3}q4+oP%ZMtKMs=C$cGY+4ICZjG
zrPiwT>TGqP+N8Fr>(!I$^VL_W_o*LJKdL^U{+;@1O`s-0qt*0kMl`o*Zqw}3JgWJv
z<|EA~T3)NrMrvcU$y${*S8LHWX%}gi=?uDD-76Vk8M=(IjL$M}%)BG>vCPLapU!+X
z^X1G}4Hp#(g{k*ldv<1K26&+63OO6_?_m;8@<|CPBQ?ZHI%(VYkl&C$kdu6(x9vyx
zw**d5c-pQP%(U%KLfbhpMa&S(#d>jxI4-UecZs{jTgAJ?hsA^96SVDbi6_KQ#V?@k
zYSlJq`yO?yI#HdfRzusfpzX!#GTQd_>TPq{9uIBr(F|#JLE9hDJfe9_bJE-P=vi$a
z)#+*5L!s@9GCs?g$-F6Zf9Ao=r!o&`z65RGNZWo5YdgkMfBK)amt5$BN}ov&lm8(v
z$`H~>stA-LxzbVTNsg01=BHnQ^tGI0_I0{<`WhK>`j2>b{3DJ#@eYHXc=W`LAGUs&
zbmFxWeJ85nPk-X^6OWwu^@#^g+;`%x+ih|<1eHZM!@$#
ze>}OJ9OSq0`}jxrgZyFs6&^a^9?}u{D$%$2t&dJLY)xC|C9fUe_t5o
z-{3#z|HL2X-{g-8tAsIOoPS3^P9<^4FhVjo3uoo3xLU4_>*RVkuvWN*+(vFQcRqI^
zcPV!jXv*8TySTf-!uW`pE(^Nv6m~vWZ;Ge=W4}r-e!WN&anNKv*IS3QL7{p+Q_HILMnqpfDvgidzMz
zFv0KV9}_nS=L_}xd*X%sqvV(n#NWaHir>S(%f*1s3*i!A6(XKX6k`4ElD3u$()O`x$pN=-TVJd$>)ck2?u^^gZrx+zIYe?lbO7m<_(=
zzT;*{3dl_YN#wo(Jy`+T!VH?STv!34VksFRi^yWKnY55CpkY4-UHc}dgXD1O{ISJ=#NGw-PVz^e4
z&N)dM*FsdBgDANUqT{-Wo^z2b&P{T-UXsoAkz8(&U)a2JtoZaeAawh-Y@bz-RHqW0@%#{<^IBjbAN?d`vWeLI}UUA`>?`y4Cd&!U|s5ME{J=F3*i3D6%fJ|
z5rHcq3a|^lht;GXxm>V1jGV+}b3bs+B$aC<$=n#RbK|gXw1il=QPA4UNEx@B*tn&{
z%B?2V+ytrR){;7IlGJl+NDa4>Ea7&MQSJt^oV$^XaW|2b+-@??{hX}e?k8Kh`^W`g
zcU;KrBNqt)Fbfhv084-uI)qNJ3A%(P{x<%0{!Za?VVkf+xKy}OxJtM}*e+ZmTqbN1
zmI-Tw)q-1a3H?G3zfO!3z7RFS4`Q-#T1*kX5fg-eiiyHkVytjNcu)9;7$bZjyd!)r
zs$o_&3*U?BVwq?X&cZC)D3%C+5$lByMVlaj&EXJ>g?Gg|;kZ~TOpDPl7wd(;3U7MR
zydu0Vd?LIh`~kH4pM{sjO0hurqgWy43$KYzu|{}PY!<79H-wLbW5Vx6yO<}uDmIB#
z!k<9r2a3U<_rpQ!hl)xuRZJ42gfC%s_*(c(j2F{jrua^{Pq;(aBit$6CEP9CBit+O
z6@DW;Ec{w{NO(}#2Nu;MU{gIUJR{sMJTLsW@Pcqucv1Kt;fU~(@SJd1c$WJOx#SR;
zx$F@UO-0Z&`C!L5xNgvCYd|kw#$69O`2p?__X604?|~ls8uV8vXfGWpAmwBjESzV^
zOXL_iPX0l@1G&uP3t>!jfsC%?H}Ko}U7$^>`SvK_3xwJ^%B0K4yYn3WES&xx;y?}%pu#DJ)PynwQR`T$qJqJUKa
z8w0in+!1gn;02ha-VOLf5v8bAv?~S`%M}|G+ZER-Zc*H;ctr7(;;7;c#Yx52fkI$x
zpf1o9SQY3DbO+uT_;}!ZLGeM2K|MhWgT{mQ2R#$?QqY@0$Adl(P7BTtE(>l5?hGCZ
zK0o-f;Om2L3w|K@K=9$7ap+f}Qdn?Ud01muPuNJ<
zc-Y3U%ffC7+Y|Os*pp$u5Bqc2iLfuir0|e%Wq3w-NqAj&Pxwgq%JB2UuL{30d{6j8
z;on4X5up+B5g8E$5w?hih|Y+ih~*LMBQB1(I^w2?JrNH^9E^B2;^m09B0h}xBC;*=
z^2pyt9*+EdteUZUK@LJ?4z+y#~zLSAolavZ{y