diff --git a/harness-engineering-bench/figures/figure_01_prevalence/figure_01_prevalence.md b/harness-engineering-bench/figures/figure_01_prevalence/figure_01_prevalence.md new file mode 100644 index 00000000..8e86080b --- /dev/null +++ b/harness-engineering-bench/figures/figure_01_prevalence/figure_01_prevalence.md @@ -0,0 +1,85 @@ +--- +id: figure_01_prevalence +archetype: heatmap +script: vero.interpret.analysis.paper_figures::fig_prevalence +outputs: [figure_01_prevalence.pdf, figure_01_prevalence.png] +status: review +--- + +## Takeaway + +Every optimizer edits the instruction prompt and the control loop, but which other +parts of the harness they touch is dictated by the benchmark, not by the optimizer. + +## Caption + +Share of optimization runs that made at least one edit of each kind, per benchmark +(20 runs each, 100 total). Cell shading is the share; the annotation is the count. +Prompt and control-loop edits are near-universal, while the remaining categories vary +sharply by benchmark — submission-path edits appear in 15 of 20 swe-atlas-qna runs but +2 of 20 terminal-bench runs, and retrieval edits appear almost only in +browsecomp-plus, the sole benchmark with a retrieval corpus. Counted per run rather +than per edit, since runs produced between 1 and 18 candidates and an edit-weighted +count would measure verbosity instead of coverage. The left colour bar groups +categories. \textsuperscript{*}gaia-shell's seed is an empty skeleton, so every +category is present there by construction and its column is not comparable with the +others. Reward is not shown; see \Cref{tab:signal-validity} for why score-versus-category +comparisons are unsupportable in this corpus. + +## What the reader should see + +- Read down the first two rows first: `prompt` and `control_loop` are dark across every + column. That is the universal behaviour. +- Then read across `submission`, `retrieval`, `tool_impl` — the variance between columns + is the finding. Benchmark, not optimizer, selects the target. +- Shading encodes the same quantity as the annotation, deliberately: shading carries the + pattern at a glance, the fraction carries the exact read. Nothing else is encoded. +- `gaia-shell` is marked and excluded from any cross-benchmark claim. +- Absent by design: reward, and any ordering of benchmarks by score. + +## Data + +Cells that made ≥1 edit of each kind, out of 20 per benchmark. Rows ordered by mean +share across benchmarks, which is the order the figure uses. + +| role | BrowseComp-Plus | OfficeQA | SWE-Atlas-QnA | Terminal-Bench | GAIA-Shell‡ | +|---|---|---|---|---|---| +| Prompt | 20/20 | 19/20 | 20/20 | 20/20 | 18/20 | +| Control Loop | 17/20 | 19/20 | 18/20 | 17/20 | 20/20 | +| Turn Budget | 15/20 | 17/20 | 16/20 | 18/20 | 11/20 | +| Tool Surface | 15/20 | 7/20 | 15/20 | 10/20 | 18/20 | +| Tool Implementation | 16/20 | 7/20 | 11/20 | 10/20 | 18/20 | +| Other | 14/20 | 8/20 | 9/20 | 9/20 | 20/20 | +| Tests | 11/20 | 7/20 | 14/20 | 10/20 | 16/20 | +| Model Client | 11/20 | 14/20 | 10/20 | 8/20 | 11/20 | +| Metadata | 9/20 | 7/20 | 6/20 | 11/20 | 19/20 | +| Submission | 7/20 | 8/20 | 15/20 | 2/20 | 18/20 | +| Output Cap | 5/20 | 13/20 | 11/20 | 7/20 | 7/20 | +| Wall-Clock Budget | 5/20 | 7/20 | 7/20 | 6/20 | 12/20 | +| Context Management | 7/20 | 7/20 | 6/20 | 10/20 | 7/20 | +| Initialization | 6/20 | 4/20 | 4/20 | 5/20 | 17/20 | +| Environment Setup | 7/20 | 5/20 | 3/20 | 7/20 | 13/20 | +| Retrieval | 9/20 | 0/20 | 1/20 | 0/20 | 2/20 | + +## Style notes + +- Column tick labels sit on top, not bottom: with 16 rows the eye enters at the top and + the header should be adjacent to the first row it applies to. +- Benchmark names are abbreviated to keep all five columns inside the text width without + rotation; rotated column headers cost more legibility than the abbreviation does. + +## Provenance + +Identifiers are snake_case in the data (`control_loop`, `browsecomp-plus`) and +title-cased for display via `analysis.display`; the table above uses the display +names, which are what the figure shows. + +``` +vero interpret extract --runs runs/{officeqa,browsecomp-plus,terminal-bench,swe-atlas-qna,gaia-shell} \ + --cells-file scope100.json +vero interpret edits +vero interpret label --model gpt-5.4-mini +python -c "from vero.interpret.analysis import stats; stats.prevalence(rows)" +``` +3,986 symbol-scoped edits over 100 runs. Roles set by deterministic rule where the +path, symbol kind or name settles it (2,114 edits) and by model otherwise (1,872). diff --git a/harness-engineering-bench/figures/figure_02_diversity/figure_02_diversity.md b/harness-engineering-bench/figures/figure_02_diversity/figure_02_diversity.md new file mode 100644 index 00000000..2267c803 --- /dev/null +++ b/harness-engineering-bench/figures/figure_02_diversity/figure_02_diversity.md @@ -0,0 +1,67 @@ +--- +id: figure_02_diversity +archetype: dumbbell +script: vero.interpret.analysis.paper_figures::fig_diversity +outputs: [figure_02_diversity.pdf, figure_02_diversity.png] +status: review +--- + +## Takeaway + +Independent optimizers converge on more similar repertoires of edits than chance +allows, in every benchmark. + +## Caption + +Mean pairwise Jaccard distance between the sets of edit categories that different +optimization runs touched (filled marker), against a permutation null that holds each +run's repertoire size and the corpus-wide category frequencies fixed and reshuffles the +assignment (open marker, null mean; band, null 95\% interval). All five benchmarks fall +below their null, so runs are more alike than independent draws from the same marginal +would be. The raw distance alone carries no information without the null: a value near +0.5 is equally consistent with genuine diversity and with every run sampling a few +categories from one skewed distribution. 20 runs per benchmark, 190 pairs each. + +## What the reader should see + +- The filled marker sits left of the grey band in every row. That gap is the whole + finding: left of the null means more similar than chance. +- Position on the x-axis is the only quantitative channel. The connector shows the size + of the departure from the null mean; it encodes nothing extra. +- Open versus filled distinguishes null from observed. Colour does not carry benchmark + identity here, because the y-axis label already does. +- Absent by design: a significance star or p-value. The null interval is shown directly + so the reader judges the margin rather than a threshold. + +## Data + +| benchmark | runs | observed | null 2.5% | null 97.5% | null mean | verdict | +|---|---|---|---|---|---|---| +| BrowseComp-Plus | 20 | 0.561 | 0.630 | 0.667 | 0.651 | converged | +| OfficeQA | 20 | 0.564 | 0.663 | 0.709 | 0.687 | converged | +| SWE-Atlas-QnA | 20 | 0.538 | 0.601 | 0.653 | 0.630 | converged | +| Terminal-Bench | 20 | 0.572 | 0.643 | 0.691 | 0.670 | converged | +| GAIA-Shell | 20 | 0.341 | 0.399 | 0.446 | 0.425 | converged | + +500 permutations per benchmark, seed 0. + +## Style notes + +- Value labels sit above the marker, not beside it: gaia-shell's observed value is at + the far left of the axis and a left-placed label collided with its tick label. +- Legend is upper-left. Lower-right — the usual choice — overlapped the terminal-bench + row, whose null band extends furthest right. +- gaia-shell is included here, unlike Figure 1, because this statistic is computed + within a benchmark and never compared across them, so its constructed seed does not + distort it. + +## Provenance + +Identifiers are snake_case in the data (`control_loop`, `browsecomp-plus`) and +title-cased for display via `analysis.display`; the table above uses the display +names, which are what the figure shows. + +``` +python -c "from vero.interpret.analysis import stats; stats.jaccard(rows, trials=500, seed=0)" +``` +Same 3,986-edit label set as Figure 1. diff --git a/harness-engineering-bench/figures/figure_03_rarefaction/figure_03_rarefaction.md b/harness-engineering-bench/figures/figure_03_rarefaction/figure_03_rarefaction.md new file mode 100644 index 00000000..ea5cfa0c --- /dev/null +++ b/harness-engineering-bench/figures/figure_03_rarefaction/figure_03_rarefaction.md @@ -0,0 +1,76 @@ +--- +id: figure_03_rarefaction +archetype: small_multiples +script: vero.interpret.analysis.paper_figures::fig_rarefaction +outputs: [figure_03_rarefaction.pdf, figure_03_rarefaction.png] +status: review +--- + +## Takeaway + +Five independent optimizers exhaust almost the entire repertoire of edit categories; +the next fifteen add nearly nothing. + +## Caption + +Distinct edit categories discovered as optimization runs are added, one panel per +benchmark (16 categories available). Line, mean over 200 random orderings of the 20 +runs; band, the 10th-90th percentile across those orderings — the spread attributable +to which runs happen to come first. Every curve is within one category of its final +value by the fifth run and flat thereafter, and the band collapses over the same +interval: at one run the number of categories seen spans roughly 2 to 16, by ten runs +it is a single value. The marginal contribution of an additional independent optimizer +is therefore close to zero, and that conclusion does not depend on the draw. Same +convergence \Cref{fig:diversity} establishes against a null, seen as saturation rather +than as a distance. + +## What the reader should see + +- The shape, not the ordering: every curve bends hard before x=5 and is flat by x=8. +- Vertical position at the right edge is the ceiling each benchmark reached (15 or 16 of + 16). The gap between curves is not the point and should not be over-read. +- The band narrowing left-to-right is as much the finding as the curve flattening: it + says the result is insensitive to which runs you happened to have. +- Colour distinguishes benchmark only, redundantly with the panel title. Panels share + both axes so heights are directly comparable across them. +- Absent by design: a cross-benchmark overlay. The claim is per-benchmark, and five + overlapping bands produced grey that attributed to no series. + +## Data + +Mean distinct categories after k runs, of 16 available. + +Mean, with the 10th-90th percentile across orderings in brackets. + +| benchmark | k=1 | k=2 | k=5 | k=10 | k=20 | +|---|---|---|---|---|---| +| BrowseComp-Plus | 8.3 [2-16] | 12.0 [8-16] | 15.2 [13-16] | 16.0 [16-16] | 16.0 [16-16] | +| OfficeQA | 7.7 [3-15] | 10.6 [6-14] | 13.9 [11-15] | 14.9 [15-15] | 15.0 [15-15] | +| SWE-Atlas-QnA | 8.8 [3-14] | 11.7 [8-15] | 14.3 [12-16] | 15.4 [14-16] | 16.0 [16-16] | +| Terminal-Bench | 7.5 [2-14] | 10.5 [6-14] | 13.8 [12-15] | 14.7 [14-15] | 15.0 [15-15] | +| GAIA-Shell | 11.4 [8-15] | 13.9 [11-16] | 15.2 [14-16] | 15.8 [15-16] | 16.0 [16-16] | + +200 random orderings per benchmark, seed 0. + +## Style notes + +- Small multiples rather than one overlaid axis. Adding percentile bands to five + overlaid series produced unattributable grey below k=5 and stretched the y-axis to 2, + compressing the saturation region that carries the claim. Faceting also retires the + earlier problem that made a legend necessary — three benchmarks land on exactly 16.0 + categories, so endpoint labels could not separate them. +- Percentiles, not a standard deviation. The quantity is a bounded count skewed hard + against its ceiling; a symmetric band would extend past the 16 categories available. +- x ticks are 1/10/20 only. Panels are ~1.2 in wide and a denser locator produced + half-steps, which is not a run count. + +## Provenance + +Identifiers are snake_case in the data (`control_loop`, `browsecomp-plus`) and +title-cased for display via `analysis.display`; the table above uses the display +names, which are what the figure shows. + +``` +python -c "from vero.interpret.analysis import stats; stats.rarefaction_bands(rows, trials=200, seed=0)" +``` +Same 3,986-edit label set as Figure 1. diff --git a/harness-engineering-bench/figures/figure_04_knob_direction/figure_04_knob_direction.md b/harness-engineering-bench/figures/figure_04_knob_direction/figure_04_knob_direction.md new file mode 100644 index 00000000..b91dc305 --- /dev/null +++ b/harness-engineering-bench/figures/figure_04_knob_direction/figure_04_knob_direction.md @@ -0,0 +1,67 @@ +--- +id: figure_04_knob_direction +archetype: dumbbell +script: vero.interpret.analysis.paper_figures::fig_knob_direction +outputs: [figure_04_knob_direction.pdf, figure_04_knob_direction.png] +status: review +--- + +## Takeaway + +Optimizers raise turn and step budgets almost without exception, but move output caps +and timeouts in both directions. + +## Caption + +Numeric constants most often changed, by direction of change (filled marker, raised; +open marker, lowered). Turn and step budgets move overwhelmingly upward — `MAX_TURNS` +raised in 63 edits against 14 lowered — while output-truncation caps and per-command +timeouts are as often reduced as increased. Direction is derived by comparing the +literal before and after values, not inferred from the commit message. Constants +touched by reformatting without a value change are excluded. Counted per edit rather +than per run, since one run may retune the same constant several times and each +retuning is a separate decision. + +## What the reader should see + +- The top two rows against the rest: budgets go up, everything else is mixed. +- `MAX_TOOL_OUTPUT_CHARS` is the notable inversion — lowered slightly more often than + raised, the only frequently-touched constant where that holds. +- Horizontal position is a count, and the connector's length is the imbalance between + the two directions. Open versus filled is the only other channel. +- Absent by design: the magnitudes of the changes. A constant moved 24→100 and 24→32 + count the same here; direction is the claim, size is not. + +## Data + +| constant | raised | lowered | +|---|---|---| +| MAX_TURNS | 63 | 14 | +| MAX_STEPS | 26 | 5 | +| MAX_TOOL_OUTPUT_CHARS | 13 | 16 | +| MAX_OUTPUT_CHARS | 3 | 3 | +| SOFT_DEADLINE_SEC | 2 | 3 | +| COMMAND_TIMEOUT_SEC | 1 | 4 | +| N_ATTEMPTS | 1 | 2 | +| MAX_HISTORY_CHARS | 0 | 2 | +| MAX_CONCURRENT_SHELLS | 2 | 0 | +| RESEARCH_DEADLINE_SEC | 0 | 2 | + +Corpus totals across all scalar constants: 137 raised, 78 lowered. + +## Style notes + +- Top 10 constants only, by total edits. The tail is single-digit and would add rows + without adding signal; the cut is stated here so it is not read as the full set. + +## Provenance + +Identifiers are snake_case in the data (`control_loop`, `browsecomp-plus`) and +title-cased for display via `analysis.display`; the table above uses the display +names, which are what the figure shows. + +``` +python -c "from vero.interpret.analysis import stats; stats.tuning_direction(rows, edits, top=10)" +``` +Direction from `taxonomy.direction_of` over captured before/after literals; same +3,986-edit set as Figure 1. diff --git a/vero/pyproject.toml b/vero/pyproject.toml index fb743449..6da8e97d 100644 --- a/vero/pyproject.toml +++ b/vero/pyproject.toml @@ -29,6 +29,9 @@ harbor = [ claude = [ "claude-agent-sdk>=0.1.56", ] +interpret = [ + "openai>=1.0", +] optimize = [ "async-lru>=2.0.5", "beautifulsoup4>=4.14.2", diff --git a/vero/src/vero/interpret/__init__.py b/vero/src/vero/interpret/__init__.py new file mode 100644 index 00000000..72261d06 --- /dev/null +++ b/vero/src/vero/interpret/__init__.py @@ -0,0 +1,26 @@ +"""Interpretability analysis over optimization runs. + +Source artifacts are canonicalised by an adapter (`artifacts`), split into +symbol-scoped edits deterministically (`edits`), labelled with a model +(`labeling`), and aggregated (`analysis`). Only `labeling` is non-deterministic. +""" + +from vero.interpret.models import ( + Candidate, + CellRef, + Corpus, + Edit, + EvalRecord, + SymbolKind, + Trajectory, +) + +__all__ = [ + "Candidate", + "CellRef", + "Corpus", + "Edit", + "EvalRecord", + "SymbolKind", + "Trajectory", +] diff --git a/vero/src/vero/interpret/analysis/__init__.py b/vero/src/vero/interpret/analysis/__init__.py new file mode 100644 index 00000000..87387cef --- /dev/null +++ b/vero/src/vero/interpret/analysis/__init__.py @@ -0,0 +1,5 @@ +"""Aggregation and figures over labelled edits.""" + +from vero.interpret.analysis import figures, stats + +__all__ = ["figures", "stats"] diff --git a/vero/src/vero/interpret/analysis/brand/fonts/GeistMono.ttf b/vero/src/vero/interpret/analysis/brand/fonts/GeistMono.ttf new file mode 100644 index 00000000..173867dc Binary files /dev/null and b/vero/src/vero/interpret/analysis/brand/fonts/GeistMono.ttf differ diff --git a/vero/src/vero/interpret/analysis/brand/fonts/HostGrotesk.ttf b/vero/src/vero/interpret/analysis/brand/fonts/HostGrotesk.ttf new file mode 100644 index 00000000..25174036 Binary files /dev/null and b/vero/src/vero/interpret/analysis/brand/fonts/HostGrotesk.ttf differ diff --git a/vero/src/vero/interpret/analysis/brand/house_style.py b/vero/src/vero/interpret/analysis/brand/house_style.py new file mode 100644 index 00000000..bf2564fa --- /dev/null +++ b/vero/src/vero/interpret/analysis/brand/house_style.py @@ -0,0 +1,137 @@ +"""House plot style: muted palette on white, sans throughout, print-legible. + +Palette, typography and rcParams for publication figures. Fonts are bundled under +``fonts/`` (OFL, redistributable) and registered at import, so no system install is +needed. Figures are designed at the target placement width so point sizes render 1:1. +""" +from __future__ import annotations + +import os +import matplotlib +import matplotlib.pyplot as plt +from matplotlib import font_manager + +# ---- paper geometry (letter, 0.82in L/R margins) ---- +TEXT_WIDTH_IN = 6.86 # \textwidth == \linewidth (single column) + +# ---- fonts: brand Aeonik -> OSS fallbacks Host Grotesk / Geist Mono ---- +# Prefer the copies bundled next to this module (assets/fonts); fall back to a +# a user font directory. Both are OFL and redistributable. +_FONT_DIRS = [ + os.path.join(os.path.dirname(os.path.abspath(__file__)), "fonts"), + os.path.expanduser("~/.fonts/house"), +] +_FILES = ("HostGrotesk.ttf", "GeistMono.ttf") + +FAMILY = "sans-serif" # -> Host Grotesk +MONO = "monospace" # -> Geist Mono + +# ---- house palette (the house palette) ---- +PALETTE = { + "black": "#000000", "white": "#FFFFFF", + "evergreen": "#193A29", # Evergreen Core + "atlas": "#273252", # Atlas Blue + "tan": "#A8927C", # Foundry Tan + "purple": "#79648C", # Archive Purple + "slate": "#839CB2", # Cloud Slate + "gray_soft": "#929292", "gray_medium": "#C7C7C7", "gray_whisper": "#EAEAEA", +} +INK = "#111111" # titles / body +INK_SUB = "#5C5C5C" # subtitle +INK_FAINT = "#7A7A7A" # source note / faint furniture + +CATEGORICAL = [PALETTE["atlas"], PALETTE["tan"], PALETTE["evergreen"], + PALETTE["purple"], PALETTE["slate"], PALETTE["gray_soft"]] + +# Print-legible type scale (points, at 1:1 final size) +FS = {"title": 11.0, "subtitle": 8.2, "axis": 8.5, "tick": 8.0, + "label": 8.0, "value": 7.2, "annot": 7.5, "legend": 7.8, "source": 6.8} + + +def _register_fonts(): + for f in _FILES: + for d in _FONT_DIRS: + p = os.path.join(d, f) + if os.path.exists(p): + try: + font_manager.fontManager.addfont(p) + except Exception: + pass + break # first copy found wins + names = {f.name for f in font_manager.fontManager.ttflist} + sans = [n for n in ("Host Grotesk", "Helvetica Neue", "Helvetica", "Arial") if n in names] + sans.append("DejaVu Sans") + mono = [n for n in ("Geist Mono",) if n in names] + ["Menlo", "DejaVu Sans Mono"] + return sans, mono + + +def apply_house_style(): + sans, mono = _register_fonts() + plt.rcParams.update({ + "font.family": "sans-serif", + "font.sans-serif": sans, + "font.monospace": mono, + "font.size": FS["tick"], + "text.color": INK, + "axes.edgecolor": PALETTE["gray_medium"], + "axes.labelcolor": INK, + "axes.labelsize": FS["axis"], + "axes.linewidth": 0.8, + "axes.grid": False, + "axes.spines.top": False, + "axes.spines.right": False, + "xtick.color": INK_FAINT, "ytick.color": INK, + "xtick.labelcolor": INK, "ytick.labelcolor": INK, + "xtick.labelsize": FS["tick"], "ytick.labelsize": FS["tick"], + "grid.color": PALETTE["gray_whisper"], "grid.linewidth": 0.7, + "figure.facecolor": PALETTE["white"], "axes.facecolor": PALETTE["white"], + "savefig.facecolor": PALETTE["white"], "savefig.dpi": 400, + "legend.frameon": False, "legend.fontsize": FS["legend"], + "lines.linewidth": 1.3, + "pdf.fonttype": 42, "ps.fonttype": 42, "svg.fonttype": "none", + }) + + +def title_block(fig, title, subtitle=None, x=0.012, y=0.975, sub_dy=0.052): + """Bold takeaway title with a lighter gray finding line above it.""" + if subtitle: + fig.text(x, y, subtitle, ha="left", va="top", + fontsize=FS["subtitle"], color=INK_SUB) + fig.text(x, y - sub_dy, title, ha="left", va="top", + fontsize=FS["title"], fontweight="bold", color=INK) + else: + fig.text(x, y, title, ha="left", va="top", + fontsize=FS["title"], fontweight="bold", color=INK) + + +def source_note(fig, text, x=0.5, y=0.013): + fig.text(x, y, text, ha="center", va="bottom", + fontsize=FS["source"], style="italic", color=INK_FAINT) + + +# Ordered accent sequence for categorical encoding (distinct, muted, print-safe). +# Keep to as few as the data needs; black/white and the gray ramp do most work. +ACCENTS = [PALETTE["atlas"], PALETTE["tan"], PALETTE["purple"], + PALETTE["evergreen"], PALETTE["slate"], PALETTE["gray_soft"]] + + +def family_colors(categories): + """Map category names -> accent colours in first-seen order. + + Use this to colour a series by group (e.g. harness family, cohort, method) + with a stable, brand-consistent palette instead of hand-picking colours. + """ + seen = [] + for c in categories: + if c not in seen: + seen.append(c) + return {c: ACCENTS[i % len(ACCENTS)] for i, c in enumerate(seen)} + + +def save(fig, path_no_ext, png_dpi=200): + """Save a figure as vector PDF (drop-in for LaTeX) plus a PNG preview. + + Pass a path without extension; writes ``.pdf`` and ``.png``. + """ + fig.savefig(f"{path_no_ext}.pdf") + fig.savefig(f"{path_no_ext}.png", dpi=png_dpi) diff --git a/vero/src/vero/interpret/analysis/display.py b/vero/src/vero/interpret/analysis/display.py new file mode 100644 index 00000000..4c499a91 --- /dev/null +++ b/vero/src/vero/interpret/analysis/display.py @@ -0,0 +1,88 @@ +"""Display names for machine identifiers. + +The taxonomy uses snake_case values because they are dictionary keys, cache-key +components and JSON fields; renaming them would invalidate every cached label and +break the data contract in the figure specs. Presentation is a separate concern, so +the mapping lives here and both the print figures and the HTML page read from it. + +Benchmark names follow the published spelling rather than a mechanical +transformation: these appear in a paper, where `Terminal-Bench` and `GAIA` are the +names readers will look up. + +Code symbols are deliberately absent. `MAX_TURNS` is a real identifier in the +optimized source, and prettifying it would misrepresent what the optimizer edited. +""" + +from __future__ import annotations + +ROLE: dict[str, str] = { + "prompt": "Prompt", + "control_loop": "Control Loop", + "tool_surface": "Tool Surface", + "tool_impl": "Tool Implementation", + "submission": "Submission", + "model_client": "Model Client", + "budget_turns": "Turn Budget", + "budget_output": "Output Cap", + "budget_wallclock": "Wall-Clock Budget", + "context_mgmt": "Context Management", + "retrieval": "Retrieval", + "env_setup": "Environment Setup", + "initialization": "Initialization", + "tests": "Tests", + "metadata": "Metadata", + "other": "Other", +} + +ACTION: dict[str, str] = { + "fix": "Fix", + "add": "Add", + "remove": "Remove", + "tune": "Tune", + "restructure": "Restructure", + "reword": "Reword", + "revert": "Revert", + "cosmetic": "Cosmetic", +} + +PROVENANCE: dict[str, str] = { + "seed": "Seed defect", + "own": "Own defect", + "unknown": "Unknown", +} + +BENCHMARK: dict[str, str] = { + "browsecomp-plus": "BrowseComp-Plus", + "officeqa": "OfficeQA", + "swe-atlas-qna": "SWE-Atlas-QnA", + "terminal-bench": "Terminal-Bench", + "gaia-shell": "GAIA-Shell", +} + +# Shorter forms for axis ticks where the full name will not fit five across a +# text-width figure. Same names, dropped qualifiers -- never a different name. +BENCHMARK_SHORT: dict[str, str] = { + "browsecomp-plus": "BrowseComp+", + "officeqa": "OfficeQA", + "swe-atlas-qna": "SWE-Atlas", + "terminal-bench": "Terminal-Bench", + "gaia-shell": "GAIA-Shell", +} + + +def role(key: str) -> str: + """Display name, falling back to a readable form for anything unmapped.""" + return ROLE.get(key, key.replace("_", " ").title()) + + +def action(key: str) -> str: + return ACTION.get(key, key.replace("_", " ").title()) + + +def provenance(key: str) -> str: + return PROVENANCE.get(key, key.replace("_", " ").title()) + + +def benchmark(key: str, *, short: bool = False) -> str: + table = BENCHMARK_SHORT if short else BENCHMARK + return table.get(key, key) diff --git a/vero/src/vero/interpret/analysis/figures.py b/vero/src/vero/interpret/analysis/figures.py new file mode 100644 index 00000000..a4d0b5ae --- /dev/null +++ b/vero/src/vero/interpret/analysis/figures.py @@ -0,0 +1,365 @@ +"""Render the analysis as one self-contained HTML page. + +Inline SVG, no build step and no CDN: the output is a single file that opens from +disk and can be handed to someone. Colour follows the validated reference palette — +sequential blue for magnitude, the fixed categorical order for identity, blue/red +diverging for polarity — and every chart carries a table view, because several +palette steps sit below 3:1 on the light surface and the relief rule applies. +""" + +from __future__ import annotations + +import html +import json +from collections import Counter + +from vero.interpret.analysis import display, preamble, stats + +CAT_LIGHT = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#4a3aa7", "#e34948"] +CAT_DARK = ["#3987e5", "#d95926", "#199e70", "#c98500", "#d55181", "#008300", "#9085e9", "#e66767"] +SEQ = ["#cde2fb", "#b7d3f6", "#9ec5f4", "#86b6ef", "#6da7ec", "#5598e7", "#3987e5", "#2a78d6", "#256abf", "#1c5cab", "#184f95"] +POS, NEG = "#2a78d6", "#e34948" + +CSS = """ +.viz-root{color-scheme:light;--surface-1:#fcfcfb;--surface-2:#f4f3f0;--text-primary:#0b0b0b; +--text-secondary:#52514e;--text-muted:#7a7975;--grid:#e6e5e1;--pos:#2a78d6;--neg:#e34948; +background:var(--surface-1);color:var(--text-primary); +font:14px/1.5 ui-sans-serif,-apple-system,"Segoe UI",Roboto,sans-serif;padding:28px 32px;max-width:1180px;margin:0 auto} +@media (prefers-color-scheme:dark){:root:where(:not([data-theme="light"])) .viz-root{color-scheme:dark; +--surface-1:#1a1a19;--surface-2:#242422;--text-primary:#fff;--text-secondary:#c3c2b7; +--text-muted:#8f8e86;--grid:#343430;--pos:#3987e5;--neg:#e66767}} +:root[data-theme="dark"] .viz-root{color-scheme:dark;--surface-1:#1a1a19;--surface-2:#242422; +--text-primary:#fff;--text-secondary:#c3c2b7;--text-muted:#8f8e86;--grid:#343430;--pos:#3987e5;--neg:#e66767} +h1{font-size:22px;margin:0 0 4px} h2{font-size:16px;margin:34px 0 2px} +p.note{color:var(--text-secondary);margin:2px 0 14px;max-width:74ch;font-size:13px} +.fig{background:var(--surface-1);border:1px solid var(--grid);border-radius:10px;padding:14px 16px;margin-bottom:6px} +.legend{display:flex;gap:16px;flex-wrap:wrap;margin:8px 0 0;font-size:12px;color:var(--text-secondary)} +.legend i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:6px;vertical-align:-1px} +text{font:11px ui-sans-serif,-apple-system,sans-serif} +.axis{fill:var(--text-secondary)} .muted{fill:var(--text-muted)} .ink{fill:var(--text-primary)} +.grid{stroke:var(--grid);stroke-width:1} +details{margin:6px 0 0} summary{cursor:pointer;color:var(--text-secondary);font-size:12px} +table{border-collapse:collapse;font-size:12px;margin-top:8px} th,td{border:1px solid var(--grid);padding:3px 8px;text-align:right} +th:first-child,td:first-child{text-align:left} th{color:var(--text-secondary);font-weight:600} +#tip{position:fixed;pointer-events:none;opacity:0;background:var(--surface-2);color:var(--text-primary); +border:1px solid var(--grid);border-radius:6px;padding:6px 9px;font-size:12px;max-width:320px; +box-shadow:0 2px 10px rgba(0,0,0,.16);z-index:50;transition:opacity .08s} +.hit{cursor:crosshair} +.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px} +.fig table{width:100%} +.toggle{float:right;font-size:12px;color:var(--text-secondary);cursor:pointer;border:1px solid var(--grid); +border-radius:6px;padding:3px 9px;background:var(--surface-2)} +""" + +JS = """ +const tip=document.getElementById('tip'); +document.querySelectorAll('[data-tip]').forEach(el=>{ + el.addEventListener('mousemove',e=>{tip.innerHTML=el.dataset.tip;tip.style.opacity=1; + const p=12;let x=e.clientX+p,y=e.clientY+p; + if(x+tip.offsetWidth>innerWidth)x=e.clientX-tip.offsetWidth-p; + if(y+tip.offsetHeight>innerHeight)y=e.clientY-tip.offsetHeight-p; + tip.style.left=x+'px';tip.style.top=y+'px';}); + el.addEventListener('mouseleave',()=>tip.style.opacity=0);}); +document.getElementById('themebtn').onclick=()=>{ + const r=document.documentElement; + const dark=r.getAttribute('data-theme')==='dark'; + r.setAttribute('data-theme',dark?'light':'dark');}; +""" + + +def _esc(s) -> str: + return html.escape(str(s), quote=True) + + +def _seq(frac: float) -> str: + return SEQ[max(0, min(len(SEQ) - 1, round(frac * (len(SEQ) - 1))))] + + +def _table(headers: list[str], rows: list[list], caption: str) -> str: + head = "".join(f"{_esc(h)}" for h in headers) + body = "".join( + "" + "".join(f"{_esc(c)}" for c in r) + "" for r in rows + ) + return ( + f"
{_esc(caption)}" + f"{head}{body}
" + ) + + +def fig_prevalence(rows: list[dict]) -> str: + roles, table = stats.prevalence(rows) + benches = [b for b in stats.BENCH_ORDER if b in next(iter(table.values()))] + cw, ch, lw, top = 132, 26, 150, 46 + w, h = lw + cw * len(benches) + 30, top + ch * len(roles) + 26 + out = [f''] + for j, b in enumerate(benches): + x = lw + j * cw + cw / 2 + mark = " ‡" if b in stats.CONSTRUCTED_SEED else "" + out.append(f'{_esc(display.benchmark(b, short=True))}{mark}') + for i, role in enumerate(roles): + y = top + i * ch + out.append(f'{_esc(display.role(role))}') + for j, b in enumerate(benches): + hit, tot = table[role][b] + frac = hit / tot if tot else 0 + x = lw + j * cw + tip = f"{_esc(role)}
{_esc(b)}
{hit} of {tot} cells ({frac:.0%})" + # 2px surface gap between adjacent fills. + out.append( + f'' + ) + ink = "#0b0b0b" if frac < 0.62 else "#ffffff" + out.append( + f'{hit}/{tot}' + ) + out.append("") + tbl = _table( + ["role"] + benches, + [[r] + [f"{table[r][b][0]}/{table[r][b][1]}" for b in benches] for r in roles], + "Table view", + ) + return "".join(out) + tbl + + +def fig_rarefaction(rows: list[dict]) -> str: + curves = stats.rarefaction(rows) + benches = [b for b in stats.BENCH_ORDER if b in curves] + w, h, pad = 760, 300, 46 + maxy = max(max(v) for v in curves.values()) + maxx = max(len(v) for v in curves.values()) + sx = lambda i: pad + i * (w - pad - 150) / max(maxx - 1, 1) + sy = lambda v: h - pad - v * (h - 2 * pad) / maxy + out = [f''] + for g in range(0, int(maxy) + 1, 2): + out.append(f'') + out.append(f'{g}') + for k, b in enumerate(benches): + pts = curves[b] + d = " ".join(f"{'M' if i==0 else 'L'}{sx(i):.1f},{sy(v):.1f}" for i, v in enumerate(pts)) + out.append(f'') + for i, v in enumerate(pts): + tip = f"{_esc(b)}
after {i+1} cells: {v:.1f} distinct roles" + out.append(f'') + out.append( + f'{_esc(b)}' + ) + out.append(f'cells sampled') + out.append("") + tbl = _table( + ["cells"] + benches, + [[i + 1] + [f"{curves[b][i]:.1f}" if i < len(curves[b]) else "" for b in benches] + for i in range(maxx)], + "Table view", + ) + return "".join(out) + tbl + + +def fig_jaccard(rows: list[dict]) -> str: + data = stats.jaccard(rows) + benches = [b for b in stats.BENCH_ORDER if b in data] + rh, lw, w = 42, 150, 760 + h = 40 + rh * len(benches) + x0, x1 = lw, w - 130 + lo = min(min(d["null_lo"], d["observed"]) for d in data.values()) - 0.04 + hi = max(max(d["null_hi"], d["observed"]) for d in data.values()) + 0.04 + sx = lambda v: x0 + (v - lo) * (x1 - x0) / (hi - lo) + out = [f''] + for t in [round(lo + i * (hi - lo) / 4, 2) for i in range(5)]: + out.append(f'') + out.append(f'{t:.2f}') + for i, b in enumerate(benches): + d = data[b] + y = 40 + i * rh + out.append(f'{_esc(display.benchmark(b, short=True))}') + tip_null = f"{_esc(b)}
null 95%: {d['null_lo']:.3f}–{d['null_hi']:.3f}
null mean {d['null_mean']:.3f}" + out.append( + f'' + ) + col = POS if d["verdict"] == "converged" else (NEG if d["verdict"] == "diverged" else "#7a7975") + tip = (f"{_esc(b)}
observed {d['observed']:.3f} ({d['n_cells']} cells)" + f"
null {d['null_lo']:.3f}–{d['null_hi']:.3f}
{_esc(d['verdict'])}") + out.append( + f'' + ) + out.append(f'{d["verdict"]}') + out.append("") + tbl = _table( + ["benchmark", "cells", "observed", "null 2.5%", "null 97.5%", "verdict"], + [[b, data[b]["n_cells"], f"{data[b]['observed']:.3f}", f"{data[b]['null_lo']:.3f}", + f"{data[b]['null_hi']:.3f}", data[b]["verdict"]] for b in benches], + "Table view", + ) + return "".join(out) + tbl + + +def fig_action_role(rows: list[dict]) -> str: + roles, actions, counts = stats.action_by_role(rows) + lw, w, rh = 150, 760, 30 + h = 40 + rh * len(roles) + maxv = max(sum(counts.get((r, a), 0) for a in actions) for r in roles) + bw = w - lw - 60 + out = [f''] + for i, role in enumerate(roles): + y = 34 + i * rh + out.append(f'{_esc(display.role(role))}') + x = lw + total = sum(counts.get((role, a), 0) for a in actions) + for k, a in enumerate(actions): + n = counts.get((role, a), 0) + if not n: + continue + seg = n * bw / maxv + tip = f"{_esc(role)}
{_esc(a)}: {n} edits ({n/total:.0%} of this role)" + out.append( + f'' + ) + x += seg + out.append(f'{total}') + out.append("") + leg = '
' + "".join( + f'{_esc(display.action(a))}' + for k, a in enumerate(actions) + ) + "
" + tbl = _table(["role"] + actions, + [[r] + [counts.get((r, a), 0) for a in actions] for r in roles], + "Table view") + return "".join(out) + leg + tbl + + +def fig_direction(rows: list[dict], edits: dict[str, dict]) -> str: + data = stats.tuning_direction(rows, edits) + if not data: + return "

No scalar constants changed value.

" + lw, w, rh = 210, 760, 28 + h = 46 + rh * len(data) + mx = max(max(u, d) for _, u, d in data) or 1 + mid = lw + (w - lw - 40) / 2 + half = (w - lw - 60) / 2 + out = [f''] + out.append(f'') + out.append(f'lowered') + out.append(f'raised') + for i, (sym, up, dn) in enumerate(data): + y = 32 + i * rh + out.append(f'{_esc(sym[:26])}') + if dn: + wpx = dn * half / mx + out.append(f'') + out.append(f'{dn}') + if up: + wpx = up * half / mx + out.append(f'') + out.append(f'{up}') + out.append("") + leg = ('
lowered' + 'raised
') + tbl = _table(["constant", "raised", "lowered"], [[s, u, d] for s, u, d in data], "Table view") + return "".join(out) + leg + tbl + + +def fig_provenance(rows: list[dict]) -> str: + """Whose defect each fix repaired. Derived from the seed tree, not model-assigned.""" + data = stats.provenance_of_fixes(rows) + benches = [b for b in stats.BENCH_ORDER if b in data] + if not benches: + return "

No fixes labelled.

" + order = ["seed", "own", "unknown"] + cols = {"seed": CAT_LIGHT[0], "own": CAT_LIGHT[1], "unknown": "#7a7975"} + lw, w, rh = 150, 760, 32 + h = 34 + rh * len(benches) + maxv = max(sum(data[b].values()) for b in benches) + bw = w - lw - 70 + out = [f''] + for i, b in enumerate(benches): + y = 22 + i * rh + total = sum(data[b].values()) + out.append(f'{_esc(b)}') + x = lw + for key in order: + n = data[b].get(key, 0) + if not n: + continue + seg = n * bw / maxv + tip = f"{_esc(b)}
{_esc(key)}: {n} of {total} fixes ({n/total:.0%})" + out.append(f'') + x += seg + out.append(f'{total}') + out.append("") + leg = '
' + "".join( + f'{_esc(display.provenance(k))}' for k in order) + "
" + tbl = _table(["benchmark"] + order, + [[b] + [data[b].get(k, 0) for k in order] for b in benches], "Table view") + return "".join(out) + leg + tbl + + +def render(rows: list[dict], edits: dict[str, dict], meta: dict) -> str: + ag = stats.hint_agreement(rows) + figs = [ + ("Which kinds of edit did optimizers make, and how universally?", + "Share of cells in each benchmark that ever made an edit of this kind. Counted per " + "cell, not per edit: cells produced between 1 and 18 candidates, so edit-weighted " + "counts would measure which cells were prolific. ‡ gaia-shell's seed is an empty " + "shell, so every role is present there by construction — do not pool that column.", + fig_prevalence(rows)), + ("Does the next optimizer try anything new?", + "Distinct roles discovered as cells are added, averaged over 200 random orderings. " + "A curve that flattens says the repertoire was exhausted early; one still climbing " + "at the right edge says it was not.", + fig_rarefaction(rows)), + ("Do different optimizers explore different things?", + "Mean pairwise Jaccard distance between cells' role repertoires (dot) against a " + "permutation null holding each cell's repertoire size and the corpus role " + "frequencies fixed (grey band, 95%). Left of the band means cells are more alike " + "than chance — convergence. The raw distance alone says nothing without this.", + fig_jaccard(rows)), + ("What was done to each part of the harness?", + "Edits per role, split by action. This is the one view where edit counts are the " + "right unit, since the question is about the composition of the work.", + fig_action_role(rows)), + ("Whose defect did each fix repair?", + "Fixes split by whether the repaired code was still exactly as the seed wrote it " + "(seed defect) or had already been rewritten by an earlier candidate in the same " + "cell (the optimizer's own). Derived by comparing trees, not asked of the model: " + "asking returned 452 own against 3 seed and called 21 of 22 swe-atlas submission " + "fixes self-inflicted, when 15 of them repair a defect in the seed's answer parser.", + fig_provenance(rows)), + ("Which way did the knobs go?", + "Scalar constants whose value actually changed, by direction. Constants touched by " + "reformatting without a value change are excluded.", + fig_direction(rows, edits)), + ] + body = "".join( + f'

{_esc(t)}

{_esc(n)}

{svg}
' + for t, n, svg in figs + ) + # Method, vocabulary and the error inventory come BEFORE the figures: a reader who + # is going to use these numbers needs to know how they were made and where they + # are wrong first, not in a footnote. + body = preamble.render_sections(rows, edits) + body + return f""" + +What optimizers modified +
+ +

What optimizers modified

+

{_esc(meta['cells'])} cells, {_esc(meta['edits'])} symbol-scoped edits, +{_esc(meta['labels'])} labelled. Roles were assigned by deterministic rule where the +file, symbol kind or name settles it ({ag['hinted']} edits) and by model otherwise +({ag['model_decided']}). Every edit was sent to the model regardless, because the action +facet is never derivable — so the rule-versus-model comparison is complete, not sampled: +{ag['disagreements']} of {ag['hinted']} hinted edits disagreed and both readings are kept. +Reward is deliberately absent: measurement noise in this corpus makes +category-versus-score comparisons unsupportable.

+{body} +

Generated by vero interpret report. Colour: sequential blue for +magnitude, fixed categorical order for identity, blue/red diverging for polarity; palette +validated for colour-vision deficiency. Every figure has a table view.

+
""" diff --git a/vero/src/vero/interpret/analysis/paper_figures.py b/vero/src/vero/interpret/analysis/paper_figures.py new file mode 100644 index 00000000..d9cde222 --- /dev/null +++ b/vero/src/vero/interpret/analysis/paper_figures.py @@ -0,0 +1,258 @@ +"""Print figures in the house style: vector PDF for LaTeX, PNG to review. + +Separate from `figures.py` on purpose. That module builds one interactive HTML page +for colleagues to explore — hover, dark mode, table views. This one produces +caption-driven figures for a paper: no baked-in titles, sized at the real placement +width so point sizes render 1:1, muted palette on white. + +Archetypes are chosen by the shape of the comparison rather than invented. Role prevalence is a +bounded-metric grid, so it is a sequential heatmap. Diversity-versus-null and +knob direction each have two values per item where the gap is the story, so both are +dumbbells. Rarefaction is a plain line plot — no archetype fits a saturation curve, +and forcing one would obscure the shape that matters. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +BRAND = Path(__file__).parent / "brand" +sys.path.insert(0, str(BRAND)) + +import matplotlib # noqa: E402 + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +import numpy as np # noqa: E402 +from matplotlib.colors import LinearSegmentedColormap # noqa: E402 +from matplotlib.patches import Rectangle # noqa: E402 +from house_style import ( # noqa: E402 + FS, + INK, + INK_FAINT, + PALETTE, + TEXT_WIDTH_IN, + apply_house_style, + family_colors, + save, +) + +from vero.interpret.analysis import display, stats # noqa: E402 + +# Roles grouped so the heatmap's left colour bar means something. Rows must be +# contiguous by group, so this ordering is load-bearing, not cosmetic. +ROLE_GROUPS: list[tuple[str, list[str]]] = [ + ("Agent behaviour", ["prompt", "control_loop", "tool_surface", "tool_impl", + "retrieval", "submission"]), + ("Budgets", ["budget_turns", "budget_output", "budget_wallclock", "context_mgmt"]), + ("Plumbing", ["model_client", "initialization", "env_setup"]), + ("Not the agent", ["tests", "metadata", "other"]), +] + + +def fig_prevalence(rows: list[dict], stem: Path) -> None: + """Heatmap: share of cells per benchmark that ever made each kind of edit.""" + roles, table = stats.prevalence(rows) + benches = [b for b in stats.BENCH_ORDER if b in next(iter(table.values()))] + + ordered: list[tuple[str, str, list[float]]] = [] + for group, members in ROLE_GROUPS: + for role in members: + if role in table: + ordered.append( + (group, role, [table[role][b][0] / table[role][b][1] for b in benches]) + ) + counts = { + (r, b): table[r][b] for _, r, _ in ordered for b in benches + } + + M = np.array([r[2] for r in ordered]) + labels = [r[1] for r in ordered] + color = family_colors([r[0] for r in ordered]) + groups: list[list] = [] + for i, (cat, _, _) in enumerate(ordered): + if groups and groups[-1][0] == cat: + groups[-1][2] = i + else: + groups.append([cat, i, i]) + + cmap = LinearSegmentedColormap.from_list( + "sequential_blue", ["#FFFFFF", PALETTE["slate"], PALETTE["atlas"]] + ) + nrow, ncol = M.shape + fig, ax = plt.subplots(figsize=(TEXT_WIDTH_IN, 0.30 * nrow + 0.55)) + fig.subplots_adjust(left=0.32, right=0.985, top=0.93, bottom=0.015) + ax.imshow(M, cmap=cmap, vmin=0, vmax=1, aspect="auto") + for i in range(nrow): + for j in range(ncol): + hit, tot = counts[(labels[i], benches[j])] + ax.text(j, i, f"{hit}/{tot}", ha="center", va="center", + fontsize=FS["value"], color="white" if M[i, j] > 0.62 else INK) + ax.set_xticks(range(ncol)) + ax.set_xticklabels( + [display.benchmark(b, short=True) + + ("\u2021" if b in stats.CONSTRUCTED_SEED else "") + for b in benches], + fontsize=FS["label"], color=INK, + ) + ax.xaxis.set_ticks_position("top") + ax.set_yticks(range(nrow)) + ax.set_yticklabels([display.role(x) for x in labels], fontsize=FS["label"], color=INK) + ax.tick_params(length=0) + for s in ax.spines.values(): + s.set_visible(False) + ax.set_xticks(np.arange(-0.5, ncol, 1), minor=True) + ax.set_yticks(np.arange(-0.5, nrow, 1), minor=True) + ax.grid(which="minor", color="white", lw=1.2) + ax.tick_params(which="minor", length=0) + + trans = ax.get_yaxis_transform() + for cat, r0, r1 in groups: + ax.add_patch(Rectangle((-0.32, r0 - 0.5), 0.024, (r1 - r0 + 1), transform=trans, + facecolor=color[cat], edgecolor="white", lw=0.6, + clip_on=False, zorder=5)) + ax.text(-0.345, (r0 + r1) / 2, cat, transform=trans, rotation=90, + ha="center", va="center", fontsize=6.4, color=INK) + save(fig, str(stem)) + plt.close(fig) + + +def fig_diversity(rows: list[dict], stem: Path) -> None: + """Dumbbell: observed repertoire distance against the permutation null.""" + data = stats.jaccard(rows) + benches = [b for b in stats.BENCH_ORDER if b in data][::-1] + y = np.arange(len(benches)) + + fig, ax = plt.subplots(figsize=(TEXT_WIDTH_IN, 0.48 * len(benches) + 0.95)) + fig.subplots_adjust(left=0.20, right=0.965, top=0.93, bottom=0.175) + for i, b in enumerate(benches): + d = data[b] + # Null interval as a light band, so the dumbbell reads against it. + ax.plot([d["null_lo"], d["null_hi"]], [i, i], lw=6.0, + color=PALETTE["gray_whisper"], solid_capstyle="round", zorder=1) + ax.plot([d["observed"], d["null_mean"]], [i, i], lw=1.4, + color=PALETTE["atlas"], zorder=2) + ax.plot(d["null_mean"], i, "o", ms=6, mfc="white", + mec=PALETTE["gray_soft"], mew=1.4, zorder=3) + ax.plot(d["observed"], i, "o", ms=6.5, color=PALETTE["evergreen"], zorder=4) + ax.text(d["observed"], i + 0.22, f"{d['observed']:.3f}", ha="center", + va="bottom", fontsize=FS["value"], color=INK) + ax.set_yticks(y) + ax.set_yticklabels([display.benchmark(b, short=True) for b in benches], fontsize=FS["label"], color=INK) + ax.set_xlabel("mean pairwise Jaccard distance between cells' edit repertoires", + fontsize=FS["axis"], color=INK) + ax.tick_params(length=0) + ax.grid(axis="x", color=PALETTE["gray_whisper"], lw=0.8) + ax.set_axisbelow(True) + for s in ("top", "right", "left"): + ax.spines[s].set_visible(False) + ax.spines["bottom"].set_color(PALETTE["gray_medium"]) + handles = [ + plt.Line2D([], [], marker="o", ls="none", ms=6, color=PALETTE["evergreen"], + label="observed"), + plt.Line2D([], [], marker="o", ls="none", ms=6, mfc="white", + mec=PALETTE["gray_soft"], mew=1.4, label="null mean"), + plt.Line2D([], [], lw=6, color=PALETTE["gray_whisper"], label="null 95%"), + ] + ax.legend(handles=handles, loc="upper left", frameon=False, + fontsize=FS["legend"], handletextpad=0.5, borderaxespad=0.2) + save(fig, str(stem)) + plt.close(fig) + + +def fig_rarefaction(rows: list[dict], stem: Path) -> None: + """Small multiples: discovery curve per benchmark, with its ordering spread.""" + bands = stats.rarefaction_bands(rows) + benches = [b for b in stats.BENCH_ORDER if b in bands] + colors = family_colors(benches) + ncol = len(benches) + + fig, axes = plt.subplots( + 1, ncol, figsize=(TEXT_WIDTH_IN, 2.15), sharey=True, sharex=True + ) + fig.subplots_adjust(left=0.078, right=0.978, top=0.86, bottom=0.235, wspace=0.26) + ceiling = max(hi[-1] for _, _, hi in bands.values()) + + for ax, b in zip(axes, benches): + mean, lo, hi = bands[b] + x = np.arange(1, len(mean) + 1) + ax.fill_between(x, lo, hi, color=colors[b], alpha=0.22, lw=0, zorder=1) + ax.plot(x, mean, lw=1.6, color=colors[b], zorder=3) + ax.set_title(display.benchmark(b, short=True), fontsize=FS["label"], color=INK, + pad=5) + ax.set_xlim(1, len(mean)) + ax.set_ylim(0, ceiling + 0.6) + ax.set_xticks([1, 10, 20]) + ax.grid(color=PALETTE["gray_whisper"], lw=0.7) + ax.set_axisbelow(True) + for s in ("top", "right"): + ax.spines[s].set_visible(False) + for s in ("bottom", "left"): + ax.spines[s].set_color(PALETTE["gray_medium"]) + ax.tick_params(colors=INK_FAINT, labelsize=FS["tick"], length=2) + + axes[0].set_ylabel("distinct edit kinds seen", fontsize=FS["axis"], color=INK) + fig.text(0.535, 0.045, "optimization runs sampled", ha="center", + fontsize=FS["axis"], color=INK) + save(fig, str(stem)) + plt.close(fig) + + +def fig_knob_direction(rows: list[dict], edits: dict[str, dict], stem: Path) -> None: + """Dumbbell: two directional counts per constant — raised against lowered.""" + data = stats.tuning_direction(rows, edits, top=10) + if not data: + return + data = data[::-1] + fig, ax = plt.subplots(figsize=(TEXT_WIDTH_IN, 0.36 * len(data) + 0.85)) + fig.subplots_adjust(left=0.34, right=0.965, top=0.965, bottom=0.155) + for i, (sym, up, dn) in enumerate(data): + ax.plot([dn, up], [i, i], lw=1.3, color=PALETTE["gray_medium"], zorder=1) + ax.plot(dn, i, "o", ms=6, mfc="white", mec=PALETTE["tan"], mew=1.5, zorder=3) + ax.plot(up, i, "o", ms=6.5, color=PALETTE["evergreen"], zorder=4) + ax.set_yticks(range(len(data))) + ax.set_yticklabels([s for s, _, _ in data], fontsize=FS["label"], color=INK) + ax.set_xlabel("edits changing this constant", fontsize=FS["axis"], color=INK) + ax.tick_params(length=0) + ax.grid(axis="x", color=PALETTE["gray_whisper"], lw=0.8) + ax.set_axisbelow(True) + for s in ("top", "right", "left"): + ax.spines[s].set_visible(False) + ax.spines["bottom"].set_color(PALETTE["gray_medium"]) + ax.legend(handles=[ + plt.Line2D([], [], marker="o", ls="none", ms=6.5, + color=PALETTE["evergreen"], label="raised"), + plt.Line2D([], [], marker="o", ls="none", ms=6, mfc="white", + mec=PALETTE["tan"], mew=1.5, label="lowered"), + ], loc="lower right", frameon=False, fontsize=FS["legend"], + handletextpad=0.5, borderaxespad=0.1) + save(fig, str(stem)) + plt.close(fig) + + +# One directory per figure, named for its id and holding the spec beside its outputs. +# A flat directory is tolerable for four figures and unusable for forty: outputs, +# specs and stale renders interleave alphabetically and nothing travels as a unit. +FIGURES: list[tuple[str, str]] = [ + ("figure_01_prevalence", "fig_prevalence"), + ("figure_02_diversity", "fig_diversity"), + ("figure_03_rarefaction", "fig_rarefaction"), + ("figure_04_knob_direction", "fig_knob_direction"), +] + + +def render_all(rows: list[dict], edits: dict[str, dict], out: Path) -> list[str]: + apply_house_style() + written: list[str] = [] + for fid, func in FIGURES: + folder = out / fid + folder.mkdir(parents=True, exist_ok=True) + stem = folder / fid + renderer = globals()[func] + if func == "fig_knob_direction": + renderer(rows, edits, stem) + else: + renderer(rows, stem) + written.append(f"{fid}/{fid}.{{pdf,png}}") + return written diff --git a/vero/src/vero/interpret/analysis/preamble.py b/vero/src/vero/interpret/analysis/preamble.py new file mode 100644 index 00000000..01392fe8 --- /dev/null +++ b/vero/src/vero/interpret/analysis/preamble.py @@ -0,0 +1,196 @@ +"""The narrative sections above the figures: method, vocabulary, and known errors. + +Computed from the same data as the figures rather than written out, so they cannot +drift from what the pipeline actually produced. The error inventory in particular is +measured: it exists because several facets are wrong in specific, findable ways, and +a reader who is going to use these figures needs that before the figures, not after. +""" + +from __future__ import annotations + +import html +import re +from collections import Counter + +_DIS = re.compile(r"\[hint=(\S+) model=(\S+)\]") + + +def _esc(s) -> str: + return html.escape(str(s), quote=True) + + +def _pick_example(rows: list[dict], edits: dict[str, dict]): + """A candidate that shows the split clearly: several edits spanning several kinds.""" + by_cand: dict[tuple[str, str], list[dict]] = {} + for r in rows: + edit = edits.get(r["edit_id"]) + if edit: + by_cand.setdefault((edit["cell_key"], edit["candidate_sha"]), []).append( + {**r, **edit} + ) + best = None + for (cell, sha), group in sorted(by_cand.items()): + kinds = len({g["symbol_kind"] for g in group}) + if 6 <= len(group) <= 9 and (best is None or kinds > best[0]): + best = (kinds, cell, sha, group) + if best is None: + (cell, sha), group = next(iter(by_cand.items())) + return cell, sha, group + return best[1], best[2], best[3] + + +def worked_example(rows: list[dict], edits: dict[str, dict]) -> str: + cell, sha, group = _pick_example(rows, edits) + added = sum(g["added"] for g in group) + kinds = len({g["symbol_kind"] for g in group}) + body = [] + for g in sorted(group, key=lambda x: -x["added"]): + val = "" + if g.get("before_value"): + val = ( + f' {_esc(g["before_value"])}' + f'→{_esc(g["after_value"])}' + ) + body.append( + f'{_esc(g["symbol"][:38])}' + f'{_esc(g["symbol_kind"])}+{g["added"]}' + f'{_esc(g["role"])}{_esc(g["action"])}{val}' + ) + return ( + f'

One commit is not one change. Candidate ' + f'{_esc(sha[:12])} of ' + f'{_esc(cell.split("/", 1)[1])} touches {added} lines under a ' + f'single subject line; labelling it as one item would give one category to all of ' + f'them. Mapping each changed line through the syntax tree to its innermost enclosing ' + f'definition splits it into {len(group)} edits across {kinds} kinds, each ' + f'labelled on its own:

' + f'
' + f'' + f'{"".join(body)}
symbolkind+linesroleaction
' + f'

Git\'s own hunk header cannot do this. Its Python matcher reports the ' + f'enclosing class for a method, so a one-line fix inside a shell helper is ' + f'attributed to the whole agent. Module-level bindings are split further by target ' + f'name and value shape, which is what separates a system prompt from a tuning ' + f'constant when both sit at the top of the same file.

' + ) + + +def kinds_table(rows: list[dict], edits: dict[str, dict]) -> str: + seen: dict[str, dict] = {} + for r in rows: + edit = edits.get(r["edit_id"]) + if not edit: + continue + kind = edit["symbol_kind"] + # Prefer an exemplar that carries a value change; it shows more of the schema. + if kind not in seen or (edit.get("before_value") and not seen[kind].get("before_value")): + seen[kind] = {**r, **edit} + order = [ + "prompt_text", "scalar_const", "collection", "regex", "method", + "function", "class", "module", "non_python", + ] + body = [] + for kind in order: + g = seen.get(kind) + if not g: + continue + val = ( + f'{_esc(g["before_value"])}→{_esc(g["after_value"])}' + if g.get("before_value") else "—" + ) + body.append( + f'{_esc(kind)}{_esc(g["symbol"][:34])}' + f'{val}{_esc(g["role"])}' + f'{_esc(g["action"])}' + f'{"rule" if g["hinted"] else "model"}' + ) + return ( + '

Every changed line lands on a named symbol with a kind resolved from ' + 'the syntax tree. The role is set by deterministic rule where the path, kind or name ' + 'settles it and by model otherwise. Before/after values are captured for scalar ' + 'constants, which is what makes tuning direction derived rather than guessed.

' + '
' + '' + f'{"".join(body)}
kindexample symbolvalue changeroleactionrole from
' + ) + + +def false_labels(rows: list[dict], edits: dict[str, dict]) -> str: + dis = [r for r in rows if r["mechanism"].startswith("[hint=")] + hinted = sum(1 for r in rows if r["hinted"]) + pairs: Counter = Counter() + for r in dis: + m = _DIS.match(r["mechanism"]) + if m: + pairs[(m.group(1), m.group(2))] += 1 + + inert = [ + r for r in rows + if (e := edits.get(r["edit_id"])) and e["symbol_kind"] == "non_python" + ] + inert_wrong = sum(1 for r in inert if r["action"] != "cosmetic") + acts = Counter(r["action"] for r in rows) + regex_roles = Counter( + r["role"] for r in rows + if (e := edits.get(r["edit_id"])) and e["symbol_kind"] == "regex" + ) + rate = len(dis) / hinted if hinted else 0.0 + + confusion = "".join( + f"{n}{_esc(h)}{_esc(m)}" + for (h, m), n in pairs.most_common(6) + ) + return f""" +

Where these labels are known to be wrong. Measured, not estimated, and +every figure below inherits it.

+
+

Fix provenance could not be labelled at all, and is now derived instead. Asked of +the model it returned 452 “own” against 3 “seed”, and called 21 of 22 +swe-atlas submission fixes self-inflicted — those repair a defect in the seed's answer +parser that 15 of 20 cells independently patched. An edit shown alone carries no history, so +the model sees repair inside the optimizer's own file and answers “own” +confidently rather than abstaining. Comparing the repaired symbol against the seed tree gives +281 seed / 226 own, and 15 of those 22 swe-atlas fixes now read as seed defects — +the same count reached independently by reading diffs.

+ +

Inert edits are read as work. {inert_wrong} of {len(inert)} non-Python edits, almost +all .gitignore, were labelled env_setup / add +rather than cosmetic: the model reads “ignore build artifacts” as environment +setup. Corpus-wide only {acts.get('cosmetic', 0)} edits were called cosmetic, which is +certainly too few.

+ +

The action facet under-uses its own vocabulary. +add {acts.get('add', 0)} against reword +{acts.get('reword', 0)} — prompt rewrites are being counted as additions. And +revert {acts.get('revert', 0)} is far too low: a revert is a property +of a commit, and at symbol scope it looks like ordinary edits, so this decomposition +structurally cannot see it. Read the action column as a coarse split, not a fine one.

+ +

Rule and model disagree on {len(dis)} of {hinted} hinted edits ({rate:.0%}), and the +pattern is systematic rather than noise: the rule labels by location, the model by purpose. +Adjudicated — _complete (rule +model_client, model control_loop) is a +client call used to force a final answer, so the rule is right about where it is and the model +is describing what it is for; TOOLS (rule +tool_surface, model prompt) is a tool +description, which is genuinely both. The rule wins on conflict, so these are recorded rather +than resolved — but they mark where a single-role facet is the wrong shape and +multi-label would fit better.

+ +{confusion}
editsrule saidmodel said
+ +

Answer-parsing regexes scatter. Regex-kind edits spread across +{len(regex_roles)} roles ({_esc(', '.join(f'{k} {v}' for k, v in regex_roles.most_common(4)))}), +and several labelled prompt are answer-extraction patterns belonging +to submission. No rule covers regex bindings, so the model decides +unaided.

+
+""" + + +def render_sections(rows: list[dict], edits: dict[str, dict]) -> str: + return ( + "

How an edit is extracted

" + worked_example(rows, edits) + + "

Symbols, kinds, and what they were assigned

" + kinds_table(rows, edits) + + "

Where these labels are wrong

" + false_labels(rows, edits) + ) diff --git a/vero/src/vero/interpret/analysis/stats.py b/vero/src/vero/interpret/analysis/stats.py new file mode 100644 index 00000000..67ecea9c --- /dev/null +++ b/vero/src/vero/interpret/analysis/stats.py @@ -0,0 +1,236 @@ +"""Aggregations behind the figures. Pure functions over labelled edits. + +Two rules run through all of these. + +Prevalence counts **cells, not edits**. Cells produced between 1 and 18 candidates, +so an edit-weighted count answers "which cells were prolific" while pretending to +answer "what did optimizers try". + +Diversity needs a **null**. A mean pairwise Jaccard distance of 0.5 is +uninterpretable on its own: it could mean cells explore genuinely different +repertoires, or simply that each drew a few roles from the same skewed marginal. The +permutation null holds each cell's repertoire *size* and the corpus-wide role +frequencies fixed and reshuffles which cell got what, so the comparison isolates +whether cells differ beyond chance. +""" + +from __future__ import annotations + +import itertools +import random +import statistics as st +from collections import Counter, defaultdict + +BENCH_ORDER = [ + "browsecomp-plus", + "officeqa", + "swe-atlas-qna", + "terminal-bench", + "gaia-shell", +] + +# gaia-shell's seed is an empty shell, so every role is "present" by construction +# rather than by choice. It is shown but never pooled with the rest. +CONSTRUCTED_SEED = {"gaia-shell"} + + +def cell_roles(rows: list[dict]) -> dict[str, set[str]]: + """cell_key -> the set of roles it ever touched.""" + out: dict[str, set[str]] = defaultdict(set) + for r in rows: + out[r["cell_key"]].add(r["role"]) + return dict(out) + + +def benchmark_cells(rows: list[dict]) -> dict[str, set[str]]: + out: dict[str, set[str]] = defaultdict(set) + for r in rows: + out[r["cell_key"].split("/")[1]].add(r["cell_key"]) + return dict(out) + + +def prevalence(rows: list[dict]) -> tuple[list[str], dict[str, dict[str, tuple[int, int]]]]: + """role -> benchmark -> (cells that used it, cells in the benchmark).""" + roles_by_cell = cell_roles(rows) + cells_by_bench = benchmark_cells(rows) + roles = sorted({r for s in roles_by_cell.values() for r in s}) + table: dict[str, dict[str, tuple[int, int]]] = {} + for role in roles: + table[role] = {} + for bench, cells in cells_by_bench.items(): + hit = sum(1 for c in cells if role in roles_by_cell.get(c, set())) + table[role][bench] = (hit, len(cells)) + # Order roles by how universal they are, so the figure reads top-down. + roles.sort(key=lambda r: -sum(h / t for h, t in table[r].values())) + return roles, table + + +def _rarefaction_trials( + rows: list[dict], *, trials: int, seed: int +) -> dict[str, list[list[int]]]: + """Per benchmark, `trials` curves of distinct-roles-after-k-cells.""" + rng = random.Random(seed) + roles_by_cell = cell_roles(rows) + out: dict[str, list[list[int]]] = {} + for bench, cells in benchmark_cells(rows).items(): + members = sorted(cells) + curves: list[list[int]] = [] + for _ in range(trials): + rng.shuffle(members) + seen: set[str] = set() + curve: list[int] = [] + for cell in members: + seen |= roles_by_cell.get(cell, set()) + curve.append(len(seen)) + curves.append(curve) + out[bench] = curves + return out + + +def _percentile(sorted_vals: list[int], q: float) -> float: + """Linear-interpolated percentile; avoids a numpy dependency in this module.""" + if not sorted_vals: + return float("nan") + if len(sorted_vals) == 1: + return float(sorted_vals[0]) + pos = q * (len(sorted_vals) - 1) + lo = int(pos) + hi = min(lo + 1, len(sorted_vals) - 1) + return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * (pos - lo) + + +def rarefaction(rows: list[dict], *, trials: int = 200, seed: int = 0) -> dict[str, list[float]]: + """Mean distinct roles discovered after k cells, averaged over orderings. + + A curve that flattens says the k-th optimizer tried nothing the first k-1 had + not already tried; one still climbing at k=20 says the repertoire is not + exhausted by the sample. + """ + return { + bench: [st.mean(c[i] for c in curves) for i in range(len(curves[0]))] + for bench, curves in _rarefaction_trials(rows, trials=trials, seed=seed).items() + } + + +def rarefaction_bands( + rows: list[dict], *, trials: int = 200, seed: int = 0, lo: float = 0.10, hi: float = 0.90 +) -> dict[str, tuple[list[float], list[float], list[float]]]: + """(mean, lo, hi) per benchmark, as percentiles across orderings. + + The band is the spread over which cells happened to come first, which is the + honest uncertainty here: it answers what a reader would have seen from a + different draw of the same size. Percentiles rather than a standard deviation, + because the quantity is a bounded count and its distribution is skewed near the + ceiling — a symmetric band would extend past the number of categories available. + """ + out: dict[str, tuple[list[float], list[float], list[float]]] = {} + for bench, curves in _rarefaction_trials(rows, trials=trials, seed=seed).items(): + k = len(curves[0]) + means, los, his = [], [], [] + for i in range(k): + col = sorted(c[i] for c in curves) + means.append(st.mean(col)) + los.append(_percentile(col, lo)) + his.append(_percentile(col, hi)) + out[bench] = (means, los, his) + return out + + +def jaccard(rows: list[dict], *, trials: int = 500, seed: int = 0) -> dict[str, dict]: + """Observed mean pairwise distance per benchmark, against a permutation null.""" + rng = random.Random(seed) + roles_by_cell = cell_roles(rows) + out: dict[str, dict] = {} + for bench, cells in benchmark_cells(rows).items(): + sets = [roles_by_cell.get(c, set()) for c in sorted(cells)] + sets = [s for s in sets if s] + if len(sets) < 3: + continue + observed = st.mean( + 1 - len(a & b) / len(a | b) for a, b in itertools.combinations(sets, 2) + ) + # Null: keep each cell's repertoire size and the corpus role frequencies, + # reshuffle the assignment. + pool: list[str] = [] + for s in sets: + pool.extend(s) + freq = Counter(pool) + vocab = list(freq) + weights = [freq[v] for v in vocab] + null: list[float] = [] + for _ in range(trials): + drawn = [] + for s in sets: + picked: set[str] = set() + while len(picked) < len(s): + picked.add(rng.choices(vocab, weights=weights, k=1)[0]) + drawn.append(picked) + null.append( + st.mean( + 1 - len(a & b) / len(a | b) for a, b in itertools.combinations(drawn, 2) + ) + ) + null.sort() + lo, hi = null[int(0.025 * len(null))], null[int(0.975 * len(null)) - 1] + out[bench] = { + "observed": observed, + "null_mean": st.mean(null), + "null_lo": lo, + "null_hi": hi, + "n_cells": len(sets), + # Below the null: cells are MORE alike than chance -> convergence. + "verdict": "converged" if observed < lo else ("diverged" if observed > hi else "as chance"), + } + return out + + +def action_by_role(rows: list[dict], *, top_roles: int = 10) -> tuple[list[str], list[str], dict]: + counts: dict[tuple[str, str], int] = Counter() + for r in rows: + counts[(r["role"], r["action"])] += 1 + role_totals = Counter() + for (role, _), n in counts.items(): + role_totals[role] += n + roles = [r for r, _ in role_totals.most_common(top_roles)] + actions = [a for a, _ in Counter(r["action"] for r in rows).most_common()] + return roles, actions, {k: v for k, v in counts.items() if k[0] in roles} + + +def tuning_direction(rows: list[dict], edits: dict[str, dict], *, top: int = 12) -> list[tuple[str, int, int]]: + """(symbol, ups, downs) for scalar constants that actually changed.""" + counts: dict[str, list[int]] = defaultdict(lambda: [0, 0]) + for r in rows: + edit = edits.get(r["edit_id"]) + if not edit or edit["symbol_kind"] != "scalar_const": + continue + if r["direction"] == "up": + counts[edit["symbol"]][0] += 1 + elif r["direction"] == "down": + counts[edit["symbol"]][1] += 1 + ranked = sorted(counts.items(), key=lambda kv: -(kv[1][0] + kv[1][1])) + return [(s, u, d) for s, (u, d) in ranked[:top]] + + +def provenance_of_fixes(rows: list[dict]) -> dict[str, Counter]: + out: dict[str, Counter] = defaultdict(Counter) + for r in rows: + if r["action"] != "fix": + continue + out[r["cell_key"].split("/")[1]][r["provenance"]] += 1 + return dict(out) + + +def hint_agreement(rows: list[dict]) -> dict[str, int]: + """How often the model's role matched the deterministic hint, where audited. + + Disagreements are recorded in `mechanism` as a "[hint=… model=…]" prefix, which + is the only place both readings survive. + """ + audited = [r for r in rows if r["hinted"] and r["mechanism"].startswith("[hint=")] + hinted = [r for r in rows if r["hinted"]] + return { + "hinted": len(hinted), + "disagreements": len(audited), + "model_decided": len(rows) - len(hinted), + "total": len(rows), + } diff --git a/vero/src/vero/interpret/artifacts/__init__.py b/vero/src/vero/interpret/artifacts/__init__.py new file mode 100644 index 00000000..36b309e2 --- /dev/null +++ b/vero/src/vero/interpret/artifacts/__init__.py @@ -0,0 +1,5 @@ +"""Source adapters: raw run artifacts in, canonical models out.""" + +from vero.interpret.artifacts.base import SourceAdapter, available, get, register + +__all__ = ["SourceAdapter", "available", "get", "register"] diff --git a/vero/src/vero/interpret/artifacts/base.py b/vero/src/vero/interpret/artifacts/base.py new file mode 100644 index 00000000..7a7aeaf5 --- /dev/null +++ b/vero/src/vero/interpret/artifacts/base.py @@ -0,0 +1,53 @@ +"""The adapter boundary. + +An adapter takes raw run artifacts from one producer and canonicalises them into +`models`. Everything downstream — edit decomposition, labelling, analysis — consumes +only the canonical types, so supporting a second producer of optimization runs is a +new adapter and nothing else. + +Adapters do two things and no more: find runs under the roots they are given, and +load one run. They must not interpret, classify, or score anything; that judgement +belongs in `edits/` (deterministic) and `labeling/` (model-assisted), which is what +keeps the boundary useful. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Iterable, Protocol, runtime_checkable + +from vero.interpret.models import CellRef, Trajectory + + +@runtime_checkable +class SourceAdapter(Protocol): + """Canonicalises one producer's run artifacts.""" + + name: str + + def discover(self, roots: Iterable[Path]) -> list[CellRef]: + """Find every run under these roots. Cheap: no archives are opened.""" + ... + + def load(self, ref: CellRef) -> Trajectory: + """Materialise one run. May be expensive; results are cached by the caller.""" + ... + + +_REGISTRY: dict[str, SourceAdapter] = {} + + +def register(adapter: SourceAdapter) -> SourceAdapter: + _REGISTRY[adapter.name] = adapter + return adapter + + +def get(name: str) -> SourceAdapter: + if name not in _REGISTRY: + known = ", ".join(sorted(_REGISTRY)) or "none registered" + raise KeyError(f"unknown source adapter {name!r} (known: {known})") + return _REGISTRY[name] + + +def available() -> list[str]: + return sorted(_REGISTRY) diff --git a/vero/src/vero/interpret/artifacts/harbor/__init__.py b/vero/src/vero/interpret/artifacts/harbor/__init__.py new file mode 100644 index 00000000..b902360b --- /dev/null +++ b/vero/src/vero/interpret/artifacts/harbor/__init__.py @@ -0,0 +1,5 @@ +"""Adapter for `vero harbor run` job trees.""" + +from vero.interpret.artifacts.harbor.adapter import HarborAdapter, build + +__all__ = ["HarborAdapter", "build"] diff --git a/vero/src/vero/interpret/artifacts/harbor/adapter.py b/vero/src/vero/interpret/artifacts/harbor/adapter.py new file mode 100644 index 00000000..449cde9d --- /dev/null +++ b/vero/src/vero/interpret/artifacts/harbor/adapter.py @@ -0,0 +1,163 @@ +"""Harbor adapter: raw job artifacts in, canonical `Trajectory` out. + +Layout it expects, which is what `vero harbor run` writes: + + ///jobs//task__*/verifier/ + finalization.json the shipped candidate and its held-out reward + session.tar.gz the candidate repo and the sidecar evaluation records + +`` may be a benchmark directory or a tree of them; discovery handles both, so +callers can point at one benchmark or a whole runs/ tree without knowing the depth. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Iterable + +from vero.interpret.artifacts.base import register +from vero.interpret.artifacts.harbor import session as session_mod +from vero.interpret.artifacts.harbor.repo import CandidateRepo +from vero.interpret.cache import Cache +from vero.interpret.models import Candidate, CellRef, EvalRecord, Trajectory + +# `shipped AND error_rate == 0.0 AND tokens > 0` is the flag harbor's own tooling +# uses, but every benchmark's build config sets error_rate_threshold: 0.1, so the +# exact-zero test discards good runs that lost one case to a platform hiccup. The +# benchmark's own threshold is the defensible line. +DEFAULT_MAX_ERROR_RATE = 0.1 + + +def latest_verifier_dir(cell_dir: Path) -> Path | None: + """The verifier directory of the most recent job for this cell. + + A cell re-run in place leaves earlier attempts behind. Every stage must agree on + which one is authoritative: picking the first glob match in one stage and the last + in another opens a repository that does not contain the other stage's commits, and + the diffs come back empty rather than erroring, silently dropping the whole cell. + """ + finals = sorted(cell_dir.glob("jobs/*/task__*/verifier/finalization.json")) + return finals[-1].parent if finals else None + + +class HarborAdapter: + name = "harbor" + + def __init__(self, cache: Cache) -> None: + self.cache = cache + + # -- discovery ------------------------------------------------------------ + + def discover(self, roots: Iterable[Path]) -> list[CellRef]: + refs: list[CellRef] = [] + seen: set[tuple[str, str]] = set() + for root in roots: + root = Path(root) + if not root.is_dir(): + continue + for final in sorted(root.glob("**/jobs/*/task__*/verifier/finalization.json")): + # ...///jobs//task__*/verifier/ + job_dir = final.parents[2] + cell_dir = final.parents[4] + benchmark = final.parents[5].name + identity = (benchmark, cell_dir.name) + if identity in seen: + continue + seen.add(identity) + refs.append( + CellRef( + source=self.name, + benchmark=benchmark, + cell=cell_dir.name, + root=str(root), + cell_dir=str(cell_dir), + job_dir=str(job_dir), + ) + ) + return refs + + # -- loading -------------------------------------------------------------- + + def load(self, ref: CellRef) -> Trajectory: + verifier = latest_verifier_dir(Path(ref.cell_dir)) + if verifier is None: + return Trajectory(ref=ref) + + final_path = verifier / "finalization.json" + final = json.loads(final_path.read_text()) + metrics = (final.get("reward_metrics") or {}).get("reward", {}) or {} + shipped_sha = ((final.get("candidate") or {}).get("id") or "") + + traj = Trajectory( + ref=ref, + reward=(final.get("rewards") or {}).get("reward"), + baseline_reward=(final.get("baseline_rewards") or {}).get("reward"), + error_rate=metrics.get("error_rate"), + total_tokens=metrics.get("inference_total_tokens"), + ) + + archive = final_path.parent / "session.tar.gz" + if not archive.is_file(): + return traj + root = session_mod.unpack(archive, self.cache) + if root is None: + return traj + + repo_dir = session_mod.find_repo(root) + if repo_dir is not None: + traj.candidates = self._candidates(CandidateRepo(repo_dir), shipped_sha) + traj.evaluations = self._evaluations(session_mod.read_evaluations(root)) + return traj + + def _candidates(self, repo: CandidateRepo, shipped_sha: str) -> list[Candidate]: + out: list[Candidate] = [] + for position, (sha, subject, body) in enumerate(repo.log()): + stats = repo.numstat(f"{sha}^", sha) if position else {} + out.append( + Candidate( + sha=sha, + parent_sha=repo.parent(sha), + position=position, + subject=subject, + body=body[:4000], + files=repo.files(sha), + insertions=sum(a for a, _ in stats.values()), + deletions=sum(r for _, r in stats.values()), + tree_sha=repo.tree_sha(sha), + is_seed=position == 0, + is_shipped=bool(shipped_sha) and sha.startswith(shipped_sha[:12]), + ) + ) + return out + + @staticmethod + def _evaluations(records: list[dict]) -> list[EvalRecord]: + out: list[EvalRecord] = [] + for doc in records: + request = doc.get("request") or {} + report = doc.get("report") or {} + eval_set = request.get("evaluation_set") or {} + metrics = report.get("metrics") or {} + sha = ((request.get("candidate") or {}).get("id") or "") + partition = eval_set.get("partition") + score = metrics.get("score") + if not (sha and partition and score is not None): + continue + out.append( + EvalRecord( + candidate_sha=sha, + partition=partition, + score=score, + error_rate=metrics.get("error_rate"), + selection_kind=(eval_set.get("selection") or {}).get("kind"), + n_attempts=(request.get("limits") or {}).get("n_attempts"), + started_at=report.get("started_at"), + finished_at=report.get("finished_at"), + ) + ) + return out + + +def build(cache: Cache) -> HarborAdapter: + return register(HarborAdapter(cache)) diff --git a/vero/src/vero/interpret/artifacts/harbor/repo.py b/vero/src/vero/interpret/artifacts/harbor/repo.py new file mode 100644 index 00000000..85aef3ed --- /dev/null +++ b/vero/src/vero/interpret/artifacts/harbor/repo.py @@ -0,0 +1,77 @@ +"""Read-only git access to a candidate repository. + +Bare-repo reads only, via subprocess. No checkout ever happens: the analysis needs +trees and diffs, and materialising working copies for 100 cells would cost disk for +nothing. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + + +class CandidateRepo: + def __init__(self, git_dir: Path) -> None: + self.git_dir = Path(git_dir) + + def _run(self, *args: str) -> str: + out = subprocess.run( + ["git", "--git-dir", str(self.git_dir), *args], + capture_output=True, + text=True, + ) + return out.stdout + + def log(self) -> list[tuple[str, str, str]]: + """(sha, subject, body) oldest first, so index 0 is the seed. + + `--all` because a candidate chain can leave commits unreachable from any + branch head once the optimizer reaches back past them. + """ + raw = self._run("log", "--all", "--format=%H%x1f%s%x1f%b%x1e") + rows = [] + for record in raw.split("\x1e"): + record = record.strip("\n") + if not record: + continue + sha, subject, body = (record.split("\x1f") + ["", "", ""])[:3] + rows.append((sha, subject, body)) + rows.reverse() + return rows + + def tree_sha(self, sha: str) -> str: + return self._run("rev-parse", f"{sha}^{{tree}}").strip() + + def parent(self, sha: str) -> str | None: + out = self._run("rev-parse", f"{sha}^").strip() + return out or None + + def files(self, sha: str) -> list[str]: + return [ + f + for f in self._run("show", "--name-only", "--format=", sha).splitlines() + if f.strip() and "__pycache__" not in f + ] + + def show_file(self, sha: str, path: str) -> str: + return self._run("show", f"{sha}:{path}") + + def diff(self, a: str, b: str, path: str | None = None, context: int = 0) -> str: + args = ["diff", f"-U{context}", a, b, "--", path or ".", ":(exclude)*__pycache__*"] + return self._run(*args) + + def numstat(self, a: str, b: str) -> dict[str, tuple[int, int]]: + """path -> (added, removed). Binary files report as (0, 0).""" + out: dict[str, tuple[int, int]] = {} + raw = self._run("diff", "--numstat", a, b, "--", ".", ":(exclude)*__pycache__*") + for line in raw.splitlines(): + parts = line.split("\t") + if len(parts) != 3: + continue + add, rem, path = parts + out[path] = ( + int(add) if add.isdigit() else 0, + int(rem) if rem.isdigit() else 0, + ) + return out diff --git a/vero/src/vero/interpret/artifacts/harbor/session.py b/vero/src/vero/interpret/artifacts/harbor/session.py new file mode 100644 index 00000000..48202170 --- /dev/null +++ b/vero/src/vero/interpret/artifacts/harbor/session.py @@ -0,0 +1,79 @@ +"""Unpack the two things worth keeping out of a harbor `session.tar.gz`. + +These archives run to hundreds of megabytes, almost all of it agent transcripts and +container logs. Only the candidate git repository and the sidecar evaluation records +are needed here, so members are filtered on the way out and the result is cached by +the archive's own digest — re-running the pipeline never re-extracts. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +import tarfile +from pathlib import Path + +from vero.interpret.cache import Cache + +_WANTED_DIR = "/candidates/repository.git/" +_WANTED_FILE = "evaluation.json" + + +def digest(path: Path, *, chunk: int = 1 << 20) -> str: + """Hash the archive itself, so the cache key is independent of where it sits.""" + h = hashlib.sha256() + with path.open("rb") as fh: + while block := fh.read(chunk): + h.update(block) + return h.hexdigest() + + +def unpack(archive: Path, cache: Cache) -> Path | None: + """Extract the wanted members, returning the session root.""" + key = digest(archive) + if (hit := cache.get_dir(key)) is not None: + return hit + + dest = cache.reserve_dir(key) + try: + with tarfile.open(archive) as tar: + members = [ + m + for m in tar.getmembers() + if _WANTED_DIR in m.name or m.name.endswith(_WANTED_FILE) + ] + if not members: + return None + # filter= is 3.12+; these are our own verifier's archives, so the guard is + # about running on 3.11 rather than about untrusted input. + if sys.version_info >= (3, 12): + tar.extractall(dest, members=members, filter="data") + else: + tar.extractall(dest, members=members) + except (tarfile.TarError, OSError): + return None + + cache.commit_dir(key) + return dest + + +def find_repo(session_root: Path) -> Path | None: + hits = list(session_root.glob("**/candidates/repository.git")) + return hits[0] if hits else None + + +def read_evaluations(session_root: Path) -> list[dict]: + """Every sidecar evaluation record, unaggregated. + + Repeats of the same candidate on the same partition are kept separate: collapsing + them into a per-partition map is exactly how a re-score silently overwrites the + earlier one, which hides the corpus's only direct measurement of scoring noise. + """ + out = [] + for path in session_root.glob("**/evaluations/*/evaluation.json"): + try: + out.append(json.loads(path.read_text())) + except (OSError, json.JSONDecodeError): + continue + return out diff --git a/vero/src/vero/interpret/cache.py b/vero/src/vero/interpret/cache.py new file mode 100644 index 00000000..d7d3e9ec --- /dev/null +++ b/vero/src/vero/interpret/cache.py @@ -0,0 +1,94 @@ +"""Content-addressed file cache, shared by artifact extraction and labelling. + +Two callers with very different economics use this: unpacking a session archive costs +minutes and hundreds of megabytes, while an LLM label costs a fraction of a cent. They +get separate namespaces so revising a taxonomy re-labels everything without +re-extracting anything — the failure mode that actually wastes an afternoon. + +Writes go to a temp file and are renamed into place, so an interrupted run leaves +either a complete entry or none, never a truncated one that poisons the next attempt. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +from pathlib import Path +from typing import Any + + +def key_of(*parts: str) -> str: + return hashlib.sha256("\x00".join(parts).encode("utf-8", "replace")).hexdigest() + + +class Cache: + """A namespaced content-addressed store under one root.""" + + def __init__(self, root: Path, namespace: str, *, refresh: bool = False) -> None: + self.root = Path(root) / namespace + self.refresh = refresh + self.hits = 0 + self.misses = 0 + + def _path(self, key: str, suffix: str) -> Path: + return self.root / key[:2] / f"{key}{suffix}" + + # -- JSON entries (labels, parsed records) -------------------------------- + + def get_json(self, key: str) -> Any | None: + if self.refresh: + return None + path = self._path(key, ".json") + if not path.is_file(): + self.misses += 1 + return None + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + self.misses += 1 + return None + self.hits += 1 + return value + + def put_json(self, key: str, value: Any) -> None: + path = self._path(key, ".json") + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(value, indent=None, default=str)) + os.replace(tmp, path) + + # -- Directory entries (unpacked archives) -------------------------------- + + def get_dir(self, key: str) -> Path | None: + """A completed directory entry, or None. + + Completion is marked by a sentinel written after the unpack finishes, so a + directory left behind by a crash is treated as absent and redone rather than + silently used half-populated. + """ + path = self._path(key, ".d") + if self.refresh or not (path / ".complete").is_file(): + self.misses += 1 + return None + self.hits += 1 + return path + + def reserve_dir(self, key: str) -> Path: + """Empty directory to unpack into. Call `commit_dir` when finished.""" + path = self._path(key, ".d") + if path.exists(): + shutil.rmtree(path, ignore_errors=True) + path.mkdir(parents=True, exist_ok=True) + return path + + def commit_dir(self, key: str) -> Path: + path = self._path(key, ".d") + (path / ".complete").write_text("") + return path + + def stats(self) -> str: + total = self.hits + self.misses + pct = (100.0 * self.hits / total) if total else 0.0 + return f"{self.root.name}: {self.hits} hit / {self.misses} miss ({pct:.0f}%)" diff --git a/vero/src/vero/interpret/cli.py b/vero/src/vero/interpret/cli.py new file mode 100644 index 00000000..85bee17a --- /dev/null +++ b/vero/src/vero/interpret/cli.py @@ -0,0 +1,234 @@ +"""`vero interpret` — extract, decompose, and report on optimization runs. + +Stages write JSONL and are independently resumable, so a long extraction is never +repeated to re-run a cheap downstream step. +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +import click + +from vero.interpret.artifacts.harbor import build as build_harbor +from vero.interpret.artifacts.harbor.adapter import latest_verifier_dir +from vero.interpret.artifacts.harbor import session as session_mod +from vero.interpret.artifacts.harbor.repo import CandidateRepo +from vero.interpret.cache import Cache +from vero.interpret.edits import decompose +from vero.interpret.models import CellRef, Edit, Trajectory + +DEFAULT_CACHE = Path.home() / ".cache" / "vero-interpret" + + +def _load_scope(path: Path | None) -> dict[str, set[str]] | None: + """{benchmark: {cell, ...}} restricting which runs are in scope.""" + if path is None: + return None + raw = json.loads(Path(path).read_text()) + return {bench: set(cells) for bench, cells in raw.items()} + + +def _in_scope(ref: CellRef, scope: dict[str, set[str]] | None) -> bool: + if scope is None: + return True + return ref.cell in scope.get(ref.benchmark, set()) + + +@click.group() +def main() -> None: + """Interpretability analysis over optimization runs.""" + + +@main.command() +@click.option("--runs", "roots", multiple=True, required=True, type=click.Path(path_type=Path), + help="Directory to search for runs. Repeatable.") +@click.option("--cells-file", type=click.Path(path_type=Path), + help='JSON {"benchmark": ["cell", ...]} restricting scope.') +@click.option("--cache-dir", type=click.Path(path_type=Path), default=DEFAULT_CACHE) +@click.option("--out", type=click.Path(path_type=Path), default=Path("trajectories.jsonl")) +@click.option("--refresh", is_flag=True, help="Bypass the cache and re-extract.") +def extract(roots, cells_file, cache_dir, out, refresh) -> None: + """Canonicalise runs into trajectories.""" + cache = Cache(cache_dir, "harbor-unpack", refresh=refresh) + adapter = build_harbor(cache) + scope = _load_scope(cells_file) + + refs = [r for r in adapter.discover(roots) if _in_scope(r, scope)] + click.echo(f"{len(refs)} cells in scope") + + out = Path(out) + with out.open("w") as fh: + for i, ref in enumerate(refs, 1): + traj = adapter.load(ref) + fh.write(traj.model_dump_json() + "\n") + click.echo( + f" [{i}/{len(refs)}] {ref.key} " + f"cands={len(traj.candidates)} evals={len(traj.evaluations)}" + ) + click.echo(f"wrote {out} ({cache.stats()})") + + +@main.command() +@click.option("--in", "src", type=click.Path(path_type=Path), default=Path("trajectories.jsonl")) +@click.option("--cache-dir", type=click.Path(path_type=Path), default=DEFAULT_CACHE) +@click.option("--out", type=click.Path(path_type=Path), default=Path("edits.jsonl")) +def edits(src, cache_dir, out) -> None: + """Split every candidate into symbol-scoped edits.""" + cache = Cache(cache_dir, "harbor-unpack") + total = 0 + with Path(out).open("w") as fh: + for line in Path(src).read_text().splitlines(): + traj = Trajectory.model_validate_json(line) + verifier = latest_verifier_dir(Path(traj.ref.cell_dir)) + if verifier is None: + continue + archive = verifier / "session.tar.gz" + if not archive.is_file(): + continue + root = session_mod.unpack(archive, cache) + if root is None: + continue + repo_dir = session_mod.find_repo(root) + if repo_dir is None: + continue + repo = CandidateRepo(repo_dir) + n = 0 + seed_sha = next((c.sha for c in traj.candidates if c.is_seed), None) + # Accumulated in candidate order so prior_touches means "before this one". + seen_symbols: set[tuple[str, str]] = set() + for cand in sorted(traj.candidates, key=lambda c: c.position): + produced = decompose(repo, traj.ref.key, cand, seed_sha=seed_sha, + prior_symbols=seen_symbols) + for edit in produced: + fh.write(edit.model_dump_json() + "\n") + n += 1 + seen_symbols |= {(e.path, e.symbol) for e in produced} + total += n + click.echo(f" {traj.ref.key}: {n} edits") + click.echo(f"wrote {out} ({total} edits)") + + +@main.command() +@click.option("--edits-file", type=click.Path(path_type=Path), default=Path("edits.jsonl")) +@click.option("--trajectories", type=click.Path(path_type=Path), + default=Path("trajectories.jsonl")) +@click.option("--model", default=None, help="Defaults to a cheap model.") +@click.option("--concurrency", default=16) +@click.option("--limit", default=0, help="Label only the first N edits (a dry run).") +@click.option("--cache-dir", type=click.Path(path_type=Path), default=DEFAULT_CACHE) +@click.option("--out", type=click.Path(path_type=Path), default=Path("labels.jsonl")) +def label(edits_file, trajectories, model, concurrency, limit, cache_dir, out) -> None: + """Assign facets to edits. Cached and resumable; re-running costs nothing.""" + import asyncio as _asyncio + + from vero.interpret.config import Settings + from vero.interpret.labeling.client import AsyncLLM + from vero.interpret.labeling.labeler import Labeler + + subjects: dict[str, str] = {} + if Path(trajectories).is_file(): + for line in Path(trajectories).read_text().splitlines(): + traj = Trajectory.model_validate_json(line) + for cand in traj.candidates: + subjects[cand.sha] = cand.subject + + rows = [Edit.model_validate_json(l) for l in Path(edits_file).read_text().splitlines()] + per_candidate = Counter(e.candidate_sha for e in rows) + if limit: + rows = rows[:limit] + click.echo(f"labelling {len(rows)} edits with {model or 'default model'}") + + settings = Settings.from_env(model=model, concurrency=concurrency, + cache_dir=Path(cache_dir)) + if not settings.api_key: + raise click.ClickException( + "no OPENAI_API_KEY found; put it in .env or secrets.env" + ) + + async def run(): + llm = AsyncLLM(settings) + labeler = Labeler(llm, Cache(Path(cache_dir), "labels")) + try: + return await labeler.label_all( + [(e, subjects.get(e.candidate_sha, ""), per_candidate[e.candidate_sha]) + for e in rows], + progress=lambda i, n: click.echo(f" {i}/{n}"), + ), labeler + finally: + await llm.close() + + labels, labeler = _asyncio.run(run()) + with Path(out).open("w") as fh: + for lab in labels: + fh.write(lab.model_dump_json() + "\n") + click.echo(f"wrote {out} ({len(labels)} labels) {labeler.stats()}") + + +@main.command() +@click.option("--labels-file", type=click.Path(path_type=Path), default=Path("labels.jsonl")) +@click.option("--edits-file", type=click.Path(path_type=Path), default=Path("edits.jsonl")) +@click.option("--out", type=click.Path(path_type=Path), default=Path("analysis")) +def report(labels_file, edits_file, out) -> None: + """Render the figures as one self-contained HTML page.""" + from vero.interpret.analysis import figures + + edits = {} + for line in Path(edits_file).read_text().splitlines(): + e = json.loads(line) + edits[e["id"]] = e + # A label carries only edit_id; the aggregations key on the edit's cell and + # symbol, so join here rather than duplicating those fields into every label. + rows = [] + orphans = 0 + for line in Path(labels_file).read_text().splitlines(): + lab = json.loads(line) + edit = edits.get(lab["edit_id"]) + if edit is None: + orphans += 1 + continue + rows.append({**lab, "cell_key": edit["cell_key"], "symbol": edit["symbol"], + "symbol_kind": edit["symbol_kind"], "path": edit["path"]}) + if orphans: + click.echo(f"warning: {orphans} labels had no matching edit and were dropped") + cells = len({r["cell_key"] for r in rows}) + meta = {"cells": cells, "edits": len(edits), "labels": len(rows)} + + out = Path(out) + out.mkdir(parents=True, exist_ok=True) + page = out / "index.html" + page.write_text(figures.render(rows, edits, meta)) + (out / "labels.jsonl").write_text(Path(labels_file).read_text()) + (out / "edits.jsonl").write_text(Path(edits_file).read_text()) + click.echo(f"wrote {page} ({cells} cells, {len(rows)} labels)") + + +@main.command() +@click.option("--in", "src", type=click.Path(path_type=Path), default=Path("edits.jsonl")) +@click.option("--top", default=40, help="Symbols to show per benchmark.") +def symbols(src, top) -> None: + """Symbol frequency, the input to designing a role vocabulary.""" + per_bench: dict[str, Counter] = {} + kinds: Counter = Counter() + cells: dict[str, set[str]] = {} + for line in Path(src).read_text().splitlines(): + e = json.loads(line) + bench = e["cell_key"].split("/")[1] + name = f"{e['path'].split('/')[-1]}::{e['symbol']}" + per_bench.setdefault(bench, Counter()) + cells.setdefault(f"{bench}|{name}", set()).add(e["cell_key"]) + per_bench[bench][name] += 1 + kinds[e["symbol_kind"]] += 1 + + for bench, counter in sorted(per_bench.items()): + click.echo(f"\n=== {bench}: {sum(counter.values())} edits, {len(counter)} symbols") + for name, n in counter.most_common(top): + ncells = len(cells[f"{bench}|{name}"]) + click.echo(f" {n:>4} edits {ncells:>3} cells {name}") + click.echo("\nsymbol kinds: " + ", ".join(f"{k} {v}" for k, v in kinds.most_common())) + + +if __name__ == "__main__": + main() diff --git a/vero/src/vero/interpret/config.py b/vero/src/vero/interpret/config.py new file mode 100644 index 00000000..9ba21642 --- /dev/null +++ b/vero/src/vero/interpret/config.py @@ -0,0 +1,76 @@ +"""Settings and secret loading. + +Reads `KEY=VALUE` files, which is the convention already in use here (`secrets.env`, +`eval.secrets.env`) as well as the usual `.env`. A twenty-line parser covers both and +avoids adding a dependency for it. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from pydantic import BaseModel + +DEFAULT_CACHE = Path.home() / ".cache" / "vero-interpret" +DEFAULT_MODEL = "gpt-5.4-mini" + + +def load_env_file(path: Path, *, override: bool = False) -> dict[str, str]: + """Parse a KEY=VALUE file into the environment. Returns what it set.""" + if not path.is_file(): + return {} + loaded: dict[str, str] = {} + for raw in path.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + if line.startswith("export "): + line = line[len("export "):] + key, _, value = line.partition("=") + key, value = key.strip(), value.strip().strip('"').strip("'") + if not key: + continue + if override or key not in os.environ: + os.environ[key] = value + loaded[key] = value + return loaded + + +def load_secrets(paths: list[Path] | None = None) -> None: + """Load the first secrets file that exists, plus `.env` if present.""" + candidates = paths or [ + Path(".env"), + Path("secrets.env"), + Path("vero/secrets.env"), + ] + for path in candidates: + if path.is_file(): + load_env_file(path) + + +class Settings(BaseModel): + """Everything the labelling stage needs to run.""" + + model: str = DEFAULT_MODEL + api_key: str | None = None + base_url: str | None = None + concurrency: int = 16 + max_retries: int = 4 + request_timeout: float = 120.0 + cache_dir: Path = DEFAULT_CACHE + + @classmethod + def from_env(cls, **overrides) -> "Settings": + load_secrets() + base = os.environ.get("OPENAI_BASE_URL") + values = { + "api_key": os.environ.get("OPENAI_API_KEY"), + # The gateway's OPENAI_BASE_URL ends in "/v1/" here. The OpenAI client + # appends "/chat/completions", and the resulting double slash returns a + # flat 403 "This route is not publicly accessible" that reads like a + # permissions problem and is not one. Strip it once, here. + "base_url": base.rstrip("/") if base else None, + } + values.update({k: v for k, v in overrides.items() if v is not None}) + return cls(**values) diff --git a/vero/src/vero/interpret/edits/__init__.py b/vero/src/vero/interpret/edits/__init__.py new file mode 100644 index 00000000..1a616fb4 --- /dev/null +++ b/vero/src/vero/interpret/edits/__init__.py @@ -0,0 +1,5 @@ +"""Deterministic decomposition of candidates into symbol-scoped edits.""" + +from vero.interpret.edits.decompose import decompose + +__all__ = ["decompose"] diff --git a/vero/src/vero/interpret/edits/decompose.py b/vero/src/vero/interpret/edits/decompose.py new file mode 100644 index 00000000..8110488a --- /dev/null +++ b/vero/src/vero/interpret/edits/decompose.py @@ -0,0 +1,171 @@ +"""Split a candidate into symbol-scoped edits. + +The unit of analysis is one symbol touched by one candidate, not the candidate. A +single commit routinely bundles a genuine bug fix with unrelated tuning under one +subject line, so labelling per candidate assigns a single category to several +distinct modifications and the interesting one gets buried. + +Everything here is deterministic. No model is consulted, so the resulting table is +reproducible and can be diffed between runs. +""" + +from __future__ import annotations + +import re +from collections import defaultdict + +from vero.interpret.artifacts.harbor.repo import CandidateRepo +from vero.interpret.edits import locus +from vero.interpret.models import Candidate, Edit, SymbolKind + +_HUNK_SPLIT = re.compile(r"^(@@ .*?@@.*)$", re.M) +# __pycache__ is compiled noise. .gitignore is NOT skipped: several cells shipped a +# candidate whose only change was one, and "shipped something inert" is a finding. +_SKIP = ("__pycache__",) +# The stored diff was capped at 8000 chars, which silently truncated the largest +# rewrites before any labeller could see them. Storage is cheap; keep the whole thing. +MAX_STORED_DIFF = 60_000 +MAX_STORED_SOURCE = 20_000 + + +def _per_symbol_diff(diff: str, keep: set[int]) -> str: + """Hunks of `diff` whose post-image start line falls in `keep`.""" + parts = _HUNK_SPLIT.split(diff) + if len(parts) < 2: + return "" + chunks: list[str] = [] + for header, body in zip(parts[1::2], parts[2::2]): + match = locus._HUNK.match(header + "\n") + if not match: + continue + start = int(match.group(1)) + count = int(match.group(2) or 1) + if keep & set(range(start, start + max(count, 1))): + chunks.append(header + body.rstrip("\n")) + return "\n".join(chunks)[:MAX_STORED_DIFF] + + +def decompose( + repo: CandidateRepo, + cell_key: str, + candidate: Candidate, + *, + seed_sha: str | None = None, + prior_symbols: set[tuple[str, str]] | None = None, +) -> list[Edit]: + """Symbol-scoped edits introduced by `candidate` relative to its parent.""" + if candidate.is_seed or candidate.parent_sha is None: + return [] + + edits: list[Edit] = [] + for path in candidate.files: + if any(s in path for s in _SKIP): + continue + + diff = repo.diff(candidate.parent_sha, candidate.sha, path=path, context=0) + if not diff.strip(): + continue + + if not path.endswith(".py"): + added = sum( + 1 for line in diff.splitlines() if line.startswith("+") and line[1:2] != "+" + ) + removed = sum( + 1 for line in diff.splitlines() if line.startswith("-") and line[1:2] != "-" + ) + edits.append( + _edit(cell_key, candidate, path, "", SymbolKind.NON_PYTHON, + added, removed, diff[:MAX_STORED_DIFF], None, None, None, True, 0) + ) + continue + + after_src = repo.show_file(candidate.sha, path) + before_src = repo.show_file(candidate.parent_sha, path) + seed_src = repo.show_file(seed_sha, path) if seed_sha else "" + mapping = locus.symbol_map(after_src) + + grouped: dict[tuple[str, SymbolKind], set[int]] = defaultdict(set) + for line in locus.changed_lines(diff): + symbol, kind = mapping.get(line, ("", SymbolKind.MODULE)) + grouped[(symbol, kind)].add(line) + + # Deletions vanish from the post-image, so a symbol removed outright has no + # line to map. Attribute the whole file's removals to rather than + # dropping them: "removed the audit pass" is a modification worth counting. + removed_total = sum( + 1 for line in diff.splitlines() if line.startswith("-") and line[1:2] != "-" + ) + added_total = sum( + 1 for line in diff.splitlines() if line.startswith("+") and line[1:2] != "+" + ) + attributed_added = sum(len(v) for v in grouped.values()) + if removed_total and not grouped: + grouped[("", SymbolKind.MODULE)] = set() + + for (symbol, kind), lines in grouped.items(): + before = after = None + if kind is SymbolKind.SCALAR_CONST: + before = locus.scalar_value(before_src, symbol) + after = locus.scalar_value(after_src, symbol) + if before == after: + continue # touched by reflow, not retuned + share = len(lines) + # Context and history, so the labeller is not reading changed lines blind. + after_source = locus.symbol_source(after_src, symbol) + in_seed = bool(seed_src) and locus.symbol_source(seed_src, symbol) is not None + touches = len([1 for k in (prior_symbols or set()) if k == (path, symbol)]) + edits.append( + _edit( + cell_key, + candidate, + path, + symbol, + kind, + share, + # Removals cannot be attributed per symbol; carry the file total + # on the module row so the count is never silently lost. + removed_total if symbol == "" else 0, + _per_symbol_diff(diff, lines) or diff[:MAX_STORED_DIFF], + before, + after, + after_source[:MAX_STORED_SOURCE] if after_source else None, + in_seed, + touches, + ) + ) + if attributed_added < added_total and grouped: + pass # unattributed remainder is comment/blank churn; not an edit + return edits + + +def _edit( + cell_key: str, + candidate: Candidate, + path: str, + symbol: str, + kind: SymbolKind, + added: int, + removed: int, + diff: str, + before: str | None, + after: str | None, + after_source: str | None, + in_seed: bool, + prior_touches: int, +) -> Edit: + return Edit( + id=Edit.make_id(cell_key, candidate.sha, path, symbol, diff), + cell_key=cell_key, + candidate_sha=candidate.sha, + path=path, + symbol=symbol, + symbol_kind=kind, + added=added, + removed=removed, + before_value=before, + after_value=after, + diff=diff, + after_source=after_source, + in_seed=in_seed, + prior_touches=prior_touches, + ) diff --git a/vero/src/vero/interpret/edits/locus.py b/vero/src/vero/interpret/edits/locus.py new file mode 100644 index 00000000..8e59be64 --- /dev/null +++ b/vero/src/vero/interpret/edits/locus.py @@ -0,0 +1,164 @@ +"""Resolve which symbol an edit landed in, from the syntax tree. + +Git's hunk header is not good enough. Its Python `xfuncname` matches the last +preceding definition line, which for a method inside a class reports the *class* — so +a one-line fix to a shell-exec helper is attributed to the whole agent. Parsing the +post-image and mapping changed line numbers to the innermost enclosing definition +gives function-level locus exactly, with no model involved. + +Module-level bindings get split further by target name and value shape, because that +bucket is otherwise the largest and least informative: on one real candidate it held +170 changed lines spanning a system prompt, the tool table, nine tuning constants and +six regexes, which are four different kinds of modification. +""" + +from __future__ import annotations + +import ast +import re + +from vero.interpret.models import SymbolKind + +_HUNK = re.compile(r"^@@ -\S+ \+(\d+)(?:,(\d+))? @@", re.M) +_PROMPT_MIN_CHARS = 200 + + +def changed_lines(diff: str) -> set[int]: + """Post-image line numbers touched by a `-U0` diff of a single file.""" + lines: set[int] = set() + for match in _HUNK.finditer(diff): + start = int(match.group(1)) + count = int(match.group(2) or 1) + lines.update(range(start, start + max(count, 1))) + return lines + + +def _binding_kind(value: ast.expr) -> SymbolKind: + if isinstance(value, ast.Constant): + if isinstance(value.value, str) and len(value.value) >= _PROMPT_MIN_CHARS: + return SymbolKind.PROMPT_TEXT + return SymbolKind.SCALAR_CONST + if isinstance(value, ast.Call): + func = ast.unparse(value.func) + if func in {"re.compile", "compile"}: + return SymbolKind.REGEX + return SymbolKind.COLLECTION + if isinstance(value, (ast.Tuple, ast.List, ast.Dict, ast.Set)): + return SymbolKind.COLLECTION + if isinstance(value, ast.JoinedStr): + return SymbolKind.PROMPT_TEXT + return SymbolKind.COLLECTION + + +def symbol_map(source: str) -> dict[int, tuple[str, SymbolKind]]: + """line number -> (qualified symbol, kind). + + Definitions are laid down widest-first so that narrower spans overwrite them and + the innermost enclosing scope wins. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return {} + + spans: list[tuple[int, int, str, SymbolKind]] = [] + + def walk(node: ast.AST, prefix: str, in_class: bool) -> None: + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + qualified = f"{prefix}{child.name}" + if isinstance(child, ast.ClassDef): + kind = SymbolKind.CLASS + else: + kind = SymbolKind.METHOD if in_class else SymbolKind.FUNCTION + spans.append( + ( + child.lineno, + getattr(child, "end_lineno", child.lineno), + qualified, + kind, + ) + ) + walk(child, f"{qualified}.", isinstance(child, ast.ClassDef)) + else: + walk(child, prefix, in_class) + + walk(tree, "", False) + + # Module-level bindings: named, so tuning a constant is distinguishable from + # rewriting a system prompt. + for node in tree.body: + targets: list[str] = [] + if isinstance(node, ast.Assign): + targets = [t.id for t in node.targets if isinstance(t, ast.Name)] + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + targets = [node.target.id] + if not targets or node.value is None: + continue + spans.append( + ( + node.lineno, + getattr(node, "end_lineno", node.lineno), + targets[0], + _binding_kind(node.value), + ) + ) + + mapping: dict[int, tuple[str, SymbolKind]] = {} + for start, end, qualified, kind in sorted(spans, key=lambda s: s[1] - s[0], reverse=True): + for line in range(start, end + 1): + mapping[line] = (qualified, kind) + return mapping + + +def symbol_source(source: str, symbol: str) -> str | None: + """Full source text of one qualified symbol, for context and history comparison.""" + if not source or symbol in ("", ""): + return None + try: + tree = ast.parse(source) + except SyntaxError: + return None + want = symbol.split(".") + + def find(node: ast.AST, path: list[str]) -> ast.AST | None: + if not path: + return node + for child in ast.iter_child_nodes(node): + if getattr(child, "name", None) == path[0]: + return find(child, path[1:]) + if isinstance(child, (ast.Assign, ast.AnnAssign)) and len(path) == 1: + targets = ( + [child.target] if isinstance(child, ast.AnnAssign) else child.targets + ) + if any(getattr(t, "id", None) == path[0] for t in targets): + return child + return None + + found = find(tree, want) + if found is None: + return None + try: + return ast.unparse(found) + except Exception: + return None + + +def scalar_value(source: str, name: str) -> str | None: + """Literal text of a module-level scalar binding, for before/after capture.""" + try: + tree = ast.parse(source) + except SyntaxError: + return None + for node in tree.body: + targets: list[str] = [] + if isinstance(node, ast.Assign): + targets = [t.id for t in node.targets if isinstance(t, ast.Name)] + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + targets = [node.target.id] + if name in targets and node.value is not None: + try: + return ast.unparse(node.value)[:200] + except Exception: + return None + return None diff --git a/vero/src/vero/interpret/edits/provenance.py b/vero/src/vero/interpret/edits/provenance.py new file mode 100644 index 00000000..68a68740 --- /dev/null +++ b/vero/src/vero/interpret/edits/provenance.py @@ -0,0 +1,49 @@ +"""Whose defect was it? Derived from the seed, not asked of a model. + +Asking a model was tried and fails systematically. An edit shown in isolation +carries no history, so the model sees code being repaired inside the optimizer's own +agent file and answers "own" almost every time: on the first pass it returned 452 +own against 3 seed corpus-wide, and labelled 21 of 22 swe-atlas submission fixes as +self-inflicted when those demonstrably repair a defect in the seed's answer parser +that 15 of 20 cells independently patched. + +The question is not a judgement. If the code being repaired is still exactly as the +seed wrote it, the defect came with the seed; if an earlier candidate in the same +cell had already rewritten it, the optimizer is repairing itself. That is two tree +lookups. +""" + +from __future__ import annotations + +from vero.interpret.artifacts.harbor.repo import CandidateRepo +from vero.interpret.edits.locus import symbol_source +from vero.interpret.labeling.taxonomy import Provenance + + +def provenance_of( + repo: CandidateRepo, + seed_sha: str, + parent_sha: str, + path: str, + symbol: str, +) -> Provenance: + """SEED if the repaired code is untouched since the seed, OWN if not.""" + if not parent_sha or not seed_sha or parent_sha.startswith(seed_sha[:12]): + return Provenance.SEED + + seed_src = repo.show_file(seed_sha, path) + parent_src = repo.show_file(parent_sha, path) + if not seed_src: + # The file did not exist in the seed, so whatever is being fixed is the + # optimizer's own work by construction. + return Provenance.OWN + if not parent_src: + return Provenance.UNKNOWN + + seed_sym = symbol_source(seed_src, symbol) + parent_sym = symbol_source(parent_src, symbol) + if seed_sym is None or parent_sym is None: + # Fall back to whole-file comparison: coarser, but still decided by content + # rather than by guess. + return Provenance.SEED if seed_src == parent_src else Provenance.OWN + return Provenance.SEED if seed_sym == parent_sym else Provenance.OWN diff --git a/vero/src/vero/interpret/labeling/__init__.py b/vero/src/vero/interpret/labeling/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vero/src/vero/interpret/labeling/client.py b/vero/src/vero/interpret/labeling/client.py new file mode 100644 index 00000000..5217a0e2 --- /dev/null +++ b/vero/src/vero/interpret/labeling/client.py @@ -0,0 +1,85 @@ +"""Bounded-concurrency async client for structured labelling. + +Deliberately thin. Labelling is thousands of small independent calls against a cheap +model, so what matters is that concurrency is capped, transient failures are retried, +and a malformed response is retried rather than parsed leniently — a label that +silently degrades to a default is worse than one that is missing, because it looks +like evidence. +""" + +from __future__ import annotations + +import asyncio +import json +import random +from typing import Any + +from vero.interpret.config import Settings + + +class LLMError(RuntimeError): + pass + + +class AsyncLLM: + def __init__(self, settings: Settings) -> None: + try: + from openai import AsyncOpenAI + except ImportError as exc: # pragma: no cover - dependency guard + raise LLMError( + "the interpret extra is required: uv sync --extra interpret" + ) from exc + + self.settings = settings + self._client = AsyncOpenAI( + api_key=settings.api_key, + base_url=settings.base_url, + timeout=settings.request_timeout, + max_retries=0, # retried here, so backoff is visible and uniform + ) + self._sem = asyncio.Semaphore(settings.concurrency) + self.calls = 0 + self.retries = 0 + + async def json_call( + self, + system: str, + user: str, + schema: dict[str, Any], + schema_name: str = "label", + ) -> dict[str, Any]: + """One structured call, retried on transport error and on schema violation.""" + last: Exception | None = None + for attempt in range(self.settings.max_retries): + try: + async with self._sem: + self.calls += 1 + response = await self._client.chat.completions.create( + model=self.settings.model, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + response_format={ + "type": "json_schema", + "json_schema": { + "name": schema_name, + "schema": schema, + "strict": True, + }, + }, + ) + content = response.choices[0].message.content or "" + return json.loads(content) + except Exception as exc: # noqa: BLE001 - retry policy is uniform + last = exc + self.retries += 1 + if attempt == self.settings.max_retries - 1: + break + # Jittered backoff: these run thousands-wide, and synchronised + # retries after a rate-limit burst just reproduce the burst. + await asyncio.sleep((2**attempt) + random.random()) + raise LLMError(f"call failed after {self.settings.max_retries} attempts: {last}") + + async def close(self) -> None: + await self._client.close() diff --git a/vero/src/vero/interpret/labeling/labeler.py b/vero/src/vero/interpret/labeling/labeler.py new file mode 100644 index 00000000..3fc96978 --- /dev/null +++ b/vero/src/vero/interpret/labeling/labeler.py @@ -0,0 +1,203 @@ +"""Assign facets to edits: cached, resumable, and mostly not the model's job. + +A deterministic hint overrides the model's role where one fires, but every edit is +sent to the model regardless, because `action` is never derivable from the artifact. +That makes the role comparison free and complete rather than sampled: agreement is +measured on 100% of hinted edits at no extra cost, and a hint that quietly disagrees +with every model reading is a bug in the hint. + +The prompt shows the diff and withholds nothing except the commit message's +authority: subjects in this corpus routinely misdescribe their diffs — one reading +"Extend research and normalize wrapped answers" deletes an entire audit pass — so +the message is supplied as a claim to be checked, not as the answer. +""" + +from __future__ import annotations + +import asyncio +from typing import Iterable + +from vero.interpret.cache import Cache, key_of +from vero.interpret.labeling.client import AsyncLLM, LLMError +from vero.interpret.labeling.taxonomy import ( + ACTION_RUBRIC, + ROLE_RUBRIC, + TAXONOMY_VERSION, + Action, + Direction, + Provenance, + Role, + direction_of, + role_hint, +) +from vero.interpret.models import Edit, EditLabel + +PROMPT_VERSION = "2" + +# Diffs were capped at 6000 characters, which truncated 12% of edits mid-hunk to save +# tokens that were never the constraint: the whole corpus costs a couple of dollars. +# These bounds exist only so one pathological edit cannot blow a context window. +MAX_DIFF_CHARS = 40_000 +MAX_SOURCE_CHARS = 12_000 + +SYSTEM = """You classify individual edits made by an AI agent that was told to improve \ +another agent's harness. + +You are shown ONE edit: a diff restricted to a single symbol (a function, method, or \ +module-level binding). Classify only that edit, not the whole commit. + +The commit subject is provided for context but is frequently wrong: it may describe \ +work that is not in this diff, omit changes that are, or claim a revert while leaving \ +behaviour in place. Trust the diff. Where they disagree, say so in `mechanism`. + +Judge only what the code does. Do not speculate about whether it improved the score.""" + +_SCHEMA = { + "type": "object", + "additionalProperties": False, + "required": ["action", "role", "provenance", "mechanism", "confidence"], + "properties": { + "action": { + "type": "string", + "enum": [a.value for a in Action], + "description": "; ".join(f"{k}: {v}" for k, v in ACTION_RUBRIC.items()), + }, + "role": { + "type": "string", + "enum": [r.value for r in Role], + "description": "; ".join(f"{k}: {v}" for k, v in ROLE_RUBRIC.items()), + }, + "provenance": { + "type": "string", + "enum": [p.value for p in Provenance], + "description": "Ignored downstream -- provenance is derived from the seed " + "tree. Answer unknown unless the history line settles it.", + }, + "mechanism": { + "type": "string", + "description": "One sentence: what this edit actually does.", + }, + "confidence": {"type": "number", "minimum": 0.0, "maximum": 1.0}, + }, +} + + +def _user_prompt(edit: Edit, subject: str, siblings: int = 0) -> str: + value = "" + if edit.before_value is not None or edit.after_value is not None: + value = f"\nvalue: {edit.before_value} -> {edit.after_value}" + + # History. Without it the model cannot tell a first-time addition from a rewrite of + # the optimizer's own work, and it guessed "own" on almost every fix when asked. + if not edit.in_seed: + history = "this symbol did not exist in the seed — the optimizer created it" + elif edit.prior_touches: + history = ( + f"this symbol came from the seed and {edit.prior_touches} earlier " + f"candidate(s) in this run already modified it" + ) + else: + history = "this symbol is still as the seed wrote it; this is the first change to it" + + # The subject often describes OTHER edits in the same commit, so say how many there + # are. Warning that it "may be inaccurate" was not enough on its own. + sib = ( + f"\nnote: this commit touched {siblings} symbols in total, so the subject may " + f"describe a different one" + if siblings > 1 else "" + ) + context = ( + f"\n\nthe symbol after the edit, for context:\n{edit.after_source[:MAX_SOURCE_CHARS]}" + if edit.after_source else "" + ) + return ( + f"file: {edit.path}\n" + f"symbol: {edit.symbol} ({edit.symbol_kind.value})\n" + f"lines: +{edit.added} -{edit.removed}{value}\n" + f"history: {history}\n" + f"commit subject (may be inaccurate): {subject}{sib}\n\n" + f"diff:\n{edit.diff[:MAX_DIFF_CHARS]}{context}" + ) + + +def cache_key(edit: Edit, model: str) -> str: + return key_of(edit.id, model, PROMPT_VERSION, TAXONOMY_VERSION) + + +class Labeler: + def __init__(self, llm: AsyncLLM, cache: Cache) -> None: + self.llm = llm + self.cache = cache + self.hint_authoritative = 0 + self.disagreements = 0 + self.failed = 0 + + async def label(self, edit: Edit, subject: str = "", siblings: int = 0) -> EditLabel | None: + key = cache_key(edit, self.llm.settings.model) + if (cached := self.cache.get_json(key)) is not None: + return EditLabel.model_validate(cached) + + hint = role_hint(edit.path, edit.symbol, edit.symbol_kind.value) + # Every edit goes to the model regardless of the hint, because `action` is never + # derivable and always needs one. That makes the role comparison free rather + # than sampled: the audit covers 100% of hinted edits at no extra cost. An + # earlier version sampled it, which bought nothing and under-reported coverage. + if hint is not None: + self.hint_authoritative += 1 + + try: + raw = await self.llm.json_call( + SYSTEM, _user_prompt(edit, subject, siblings), _SCHEMA + ) + except LLMError: + self.failed += 1 + return None + + role = hint.value if hint is not None else raw["role"] + label = EditLabel( + edit_id=edit.id, + action=raw["action"], + role=role, + provenance=raw.get("provenance", Provenance.UNKNOWN.value), + direction=direction_of(edit.before_value, edit.after_value).value, + mechanism=raw.get("mechanism", "")[:300], + confidence=float(raw.get("confidence", 0.0)), + hinted=hint is not None, + model=self.llm.settings.model, + taxonomy_version=TAXONOMY_VERSION, + ) + # Disagreement is recorded, not silently resolved: it is the signal that a + # hint is wrong, and it is only visible if both readings are kept. + if hint is not None and raw["role"] != hint.value: + self.disagreements += 1 + label.mechanism = f"[hint={hint.value} model={raw['role']}] {label.mechanism}" + self.cache.put_json(key, label.model_dump()) + return label + + async def label_all( + self, + edits: Iterable[tuple[Edit, str, int]], + *, + progress=None, + ) -> list[EditLabel]: + tasks = [asyncio.create_task(self.label(e, s, n)) for e, s, n in edits] + out: list[EditLabel] = [] + for i, task in enumerate(asyncio.as_completed(tasks), 1): + label = await task + if label is not None: + out.append(label) + if progress and i % 50 == 0: + progress(i, len(tasks)) + return out + + def stats(self) -> str: + return ( + f"{self.cache.stats()}; calls={self.llm.calls} retries={self.llm.retries} " + f"rule-decided={self.hint_authoritative} (all audited) " + f"disagreements={self.disagreements} failed={self.failed}" + ) + + +def direction_only(edit: Edit) -> Direction: + """Derived facet, exposed for callers that want it without labelling.""" + return direction_of(edit.before_value, edit.after_value) diff --git a/vero/src/vero/interpret/labeling/taxonomy.py b/vero/src/vero/interpret/labeling/taxonomy.py new file mode 100644 index 00000000..aae212d1 --- /dev/null +++ b/vero/src/vero/interpret/labeling/taxonomy.py @@ -0,0 +1,182 @@ +"""Facets an edit is labelled on, and the deterministic part of assigning them. + +The vocabulary is derived from the symbol distribution the optimizers actually +touched across 100 runs, not chosen in advance: the same handful of targets dominate +in every benchmark, so the roots below are what the corpus contains rather than what +seemed plausible. + +Three facets are model-assigned (`action`, `role` where a hint does not fire, +`provenance`); the rest are derived. Keeping the model's job to semantic judgement is +what makes the output checkable — a role hint that disagrees with a model label is a +bug you can find without reading anything. + +Bump TAXONOMY_VERSION on any change here. It is part of the label cache key, so a +revision re-labels without re-extracting. +""" + +from __future__ import annotations + +import re +from enum import StrEnum + +TAXONOMY_VERSION = "2" + + +class Role(StrEnum): + """What part of the harness an edit targets. Closed and benchmark-agnostic.""" + + PROMPT = "prompt" # system/instruction text + CONTROL_LOOP = "control_loop" # the agent's main turn loop + TOOL_SURFACE = "tool_surface" # tool declarations shown to the model + TOOL_IMPL = "tool_impl" # a specific tool's implementation + SUBMISSION = "submission" # emitting/parsing the final answer + MODEL_CLIENT = "model_client" # completion params, retries, timeouts + BUDGET_TURNS = "budget_turns" + BUDGET_OUTPUT = "budget_output" # tool-output truncation caps + BUDGET_WALLCLOCK = "budget_wallclock" # deadlines that stop the loop + CONTEXT_MGMT = "context_mgmt" # compaction, history pruning + RETRIEVAL = "retrieval" # search/index parameters and strategy + ENV_SETUP = "env_setup" # setup() work, package installation + INITIALIZATION = "initialization" # __init__, wiring + TESTS = "tests" # the candidate's own test suite + METADATA = "metadata" # version/name bookkeeping + OTHER = "other" + + +class Action(StrEnum): + """What was done to it.""" + + FIX = "fix" # repairs a defect + ADD = "add" # new capability or code path + REMOVE = "remove" # deletes a capability or code path + TUNE = "tune" # changes a value, no structural change + RESTRUCTURE = "restructure" # same behaviour, different shape + REWORD = "reword" # instruction text changed, intent preserved + REVERT = "revert" # undoes the optimizer's own earlier edit + COSMETIC = "cosmetic" # cannot change behaviour + + +class Provenance(StrEnum): + """For fixes: whose defect was it? + + Distinguishing these matters. An optimizer repairing the seed is doing the task; + an optimizer repairing damage it caused two candidates ago is paying down its own + debt, and several cells in this corpus spent most of their budget that way. + """ + + SEED = "seed" + OWN = "own" + UNKNOWN = "unknown" + + +class Direction(StrEnum): + """For tunes. Derived from before/after values where both are numeric.""" + + UP = "up" + DOWN = "down" + UNCHANGED = "unchanged" + NA = "na" + + +# Symbol-name hints. These fire on the leaf symbol and are exact enough to skip the +# model entirely; anything unmatched goes to the labeller. Ordered, first match wins. +_ROLE_HINTS: list[tuple[re.Pattern[str], Role]] = [ + (re.compile(r"(PROMPT|INSTRUCTIONS?|GUIDANCE|PLAYBOOK)", re.I), Role.PROMPT), + (re.compile(r"^(MAX_TURNS|MAX_STEPS|TURN_BUDGET|STEP_BUDGET|MAX_ITER\w*)$"), Role.BUDGET_TURNS), + (re.compile(r"^(MAX_TOOL_OUTPUT\w*|MAX_OUTPUT\w*|MAX_CHARS|MAX_BYTES)$"), Role.BUDGET_OUTPUT), + (re.compile(r"\w*(DEADLINE|TIME_BUDGET|WALL_CLOCK)\w*"), Role.BUDGET_WALLCLOCK), + (re.compile(r"^(MAX_HISTORY\w*|MAX_CONTEXT\w*|.*COMPACT.*)$", re.I), Role.CONTEXT_MGMT), + (re.compile(r"^TOOLS$"), Role.TOOL_SURFACE), + (re.compile(r"\.run$"), Role.CONTROL_LOOP), + (re.compile(r"\.(_submit|submit\w*|_answer_payload|_extract\w*answer\w*)$", re.I), Role.SUBMISSION), + (re.compile(r"^(_answer_payload|_scale_variants|_format_number)$"), Role.SUBMISSION), + (re.compile(r"\.(_completion_kwargs|_complete|_create|_chat)$"), Role.MODEL_CLIENT), + (re.compile(r"\.(_run_shell|_exec|_run_python|_index_command|_open\w*|_search\w*)$"), Role.TOOL_IMPL), + (re.compile(r"\.setup$"), Role.ENV_SETUP), + (re.compile(r"\.__init__$"), Role.INITIALIZATION), + (re.compile(r"\.(version|name)$"), Role.METADATA), +] + + +def role_hint(path: str, symbol: str, symbol_kind: str | None = None) -> Role | None: + """Deterministic role where path, kind or name settles it, else None. + + Order matters. Path wins first: an edit inside the candidate's own test suite is + a test edit whatever it touches. Kind wins next, because a long module-level + string binding is a prompt regardless of what it is called — which is how + `REVIEW_INSTRUCTIONS` and friends get caught without enumerating names. + """ + if "test" in path.rsplit("/", 1)[-1]: + return Role.TESTS + if symbol_kind == "prompt_text": + return Role.PROMPT + for pattern, role in _ROLE_HINTS: + if pattern.search(symbol): + return role + return None + + +_NUMERIC = re.compile(r"-?\d+(?:\.\d+)?") + + +def direction_of(before: str | None, after: str | None) -> Direction: + """Numeric direction of a tune, where both sides parse.""" + if before is None or after is None: + return Direction.NA + if before == after: + return Direction.UNCHANGED + b, a = _NUMERIC.search(before), _NUMERIC.search(after) + if not (b and a): + return Direction.NA + return Direction.UP if float(a.group()) > float(b.group()) else Direction.DOWN + + +# Rubrics. Passing bare enum names left the model to infer sixteen role meanings from +# the names alone, which is the likeliest source of the 16% rule/model disagreement +# and of `add` swamping `reword`. These go into the JSON schema so the definition +# travels with the field rather than sitting in a system prompt the model may skim. +ROLE_RUBRIC: dict[str, str] = { + Role.PROMPT: "System/instruction text sent to the model: task guidance, worked " + "examples, output-format rules. Prose the model reads, not code.", + Role.CONTROL_LOOP: "The agent's main turn loop: how many steps run, when to stop, " + "what happens between turns, dispatch of tool calls.", + Role.TOOL_SURFACE: "The tool DECLARATIONS shown to the model — names, JSON schemas, " + "descriptions. Not the code that implements them.", + Role.TOOL_IMPL: "The implementation behind a tool: shell execution, file reading, " + "search calls, document opening.", + Role.SUBMISSION: "Producing and parsing the final answer: writing the answer file, " + "stripping markers, detecting refusals, forcing an answer.", + Role.MODEL_CLIENT: "How a completion is requested: client construction, sampling " + "parameters, per-request timeouts, SDK retries.", + Role.BUDGET_TURNS: "A limit counted in turns or steps.", + Role.BUDGET_OUTPUT: "A limit on how much tool output is kept or shown.", + Role.BUDGET_WALLCLOCK: "A limit counted in seconds: deadlines, elapsed-time checks.", + Role.CONTEXT_MGMT: "Managing conversation history: compaction, pruning, summarising.", + Role.RETRIEVAL: "Search or index behaviour: query construction, result count, " + "ranking, snippet sizing.", + Role.ENV_SETUP: "Preparing the container before work starts: installing packages, " + "writing fixtures. NOT ignoring build artifacts.", + Role.INITIALIZATION: "Constructor and wiring: state set up once when the agent is " + "created.", + Role.TESTS: "The candidate's own test suite.", + Role.METADATA: "Bookkeeping with no effect on behaviour: version strings, names.", + Role.OTHER: "None of the above, or genuinely unclear from this diff alone.", +} + +ACTION_RUBRIC: dict[str, str] = { + Action.FIX: "Repairs something broken: a crash, a type error, a parse that returned " + "the wrong thing. There must be a defect, not merely a weakness.", + Action.ADD: "Introduces a capability or code path that did not exist. Prefer " + "`reword` if existing prose was rewritten and `tune` if a value moved.", + Action.REMOVE: "Deletes a capability or code path.", + Action.TUNE: "Changes a value or threshold, leaving structure intact.", + Action.RESTRUCTURE: "Reorganises code without intending to change behaviour: " + "extracting a helper, renaming, moving logic.", + Action.REWORD: "Rewrites instruction or description TEXT while keeping its intent. " + "Use this for prompt edits that sharpen or re-phrase guidance — " + "most prompt changes are rewords, not additions.", + Action.REVERT: "Undoes an edit the same optimizer made earlier in this cell. Only " + "when the history shown says so.", + Action.COSMETIC: "Cannot change behaviour: comments, formatting, .gitignore, " + "ignoring compiled artifacts, dead code.", +} diff --git a/vero/src/vero/interpret/models.py b/vero/src/vero/interpret/models.py new file mode 100644 index 00000000..16ec367e --- /dev/null +++ b/vero/src/vero/interpret/models.py @@ -0,0 +1,184 @@ +"""Canonical schema for interpretability analysis. + +Every source adapter normalises into these types, so downstream code never sees a +harbor path, a tarball, or a git object. Adding a second producer of optimization +runs means writing an adapter, not touching anything below `artifacts/`. + +Identity is content-addressed throughout. `Edit.id` in particular is derived from the +edit's own content rather than its position, so a cached label survives re-extraction, +re-ordering, and unrelated edits appearing earlier in the same candidate. +""" + +from __future__ import annotations + +import hashlib +from enum import StrEnum + +from pydantic import BaseModel, Field + + +def content_id(*parts: str) -> str: + """Stable 16-hex identity over the given parts.""" + digest = hashlib.sha256("\x00".join(parts).encode("utf-8", "replace")) + return digest.hexdigest()[:16] + + +class SymbolKind(StrEnum): + """What kind of thing an edit landed in, resolved from the syntax tree.""" + + FUNCTION = "function" + METHOD = "method" + CLASS = "class" + PROMPT_TEXT = "prompt_text" # module-level str binding, long + SCALAR_CONST = "scalar_const" # module-level int/float/bool/short str + COLLECTION = "collection" # module-level tuple/list/dict/set + REGEX = "regex" # module-level re.compile + MODULE = "module" # module level, unattributed + NON_PYTHON = "non_python" # config, markdown, lockfiles + + +class CellRef(BaseModel): + """One optimization run: a single (source, benchmark, cell) triple.""" + + source: str # adapter name, e.g. "harbor" + benchmark: str + cell: str + root: str # directory the adapter was pointed at + cell_dir: str # resolved during discovery, not recomputed + job_dir: str | None = None + + @property + def key(self) -> str: + return f"{self.source}/{self.benchmark}/{self.cell}" + + +class Candidate(BaseModel): + """One state the optimizer produced. Position 0 is the seed.""" + + sha: str + parent_sha: str | None = None + position: int + subject: str = "" + body: str = "" + files: list[str] = Field(default_factory=list) + insertions: int = 0 + deletions: int = 0 + tree_sha: str | None = None + is_seed: bool = False + is_shipped: bool = False + + +class EvalRecord(BaseModel): + """One scoring of one candidate. + + A candidate can be scored more than once on the same partition; those repeats are + the corpus's only direct measurement of evaluation noise, so they are kept as + separate records rather than collapsed into a per-partition map. + """ + + candidate_sha: str + partition: str + score: float + error_rate: float | None = None + selection_kind: str | None = None # "all" for a full partition, else a slice + n_attempts: int | None = None + started_at: str | None = None + finished_at: str | None = None + + +class Edit(BaseModel): + """A symbol-scoped change: the unit of analysis. + + Not the commit. A single candidate routinely bundles a bug fix with unrelated + tuning under one subject line, so labelling per candidate assigns one category to + several distinct modifications. + """ + + id: str + cell_key: str + candidate_sha: str + path: str + symbol: str # "" when unattributed + symbol_kind: SymbolKind + added: int = 0 + removed: int = 0 + before_value: str | None = None # scalar constants only + after_value: str | None = None + diff: str = "" # unified diff restricted to this symbol + after_source: str | None = None # the symbol's full text after the edit + in_seed: bool = True # did this symbol exist in the seed at all? + prior_touches: int = 0 # earlier candidates in this cell that touched it + + @staticmethod + def make_id(cell_key: str, sha: str, path: str, symbol: str, diff: str) -> str: + return content_id(cell_key, sha, path, symbol, diff) + + +class EditLabel(BaseModel): + """A model-assigned reading of one edit, plus the facets that were derived. + + `hinted` records whether the role came from a deterministic rule rather than the + model, so agreement between the two can be measured instead of assumed. + """ + + edit_id: str + action: str + role: str + provenance: str = "unknown" + direction: str = "na" + mechanism: str = "" # one line, the model's own words + confidence: float = 0.0 + hinted: bool = False + model: str = "" + taxonomy_version: str = "" + + +class Trajectory(BaseModel): + """Everything known about one cell.""" + + ref: CellRef + candidates: list[Candidate] = Field(default_factory=list) + evaluations: list[EvalRecord] = Field(default_factory=list) + edits: list[Edit] = Field(default_factory=list) + labels: list[EditLabel] = Field(default_factory=list) + reward: float | None = None + baseline_reward: float | None = None + error_rate: float | None = None + total_tokens: float | None = None + + @property + def seed(self) -> Candidate | None: + return next((c for c in self.candidates if c.is_seed), None) + + @property + def shipped(self) -> Candidate | None: + return next((c for c in self.candidates if c.is_shipped), None) + + @property + def shipped_the_seed(self) -> bool: + """True when the shipped tree is byte-identical to the seed tree. + + Decided on tree hashes, never on a commit message: messages saying "revert" + routinely carry surviving behavioural change, and messages saying nothing of + the sort are sometimes total reverts. + """ + seed, shipped = self.seed, self.shipped + if not (seed and shipped and seed.tree_sha and shipped.tree_sha): + return False + return seed.tree_sha == shipped.tree_sha + + +class Corpus(BaseModel): + """A collated set of trajectories, usually one analysis scope.""" + + trajectories: list[Trajectory] = Field(default_factory=list) + + def by_benchmark(self) -> dict[str, list[Trajectory]]: + out: dict[str, list[Trajectory]] = {} + for t in self.trajectories: + out.setdefault(t.ref.benchmark, []).append(t) + return out + + def edits(self): + for t in self.trajectories: + yield from t.edits diff --git a/vero/tests/test_interpret_locus.py b/vero/tests/test_interpret_locus.py new file mode 100644 index 00000000..5943eb50 --- /dev/null +++ b/vero/tests/test_interpret_locus.py @@ -0,0 +1,75 @@ +"""Locus resolution is the load-bearing deterministic step; test it directly. + +These cases are the ones that actually went wrong on real data: git's hunk header +reports the enclosing class for a method, and module-level bindings collapse a system +prompt, a tool table and a dozen tuning constants into one bucket. +""" + +from __future__ import annotations + +from vero.interpret.edits.locus import changed_lines, scalar_value, symbol_map +from vero.interpret.models import SymbolKind + +SOURCE = ''' +MAX_TURNS = 24 +SHELL_TIMEOUT_SEC = 150 +INSTRUCTIONS = """ +{} +""" +TOOLS = [{{"name": "run_shell"}}, {{"name": "submit"}}] +_ANSWER_RE = re.compile(r"x") + + +def helper(value): + return value + + +class Agent: + def run(self): + return 1 + + def _run_shell(self, cmd): + return cmd +'''.format("guidance " * 40) + + +def test_method_resolves_to_method_not_class(): + mapping = symbol_map(SOURCE) + line = next( + i for i, text in enumerate(SOURCE.splitlines(), 1) if "return cmd" in text + ) + symbol, kind = mapping[line] + assert symbol == "Agent._run_shell" + assert kind is SymbolKind.METHOD + + +def test_module_bindings_split_by_name_and_shape(): + mapping = symbol_map(SOURCE) + got = {symbol: kind for symbol, kind in mapping.values()} + assert got["MAX_TURNS"] is SymbolKind.SCALAR_CONST + assert got["INSTRUCTIONS"] is SymbolKind.PROMPT_TEXT + assert got["TOOLS"] is SymbolKind.COLLECTION + assert got["_ANSWER_RE"] is SymbolKind.REGEX + + +def test_plain_function_is_not_a_method(): + mapping = symbol_map(SOURCE) + line = next( + i for i, text in enumerate(SOURCE.splitlines(), 1) if "return value" in text + ) + assert mapping[line] == ("helper", SymbolKind.FUNCTION) + + +def test_changed_lines_parses_zero_context_hunks(): + diff = "@@ -1 +1 @@\n-a\n+b\n@@ -10,0 +11,3 @@\n+x\n+y\n+z\n" + assert changed_lines(diff) == {1, 11, 12, 13} + + +def test_scalar_value_reads_the_literal(): + assert scalar_value(SOURCE, "MAX_TURNS") == "24" + assert scalar_value(SOURCE, "SHELL_TIMEOUT_SEC") == "150" + assert scalar_value(SOURCE, "nonexistent") is None + + +def test_unparsable_source_yields_no_map_rather_than_raising(): + assert symbol_map("def broken(:\n") == {} diff --git a/vero/uv.lock b/vero/uv.lock index 0681a8e8..cdf3d86b 100644 --- a/vero/uv.lock +++ b/vero/uv.lock @@ -1830,6 +1830,9 @@ harbor = [ { name = "pyyaml" }, { name = "uvicorn" }, ] +interpret = [ + { name = "openai" }, +] optimize = [ { name = "async-lru" }, { name = "beautifulsoup4" }, @@ -1864,6 +1867,7 @@ requires-dist = [ { name = "httpx", marker = "extra == 'optimize'", specifier = ">=0.28.1" }, { name = "jinja2", marker = "extra == 'harbor'", specifier = ">=3.1.6" }, { name = "lxml", marker = "extra == 'optimize'", specifier = ">=6.0.2" }, + { name = "openai", marker = "extra == 'interpret'", specifier = ">=1.0" }, { name = "openai-agents", extras = ["litellm"], marker = "extra == 'optimize'", specifier = ">=0.18.3,<0.19" }, { name = "orjson", marker = "extra == 'optimize'", specifier = ">=3.10" }, { name = "pydantic", specifier = ">=2.11.7" }, @@ -1874,7 +1878,7 @@ requires-dist = [ { name = "wandb", marker = "extra == 'wandb'", specifier = ">=0.19.10" }, { name = "wcmatch", specifier = ">=10.1" }, ] -provides-extras = ["harbor", "claude", "optimize", "wandb"] +provides-extras = ["harbor", "claude", "interpret", "optimize", "wandb"] [package.metadata.requires-dev] dev = [