From 68ad00eb63a10cf092469975019825c17fb9d3e9 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 7 Jul 2026 18:03:37 -0600 Subject: [PATCH] =?UTF-8?q?feat(gates):=20neutralizationGate=20=E2=80=94?= =?UTF-8?q?=20footprint-matched=20placebo=20control=20(0.107.0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A held-out gate proves a candidate beat baseline; it cannot prove the lift came from the candidate's CONTENT rather than the prompt/mount FOOTPRINT the content added. neutralizationGate closes that hole: runImprovementLoop scores a third holdout arm — the winner with its content footprint-matched-blanked (neutralizeText) — and the gate holds any win whose lift survives blanking (decorative), however large or significant its raw lift. - src/campaign/neutralize.ts: neutralizeText — layout-preserving, byte-exact (ASCII) content blanking. - src/campaign/gates/neutralization-gate.ts: composable Gate; R46 rule (reject when neutralized lift >= maxDecorativeFraction of candidate lift, default 0.5); fail-closed on no lift, fail-loud on a missing neutralized arm. - GateContext gains optional neutralizedJudgeScores/neutralizedArtifacts. - runImprovementLoop gains an optional neutralize fn; when supplied and the winner changed, it runs the placebo arm and exposes it to the gate. Additive: new exports + optional fields only; no consumer bump required. --- clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- package.json | 2 +- .../gates/neutralization-gate.test.ts | 117 +++++++++++++++ src/campaign/gates/neutralization-gate.ts | 134 ++++++++++++++++++ src/campaign/index.ts | 5 + src/campaign/neutralize.ts | 35 +++++ src/campaign/presets/run-improvement-loop.ts | 36 +++++ src/campaign/types.ts | 10 ++ 9 files changed, 340 insertions(+), 3 deletions(-) create mode 100644 src/campaign/gates/neutralization-gate.test.ts create mode 100644 src/campaign/gates/neutralization-gate.ts create mode 100644 src/campaign/neutralize.ts diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 903a8a40..0748b10e 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.106.3" +version = "0.107.0" description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index 475dd95d..e9cf6168 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -58,7 +58,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.106.3" + __version__ = "0.107.0" __all__ = [ "Client", diff --git a/package.json b/package.json index 61315f76..93f4f9b1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.106.3", + "version": "0.107.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/campaign/gates/neutralization-gate.test.ts b/src/campaign/gates/neutralization-gate.test.ts new file mode 100644 index 00000000..2faa3e26 --- /dev/null +++ b/src/campaign/gates/neutralization-gate.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from 'vitest' +import { neutralizeText } from '../neutralize' +import type { GateContext, JudgeScore, Scenario } from '../types' +import { neutralizationGate } from './neutralization-gate' + +const scenarios: Scenario[] = [ + { id: 's1' } as Scenario, + { id: 's2' } as Scenario, + { id: 's3' } as Scenario, +] + +/** Build a per-cell score map: one cell per scenario (rep 0), one judge. */ +function scoreMap(byScenario: Record): Map> { + const m = new Map>() + for (const [sid, composite] of Object.entries(byScenario)) { + m.set(`${sid}:0`, { j: { composite, dimensions: {}, notes: '' } as JudgeScore }) + } + return m +} + +function ctx( + candidate: Record, + baseline: Record, + neutralized: Record, +): GateContext { + return { + candidateArtifacts: new Map(), + judgeScores: scoreMap(candidate), + baselineJudgeScores: scoreMap(baseline), + neutralizedJudgeScores: scoreMap(neutralized), + scenarios, + cost: { candidate: 0, baseline: 0 }, + signal: new AbortController().signal, + } +} + +describe('neutralizationGate', () => { + it('SHIPS when the lift collapses under neutralization (content is causal)', async () => { + const gate = neutralizationGate({ scenarios }) + // candidate lifts +4 over baseline; blanked content recovers ~0 → causal. + const res = await gate.decide( + ctx( + { s1: 8, s2: 8, s3: 8 }, // candidate + { s1: 4, s2: 4, s3: 4 }, // baseline + { s1: 4, s2: 4, s3: 4 }, // neutralized == baseline + ), + ) + expect(res.decision).toBe('ship') + const detail = res.contributingGates[0]?.detail as { decorativeFraction: number } + expect(detail.decorativeFraction).toBeCloseTo(0, 5) + }) + + it('HOLDS when the lift survives neutralization (decorative — footprint, not content)', async () => { + const gate = neutralizationGate({ scenarios }) + // candidate +4; blanked content still recovers +3 (75% ≥ 50%) → decorative. + const res = await gate.decide( + ctx({ s1: 8, s2: 8, s3: 8 }, { s1: 4, s2: 4, s3: 4 }, { s1: 7, s2: 7, s3: 7 }), + ) + expect(res.decision).toBe('hold') + expect(res.reasons[0]).toMatch(/DECORATIVE/) + const detail = res.contributingGates[0]?.detail as { decorativeFraction: number } + expect(detail.decorativeFraction).toBeCloseTo(0.75, 5) + }) + + it('HOLDS exactly at the threshold (equality is decorative)', async () => { + const gate = neutralizationGate({ scenarios, maxDecorativeFraction: 0.5 }) + const res = await gate.decide( + ctx({ s1: 8, s2: 8, s3: 8 }, { s1: 4, s2: 4, s3: 4 }, { s1: 6, s2: 6, s3: 6 }), + ) + // neutralized +2 / candidate +4 = 0.5 → not < 0.5 → hold. + expect(res.decision).toBe('hold') + }) + + it('HOLDS (fail-closed) when the candidate has no positive lift', async () => { + const gate = neutralizationGate({ scenarios }) + const res = await gate.decide( + ctx({ s1: 4, s2: 4, s3: 4 }, { s1: 4, s2: 4, s3: 4 }, { s1: 4, s2: 4, s3: 4 }), + ) + expect(res.decision).toBe('hold') + expect(res.reasons[0]).toMatch(/no positive lift/) + }) + + it('throws when the neutralized arm is missing (composed without loop wiring)', async () => { + const gate = neutralizationGate({ scenarios }) + const bare = ctx({ s1: 8 }, { s1: 4 }, { s1: 4 }) + bare.neutralizedJudgeScores = undefined + await expect(gate.decide(bare)).rejects.toThrow(/neutralizedJudgeScores is required/) + }) +}) + +describe('neutralizeText', () => { + it('preserves whitespace/layout and blanks non-whitespace', () => { + const src = 'line one\n indented two\n' + const out = neutralizeText(src) + expect(out).toBe('#### ###\n ######## ###\n') + // same length + same whitespace positions, zero readable content + expect(out.length).toBe(src.length) + expect([...out].filter((c) => /\s/.test(c)).join('')).toBe( + [...src].filter((c) => /\s/.test(c)).join(''), + ) + expect(out).not.toMatch(/[a-z]/i) + }) + + it('is byte-length-exact for ASCII', () => { + const src = 'the frontier author knows section 1202(b)(2)' + expect(Buffer.byteLength(neutralizeText(src))).toBe(Buffer.byteLength(src)) + }) + + it('preserves character count (not byte count) for multibyte content', () => { + // Documented limitation: a 2-byte char blanks to a 1-byte filler, so byte + // length shrinks while CHARACTER count + layout are preserved. The tokenizer + // footprint tracks char/token structure, which is what the placebo needs. + const src = 'knows §1202' + expect(neutralizeText(src).length).toBe(src.length) + expect(neutralizeText(src)).toBe('##### #####') + }) +}) diff --git a/src/campaign/gates/neutralization-gate.ts b/src/campaign/gates/neutralization-gate.ts new file mode 100644 index 00000000..72775728 --- /dev/null +++ b/src/campaign/gates/neutralization-gate.ts @@ -0,0 +1,134 @@ +/** + * @module + * Composable placebo / neutralization promotion gate. + * + * A held-out gate proves a candidate beat baseline. It CANNOT prove the lift came + * from the candidate's CONTENT rather than from the prompt/mount FOOTPRINT the + * content happened to add (more bytes, a longer prompt). This gate closes that + * hole: it compares the candidate's held-out lift against the lift of a + * FOOTPRINT-MATCHED neutralized variant (same layout + length, zero content, via + * `neutralizeText`). If the neutralized variant reproduces more than + * `maxDecorativeFraction` of the candidate's lift, the lift is decorative — it + * survives blanking the content — and the candidate is HELD regardless of how + * large or significant its raw lift is. + * + * Compose it AFTER the significance gate — significance says the lift is real, + * this says the lift is CAUSED BY THE CONTENT: + * composeGate(heldOutGate({ ... }), neutralizationGate({ ... })) + * + * Requires `ctx.neutralizedJudgeScores`, populated by `runImprovementLoop` when it + * is given a `neutralize` function. A gate composed without that wiring fails + * loud rather than silently passing an unproven candidate. + */ + +import type { Gate, GateContext, GateResult, Scenario } from '../types' +import { pairHoldout } from './statistical-heldout' + +export interface NeutralizationGateOptions { + scenarios: TScenario[] + /** Reject when the neutralized (content-blanked, footprint-matched) variant + * reproduces at least this fraction of the candidate's held-out lift. Default + * 0.5 — if blanking the content keeps half the lift, the content is decorative. + * Equality rejects: a neutralized lift == threshold·candidateLift is decorative. */ + maxDecorativeFraction?: number +} + +/** Mean of a numeric array; 0 for an empty array (callers guard n separately). */ +function mean(xs: number[]): number { + return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length +} + +/** Paired mean held-out lift of `arm` over baseline, on the in-scope cells. */ +function pairedLift( + arm: Map>, + baseline: Map>, + scenarioIds: Set, +): { lift: number; n: number } { + const paired = pairHoldout(arm as never, baseline as never, scenarioIds, (s) => s.composite) + const deltas = paired.after.map((a, i) => a - (paired.before[i] ?? 0)) + return { lift: mean(deltas), n: deltas.length } +} + +/** + * Composable placebo gate: ships only when the candidate's held-out lift is NOT + * mostly reproduced by a footprint-matched neutralized variant. + */ +export function neutralizationGate( + options: NeutralizationGateOptions, +): Gate { + const maxDecorativeFraction = options.maxDecorativeFraction ?? 0.5 + return { + name: 'neutralizationGate', + async decide(ctx: GateContext): Promise { + if (!ctx.baselineJudgeScores) { + throw new Error( + 'neutralizationGate: ctx.baselineJudgeScores is required — the placebo control measures lift OVER baseline.', + ) + } + if (!ctx.neutralizedJudgeScores) { + throw new Error( + 'neutralizationGate: ctx.neutralizedJudgeScores is required. It is populated by runImprovementLoop only when a `neutralize` function is supplied — composing this gate without that wiring would pass an unproven candidate.', + ) + } + const scenarioIds = new Set(options.scenarios.map((s) => s.id)) + const cand = pairedLift( + ctx.judgeScores as never, + ctx.baselineJudgeScores as never, + scenarioIds, + ) + const neut = pairedLift( + ctx.neutralizedJudgeScores as never, + ctx.baselineJudgeScores as never, + scenarioIds, + ) + + // No positive candidate lift → nothing to attribute. Fail closed: the + // placebo control has no basis to clear the candidate (the significance + // gate is what judges lift magnitude; this one only judges its CAUSE). + if (cand.lift <= 0) { + return { + decision: 'hold', + reasons: [ + `neutralization: candidate held-out lift ${cand.lift.toFixed(3)} ≤ 0 — no positive lift to attribute to content`, + ], + contributingGates: [ + { + name: 'neutralizationGate', + passed: false, + detail: { candidateLift: cand.lift, neutralizedLift: neut.lift, n: cand.n }, + }, + ], + delta: cand.lift, + } + } + + const decorativeFraction = neut.lift / cand.lift + const passed = decorativeFraction < maxDecorativeFraction + const pct = (decorativeFraction * 100).toFixed(0) + return { + decision: passed ? 'ship' : 'hold', + reasons: passed + ? [ + `neutralization: content is causal — blanked variant reproduces ${pct}% of the lift (< ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate Δ ${cand.lift.toFixed(3)}, neutralized Δ ${neut.lift.toFixed(3)}`, + ] + : [ + `neutralization: lift is DECORATIVE — blanking the content (footprint-matched) reproduces ${pct}% of the lift (≥ ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate Δ ${cand.lift.toFixed(3)}, neutralized Δ ${neut.lift.toFixed(3)}`, + ], + contributingGates: [ + { + name: 'neutralizationGate', + passed, + detail: { + candidateLift: cand.lift, + neutralizedLift: neut.lift, + decorativeFraction, + maxDecorativeFraction, + n: cand.n, + }, + }, + ], + delta: cand.lift, + } + }, + } +} diff --git a/src/campaign/index.ts b/src/campaign/index.ts index f949c888..750eea16 100644 --- a/src/campaign/index.ts +++ b/src/campaign/index.ts @@ -67,6 +67,10 @@ export { defaultProductionGate, } from './gates/default-production-gate' export { type HeldOutGateOptions, heldOutGate } from './gates/heldout-gate' +export { + type NeutralizationGateOptions, + neutralizationGate, +} from './gates/neutralization-gate' export { type PowerPreflight, type PowerPreflightOptions, @@ -110,6 +114,7 @@ export { type FsLabeledScenarioStoreOptions, LabeledScenarioStoreError, } from './labeled-store/fs-adapter' +export { neutralizeText } from './neutralize' // ── Presets (the documented public surface) ────────────────────────── export { type CompareProposersOptions, diff --git a/src/campaign/neutralize.ts b/src/campaign/neutralize.ts new file mode 100644 index 00000000..283852b4 --- /dev/null +++ b/src/campaign/neutralize.ts @@ -0,0 +1,35 @@ +/** + * @module + * Footprint-matched neutralization — the placebo control for content-vs-footprint + * attribution in a promotion gate. + * + * A promoted surface can raise a held-out score two different ways: + * 1. its CONTENT is informative (the thing we want to promote), or + * 2. it merely added prompt/mount FOOTPRINT — more bytes, more lines, a longer + * more authoritative-looking prompt — that the model spends attention on + * regardless of what the bytes say. + * + * A held-out gate proves the candidate beat baseline; it cannot separate (1) from + * (2). `neutralizeText` produces a variant that keeps the input's layout and + * length while carrying ZERO information, so scoring it isolates the footprint + * contribution (2). Feed the neutralized variant's scores to `neutralizationGate`: + * any lift it still holds over baseline is decorative, and a candidate whose lift + * survives neutralization is rejected however large its raw lift. + */ + +/** Filler for blanked content. A single ASCII byte, so a run of it preserves an + * ASCII source's exact byte length; for multibyte sources it preserves CHARACTER + * count and layout (what the tokenizer footprint tracks), not raw byte count. */ +const FILLER = '#' + +/** + * Blank every non-whitespace character to a 1-byte filler while preserving all + * whitespace. Line count, indentation, and word/line lengths are unchanged — so + * the neutralized variant has the same layout and (for ASCII) the same byte + * footprint as the input, but no readable content. Whitespace is preserved + * deliberately: collapsing it would change the token structure and stop the + * variant from being a true footprint match. + */ +export function neutralizeText(content: string): string { + return content.replace(/\S/g, FILLER) +} diff --git a/src/campaign/presets/run-improvement-loop.ts b/src/campaign/presets/run-improvement-loop.ts index 6713c96f..f0fe76d1 100644 --- a/src/campaign/presets/run-improvement-loop.ts +++ b/src/campaign/presets/run-improvement-loop.ts @@ -58,6 +58,14 @@ export type RunImprovementLoopOptions< /** Optional render override — substrate writes a diff-shaped surface; pass * a function to format the promoted surface differently. */ renderPromotedDiff?: (winnerSurface: MutableSurface, baselineSurface: MutableSurface) => string + /** Placebo control. When supplied AND the winner differs from baseline, the + * loop scores a THIRD holdout arm: the winner surface with its content + * footprint-matched-blanked by this function (typically via `neutralizeText`). + * Its scores are exposed to the gate as `ctx.neutralizedJudgeScores`, letting + * a `neutralizationGate` reject a win whose lift survives blanking the content + * (decorative — driven by footprint, not content). Costs one extra holdout + * campaign; omit to skip. Return a byte/layout-matched blank of the winner. */ + neutralize?: (winnerSurface: MutableSurface, baselineSurface: MutableSurface) => MutableSurface } export interface RunImprovementLoopResult @@ -197,6 +205,32 @@ export async function runImprovementLoop( baselineJudgeScores.set(cell.cellId, cell.judgeScores) } + // ── (3a) placebo arm ─────────────────────────────────────────────── + // When a `neutralize` fn is wired and the winner actually changed something, + // score a third holdout arm: the winner surface with its content + // footprint-matched-blanked. A `neutralizationGate` reads these scores to + // reject a win whose lift survives blanking the content (decorative — driven + // by the added footprint, not the content). Skipped for a no-op winner (there + // is no content to blank) and when no `neutralize` is supplied. + let neutralizedArtifacts: Map | undefined + let neutralizedJudgeScores: ScoreMap | undefined + if (opts.neutralize && !winnerIsBaseline) { + const neutralizedSurface = opts.neutralize(optimization.winnerSurface, opts.baselineSurface) + const neutralizedOnHoldout = await runCampaign({ + ...opts, + dispatchTimeoutMs, + scenarios: opts.holdoutScenarios, + dispatch: (scenario, ctx) => opts.dispatchWithSurface(neutralizedSurface, scenario, ctx), + runDir: `${opts.runDir}/holdout-neutralized`, + }) + neutralizedArtifacts = new Map() + neutralizedJudgeScores = new Map() + for (const cell of neutralizedOnHoldout.cells) { + neutralizedArtifacts.set(cell.cellId, cell.artifact) + neutralizedJudgeScores.set(cell.cellId, cell.judgeScores) + } + } + // No-op guard: a winner identical to the baseline has nothing to promote, so // it never reaches the gate — otherwise the gate scores baseline-vs-itself, // sees model noise as a delta, and can "ship" an empty diff (the observed @@ -217,6 +251,8 @@ export async function runImprovementLoop( baselineArtifacts, judgeScores, baselineJudgeScores, + neutralizedArtifacts, + neutralizedJudgeScores, scenarios: opts.holdoutScenarios, cost: { candidate: winnerOnHoldout.aggregates.totalCostUsd, diff --git a/src/campaign/types.ts b/src/campaign/types.ts index 7c5bd651..0523e5ce 100644 --- a/src/campaign/types.ts +++ b/src/campaign/types.ts @@ -324,6 +324,16 @@ export interface GateContext { * cannot represent both. A gate computing a holdout delta MUST read * candidate from `judgeScores` and baseline from here. */ baselineJudgeScores?: Map> + /** Neutralized-arm judge scores, keyed by cellId — the winner surface with its + * content footprint-matched-blanked (via a `neutralize` fn). Same scenarios as + * `judgeScores`. Present ONLY when `runImprovementLoop` was given a `neutralize` + * function. A placebo gate (`neutralizationGate`) compares this arm's lift + * against the candidate's to reject decorative wins (lift from footprint, not + * content). Undefined otherwise. */ + neutralizedJudgeScores?: Map> + /** Neutralized-arm artifacts, keyed by cellId. Present alongside + * `neutralizedJudgeScores`. */ + neutralizedArtifacts?: Map scenarios: TScenario[] cost: { candidate: number; baseline: number } signal: AbortSignal