feat(gates): neutralizationGate — footprint-matched placebo control (0.107.0)

clients/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agent-eval-rpc"
-version = "0.106.3"
+version = "0.107.0"
 description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client."
 readme = "README.md"
 requires-python = ">=3.10"

clients/python/src/agent_eval_rpc/__init__.py

@@ -58,7 +58,7 @@
 try:
     __version__ = version("agent-eval-rpc")
 except PackageNotFoundError:
-    __version__ = "0.106.3"
+    __version__ = "0.107.0"
 
 __all__ = [
     "Client",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.106.3",
+  "version": "0.107.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {

src/campaign/gates/neutralization-gate.test.ts

@@ -0,0 +1,117 @@
+import { describe, expect, it } from 'vitest'
+import { neutralizeText } from '../neutralize'
+import type { GateContext, JudgeScore, Scenario } from '../types'
+import { neutralizationGate } from './neutralization-gate'
+
+const scenarios: Scenario[] = [
+  { id: 's1' } as Scenario,
+  { id: 's2' } as Scenario,
+  { id: 's3' } as Scenario,
+]
+
+/** Build a per-cell score map: one cell per scenario (rep 0), one judge. */
+function scoreMap(byScenario: Record<string, number>): Map<string, Record<string, JudgeScore>> {
+  const m = new Map<string, Record<string, JudgeScore>>()
+  for (const [sid, composite] of Object.entries(byScenario)) {
+    m.set(`${sid}:0`, { j: { composite, dimensions: {}, notes: '' } as JudgeScore })
+  }
+  return m
+}
+
+function ctx(
+  candidate: Record<string, number>,
+  baseline: Record<string, number>,
+  neutralized: Record<string, number>,
+): GateContext<unknown, Scenario> {
+  return {
+    candidateArtifacts: new Map(),
+    judgeScores: scoreMap(candidate),
+    baselineJudgeScores: scoreMap(baseline),
+    neutralizedJudgeScores: scoreMap(neutralized),
+    scenarios,
+    cost: { candidate: 0, baseline: 0 },
+    signal: new AbortController().signal,
+  }
+}
+
+describe('neutralizationGate', () => {
+  it('SHIPS when the lift collapses under neutralization (content is causal)', async () => {
+    const gate = neutralizationGate({ scenarios })
+    // candidate lifts +4 over baseline; blanked content recovers ~0 → causal.
+    const res = await gate.decide(
+      ctx(
+        { s1: 8, s2: 8, s3: 8 }, // candidate
+        { s1: 4, s2: 4, s3: 4 }, // baseline
+        { s1: 4, s2: 4, s3: 4 }, // neutralized == baseline
+      ),
+    )
+    expect(res.decision).toBe('ship')
+    const detail = res.contributingGates[0]?.detail as { decorativeFraction: number }
+    expect(detail.decorativeFraction).toBeCloseTo(0, 5)
+  })
+
+  it('HOLDS when the lift survives neutralization (decorative — footprint, not content)', async () => {
+    const gate = neutralizationGate({ scenarios })
+    // candidate +4; blanked content still recovers +3 (75% ≥ 50%) → decorative.
+    const res = await gate.decide(
+      ctx({ s1: 8, s2: 8, s3: 8 }, { s1: 4, s2: 4, s3: 4 }, { s1: 7, s2: 7, s3: 7 }),
+    )
+    expect(res.decision).toBe('hold')
+    expect(res.reasons[0]).toMatch(/DECORATIVE/)
+    const detail = res.contributingGates[0]?.detail as { decorativeFraction: number }
+    expect(detail.decorativeFraction).toBeCloseTo(0.75, 5)
+  })
+
+  it('HOLDS exactly at the threshold (equality is decorative)', async () => {
+    const gate = neutralizationGate({ scenarios, maxDecorativeFraction: 0.5 })
+    const res = await gate.decide(
+      ctx({ s1: 8, s2: 8, s3: 8 }, { s1: 4, s2: 4, s3: 4 }, { s1: 6, s2: 6, s3: 6 }),
+    )
+    // neutralized +2 / candidate +4 = 0.5 → not < 0.5 → hold.
+    expect(res.decision).toBe('hold')
+  })
+
+  it('HOLDS (fail-closed) when the candidate has no positive lift', async () => {
+    const gate = neutralizationGate({ scenarios })
+    const res = await gate.decide(
+      ctx({ s1: 4, s2: 4, s3: 4 }, { s1: 4, s2: 4, s3: 4 }, { s1: 4, s2: 4, s3: 4 }),
+    )
+    expect(res.decision).toBe('hold')
+    expect(res.reasons[0]).toMatch(/no positive lift/)
+  })
+
+  it('throws when the neutralized arm is missing (composed without loop wiring)', async () => {
+    const gate = neutralizationGate({ scenarios })
+    const bare = ctx({ s1: 8 }, { s1: 4 }, { s1: 4 })
+    bare.neutralizedJudgeScores = undefined
+    await expect(gate.decide(bare)).rejects.toThrow(/neutralizedJudgeScores is required/)
+  })
+})
+
+describe('neutralizeText', () => {
+  it('preserves whitespace/layout and blanks non-whitespace', () => {
+    const src = 'line one\n  indented two\n'
+    const out = neutralizeText(src)
+    expect(out).toBe('#### ###\n  ######## ###\n')
+    // same length + same whitespace positions, zero readable content
+    expect(out.length).toBe(src.length)
+    expect([...out].filter((c) => /\s/.test(c)).join('')).toBe(
+      [...src].filter((c) => /\s/.test(c)).join(''),
+    )
+    expect(out).not.toMatch(/[a-z]/i)
+  })
+
+  it('is byte-length-exact for ASCII', () => {
+    const src = 'the frontier author knows section 1202(b)(2)'
+    expect(Buffer.byteLength(neutralizeText(src))).toBe(Buffer.byteLength(src))
+  })
+
+  it('preserves character count (not byte count) for multibyte content', () => {
+    // Documented limitation: a 2-byte char blanks to a 1-byte filler, so byte
+    // length shrinks while CHARACTER count + layout are preserved. The tokenizer
+    // footprint tracks char/token structure, which is what the placebo needs.
+    const src = 'knows §1202'
+    expect(neutralizeText(src).length).toBe(src.length)
+    expect(neutralizeText(src)).toBe('##### #####')
+  })
+})

src/campaign/gates/neutralization-gate.ts

@@ -0,0 +1,134 @@
+/**
+ * @module
+ * Composable placebo / neutralization promotion gate.
+ *
+ * A held-out gate proves a candidate beat baseline. It CANNOT prove the lift came
+ * from the candidate's CONTENT rather than from the prompt/mount FOOTPRINT the
+ * content happened to add (more bytes, a longer prompt). This gate closes that
+ * hole: it compares the candidate's held-out lift against the lift of a
+ * FOOTPRINT-MATCHED neutralized variant (same layout + length, zero content, via
+ * `neutralizeText`). If the neutralized variant reproduces more than
+ * `maxDecorativeFraction` of the candidate's lift, the lift is decorative — it
+ * survives blanking the content — and the candidate is HELD regardless of how
+ * large or significant its raw lift is.
+ *
+ * Compose it AFTER the significance gate — significance says the lift is real,
+ * this says the lift is CAUSED BY THE CONTENT:
+ *   composeGate(heldOutGate({ ... }), neutralizationGate({ ... }))
+ *
+ * Requires `ctx.neutralizedJudgeScores`, populated by `runImprovementLoop` when it
+ * is given a `neutralize` function. A gate composed without that wiring fails
+ * loud rather than silently passing an unproven candidate.
+ */
+
+import type { Gate, GateContext, GateResult, Scenario } from '../types'
+import { pairHoldout } from './statistical-heldout'
+
+export interface NeutralizationGateOptions<TScenario extends Scenario = Scenario> {
+  scenarios: TScenario[]
+  /** Reject when the neutralized (content-blanked, footprint-matched) variant
+   *  reproduces at least this fraction of the candidate's held-out lift. Default
+   *  0.5 — if blanking the content keeps half the lift, the content is decorative.
+   *  Equality rejects: a neutralized lift == threshold·candidateLift is decorative. */
+  maxDecorativeFraction?: number
+}
+
+/** Mean of a numeric array; 0 for an empty array (callers guard n separately). */
+function mean(xs: number[]): number {
+  return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length
+}
+
+/** Paired mean held-out lift of `arm` over baseline, on the in-scope cells. */
+function pairedLift(
+  arm: Map<string, Record<string, { composite: number }>>,
+  baseline: Map<string, Record<string, { composite: number }>>,
+  scenarioIds: Set<string>,
+): { lift: number; n: number } {
+  const paired = pairHoldout(arm as never, baseline as never, scenarioIds, (s) => s.composite)
+  const deltas = paired.after.map((a, i) => a - (paired.before[i] ?? 0))
+  return { lift: mean(deltas), n: deltas.length }
+}
+
+/**
+ * Composable placebo gate: ships only when the candidate's held-out lift is NOT
+ * mostly reproduced by a footprint-matched neutralized variant.
+ */
+export function neutralizationGate<TArtifact, TScenario extends Scenario>(
+  options: NeutralizationGateOptions<TScenario>,
+): Gate<TArtifact, TScenario> {
+  const maxDecorativeFraction = options.maxDecorativeFraction ?? 0.5
+  return {
+    name: 'neutralizationGate',
+    async decide(ctx: GateContext<TArtifact, TScenario>): Promise<GateResult> {
+      if (!ctx.baselineJudgeScores) {
+        throw new Error(
+          'neutralizationGate: ctx.baselineJudgeScores is required — the placebo control measures lift OVER baseline.',
+        )
+      }
+      if (!ctx.neutralizedJudgeScores) {
+        throw new Error(
+          'neutralizationGate: ctx.neutralizedJudgeScores is required. It is populated by runImprovementLoop only when a `neutralize` function is supplied — composing this gate without that wiring would pass an unproven candidate.',
+        )
+      }
+      const scenarioIds = new Set(options.scenarios.map((s) => s.id))
+      const cand = pairedLift(
+        ctx.judgeScores as never,
+        ctx.baselineJudgeScores as never,
+        scenarioIds,
+      )
+      const neut = pairedLift(
+        ctx.neutralizedJudgeScores as never,
+        ctx.baselineJudgeScores as never,
+        scenarioIds,
+      )
+
+      // No positive candidate lift → nothing to attribute. Fail closed: the
+      // placebo control has no basis to clear the candidate (the significance
+      // gate is what judges lift magnitude; this one only judges its CAUSE).
+      if (cand.lift <= 0) {
+        return {
+          decision: 'hold',
+          reasons: [
+            `neutralization: candidate held-out lift ${cand.lift.toFixed(3)} ≤ 0 — no positive lift to attribute to content`,
+          ],
+          contributingGates: [
+            {
+              name: 'neutralizationGate',
+              passed: false,
+              detail: { candidateLift: cand.lift, neutralizedLift: neut.lift, n: cand.n },
+            },
+          ],
+          delta: cand.lift,
+        }
+      }
+
+      const decorativeFraction = neut.lift / cand.lift
+      const passed = decorativeFraction < maxDecorativeFraction
+      const pct = (decorativeFraction * 100).toFixed(0)
+      return {
+        decision: passed ? 'ship' : 'hold',
+        reasons: passed
+          ? [
+              `neutralization: content is causal — blanked variant reproduces ${pct}% of the lift (< ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate Δ ${cand.lift.toFixed(3)}, neutralized Δ ${neut.lift.toFixed(3)}`,
+            ]
+          : [
+              `neutralization: lift is DECORATIVE — blanking the content (footprint-matched) reproduces ${pct}% of the lift (≥ ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate Δ ${cand.lift.toFixed(3)}, neutralized Δ ${neut.lift.toFixed(3)}`,
+            ],
+        contributingGates: [
+          {
+            name: 'neutralizationGate',
+            passed,
+            detail: {
+              candidateLift: cand.lift,
+              neutralizedLift: neut.lift,
+              decorativeFraction,
+              maxDecorativeFraction,
+              n: cand.n,
+            },
+          },
+        ],
+        delta: cand.lift,
+      }
+    },
+  }
+}

src/campaign/index.ts

@@ -67,6 +67,10 @@ export {
   defaultProductionGate,
 } from './gates/default-production-gate'
 export { type HeldOutGateOptions, heldOutGate } from './gates/heldout-gate'
+export {
+  type NeutralizationGateOptions,
+  neutralizationGate,
+} from './gates/neutralization-gate'
 export {
   type PowerPreflight,
   type PowerPreflightOptions,
@@ -110,6 +114,7 @@ export {
   type FsLabeledScenarioStoreOptions,
   LabeledScenarioStoreError,
 } from './labeled-store/fs-adapter'
+export { neutralizeText } from './neutralize'
 // ── Presets (the documented public surface) ──────────────────────────
 export {
   type CompareProposersOptions,

src/campaign/neutralize.ts

@@ -0,0 +1,35 @@
+/**
+ * @module
+ * Footprint-matched neutralization — the placebo control for content-vs-footprint
+ * attribution in a promotion gate.
+ *
+ * A promoted surface can raise a held-out score two different ways:
+ *  1. its CONTENT is informative (the thing we want to promote), or
+ *  2. it merely added prompt/mount FOOTPRINT — more bytes, more lines, a longer
+ *     more authoritative-looking prompt — that the model spends attention on
+ *     regardless of what the bytes say.
+ *
+ * A held-out gate proves the candidate beat baseline; it cannot separate (1) from
+ * (2). `neutralizeText` produces a variant that keeps the input's layout and
+ * length while carrying ZERO information, so scoring it isolates the footprint
+ * contribution (2). Feed the neutralized variant's scores to `neutralizationGate`:
+ * any lift it still holds over baseline is decorative, and a candidate whose lift
+ * survives neutralization is rejected however large its raw lift.
+ */
+
+/** Filler for blanked content. A single ASCII byte, so a run of it preserves an
+ *  ASCII source's exact byte length; for multibyte sources it preserves CHARACTER
+ *  count and layout (what the tokenizer footprint tracks), not raw byte count. */
+const FILLER = '#'
+
+/**
+ * Blank every non-whitespace character to a 1-byte filler while preserving all
+ * whitespace. Line count, indentation, and word/line lengths are unchanged — so
+ * the neutralized variant has the same layout and (for ASCII) the same byte
+ * footprint as the input, but no readable content. Whitespace is preserved
+ * deliberately: collapsing it would change the token structure and stop the
+ * variant from being a true footprint match.
+ */
+export function neutralizeText(content: string): string {
+  return content.replace(/\S/g, FILLER)
+}

src/campaign/presets/run-improvement-loop.ts

@@ -58,6 +58,14 @@ export type RunImprovementLoopOptions<
   /** Optional render override — substrate writes a diff-shaped surface; pass
    *  a function to format the promoted surface differently. */
   renderPromotedDiff?: (winnerSurface: MutableSurface, baselineSurface: MutableSurface) => string
+  /** Placebo control. When supplied AND the winner differs from baseline, the
+   *  loop scores a THIRD holdout arm: the winner surface with its content
+   *  footprint-matched-blanked by this function (typically via `neutralizeText`).
+   *  Its scores are exposed to the gate as `ctx.neutralizedJudgeScores`, letting
+   *  a `neutralizationGate` reject a win whose lift survives blanking the content
+   *  (decorative — driven by footprint, not content). Costs one extra holdout
+   *  campaign; omit to skip. Return a byte/layout-matched blank of the winner. */
+  neutralize?: (winnerSurface: MutableSurface, baselineSurface: MutableSurface) => MutableSurface
 }
 
 export interface RunImprovementLoopResult<TArtifact, TScenario extends Scenario>
@@ -197,6 +205,32 @@ export async function runImprovementLoop<TScenario extends Scenario, TArtifact>(
     baselineJudgeScores.set(cell.cellId, cell.judgeScores)
   }
 
+  // ── (3a) placebo arm ───────────────────────────────────────────────
+  // When a `neutralize` fn is wired and the winner actually changed something,
+  // score a third holdout arm: the winner surface with its content
+  // footprint-matched-blanked. A `neutralizationGate` reads these scores to
+  // reject a win whose lift survives blanking the content (decorative — driven
+  // by the added footprint, not the content). Skipped for a no-op winner (there
+  // is no content to blank) and when no `neutralize` is supplied.
+  let neutralizedArtifacts: Map<string, TArtifact> | undefined
+  let neutralizedJudgeScores: ScoreMap | undefined
+  if (opts.neutralize && !winnerIsBaseline) {
+    const neutralizedSurface = opts.neutralize(optimization.winnerSurface, opts.baselineSurface)
+    const neutralizedOnHoldout = await runCampaign<TScenario, TArtifact>({
+      ...opts,
+      dispatchTimeoutMs,
+      scenarios: opts.holdoutScenarios,
+      dispatch: (scenario, ctx) => opts.dispatchWithSurface(neutralizedSurface, scenario, ctx),
+      runDir: `${opts.runDir}/holdout-neutralized`,
+    })
+    neutralizedArtifacts = new Map<string, TArtifact>()
+    neutralizedJudgeScores = new Map()
+    for (const cell of neutralizedOnHoldout.cells) {
+      neutralizedArtifacts.set(cell.cellId, cell.artifact)
+      neutralizedJudgeScores.set(cell.cellId, cell.judgeScores)
+    }
+  }
+
   // No-op guard: a winner identical to the baseline has nothing to promote, so
   // it never reaches the gate — otherwise the gate scores baseline-vs-itself,
   // sees model noise as a delta, and can "ship" an empty diff (the observed
@@ -217,6 +251,8 @@ export async function runImprovementLoop<TScenario extends Scenario, TArtifact>(
         baselineArtifacts,
         judgeScores,
         baselineJudgeScores,
+        neutralizedArtifacts,
+        neutralizedJudgeScores,
         scenarios: opts.holdoutScenarios,
         cost: {
           candidate: winnerOnHoldout.aggregates.totalCostUsd,