From fefb1b5399c11a674bf8820177760703be241988 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 8 Jul 2026 08:31:54 -0600 Subject: [PATCH] feat(contract): selfImprove forwards `neutralize` to the placebo arm (0.108.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.107.0 wired the footprint-matched placebo arm at runImprovementLoop but the public selfImprove facade did not forward it — the neutralizationGate was unreachable from the one-call entry point. Thread `neutralize` through SelfImproveOptions -> runImprovementLoop so selfImprove({ neutralize }) scores the placebo arm and hands ctx.neutralizedJudgeScores to the gate. Test drives selfImprove with a winning proposer + a neutralize fn and asserts the fn is called AND a capture-gate receives neutralizedJudgeScores (covers the facade passthrough + the loop's placebo-arm run). Additive optional field. --- CHANGELOG.md | 10 ++++ clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- package.json | 2 +- src/contract/self-improve.test.ts | 47 ++++++++++++++++++- src/contract/self-improve.ts | 10 ++++ 6 files changed, 69 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 562711d1..1bcef4d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.108.0] — 2026-07-08 — placebo control reaches the facades + +### Added + +- **`neutralize` passthrough on `selfImprove`.** 0.107.0 wired the footprint-matched placebo arm at `runImprovementLoop`; the public `selfImprove` facade did not forward it, so the placebo gate was unreachable from the one-call entry point. `selfImprove({ ..., neutralize })` now scores the third placebo arm and exposes `ctx.neutralizedJudgeScores` to the gate — compose `neutralizationGate` into `gate` to act on it. + +Additive (one optional field); no consumer bump required. + +--- + ## [0.107.0] — 2026-07-07 — footprint-matched placebo promotion gate ### Added diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 0748b10e..9b6d7a74 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.107.0" +version = "0.108.0" description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index e9cf6168..95cc4b7f 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -58,7 +58,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.107.0" + __version__ = "0.108.0" __all__ = [ "Client", diff --git a/package.json b/package.json index 93f4f9b1..521e3d95 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.107.0", + "version": "0.108.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/contract/self-improve.test.ts b/src/contract/self-improve.test.ts index d515d4fd..fedf26b3 100644 --- a/src/contract/self-improve.test.ts +++ b/src/contract/self-improve.test.ts @@ -8,7 +8,7 @@ */ import { describe, expect, it } from 'vitest' -import type { JudgeConfig } from '../campaign/types' +import type { Gate, JudgeConfig, SurfaceProposer } from '../campaign/types' import type { DispatchContext, Scenario } from './index' import { type SelfImproveProgressEvent, selfImprove } from './self-improve' @@ -61,3 +61,48 @@ describe('selfImprove — power analysis wiring', () => { expect(powerEvents).toHaveLength(1) }) }) + +describe('selfImprove — neutralize (placebo arm) passthrough', () => { + it('forwards `neutralize`: runs the placebo arm and hands its scores to the gate', async () => { + let neutralizeCalled = false + let gateSawNeutralized = false + + // Rewards only the surface containing "better", so the proposed candidate + // beats the baseline → the loop promotes a winner ≠ baseline → the placebo + // arm runs (it is skipped when winner == baseline). + const winJudge: JudgeConfig<{ text: string }, Scenario> = { + name: 'win-judge', + dimensions: [{ key: 'q', description: 'fixture quality' }], + score: ({ artifact }) => { + const good = artifact.text.includes('better') ? 1 : 0 + return { dimensions: { q: good }, composite: good, notes: '' } + }, + } + const proposer: SurfaceProposer = { kind: 'stub', propose: async () => ['better'] } + const captureGate: Gate<{ text: string }, Scenario> = { + name: 'capture', + async decide(ctx) { + gateSawNeutralized = (ctx.neutralizedJudgeScores?.size ?? 0) > 0 + return { decision: 'hold', reasons: [], contributingGates: [] } + }, + } + + await selfImprove({ + agent: stubAgent, + scenarios, + judge: winJudge, + baselineSurface: 'base', + proposer, + gate: captureGate, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.5 }, + neutralize: (winner) => { + neutralizeCalled = true + // footprint-matched-ish blank: same length, zero content + return '#'.repeat(String(winner).length) + }, + }) + + expect(neutralizeCalled).toBe(true) + expect(gateSawNeutralized).toBe(true) + }) +}) diff --git a/src/contract/self-improve.ts b/src/contract/self-improve.ts index db9ac0f4..2e5137c7 100644 --- a/src/contract/self-improve.ts +++ b/src/contract/self-improve.ts @@ -140,6 +140,15 @@ export interface SelfImproveOptions { * `deltaThreshold: 0.05` on the held-out split. */ gate?: Gate + /** Placebo control. When supplied AND the winner differs from baseline, the + * loop scores a THIRD held-out arm: the winner surface with its content + * footprint-matched-blanked by this fn (typically via `neutralizeText`). Its + * scores reach the gate as `ctx.neutralizedJudgeScores`, letting a + * `neutralizationGate` reject a win whose lift survives blanking the content + * (decorative — driven by footprint, not content). Costs one extra held-out + * campaign; omit to skip. Compose `neutralizationGate` into `gate` to act on it. */ + neutralize?: (winnerSurface: MutableSurface, baselineSurface: MutableSurface) => MutableSurface + /** LLM config consumed by the default `gepaProposer`. Ignored if you pass * your own `proposer`. */ llm?: SelfImproveLlm @@ -444,6 +453,7 @@ export async function selfImprove( reps: budget.reps, holdoutScenarios: holdout, gate, + neutralize: opts.neutralize, autoOnPromote: opts.autoOnPromote ?? 'none', ghOwner: opts.ghOwner, ghRepo: opts.ghRepo,