diff --git a/src/adaptive-driver.ts b/src/adaptive-driver.ts index 3d70e6e..3d3ba78 100644 --- a/src/adaptive-driver.ts +++ b/src/adaptive-driver.ts @@ -1,5 +1,5 @@ /** - * Adaptive verifier mode for `runTwoAgentResearchLoop`. + * Adaptive verifier mode for `runVerifiedResearchLoop`. * * The cost/quality A/B (`docs/results/cost-quality.md`) found the LLM relevance * verifier's cleanliness win is dominated by DE-DUPLICATION — which a diff --git a/src/claim-grounding.ts b/src/claim-grounding.ts index 244b7d1..1998aae 100644 --- a/src/claim-grounding.ts +++ b/src/claim-grounding.ts @@ -1,5 +1,5 @@ /** - * Claim-grounding mode for `runTwoAgentResearchLoop`. + * Claim-grounding mode for `runVerifiedResearchLoop`. * * The two-agent loop's existing verifier judges a source's on-topic RELEVANCE * (is this page about the goal?). On the topic sets we have measured, its @@ -252,7 +252,7 @@ export interface ClaimGroundingDriverOptions extends GroundClaimOptions { * composes a relevance verifier after grounding passes. * * The returned function matches `ResearchDriver['verifySource']`, so it drops - * straight into `runTwoAgentResearchLoop` as `{ verifySource: createClaimGroundingVerifier(...) }`. + * straight into `runVerifiedResearchLoop` as `{ verifySource: createClaimGroundingVerifier(...) }`. */ export function createClaimGroundingVerifier(options: ClaimGroundingDriverOptions = {}) { const onMissingClaim = options.onMissingClaim ?? 'reject' diff --git a/src/investment-thesis-task.ts b/src/investment-thesis-task.ts index 5b61b10..fe2fc57 100644 --- a/src/investment-thesis-task.ts +++ b/src/investment-thesis-task.ts @@ -2,7 +2,7 @@ * The INVESTMENT-THESIS research task. * * Given `{ company, ticker, cik, cutoff }`, drive the SAME two-agent research - * loop the ML deep-question A/B uses (`runTwoAgentResearchLoop` + the real web + * loop the ML deep-question A/B uses (`runVerifiedResearchLoop` + the real web * worker) to research the company AS OF the cutoff — web + SEC EDGAR, both public * — and produce an investment-thesis PAGE in the knowledge base: a judgment, the * drivers, and the risks, grounded in what it fetched. @@ -26,8 +26,8 @@ import { kbIndexToText } from './material-facts-metric' import { layoutFor } from './store' import { type ResearchDriver, - runTwoAgentResearchLoop, - type TwoAgentResearchLoopResult, + runVerifiedResearchLoop, + type VerifiedResearchLoopResult, } from './two-agent-research-loop' import { createWebResearchWorker, @@ -192,7 +192,7 @@ export interface ThesisRunOptions { } export interface ThesisRunResult { - loop: TwoAgentResearchLoopResult + loop: VerifiedResearchLoopResult /** The synthesized thesis text. */ thesis: string /** Path of the thesis page written into the KB. */ @@ -212,7 +212,7 @@ export async function runInvestmentThesisTask( const worker = createWebResearchWorker({ ...options.workerOptions, router: options.router }) const goal = `${input.company} (${input.ticker}) investment thesis as of ${input.cutoff}` - const loop = await runTwoAgentResearchLoop({ + const loop = await runVerifiedResearchLoop({ root: options.root, goal, worker, diff --git a/src/memory/adapter.ts b/src/memory/adapter.ts index 1a9017a..f29e462 100644 --- a/src/memory/adapter.ts +++ b/src/memory/adapter.ts @@ -1,3 +1,4 @@ +import { applySessionStickyRetrievalHoldout } from './holdout' import { memoryHitToSourceRecord } from './source-record' import type { AgentMemoryAdapter, @@ -12,11 +13,21 @@ export async function defaultGetMemoryContext( options: AgentMemorySearchOptions = {}, ): Promise { const hits = await adapter.search(query, options) + // The holdout applies between search (eligibility set) and render (delivered set), and only + // when explicitly configured: the unconfigured path must stay byte-identical, including + // hit-array identity, so default behavior cannot drift. + const delivered = options.holdout + ? applySessionStickyRetrievalHoldout(hits, options.holdout, { + query, + ...(options.scope !== undefined ? { scope: options.scope } : {}), + ...(options.scope?.tags?.taskId !== undefined ? { taskId: options.scope.tags.taskId } : {}), + }).delivered + : hits return { query, - hits, - sourceRecords: hits.map((hit) => memoryHitToSourceRecord(hit, { scope: options.scope })), - text: renderMemoryContext(hits), + hits: delivered, + sourceRecords: delivered.map((hit) => memoryHitToSourceRecord(hit, { scope: options.scope })), + text: renderMemoryContext(delivered), } } diff --git a/src/memory/holdout.ts b/src/memory/holdout.ts new file mode 100644 index 0000000..c84a0c8 --- /dev/null +++ b/src/memory/holdout.ts @@ -0,0 +1,237 @@ +import { randomUUID } from 'node:crypto' +import { sha256 } from '../ids' +import type { AgentMemoryHit, AgentMemoryScope } from './types' + +// Randomized retrieval holdout (epsilon-dropout) for per-item treatment-effect logging. +// Default-off: nothing in this module runs unless a consumer passes a RetrievalHoldoutConfig. +// The library never does I/O here; persistence is the consumer's job via onEvent. +// Design + estimator + sample-size analysis: research repo, +// projects/probabilistic-agent-optimization/notes/2026-07-03-DRAFT-o3-holdout-design.md (O3 / EXP-007). + +export interface RetrievalHoldoutConfig { + /** Per-session probability that one eligible watchlist item is suppressed. 0 logs the full schema without ever dropping. */ + epsilon: number + /** Item ids eligible for suppression. Empty or absent means no item can ever be dropped. */ + watchlist?: string[] + /** Ties every event to the exact epsilon/watchlist in force, for audit and replay. */ + configVersion?: string + /** Copied onto every event so multi-adapter logs stay attributable. */ + adapterId?: string + /** Corpus/store version stamp; an edited item under the same id is a different treatment. */ + corpusVersion?: string + /** + * Uniform-[0,1) generator keyed by a string. Defaults to a sha256-derived deterministic + * generator so every assignment is replayable from the logged keys alone (design rule D5). + */ + rng?: (key: string) => number + /** Receives one event per retrieval call, INCLUDING no-drop calls: control-arm membership is half the data. */ + onEvent: (event: RetrievalHoldoutEvent) => void +} + +export interface RetrievalHoldoutEligibleItem { + id: string + /** 1-based position in the post-filter hit list. */ + rank: number + score?: number + kind: string + /** sha256(hit.text) prefix; effects are estimated per (id, contentHash) pair. */ + contentHash: string +} + +export interface RetrievalHoldoutEvent { + v: 1 + eventId: string + ts: string + adapterId?: string + sessionId?: string + taskId?: string + /** 1-based call counter within the session; 0 when the call is outside session randomization. */ + callIndex: number + /** sha256(sessionId) prefix — the seed key for the replayable assignment draws. */ + rngKey?: string + queryHash?: string + scope?: AgentMemoryScope + config: { epsilon: number; watchlist: string[]; configVersion?: string } + /** False when no sessionId is available, so the fraction-under-experiment denominator stays honest. */ + holdoutEligible: boolean + /** The full post-filter eligibility set E, logged on every call (control arm + interference probes). */ + eligible: RetrievalHoldoutEligibleItem[] + /** Ids in watchlist ∩ E, in eligibility order. */ + watchlistEligible: string[] + sessionHoldout: boolean + /** The session's sticky drop target once drawn; distinguishes "target absent from E" from "not yet drawn". */ + sessionTargetId: string | null + /** The item suppressed in THIS call, or null. */ + droppedId: string | null + /** 1/|watchlist ∩ E| recorded at draw time; the exact inverse-propensity weight input. */ + pickPropensity: number | null + /** epsilon * pickPropensity, recorded at draw time so analysis never re-derives assignment probabilities. */ + dropPropensity: number | null + deliveredIds: string[] + corpusVersion?: string +} + +export interface RetrievalHoldoutSessionState { + sessionId: string + /** Calls observed so far in this session. */ + callCount: number + sessionHoldout: boolean + /** Sticky drop target; drawn once at the first call whose eligibility set intersects the watchlist. */ + targetId: string | null + pickPropensity: number | null +} + +export interface RetrievalHoldoutCallContext { + sessionId?: string + taskId?: string + /** Raw query; only its sha256 prefix is logged. */ + query?: string + scope?: AgentMemoryScope + /** State returned by the previous call of this session; threading it is what makes suppression sticky. */ + session?: RetrievalHoldoutSessionState +} + +export interface RetrievalHoldoutResult { + delivered: AgentMemoryHit[] + event: RetrievalHoldoutEvent + session?: RetrievalHoldoutSessionState +} + +/** Deterministic uniform [0,1): 13 hex chars = 52 bits, exactly representable in a double. */ +export function deterministicRng(key: string): number { + return Number.parseInt(sha256(key).slice(0, 13), 16) / 16 ** 13 +} + +/** + * Pure per-call holdout: takes post-filter hits, returns delivered hits plus the log event + * and the next session state. Suppression only removes items (no backfill), and it happens + * before rendering so a drop session's context is byte-identical to a natural smaller retrieval. + */ +export function applyRetrievalHoldout( + hits: AgentMemoryHit[], + config: RetrievalHoldoutConfig, + ctx: RetrievalHoldoutCallContext = {}, +): RetrievalHoldoutResult { + const rng = config.rng ?? deterministicRng + const watchlist = config.watchlist ?? [] + const sessionId = ctx.sessionId ?? ctx.scope?.sessionId + const holdoutEligible = typeof sessionId === 'string' && sessionId.length > 0 + + let session: RetrievalHoldoutSessionState | undefined + if (holdoutEligible) { + const prior = ctx.session?.sessionId === sessionId ? ctx.session : undefined + session = prior ?? { + sessionId, + callCount: 0, + // The epsilon coin depends only on sessionId, so assignment is independent of task + // features by construction and exactly replayable (design rules D1 + D5). + sessionHoldout: rng(`${sessionId}#holdout`) < config.epsilon, + targetId: null, + pickPropensity: null, + } + session = { ...session, callCount: session.callCount + 1 } + } + + const watchlistSet = new Set(watchlist) + const watchlistEligible = hits.filter((hit) => watchlistSet.has(hit.id)).map((hit) => hit.id) + + if (session?.sessionHoldout && session.targetId === null && watchlistEligible.length > 0) { + // Draw once, uniformly over watchlist ∩ E at the first intersecting call, then stay sticky. + // Candidates are sorted by id so the draw does not depend on retrieval ordering. + const candidates = [...watchlistEligible].sort() + const index = Math.min( + Math.floor(rng(`${session.sessionId}#pick`) * candidates.length), + candidates.length - 1, + ) + const picked = candidates[index] + if (picked !== undefined) { + session = { ...session, targetId: picked, pickPropensity: 1 / candidates.length } + } + } + + const targetId = session?.sessionHoldout ? (session.targetId ?? null) : null + const droppedId = targetId !== null && hits.some((hit) => hit.id === targetId) ? targetId : null + const delivered = droppedId === null ? hits : hits.filter((hit) => hit.id !== droppedId) + + const pickPropensity = session?.sessionHoldout ? (session.pickPropensity ?? null) : null + const event: RetrievalHoldoutEvent = { + v: 1, + eventId: randomUUID(), + ts: new Date().toISOString(), + ...(config.adapterId !== undefined ? { adapterId: config.adapterId } : {}), + ...(sessionId !== undefined ? { sessionId } : {}), + ...(ctx.taskId !== undefined ? { taskId: ctx.taskId } : {}), + callIndex: session?.callCount ?? 0, + ...(sessionId !== undefined ? { rngKey: sha256(sessionId).slice(0, 16) } : {}), + ...(ctx.query !== undefined ? { queryHash: sha256(ctx.query).slice(0, 16) } : {}), + ...(ctx.scope !== undefined ? { scope: ctx.scope } : {}), + config: { + epsilon: config.epsilon, + watchlist: [...watchlist], + ...(config.configVersion !== undefined ? { configVersion: config.configVersion } : {}), + }, + holdoutEligible, + eligible: hits.map((hit, index) => ({ + id: hit.id, + rank: index + 1, + ...(typeof (hit.normalizedScore ?? hit.score) === 'number' + ? { score: hit.normalizedScore ?? hit.score } + : {}), + kind: hit.kind, + contentHash: sha256(hit.text).slice(0, 16), + })), + watchlistEligible, + sessionHoldout: session?.sessionHoldout ?? false, + sessionTargetId: targetId, + droppedId, + pickPropensity, + dropPropensity: pickPropensity !== null ? config.epsilon * pickPropensity : null, + deliveredIds: delivered.map((hit) => hit.id), + ...(config.corpusVersion !== undefined ? { corpusVersion: config.corpusVersion } : {}), + } + config.onEvent(event) + + return droppedId === null + ? { delivered: hits, event, ...(session !== undefined ? { session } : {}) } + : { delivered, event, ...(session !== undefined ? { session } : {}) } +} + +// Session states are tiny (5 scalar fields); the cap only guards unbounded long-lived processes. +// Evicting a live session degrades detectably, not silently: a re-drawn target that differs +// shows up in the log as a mixed-exposure session, which analysis excludes and counts. +const MAX_TRACKED_SESSIONS = 10_000 +const sessionRegistry = new WeakMap< + RetrievalHoldoutConfig, + Map +>() + +/** + * Convenience wrapper that threads session state internally, keyed by config object identity. + * Reuse ONE config object across all calls of a session (normally one per harness) or + * stickiness falls back to the deterministic per-session draws alone. + */ +export function applySessionStickyRetrievalHoldout( + hits: AgentMemoryHit[], + config: RetrievalHoldoutConfig, + ctx: Omit = {}, +): RetrievalHoldoutResult { + const sessionId = ctx.sessionId ?? ctx.scope?.sessionId + let sessions = sessionRegistry.get(config) + if (!sessions) { + sessions = new Map() + sessionRegistry.set(config, sessions) + } + const prior = sessionId !== undefined ? sessions.get(sessionId) : undefined + const result = applyRetrievalHoldout(hits, config, { + ...ctx, + ...(prior ? { session: prior } : {}), + }) + if (sessionId !== undefined && result.session) { + if (!sessions.has(sessionId) && sessions.size >= MAX_TRACKED_SESSIONS) { + const oldest = sessions.keys().next().value + if (oldest !== undefined) sessions.delete(oldest) + } + sessions.set(sessionId, result.session) + } + return result +} diff --git a/src/memory/index.ts b/src/memory/index.ts index 8d7569e..da6fffb 100644 --- a/src/memory/index.ts +++ b/src/memory/index.ts @@ -1,4 +1,5 @@ export * from './adapter' +export * from './holdout' export * from './neo4j' export * from './schemas' export * from './source-record' diff --git a/src/memory/types.ts b/src/memory/types.ts index 138dda1..a538d1f 100644 --- a/src/memory/types.ts +++ b/src/memory/types.ts @@ -1,4 +1,5 @@ import type { SourceRecord } from '../types' +import type { RetrievalHoldoutConfig } from './holdout' export type AgentMemoryKind = | 'message' @@ -45,6 +46,11 @@ export interface AgentMemorySearchOptions { minScore?: number kinds?: AgentMemoryKind[] metadata?: Record + /** + * Opt-in randomized retrieval holdout (epsilon-dropout) for treatment-effect logging. + * Absent by default; when absent, retrieval behavior is unchanged. See ./holdout. + */ + holdout?: RetrievalHoldoutConfig } export interface AgentMemoryWriteInput { diff --git a/src/research-driving-driver.ts b/src/research-driving-driver.ts index 2b9d2a6..3e820c0 100644 --- a/src/research-driving-driver.ts +++ b/src/research-driving-driver.ts @@ -1,5 +1,5 @@ /** - * Research-DRIVING driver for `runTwoAgentResearchLoop`. + * Research-DRIVING driver for `runVerifiedResearchLoop`. * * The shipped drivers all FILTER the worker's sources: * - `createVerifyingResearchDriver` judges on-topic relevance, @@ -37,7 +37,7 @@ * claim is NOT done; a KB whose handful of claims are each corroborated or * contested IS. * - * It reuses `runTwoAgentResearchLoop` (it is a plain `ResearchDriver`), the web + * It reuses `runVerifiedResearchLoop` (it is a plain `ResearchDriver`), the web * worker, `sha256` (claim identity), `canonicalizeUrl` (independent-source * identity), and the `RouterClient` chat surface; it reinvents none of them. */ @@ -149,7 +149,7 @@ export interface ResearchDrivingSteer { /** * The research-driving driver. It is a `ResearchDriver` (drops straight into - * `runTwoAgentResearchLoop`) PLUS a completion oracle and live state, mirroring + * `runVerifiedResearchLoop`) PLUS a completion oracle and live state, mirroring * how `createAdaptiveResearchDriver` exposes `stats()`. */ export interface ResearchDrivingDriver extends ResearchDriver { diff --git a/src/research-supervisor.ts b/src/research-supervisor.ts index 26e2223..c0a421d 100644 --- a/src/research-supervisor.ts +++ b/src/research-supervisor.ts @@ -106,7 +106,7 @@ export function knowledgeReadinessDeliverable( * completion oracle that decides when the KB is ready. * * Needs creds (a real supervisor router brain + a worker backend), so it is the - * LIVE path. The offline two-agent loop is `runTwoAgentResearchLoop`. + * LIVE path. The offline two-agent loop is `runVerifiedResearchLoop`. */ export async function runResearchSupervisor( options: ResearchSupervisorOptions, diff --git a/src/two-agent-research-loop.ts b/src/two-agent-research-loop.ts index 239ddf0..331d6e6 100644 --- a/src/two-agent-research-loop.ts +++ b/src/two-agent-research-loop.ts @@ -134,7 +134,7 @@ export type ResearchWorker = ( ctx: WorkerResearchContext, ) => Promise | ResearchContribution -export interface TwoAgentResearchLoopOptions { +export interface VerifiedResearchLoopOptions { root: string goal: string worker: ResearchWorker @@ -154,10 +154,10 @@ export interface TwoAgentResearchLoopOptions { readiness?: Omit sourceOptions?: Pick signal?: AbortSignal - onRound?: (round: TwoAgentResearchRound) => Promise | void + onRound?: (round: VerifiedResearchRound) => Promise | void } -export interface TwoAgentResearchRound { +export interface VerifiedResearchRound { round: number /** Gaps reported at the START of the round (what the worker targeted). */ gaps: KnowledgeGap[] @@ -176,14 +176,14 @@ export interface TwoAgentResearchRound { notes: { worker?: string; driver?: string } } -export interface TwoAgentResearchLoopResult { +export interface VerifiedResearchLoopResult { root: string goal: string rounds: number ready: boolean index: KnowledgeIndex readiness?: EvalKnowledgeBundleBuildResult - steps: TwoAgentResearchRound[] + steps: VerifiedResearchRound[] } /** @@ -208,12 +208,12 @@ export interface TwoAgentResearchLoopResult { * `applyKnowledgeWriteBlocks`, `buildEvalKnowledgeBundle` (the readiness gate), * and `searchKnowledge` — and reinvents none of them. */ -export async function runTwoAgentResearchLoop( - options: TwoAgentResearchLoopOptions, -): Promise { +export async function runVerifiedResearchLoop( + options: VerifiedResearchLoopOptions, +): Promise { const maxRounds = Math.max(1, options.maxRounds ?? 3) await initKnowledgeBase(options.root) - const steps: TwoAgentResearchRound[] = [] + const steps: VerifiedResearchRound[] = [] let index = await buildKnowledgeIndex(options.root) let readiness = readinessFor(options, index) let ready = isReady(readiness?.report) @@ -305,7 +305,7 @@ export async function runTwoAgentResearchLoop( const remainingGaps = gapsFromReadiness(readiness) steer = ready ? undefined : foldGaps(options.driver, remainingGaps) - const step: TwoAgentResearchRound = { + const step: VerifiedResearchRound = { round, gaps, acceptedWorkerSources, @@ -395,7 +395,7 @@ function isDuplicate( } async function registerSources( - options: TwoAgentResearchLoopOptions, + options: VerifiedResearchLoopOptions, sources: ResearchSourceProposal[], ): Promise { const records: SourceRecord[] = [] @@ -428,7 +428,7 @@ async function applyPages( function requireReadiness( readiness: EvalKnowledgeBundleBuildResult | undefined, - options: TwoAgentResearchLoopOptions, + options: VerifiedResearchLoopOptions, ): EvalKnowledgeBundleBuildResult { if (readiness) return readiness // The worker/driver contexts type `readiness` as required for ergonomics; when @@ -474,3 +474,17 @@ export function sourceMatchesGaps( } return hits } + +// ── Deprecated aliases ─────────────────────────────────────────────────────────── +// The old "TwoAgent" names described the MECHANISM (a proposer worker + a verifier driver) +// instead of the VALUE (research whose sources are verified before admission). Kept as +// aliases so existing importers do not break; prefer the `Verified*` names. + +/** @deprecated Renamed to {@link runVerifiedResearchLoop}. */ +export const runTwoAgentResearchLoop = runVerifiedResearchLoop +/** @deprecated Renamed to {@link VerifiedResearchLoopOptions}. */ +export type TwoAgentResearchLoopOptions = VerifiedResearchLoopOptions +/** @deprecated Renamed to {@link VerifiedResearchLoopResult}. */ +export type TwoAgentResearchLoopResult = VerifiedResearchLoopResult +/** @deprecated Renamed to {@link VerifiedResearchRound}. */ +export type TwoAgentResearchRound = VerifiedResearchRound diff --git a/src/web-research-worker.ts b/src/web-research-worker.ts index 1b6ddb2..1f359ca 100644 --- a/src/web-research-worker.ts +++ b/src/web-research-worker.ts @@ -1,5 +1,5 @@ /** - * Real web-research worker + verifying driver for `runTwoAgentResearchLoop`. + * Real web-research worker + verifying driver for `runVerifiedResearchLoop`. * * This is the GENERAL, any-topic implementation behind the two-agent research * loop's live arm. Given the open knowledge gaps the readiness gate surfaces, diff --git a/tests/memory-holdout.test.ts b/tests/memory-holdout.test.ts new file mode 100644 index 0000000..a33f302 --- /dev/null +++ b/tests/memory-holdout.test.ts @@ -0,0 +1,266 @@ +import { describe, expect, it } from 'vitest' +import { sha256 } from '../src/ids' +import type { + AgentMemoryHit, + RetrievalHoldoutConfig, + RetrievalHoldoutEvent, +} from '../src/memory/index' +import { + applyRetrievalHoldout, + applySessionStickyRetrievalHoldout, + defaultGetMemoryContext, + deterministicRng, + renderMemoryContext, +} from '../src/memory/index' + +function hit(id: string, text: string, score: number): AgentMemoryHit { + return { id, uri: `memory://test/${id}`, kind: 'fact', text, normalizedScore: score } +} + +function collectingConfig( + overrides: Partial & Pick, +): { config: RetrievalHoldoutConfig; events: RetrievalHoldoutEvent[] } { + const events: RetrievalHoldoutEvent[] = [] + return { config: { onEvent: (event) => events.push(event), ...overrides }, events } +} + +describe('retrieval holdout: default-off is a byte-identical no-op', () => { + it('leaves defaultGetMemoryContext untouched when no holdout is configured', async () => { + const hits = [hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8)] + const context = await defaultGetMemoryContext({ search: async () => hits }, 'q', { + scope: { sessionId: 's-1' }, + }) + // Same array instance: the unconfigured code path never copies or filters. + expect(context.hits).toBe(hits) + // Frozen from the pre-holdout renderer output; a byte-level drift here fails the test. + expect(context.text).toBe('[1] fact:m1 score=0.900\nalpha\n\n[2] fact:m7 score=0.800\nbeta') + expect(context.sourceRecords).toHaveLength(2) + }) +}) + +describe('retrieval holdout: enabled behavior and event schema', () => { + // Forces holdout (epsilon 1) and picks index floor(0.6 * |candidates|) deterministically. + const forcedRng = (key: string) => (key.endsWith('#pick') ? 0.6 : 0) + + it('drops exactly one watchlist item and emits the registered event schema', () => { + const { config, events } = collectingConfig({ + epsilon: 1, + watchlist: ['m1', 'm7'], + configVersion: '2026-07-04a', + adapterId: 'test-adapter', + corpusVersion: 'kb-test-1', + rng: forcedRng, + }) + const hits = [hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8), hit('m3', 'gamma', 0.7)] + const { delivered, event } = applyRetrievalHoldout(hits, config, { + sessionId: 's-1', + taskId: 'task-17', + query: 'the query', + scope: { sessionId: 's-1', namespace: 'ns' }, + }) + + // sorted(watchlist ∩ E) = ['m1', 'm7']; floor(0.6 * 2) = 1 → 'm7'. + expect(delivered.map((h) => h.id)).toEqual(['m1', 'm3']) + expect(events).toHaveLength(1) + expect(Object.keys(event).sort()).toEqual( + [ + 'adapterId', + 'callIndex', + 'config', + 'corpusVersion', + 'deliveredIds', + 'droppedId', + 'dropPropensity', + 'eligible', + 'eventId', + 'holdoutEligible', + 'pickPropensity', + 'queryHash', + 'rngKey', + 'scope', + 'sessionHoldout', + 'sessionId', + 'sessionTargetId', + 'taskId', + 'ts', + 'v', + 'watchlistEligible', + ].sort(), + ) + expect(event.v).toBe(1) + expect(event.sessionId).toBe('s-1') + expect(event.taskId).toBe('task-17') + expect(event.callIndex).toBe(1) + expect(event.holdoutEligible).toBe(true) + expect(event.sessionHoldout).toBe(true) + expect(event.sessionTargetId).toBe('m7') + expect(event.droppedId).toBe('m7') + // Draw rule: pick uniform over |watchlist ∩ E| = 2, drop = epsilon * pick. + expect(event.pickPropensity).toBe(0.5) + expect(event.dropPropensity).toBe(0.5) + expect(event.deliveredIds).toEqual(['m1', 'm3']) + expect(event.watchlistEligible).toEqual(['m1', 'm7']) + expect(event.eligible).toEqual([ + { id: 'm1', rank: 1, score: 0.9, kind: 'fact', contentHash: sha256('alpha').slice(0, 16) }, + { id: 'm7', rank: 2, score: 0.8, kind: 'fact', contentHash: sha256('beta').slice(0, 16) }, + { id: 'm3', rank: 3, score: 0.7, kind: 'fact', contentHash: sha256('gamma').slice(0, 16) }, + ]) + expect(event.rngKey).toBe(sha256('s-1').slice(0, 16)) + expect(event.queryHash).toBe(sha256('the query').slice(0, 16)) + expect(event.config).toEqual({ + epsilon: 1, + watchlist: ['m1', 'm7'], + configVersion: '2026-07-04a', + }) + expect(event.adapterId).toBe('test-adapter') + expect(event.corpusVersion).toBe('kb-test-1') + // The manipulation is invisible: the rendered context renumbers with no gap (design rule D6). + expect(renderMemoryContext(delivered)).toBe( + '[1] fact:m1 score=0.900\nalpha\n\n[2] fact:m3 score=0.700\ngamma', + ) + }) + + it('suppresses the same target on every call of the session (sticky), via the adapter path', async () => { + const { config, events } = collectingConfig({ + epsilon: 1, + watchlist: ['m1', 'm7'], + rng: forcedRng, + }) + const options = { scope: { sessionId: 's-2', tags: { taskId: 'task-9' } }, holdout: config } + const call1 = await defaultGetMemoryContext( + { search: async () => [hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8)] }, + 'q1', + options, + ) + const call2 = await defaultGetMemoryContext( + { search: async () => [hit('m7', 'beta', 0.8), hit('m3', 'gamma', 0.7)] }, + 'q2', + options, + ) + // Third call's eligibility set no longer contains the target: nothing is dropped, + // but the sticky target is still reported for mixed-exposure audits. + const call3 = await defaultGetMemoryContext( + { search: async () => [hit('m3', 'gamma', 0.7)] }, + 'q3', + options, + ) + + expect(call1.hits.map((h) => h.id)).toEqual(['m1']) + expect(call2.hits.map((h) => h.id)).toEqual(['m3']) + expect(call3.hits.map((h) => h.id)).toEqual(['m3']) + expect(events.map((e) => e.callIndex)).toEqual([1, 2, 3]) + expect(events.map((e) => e.droppedId)).toEqual(['m7', 'm7', null]) + expect(events.map((e) => e.sessionTargetId)).toEqual(['m7', 'm7', 'm7']) + expect(events.map((e) => e.taskId)).toEqual(['task-9', 'task-9', 'task-9']) + expect(events.every((e) => e.sessionHoldout)).toBe(true) + }) + + it('emits control-arm events on no-drop calls, including at epsilon 0', () => { + const { config, events } = collectingConfig({ + epsilon: 0, + watchlist: ['m1'], + }) + const hits = [hit('m1', 'alpha', 0.9), hit('m2', 'beta', 0.8)] + const { delivered, event } = applyRetrievalHoldout(hits, config, { sessionId: 's-3' }) + + expect(delivered).toBe(hits) + expect(events).toHaveLength(1) + expect(event.holdoutEligible).toBe(true) + expect(event.sessionHoldout).toBe(false) + expect(event.droppedId).toBeNull() + expect(event.sessionTargetId).toBeNull() + expect(event.pickPropensity).toBeNull() + expect(event.dropPropensity).toBeNull() + // The full eligibility set is still logged: control-arm membership is half the data. + expect(event.eligible.map((e) => e.id)).toEqual(['m1', 'm2']) + expect(event.deliveredIds).toEqual(['m1', 'm2']) + }) + + it('records pickPropensity 1 and dropPropensity epsilon when one watchlist item is eligible', () => { + const { config } = collectingConfig({ + epsilon: 0.2, + watchlist: ['m7', 'm-absent'], + rng: () => 0, + }) + const { event } = applyRetrievalHoldout( + [hit('m7', 'beta', 0.8), hit('m3', 'gamma', 0.7)], + config, + { + sessionId: 's-4', + }, + ) + expect(event.droppedId).toBe('m7') + expect(event.pickPropensity).toBe(1) + expect(event.dropPropensity).toBeCloseTo(0.2, 12) + }) + + it('marks calls without a sessionId as holdout-ineligible and never drops', () => { + const { config, events } = collectingConfig({ epsilon: 1, watchlist: ['m1'], rng: () => 0 }) + const hits = [hit('m1', 'alpha', 0.9)] + const { delivered, event } = applyRetrievalHoldout(hits, config, {}) + + expect(delivered).toBe(hits) + expect(events).toHaveLength(1) + expect(event.holdoutEligible).toBe(false) + expect(event.sessionHoldout).toBe(false) + expect(event.droppedId).toBeNull() + expect(event.callIndex).toBe(0) + }) + + it('threads explicit session state through the pure function identically to the sticky wrapper', () => { + const { config: pureConfig } = collectingConfig({ + epsilon: 1, + watchlist: ['m1', 'm7'], + rng: forcedRng, + }) + const { config: stickyConfig } = collectingConfig({ + epsilon: 1, + watchlist: ['m1', 'm7'], + rng: forcedRng, + }) + const call1Hits = [hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8)] + const call2Hits = [hit('m7', 'beta', 0.8), hit('m3', 'gamma', 0.7)] + + const pure1 = applyRetrievalHoldout(call1Hits, pureConfig, { sessionId: 's-5' }) + const pure2 = applyRetrievalHoldout(call2Hits, pureConfig, { + sessionId: 's-5', + session: pure1.session, + }) + const sticky1 = applySessionStickyRetrievalHoldout(call1Hits, stickyConfig, { + sessionId: 's-5', + }) + const sticky2 = applySessionStickyRetrievalHoldout(call2Hits, stickyConfig, { + sessionId: 's-5', + }) + + expect(pure1.event.droppedId).toBe(sticky1.event.droppedId) + expect(pure2.event.droppedId).toBe(sticky2.event.droppedId) + expect(pure2.event.droppedId).toBe('m7') + expect(pure2.event.callIndex).toBe(2) + }) +}) + +describe('retrieval holdout: empirical calibration of the default rng', () => { + it('drops at a rate matching epsilon over 10^4 simulated sessions', () => { + const epsilon = 0.2 + const { config, events } = collectingConfig({ epsilon, watchlist: ['m1', 'm7'] }) + const n = 10_000 + for (let i = 0; i < n; i += 1) { + applyRetrievalHoldout([hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8)], config, { + sessionId: `session-${i}`, + }) + } + const drops = events.filter((e) => e.droppedId !== null) + const dropRate = drops.length / n + // Binomial sd at n=1e4, p=0.2 is 0.004; ±0.015 is ~3.75 sd, and the default rng is + // deterministic so this either always passes or flags a real bias. + expect(dropRate).toBeGreaterThan(epsilon - 0.015) + expect(dropRate).toBeLessThan(epsilon + 0.015) + // The uniform pick over two sorted candidates should split drops roughly evenly. + const m1Share = drops.filter((e) => e.droppedId === 'm1').length / drops.length + expect(m1Share).toBeGreaterThan(0.4) + expect(m1Share).toBeLessThan(0.6) + // Replayability (design rule D5): the same sessionId reproduces the same decision. + expect(deterministicRng('session-0#holdout')).toBe(deterministicRng('session-0#holdout')) + }) +})