diff --git a/src/memory/holdout.ts b/src/memory/holdout.ts index c84a0c8..f142e8b 100644 --- a/src/memory/holdout.ts +++ b/src/memory/holdout.ts @@ -1,4 +1,6 @@ import { randomUUID } from 'node:crypto' +import { mulberry32 } from '@tangle-network/agent-eval' +import type { OffPolicyTrajectory } from '@tangle-network/agent-eval/rl' import { sha256 } from '../ids' import type { AgentMemoryHit, AgentMemoryScope } from './types' @@ -19,6 +21,19 @@ export interface RetrievalHoldoutConfig { adapterId?: string /** Corpus/store version stamp; an edited item under the same id is a different treatment. */ corpusVersion?: string + /** + * Emit plaintext sessionId and scope on events. Default false: events carry only + * sessionIdHash/scopeHash, so PII-bearing identifiers (tenantId/userId/tags) never reach a + * consumer-controlled sink unless the consumer explicitly owns that decision. Note that + * replaying assignment draws from logs alone needs the plaintext sessionId, so + * privacy-default logs require the consumer's own sessionId mapping for replay audits. + */ + includePlaintextIdentifiers?: boolean + /** + * Cap on tracked sessions per experiment config in the sticky wrapper's registry. + * Exists so tests can exercise eviction; production should keep the default (10,000). + */ + maxTrackedSessions?: number /** * Uniform-[0,1) generator keyed by a string. Defaults to a sha256-derived deterministic * generator so every assignment is replayable from the logged keys alone (design rule D5). @@ -43,17 +58,35 @@ export interface RetrievalHoldoutEvent { eventId: string ts: string adapterId?: string + /** Plaintext session id — emitted ONLY when config.includePlaintextIdentifiers is true. */ sessionId?: string + /** Consumer-supplied experiment/outcome join id (scope.tags.taskId); deliberately plaintext. */ taskId?: string /** 1-based call counter within the session; 0 when the call is outside session randomization. */ callIndex: number - /** sha256(sessionId) prefix — the seed key for the replayable assignment draws. */ - rngKey?: string + /** + * sha256(sessionId) prefix — the default privacy-preserving session join key AND the seed-key + * reference for the assignment draws (previously named rngKey; identical derivation, deduped). + */ + sessionIdHash?: string queryHash?: string + /** Verbatim scope — emitted ONLY when config.includePlaintextIdentifiers is true (PII risk). */ scope?: AgentMemoryScope + /** sha256 prefix of the canonical-JSON scope (keys sorted, undefined stripped). */ + scopeHash?: string config: { epsilon: number; watchlist: string[]; configVersion?: string } - /** False when no sessionId is available, so the fraction-under-experiment denominator stays honest. */ + /** + * Value-hash of the experiment-defining knobs, sha256({epsilon, sorted watchlist}) prefix. + * The estimator groups events by it; the sticky-session registry is keyed by it. + */ + configHash: string + /** + * False when no sessionId is available or the adapter answered without retrieval + * (see bypassReason), so the fraction-under-experiment denominator stays honest. + */ holdoutEligible: boolean + /** Present only on adapter paths that bypassed retrieval, where no suppression could apply. */ + bypassReason?: RetrievalHoldoutBypassReason /** The full post-filter eligibility set E, logged on every call (control arm + interference probes). */ eligible: RetrievalHoldoutEligibleItem[] /** Ids in watchlist ∩ E, in eligibility order. */ @@ -97,9 +130,79 @@ export interface RetrievalHoldoutResult { session?: RetrievalHoldoutSessionState } -/** Deterministic uniform [0,1): 13 hex chars = 52 bits, exactly representable in a double. */ +/** Adapter context paths that answer without retrieval, so no holdout draw can happen. */ +export type RetrievalHoldoutBypassReason = 'short-term-context' | 'raw-string-context' + +/** + * Deterministic uniform [0,1) for assignment draws. The sha256 key derivation is ours — it makes + * every draw replayable from the logged keys alone (design rule D5) — while the generator core is + * the substrate's `mulberry32` (@tangle-network/agent-eval statistics vocabulary), seeded with the + * top 32 bits of the digest, so the statistical machinery is reused rather than forked. + */ export function deterministicRng(key: string): number { - return Number.parseInt(sha256(key).slice(0, 13), 16) / 16 ** 13 + return mulberry32(Number.parseInt(sha256(key).slice(0, 8), 16))() +} + +// Deterministic serialization for hashing: keys sorted, undefined-valued entries stripped, so +// the same logical scope/config always produces the same hash regardless of construction order. +// Known limitation: NaN and Infinity serialize as null (JSON.stringify semantics) — acceptable +// for hashing because it is deterministic; bigint has no JSON form at all, so it fails loud. +function canonicalJson(value: unknown): string { + if (typeof value === 'bigint') { + throw new TypeError('canonicalJson: bigint values are not supported in holdout hashing') + } + if (value === undefined) return 'null' + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]` + if (value !== null && typeof value === 'object') { + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`) + return `{${entries.join(',')}}` + } + return JSON.stringify(value) +} + +/** + * Value-hash of the knobs that DEFINE the experiment (epsilon + watchlist, order-independent). + * Everything else on the config (callbacks, attribution stamps, privacy flags) does not change + * which assignments are drawn, so it stays out of the hash. + */ +export function retrievalHoldoutConfigHash( + config: Pick, +): string { + return sha256( + canonicalJson({ epsilon: config.epsilon, watchlist: [...(config.watchlist ?? [])].sort() }), + ).slice(0, 16) +} + +// Malformed knobs silently corrupt propensities (epsilon>1 forces holdout with dropPropensity>1; +// epsilon<0 disables it while still logging a control-arm-shaped stream), so fail loud instead. +function assertValidHoldoutConfig(config: RetrievalHoldoutConfig): void { + const { epsilon, watchlist } = config + if (typeof epsilon !== 'number' || !Number.isFinite(epsilon) || epsilon < 0 || epsilon > 1) { + throw new Error(`retrieval holdout epsilon must be a number in [0, 1], got ${String(epsilon)}`) + } + if ( + watchlist !== undefined && + (!Array.isArray(watchlist) || watchlist.some((id) => typeof id !== 'string')) + ) { + throw new Error('retrieval holdout watchlist must be an array of item-id strings') + } +} + +// Observability must never break retrieval: a throwing onEvent sink is logged and swallowed, +// deliberately identical on the randomized and bypass paths. The cost is a lost event — a hole +// the analysis can see as a callIndex gap — never a lost or altered retrieval result. +function safeEmit(config: RetrievalHoldoutConfig, event: RetrievalHoldoutEvent): void { + try { + config.onEvent(event) + } catch (error) { + console.error( + '[agent-knowledge] retrieval holdout onEvent sink threw; retrieval continues', + error, + ) + } } /** @@ -112,10 +215,11 @@ export function applyRetrievalHoldout( config: RetrievalHoldoutConfig, ctx: RetrievalHoldoutCallContext = {}, ): RetrievalHoldoutResult { + assertValidHoldoutConfig(config) const rng = config.rng ?? deterministicRng - const watchlist = config.watchlist ?? [] const sessionId = ctx.sessionId ?? ctx.scope?.sessionId const holdoutEligible = typeof sessionId === 'string' && sessionId.length > 0 + const base = holdoutEventBase(hits, config, ctx) let session: RetrievalHoldoutSessionState | undefined if (holdoutEligible) { @@ -132,8 +236,7 @@ export function applyRetrievalHoldout( session = { ...session, callCount: session.callCount + 1 } } - const watchlistSet = new Set(watchlist) - const watchlistEligible = hits.filter((hit) => watchlistSet.has(hit.id)).map((hit) => hit.id) + const watchlistEligible = base.watchlistEligible if (session?.sessionHoldout && session.targetId === null && watchlistEligible.length > 0) { // Draw once, uniformly over watchlist ∩ E at the first intersecting call, then stay sticky. @@ -155,22 +258,52 @@ export function applyRetrievalHoldout( const pickPropensity = session?.sessionHoldout ? (session.pickPropensity ?? null) : null const event: RetrievalHoldoutEvent = { - v: 1, + ...base, + callIndex: session?.callCount ?? 0, + holdoutEligible, + sessionHoldout: session?.sessionHoldout ?? false, + sessionTargetId: targetId, + droppedId, + pickPropensity, + dropPropensity: pickPropensity !== null ? config.epsilon * pickPropensity : null, + deliveredIds: delivered.map((hit) => hit.id), + } + safeEmit(config, event) + + return droppedId === null + ? { delivered: hits, event, ...(session !== undefined ? { session } : {}) } + : { delivered, event, ...(session !== undefined ? { session } : {}) } +} + +/** Fields shared by randomized and bypass events: identity, attribution, config echo, eligibility. */ +function holdoutEventBase( + hits: AgentMemoryHit[], + config: RetrievalHoldoutConfig, + ctx: Omit, +) { + const sessionId = ctx.sessionId ?? ctx.scope?.sessionId + const watchlist = config.watchlist ?? [] + const watchlistSet = new Set(watchlist) + const plaintext = config.includePlaintextIdentifiers === true + return { + v: 1 as const, eventId: randomUUID(), ts: new Date().toISOString(), ...(config.adapterId !== undefined ? { adapterId: config.adapterId } : {}), - ...(sessionId !== undefined ? { sessionId } : {}), + ...(plaintext && sessionId !== undefined ? { sessionId } : {}), ...(ctx.taskId !== undefined ? { taskId: ctx.taskId } : {}), - callIndex: session?.callCount ?? 0, - ...(sessionId !== undefined ? { rngKey: sha256(sessionId).slice(0, 16) } : {}), + ...(sessionId !== undefined ? { sessionIdHash: sha256(sessionId).slice(0, 16) } : {}), ...(ctx.query !== undefined ? { queryHash: sha256(ctx.query).slice(0, 16) } : {}), - ...(ctx.scope !== undefined ? { scope: ctx.scope } : {}), + ...(plaintext && ctx.scope !== undefined ? { scope: ctx.scope } : {}), + ...(ctx.scope !== undefined + ? { scopeHash: sha256(canonicalJson(ctx.scope)).slice(0, 16) } + : {}), config: { epsilon: config.epsilon, watchlist: [...watchlist], ...(config.configVersion !== undefined ? { configVersion: config.configVersion } : {}), }, - holdoutEligible, + configHash: retrievalHoldoutConfigHash(config), eligible: hits.map((hit, index) => ({ id: hit.id, rank: index + 1, @@ -180,46 +313,81 @@ export function applyRetrievalHoldout( kind: hit.kind, contentHash: sha256(hit.text).slice(0, 16), })), - watchlistEligible, - sessionHoldout: session?.sessionHoldout ?? false, - sessionTargetId: targetId, - droppedId, - pickPropensity, - dropPropensity: pickPropensity !== null ? config.epsilon * pickPropensity : null, - deliveredIds: delivered.map((hit) => hit.id), + watchlistEligible: hits.filter((hit) => watchlistSet.has(hit.id)).map((hit) => hit.id), ...(config.corpusVersion !== undefined ? { corpusVersion: config.corpusVersion } : {}), } - config.onEvent(event) +} - return droppedId === null - ? { delivered: hits, event, ...(session !== undefined ? { session } : {}) } - : { delivered, event, ...(session !== undefined ? { session } : {}) } +/** + * Logs a holdout event for adapter context paths that answer WITHOUT going through the + * search→render seam (Neo4j short-term conversation context, raw-string getContext results), + * so a consumer with a holdout configured still sees every call and the fraction-under-experiment + * denominator stays honest instead of silently losing these calls. No suppression is applied: + * dropping is only meaningful for retrieved memory hits, never for conversation context. + */ +export function emitRetrievalHoldoutBypass( + hits: AgentMemoryHit[], + config: RetrievalHoldoutConfig, + ctx: Omit, + bypassReason: RetrievalHoldoutBypassReason, +): RetrievalHoldoutEvent { + assertValidHoldoutConfig(config) + const event: RetrievalHoldoutEvent = { + ...holdoutEventBase(hits, config, ctx), + callIndex: 0, + holdoutEligible: false, + sessionHoldout: false, + sessionTargetId: null, + droppedId: null, + pickPropensity: null, + dropPropensity: null, + deliveredIds: hits.map((hit) => hit.id), + bypassReason, + } + safeEmit(config, event) + return event } // Session states are tiny (5 scalar fields); the cap only guards unbounded long-lived processes. // Evicting a live session degrades detectably, not silently: a re-drawn target that differs -// shows up in the log as a mixed-exposure session, which analysis excludes and counts. -const MAX_TRACKED_SESSIONS = 10_000 -const sessionRegistry = new WeakMap< - RetrievalHoldoutConfig, - Map ->() +// shows up in the log as a mixed-exposure session (same sessionIdHash, different sessionTargetId, +// callIndex restarting at 1), which analysis excludes and counts. +const DEFAULT_MAX_TRACKED_SESSIONS = 10_000 +// Keyed by configHash VALUE, not config object identity: callers building options inline pass a +// fresh config object per call (the natural adapter pattern), and identity-keying would silently +// reset callIndex and re-draw the "sticky" target mid-session — one session logged as +// under-treatment for two different items, invalidating the per-item estimate. The outer map is +// bounded by the number of distinct experiment configs, in practice a handful. +const sessionRegistry = new Map>() /** - * Convenience wrapper that threads session state internally, keyed by config object identity. - * Reuse ONE config object across all calls of a session (normally one per harness) or - * stickiness falls back to the deterministic per-session draws alone. + * Clears all sticky-session state. For test isolation and experiment-epoch boundaries; + * sessions in flight afterwards re-draw deterministically (same rng keys), so with the default + * rng a reset is invisible in the logs unless eligibility sets changed in between. + */ +export function resetRetrievalHoldoutRegistry(): void { + sessionRegistry.clear() +} + +/** + * Convenience wrapper that threads session state internally, keyed by the VALUE of the + * experiment-defining knobs (configHash = epsilon + sorted watchlist) plus sessionId, so a fresh + * config object per call — the natural pattern when options are built inline — keeps full + * stickiness. Distinct experiments never share session state; two config objects with the same + * epsilon/watchlist are the same experiment by definition. */ export function applySessionStickyRetrievalHoldout( hits: AgentMemoryHit[], config: RetrievalHoldoutConfig, ctx: Omit = {}, ): RetrievalHoldoutResult { + assertValidHoldoutConfig(config) const sessionId = ctx.sessionId ?? ctx.scope?.sessionId - let sessions = sessionRegistry.get(config) + const configHash = retrievalHoldoutConfigHash(config) + let sessions = sessionRegistry.get(configHash) if (!sessions) { sessions = new Map() - sessionRegistry.set(config, sessions) + sessionRegistry.set(configHash, sessions) } const prior = sessionId !== undefined ? sessions.get(sessionId) : undefined const result = applyRetrievalHoldout(hits, config, { @@ -227,7 +395,9 @@ export function applySessionStickyRetrievalHoldout( ...(prior ? { session: prior } : {}), }) if (sessionId !== undefined && result.session) { - if (!sessions.has(sessionId) && sessions.size >= MAX_TRACKED_SESSIONS) { + const cap = config.maxTrackedSessions ?? DEFAULT_MAX_TRACKED_SESSIONS + if (!sessions.has(sessionId) && sessions.size >= cap) { + // Oldest-inserted-first: Map preserves insertion order and updates keep position. const oldest = sessions.keys().next().value if (oldest !== undefined) sessions.delete(oldest) } @@ -235,3 +405,206 @@ export function applySessionStickyRetrievalHoldout( } return result } + +/** Aggregated view of one session's holdout exposure — the randomization unit of this design. */ +export interface RetrievalHoldoutSessionSummary { + /** `${configHash}:${sessionIdHash}` — the trajectory's runId. */ + runId: string + configHash: string + sessionIdHash: string + sessionHoldout: boolean + sessionTargetId: string | null + /** Item actually suppressed in this session (equals sessionTargetId when a drop occurred). */ + droppedId: string | null + /** |watchlist ∩ E| at the session's first intersecting randomized call; 0 when never intersecting. */ + firstCandidateCount: number + /** Randomized retrieval calls observed for this session. */ + callCount: number + /** Adapter bypass calls observed for this session (logged, never randomized). */ + bypassCallCount: number + /** Session-level probability of the OBSERVED assignment under the behavior policy. */ + behaviorProb: number + /** True when the session's events disagree on arm or target (e.g. registry eviction re-draw). */ + mixedExposure: boolean + /** Present only on sessions surfaced in `excluded` rather than converted. */ + exclusionReason?: 'mixed-exposure' | 'no-randomized-calls' +} + +export interface RetrievalHoldoutOffPolicyOptions { + /** + * Realized outcome PER SESSION, keyed by sessionIdHash — the session is the randomization + * unit, so rewards are per session, never per call. A missing entry for an included session + * throws: silently dropping unscored sessions would corrupt the estimator's denominator, so + * filter events to scored sessions upstream instead. + */ + rewards: Record + /** + * Target-policy probability of the session's observed assignment. Default models the + * always-deliver-in-full policy: 1 for full-delivery sessions, 0 for drop sessions. + */ + targetProb?: (session: RetrievalHoldoutSessionSummary) => number + /** Per-session reward-model prediction enabling `doublyRobust`; when absent, qHat is null. */ + qHat?: (session: RetrievalHoldoutSessionSummary) => number | null +} + +export interface RetrievalHoldoutOffPolicyResult { + /** One trajectory per (configHash, sessionIdHash) — never per call. */ + trajectories: OffPolicyTrajectory[] + /** 1:1 with `trajectories` (same order): the diagnostic view of each converted session. */ + sessions: RetrievalHoldoutSessionSummary[] + /** Sessions surfaced but NOT converted (mixed exposure, bypass-only) — counted, never hidden. */ + excluded: RetrievalHoldoutSessionSummary[] + /** Events with no sessionIdHash: outside session randomization, cannot join a reward. */ + unattributableEvents: number +} + +/** + * Maps holdout logs onto agent-eval's `OffPolicyTrajectory`, ONE TRAJECTORY PER SESSION, so + * EXP-007's analysis consumes the substrate's `inverseProbabilityWeighting` / + * `selfNormalizedImportanceWeighting` / `doublyRobust` estimators directly. This matches the + * PREREG estimator (session-sticky self-normalized IPW): the design randomizes per SESSION — + * the arm coin is flipped once (P(holdout) = epsilon) and the sticky target is drawn once, + * uniformly over watchlist ∩ E at the session's first intersecting call. + * + * Session-level action space and behavior probabilities (they sum to 1 by construction): + * - full delivery (control arm observed): `1 − epsilon`; + * - drop candidate i of k = |watchlist ∩ E_first|: `epsilon / k` each (the draw event's logged + * `dropPropensity`); + * - sessions whose eligibility sets never intersect the watchlist: probability 1 — full + * delivery was certain in either arm. + * + * Per-call events are repeated observations WITHIN one session-level randomization; per-call + * IPW is statistically invalid for this design (per-call "propensities" imply an action + * distribution summing to more than 1 and bias IPS downward). Calls where the sticky target is + * absent from E, and adapter bypass calls, fold into the session summary (callCount / + * bypassCallCount) instead of generating independent propensities. + * + * Join contract: `rewards` is keyed by `sessionIdHash` (sha256(sessionId) first 16 hex — compute + * the same prefix over the outcome table's session ids, or log with + * `includePlaintextIdentifiers: true` and join on raw ids upstream). Events are grouped per + * (configHash, sessionIdHash) so different experiment configs are never pooled. Mixed-exposure + * sessions (arm or target disagreement, e.g. after registry eviction) are excluded from the + * trajectories and surfaced in `excluded` for the analysis to count. + * Pre-registration for EXP-007 stays in the research repo by design; the manifest vocabulary for + * it is agent-eval's `HypothesisManifest` / `signManifest` / `evaluateHypothesis` exports. + */ +export function toOffPolicyTrajectory( + events: RetrievalHoldoutEvent[], + options: RetrievalHoldoutOffPolicyOptions, +): RetrievalHoldoutOffPolicyResult { + const groups = new Map() + let unattributableEvents = 0 + for (const event of events) { + if (event.sessionIdHash === undefined) { + unattributableEvents += 1 + continue + } + const key = `${event.configHash}:${event.sessionIdHash}` + const group = groups.get(key) + if (group) group.push(event) + else groups.set(key, [event]) + } + + const trajectories: OffPolicyTrajectory[] = [] + const sessions: RetrievalHoldoutSessionSummary[] = [] + const excluded: RetrievalHoldoutSessionSummary[] = [] + + for (const [runId, group] of groups) { + const ordered = [...group].sort((a, b) => a.callIndex - b.callIndex || a.ts.localeCompare(b.ts)) + const randomized = ordered.filter((event) => event.holdoutEligible) + const firstEvent = ordered[0] + if (firstEvent?.sessionIdHash === undefined) continue + const base = { + runId, + configHash: firstEvent.configHash, + sessionIdHash: firstEvent.sessionIdHash, + callCount: randomized.length, + bypassCallCount: ordered.length - randomized.length, + } + const first = randomized[0] + if (first === undefined) { + excluded.push({ + ...base, + sessionHoldout: false, + sessionTargetId: null, + droppedId: null, + firstCandidateCount: 0, + behaviorProb: 1, + mixedExposure: false, + exclusionReason: 'no-randomized-calls', + }) + continue + } + + const targets = new Set( + randomized + .map((event) => event.sessionTargetId) + .filter((target): target is string => target !== null), + ) + const arms = new Set(randomized.map((event) => event.sessionHoldout)) + const mixedExposure = targets.size > 1 || arms.size > 1 + const dropEvent = randomized.find((event) => event.droppedId !== null) + const firstIntersecting = randomized.find((event) => event.watchlistEligible.length > 0) + const sessionTargetId = dropEvent?.sessionTargetId ?? [...targets][0] ?? null + + let behaviorProb: number + if (dropEvent !== undefined) { + if (dropEvent.dropPropensity === null) { + throw new Error(`holdout event ${dropEvent.eventId} recorded a drop without dropPropensity`) + } + // epsilon / |watchlist ∩ E_first|, logged at draw time. + behaviorProb = dropEvent.dropPropensity + } else if (!mixedExposure && sessionTargetId !== null) { + // A drawn target implies its draw call dropped it, so a target with no drop event means + // the batch is missing that session's calls. Weighting partial sessions biases the + // estimate, so fail loud instead. + throw new Error( + `holdout session ${base.sessionIdHash} has a drawn target but no drop event: ` + + 'the event batch is missing this session draw call', + ) + } else if (firstIntersecting === undefined) { + // No call could ever drop anything: full delivery was certain in either arm. + behaviorProb = 1 + } else { + // Control arm observed with a drop possible: the session-level assignment probability + // is 1 − epsilon (NOT 1 − epsilon·pick, which is a per-item survival probability and + // makes the implied action distribution sum past 1). + behaviorProb = 1 - first.config.epsilon + } + + const summary: RetrievalHoldoutSessionSummary = { + ...base, + sessionHoldout: first.sessionHoldout, + sessionTargetId, + droppedId: dropEvent?.droppedId ?? null, + firstCandidateCount: firstIntersecting + ? new Set(firstIntersecting.watchlistEligible).size + : 0, + behaviorProb, + mixedExposure, + ...(mixedExposure ? { exclusionReason: 'mixed-exposure' as const } : {}), + } + if (mixedExposure) { + excluded.push(summary) + continue + } + + const reward = options.rewards[summary.sessionIdHash] + if (reward === undefined) { + throw new Error( + `no reward for session ${summary.sessionIdHash}: ` + + 'filter events to scored sessions before converting', + ) + } + trajectories.push({ + runId, + reward, + behaviorProb, + targetProb: options.targetProb?.(summary) ?? (summary.droppedId === null ? 1 : 0), + qHat: options.qHat?.(summary) ?? null, + }) + sessions.push(summary) + } + + return { trajectories, sessions, excluded, unattributableEvents } +} diff --git a/src/memory/neo4j.ts b/src/memory/neo4j.ts index 162e3ac..a6e73db 100644 --- a/src/memory/neo4j.ts +++ b/src/memory/neo4j.ts @@ -1,9 +1,11 @@ import { stableId } from '../ids' import { defaultGetMemoryContext } from './adapter' +import { emitRetrievalHoldoutBypass } from './holdout' import { memoryHitToSourceRecord, memoryWriteResultToSourceRecord } from './source-record' import type { AgentMemoryAdapter, AgentMemoryHit, + AgentMemoryScope, AgentMemorySearchOptions, AgentMemoryWriteInput, AgentMemoryWriteResult, @@ -58,6 +60,14 @@ export function createNeo4jAgentMemoryAdapter( : hits.length > 0 ? renderHits(hits) : '') + if (searchOptions.holdout) { + emitRetrievalHoldoutBypass( + hits, + searchOptions.holdout, + holdoutBypassContext(query, searchOptions), + 'short-term-context', + ) + } return { query, text, @@ -102,6 +112,14 @@ export function createNeo4jAgentMemoryAdapter( title: 'Memory context', normalizedScore: 1, } + if (searchOptions.holdout) { + emitRetrievalHoldoutBypass( + [hit], + searchOptions.holdout, + holdoutBypassContext(query, searchOptions), + 'raw-string-context', + ) + } return { query, text: result, @@ -285,6 +303,18 @@ async function searchNeo4jMemory( .slice(0, options.limit) } +// Mirrors defaultGetMemoryContext's holdout call context so bypass events join the same log stream. +function holdoutBypassContext( + query: string, + options: AgentMemorySearchOptions, +): { query: string; scope?: AgentMemoryScope; taskId?: string } { + return { + query, + ...(options.scope !== undefined ? { scope: options.scope } : {}), + ...(options.scope?.tags?.taskId !== undefined ? { taskId: options.scope.tags.taskId } : {}), + } +} + function neo4jOptions(options: AgentMemorySearchOptions): Record { return { limit: options.limit, diff --git a/tests/memory-holdout.test.ts b/tests/memory-holdout.test.ts index a33f302..c6f6756 100644 --- a/tests/memory-holdout.test.ts +++ b/tests/memory-holdout.test.ts @@ -1,4 +1,9 @@ -import { describe, expect, it } from 'vitest' +import { + doublyRobust, + inverseProbabilityWeighting, + selfNormalizedImportanceWeighting, +} from '@tangle-network/agent-eval/rl' +import { beforeEach, describe, expect, it, vi } from 'vitest' import { sha256 } from '../src/ids' import type { AgentMemoryHit, @@ -8,11 +13,21 @@ import type { import { applyRetrievalHoldout, applySessionStickyRetrievalHoldout, + createNeo4jAgentMemoryAdapter, defaultGetMemoryContext, deterministicRng, + emitRetrievalHoldoutBypass, renderMemoryContext, + resetRetrievalHoldoutRegistry, + retrievalHoldoutConfigHash, + toOffPolicyTrajectory, } from '../src/memory/index' +/** sessionIdHash as events carry it: sha256(sessionId) first 16 hex. */ +function sid(sessionId: string): string { + return sha256(sessionId).slice(0, 16) +} + function hit(id: string, text: string, score: number): AgentMemoryHit { return { id, uri: `memory://test/${id}`, kind: 'fact', text, normalizedScore: score } } @@ -67,6 +82,7 @@ describe('retrieval holdout: enabled behavior and event schema', () => { 'adapterId', 'callIndex', 'config', + 'configHash', 'corpusVersion', 'deliveredIds', 'droppedId', @@ -76,10 +92,9 @@ describe('retrieval holdout: enabled behavior and event schema', () => { 'holdoutEligible', 'pickPropensity', 'queryHash', - 'rngKey', - 'scope', + 'scopeHash', 'sessionHoldout', - 'sessionId', + 'sessionIdHash', 'sessionTargetId', 'taskId', 'ts', @@ -88,7 +103,9 @@ describe('retrieval holdout: enabled behavior and event schema', () => { ].sort(), ) expect(event.v).toBe(1) - expect(event.sessionId).toBe('s-1') + // Default-private: plaintext sessionId/scope are opt-in via includePlaintextIdentifiers. + expect(event.sessionId).toBeUndefined() + expect(event.scope).toBeUndefined() expect(event.taskId).toBe('task-17') expect(event.callIndex).toBe(1) expect(event.holdoutEligible).toBe(true) @@ -105,13 +122,17 @@ describe('retrieval holdout: enabled behavior and event schema', () => { { id: 'm7', rank: 2, score: 0.8, kind: 'fact', contentHash: sha256('beta').slice(0, 16) }, { id: 'm3', rank: 3, score: 0.7, kind: 'fact', contentHash: sha256('gamma').slice(0, 16) }, ]) - expect(event.rngKey).toBe(sha256('s-1').slice(0, 16)) + expect(event.sessionIdHash).toBe(sha256('s-1').slice(0, 16)) + expect(event.scopeHash).toBe(sha256('{"namespace":"ns","sessionId":"s-1"}').slice(0, 16)) expect(event.queryHash).toBe(sha256('the query').slice(0, 16)) expect(event.config).toEqual({ epsilon: 1, watchlist: ['m1', 'm7'], configVersion: '2026-07-04a', }) + expect(event.configHash).toBe( + retrievalHoldoutConfigHash({ epsilon: 1, watchlist: ['m1', 'm7'] }), + ) expect(event.adapterId).toBe('test-adapter') expect(event.corpusVersion).toBe('kb-test-1') // The manipulation is invisible: the rendered context renumbers with no gap (design rule D6). @@ -240,6 +261,611 @@ describe('retrieval holdout: enabled behavior and event schema', () => { }) }) +describe('retrieval holdout: adapter bypass paths still log the call', () => { + it('emits a short-term-context bypass event instead of silently skipping the hook', async () => { + const { config, events } = collectingConfig({ + epsilon: 1, + watchlist: ['obs-1'], + adapterId: 'neo4j-test', + rng: () => 0, + }) + const client = { + shortTerm: { + async getContext(conversationId: string) { + return { + observations: [{ id: 'obs-1', content: `Observation for ${conversationId}` }], + recentMessages: [{ id: 'msg-1', role: 'user', content: 'Keep it brief.' }], + } + }, + }, + } + const adapter = createNeo4jAgentMemoryAdapter({ client }) + + const context = await adapter.getContext('style', { + scope: { sessionId: 'conv-1', tags: { taskId: 'task-3' } }, + holdout: config, + }) + + // The bypass never suppresses: dropping is only meaningful for retrieved memory hits. + expect(context.hits.map((h) => h.id)).toEqual(['obs-1', 'msg-1']) + expect(context.text).toContain('Observation for conv-1') + expect(events).toHaveLength(1) + const event = events[0]! + expect(event.bypassReason).toBe('short-term-context') + expect(event.holdoutEligible).toBe(false) + expect(event.sessionHoldout).toBe(false) + expect(event.droppedId).toBeNull() + expect(event.sessionTargetId).toBeNull() + expect(event.pickPropensity).toBeNull() + expect(event.dropPropensity).toBeNull() + expect(event.callIndex).toBe(0) + expect(event.sessionId).toBeUndefined() + expect(event.sessionIdHash).toBe(sha256('conv-1').slice(0, 16)) + expect(event.taskId).toBe('task-3') + expect(event.adapterId).toBe('neo4j-test') + expect(event.watchlistEligible).toEqual(['obs-1']) + expect(event.eligible.map((e) => e.id)).toEqual(['obs-1', 'msg-1']) + expect(event.deliveredIds).toEqual(['obs-1', 'msg-1']) + expect(event.queryHash).toBe(sha256('style').slice(0, 16)) + }) + + it('emits a raw-string-context bypass event for string getContext results', async () => { + const { config, events } = collectingConfig({ epsilon: 1, watchlist: ['w-1'], rng: () => 0 }) + const client = { + async getContext() { + return 'Use the private project namespace.' + }, + } + const adapter = createNeo4jAgentMemoryAdapter({ client, id: 'neo4j-private' }) + + const context = await adapter.getContext('project namespace', { holdout: config }) + + expect(context.text).toBe('Use the private project namespace.') + expect(context.hits).toHaveLength(1) + expect(events).toHaveLength(1) + const event = events[0]! + expect(event.bypassReason).toBe('raw-string-context') + expect(event.holdoutEligible).toBe(false) + expect(event.droppedId).toBeNull() + expect(event.eligible).toEqual([ + { + id: context.hits[0]!.id, + rank: 1, + score: 1, + kind: 'fact', + contentHash: sha256('Use the private project namespace.').slice(0, 16), + }, + ]) + expect(event.deliveredIds).toEqual([context.hits[0]!.id]) + expect(event.sessionId).toBeUndefined() + expect(event.sessionIdHash).toBeUndefined() + }) +}) + +describe('retrieval holdout: value-keyed session registry', () => { + const forcedRng = (key: string) => (key.endsWith('#pick') ? 0.6 : 0) + + beforeEach(() => { + resetRetrievalHoldoutRegistry() + }) + + it('keeps stickiness when a FRESH config object is built per call', async () => { + const events: RetrievalHoldoutEvent[] = [] + // The natural adapter pattern: options (and the holdout config inside them) built inline + // per retrieval. Identity-keyed state would miss on every call. + const freshConfig = (): RetrievalHoldoutConfig => ({ + epsilon: 1, + watchlist: ['m1', 'm3', 'm7'], + rng: forcedRng, + onEvent: (event) => events.push(event), + }) + + // Reviewer repro: call1 candidates sorted(watchlist ∩ E) = [m1, m7] → floor(0.6·2) = 1 → m7. + // A registry miss on call2 would re-draw over [m1, m3, m7] (floor(0.6·3) = 1 → m3), + // logging one session as under-treatment for two different items. + const call1 = await defaultGetMemoryContext( + { search: async () => [hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8)] }, + 'q1', + { scope: { sessionId: 's-fresh' }, holdout: freshConfig() }, + ) + const call2 = await defaultGetMemoryContext( + { + search: async () => [ + hit('m1', 'alpha', 0.9), + hit('m3', 'gamma', 0.7), + hit('m7', 'beta', 0.8), + ], + }, + 'q2', + { scope: { sessionId: 's-fresh' }, holdout: freshConfig() }, + ) + + expect(call1.hits.map((h) => h.id)).toEqual(['m1']) + expect(call2.hits.map((h) => h.id)).toEqual(['m1', 'm3']) + expect(events.map((e) => e.callIndex)).toEqual([1, 2]) + expect(events.map((e) => e.droppedId)).toEqual(['m7', 'm7']) + expect(events.map((e) => e.sessionTargetId)).toEqual(['m7', 'm7']) + expect(new Set(events.map((e) => e.configHash)).size).toBe(1) + }) + + it('computes configHash order-independently and only from experiment-defining knobs', () => { + expect(retrievalHoldoutConfigHash({ epsilon: 0.2, watchlist: ['b', 'a'] })).toBe( + retrievalHoldoutConfigHash({ epsilon: 0.2, watchlist: ['a', 'b'] }), + ) + expect(retrievalHoldoutConfigHash({ epsilon: 0.2, watchlist: ['a'] })).not.toBe( + retrievalHoldoutConfigHash({ epsilon: 0.3, watchlist: ['a'] }), + ) + expect(retrievalHoldoutConfigHash({ epsilon: 0.2 })).toBe( + retrievalHoldoutConfigHash({ epsilon: 0.2, watchlist: [] }), + ) + }) + + it('evicts oldest-first at the cap and re-drawn targets surface as mixed exposure', () => { + const events: RetrievalHoldoutEvent[] = [] + const config: RetrievalHoldoutConfig = { + epsilon: 1, + watchlist: ['e1', 'e5', 'e9'], + maxTrackedSessions: 2, + rng: (key) => (key.endsWith('#pick') ? 0.6 : 0), + onEvent: (event) => events.push(event), + } + // Candidates sorted(watchlist ∩ E) = [e1, e9] → floor(0.6·2) = 1 → e9. + const richHits = [hit('e1', 'one', 0.9), hit('e9', 'nine', 0.8)] + + const a1 = applySessionStickyRetrievalHoldout(richHits, config, { sessionId: 'evict-a' }) + expect(a1.event.sessionTargetId).toBe('e9') + applySessionStickyRetrievalHoldout(richHits, config, { sessionId: 'evict-b' }) + // Inserting C at cap 2 evicts A, the oldest-inserted session. + applySessionStickyRetrievalHoldout(richHits, config, { sessionId: 'evict-c' }) + + // B survived: its second call continues callIndex 2 with the same sticky target. + const b2 = applySessionStickyRetrievalHoldout(richHits, config, { sessionId: 'evict-b' }) + expect(b2.event.callIndex).toBe(2) + expect(b2.event.sessionTargetId).toBe('e9') + + // A returns after eviction with a narrower eligibility set: state was lost, so the target + // is re-drawn over [e1] alone — a different item than A's first exposure. + const a2 = applySessionStickyRetrievalHoldout([hit('e1', 'one', 0.9)], config, { + sessionId: 'evict-a', + }) + expect(a2.event.sessionTargetId).toBe('e1') + expect(a2.event.droppedId).toBe('e1') + + // The detectable mixed-exposure signature the analysis excludes and counts: + // same session hash, two different targets, callIndex reset to 1. + const aEvents = events.filter((e) => e.sessionIdHash === sha256('evict-a').slice(0, 16)) + expect(aEvents.map((e) => e.sessionTargetId)).toEqual(['e9', 'e1']) + expect(aEvents.map((e) => e.callIndex)).toEqual([1, 1]) + }) +}) + +describe('retrieval holdout: privacy defaults on event identifiers', () => { + it('emits plaintext sessionId and scope only with includePlaintextIdentifiers', () => { + const scope = { sessionId: 's-pii', userId: 'user-9', tags: { team: 'gtm' } } + const { config, events } = collectingConfig({ epsilon: 0, watchlist: ['m1'] }) + applyRetrievalHoldout([hit('m1', 'alpha', 0.9)], config, { scope }) + const { config: openConfig, events: openEvents } = collectingConfig({ + epsilon: 0, + watchlist: ['m1'], + includePlaintextIdentifiers: true, + }) + applyRetrievalHoldout([hit('m1', 'alpha', 0.9)], openConfig, { scope }) + + const priv = events[0]! + expect(priv.sessionId).toBeUndefined() + expect(priv.scope).toBeUndefined() + expect(priv.sessionIdHash).toBe(sha256('s-pii').slice(0, 16)) + expect(priv.scopeHash).toBe( + sha256('{"sessionId":"s-pii","tags":{"team":"gtm"},"userId":"user-9"}').slice(0, 16), + ) + + const open = openEvents[0]! + expect(open.sessionId).toBe('s-pii') + expect(open.scope).toEqual(scope) + // Hashes stay present either way, so both log shapes share the same join keys. + expect(open.sessionIdHash).toBe(priv.sessionIdHash) + expect(open.scopeHash).toBe(priv.scopeHash) + }) + + it('canonical hashing rejects bigint values instead of hashing a lossy form', () => { + const { config } = collectingConfig({ epsilon: 0 }) + expect(() => + applyRetrievalHoldout([], config, { + scope: { tags: { n: 1n as unknown as string } }, + }), + ).toThrow(TypeError) + }) +}) + +describe('retrieval holdout: onEvent sink failures never break retrieval', () => { + it('logs and continues when the sink throws, identically on randomized and bypass paths', () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + try { + const config: RetrievalHoldoutConfig = { + epsilon: 1, + watchlist: ['m1'], + rng: () => 0, + onEvent: () => { + throw new Error('sink down') + }, + } + const hits = [hit('m1', 'alpha', 0.9), hit('m2', 'beta', 0.8)] + const { delivered, event } = applyRetrievalHoldout(hits, config, { sessionId: 's-sink' }) + // Suppression still applied and the event still returned: only persistence was lost. + expect(delivered.map((h) => h.id)).toEqual(['m2']) + expect(event.droppedId).toBe('m1') + const bypass = emitRetrievalHoldoutBypass(hits, config, { query: 'q' }, 'raw-string-context') + expect(bypass.bypassReason).toBe('raw-string-context') + expect(errorSpy).toHaveBeenCalledTimes(2) + } finally { + errorSpy.mockRestore() + } + }) +}) + +describe('retrieval holdout: config validation fails loud', () => { + const onEvent = () => {} + + it('rejects epsilon outside [0, 1]', () => { + expect(() => + applyRetrievalHoldout( + [hit('m1', 'alpha', 0.9)], + { epsilon: 1.5, watchlist: ['m1'], onEvent }, + { sessionId: 's-v1' }, + ), + ).toThrow('epsilon must be a number in [0, 1]') + expect(() => applyRetrievalHoldout([], { epsilon: -0.1, onEvent })).toThrow( + 'epsilon must be a number in [0, 1]', + ) + expect(() => applyRetrievalHoldout([], { epsilon: Number.NaN, onEvent })).toThrow('epsilon') + }) + + it('rejects a watchlist that is not an array of strings', () => { + expect(() => + applyRetrievalHoldout([], { epsilon: 0.5, watchlist: [7 as unknown as string], onEvent }), + ).toThrow('watchlist must be an array of item-id strings') + expect(() => + emitRetrievalHoldoutBypass( + [], + { epsilon: 0.5, watchlist: 'm1' as unknown as string[], onEvent }, + {}, + 'raw-string-context', + ), + ).toThrow('watchlist must be an array of item-id strings') + }) + + it('validates before the sticky wrapper touches the registry', () => { + expect(() => + applySessionStickyRetrievalHoldout([], { epsilon: 2, onEvent }, { sessionId: 's-v2' }), + ).toThrow('epsilon must be a number in [0, 1]') + }) +}) + +describe('retrieval holdout: watchlist and score edge cases', () => { + it('with an empty or omitted watchlist the event is still emitted and nothing is ever dropped', () => { + const { config, events } = collectingConfig({ epsilon: 1, watchlist: [], rng: () => 0 }) + const hits = [hit('m1', 'alpha', 0.9), hit('m2', 'beta', 0.8)] + const { delivered, event } = applyRetrievalHoldout(hits, config, { sessionId: 's-empty' }) + + // Delivery identity is preserved on the no-drop path even in a holdout-arm session. + expect(delivered).toBe(hits) + expect(events).toHaveLength(1) + // The epsilon coin is independent of the watchlist, but no target can ever be drawn. + expect(event.sessionHoldout).toBe(true) + expect(event.sessionTargetId).toBeNull() + expect(event.droppedId).toBeNull() + expect(event.pickPropensity).toBeNull() + expect(event.watchlistEligible).toEqual([]) + expect(event.eligible.map((e) => e.id)).toEqual(['m1', 'm2']) + expect(event.deliveredIds).toEqual(['m1', 'm2']) + + const { config: omitted, events: omittedEvents } = collectingConfig({ + epsilon: 1, + rng: () => 0, + }) + const result = applyRetrievalHoldout(hits, omitted, { sessionId: 's-omitted' }) + expect(result.delivered).toBe(hits) + expect(omittedEvents).toHaveLength(1) + expect(omittedEvents[0]!.droppedId).toBeNull() + expect(omittedEvents[0]!.config.watchlist).toEqual([]) + }) + + it('falls back to the raw score when normalizedScore is absent', () => { + const { config } = collectingConfig({ epsilon: 0 }) + const scoreOnly: AgentMemoryHit = { + id: 'm-s', + uri: 'memory://test/m-s', + kind: 'fact', + text: 'raw', + score: 0.42, + } + const { event } = applyRetrievalHoldout([scoreOnly], config, { sessionId: 's-score' }) + expect(event.eligible[0]).toEqual({ + id: 'm-s', + rank: 1, + score: 0.42, + kind: 'fact', + contentHash: sha256('raw').slice(0, 16), + }) + }) +}) + +describe('retrieval holdout: session-level off-policy conversion', () => { + // Forces holdout (epsilon 1) and picks index floor(0.6 * |candidates|) deterministically. + const forcedRng = (key: string) => (key.endsWith('#pick') ? 0.6 : 0) + + it('emits ONE trajectory per session; t>1 target-absent calls fold in without a propensity', () => { + const { config, events } = collectingConfig({ + epsilon: 1, + watchlist: ['m1', 'm7'], + rng: forcedRng, + }) + const drop = applyRetrievalHoldout([hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8)], config, { + sessionId: 's-ope', + }) + // Second call of the SAME session with the sticky target absent from E: this is a repeated + // observation within one randomization, never an independent draw. + applyRetrievalHoldout([hit('m3', 'gamma', 0.7)], config, { + sessionId: 's-ope', + session: drop.session, + }) + + const result = toOffPolicyTrajectory(events, { rewards: { [sid('s-ope')]: 0 } }) + expect(result.trajectories).toHaveLength(1) + expect(result.trajectories[0]).toEqual({ + runId: `${events[0]!.configHash}:${sid('s-ope')}`, + reward: 0, + // Session-level P(drop m7) = epsilon / |watchlist ∩ E_first| = 1/2, the logged + // draw-time dropPropensity. + behaviorProb: 0.5, + targetProb: 0, + qHat: null, + }) + expect(result.sessions[0]).toMatchObject({ + callCount: 2, + bypassCallCount: 0, + droppedId: 'm7', + sessionTargetId: 'm7', + firstCandidateCount: 2, + mixedExposure: false, + }) + expect(result.excluded).toHaveLength(0) + expect(result.unattributableEvents).toBe(0) + }) + + it('assigns control sessions behaviorProb 1 − epsilon, the session-level assignment probability', () => { + const { config, events } = collectingConfig({ + epsilon: 0.2, + watchlist: ['m1', 'm7'], + // Coin 0.99 >= epsilon: control arm. + rng: () => 0.99, + }) + applyRetrievalHoldout( + [hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8), hit('m3', 'gamma', 0.7)], + config, + { sessionId: 's-ctl' }, + ) + const { trajectories } = toOffPolicyTrajectory(events, { rewards: { [sid('s-ctl')]: 1 } }) + // NOT 1 − epsilon·pick (a per-item survival probability): P(control) = 1 − epsilon. + expect(trajectories[0]!.behaviorProb).toBeCloseTo(0.8, 12) + expect(trajectories[0]!.targetProb).toBe(1) + expect(trajectories[0]!.reward).toBe(1) + }) + + it('property: session-level action probabilities sum to 1 over the (epsilon, k) grid', () => { + for (const epsilon of [0.1, 0.5, 0.9]) { + for (const k of [1, 2, 5]) { + const ids = Array.from({ length: k }, (_, i) => `w${i}`) + const hits = ids.map((id, i) => hit(id, `text-${id}`, 1 - i * 0.01)) + const rewards: Record = {} + const events: RetrievalHoldoutEvent[] = [] + const mkConfig = (rig: (key: string) => number): RetrievalHoldoutConfig => ({ + epsilon, + watchlist: ids, + rng: rig, + onEvent: (event) => events.push(event), + }) + // One control session plus k holdout sessions realizing each drop action once + // (the pick rig lands on candidate i exactly: floor(((i + 0.5) / k) · k) = i). + const ctlSession = `grid-ctl-${epsilon}-${k}` + applyRetrievalHoldout( + hits, + mkConfig(() => 0.999999), + { sessionId: ctlSession }, + ) + rewards[sid(ctlSession)] = 1 + for (let i = 0; i < k; i += 1) { + const holdoutSession = `grid-h-${epsilon}-${k}-${i}` + applyRetrievalHoldout( + hits, + mkConfig((key) => (key.endsWith('#pick') ? (i + 0.5) / k : 0)), + { sessionId: holdoutSession }, + ) + rewards[sid(holdoutSession)] = 1 + } + + const { trajectories, sessions } = toOffPolicyTrajectory(events, { rewards }) + expect(trajectories).toHaveLength(k + 1) + const drops = sessions.filter((s) => s.droppedId !== null) + // All k drop actions enumerated, each with probability epsilon / k. + expect(new Set(drops.map((s) => s.droppedId)).size).toBe(k) + for (const s of drops) expect(s.behaviorProb).toBeCloseTo(epsilon / k, 12) + const control = sessions.find((s) => s.droppedId === null) + expect(control!.behaviorProb).toBeCloseTo(1 - epsilon, 12) + const total = control!.behaviorProb + drops.reduce((acc, s) => acc + s.behaviorProb, 0) + expect(total).toBeCloseTo(1, 12) + } + } + }) + + it('regression: IPS recovers 1.000 on the audit 100-session scenario (per-call gave 0.667)', () => { + const rewards: Record = {} + const { config, events } = collectingConfig({ + epsilon: 0.5, + watchlist: ['m1', 'm7'], + rng: (key) => (key.includes(':h:') ? (key.endsWith('#pick') ? 0.6 : 0) : 0.99), + }) + for (let i = 0; i < 50; i += 1) { + const holdoutSession = `sim:h:${i}` + const controlSession = `sim:c:${i}` + applyRetrievalHoldout([hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8)], config, { + sessionId: holdoutSession, + }) + applyRetrievalHoldout([hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8)], config, { + sessionId: controlSession, + }) + rewards[sid(holdoutSession)] = 0 + rewards[sid(controlSession)] = 1 + } + + const { trajectories, sessions } = toOffPolicyTrajectory(events, { rewards }) + expect(trajectories).toHaveLength(100) + expect(sessions.filter((s) => s.droppedId !== null)).toHaveLength(50) + const ips = inverseProbabilityWeighting(trajectories) + const snips = selfNormalizedImportanceWeighting(trajectories) + expect(ips.n).toBe(100) + // True always-deliver value is 1: controls weigh 1/(1−ε) = 2, drops weigh 0. + // The per-call converter weighed controls 1/(1−ε·pick) = 4/3 and measured 0.667 here. + expect(ips.value).toBeCloseTo(1, 12) + expect(snips.value).toBeCloseTo(1, 12) + expect(ips.maxImportanceWeight).toBeCloseTo(2, 12) + }) + + it('IPS and SNIPS diverge under arm imbalance with mixed candidate counts; DR recovers with a truthful qHat', () => { + const rewards: Record = {} + const events: RetrievalHoldoutEvent[] = [] + const cfg = (rig: (key: string) => number): RetrievalHoldoutConfig => ({ + epsilon: 0.5, + watchlist: ['m1', 'm7'], + rng: rig, + onEvent: (event) => events.push(event), + }) + for (let i = 0; i < 4; i += 1) { + const s = `div-c-${i}` + applyRetrievalHoldout( + [hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8)], + cfg(() => 0.99), + { sessionId: s }, + ) + rewards[sid(s)] = 1 + } + // Two holdout sessions with DIFFERENT candidate-set sizes: k=2 → ε/k = 0.25, k=1 → 0.5. + applyRetrievalHoldout( + [hit('m1', 'alpha', 0.9), hit('m7', 'beta', 0.8)], + cfg((key) => (key.endsWith('#pick') ? 0.6 : 0)), + { sessionId: 'div-h-k2' }, + ) + rewards[sid('div-h-k2')] = 0 + applyRetrievalHoldout( + [hit('m7', 'beta', 0.8)], + cfg(() => 0), + { sessionId: 'div-h-k1' }, + ) + rewards[sid('div-h-k1')] = 0 + + const { trajectories, sessions } = toOffPolicyTrajectory(events, { rewards }) + expect( + sessions + .filter((s) => s.droppedId !== null) + .map((s) => s.behaviorProb) + .sort((a, b) => a - b), + ).toEqual([0.25, 0.5]) + const ips = inverseProbabilityWeighting(trajectories) + const snips = selfNormalizedImportanceWeighting(trajectories) + // 4 controls at weight 2, 2 drops at weight 0: Σw·r = 8 over n = 6 vs Σw = 8. + expect(ips.value).toBeCloseTo(8 / 6, 12) + expect(snips.value).toBeCloseTo(1, 12) + expect(Math.abs(ips.value - snips.value)).toBeGreaterThan(0.2) + // Doubly-robust with a truthful per-session qHat is exact despite the imbalance. + const dr = doublyRobust(toOffPolicyTrajectory(events, { rewards, qHat: () => 1 }).trajectories) + expect(dr.value).toBeCloseTo(1, 12) + }) + + it('honors custom targetProb and qHat callbacks per session', () => { + const { config, events } = collectingConfig({ + epsilon: 0.2, + watchlist: ['m1'], + rng: () => 0.99, + }) + applyRetrievalHoldout([hit('m1', 'alpha', 0.9)], config, { sessionId: 's-cb' }) + const { trajectories } = toOffPolicyTrajectory(events, { + rewards: { [sid('s-cb')]: 0.75 }, + targetProb: (session) => (session.droppedId === null ? 0.25 : 0), + qHat: () => 0.5, + }) + expect(trajectories[0]).toMatchObject({ + reward: 0.75, + behaviorProb: 0.8, + targetProb: 0.25, + qHat: 0.5, + }) + }) + + it('excludes mixed-exposure and bypass-only sessions, surfacing them with reasons', () => { + const { config, events } = collectingConfig({ + epsilon: 1, + watchlist: ['e1', 'e9'], + rng: (key) => (key.endsWith('#pick') ? 0.6 : 0), + }) + // Two calls of one session WITHOUT threaded state and with different eligibility sets: + // fresh draws land on different targets — the mixed-exposure signature. + applyRetrievalHoldout([hit('e1', 'one', 0.9), hit('e9', 'nine', 0.8)], config, { + sessionId: 's-mixed', + }) + applyRetrievalHoldout([hit('e1', 'one', 0.9)], config, { sessionId: 's-mixed' }) + // A session that only ever hit adapter bypass paths. + emitRetrievalHoldoutBypass( + [hit('m1', 'alpha', 0.9)], + config, + { query: 'q', scope: { sessionId: 's-bypass-only' } }, + 'short-term-context', + ) + // An event with no session at all. + applyRetrievalHoldout([hit('e1', 'one', 0.9)], config, {}) + + const result = toOffPolicyTrajectory(events, { rewards: {} }) + expect(result.trajectories).toHaveLength(0) + expect(result.excluded.map((s) => s.exclusionReason).sort()).toEqual([ + 'mixed-exposure', + 'no-randomized-calls', + ]) + const mixed = result.excluded.find((s) => s.exclusionReason === 'mixed-exposure') + expect(mixed).toMatchObject({ mixedExposure: true, sessionIdHash: sid('s-mixed') }) + const bypassOnly = result.excluded.find((s) => s.exclusionReason === 'no-randomized-calls') + expect(bypassOnly).toMatchObject({ callCount: 0, bypassCallCount: 1 }) + expect(result.unattributableEvents).toBe(1) + }) + + it('fails loud when an included session has no reward', () => { + const { config, events } = collectingConfig({ epsilon: 0, watchlist: ['m1'] }) + applyRetrievalHoldout([hit('m1', 'alpha', 0.9)], config, { sessionId: 's-unscored' }) + expect(() => toOffPolicyTrajectory(events, { rewards: {} })).toThrow('no reward for session') + }) + + it('fails loud on a corrupt drop event and on a batch missing the draw call', () => { + const { config, events } = collectingConfig({ epsilon: 1, watchlist: ['m1'], rng: () => 0 }) + const drop = applyRetrievalHoldout([hit('m1', 'alpha', 0.9)], config, { + sessionId: 's-corrupt', + }) + const corrupted = events.map((event) => ({ ...event, dropPropensity: null })) + expect(() => toOffPolicyTrajectory(corrupted, { rewards: { [sid('s-corrupt')]: 0 } })).toThrow( + 'without dropPropensity', + ) + + // Batch containing only the t=2 target-absent call: the drawn target proves the draw call + // is missing, and weighting the partial session would bias the estimate. + const { event: absentCall } = applyRetrievalHoldout([hit('m3', 'gamma', 0.7)], config, { + sessionId: 's-corrupt', + session: drop.session, + }) + expect(absentCall.sessionTargetId).toBe('m1') + expect(absentCall.droppedId).toBeNull() + expect(() => + toOffPolicyTrajectory([absentCall], { rewards: { [sid('s-corrupt')]: 0 } }), + ).toThrow('missing this session draw call') + }) +}) + describe('retrieval holdout: empirical calibration of the default rng', () => { it('drops at a rate matching epsilon over 10^4 simulated sessions', () => { const epsilon = 0.2