From f3ee4519786728ae301e8ead2e690dbbc73d87e2 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 7 Jul 2026 11:24:45 -0600 Subject: [PATCH] =?UTF-8?q?feat(campaign):=20discriminative=20scenario=20s?= =?UTF-8?q?election=20=E2=80=94=20pick=20holdout=20by=20signal,=20drop=20s?= =?UTF-8?q?aturated=20ties=20(E2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/campaign/index.ts | 7 ++ src/campaign/scenario-selection.test.ts | 111 +++++++++++++++++++++++ src/campaign/scenario-selection.ts | 116 ++++++++++++++++++++++++ 3 files changed, 234 insertions(+) create mode 100644 src/campaign/scenario-selection.test.ts create mode 100644 src/campaign/scenario-selection.ts diff --git a/src/campaign/index.ts b/src/campaign/index.ts index 85280c42..b5aef2e9 100644 --- a/src/campaign/index.ts +++ b/src/campaign/index.ts @@ -238,6 +238,13 @@ export { runCampaign, } from './run-campaign' export { resolveRunDir, tangleTracesRoot } from './run-dir.js' +// ── Discriminative holdout selection (drop saturated ties) ─────────── +export { + type DiscriminationScore, + type ScenarioSignal, + scoreDiscrimination, + selectDiscriminative, +} from './scenario-selection' export { type CampaignBreakdown, campaignBreakdown, campaignMeanComposite } from './score-utils' export { type ApplySkillPatchResult, diff --git a/src/campaign/scenario-selection.test.ts b/src/campaign/scenario-selection.test.ts new file mode 100644 index 00000000..bf47e76f --- /dev/null +++ b/src/campaign/scenario-selection.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from 'vitest' +import { + type ScenarioSignal, + scoreDiscrimination, + selectDiscriminative, +} from './scenario-selection' + +describe('scoreDiscrimination', () => { + it('ranks a high-variance scenario above a low-variance one', () => { + const signals: ScenarioSignal[] = [ + { scenarioId: 'flat', scores: [0.5, 0.5, 0.5] }, + { scenarioId: 'spread', scores: [0.1, 0.9, 0.5] }, + ] + const ranked = scoreDiscrimination(signals) + expect(ranked.map((s) => s.scenarioId)).toEqual(['spread', 'flat']) + expect(ranked[0]!.discrimination).toBeGreaterThan(ranked[1]!.discrimination) + }) + + it('flags a scenario every candidate scores 1.0 as tied', () => { + const signals: ScenarioSignal[] = [ + { scenarioId: 'saturated', scores: [1.0, 1.0, 1.0] }, + { scenarioId: 'live', scores: [0.2, 0.8] }, + ] + const byId = new Map(scoreDiscrimination(signals).map((s) => [s.scenarioId, s])) + const saturated = byId.get('saturated') + expect(saturated?.tied).toBe(true) + expect(saturated?.variance ?? 1).toBeLessThan(1e-9) + expect(byId.get('live')?.tied).toBe(false) + }) + + it('does not flag an all-equal-but-low scenario as tied (room to improve)', () => { + // Every candidate scores 0.5 identically: zero variance, but NOT saturated — + // there is headroom, so it is not a wasted tie. + const signals: ScenarioSignal[] = [{ scenarioId: 'hard-flat', scores: [0.5, 0.5, 0.5] }] + const [score] = scoreDiscrimination(signals) + expect(score!.variance).toBeLessThan(1e-9) + expect(score!.tied).toBe(false) + }) + + it('breaks discrimination ties by lower meanScore (more headroom first)', () => { + // Same spread (variance) but different difficulty: the harder one ranks first. + // Dyadic values (±0.125 deviations) so the variances are bit-identical and the + // meanScore tiebreak — not float noise — decides the order. + const signals: ScenarioSignal[] = [ + { scenarioId: 'easy', scores: [0.5, 0.75] }, + { scenarioId: 'hard', scores: [0.125, 0.375] }, + ] + const ranked = scoreDiscrimination(signals) + expect(ranked[0]!.variance).toBeCloseTo(ranked[1]!.variance, 12) + expect(ranked.map((s) => s.scenarioId)).toEqual(['hard', 'easy']) + }) + + it('honors a custom saturationCeiling', () => { + const signals: ScenarioSignal[] = [{ scenarioId: 's', scores: [0.9, 0.9] }] + expect(scoreDiscrimination(signals, { saturationCeiling: 0.8 })[0]!.tied).toBe(true) + expect(scoreDiscrimination(signals, { saturationCeiling: 0.95 })[0]!.tied).toBe(false) + }) +}) + +describe('selectDiscriminative', () => { + const mixed: ScenarioSignal[] = [ + { scenarioId: 'tie-a', scores: [1.0, 1.0, 1.0] }, + { scenarioId: 'tie-b', scores: [1.0, 1.0] }, + { scenarioId: 'live-lo', scores: [0.2, 0.4] }, + { scenarioId: 'live-hi', scores: [0.0, 1.0] }, + ] + + it('excludes saturated ties when enough non-tied scenarios exist', () => { + const picked = selectDiscriminative(mixed, 2) + expect(picked).toEqual(['live-hi', 'live-lo']) + expect(picked).not.toContain('tie-a') + expect(picked).not.toContain('tie-b') + }) + + it('falls back to least-saturated ties when non-tied < k', () => { + const signals: ScenarioSignal[] = [ + { scenarioId: 'live', scores: [0.1, 0.9] }, + { scenarioId: 'tie-high', scores: [1.0, 1.0] }, + { scenarioId: 'tie-low', scores: [0.999, 0.999] }, + ] + // Only 1 non-tied; k=2 must fill with the least-saturated tie (tie-low < tie-high). + const picked = selectDiscriminative(signals, 2) + expect(picked[0]!).toBe('live') + expect(picked[1]!).toBe('tie-low') + expect(picked).not.toContain('tie-high') + }) + + it('is deterministic: same input twice ⇒ identical output', () => { + const a = selectDiscriminative(mixed, 3) + const b = selectDiscriminative(mixed, 3) + expect(a).toEqual(b) + }) + + it('throws if k < 1', () => { + expect(() => selectDiscriminative(mixed, 0)).toThrow() + expect(() => selectDiscriminative(mixed, -1)).toThrow() + }) + + it('returns all ids in discrimination order when k >= n', () => { + const all = selectDiscriminative(mixed, mixed.length) + expect(all).toHaveLength(mixed.length) + expect([...all].sort()).toEqual(['live-hi', 'live-lo', 'tie-a', 'tie-b']) + // Order matches scoreDiscrimination ranking, including ties (not dropped here). + expect(all).toEqual(scoreDiscrimination(mixed).map((s) => s.scenarioId)) + }) + + it('returns all ids when k exceeds n', () => { + const all = selectDiscriminative(mixed, mixed.length + 5) + expect(all).toHaveLength(mixed.length) + }) +}) diff --git a/src/campaign/scenario-selection.ts b/src/campaign/scenario-selection.ts new file mode 100644 index 00000000..586f8637 --- /dev/null +++ b/src/campaign/scenario-selection.ts @@ -0,0 +1,116 @@ +/** + * Discriminative scenario selection (research claim E2). + * + * The OR benchmark is SATURATING: run 7 measured ~75% tied holdout cells — most + * problems are solved optimally by the baseline AND every candidate, so those + * paired cells carry zero signal. A random/balanced holdout split spends its + * budget on scenarios that cannot separate candidates. + * + * This picks the holdout by DISCRIMINATION power instead: a scenario every + * candidate scores identically (variance ~0) carries no signal; one where the + * scores spread carries the most. We drop fully saturated ties so each paired + * holdout cell is spent on a scenario that can actually move a verdict. + */ + +/** Per-scenario observation: the composite scores each candidate earned on it. */ +export interface ScenarioSignal { + scenarioId: string + /** Per-candidate composite scores observed for this scenario (>=1 values). */ + scores: number[] +} + +export interface DiscriminationScore { + scenarioId: string + /** Higher = separates candidates more (spread of their scores). */ + discrimination: number + /** Higher = easier / more-saturated (mean of candidate scores). */ + meanScore: number + variance: number + /** variance ~0 AND meanScore at/above the ceiling ⇒ a saturated tie, no signal. */ + tied: boolean +} + +const DEFAULT_SATURATION_CEILING = 0.999 +/** Variance below this is treated as "every candidate scored the same". */ +const VARIANCE_EPSILON = 1e-9 + +interface Moments { + meanScore: number + variance: number +} + +/** Population mean + variance of the candidate scores. Empty ⇒ zeros (a signal + * with no observations cannot discriminate). */ +function moments(scores: number[]): Moments { + const n = scores.length + if (n === 0) return { meanScore: 0, variance: 0 } + let sum = 0 + for (const s of scores) sum += s + const meanScore = sum / n + let sqDev = 0 + for (const s of scores) { + const d = s - meanScore + sqDev += d * d + } + return { meanScore, variance: sqDev / n } +} + +/** Deterministic ordering: discrimination desc, then meanScore asc (more + * headroom first), then scenarioId asc. */ +function compareDiscrimination(a: DiscriminationScore, b: DiscriminationScore): number { + if (b.discrimination !== a.discrimination) return b.discrimination - a.discrimination + if (a.meanScore !== b.meanScore) return a.meanScore - b.meanScore + return a.scenarioId < b.scenarioId ? -1 : a.scenarioId > b.scenarioId ? 1 : 0 +} + +/** + * Rank scenarios by how well they DISCRIMINATE candidates. + * + * `discrimination = variance` (spread of the candidate scores) — kept simple on + * purpose; the headroom term (`saturationCeiling - meanScore`) only breaks ties + * so that, among equally spread scenarios, the one with more room to improve + * ranks first. Returned sorted by the deterministic order above. + */ +export function scoreDiscrimination( + signals: ScenarioSignal[], + opts?: { saturationCeiling?: number }, +): DiscriminationScore[] { + const saturationCeiling = opts?.saturationCeiling ?? DEFAULT_SATURATION_CEILING + const scored = signals.map((signal): DiscriminationScore => { + const { meanScore, variance } = moments(signal.scores) + const tied = variance < VARIANCE_EPSILON && meanScore >= saturationCeiling + return { + scenarioId: signal.scenarioId, + discrimination: variance, + meanScore, + variance, + tied, + } + }) + return scored.sort(compareDiscrimination) +} + +/** + * Select the top-`k` most discriminative scenario ids for a holdout, EXCLUDING + * fully saturated ties when enough non-tied scenarios exist (a tie in the + * holdout wastes a paired cell). + * + * Prefers non-tied scenarios; if fewer than `k` non-tied exist, fills with the + * least-saturated tied ones (tied scenarios are already ordered least-saturated + * first by `meanScore` asc). Deterministic. Throws if `k < 1`. If + * `signals.length <= k`, returns all ids in discrimination order. + */ +export function selectDiscriminative( + signals: ScenarioSignal[], + k: number, + opts?: { saturationCeiling?: number }, +): string[] { + if (k < 1) throw new Error(`selectDiscriminative: k must be >= 1 (got ${k})`) + const ranked = scoreDiscrimination(signals, opts) + if (ranked.length <= k) return ranked.map((s) => s.scenarioId) + const nonTied = ranked.filter((s) => !s.tied) + if (nonTied.length >= k) return nonTied.slice(0, k).map((s) => s.scenarioId) + const tied = ranked.filter((s) => s.tied) + const fill = tied.slice(0, k - nonTied.length) + return [...nonTied, ...fill].map((s) => s.scenarioId) +}