feat(campaign): discriminative scenario selection (E2)

src/campaign/index.ts

@@ -238,6 +238,13 @@ export {
   runCampaign,
 } from './run-campaign'
 export { resolveRunDir, tangleTracesRoot } from './run-dir.js'
+// ── Discriminative holdout selection (drop saturated ties) ───────────
+export {
+  type DiscriminationScore,
+  type ScenarioSignal,
+  scoreDiscrimination,
+  selectDiscriminative,
+} from './scenario-selection'
 export { type CampaignBreakdown, campaignBreakdown, campaignMeanComposite } from './score-utils'
 export {
   type ApplySkillPatchResult,

src/campaign/scenario-selection.test.ts

@@ -0,0 +1,111 @@
+import { describe, expect, it } from 'vitest'
+import {
+  type ScenarioSignal,
+  scoreDiscrimination,
+  selectDiscriminative,
+} from './scenario-selection'
+
+describe('scoreDiscrimination', () => {
+  it('ranks a high-variance scenario above a low-variance one', () => {
+    const signals: ScenarioSignal[] = [
+      { scenarioId: 'flat', scores: [0.5, 0.5, 0.5] },
+      { scenarioId: 'spread', scores: [0.1, 0.9, 0.5] },
+    ]
+    const ranked = scoreDiscrimination(signals)
+    expect(ranked.map((s) => s.scenarioId)).toEqual(['spread', 'flat'])
+    expect(ranked[0]!.discrimination).toBeGreaterThan(ranked[1]!.discrimination)
+  })
+
+  it('flags a scenario every candidate scores 1.0 as tied', () => {
+    const signals: ScenarioSignal[] = [
+      { scenarioId: 'saturated', scores: [1.0, 1.0, 1.0] },
+      { scenarioId: 'live', scores: [0.2, 0.8] },
+    ]
+    const byId = new Map(scoreDiscrimination(signals).map((s) => [s.scenarioId, s]))
+    const saturated = byId.get('saturated')
+    expect(saturated?.tied).toBe(true)
+    expect(saturated?.variance ?? 1).toBeLessThan(1e-9)
+    expect(byId.get('live')?.tied).toBe(false)
+  })
+
+  it('does not flag an all-equal-but-low scenario as tied (room to improve)', () => {
+    // Every candidate scores 0.5 identically: zero variance, but NOT saturated —
+    // there is headroom, so it is not a wasted tie.
+    const signals: ScenarioSignal[] = [{ scenarioId: 'hard-flat', scores: [0.5, 0.5, 0.5] }]
+    const [score] = scoreDiscrimination(signals)
+    expect(score!.variance).toBeLessThan(1e-9)
+    expect(score!.tied).toBe(false)
+  })
+
+  it('breaks discrimination ties by lower meanScore (more headroom first)', () => {
+    // Same spread (variance) but different difficulty: the harder one ranks first.
+    // Dyadic values (±0.125 deviations) so the variances are bit-identical and the
+    // meanScore tiebreak — not float noise — decides the order.
+    const signals: ScenarioSignal[] = [
+      { scenarioId: 'easy', scores: [0.5, 0.75] },
+      { scenarioId: 'hard', scores: [0.125, 0.375] },
+    ]
+    const ranked = scoreDiscrimination(signals)
+    expect(ranked[0]!.variance).toBeCloseTo(ranked[1]!.variance, 12)
+    expect(ranked.map((s) => s.scenarioId)).toEqual(['hard', 'easy'])
+  })
+
+  it('honors a custom saturationCeiling', () => {
+    const signals: ScenarioSignal[] = [{ scenarioId: 's', scores: [0.9, 0.9] }]
+    expect(scoreDiscrimination(signals, { saturationCeiling: 0.8 })[0]!.tied).toBe(true)
+    expect(scoreDiscrimination(signals, { saturationCeiling: 0.95 })[0]!.tied).toBe(false)
+  })
+})
+
+describe('selectDiscriminative', () => {
+  const mixed: ScenarioSignal[] = [
+    { scenarioId: 'tie-a', scores: [1.0, 1.0, 1.0] },
+    { scenarioId: 'tie-b', scores: [1.0, 1.0] },
+    { scenarioId: 'live-lo', scores: [0.2, 0.4] },
+    { scenarioId: 'live-hi', scores: [0.0, 1.0] },
+  ]
+
+  it('excludes saturated ties when enough non-tied scenarios exist', () => {
+    const picked = selectDiscriminative(mixed, 2)
+    expect(picked).toEqual(['live-hi', 'live-lo'])
+    expect(picked).not.toContain('tie-a')
+    expect(picked).not.toContain('tie-b')
+  })
+
+  it('falls back to least-saturated ties when non-tied < k', () => {
+    const signals: ScenarioSignal[] = [
+      { scenarioId: 'live', scores: [0.1, 0.9] },
+      { scenarioId: 'tie-high', scores: [1.0, 1.0] },
+      { scenarioId: 'tie-low', scores: [0.999, 0.999] },
+    ]
+    // Only 1 non-tied; k=2 must fill with the least-saturated tie (tie-low < tie-high).
+    const picked = selectDiscriminative(signals, 2)
+    expect(picked[0]!).toBe('live')
+    expect(picked[1]!).toBe('tie-low')
+    expect(picked).not.toContain('tie-high')
+  })
+
+  it('is deterministic: same input twice ⇒ identical output', () => {
+    const a = selectDiscriminative(mixed, 3)
+    const b = selectDiscriminative(mixed, 3)
+    expect(a).toEqual(b)
+  })
+
+  it('throws if k < 1', () => {
+    expect(() => selectDiscriminative(mixed, 0)).toThrow()
+    expect(() => selectDiscriminative(mixed, -1)).toThrow()
+  })
+
+  it('returns all ids in discrimination order when k >= n', () => {
+    const all = selectDiscriminative(mixed, mixed.length)
+    expect(all).toHaveLength(mixed.length)
+    expect([...all].sort()).toEqual(['live-hi', 'live-lo', 'tie-a', 'tie-b'])
+    // Order matches scoreDiscrimination ranking, including ties (not dropped here).
+    expect(all).toEqual(scoreDiscrimination(mixed).map((s) => s.scenarioId))
+  })
+
+  it('returns all ids when k exceeds n', () => {
+    const all = selectDiscriminative(mixed, mixed.length + 5)
+    expect(all).toHaveLength(mixed.length)
+  })
+})

src/campaign/scenario-selection.ts

@@ -0,0 +1,116 @@
+/**
+ * Discriminative scenario selection (research claim E2).
+ *
+ * The OR benchmark is SATURATING: run 7 measured ~75% tied holdout cells — most
+ * problems are solved optimally by the baseline AND every candidate, so those
+ * paired cells carry zero signal. A random/balanced holdout split spends its
+ * budget on scenarios that cannot separate candidates.
+ *
+ * This picks the holdout by DISCRIMINATION power instead: a scenario every
+ * candidate scores identically (variance ~0) carries no signal; one where the
+ * scores spread carries the most. We drop fully saturated ties so each paired
+ * holdout cell is spent on a scenario that can actually move a verdict.
+ */
+
+/** Per-scenario observation: the composite scores each candidate earned on it. */
+export interface ScenarioSignal {
+  scenarioId: string
+  /** Per-candidate composite scores observed for this scenario (>=1 values). */
+  scores: number[]
+}
+
+export interface DiscriminationScore {
+  scenarioId: string
+  /** Higher = separates candidates more (spread of their scores). */
+  discrimination: number
+  /** Higher = easier / more-saturated (mean of candidate scores). */
+  meanScore: number
+  variance: number
+  /** variance ~0 AND meanScore at/above the ceiling ⇒ a saturated tie, no signal. */
+  tied: boolean
+}
+
+const DEFAULT_SATURATION_CEILING = 0.999
+/** Variance below this is treated as "every candidate scored the same". */
+const VARIANCE_EPSILON = 1e-9
+
+interface Moments {
+  meanScore: number
+  variance: number
+}
+
+/** Population mean + variance of the candidate scores. Empty ⇒ zeros (a signal
+ *  with no observations cannot discriminate). */
+function moments(scores: number[]): Moments {
+  const n = scores.length
+  if (n === 0) return { meanScore: 0, variance: 0 }
+  let sum = 0
+  for (const s of scores) sum += s
+  const meanScore = sum / n
+  let sqDev = 0
+  for (const s of scores) {
+    const d = s - meanScore
+    sqDev += d * d
+  }
+  return { meanScore, variance: sqDev / n }
+}
+
+/** Deterministic ordering: discrimination desc, then meanScore asc (more
+ *  headroom first), then scenarioId asc. */
+function compareDiscrimination(a: DiscriminationScore, b: DiscriminationScore): number {
+  if (b.discrimination !== a.discrimination) return b.discrimination - a.discrimination
+  if (a.meanScore !== b.meanScore) return a.meanScore - b.meanScore
+  return a.scenarioId < b.scenarioId ? -1 : a.scenarioId > b.scenarioId ? 1 : 0
+}
+
+/**
+ * Rank scenarios by how well they DISCRIMINATE candidates.
+ *
+ * `discrimination = variance` (spread of the candidate scores) — kept simple on
+ * purpose; the headroom term (`saturationCeiling - meanScore`) only breaks ties
+ * so that, among equally spread scenarios, the one with more room to improve
+ * ranks first. Returned sorted by the deterministic order above.
+ */
+export function scoreDiscrimination(
+  signals: ScenarioSignal[],
+  opts?: { saturationCeiling?: number },
+): DiscriminationScore[] {
+  const saturationCeiling = opts?.saturationCeiling ?? DEFAULT_SATURATION_CEILING
+  const scored = signals.map((signal): DiscriminationScore => {
+    const { meanScore, variance } = moments(signal.scores)
+    const tied = variance < VARIANCE_EPSILON && meanScore >= saturationCeiling
+    return {
+      scenarioId: signal.scenarioId,
+      discrimination: variance,
+      meanScore,
+      variance,
+      tied,
+    }
+  })
+  return scored.sort(compareDiscrimination)
+}
+
+/**
+ * Select the top-`k` most discriminative scenario ids for a holdout, EXCLUDING
+ * fully saturated ties when enough non-tied scenarios exist (a tie in the
+ * holdout wastes a paired cell).
+ *
+ * Prefers non-tied scenarios; if fewer than `k` non-tied exist, fills with the
+ * least-saturated tied ones (tied scenarios are already ordered least-saturated
+ * first by `meanScore` asc). Deterministic. Throws if `k < 1`. If
+ * `signals.length <= k`, returns all ids in discrimination order.
+ */
+export function selectDiscriminative(
+  signals: ScenarioSignal[],
+  k: number,
+  opts?: { saturationCeiling?: number },
+): string[] {
+  if (k < 1) throw new Error(`selectDiscriminative: k must be >= 1 (got ${k})`)
+  const ranked = scoreDiscrimination(signals, opts)
+  if (ranked.length <= k) return ranked.map((s) => s.scenarioId)
+  const nonTied = ranked.filter((s) => !s.tied)
+  if (nonTied.length >= k) return nonTied.slice(0, k).map((s) => s.scenarioId)
+  const tied = ranked.filter((s) => s.tied)
+  const fill = tied.slice(0, k - nonTied.length)
+  return [...nonTied, ...fill].map((s) => s.scenarioId)
+}