From c78df7c965e3389058db96d5de46849af2234971 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 7 Jul 2026 19:08:17 -0600 Subject: [PATCH] feat(benchmarks): add unified benchmark runner --- README.md | 2 +- examples/benchmarks/README.md | 40 +++ src/benchmarks/calibration.ts | 60 +++++ src/benchmarks/index.ts | 44 ++++ src/benchmarks/routing/index.ts | 6 + src/benchmarks/runner.ts | 381 +++++++++++++++++++++++++++++ src/benchmarks/standard-formats.ts | 365 +++++++++++++++++++++++++++ src/benchmarks/types.ts | 75 +++++- src/index.ts | 5 + tests/benchmarks.test.ts | 143 +++++++++++ 10 files changed, 1118 insertions(+), 3 deletions(-) create mode 100644 src/benchmarks/calibration.ts create mode 100644 src/benchmarks/runner.ts create mode 100644 src/benchmarks/standard-formats.ts diff --git a/README.md b/README.md index c7052b13..3d7e98de 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ Each example: `README.md` + a single `index.ts` runnable via `pnpm tsx`. Prints | `…/matrix` | `runAgentMatrix` — an N-axis cartesian over caller-supplied substrate values, per-axis pass/score/cost/duration | | `…/multishot` | N-shot persona × shot matrix runner (`runMultishot` / `runMultishotMatrix`) | | `…/wire` | The cross-language HTTP/RPC server + Zod schemas (the source-of-truth protocol the Python client speaks) + the built-in rubric registry | -| `…/benchmarks` | `BenchmarkAdapter` contract + `deterministicSplit` + the bundled `routing` reference benchmark | +| `…/benchmarks` | `BenchmarkAdapter` contract, `runBenchmarkAdapter`, `calibrateBenchmarkMetric`, standard retrieval parsers + ranked retrieval metrics, `deterministicSplit`, and the bundled `routing` reference benchmark | **Specialized surfaces** (subpath-only): `…/prm` (process-reward grading + best-of-N), `…/meta-eval` (judge calibration + the deployment-outcome store), `…/belief-state` (decision-point extraction + selective-policy reports), `…/pipelines` (trace-diagnostic views: budget breach, failure cluster, stuck loop, …), `…/governance` (EU AI Act / NIST AI RMF / SOC2 reports), `…/knowledge` (knowledge-readiness gating before a run), `…/builder-eval` (code-generator three-layer eval), `…/storyboard` (trace → watchable replay), `…/authenticity` (anti-Goodhart "real or convincing BS" scorer over produced files), `…/workflow` (workflow-trace eval + partner export), `…/telemetry` (Workers-safe telemetry client), `…/testing` (test-only reset helpers). diff --git a/examples/benchmarks/README.md b/examples/benchmarks/README.md index 1fc647c0..97fd2d53 100644 --- a/examples/benchmarks/README.md +++ b/examples/benchmarks/README.md @@ -26,6 +26,46 @@ assignSplit(itemId: string): 'search' | 'dev' | 'holdout' `assignSplit` uses `deterministicSplit(itemId, BENCHMARK_SPLIT_SEED)` — same item gets the same split everywhere. Don't change the seed; it's load-bearing for reproducibility. +## Running a benchmark + +Use `runBenchmarkAdapter` when you want a campaign-backed run with resumability, traces, cost, latency, split reporting, and persisted report artifacts. + +```ts +import { runBenchmarkAdapter, routing } from '@tangle-network/agent-eval/benchmarks' + +const result = await runBenchmarkAdapter({ + adapter: new routing.RoutingAdapter(), + runDir: 'routing-smoke', + respond: async ({ item, context }) => { + const route = await callRouter(item.payload.prompt) + context.cost.observe(0.001, 'router') + return route + }, +}) + +console.log(result.report.score.mean) +console.log(result.reportMarkdownPath) +``` + +Before treating a benchmark metric as real evidence, run `calibrateBenchmarkMetric` with an intentionally weak and intentionally strong artifact. +The default pass condition is weak score at most `0.3`, strong score at least `0.7`, and gap at least `0.4`. + +## Standard retrieval formats + +`@tangle-network/agent-eval/benchmarks` also exports dependency-free helpers for BEIR/MTEB/MS MARCO/TREC/MIRACL-style files: + +- `parseBeirCorpusJsonl` +- `parseBeirQueriesJsonl` +- `parseQrels` +- `buildStandardRetrievalItems` +- `createRetrievalIdBenchmarkAdapter` +- `evaluateStandardRetrieval` + +These helpers normalize public retrieval datasets into the same `BenchmarkDatasetItem` shape. +The retrieval evaluator accepts ranked document IDs and reports nDCG@k, recall@k, precision@k, MRR@k, and hit@k. +It does not copy the full corpus into every query payload unless `includeCorpusInPayload` is explicitly set. +Domain packages such as `agent-knowledge` can then map those items into their own RAG scenario types instead of re-parsing every public benchmark. + ## Adding a new benchmark 1. Create `examples/benchmarks//index.ts`. diff --git a/src/benchmarks/calibration.ts b/src/benchmarks/calibration.ts new file mode 100644 index 00000000..8dc84f13 --- /dev/null +++ b/src/benchmarks/calibration.ts @@ -0,0 +1,60 @@ +import type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation } from './types' + +export interface BenchmarkMetricCalibrationOptions { + adapter: BenchmarkAdapter, TPayload, TArtifact> + item: BenchmarkDatasetItem + weakArtifact: TArtifact + strongArtifact: TArtifact + maxWeakScore?: number + minStrongScore?: number + minGap?: number +} + +export interface BenchmarkMetricCalibrationResult { + passed: boolean + weak: BenchmarkEvaluation + strong: BenchmarkEvaluation + weakScore: number + strongScore: number + gap: number + reasons: string[] +} + +export async function calibrateBenchmarkMetric( + options: BenchmarkMetricCalibrationOptions, +): Promise { + const weak = await options.adapter.evaluate(options.item, options.weakArtifact) + const strong = await options.adapter.evaluate(options.item, options.strongArtifact) + const weakScore = clamp01(weak.score) + const strongScore = clamp01(strong.score) + const maxWeakScore = options.maxWeakScore ?? 0.3 + const minStrongScore = options.minStrongScore ?? 0.7 + const minGap = options.minGap ?? 0.4 + const gap = strongScore - weakScore + const reasons = [ + weakScore <= maxWeakScore + ? undefined + : `weak score ${weakScore.toFixed(3)} exceeds max ${maxWeakScore.toFixed(3)}`, + strongScore >= minStrongScore + ? undefined + : `strong score ${strongScore.toFixed(3)} below min ${minStrongScore.toFixed(3)}`, + gap >= minGap ? undefined : `gap ${gap.toFixed(3)} below min ${minGap.toFixed(3)}`, + ].filter((reason): reason is string => Boolean(reason)) + + return { + passed: reasons.length === 0, + weak, + strong, + weakScore, + strongScore, + gap, + reasons, + } +} + +function clamp01(value: number): number { + if (!Number.isFinite(value)) return 0 + if (value < 0) return 0 + if (value > 1) return 1 + return value +} diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index 15068678..0d982254 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -3,6 +3,9 @@ * * Core surface (exported here): * - The `BenchmarkAdapter` contract. + * - `runBenchmarkAdapter` for campaign-backed benchmark execution. + * - `calibrateBenchmarkMetric` for weak/strong metric checks. + * - Standard retrieval parsers for BEIR/MTEB/MS MARCO/TREC/MIRACL-style files. * - `deterministicSplit` + `BENCHMARK_SPLIT_SEED` for split assignment. * - `routing` — synthetic 16-task router benchmark. The only novel * benchmark we built; ships in the package. @@ -18,10 +21,51 @@ * entry — every team will configure them differently. */ +export { + type BenchmarkMetricCalibrationOptions, + type BenchmarkMetricCalibrationResult, + calibrateBenchmarkMetric, +} from './calibration' export * as routing from './routing/index' +export { + type BenchmarkDistribution, + type BenchmarkReport, + type BenchmarkRunOptions, + type BenchmarkRunResult, + type BenchmarkSliceSummary, + renderBenchmarkReportMarkdown, + runBenchmarkAdapter, + summarizeBenchmarkCampaign, +} from './runner' +export { + type BuildStandardRetrievalItemsOptions, + buildStandardRetrievalItems, + createRetrievalIdBenchmarkAdapter, + evaluateStandardRetrieval, + normalizeRetrievedDocumentIds, + parseBeirCorpusJsonl, + parseBeirQueriesJsonl, + parseJsonlRows, + parseQrels, + parseTsvRows, + type RetrievalIdAdapterOptions, + retrievalMetricsAtCutoff, + type StandardRetrievalArtifact, + type StandardRetrievalDocument, + type StandardRetrievalEvaluationOptions, + type StandardRetrievalPayload, + type StandardRetrievalQrel, + type StandardRetrievalQuery, + type StandardRetrievalResult, +} from './standard-formats' export type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation, + BenchmarkFamily, + BenchmarkResponder, + BenchmarkScenario, + BenchmarkSource, + BenchmarkTaskKind, } from './types' export { BENCHMARK_SPLIT_SEED, deterministicSplit } from './types' diff --git a/src/benchmarks/routing/index.ts b/src/benchmarks/routing/index.ts index 829f0f6f..e2cdc239 100644 --- a/src/benchmarks/routing/index.ts +++ b/src/benchmarks/routing/index.ts @@ -20,6 +20,12 @@ export type RoutingPayload = RoutingItem export type RoutingDatasetItem = BenchmarkDatasetItem class RoutingAdapter implements BenchmarkAdapter { + readonly id = 'first-party/routing' + readonly family = 'first-party' + readonly taskKind = 'routing' + readonly description = 'Synthetic fixed-route classification smoke benchmark' + readonly defaultMetric = 'route_exact_match' + async loadDataset(split: RunSplitTag): Promise { return ROUTING_DATASET.map((item) => ({ id: item.id, payload: item })).filter( (it) => assignSplitImpl(it.id) === split, diff --git a/src/benchmarks/runner.ts b/src/benchmarks/runner.ts new file mode 100644 index 00000000..0502d0a9 --- /dev/null +++ b/src/benchmarks/runner.ts @@ -0,0 +1,381 @@ +import { join } from 'node:path' +import { + type CampaignResult, + type DispatchFn, + type JudgeConfig, + type JudgeScore, + runCampaign, + type Scenario, +} from '../campaign' +import { type CampaignStorage, fsCampaignStorage } from '../campaign/storage' +import type { RunSplitTag } from '../run-record' +import type { + BenchmarkAdapter, + BenchmarkDatasetItem, + BenchmarkEvaluation, + BenchmarkFamily, + BenchmarkResponder, + BenchmarkScenario, + BenchmarkTaskKind, +} from './types' + +export interface BenchmarkRunOptions { + adapter: BenchmarkAdapter, TPayload, TArtifact> + respond: BenchmarkResponder + splits?: readonly RunSplitTag[] + runDir: string + repo?: string + seed?: number + reps?: number + resumable?: boolean + costCeiling?: number + maxConcurrency?: number + dispatchTimeoutMs?: number + expectUsage?: 'assert' | 'warn' | 'off' + storage?: CampaignStorage + now?: () => Date +} + +export interface BenchmarkReport { + benchmarkId: string + family: BenchmarkFamily | string + taskKind: BenchmarkTaskKind | string + source?: BenchmarkAdapter['source'] + runDir: string + manifestHash: string + seed: number + startedAt: string + endedAt: string + durationMs: number + totalItems: number + totalCells: number + cellsFailed: number + cellsCached: number + totalCostUsd: number + splits: Record + tags: Record + dimensions: Record + score: BenchmarkDistribution + costUsd: BenchmarkDistribution + latencyMs: BenchmarkDistribution +} + +export interface BenchmarkSliceSummary { + n: number + meanScore: number + passRate: number + score: BenchmarkDistribution + costUsd: BenchmarkDistribution + latencyMs: BenchmarkDistribution +} + +export interface BenchmarkDistribution { + n: number + min: number + mean: number + median: number + p90: number + max: number +} + +export interface BenchmarkRunResult { + scenarios: Array> + campaign: CampaignResult> + report: BenchmarkReport + reportJsonPath: string + reportMarkdownPath: string +} + +export async function runBenchmarkAdapter( + options: BenchmarkRunOptions, +): Promise> { + const storage = options.storage ?? fsCampaignStorage() + const benchmarkId = benchmarkIdFor(options.adapter) + const scenarios = await loadBenchmarkScenarios(options.adapter, options.splits) + const judge = benchmarkAdapterJudge(options.adapter) + const dispatch: DispatchFn, TArtifact> = async ( + scenario, + context, + ) => { + return options.respond({ scenario, item: scenario.item, context }) + } + + const campaign = await runCampaign, TArtifact>({ + scenarios, + dispatch, + dispatchRef: `benchmark:${benchmarkId}`, + judges: [judge], + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + costCeiling: options.costCeiling, + maxConcurrency: options.maxConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + expectUsage: options.expectUsage ?? 'off', + runDir: options.runDir, + repo: options.repo, + storage, + now: options.now, + }) + const report = summarizeBenchmarkCampaign({ + adapter: options.adapter, + scenarios, + campaign, + }) + storage.ensureDir(campaign.runDir) + const reportJsonPath = join(campaign.runDir, 'benchmark-report.json') + const reportMarkdownPath = join(campaign.runDir, 'benchmark-report.md') + storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}\n`) + storage.write(reportMarkdownPath, renderBenchmarkReportMarkdown(report)) + return { scenarios, campaign, report, reportJsonPath, reportMarkdownPath } +} + +export async function loadBenchmarkScenarios( + adapter: BenchmarkAdapter, TPayload, unknown>, + splits: readonly RunSplitTag[] = ['search', 'dev', 'holdout'], +): Promise>> { + const benchmarkId = benchmarkIdFor(adapter) + const family = adapter.family ?? 'custom' + const taskKind = adapter.taskKind ?? 'custom' + const out: Array> = [] + const seen = new Set() + for (const split of splits) { + const items = await adapter.loadDataset(split) + for (const item of items) { + const splitTag = item.split ?? adapter.assignSplit(item.id) + if (splitTag !== split) continue + const key = `${benchmarkId}:${item.id}` + if (seen.has(key)) continue + seen.add(key) + out.push({ + id: key, + kind: 'benchmark', + benchmarkId, + family: item.family ?? family, + taskKind: item.taskKind ?? taskKind, + splitTag, + tags: [...new Set([splitTag, ...(item.tags ?? [])])], + item, + }) + } + } + return out +} + +export function benchmarkAdapterJudge( + adapter: BenchmarkAdapter, TPayload, TArtifact>, +): JudgeConfig> { + const benchmarkId = benchmarkIdFor(adapter) + return { + name: `${benchmarkId}:score`, + dimensions: [ + { key: 'score', description: `Primary ${benchmarkId} benchmark score` }, + { key: 'passed', description: 'Binary pass projection for aggregate reporting' }, + ], + appliesTo: (scenario: Scenario): scenario is BenchmarkScenario => { + return scenario.kind === 'benchmark' && scenario.id.startsWith(`${benchmarkId}:`) + }, + async score({ artifact, scenario }) { + const evaluation = await adapter.evaluate(scenario.item, artifact) + const dimensions = normalizeEvaluationDimensions(evaluation) + return { + dimensions, + composite: clamp01(evaluation.score), + notes: evaluation.notes ?? '', + } + }, + } +} + +export function summarizeBenchmarkCampaign(input: { + adapter: BenchmarkAdapter, TPayload, TArtifact> + scenarios: Array> + campaign: CampaignResult> +}): BenchmarkReport { + const scenarioById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario])) + const rows = input.campaign.cells.map((cell) => { + const scenario = scenarioById.get(cell.scenarioId) + const judge = firstJudgeScore(cell.judgeScores) + const score = judge?.composite ?? 0 + return { + cell, + scenario, + score, + passed: (judge?.dimensions.passed ?? (score > 0 ? 1 : 0)) >= 1, + dimensions: judge?.dimensions ?? {}, + } + }) + const successful = rows.filter((row) => !row.cell.error) + const scoreValues = successful.map((row) => row.score) + return { + benchmarkId: benchmarkIdFor(input.adapter), + family: input.adapter.family ?? 'custom', + taskKind: input.adapter.taskKind ?? 'custom', + ...(input.adapter.source ? { source: input.adapter.source } : {}), + runDir: input.campaign.runDir, + manifestHash: input.campaign.manifestHash, + seed: input.campaign.seed, + startedAt: input.campaign.startedAt, + endedAt: input.campaign.endedAt, + durationMs: input.campaign.durationMs, + totalItems: input.scenarios.length, + totalCells: input.campaign.cells.length, + cellsFailed: input.campaign.aggregates.cellsFailed, + cellsCached: input.campaign.aggregates.cellsCached, + totalCostUsd: input.campaign.aggregates.totalCostUsd, + splits: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? 'unknown', [ + 'search', + 'dev', + 'holdout', + ]), + tags: summarizeSlices(successful, (row) => row.scenario?.tags ?? []), + dimensions: summarizeDimensions(successful.map((row) => row.dimensions)), + score: distribution(scoreValues), + costUsd: distribution(successful.map((row) => row.cell.costUsd)), + latencyMs: distribution(successful.map((row) => row.cell.durationMs)), + } +} + +export function renderBenchmarkReportMarkdown(report: BenchmarkReport): string { + const splitRows = Object.entries(report.splits) + .map(([split, summary]) => { + return `| ${split} | ${summary.n} | ${fmt(summary.meanScore)} | ${fmt(summary.passRate)} | ${fmt(summary.score.p90)} | ${fmt(summary.costUsd.mean)} | ${fmt(summary.latencyMs.p90)} |` + }) + .join('\n') + const dimRows = Object.entries(report.dimensions) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, dist]) => `| ${key} | ${dist.n} | ${fmt(dist.mean)} | ${fmt(dist.p90)} |`) + .join('\n') + return [ + `# Benchmark Report: ${report.benchmarkId}`, + '', + `- family: ${report.family}`, + `- task kind: ${report.taskKind}`, + `- run dir: ${report.runDir}`, + `- manifest: ${report.manifestHash}`, + `- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`, + `- cost: $${fmt(report.totalCostUsd)}`, + `- score: mean ${fmt(report.score.mean)}, median ${fmt(report.score.median)}, p90 ${fmt(report.score.p90)}, n=${report.score.n}`, + '', + '## Splits', + '', + '| split | n | mean score | pass rate | score p90 | mean cost | latency p90 ms |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + splitRows || '| none | 0 | 0 | 0 | 0 | 0 | 0 |', + '', + '## Dimensions', + '', + '| dimension | n | mean | p90 |', + '| --- | ---: | ---: | ---: |', + dimRows || '| none | 0 | 0 | 0 |', + '', + ].join('\n') +} + +function benchmarkIdFor(adapter: Pick): string { + return adapter.id ?? `${adapter.family ?? 'custom'}/${adapter.taskKind ?? 'custom'}` +} + +function normalizeEvaluationDimensions(evaluation: BenchmarkEvaluation): Record { + const dimensions: Record = { score: clamp01(evaluation.score) } + for (const [key, value] of Object.entries(evaluation.dimensions ?? {})) { + if (Number.isFinite(value)) dimensions[key] = value + } + dimensions.passed = (evaluation.passed ?? evaluation.score > 0) ? 1 : 0 + return dimensions +} + +function summarizeDimensions( + rows: Array>, +): Record { + const values = new Map() + for (const row of rows) { + for (const [key, value] of Object.entries(row)) { + if (!Number.isFinite(value)) continue + const list = values.get(key) ?? [] + list.push(value) + values.set(key, list) + } + } + return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)])) +} + +function summarizeSlices( + rows: T[], + keyOf: (row: T) => string | string[], + knownKeys: readonly string[] = [], +): Record { + const grouped = new Map() + for (const key of knownKeys) grouped.set(key, []) + for (const row of rows) { + const keys = keyOf(row) + for (const key of Array.isArray(keys) ? keys : [keys]) { + const list = grouped.get(key) ?? [] + list.push(row) + grouped.set(key, list) + } + } + const out: Record = {} + for (const [key, list] of grouped) { + const withShape = list as Array<{ + score: number + passed: boolean + cell: { costUsd: number; durationMs: number } + }> + out[key] = { + n: list.length, + meanScore: mean(withShape.map((row) => row.score)), + passRate: mean(withShape.map((row) => (row.passed ? 1 : 0))), + score: distribution(withShape.map((row) => row.score)), + costUsd: distribution(withShape.map((row) => row.cell.costUsd)), + latencyMs: distribution(withShape.map((row) => row.cell.durationMs)), + } + } + return out +} + +function firstJudgeScore( + judgeScores: CampaignResult['cells'][number]['judgeScores'], +): JudgeScore | undefined { + return Object.values(judgeScores)[0] +} + +function distribution(values: readonly number[]): BenchmarkDistribution { + const finite = [...values].filter(Number.isFinite).sort((a, b) => a - b) + if (finite.length === 0) return { n: 0, min: 0, mean: 0, median: 0, p90: 0, max: 0 } + return { + n: finite.length, + min: finite[0]!, + mean: mean(finite), + median: percentile(finite, 0.5), + p90: percentile(finite, 0.9), + max: finite[finite.length - 1]!, + } +} + +function percentile(sortedValues: readonly number[], p: number): number { + if (sortedValues.length === 0) return 0 + const index = Math.min( + sortedValues.length - 1, + Math.max(0, Math.ceil(p * sortedValues.length) - 1), + ) + return sortedValues[index]! +} + +function mean(values: readonly number[]): number { + const finite = values.filter(Number.isFinite) + if (finite.length === 0) return 0 + return finite.reduce((sum, value) => sum + value, 0) / finite.length +} + +function clamp01(value: number): number { + if (!Number.isFinite(value)) return 0 + if (value < 0) return 0 + if (value > 1) return 1 + return value +} + +function fmt(value: number): string { + if (!Number.isFinite(value)) return '0' + return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3) +} diff --git a/src/benchmarks/standard-formats.ts b/src/benchmarks/standard-formats.ts new file mode 100644 index 00000000..56e03eb4 --- /dev/null +++ b/src/benchmarks/standard-formats.ts @@ -0,0 +1,365 @@ +import type { RunSplitTag } from '../run-record' +import type { + BenchmarkAdapter, + BenchmarkDatasetItem, + BenchmarkFamily, + BenchmarkSource, + BenchmarkTaskKind, +} from './types' +import { deterministicSplit } from './types' + +export interface StandardRetrievalDocument { + id: string + title?: string + text: string + metadata?: Record +} + +export interface StandardRetrievalQuery { + id: string + text: string + metadata?: Record +} + +export interface StandardRetrievalQrel { + queryId: string + documentId: string + score: number +} + +export interface StandardRetrievalPayload { + queryId: string + query: string + expectedDocumentIds: string[] + expectedScores: Record + corpus?: Record + metadata?: Record +} + +export interface BuildStandardRetrievalItemsOptions { + benchmarkId: string + family: BenchmarkFamily | string + queries: readonly StandardRetrievalQuery[] + qrels: readonly StandardRetrievalQrel[] + corpus?: readonly StandardRetrievalDocument[] + includeCorpusInPayload?: boolean + source?: BenchmarkSource + tags?: readonly string[] + splitOf?: (queryId: string) => RunSplitTag +} + +export interface RetrievalIdAdapterOptions extends BuildStandardRetrievalItemsOptions { + responseIdPattern?: RegExp + cutoffs?: readonly number[] + primaryMetric?: string + passMetric?: string + passThreshold?: number +} + +export interface StandardRetrievalResult { + id?: string + documentId?: string + docId?: string + score?: number +} + +export type StandardRetrievalArtifact = + | string + | readonly string[] + | readonly StandardRetrievalResult[] + | { + ids?: readonly string[] + documentIds?: readonly string[] + results?: readonly StandardRetrievalResult[] + } + +export interface StandardRetrievalEvaluationOptions { + responseIdPattern?: RegExp + cutoffs?: readonly number[] + primaryMetric?: string + passMetric?: string + passThreshold?: number +} + +export function parseJsonlRows(text: string): T[] { + return text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .map((line, index) => { + try { + return JSON.parse(line) as T + } catch (error) { + throw new Error(`invalid JSONL row ${index + 1}: ${(error as Error).message}`) + } + }) +} + +export function parseTsvRows(text: string): string[][] { + return text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .filter((line) => !line.startsWith('#')) + .map((line) => line.split(/\t|\s+/)) +} + +export function parseQrels(text: string): StandardRetrievalQrel[] { + return parseTsvRows(text).flatMap((parts, index) => { + if (parts.length < 3) return [] + const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts + if (!queryId || !maybeZeroOrDocId || !maybeDocIdOrScore) return [] + if (queryId.toLowerCase() === 'query-id' || queryId.toLowerCase() === 'qid') return [] + const documentId = maybeScore === undefined ? maybeZeroOrDocId : maybeDocIdOrScore + const scoreText = maybeScore === undefined ? maybeDocIdOrScore : maybeScore + const score = Number(scoreText) + if (!documentId || !Number.isFinite(score)) { + throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`) + } + return [{ queryId, documentId, score }] + }) +} + +export function parseBeirCorpusJsonl(text: string): StandardRetrievalDocument[] { + return parseJsonlRows>(text).map((row, index) => { + const id = stringField(row, '_id') ?? stringField(row, 'id') + const body = stringField(row, 'text') ?? stringField(row, 'contents') + if (!id || body === undefined) { + throw new Error(`invalid BEIR corpus row ${index + 1}: expected _id/id and text/contents`) + } + return { + id, + title: stringField(row, 'title'), + text: body, + metadata: stripKnown(row, ['_id', 'id', 'title', 'text', 'contents']), + } + }) +} + +export function parseBeirQueriesJsonl(text: string): StandardRetrievalQuery[] { + return parseJsonlRows>(text).map((row, index) => { + const id = stringField(row, '_id') ?? stringField(row, 'id') ?? stringField(row, 'query_id') + const query = stringField(row, 'text') ?? stringField(row, 'query') + if (!id || !query) { + throw new Error(`invalid BEIR query row ${index + 1}: expected _id/id and text/query`) + } + return { + id, + text: query, + metadata: stripKnown(row, ['_id', 'id', 'query_id', 'text', 'query']), + } + }) +} + +export function buildStandardRetrievalItems( + options: BuildStandardRetrievalItemsOptions, +): Array> { + const qrelsByQuery = new Map() + for (const qrel of options.qrels) { + if (qrel.score <= 0) continue + const list = qrelsByQuery.get(qrel.queryId) ?? [] + list.push(qrel) + qrelsByQuery.set(qrel.queryId, list) + } + const corpus = + options.includeCorpusInPayload && options.corpus + ? Object.fromEntries(options.corpus.map((document) => [document.id, document])) + : undefined + return options.queries.flatMap((query) => { + const qrels = qrelsByQuery.get(query.id) ?? [] + if (qrels.length === 0) return [] + const split = + options.splitOf?.(query.id) ?? deterministicSplit(`${options.benchmarkId}:${query.id}`) + return [ + { + id: query.id, + split, + family: options.family, + taskKind: 'retrieval', + tags: [...new Set([...(options.tags ?? []), split])], + ...(options.source ? { source: options.source } : {}), + ...(query.metadata ? { metadata: query.metadata } : {}), + payload: { + queryId: query.id, + query: query.text, + expectedDocumentIds: qrels.map((qrel) => qrel.documentId), + expectedScores: Object.fromEntries(qrels.map((qrel) => [qrel.documentId, qrel.score])), + ...(corpus ? { corpus } : {}), + ...(query.metadata ? { metadata: query.metadata } : {}), + }, + }, + ] + }) +} + +export function createRetrievalIdBenchmarkAdapter( + options: RetrievalIdAdapterOptions, +): BenchmarkAdapter< + BenchmarkDatasetItem, + StandardRetrievalPayload, + StandardRetrievalArtifact +> { + const items = buildStandardRetrievalItems(options) + return { + id: options.benchmarkId, + family: options.family, + taskKind: 'retrieval' satisfies BenchmarkTaskKind, + source: options.source, + defaultMetric: options.primaryMetric ?? 'ndcg@10', + async loadDataset(split) { + return items.filter((item) => item.split === split) + }, + async evaluate(item, artifact) { + return evaluateStandardRetrieval(item.payload, artifact, options) + }, + assignSplit(itemId) { + return options.splitOf?.(itemId) ?? deterministicSplit(`${options.benchmarkId}:${itemId}`) + }, + } +} + +export function evaluateStandardRetrieval( + payload: StandardRetrievalPayload, + artifact: StandardRetrievalArtifact, + options: StandardRetrievalEvaluationOptions = {}, +) { + const rankedDocumentIds = normalizeRetrievedDocumentIds( + artifact, + options.responseIdPattern ?? /[A-Za-z0-9_.:/-]+/g, + ) + const cutoffs = normalizeCutoffs(options.cutoffs ?? [1, 3, 5, 10]) + const dimensions: Record = { + expected_count: payload.expectedDocumentIds.length, + returned_count: rankedDocumentIds.length, + } + for (const cutoff of cutoffs) { + Object.assign( + dimensions, + retrievalMetricsAtCutoff({ + rankedDocumentIds, + expectedScores: payload.expectedScores, + cutoff, + }), + ) + } + const primaryMetric = options.primaryMetric ?? 'ndcg@10' + const passMetric = options.passMetric ?? 'hit@10' + const score = dimensions[primaryMetric] ?? dimensions[`ndcg@${cutoffs[cutoffs.length - 1]}`] ?? 0 + const passScore = + dimensions[passMetric] ?? dimensions[`hit@${cutoffs[cutoffs.length - 1]}`] ?? score + return { + score, + passed: passScore >= (options.passThreshold ?? 1), + dimensions, + raw: { + rankedDocumentIds, + expectedDocumentIds: payload.expectedDocumentIds, + expectedScores: payload.expectedScores, + }, + } +} + +export function normalizeRetrievedDocumentIds( + artifact: StandardRetrievalArtifact, + responseIdPattern: RegExp = /[A-Za-z0-9_.:/-]+/g, +): string[] { + if (typeof artifact === 'string') { + return uniqueOrdered((artifact.match(responseIdPattern) ?? []).map((value) => value.trim())) + } + if (Array.isArray(artifact)) { + const entries = artifact as readonly (string | StandardRetrievalResult)[] + return uniqueOrdered( + entries.flatMap((entry) => { + if (typeof entry === 'string') return [entry] + return [entry.documentId ?? entry.docId ?? entry.id ?? ''] + }), + ) + } + const objectArtifact = artifact as Exclude + return uniqueOrdered( + [ + ...(objectArtifact.documentIds ?? []), + ...(objectArtifact.ids ?? []), + ...(objectArtifact.results ?? []).map( + (entry) => entry.documentId ?? entry.docId ?? entry.id ?? '', + ), + ].map((value) => value.trim()), + ) +} + +export function retrievalMetricsAtCutoff(input: { + rankedDocumentIds: readonly string[] + expectedScores: Record + cutoff: number +}): Record { + const top = input.rankedDocumentIds.slice(0, input.cutoff) + const relevantIds = Object.keys(input.expectedScores).filter( + (id) => input.expectedScores[id]! > 0, + ) + const relevant = new Set(relevantIds) + const hits = top.filter((id) => relevant.has(id)).length + const firstRelevantRank = top.findIndex((id) => relevant.has(id)) + const prefix = `@${input.cutoff}` + return { + [`hit${prefix}`]: hits > 0 ? 1 : 0, + [`recall${prefix}`]: relevant.size === 0 ? 1 : hits / relevant.size, + [`precision${prefix}`]: hits / input.cutoff, + [`mrr${prefix}`]: firstRelevantRank === -1 ? 0 : 1 / (firstRelevantRank + 1), + [`ndcg${prefix}`]: ndcgAt(input.rankedDocumentIds, input.expectedScores, input.cutoff), + } +} + +function stringField(row: Record, key: string): string | undefined { + const value = row[key] + return typeof value === 'string' ? value : undefined +} + +function stripKnown( + row: Record, + keys: readonly string[], +): Record { + const known = new Set(keys) + return Object.fromEntries(Object.entries(row).filter(([key]) => !known.has(key))) +} + +function normalizeCutoffs(cutoffs: readonly number[]): number[] { + return [ + ...new Set(cutoffs.map((cutoff) => Math.trunc(cutoff)).filter((cutoff) => cutoff > 0)), + ].sort((a, b) => a - b) +} + +function uniqueOrdered(values: readonly string[]): string[] { + const seen = new Set() + const out: string[] = [] + for (const value of values) { + const trimmed = value.trim() + if (!trimmed || seen.has(trimmed)) continue + seen.add(trimmed) + out.push(trimmed) + } + return out +} + +function ndcgAt( + rankedDocumentIds: readonly string[], + expectedScores: Record, + cutoff: number, +): number { + const dcg = rankedDocumentIds + .slice(0, cutoff) + .reduce( + (sum, documentId, index) => sum + discountedGain(expectedScores[documentId] ?? 0, index), + 0, + ) + const ideal = Object.values(expectedScores) + .filter((score) => score > 0) + .sort((a, b) => b - a) + .slice(0, cutoff) + .reduce((sum, score, index) => sum + discountedGain(score, index), 0) + return ideal === 0 ? 1 : dcg / ideal +} + +function discountedGain(relevance: number, zeroBasedRank: number): number { + if (relevance <= 0) return 0 + return (2 ** relevance - 1) / Math.log2(zeroBasedRank + 2) +} diff --git a/src/benchmarks/types.ts b/src/benchmarks/types.ts index 1cdd6163..7ec1069d 100644 --- a/src/benchmarks/types.ts +++ b/src/benchmarks/types.ts @@ -4,14 +4,50 @@ * `BenchmarkAdapter` plus its own typed `DatasetItem` shape. */ +import type { DispatchContext, Scenario } from '../campaign/types' import type { RunSplitTag } from '../run-record' +export type BenchmarkTaskKind = + | 'retrieval' + | 'rag-answer' + | 'hallucination' + | 'kb-improvement' + | 'routing' + | 'custom' + +export type BenchmarkFamily = + | 'beir' + | 'mteb-retrieval' + | 'msmarco' + | 'trec-dl' + | 'miracl' + | 'lotte' + | 'bright' + | 'crag' + | 'hotpotqa' + | 'kilt' + | 'ragtruth' + | 'faithbench' + | 'first-party' + | 'custom' + export interface BenchmarkDatasetItem { /** Stable dataset-local item id (used for split assignment + paper * references). Unique within a benchmark. */ id: string /** Free-form payload. Each benchmark defines its own shape. */ payload: TPayload + /** Optional precomputed split. When absent, adapters use assignSplit(id). */ + split?: RunSplitTag + /** Benchmark family this row came from, e.g. `beir` or `crag`. */ + family?: BenchmarkFamily | string + /** Benchmark-local task kind, e.g. retrieval vs answer quality. */ + taskKind?: BenchmarkTaskKind | string + /** Slice labels such as language, domain, freshness, multihop, long-tail. */ + tags?: string[] + /** Dataset provenance, version, URL, or license notes. */ + source?: BenchmarkSource + metadata?: Record } export interface BenchmarkEvaluation { @@ -19,9 +55,22 @@ export interface BenchmarkEvaluation { * benchmarks use 0/1; partial-credit benchmarks may return * fractional values. */ score: number + /** Optional pass/fail projection. Defaults to `score > 0` when absent. */ + passed?: boolean + /** Numeric sub-metrics. These become report dimensions. */ + dimensions?: Record /** Optional bag of raw scoring signals — e.g. parsed numeric * answer, regex match, judge sub-scores. */ raw: Record + notes?: string +} + +export interface BenchmarkSource { + name?: string + url?: string + version?: string + license?: string + citation?: string } /** Common signature implemented by every adapter under `src/benchmarks/*`. */ @@ -29,20 +78,42 @@ export interface BenchmarkEvaluation { // downstream type-narrowing extensions (a richer `BenchmarkDatasetItem` // subclass that adds e.g. provenance metadata) but is intentionally // unused here. `noUnusedLocals` requires the leading underscore. -export interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown> { +export interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown, TArtifact = string> { + /** Stable benchmark id such as `beir/nfcorpus` or `crag/smoke`. */ + id?: string + family?: BenchmarkFamily | string + taskKind?: BenchmarkTaskKind | string + description?: string + source?: BenchmarkSource + defaultMetric?: string /** Load the dataset for the given split. May hit the network on * first call but should be cache-friendly. Adapters that don't * ship the dataset itself MUST throw a clearly-marked error * pointing the caller at the loader script. */ loadDataset(split: RunSplitTag): Promise[]> /** Score a single response. Pure with respect to the inputs. */ - evaluate(item: BenchmarkDatasetItem, response: string): Promise + evaluate(item: BenchmarkDatasetItem, artifact: TArtifact): Promise /** Deterministic split assignment via item id hashing. The * fraction of items in each split is implementation-defined but * MUST be stable across processes and platforms. */ assignSplit(itemId: string): RunSplitTag } +export interface BenchmarkScenario extends Scenario { + kind: 'benchmark' + benchmarkId: string + family: BenchmarkFamily | string + taskKind: BenchmarkTaskKind | string + splitTag: RunSplitTag + item: BenchmarkDatasetItem +} + +export type BenchmarkResponder = (input: { + scenario: BenchmarkScenario + item: BenchmarkDatasetItem + context: DispatchContext +}) => Promise | TArtifact + // ── Deterministic split assignment ─────────────────────────────────── /** diff --git a/src/index.ts b/src/index.ts index 558634b7..5f797085 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1181,6 +1181,11 @@ export type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation, + BenchmarkFamily, + BenchmarkResponder, + BenchmarkScenario, + BenchmarkSource, + BenchmarkTaskKind, } from './benchmarks/types' export { BENCHMARK_SPLIT_SEED, diff --git a/tests/benchmarks.test.ts b/tests/benchmarks.test.ts index 651a9087..213bc616 100644 --- a/tests/benchmarks.test.ts +++ b/tests/benchmarks.test.ts @@ -5,8 +5,19 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest' import * as gsm8k from '../examples/benchmarks/gsm8k/index' import * as swebenchLite from '../examples/benchmarks/swebench-lite/index' +import { + buildStandardRetrievalItems, + calibrateBenchmarkMetric, + createRetrievalIdBenchmarkAdapter, + parseBeirCorpusJsonl, + parseBeirQueriesJsonl, + parseJsonlRows, + parseQrels, + runBenchmarkAdapter, +} from '../src/benchmarks' import * as routing from '../src/benchmarks/routing/index' import { BENCHMARK_SPLIT_SEED, deterministicSplit } from '../src/benchmarks/types' +import { inMemoryCampaignStorage } from '../src/campaign' describe('deterministicSplit', () => { it('always returns one of search|dev|holdout', () => { @@ -103,6 +114,138 @@ describe('routing benchmark', () => { it('assignSplit is stable per id', () => { expect(routing.assignSplit('chat_001')).toBe(routing.assignSplit('chat_001')) }) + + it('calibrates the benchmark metric with weak and strong artifacts', async () => { + const items = [ + ...(await routing.loadDataset('search')), + ...(await routing.loadDataset('dev')), + ...(await routing.loadDataset('holdout')), + ] + const item = items[0]! + const result = await calibrateBenchmarkMetric({ + adapter: new routing.RoutingAdapter(), + item, + weakArtifact: 'route=chat.reply', + strongArtifact: `route=${item.payload.route}`, + }) + expect(result.passed).toBe(true) + expect(result.weakScore).toBeLessThanOrEqual(0.3) + expect(result.strongScore).toBeGreaterThanOrEqual(0.7) + }) + + it('runs a benchmark adapter through campaign and writes report artifacts', async () => { + const storage = inMemoryCampaignStorage() + const result = await runBenchmarkAdapter({ + adapter: new routing.RoutingAdapter(), + splits: ['search', 'dev', 'holdout'], + runDir: '/runs/routing-smoke', + storage, + respond: ({ item, context }) => { + context.cost.observe(0.001, 'unit-test') + context.cost.observeTokens({ input: 3, output: 2 }) + return `route=${item.payload.route}` + }, + }) + + expect(result.scenarios.length).toBe(routing.ROUTING_DATASET.length) + expect(result.report.totalCells).toBe(routing.ROUTING_DATASET.length) + expect(result.report.score.mean).toBe(1) + expect( + result.report.splits.search.n + result.report.splits.dev.n + result.report.splits.holdout.n, + ).toBe(routing.ROUTING_DATASET.length) + expect(storage.read(result.reportJsonPath)).toContain('"benchmarkId": "first-party/routing"') + expect(storage.read(result.reportMarkdownPath)).toContain( + '# Benchmark Report: first-party/routing', + ) + }) +}) + +// ── standard retrieval formats — BEIR/MTEB/MS MARCO/TREC/MIRACL shape ── + +describe('standard retrieval benchmark formats', () => { + it('parses JSONL, BEIR corpus/query rows, and qrels', () => { + expect(parseJsonlRows('{"a":1}\n{"a":2}\n')).toEqual([{ a: 1 }, { a: 2 }]) + expect( + parseBeirCorpusJsonl( + [ + JSON.stringify({ _id: 'd1', title: 'Doc 1', text: 'Refunds are available.' }), + JSON.stringify({ _id: 'd2', text: 'Shipping takes two days.' }), + ].join('\n'), + ), + ).toMatchObject([{ id: 'd1', title: 'Doc 1' }, { id: 'd2' }]) + expect(parseBeirQueriesJsonl(JSON.stringify({ _id: 'q1', text: 'refund policy' }))).toEqual([ + { id: 'q1', text: 'refund policy', metadata: {} }, + ]) + expect(parseQrels('q1 0 d1 1\nq1 0 d2 0\nq2 d3 2')).toEqual([ + { queryId: 'q1', documentId: 'd1', score: 1 }, + { queryId: 'q1', documentId: 'd2', score: 0 }, + { queryId: 'q2', documentId: 'd3', score: 2 }, + ]) + }) + + it('builds retrieval benchmark items and scores returned document ids', async () => { + const queries = [{ id: 'q1', text: 'refund policy' }] + const qrels = [ + { queryId: 'q1', documentId: 'd1', score: 1 }, + { queryId: 'q1', documentId: 'd2', score: 1 }, + ] + const items = buildStandardRetrievalItems({ + benchmarkId: 'beir/smoke', + family: 'beir', + queries, + qrels, + splitOf: () => 'search', + }) + expect(items).toHaveLength(1) + expect(items[0]?.payload.expectedDocumentIds).toEqual(['d1', 'd2']) + expect(items[0]?.payload.corpus).toBeUndefined() + + const adapter = createRetrievalIdBenchmarkAdapter({ + benchmarkId: 'beir/smoke', + family: 'beir', + queries, + qrels, + cutoffs: [1, 2, 10], + splitOf: () => 'search', + }) + const [item] = await adapter.loadDataset('search') + const partial = await adapter.evaluate(item!, ['d1']) + expect(partial.score).toBeGreaterThan(0.6) + expect(partial.dimensions?.['recall@10']).toBe(0.5) + expect(partial.dimensions?.['precision@1']).toBe(1) + expect(partial.dimensions?.['hit@10']).toBe(1) + const complete = await adapter.evaluate(item!, [ + { documentId: 'd1' }, + { id: 'd2' }, + { id: 'distractor' }, + ]) + expect(complete.score).toBe(1) + expect(complete.dimensions?.['ndcg@10']).toBe(1) + expect(complete.dimensions?.['precision@10']).toBe(0.2) + expect(complete.passed).toBe(true) + }) + + it('runs retrieval items through campaign without undefined manifest fields', async () => { + const storage = inMemoryCampaignStorage() + const adapter = createRetrievalIdBenchmarkAdapter({ + benchmarkId: 'beir/manifest-smoke', + family: 'beir', + queries: [{ id: 'q1', text: 'refund policy' }], + qrels: [{ queryId: 'q1', documentId: 'd1', score: 1 }], + splitOf: () => 'search', + }) + const result = await runBenchmarkAdapter({ + adapter, + splits: ['search'], + runDir: '/runs/retrieval-smoke', + storage, + respond: () => ['d1'], + }) + + expect(result.report.totalCells).toBe(1) + expect(result.report.score.mean).toBe(1) + expect(storage.read(result.reportJsonPath)).toContain('"ndcg@10"') + }) }) // ── gsm8k — JSONL loader ────────────────────────────────────────────