feat(benchmarks): add unified benchmark runner

README.md

@@ -136,7 +136,7 @@ Each example: `README.md` + a single `index.ts` runnable via `pnpm tsx`. Prints
 | `…/matrix` | `runAgentMatrix` — an N-axis cartesian over caller-supplied substrate values, per-axis pass/score/cost/duration |
 | `…/multishot` | N-shot persona × shot matrix runner (`runMultishot` / `runMultishotMatrix`) |
 | `…/wire` | The cross-language HTTP/RPC server + Zod schemas (the source-of-truth protocol the Python client speaks) + the built-in rubric registry |
-| `…/benchmarks` | `BenchmarkAdapter` contract + `deterministicSplit` + the bundled `routing` reference benchmark |
+| `…/benchmarks` | `BenchmarkAdapter` contract, `runBenchmarkAdapter`, `calibrateBenchmarkMetric`, standard retrieval parsers + ranked retrieval metrics, `deterministicSplit`, and the bundled `routing` reference benchmark |
 
 **Specialized surfaces** (subpath-only): `…/prm` (process-reward grading + best-of-N), `…/meta-eval` (judge calibration + the deployment-outcome store), `…/belief-state` (decision-point extraction + selective-policy reports), `…/pipelines` (trace-diagnostic views: budget breach, failure cluster, stuck loop, …), `…/governance` (EU AI Act / NIST AI RMF / SOC2 reports), `…/knowledge` (knowledge-readiness gating before a run), `…/builder-eval` (code-generator three-layer eval), `…/storyboard` (trace → watchable replay), `…/authenticity` (anti-Goodhart "real or convincing BS" scorer over produced files), `…/workflow` (workflow-trace eval + partner export), `…/telemetry` (Workers-safe telemetry client), `…/testing` (test-only reset helpers).
 

examples/benchmarks/README.md

@@ -26,6 +26,46 @@ assignSplit(itemId: string): 'search' | 'dev' | 'holdout'
 
 `assignSplit` uses `deterministicSplit(itemId, BENCHMARK_SPLIT_SEED)` — same item gets the same split everywhere. Don't change the seed; it's load-bearing for reproducibility.
 
+## Running a benchmark
+
+Use `runBenchmarkAdapter` when you want a campaign-backed run with resumability, traces, cost, latency, split reporting, and persisted report artifacts.
+
+```ts
+import { runBenchmarkAdapter, routing } from '@tangle-network/agent-eval/benchmarks'
+
+const result = await runBenchmarkAdapter({
+  adapter: new routing.RoutingAdapter(),
+  runDir: 'routing-smoke',
+  respond: async ({ item, context }) => {
+    const route = await callRouter(item.payload.prompt)
+    context.cost.observe(0.001, 'router')
+    return route
+  },
+})
+
+console.log(result.report.score.mean)
+console.log(result.reportMarkdownPath)
+```
+
+Before treating a benchmark metric as real evidence, run `calibrateBenchmarkMetric` with an intentionally weak and intentionally strong artifact.
+The default pass condition is weak score at most `0.3`, strong score at least `0.7`, and gap at least `0.4`.
+
+## Standard retrieval formats
+
+`@tangle-network/agent-eval/benchmarks` also exports dependency-free helpers for BEIR/MTEB/MS MARCO/TREC/MIRACL-style files:
+
+- `parseBeirCorpusJsonl`
+- `parseBeirQueriesJsonl`
+- `parseQrels`
+- `buildStandardRetrievalItems`
+- `createRetrievalIdBenchmarkAdapter`
+- `evaluateStandardRetrieval`
+
+These helpers normalize public retrieval datasets into the same `BenchmarkDatasetItem` shape.
+The retrieval evaluator accepts ranked document IDs and reports nDCG@k, recall@k, precision@k, MRR@k, and hit@k.
+It does not copy the full corpus into every query payload unless `includeCorpusInPayload` is explicitly set.
+Domain packages such as `agent-knowledge` can then map those items into their own RAG scenario types instead of re-parsing every public benchmark.
+
 ## Adding a new benchmark
 
 1. Create `examples/benchmarks/<your-benchmark>/index.ts`.

src/benchmarks/calibration.ts

@@ -0,0 +1,60 @@
+import type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation } from './types'
+
+export interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {
+  adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>
+  item: BenchmarkDatasetItem<TPayload>
+  weakArtifact: TArtifact
+  strongArtifact: TArtifact
+  maxWeakScore?: number
+  minStrongScore?: number
+  minGap?: number
+}
+
+export interface BenchmarkMetricCalibrationResult {
+  passed: boolean
+  weak: BenchmarkEvaluation
+  strong: BenchmarkEvaluation
+  weakScore: number
+  strongScore: number
+  gap: number
+  reasons: string[]
+}
+
+export async function calibrateBenchmarkMetric<TPayload = unknown, TArtifact = string>(
+  options: BenchmarkMetricCalibrationOptions<TPayload, TArtifact>,
+): Promise<BenchmarkMetricCalibrationResult> {
+  const weak = await options.adapter.evaluate(options.item, options.weakArtifact)
+  const strong = await options.adapter.evaluate(options.item, options.strongArtifact)
+  const weakScore = clamp01(weak.score)
+  const strongScore = clamp01(strong.score)
+  const maxWeakScore = options.maxWeakScore ?? 0.3
+  const minStrongScore = options.minStrongScore ?? 0.7
+  const minGap = options.minGap ?? 0.4
+  const gap = strongScore - weakScore
+  const reasons = [
+    weakScore <= maxWeakScore
+      ? undefined
+      : `weak score ${weakScore.toFixed(3)} exceeds max ${maxWeakScore.toFixed(3)}`,
+    strongScore >= minStrongScore
+      ? undefined
+      : `strong score ${strongScore.toFixed(3)} below min ${minStrongScore.toFixed(3)}`,
+    gap >= minGap ? undefined : `gap ${gap.toFixed(3)} below min ${minGap.toFixed(3)}`,
+  ].filter((reason): reason is string => Boolean(reason))
+
+  return {
+    passed: reasons.length === 0,
+    weak,
+    strong,
+    weakScore,
+    strongScore,
+    gap,
+    reasons,
+  }
+}
+
+function clamp01(value: number): number {
+  if (!Number.isFinite(value)) return 0
+  if (value < 0) return 0
+  if (value > 1) return 1
+  return value
+}

src/benchmarks/index.ts

@@ -3,6 +3,9 @@
  *
  * Core surface (exported here):
  *   - The `BenchmarkAdapter` contract.
+ *   - `runBenchmarkAdapter` for campaign-backed benchmark execution.
+ *   - `calibrateBenchmarkMetric` for weak/strong metric checks.
+ *   - Standard retrieval parsers for BEIR/MTEB/MS MARCO/TREC/MIRACL-style files.
  *   - `deterministicSplit` + `BENCHMARK_SPLIT_SEED` for split assignment.
  *   - `routing` — synthetic 16-task router benchmark. The only novel
  *     benchmark we built; ships in the package.
@@ -18,10 +21,51 @@
  * entry — every team will configure them differently.
  */
 
+export {
+  type BenchmarkMetricCalibrationOptions,
+  type BenchmarkMetricCalibrationResult,
+  calibrateBenchmarkMetric,
+} from './calibration'
 export * as routing from './routing/index'
+export {
+  type BenchmarkDistribution,
+  type BenchmarkReport,
+  type BenchmarkRunOptions,
+  type BenchmarkRunResult,
+  type BenchmarkSliceSummary,
+  renderBenchmarkReportMarkdown,
+  runBenchmarkAdapter,
+  summarizeBenchmarkCampaign,
+} from './runner'
+export {
+  type BuildStandardRetrievalItemsOptions,
+  buildStandardRetrievalItems,
+  createRetrievalIdBenchmarkAdapter,
+  evaluateStandardRetrieval,
+  normalizeRetrievedDocumentIds,
+  parseBeirCorpusJsonl,
+  parseBeirQueriesJsonl,
+  parseJsonlRows,
+  parseQrels,
+  parseTsvRows,
+  type RetrievalIdAdapterOptions,
+  retrievalMetricsAtCutoff,
+  type StandardRetrievalArtifact,
+  type StandardRetrievalDocument,
+  type StandardRetrievalEvaluationOptions,
+  type StandardRetrievalPayload,
+  type StandardRetrievalQrel,
+  type StandardRetrievalQuery,
+  type StandardRetrievalResult,
+} from './standard-formats'
 export type {
   BenchmarkAdapter,
   BenchmarkDatasetItem,
   BenchmarkEvaluation,
+  BenchmarkFamily,
+  BenchmarkResponder,
+  BenchmarkScenario,
+  BenchmarkSource,
+  BenchmarkTaskKind,
 } from './types'
 export { BENCHMARK_SPLIT_SEED, deterministicSplit } from './types'

src/benchmarks/routing/index.ts

@@ -20,6 +20,12 @@ export type RoutingPayload = RoutingItem
 export type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>
 
 class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
+  readonly id = 'first-party/routing'
+  readonly family = 'first-party'
+  readonly taskKind = 'routing'
+  readonly description = 'Synthetic fixed-route classification smoke benchmark'
+  readonly defaultMetric = 'route_exact_match'
+
   async loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]> {
     return ROUTING_DATASET.map((item) => ({ id: item.id, payload: item })).filter(
       (it) => assignSplitImpl(it.id) === split,