From a95bb687a4deb3d0aa5e53a1e3c6ef90f0ea5bcf Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 7 Jul 2026 19:24:18 -0600 Subject: [PATCH] test(benchmarks): add all-family smoke pack --- README.md | 9 +++ src/benchmarks/index.ts | 145 +++++++++++++++++++++++++++++++++++++++ tests/benchmarks.test.ts | 29 ++++++++ 3 files changed, 183 insertions(+) diff --git a/README.md b/README.md index e8d27b7..0afd653 100644 --- a/README.md +++ b/README.md @@ -142,11 +142,19 @@ The module also exports `INDUSTRY_RAG_BENCHMARKS`, a compact manifest for BEIR, ```ts import { + buildIndustryRagBenchmarkSmokeCases, buildRetrievalBenchmarkCasesFromQrels, parseKnowledgeBenchmarkQrels, + respondToIndustryRagBenchmarkSmokeCase, runKnowledgeBenchmarkSuite, } from '@tangle-network/agent-knowledge/benchmarks' +await runKnowledgeBenchmarkSuite({ + cases: buildIndustryRagBenchmarkSmokeCases(), + runDir: '.agent-knowledge/benchmark-runs/industry-smoke', + respond: respondToIndustryRagBenchmarkSmokeCase, +}) + const cases = buildRetrievalBenchmarkCasesFromQrels({ benchmarkId: 'beir/nfcorpus', family: 'beir', @@ -170,6 +178,7 @@ console.log(result.report.score.mean) ``` Use `buildRetrievalBenchmarkCasesFromQrels()` for qrels-backed retrieval datasets. +The smoke pack proves that every declared benchmark family is wired through the runner; full BEIR, MTEB, MS MARCO, TREC DL, MIRACL, LoTTE, BRIGHT, CRAG, HotpotQA, KILT, RAGTruth, and FaithBench runs should pass real dataset rows through the same case shapes. Use `KnowledgeAnswerBenchmarkCase` for CRAG/HotpotQA/KILT-style answer checks and RAGTruth/FaithBench-style hallucination checks by encoding required claims, forbidden claims, and expected source IDs. Use `taskKind: 'kb-improvement'` when the artifact is candidate KB text produced by `improveKnowledgeBase()`. diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index daba917..fab70af 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -315,6 +315,95 @@ export const INDUSTRY_RAG_BENCHMARKS: readonly KnowledgeBenchmarkSpec[] = [ }, ] +export function buildIndustryRagBenchmarkSmokeCases( + specs: readonly KnowledgeBenchmarkSpec[] = INDUSTRY_RAG_BENCHMARKS, +): KnowledgeBenchmarkCase[] { + return specs.map((spec) => { + const source = { + name: spec.id, + version: 'smoke', + } + const split = spec.taskKind === 'retrieval' ? 'search' : 'holdout' + const tags = unique(['industry-smoke', spec.id, spec.family, spec.taskKind]) + if (spec.taskKind === 'retrieval') { + return { + id: `${spec.id}/smoke:q1`, + family: spec.family, + taskKind: 'retrieval', + split, + tags, + source, + query: `${spec.id} smoke retrieval query`, + expected: [{ kind: 'page', pageId: `${spec.id}:doc-1` }], + k: 5, + metadata: { + adapter: spec.adapter, + primaryMetrics: spec.primaryMetrics, + }, + } + } + + return { + id: `${spec.id}/smoke:q1`, + family: spec.family, + taskKind: spec.taskKind, + split, + tags, + source, + prompt: `${spec.id} smoke benchmark prompt`, + requiredClaims: [ + { + id: `${spec.id}:required`, + anyOf: [`${spec.id} supported answer`], + }, + ], + forbiddenClaims: [ + { + id: `${spec.id}:unsupported`, + anyOf: [`${spec.id} unsupported claim`], + }, + ], + expectedSourceIds: [`${spec.id}:source-1`], + referenceAnswer: `${spec.id} supported answer`, + metadata: { + adapter: spec.adapter, + primaryMetrics: spec.primaryMetrics, + }, + } + }) +} + +export function respondToIndustryRagBenchmarkSmokeCase(input: { + case: KnowledgeBenchmarkCase +}): KnowledgeBenchmarkArtifact { + const testCase = input.case + if (testCase.taskKind === 'retrieval') { + const expected = Array.isArray(testCase.expected) ? testCase.expected[0] : testCase.expected + const hit = hitForExpectedTarget(expected, testCase.id) + return { + hits: [hit], + costUsd: 0.001, + durationMs: 1, + metadata: { + smoke: true, + }, + } + } + + return { + answer: (testCase.requiredClaims ?? []) + .map((claim) => claim.anyOf[0]) + .filter((fragment): fragment is string => Boolean(fragment)) + .join(' '), + citedSourceIds: testCase.expectedSourceIds ?? [], + costUsd: 0.001, + durationMs: 1, + metadata: { + smoke: true, + }, + } +} + export function parseKnowledgeBenchmarkJsonl(text: string): T[] { return text .split(/\r?\n/) @@ -648,6 +737,62 @@ function defaultDocumentTarget( } } +function hitForExpectedTarget( + expected: RetrievalGoldTarget | undefined, + fallbackId: string, +): RetrievedKnowledgeHit { + if (!expected) { + return { + pageId: fallbackId, + path: `${fallbackId}.md`, + rank: 1, + } + } + switch (expected.kind) { + case 'page': + return { + pageId: expected.pageId, + path: `${expected.pageId}.md`, + rank: 1, + } + case 'page-path': + return { + pageId: expected.path, + path: expected.path, + rank: 1, + } + case 'source': + return { + pageId: expected.sourceId, + path: `${expected.sourceId}.md`, + sourceIds: [expected.sourceId], + rank: 1, + } + case 'source-anchor': + return { + pageId: expected.sourceId, + path: `${expected.sourceId}.md`, + sourceIds: [expected.sourceId], + sourceSpans: [{ sourceId: expected.sourceId, anchorId: expected.anchorId }], + rank: 1, + } + case 'source-span': + return { + pageId: expected.sourceId, + path: `${expected.sourceId}.md`, + sourceIds: [expected.sourceId], + sourceSpans: [ + { + sourceId: expected.sourceId, + charStart: expected.charStart, + charEnd: expected.charEnd, + }, + ], + rank: 1, + } + } +} + function scoreClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { let matched = 0 let matchedWeight = 0 diff --git a/tests/benchmarks.test.ts b/tests/benchmarks.test.ts index a972421..4d43786 100644 --- a/tests/benchmarks.test.ts +++ b/tests/benchmarks.test.ts @@ -1,11 +1,13 @@ import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' import { + buildIndustryRagBenchmarkSmokeCases, buildRetrievalBenchmarkCasesFromQrels, INDUSTRY_RAG_BENCHMARKS, type KnowledgeAnswerBenchmarkCase, parseKnowledgeBenchmarkJsonl, parseKnowledgeBenchmarkQrels, + respondToIndustryRagBenchmarkSmokeCase, runKnowledgeBenchmarkSuite, scoreKnowledgeBenchmarkArtifact, } from '../src/benchmarks/index' @@ -134,4 +136,31 @@ describe('knowledge benchmark adapters', () => { expect(ids.has(id)).toBe(true) } }) + + it('runs one persisted benchmark cell for every declared industry family', async () => { + const storage = inMemoryCampaignStorage() + const cases = buildIndustryRagBenchmarkSmokeCases() + const result = await runKnowledgeBenchmarkSuite({ + cases, + runDir: '/runs/knowledge-benchmark-family-smoke', + storage, + respond: respondToIndustryRagBenchmarkSmokeCase, + }) + + expect(cases).toHaveLength(INDUSTRY_RAG_BENCHMARKS.length) + expect(result.report.totalCases).toBe(INDUSTRY_RAG_BENCHMARKS.length) + expect(result.report.totalCells).toBe(INDUSTRY_RAG_BENCHMARKS.length) + expect(result.report.cellsFailed).toBe(0) + expect(result.report.score.mean).toBe(1) + expect(result.report.byTaskKind.retrieval?.n).toBe(7) + expect(result.report.byTaskKind['rag-answer']?.n).toBe(3) + expect(result.report.byTaskKind.hallucination?.n).toBe(2) + expect(result.report.byTaskKind['kb-improvement']?.n).toBe(1) + expect(result.report.totalCostUsd).toBeCloseTo(INDUSTRY_RAG_BENCHMARKS.length * 0.001) + for (const benchmark of INDUSTRY_RAG_BENCHMARKS) { + expect(result.report.byFamily[benchmark.family]?.n).toBeGreaterThanOrEqual(1) + } + expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 13') + expect(storage.read(result.reportMarkdownPath)).toContain('## Task Kinds') + }) })