test(benchmarks): add all-family smoke pack

README.md

@@ -142,11 +142,19 @@ The module also exports `INDUSTRY_RAG_BENCHMARKS`, a compact manifest for BEIR,
 
 ```ts
 import {
+  buildIndustryRagBenchmarkSmokeCases,
   buildRetrievalBenchmarkCasesFromQrels,
   parseKnowledgeBenchmarkQrels,
+  respondToIndustryRagBenchmarkSmokeCase,
   runKnowledgeBenchmarkSuite,
 } from '@tangle-network/agent-knowledge/benchmarks'
 
+await runKnowledgeBenchmarkSuite({
+  cases: buildIndustryRagBenchmarkSmokeCases(),
+  runDir: '.agent-knowledge/benchmark-runs/industry-smoke',
+  respond: respondToIndustryRagBenchmarkSmokeCase,
+})
+
 const cases = buildRetrievalBenchmarkCasesFromQrels({
   benchmarkId: 'beir/nfcorpus',
   family: 'beir',
@@ -170,6 +178,7 @@ console.log(result.report.score.mean)
 ```
 
 Use `buildRetrievalBenchmarkCasesFromQrels()` for qrels-backed retrieval datasets.
+The smoke pack proves that every declared benchmark family is wired through the runner; full BEIR, MTEB, MS MARCO, TREC DL, MIRACL, LoTTE, BRIGHT, CRAG, HotpotQA, KILT, RAGTruth, and FaithBench runs should pass real dataset rows through the same case shapes.
 Use `KnowledgeAnswerBenchmarkCase` for CRAG/HotpotQA/KILT-style answer checks and RAGTruth/FaithBench-style hallucination checks by encoding required claims, forbidden claims, and expected source IDs.
 Use `taskKind: 'kb-improvement'` when the artifact is candidate KB text produced by `improveKnowledgeBase()`.
 

src/benchmarks/index.ts

@@ -315,6 +315,95 @@ export const INDUSTRY_RAG_BENCHMARKS: readonly KnowledgeBenchmarkSpec[] = [
   },
 ]
 
+export function buildIndustryRagBenchmarkSmokeCases(
+  specs: readonly KnowledgeBenchmarkSpec[] = INDUSTRY_RAG_BENCHMARKS,
+): KnowledgeBenchmarkCase[] {
+  return specs.map((spec) => {
+    const source = {
+      name: spec.id,
+      version: 'smoke',
+    }
+    const split = spec.taskKind === 'retrieval' ? 'search' : 'holdout'
+    const tags = unique(['industry-smoke', spec.id, spec.family, spec.taskKind])
+    if (spec.taskKind === 'retrieval') {
+      return {
+        id: `${spec.id}/smoke:q1`,
+        family: spec.family,
+        taskKind: 'retrieval',
+        split,
+        tags,
+        source,
+        query: `${spec.id} smoke retrieval query`,
+        expected: [{ kind: 'page', pageId: `${spec.id}:doc-1` }],
+        k: 5,
+        metadata: {
+          adapter: spec.adapter,
+          primaryMetrics: spec.primaryMetrics,
+        },
+      }
+    }
+
+    return {
+      id: `${spec.id}/smoke:q1`,
+      family: spec.family,
+      taskKind: spec.taskKind,
+      split,
+      tags,
+      source,
+      prompt: `${spec.id} smoke benchmark prompt`,
+      requiredClaims: [
+        {
+          id: `${spec.id}:required`,
+          anyOf: [`${spec.id} supported answer`],
+        },
+      ],
+      forbiddenClaims: [
+        {
+          id: `${spec.id}:unsupported`,
+          anyOf: [`${spec.id} unsupported claim`],
+        },
+      ],
+      expectedSourceIds: [`${spec.id}:source-1`],
+      referenceAnswer: `${spec.id} supported answer`,
+      metadata: {
+        adapter: spec.adapter,
+        primaryMetrics: spec.primaryMetrics,
+      },
+    }
+  })
+}
+
+export function respondToIndustryRagBenchmarkSmokeCase(input: {
+  case: KnowledgeBenchmarkCase
+}): KnowledgeBenchmarkArtifact {
+  const testCase = input.case
+  if (testCase.taskKind === 'retrieval') {
+    const expected = Array.isArray(testCase.expected) ? testCase.expected[0] : testCase.expected
+    const hit = hitForExpectedTarget(expected, testCase.id)
+    return {
+      hits: [hit],
+      costUsd: 0.001,
+      durationMs: 1,
+      metadata: {
+        smoke: true,
+      },
+    }
+  }
+
+  return {
+    answer: (testCase.requiredClaims ?? [])
+      .map((claim) => claim.anyOf[0])
+      .filter((fragment): fragment is string => Boolean(fragment))
+      .join(' '),
+    citedSourceIds: testCase.expectedSourceIds ?? [],
+    costUsd: 0.001,
+    durationMs: 1,
+    metadata: {
+      smoke: true,
+    },
+  }
+}
+
 export function parseKnowledgeBenchmarkJsonl<T = unknown>(text: string): T[] {
   return text
     .split(/\r?\n/)
@@ -648,6 +737,62 @@ function defaultDocumentTarget(
   }
 }
 
+function hitForExpectedTarget(
+  expected: RetrievalGoldTarget | undefined,
+  fallbackId: string,
+): RetrievedKnowledgeHit {
+  if (!expected) {
+    return {
+      pageId: fallbackId,
+      path: `${fallbackId}.md`,
+      rank: 1,
+    }
+  }
+  switch (expected.kind) {
+    case 'page':
+      return {
+        pageId: expected.pageId,
+        path: `${expected.pageId}.md`,
+        rank: 1,
+      }
+    case 'page-path':
+      return {
+        pageId: expected.path,
+        path: expected.path,
+        rank: 1,
+      }
+    case 'source':
+      return {
+        pageId: expected.sourceId,
+        path: `${expected.sourceId}.md`,
+        sourceIds: [expected.sourceId],
+        rank: 1,
+      }
+    case 'source-anchor':
+      return {
+        pageId: expected.sourceId,
+        path: `${expected.sourceId}.md`,
+        sourceIds: [expected.sourceId],
+        sourceSpans: [{ sourceId: expected.sourceId, anchorId: expected.anchorId }],
+        rank: 1,
+      }
+    case 'source-span':
+      return {
+        pageId: expected.sourceId,
+        path: `${expected.sourceId}.md`,
+        sourceIds: [expected.sourceId],
+        sourceSpans: [
+          {
+            sourceId: expected.sourceId,
+            charStart: expected.charStart,
+            charEnd: expected.charEnd,
+          },
+        ],
+        rank: 1,
+      }
+  }
+}
+
 function scoreClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) {
   let matched = 0
   let matchedWeight = 0

tests/benchmarks.test.ts

@@ -1,11 +1,13 @@
 import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign'
 import { describe, expect, it } from 'vitest'
 import {
+  buildIndustryRagBenchmarkSmokeCases,
   buildRetrievalBenchmarkCasesFromQrels,
   INDUSTRY_RAG_BENCHMARKS,
   type KnowledgeAnswerBenchmarkCase,
   parseKnowledgeBenchmarkJsonl,
   parseKnowledgeBenchmarkQrels,
+  respondToIndustryRagBenchmarkSmokeCase,
   runKnowledgeBenchmarkSuite,
   scoreKnowledgeBenchmarkArtifact,
 } from '../src/benchmarks/index'
@@ -134,4 +136,31 @@ describe('knowledge benchmark adapters', () => {
       expect(ids.has(id)).toBe(true)
     }
   })
+
+  it('runs one persisted benchmark cell for every declared industry family', async () => {
+    const storage = inMemoryCampaignStorage()
+    const cases = buildIndustryRagBenchmarkSmokeCases()
+    const result = await runKnowledgeBenchmarkSuite({
+      cases,
+      runDir: '/runs/knowledge-benchmark-family-smoke',
+      storage,
+      respond: respondToIndustryRagBenchmarkSmokeCase,
+    })
+
+    expect(cases).toHaveLength(INDUSTRY_RAG_BENCHMARKS.length)
+    expect(result.report.totalCases).toBe(INDUSTRY_RAG_BENCHMARKS.length)
+    expect(result.report.totalCells).toBe(INDUSTRY_RAG_BENCHMARKS.length)
+    expect(result.report.cellsFailed).toBe(0)
+    expect(result.report.score.mean).toBe(1)
+    expect(result.report.byTaskKind.retrieval?.n).toBe(7)
+    expect(result.report.byTaskKind['rag-answer']?.n).toBe(3)
+    expect(result.report.byTaskKind.hallucination?.n).toBe(2)
+    expect(result.report.byTaskKind['kb-improvement']?.n).toBe(1)
+    expect(result.report.totalCostUsd).toBeCloseTo(INDUSTRY_RAG_BENCHMARKS.length * 0.001)
+    for (const benchmark of INDUSTRY_RAG_BENCHMARKS) {
+      expect(result.report.byFamily[benchmark.family]?.n).toBeGreaterThanOrEqual(1)
+    }
+    expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 13')
+    expect(storage.read(result.reportMarkdownPath)).toContain('## Task Kinds')
+  })
 })