diff --git a/README.md b/README.md index 56853f6..e8d27b7 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ This package turns raw sources and generated markdown knowledge into a versionab - [Start here](#start-here) — pick CLI vs programmatic - [CLI](#cli) — `init` → `source-add` → `index` → `search` → `lint` - [Design](#design) — the invariants (immutable sources, cited claims, deterministic graph) +- [Benchmark harness](#benchmark-harness) — BEIR/MTEB/qrels, RAG answer, hallucination, KB-improvement cases - [Agent-Eval integration](#agent-eval-integration) — retrieval eval + readiness bundles + release reports - [Memory adapters](#memory-adapters) — generic memory contract + Neo4j Agent Memory bridge - [Research loop](#research-loop) — `runKnowledgeResearchLoop` + control-loop adapter @@ -37,6 +38,7 @@ Two ways in, depending on what you're doing: - *"Expose the lower-level RAG lifecycle phases"* → `runRagKnowledgeImprovementLoop` in the [Agent-Eval integration](#agent-eval-integration) section. It exposes retrieval tuning, gap diagnosis, knowledge acquisition/update, answer-quality checks, and promotion as one typed lifecycle. - *"Evaluate RAG answers or a wiki/KB"* → `ragAnswerQualityJudge`, `createRagAnswerQualityHook`, and `scoreKnowledgeBaseIndex` in the [Agent-Eval integration](#agent-eval-integration) section. + - *"Run standard RAG/KB benchmarks"* → `runKnowledgeBenchmarkSuite` in the [Benchmark harness](#benchmark-harness) section. - *"Does this candidate KB actually improve task success?"* → run an [agent-eval improvement loop](#agent-eval-integration) over KB variants, then `knowledgeReleaseReport` for the promotion decision. - *"Keep live authorities fresh"* → [pluggable sources](#pluggable-knowledge-sources) + `detectChanges` → eval re-runs. @@ -133,6 +135,44 @@ The `/memory` subpath exports an optional memory adapter contract. Use it to bridge episodic or graph-native memory systems into the same source-grounded readiness/eval machinery without making `agent-knowledge` own the database. +## Benchmark Harness + +Use `runKnowledgeBenchmarkSuite()` when the product goal is to run a fixed RAG/KB benchmark pack and get one report across retrieval, answer quality, hallucination, and candidate-KB improvement cases. +The module also exports `INDUSTRY_RAG_BENCHMARKS`, a compact manifest for BEIR, MTEB retrieval, MS MARCO, TREC DL, MIRACL, LoTTE, BRIGHT, CRAG, HotpotQA, KILT, RAGTruth, FaithBench, and first-party KB-improvement tasks. + +```ts +import { + buildRetrievalBenchmarkCasesFromQrels, + parseKnowledgeBenchmarkQrels, + runKnowledgeBenchmarkSuite, +} from '@tangle-network/agent-knowledge/benchmarks' + +const cases = buildRetrievalBenchmarkCasesFromQrels({ + benchmarkId: 'beir/nfcorpus', + family: 'beir', + queries: [{ id: 'q1', text: 'aspirin heart attack prevention', split: 'holdout' }], + qrels: parseKnowledgeBenchmarkQrels('q1 0 src-aspirin 1'), + targetKind: 'source', + k: 10, +}) + +const result = await runKnowledgeBenchmarkSuite({ + cases, + runDir: '.agent-knowledge/benchmark-runs/beir-nfcorpus-smoke', + respond: async ({ case: testCase }) => { + if (testCase.taskKind !== 'retrieval') return { hits: [] } + const hits = await retrieveFromYourKb(testCase.query) + return { hits, costUsd: 0.001 } + }, +}) + +console.log(result.report.score.mean) +``` + +Use `buildRetrievalBenchmarkCasesFromQrels()` for qrels-backed retrieval datasets. +Use `KnowledgeAnswerBenchmarkCase` for CRAG/HotpotQA/KILT-style answer checks and RAGTruth/FaithBench-style hallucination checks by encoding required claims, forbidden claims, and expected source IDs. +Use `taskKind: 'kb-improvement'` when the artifact is candidate KB text produced by `improveKnowledgeBase()`. + ## Agent-Eval Integration Use `ragAnswerQualityJudge` or `createRagAnswerQualityHook` when the product already has answer traces and needs SOTA-style RAG scoring without rebuilding metrics. diff --git a/package.json b/package.json index 4433d5c..d6c4557 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,11 @@ "types": "./dist/autodata/index.d.ts", "import": "./dist/autodata/index.js", "default": "./dist/autodata/index.js" + }, + "./benchmarks": { + "types": "./dist/benchmarks/index.d.ts", + "import": "./dist/benchmarks/index.js", + "default": "./dist/benchmarks/index.js" } }, "bin": { diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts new file mode 100644 index 0000000..daba917 --- /dev/null +++ b/src/benchmarks/index.ts @@ -0,0 +1,806 @@ +import { join } from 'node:path' +import { + type CampaignResult, + type CampaignStorage, + type DispatchContext, + fsCampaignStorage, + type JudgeConfig, + type RunCampaignOptions, + runCampaign, + type Scenario, +} from '@tangle-network/agent-eval/campaign' +import { + type RetrievalEvalArtifact, + type RetrievalEvalScenario, + type RetrievalGoldTarget, + type RetrievedKnowledgeHit, + scoreRetrievalArtifact, +} from '../retrieval-eval' + +export type KnowledgeBenchmarkTaskKind = + | 'retrieval' + | 'rag-answer' + | 'hallucination' + | 'kb-improvement' + +export type KnowledgeBenchmarkFamily = + | 'beir' + | 'mteb-retrieval' + | 'msmarco' + | 'trec-dl' + | 'miracl' + | 'lotte' + | 'bright' + | 'crag' + | 'hotpotqa' + | 'kilt' + | 'ragtruth' + | 'faithbench' + | 'first-party' + | 'custom' + +export type KnowledgeBenchmarkSplit = 'search' | 'dev' | 'holdout' | string + +export interface KnowledgeBenchmarkSource { + name?: string + url?: string + version?: string + license?: string + citation?: string +} + +export interface KnowledgeBenchmarkSpec { + id: string + family: KnowledgeBenchmarkFamily + taskKind: KnowledgeBenchmarkTaskKind + primaryMetrics: readonly string[] + adapter: string + notes: string +} + +export interface KnowledgeBenchmarkCaseBase { + id: string + family: KnowledgeBenchmarkFamily | string + taskKind: KnowledgeBenchmarkTaskKind + split?: KnowledgeBenchmarkSplit + tags?: readonly string[] + source?: KnowledgeBenchmarkSource + metadata?: Record +} + +export interface KnowledgeRetrievalBenchmarkCase extends KnowledgeBenchmarkCaseBase { + taskKind: 'retrieval' + query: string + expected: RetrievalGoldTarget | readonly RetrievalGoldTarget[] + k?: number +} + +export interface KnowledgeClaimMatcher { + id: string + anyOf: readonly string[] + weight?: number +} + +export interface KnowledgeAnswerBenchmarkCase extends KnowledgeBenchmarkCaseBase { + taskKind: 'rag-answer' | 'hallucination' | 'kb-improvement' + prompt: string + requiredClaims?: readonly KnowledgeClaimMatcher[] + forbiddenClaims?: readonly KnowledgeClaimMatcher[] + expectedSourceIds?: readonly string[] + referenceAnswer?: string +} + +export type KnowledgeBenchmarkCase = KnowledgeRetrievalBenchmarkCase | KnowledgeAnswerBenchmarkCase + +export interface KnowledgeBenchmarkArtifact { + answer?: string + text?: string + hits?: readonly RetrievedKnowledgeHit[] + citedSourceIds?: readonly string[] + costUsd?: number + durationMs?: number + metadata?: Record +} + +export interface KnowledgeBenchmarkEvaluation { + score: number + passed: boolean + dimensions: Record + notes: string + raw: Record +} + +export interface KnowledgeBenchmarkScenario extends Scenario { + kind: 'knowledge-benchmark' + family: KnowledgeBenchmarkFamily | string + taskKind: KnowledgeBenchmarkTaskKind + splitTag: KnowledgeBenchmarkSplit + case: KnowledgeBenchmarkCase +} + +export type KnowledgeBenchmarkResponder = (input: { + case: KnowledgeBenchmarkCase + scenario: KnowledgeBenchmarkScenario + context: DispatchContext +}) => Promise | TArtifact + +export interface RunKnowledgeBenchmarkSuiteOptions { + cases: readonly KnowledgeBenchmarkCase[] + respond: KnowledgeBenchmarkResponder + runDir: string + splits?: readonly KnowledgeBenchmarkSplit[] + repo?: string + seed?: number + reps?: number + resumable?: boolean + costCeiling?: number + maxConcurrency?: number + dispatchTimeoutMs?: number + expectUsage?: 'assert' | 'warn' | 'off' + storage?: CampaignStorage + now?: () => Date +} + +export interface KnowledgeBenchmarkDistribution { + n: number + min: number + mean: number + median: number + p90: number + max: number +} + +export interface KnowledgeBenchmarkSliceSummary { + n: number + meanScore: number + passRate: number + score: KnowledgeBenchmarkDistribution +} + +export interface KnowledgeBenchmarkReport { + totalCases: number + totalCells: number + cellsFailed: number + cellsCached: number + totalCostUsd: number + bySplit: Record + byFamily: Record + byTaskKind: Record + dimensions: Record + score: KnowledgeBenchmarkDistribution +} + +export interface RunKnowledgeBenchmarkSuiteResult { + scenarios: readonly KnowledgeBenchmarkScenario[] + campaign: CampaignResult + report: KnowledgeBenchmarkReport + reportJsonPath: string + reportMarkdownPath: string +} + +export interface KnowledgeRetrievalBenchmarkQuery { + id: string + text: string + split?: KnowledgeBenchmarkSplit + tags?: readonly string[] + metadata?: Record +} + +export interface KnowledgeRetrievalBenchmarkQrel { + queryId: string + documentId: string + score: number +} + +export interface BuildRetrievalBenchmarkCasesFromQrelsOptions { + benchmarkId: string + family: KnowledgeBenchmarkFamily | string + queries: readonly KnowledgeRetrievalBenchmarkQuery[] + qrels: readonly KnowledgeRetrievalBenchmarkQrel[] + source?: KnowledgeBenchmarkSource + tags?: readonly string[] + k?: number + targetKind?: 'page' | 'page-path' | 'source' + documentTarget?: ( + documentId: string, + qrel: KnowledgeRetrievalBenchmarkQrel, + ) => RetrievalGoldTarget + splitOf?: (queryId: string) => KnowledgeBenchmarkSplit +} + +export const INDUSTRY_RAG_BENCHMARKS: readonly KnowledgeBenchmarkSpec[] = [ + { + id: 'beir', + family: 'beir', + taskKind: 'retrieval', + primaryMetrics: ['nDCG@10', 'Recall@100', 'MRR@10'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Classic zero-shot retrieval suites using query/corpus/qrels files.', + }, + { + id: 'mteb-retrieval', + family: 'mteb-retrieval', + taskKind: 'retrieval', + primaryMetrics: ['nDCG@10', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'MTEB retrieval task shape; same qrels bridge, different dataset provenance.', + }, + { + id: 'msmarco', + family: 'msmarco', + taskKind: 'retrieval', + primaryMetrics: ['MRR@10', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Passage retrieval and reranking smoke for web-style questions.', + }, + { + id: 'trec-dl', + family: 'trec-dl', + taskKind: 'retrieval', + primaryMetrics: ['nDCG@10', 'MAP', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Deep Learning Track judgments over MS MARCO-derived corpora.', + }, + { + id: 'miracl', + family: 'miracl', + taskKind: 'retrieval', + primaryMetrics: ['nDCG@10', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Multilingual retrieval; use language tags on cases.', + }, + { + id: 'lotte', + family: 'lotte', + taskKind: 'retrieval', + primaryMetrics: ['Success@5', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Long-tail search tasks; map collection/domain into tags.', + }, + { + id: 'bright', + family: 'bright', + taskKind: 'retrieval', + primaryMetrics: ['nDCG@10', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Reasoning-heavy retrieval; preserve domain tags for slice reporting.', + }, + { + id: 'crag', + family: 'crag', + taskKind: 'rag-answer', + primaryMetrics: ['claim_recall', 'citation_recall', 'hallucination_safe'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Answer quality and freshness cases; use required/forbidden claims plus citations.', + }, + { + id: 'hotpotqa', + family: 'hotpotqa', + taskKind: 'rag-answer', + primaryMetrics: ['claim_recall', 'citation_recall'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Multihop QA; encode each supporting fact as a required claim.', + }, + { + id: 'kilt', + family: 'kilt', + taskKind: 'rag-answer', + primaryMetrics: ['claim_recall', 'citation_recall'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Knowledge-intensive generation with provenance; encode expected pages/sources.', + }, + { + id: 'ragtruth', + family: 'ragtruth', + taskKind: 'hallucination', + primaryMetrics: ['hallucination_safe', 'forbidden_claim_rate'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Hallucination detection; encode hallucinated spans as forbidden claims.', + }, + { + id: 'faithbench', + family: 'faithbench', + taskKind: 'hallucination', + primaryMetrics: ['hallucination_safe', 'forbidden_claim_rate'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Faithfulness benchmark; score unsupported claims as forbidden claims.', + }, + { + id: 'first-party/kb-improvement', + family: 'first-party', + taskKind: 'kb-improvement', + primaryMetrics: ['claim_recall', 'hallucination_safe', 'score'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Project-owned candidate-KB validation; grade the produced KB text or answer bundle.', + }, +] + +export function parseKnowledgeBenchmarkJsonl(text: string): T[] { + return text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .map((line, index) => { + try { + return JSON.parse(line) as T + } catch (error) { + throw new Error(`invalid JSONL row ${index + 1}: ${(error as Error).message}`) + } + }) +} + +export function parseKnowledgeBenchmarkQrels(text: string): KnowledgeRetrievalBenchmarkQrel[] { + return text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .flatMap((line, index) => { + const parts = line.split(/\t|\s+/) + if (parts.length < 3) return [] + const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts + if (!queryId || !maybeZeroOrDocId || !maybeDocIdOrScore) return [] + if (queryId.toLowerCase() === 'qid' || queryId.toLowerCase() === 'query-id') return [] + const documentId = maybeScore === undefined ? maybeZeroOrDocId : maybeDocIdOrScore + const scoreText = maybeScore === undefined ? maybeDocIdOrScore : maybeScore + const score = Number(scoreText) + if (!documentId || !Number.isFinite(score)) { + throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`) + } + return [{ queryId, documentId, score }] + }) +} + +export function buildRetrievalBenchmarkCasesFromQrels( + options: BuildRetrievalBenchmarkCasesFromQrelsOptions, +): KnowledgeRetrievalBenchmarkCase[] { + const qrelsByQuery = new Map() + for (const qrel of options.qrels) { + if (qrel.score <= 0) continue + const list = qrelsByQuery.get(qrel.queryId) ?? [] + list.push(qrel) + qrelsByQuery.set(qrel.queryId, list) + } + + return options.queries.flatMap((query) => { + const qrels = qrelsByQuery.get(query.id) ?? [] + if (qrels.length === 0) return [] + const split = query.split ?? options.splitOf?.(query.id) + const expected = qrels.map((qrel) => + options.documentTarget + ? options.documentTarget(qrel.documentId, qrel) + : defaultDocumentTarget(qrel.documentId, options.targetKind ?? 'page'), + ) + return [ + compactObject({ + id: `${options.benchmarkId}:${query.id}`, + family: options.family, + taskKind: 'retrieval' as const, + query: query.text, + expected, + k: options.k, + split, + tags: unique([...(options.tags ?? []), ...(query.tags ?? []), ...(split ? [split] : [])]), + source: options.source, + metadata: query.metadata, + }) as KnowledgeRetrievalBenchmarkCase, + ] + }) +} + +export async function runKnowledgeBenchmarkSuite( + options: RunKnowledgeBenchmarkSuiteOptions, +): Promise> { + const storage = options.storage ?? fsCampaignStorage() + const scenarios = buildKnowledgeBenchmarkScenarios(options.cases, options.splits) + const dispatch: RunCampaignOptions['dispatch'] = async ( + scenario, + context, + ) => { + const artifact = await options.respond({ case: scenario.case, scenario, context }) + observeArtifactCost(context, artifact) + return artifact + } + const campaign = await runCampaign({ + scenarios, + dispatch, + dispatchRef: 'agent-knowledge:benchmark-suite', + judges: [knowledgeBenchmarkJudge()], + runDir: options.runDir, + repo: options.repo, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + costCeiling: options.costCeiling, + maxConcurrency: options.maxConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + expectUsage: options.expectUsage ?? 'off', + storage, + now: options.now, + }) + const report = summarizeKnowledgeBenchmarkCampaign({ scenarios, campaign }) + const reportJsonPath = join(campaign.runDir, 'knowledge-benchmark-report.json') + const reportMarkdownPath = join(campaign.runDir, 'knowledge-benchmark-report.md') + storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}\n`) + storage.write(reportMarkdownPath, renderKnowledgeBenchmarkReportMarkdown(report)) + return { + scenarios, + campaign, + report, + reportJsonPath, + reportMarkdownPath, + } +} + +export function renderKnowledgeBenchmarkReportMarkdown(report: KnowledgeBenchmarkReport): string { + return [ + '# Knowledge Benchmark Report', + '', + `- cases: ${report.totalCases}`, + `- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`, + `- cost: $${formatNumber(report.totalCostUsd)}`, + `- score: mean ${formatNumber(report.score.mean)}, median ${formatNumber(report.score.median)}, p90 ${formatNumber(report.score.p90)}, n=${report.score.n}`, + '', + '## Task Kinds', + '', + renderSliceTable(report.byTaskKind), + '', + '## Splits', + '', + renderSliceTable(report.bySplit), + '', + '## Dimensions', + '', + '| dimension | n | mean | p90 |', + '| --- | ---: | ---: | ---: |', + ...Object.entries(report.dimensions) + .sort(([a], [b]) => a.localeCompare(b)) + .map( + ([key, dist]) => + `| ${key} | ${dist.n} | ${formatNumber(dist.mean)} | ${formatNumber(dist.p90)} |`, + ), + '', + ].join('\n') +} + +export function buildKnowledgeBenchmarkScenarios( + cases: readonly KnowledgeBenchmarkCase[], + splits?: readonly KnowledgeBenchmarkSplit[], +): KnowledgeBenchmarkScenario[] { + const splitSet = splits ? new Set(splits) : null + return cases.flatMap((testCase) => { + const splitTag = testCase.split ?? 'dev' + if (splitSet && !splitSet.has(splitTag)) return [] + return [ + compactObject({ + id: testCase.id, + kind: 'knowledge-benchmark' as const, + family: testCase.family, + taskKind: testCase.taskKind, + splitTag, + tags: unique([splitTag, ...(testCase.tags ?? [])]), + case: compactObject(testCase), + }) as KnowledgeBenchmarkScenario, + ] + }) +} + +export function knowledgeBenchmarkJudge(): JudgeConfig< + TArtifact, + KnowledgeBenchmarkScenario +> { + return { + name: 'knowledge-benchmark', + dimensions: [ + { key: 'score', description: 'primary knowledge benchmark score' }, + { key: 'passed', description: '1 when the benchmark case passes' }, + { key: 'claim_recall', description: 'required claim coverage' }, + { key: 'citation_recall', description: 'expected citation/source coverage' }, + { key: 'hallucination_safe', description: '1 when no forbidden claim appears' }, + ], + appliesTo: (scenario) => scenario.kind === 'knowledge-benchmark', + score({ artifact, scenario }) { + const evaluation = scoreKnowledgeBenchmarkArtifact(scenario.case, artifact) + return { + dimensions: { + score: evaluation.score, + passed: evaluation.passed ? 1 : 0, + ...evaluation.dimensions, + }, + composite: evaluation.score, + notes: evaluation.notes, + } + }, + } +} + +export function scoreKnowledgeBenchmarkArtifact( + testCase: KnowledgeBenchmarkCase, + artifact: TArtifact, +): KnowledgeBenchmarkEvaluation { + if (testCase.taskKind === 'retrieval') { + const retrievalArtifact = normalizeRetrievalArtifact(testCase, artifact) + const metrics = scoreRetrievalArtifact(retrievalArtifact, retrievalScenarioForCase(testCase)) + return { + score: metrics.recall, + passed: metrics.recall >= 1, + dimensions: { + recall: metrics.recall, + mrr: metrics.mrr, + ndcg: metrics.ndcg, + precision_at_k: metrics.precisionAtK, + expected_count: metrics.expectedCount, + matched_count: metrics.matchedCount, + }, + notes: `matched ${metrics.matchedCount}/${metrics.expectedCount}; first_hit_rank=${metrics.firstHitRank ?? 'none'}`, + raw: { matchedTargetIds: metrics.matchedTargetIds }, + } + } + + const answerArtifact = artifact as KnowledgeBenchmarkArtifact + const text = answerArtifact.text ?? answerArtifact.answer ?? '' + const required = scoreClaims(text, testCase.requiredClaims ?? []) + const forbidden = scoreForbiddenClaims(text, testCase.forbiddenClaims ?? []) + const citation = scoreCitationRecall( + answerArtifact.citedSourceIds ?? [], + testCase.expectedSourceIds ?? [], + ) + const components = [ + required.totalWeight > 0 ? required.recall : undefined, + testCase.expectedSourceIds && testCase.expectedSourceIds.length > 0 ? citation : undefined, + forbidden.safe, + ].filter((value): value is number => value !== undefined) + const score = mean(components) + return { + score, + passed: score >= 1, + dimensions: { + claim_recall: required.recall, + citation_recall: citation, + hallucination_safe: forbidden.safe, + forbidden_claim_rate: forbidden.rate, + required_claim_count: required.total, + matched_claim_count: required.matched, + forbidden_claim_count: forbidden.total, + matched_forbidden_claim_count: forbidden.matched, + }, + notes: `required=${required.matched}/${required.total}; forbidden=${forbidden.matched}/${forbidden.total}; citation_recall=${citation.toFixed(3)}`, + raw: { + matchedRequiredClaimIds: required.matchedIds, + matchedForbiddenClaimIds: forbidden.matchedIds, + }, + } +} + +export function summarizeKnowledgeBenchmarkCampaign(input: { + scenarios: readonly KnowledgeBenchmarkScenario[] + campaign: CampaignResult +}): KnowledgeBenchmarkReport { + const scenariosById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario])) + const rows = input.campaign.cells.map((cell) => { + const score = Object.values(cell.judgeScores)[0] + const scenario = scenariosById.get(cell.scenarioId) + return { + cell, + scenario, + composite: score?.composite ?? 0, + passed: (score?.dimensions.passed ?? 0) >= 1, + dimensions: score?.dimensions ?? {}, + } + }) + const successful = rows.filter((row) => !row.cell.error) + return { + totalCases: input.scenarios.length, + totalCells: input.campaign.cells.length, + cellsFailed: input.campaign.aggregates.cellsFailed, + cellsCached: input.campaign.aggregates.cellsCached, + totalCostUsd: input.campaign.aggregates.totalCostUsd, + bySplit: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? 'unknown'), + byFamily: summarizeSlices(successful, (row) => row.scenario?.family ?? 'unknown'), + byTaskKind: summarizeSlices(successful, (row) => row.scenario?.taskKind ?? 'unknown'), + dimensions: summarizeDimensions(successful.map((row) => row.dimensions)), + score: distribution(successful.map((row) => row.composite)), + } +} + +function retrievalScenarioForCase( + testCase: KnowledgeRetrievalBenchmarkCase, +): RetrievalEvalScenario { + return { + id: testCase.id, + kind: 'retrieval-eval', + query: testCase.query, + expected: testCase.expected, + ...(testCase.k !== undefined ? { k: testCase.k } : {}), + } +} + +function normalizeRetrievalArtifact( + testCase: KnowledgeRetrievalBenchmarkCase, + artifact: TArtifact, +): RetrievalEvalArtifact { + const maybe = artifact as Partial & KnowledgeBenchmarkArtifact + const hits = maybe.hits ?? [] + if (Array.isArray(maybe.hits) && maybe.query && maybe.requestedK !== undefined) { + return maybe as RetrievalEvalArtifact + } + return { + config: {}, + query: testCase.query, + requestedK: testCase.k ?? Math.max(1, hits.length), + hits, + durationMs: maybe.durationMs ?? 0, + ...(maybe.costUsd !== undefined ? { costUsd: maybe.costUsd } : {}), + ...(maybe.metadata ? { metadata: maybe.metadata } : {}), + } +} + +function defaultDocumentTarget( + documentId: string, + targetKind: 'page' | 'page-path' | 'source', +): RetrievalGoldTarget { + switch (targetKind) { + case 'page': + return { kind: 'page', pageId: documentId } + case 'page-path': + return { kind: 'page-path', path: documentId } + case 'source': + return { kind: 'source', sourceId: documentId } + } +} + +function scoreClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { + let matched = 0 + let matchedWeight = 0 + let totalWeight = 0 + const matchedIds: string[] = [] + const haystack = text.toLowerCase() + for (const claim of claims) { + const weight = claim.weight ?? 1 + totalWeight += weight + if (claim.anyOf.some((fragment) => haystack.includes(fragment.toLowerCase()))) { + matched += 1 + matchedWeight += weight + matchedIds.push(claim.id) + } + } + return { + total: claims.length, + matched, + totalWeight, + recall: totalWeight === 0 ? 1 : matchedWeight / totalWeight, + matchedIds, + } +} + +function scoreForbiddenClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { + const matched = scoreClaims(text, claims) + return { + total: claims.length, + matched: matched.matched, + matchedIds: matched.matchedIds, + rate: claims.length === 0 ? 0 : matched.matched / claims.length, + safe: matched.matched === 0 ? 1 : 0, + } +} + +function scoreCitationRecall( + citedSourceIds: readonly string[], + expectedSourceIds: readonly string[], +): number { + if (expectedSourceIds.length === 0) return 1 + const cited = new Set(citedSourceIds) + const matched = expectedSourceIds.filter((sourceId) => cited.has(sourceId)).length + return matched / expectedSourceIds.length +} + +function summarizeDimensions( + rows: Array>, +): Record { + const values = new Map() + for (const row of rows) { + for (const [key, value] of Object.entries(row)) { + if (!Number.isFinite(value)) continue + const list = values.get(key) ?? [] + list.push(value) + values.set(key, list) + } + } + return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)])) +} + +function summarizeSlices( + rows: T[], + keyOf: (row: T) => string, +): Record { + const grouped = new Map() + for (const row of rows) { + const key = keyOf(row) + const list = grouped.get(key) ?? [] + list.push(row) + grouped.set(key, list) + } + return Object.fromEntries( + [...grouped.entries()].map(([key, list]) => { + const withShape = list as Array<{ composite: number; passed: boolean }> + return [ + key, + { + n: list.length, + meanScore: mean(withShape.map((row) => row.composite)), + passRate: mean(withShape.map((row) => (row.passed ? 1 : 0))), + score: distribution(withShape.map((row) => row.composite)), + }, + ] + }), + ) +} + +function distribution(values: readonly number[]): KnowledgeBenchmarkDistribution { + const finite = [...values].filter(Number.isFinite).sort((a, b) => a - b) + if (finite.length === 0) return { n: 0, min: 0, mean: 0, median: 0, p90: 0, max: 0 } + return { + n: finite.length, + min: finite[0]!, + mean: mean(finite), + median: percentile(finite, 0.5), + p90: percentile(finite, 0.9), + max: finite[finite.length - 1]!, + } +} + +function percentile(sortedValues: readonly number[], p: number): number { + if (sortedValues.length === 0) return 0 + const index = Math.min( + sortedValues.length - 1, + Math.max(0, Math.ceil(p * sortedValues.length) - 1), + ) + return sortedValues[index]! +} + +function mean(values: readonly number[]): number { + const finite = values.filter(Number.isFinite) + if (finite.length === 0) return 0 + return finite.reduce((sum, value) => sum + value, 0) / finite.length +} + +function unique(values: readonly string[]): string[] { + return [...new Set(values.filter(Boolean))] +} + +function renderSliceTable(slices: Record): string { + const rows = Object.entries(slices).map( + ([key, slice]) => + `| ${key} | ${slice.n} | ${formatNumber(slice.meanScore)} | ${formatNumber(slice.passRate)} | ${formatNumber(slice.score.p90)} |`, + ) + return [ + '| slice | n | mean score | pass rate | score p90 |', + '| --- | ---: | ---: | ---: | ---: |', + ...(rows.length ? rows : ['| none | 0 | 0 | 0 | 0 |']), + ].join('\n') +} + +function formatNumber(value: number): string { + if (!Number.isFinite(value)) return '0' + return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3) +} + +function observeArtifactCost(context: DispatchContext, artifact: unknown): void { + const costUsd = (artifact as { costUsd?: unknown })?.costUsd + if (costUsd === undefined) return + if (typeof costUsd !== 'number' || !Number.isFinite(costUsd) || costUsd < 0) { + throw new Error( + `benchmark artifact costUsd must be non-negative finite, got ${String(costUsd)}`, + ) + } + context.cost.observe(costUsd, 'agent-knowledge:benchmark') +} + +function compactObject(value: unknown): unknown { + if (Array.isArray(value)) return value.map(compactObject) + if (!value || typeof value !== 'object') return value + return Object.fromEntries( + Object.entries(value as Record) + .filter(([, entry]) => entry !== undefined) + .map(([key, entry]) => [key, compactObject(entry)]), + ) +} diff --git a/src/index.ts b/src/index.ts index e033813..23b0e3c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ export * from './adapters' export * from './adaptive-driver' +export * from './benchmarks/index' export * from './changes' export * from './chunking' export * from './claim-grounding' diff --git a/tests/benchmarks.test.ts b/tests/benchmarks.test.ts new file mode 100644 index 0000000..a972421 --- /dev/null +++ b/tests/benchmarks.test.ts @@ -0,0 +1,137 @@ +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import { + buildRetrievalBenchmarkCasesFromQrels, + INDUSTRY_RAG_BENCHMARKS, + type KnowledgeAnswerBenchmarkCase, + parseKnowledgeBenchmarkJsonl, + parseKnowledgeBenchmarkQrels, + runKnowledgeBenchmarkSuite, + scoreKnowledgeBenchmarkArtifact, +} from '../src/benchmarks/index' + +describe('knowledge benchmark adapters', () => { + it('parses qrels/jsonl and builds retrieval cases for public benchmark formats', () => { + expect(parseKnowledgeBenchmarkJsonl('{"id":"q1"}\n{"id":"q2"}\n')).toEqual([ + { id: 'q1' }, + { id: 'q2' }, + ]) + const qrels = parseKnowledgeBenchmarkQrels('q1 0 page-1 1\nq1 0 page-2 0\nq2 page-3 2') + expect(qrels).toEqual([ + { queryId: 'q1', documentId: 'page-1', score: 1 }, + { queryId: 'q1', documentId: 'page-2', score: 0 }, + { queryId: 'q2', documentId: 'page-3', score: 2 }, + ]) + + const cases = buildRetrievalBenchmarkCasesFromQrels({ + benchmarkId: 'beir/smoke', + family: 'beir', + queries: [ + { id: 'q1', text: 'refund policy', split: 'search' }, + { id: 'q2', text: 'shipping speed', split: 'holdout', tags: ['commerce'] }, + ], + qrels, + targetKind: 'page', + }) + + expect(cases).toHaveLength(2) + expect(cases[0]).toMatchObject({ + id: 'beir/smoke:q1', + taskKind: 'retrieval', + split: 'search', + expected: [{ kind: 'page', pageId: 'page-1' }], + }) + expect(cases[1]?.tags).toEqual(['commerce', 'holdout']) + }) + + it('runs retrieval benchmark cases through the campaign-backed suite', async () => { + const storage = inMemoryCampaignStorage() + const cases = buildRetrievalBenchmarkCasesFromQrels({ + benchmarkId: 'beir/suite-smoke', + family: 'beir', + queries: [ + { id: 'q1', text: 'refund policy', split: 'search' }, + { id: 'q2', text: 'shipping speed', split: 'holdout' }, + ], + qrels: [ + { queryId: 'q1', documentId: 'page-1', score: 1 }, + { queryId: 'q2', documentId: 'page-2', score: 1 }, + ], + targetKind: 'page', + k: 2, + }) + const result = await runKnowledgeBenchmarkSuite({ + cases, + runDir: '/runs/knowledge-benchmark-smoke', + storage, + respond: ({ case: testCase }) => ({ + costUsd: 0.01, + hits: [ + testCase.id.endsWith('q1') + ? { pageId: 'page-1', path: 'knowledge/page-1.md', rank: 1 } + : { pageId: 'miss', path: 'knowledge/miss.md', rank: 1 }, + ], + }), + }) + + expect(result.report.totalCases).toBe(2) + expect(result.report.cellsFailed).toBe(0) + expect(result.report.score.mean).toBe(0.5) + expect(result.report.byFamily.beir?.n).toBe(2) + expect(result.report.bySplit.search?.meanScore).toBe(1) + expect(result.report.bySplit.holdout?.meanScore).toBe(0) + expect(result.report.totalCostUsd).toBe(0.02) + expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 2') + expect(storage.read(result.reportMarkdownPath)).toContain('# Knowledge Benchmark Report') + }) + + it('scores RAG answer, hallucination, and KB-improvement cases with claim/source checks', () => { + const testCase: KnowledgeAnswerBenchmarkCase = { + id: 'crag/smoke:q1', + family: 'crag', + taskKind: 'rag-answer', + prompt: 'What is the refund policy?', + requiredClaims: [{ id: 'refund-window', anyOf: ['30 day refund', '30-day refund'] }], + forbiddenClaims: [{ id: 'unsupported-lifetime', anyOf: ['lifetime refund'] }], + expectedSourceIds: ['src-policy', 'src-terms'], + } + + const partial = scoreKnowledgeBenchmarkArtifact(testCase, { + answer: 'The product has a 30-day refund period.', + citedSourceIds: ['src-policy'], + }) + expect(partial.dimensions.claim_recall).toBe(1) + expect(partial.dimensions.citation_recall).toBe(0.5) + expect(partial.dimensions.hallucination_safe).toBe(1) + expect(partial.score).toBeCloseTo(5 / 6) + expect(partial.passed).toBe(false) + + const hallucinated = scoreKnowledgeBenchmarkArtifact( + { ...testCase, taskKind: 'hallucination' }, + { answer: 'The product has a lifetime refund.' }, + ) + expect(hallucinated.dimensions.hallucination_safe).toBe(0) + expect(hallucinated.raw.matchedForbiddenClaimIds).toEqual(['unsupported-lifetime']) + }) + + it('declares every requested industry benchmark family', () => { + const ids = new Set(INDUSTRY_RAG_BENCHMARKS.map((benchmark) => benchmark.id)) + for (const id of [ + 'beir', + 'mteb-retrieval', + 'msmarco', + 'trec-dl', + 'miracl', + 'lotte', + 'bright', + 'crag', + 'hotpotqa', + 'kilt', + 'ragtruth', + 'faithbench', + 'first-party/kb-improvement', + ]) { + expect(ids.has(id)).toBe(true) + } + }) +}) diff --git a/tsup.config.ts b/tsup.config.ts index 3aed29f..84f7eab 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -9,6 +9,7 @@ export default defineConfig({ 'sources/index': 'src/sources/index.ts', 'profiles/index': 'src/profiles/index.ts', 'autodata/index': 'src/autodata/index.ts', + 'benchmarks/index': 'src/benchmarks/index.ts', }, format: ['esm'], dts: true,