diff --git a/bench/HARNESS.md b/bench/HARNESS.md index bba10ee0..5f59d38c 100644 --- a/bench/HARNESS.md +++ b/bench/HARNESS.md @@ -3,7 +3,7 @@ If you're an agent picking this up: read this page, then run `pnpm help` + `pnpm gate` — do NOT re-derive the harness from source. This map is SHORT on purpose; if it disagrees with the code, the code wins — fix this page in the same turn (the anti-rediscovery law). -Verified against source 2026-06-10 · agent-eval pinned `^0.83.0`. The CANONICAL surface is now +Verified against source 2026-07-07 · agent-eval pinned `^0.106.1`. The CANONICAL surface is now the published optimization suite (`@tangle-network/agent-runtime/loops`): `Environment` + `Strategy`/`defineStrategy` + `runBenchmark` — see the section below FIRST. The recursive diverse-vs-blind gate runs through the keystone (`gate-cli.mts` → `runGate`); @@ -202,11 +202,11 @@ The code-benches share `benchmarks/_harness.ts` (stage artifact → run the benc in a `.venv`/Docker subprocess → parse its JSON report → `{resolved,score}`). No per-adapter copy of the process/venv/Docker/temp/report plumbing; commit0+appworld also share its stdin-piping runner (`runVenvScriptStdin`). -- **Real, runnable with ZERO extra deps:** finsearchcomp (GitHub dataset + fixtures + LLM judge — the gate bench), hotpotqa + simpleqa + frames (HF/web QA + F1/LLM judge; `*_FIXTURES=1` offline), **aec-bench** (real GitHub task tree + fixtures; judge = the task's own `tests/verify.py` over python3 stdlib — **deterministic, graded per-field partial credit, no Docker, no LLM** → the candidate non-oracle correctable-middle-band bench for the open gate). +- **Real, runnable with ZERO extra deps:** finsearchcomp (GitHub dataset + fixtures + LLM judge — the gate bench), hotpotqa + simpleqa + frames (HF/web QA + F1/LLM judge; `*_FIXTURES=1` offline), **ragbench**, **crag**, **nomiracl**, **open-rag-bench**, **t2-ragbench** (SOTA RAG/knowledge benchmarks with committed fixtures and deterministic answer/relevance judges; live mode reads explicit `*_DATA_FILE` JSON/JSONL exports), **aec-bench** (real GitHub task tree + fixtures; judge = the task's own `tests/verify.py` over python3 stdlib — **deterministic, graded per-field partial credit, no Docker, no LLM** → the candidate non-oracle correctable-middle-band bench for the open gate). - **Real code, needs an external harness/tools to run (fail loud with the exact install/Docker fix; never a fabricated score):** swe-bench + terminal-bench (`bench/.venv` + Docker), **commit0** (ISOLATED `bench/.venv-commit0` via `python3 -m venv bench/.venv-commit0 && bench/.venv-commit0/bin/pip install commit0 datasets` — its deps conflict with the shared `.venv`; override dir with `COMMIT0_VENV` — plus Docker; judge = official pytest harness, graded (passed+xfail)/total; the rollout prompt stages in-box (clones `commit-0/` @ `base_commit`, emits `git diff`); `COMMIT0_FIXTURES=1` for offline listing), **programbench** (`pip install programbench` + Docker on linux/amd64 + HF blobs; judge = official cleanroom eval, graded passed/total; `PROGRAMBENCH_FIXTURES=1` offline), **appworld** (`pip install appworld` + `appworld install` + `appworld download data`; judge = AppWorld's own `world.evaluate()`, graded passes/num_tests — NO committed fixture: task data exists only after `download data`, so loadTasks fails loud rather than fabricate a task), **dabstep** (`DABSTEP_DIR=/path/to/EnvCommons/DABStep` with the released `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`; judge delegates to official `grade.py`; `DABSTEP_FIXTURES=1` only tests adapter plumbing and does not fabricate benchmark scores), **webarena-verified** (`WEBARENA_VERIFIED_DIR=/path/to/webarena-verified`; judge delegates to official `eval-tasks` over a run output directory), **tau2-bench** (`TAU2_BENCH_DIR=/path/to/tau2-bench`; judge recomputes tau2 trajectory rewards), **tau3-banking** (`TAU3_BENCH_DIR=/path/to/tau2-bench`; default domain `banking_knowledge`; judge recomputes tau trajectory rewards through the upstream tau3 package), **agentbench** DBBench subset (`AGENTBENCH_DIR=/path/to/AgentBench`; exact-match deterministic label judge), **bfcl** deterministic function-call subset (`BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard`; loads official BFCL JSONL + `possible_answer`; score = structured call/argument match, not the full BFCL leaderboard evaluator), **toollm** API-selection subset (`TOOLBENCH_DIR=/path/to/ToolBench`; score = recall of ToolBench `relevant APIs` labels, resolved only when the worker emits the requested structured JSON call list; official ToolEval pass rate remains LLM-judged/stochastic), **finresearchbench** (`FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl`; rows must carry official `judge_system_prompt` + `judge_prompt_template`; no self-authored live judge), mind2web, cad-design + cadbench + cadgenbench (openscad/blender/build123d). - **goldArtifact:** aec-bench returns the task's real `golden_pass.md` (verify-judge works fully offline). commit0 / programbench / appworld return `undefined` — the oracle is a git ref / stripped source / engine-bundled solution, not a portable string; judge correctness is proven by a real solve through the harness, not a synthetic gold (documented + fail-loud, not a fake). - **Absent (not built):** swe-gym, swe-bench-multimodal, and the rest of the survey set. -Every unbuilt/scaffold adapter fails LOUD (throws with the integration step) rather than faking a score — no silent zeros in any corpus. Offline fixture tests: `benchmarks/{aec-bench,commit0,programbench,appworld}.test.mts` (`tsx --test`). +Every unbuilt/scaffold adapter fails LOUD (throws with the integration step) rather than faking a score — no silent zeros in any corpus. Offline fixture tests: `benchmarks/{aec-bench,commit0,programbench,appworld,rag-benchmarks}.test.mts` (`tsx --test`). ## Is it runnable RIGHT NOW? (verify the map, don't trust it blindly) ``` diff --git a/bench/fixtures/crag.json b/bench/fixtures/crag.json new file mode 100644 index 00000000..f81ef269 --- /dev/null +++ b/bench/fixtures/crag.json @@ -0,0 +1,10 @@ +[ + { + "id": "crag-fixture-0", + "domain": "finance", + "question_type": "simple", + "static_or_dynamic": "slow-changing", + "query": "What was Contoso's reported revenue in 2025?", + "answer": ["$12.4 billion", "12.4 billion dollars"] + } +] diff --git a/bench/fixtures/nomiracl.json b/bench/fixtures/nomiracl.json new file mode 100644 index 00000000..05278ef5 --- /dev/null +++ b/bench/fixtures/nomiracl.json @@ -0,0 +1,26 @@ +[ + { + "id": "nomiracl-fixture-relevant", + "language": "en", + "query": "What is the refund window?", + "positive_passages": [ + { + "docid": "p1", + "title": "Refund policy", + "text": "The refund window is 30 days." + } + ] + }, + { + "id": "nomiracl-fixture-non-relevant", + "language": "en", + "query": "What is the refund window?", + "negative_passages": [ + { + "docid": "p2", + "title": "Shipping policy", + "text": "Shipping labels expire after 7 days." + } + ] + } +] diff --git a/bench/fixtures/open-rag-bench.json b/bench/fixtures/open-rag-bench.json new file mode 100644 index 00000000..6dab73c3 --- /dev/null +++ b/bench/fixtures/open-rag-bench.json @@ -0,0 +1,16 @@ +[ + { + "id": "open-rag-bench-fixture-0", + "file_name": "arxiv-0001.pdf", + "modality": "table", + "question": "Which method has the highest accuracy in the table?", + "answer": "Hybrid BM25", + "contexts": [ + { + "id": "table-1", + "title": "Accuracy table", + "text": "Method | Accuracy\nDense | 82.1\nHybrid BM25 | 87.4\nSparse | 79.0" + } + ] + } +] diff --git a/bench/fixtures/ragbench.json b/bench/fixtures/ragbench.json new file mode 100644 index 00000000..3f753acf --- /dev/null +++ b/bench/fixtures/ragbench.json @@ -0,0 +1,21 @@ +[ + { + "id": "ragbench-fixture-0", + "dataset": "support", + "question": "How long is the refund window?", + "response": "30 days", + "documents": [ + [ + ["0a", "Title: Refund policy"], + ["0b", "Customers can request a refund within 30 days of purchase."] + ], + [ + ["1a", "Title: Shipping policy"], + ["1b", "Shipping labels expire after 7 days."] + ] + ], + "adherence": 1, + "completeness": 1, + "relevance": 1 + } +] diff --git a/bench/fixtures/t2-ragbench.json b/bench/fixtures/t2-ragbench.json new file mode 100644 index 00000000..f69e06e4 --- /dev/null +++ b/bench/fixtures/t2-ragbench.json @@ -0,0 +1,13 @@ +[ + { + "id": "t2-ragbench-fixture-0", + "subset": "finqa", + "context_id": "10k-0001", + "question": "What is the percentage increase from 100 to 112?", + "program_answer": "12%", + "original_answer": "12.0 percent", + "context": "Revenue increased from 100 in 2024 to 112 in 2025.", + "table": "Year | Revenue\n2024 | 100\n2025 | 112", + "file_name": "10k-0001.pdf" + } +] diff --git a/bench/package.json b/bench/package.json index 1bd879e7..8d4aad86 100644 --- a/bench/package.json +++ b/bench/package.json @@ -2,7 +2,7 @@ "name": "@tangle-network/agent-bench", "version": "0.1.0", "type": "module", - "description": "The unified benchmark suite for agent-runtime agents: 26 adapters (commit0, enterpriseops-gym, tau3-banking, bfcl, finresearchbench, trata-hedge, finsearchcomp, dabstep, webarena-verified, tau2-bench, agentbench, swe-bench, humaneval, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.", + "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.", "main": "src/index.ts", "types": "src/index.ts", "exports": { @@ -18,9 +18,9 @@ "terminal-compare": "tsx src/terminal-compare.ts" }, "dependencies": { - "@tangle-network/agent-eval": "^0.103.2", - "@tangle-network/agent-runtime": "^0.88.0", - "@tangle-network/sandbox": "^0.9.5" + "@tangle-network/agent-eval": "^0.106.3", + "@tangle-network/agent-runtime": "^0.89.0", + "@tangle-network/sandbox": "^0.9.7" }, "devDependencies": { "tsx": "^4.19.0", diff --git a/bench/pnpm-lock.yaml b/bench/pnpm-lock.yaml index 8526b7af..186f3f70 100644 --- a/bench/pnpm-lock.yaml +++ b/bench/pnpm-lock.yaml @@ -9,14 +9,14 @@ importers: .: dependencies: '@tangle-network/agent-eval': - specifier: ^0.100.0 - version: 0.100.0(typescript@6.0.3) + specifier: ^0.106.3 + version: 0.106.3(typescript@6.0.3) '@tangle-network/agent-runtime': - specifier: ^0.79.3 - version: 0.79.3(@tangle-network/agent-eval@0.100.0(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.5(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))) + specifier: ^0.89.0 + version: 0.89.0(@tangle-network/agent-eval@0.106.3(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))) '@tangle-network/sandbox': - specifier: ^0.9.3 - version: 0.9.5(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) + specifier: ^0.9.7 + version: 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) devDependencies: tsx: specifier: ^4.19.0 @@ -251,13 +251,8 @@ packages: '@tangle-network/agent-core@0.3.4': resolution: {integrity: sha512-Hvz3ABRouNtBmRvGqPxifAO2yuILneJMylWH5jW/jeS2F03RvqkGYuXyGXWWLqosYbb3hVAvSEe4Ykm2FMGEDQ==} - '@tangle-network/agent-eval@0.100.0': - resolution: {integrity: sha512-yBupVJJAqHozhe1BL5xBuDObjvNsoY+XmJo7qfpw/w7rehAXbKliBb4k3XS1G55+GaYPjFA+xwPzlEDQISpMRw==} - engines: {node: '>=20'} - hasBin: true - - '@tangle-network/agent-integrations@0.29.0': - resolution: {integrity: sha512-Avn4oBDTRP5v/3o1xq++uu/9+Rhl2hscIggeFPBGjtVYwhvbsSZL9pRrF3LfjqL9rjx9AocZOdsZC6MXrxKnkg==} + '@tangle-network/agent-eval@0.106.3': + resolution: {integrity: sha512-5mLpTNoR4YcAbHueCrYcdoBRo+I9zmjXipzWaYfJ2FGj4byA6L4wrTPdTneOLjvxOGlrq+H/CRmHB738fP8R/Q==} engines: {node: '>=20'} hasBin: true @@ -270,12 +265,12 @@ packages: '@tangle-network/agent-interface@0.14.0': resolution: {integrity: sha512-9CyGhIpl90E7v4MTm3b1ti3Bp7BfPigk2Nafgi21Lg0U+QxlNB656F2JmVpUuSbOo9aGZPtg5nXu5EBTlV5a1g==} - '@tangle-network/agent-runtime@0.79.3': - resolution: {integrity: sha512-CIQ09F9zK8agXbPvilRySCX3QB8XssnYx95VHsonWs5D4M5kXn3v+dXzz1aPbnOCxveEHLyiE7zApUyj3WU1yA==} + '@tangle-network/agent-runtime@0.89.0': + resolution: {integrity: sha512-tjQ/uJORuLOAE/E99w0NxWb5O0uP/nwescMRyIqxsqLXzry7WhNh9DK/BHKTEfb6XMvcIaT92J/3T1ZJTtXZ9Q==} engines: {node: '>=20'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': '>=0.97.0 <1.0.0' + '@tangle-network/agent-eval': '>=0.101.0 <1.0.0' '@tangle-network/agent-interface': '>=0.14.0 <1.0.0' '@tangle-network/sandbox': '>=0.8.0 <1.0.0' playwright: ^1.40.0 @@ -287,28 +282,8 @@ packages: playwright: optional: true - '@tangle-network/sandbox@0.3.0': - resolution: {integrity: sha512-KfgvKhsUaOpkJe3AD19w7s4hdQekBlXQGoNx0xS4u6vuQk5YnFzBgv+EQeHCkkgETpYOWS2AN+6u/JhSyWStMw==} - peerDependencies: - '@mastra/core': '*' - '@modelcontextprotocol/sdk': '*' - ai: '*' - openai: ^6.36.0 - viem: ^2.0.0 - peerDependenciesMeta: - '@mastra/core': - optional: true - '@modelcontextprotocol/sdk': - optional: true - ai: - optional: true - openai: - optional: true - viem: - optional: true - - '@tangle-network/sandbox@0.9.5': - resolution: {integrity: sha512-yvX2OX6uISBVnMQ+v6Upkesa3u8yj6BHxsfcS6p8Vze+M4WBpyhkwA+onzFHuo9rti557ItZn8yDu4a/klljvQ==} + '@tangle-network/sandbox@0.9.7': + resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} peerDependencies: '@mastra/core': ^1.36.0 '@modelcontextprotocol/sdk': ^1.29.0 @@ -331,8 +306,8 @@ packages: resolution: {integrity: sha512-+TAF9s5t1jOWGyGHvKhIWe2FYmG7puVaxmmg0Et67ylAjGa7GqUAvISXGjG/6dzld7A170V0kQHK0WVdh2Wh0Q==} engines: {node: '>=18'} - '@tangle-network/tcloud@0.4.12': - resolution: {integrity: sha512-3Qs90sV0P3LBtrTGC9HW2rwCMDjbScyhZIQU6H2/dVd84S5uKN+tCsURnXE6uu54U766Xa/V3Rcdqqjmgv7AXg==} + '@tangle-network/tcloud@0.4.14': + resolution: {integrity: sha512-jWYt//cGdLBDOv0luLH6xAGS4gbuOt8uHIkaCWwDDpQ1zp0FUPATHIrA3RMuF0qtQq9Vq00IhLrmCnHdHBP+dg==} engines: {node: '>=18'} hasBin: true @@ -570,13 +545,13 @@ snapshots: '@tangle-network/agent-interface': 0.14.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.100.0(typescript@6.0.3)': + '@tangle-network/agent-eval@0.106.3(typescript@6.0.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 19.0.45(zod@4.4.3) '@hono/node-server': 2.0.4(hono@4.12.23) '@tangle-network/agent-interface': 0.10.1 - '@tangle-network/tcloud': 0.4.12(typescript@6.0.3)(zod@4.4.3) + '@tangle-network/tcloud': 0.4.14(typescript@6.0.3)(zod@4.4.3) hono: 4.12.23 zod: 4.4.3 transitivePeerDependencies: @@ -588,8 +563,6 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-integrations@0.29.0': {} - '@tangle-network/agent-interface@0.10.1': dependencies: zod: 4.4.3 @@ -602,20 +575,14 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-runtime@0.79.3(@tangle-network/agent-eval@0.100.0(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.5(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))': + '@tangle-network/agent-runtime@0.89.0(@tangle-network/agent-eval@0.106.3(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))': dependencies: - '@tangle-network/agent-eval': 0.100.0(typescript@6.0.3) + '@tangle-network/agent-eval': 0.106.3(typescript@6.0.3) optionalDependencies: '@tangle-network/agent-interface': 0.14.0 - '@tangle-network/sandbox': 0.9.5(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) - - '@tangle-network/sandbox@0.3.0(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))': - dependencies: - '@tangle-network/agent-integrations': 0.29.0 - optionalDependencies: - viem: 2.52.0(typescript@6.0.3)(zod@4.4.3) + '@tangle-network/sandbox': 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) - '@tangle-network/sandbox@0.9.5(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))': + '@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))': dependencies: '@tangle-network/agent-core': 0.3.4 '@tangle-network/agent-interface': 0.13.0 @@ -624,11 +591,11 @@ snapshots: '@tangle-network/tcloud-attestation@0.1.1': {} - '@tangle-network/tcloud@0.4.12(typescript@6.0.3)(zod@4.4.3)': + '@tangle-network/tcloud@0.4.14(typescript@6.0.3)(zod@4.4.3)': dependencies: '@scure/bip32': 2.2.0 '@scure/bip39': 2.2.0 - '@tangle-network/sandbox': 0.3.0(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) + '@tangle-network/sandbox': 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) '@tangle-network/tcloud-attestation': 0.1.1 commander: 14.0.3 viem: 2.52.0(typescript@6.0.3)(zod@4.4.3) diff --git a/bench/src/adapters.ts b/bench/src/adapters.ts index f7cc7b7d..21b37140 100644 --- a/bench/src/adapters.ts +++ b/bench/src/adapters.ts @@ -13,6 +13,7 @@ import { createCadBenchAdapter } from './benchmarks/cadbench' import { createCadDesignAdapter } from './benchmarks/cad-design' import { createCadGenBenchAdapter } from './benchmarks/cadgenbench' import { createCommit0Adapter } from './benchmarks/commit0' +import { createCragAdapter } from './benchmarks/crag' import { createDabstepAdapter } from './benchmarks/dabstep' import { createEnterpriseOpsGymAdapter } from './benchmarks/enterpriseops-gym' import { createFinResearchBenchAdapter } from './benchmarks/finresearchbench' @@ -21,9 +22,13 @@ import { createFramesAdapter } from './benchmarks/frames' import { createHotpotqaAdapter } from './benchmarks/hotpotqa' import { createHumanEvalAdapter } from './benchmarks/humaneval' import { createMind2WebAdapter } from './benchmarks/mind2web' +import { createNoMiraclAdapter } from './benchmarks/nomiracl' +import { createOpenRagBenchAdapter } from './benchmarks/open-rag-bench' import { createProgrambenchAdapter } from './benchmarks/programbench' +import { createRagBenchAdapter } from './benchmarks/ragbench' import { createSimpleQaAdapter } from './benchmarks/simpleqa' import { createSweBenchAdapter } from './benchmarks/swe-bench' +import { createT2RagBenchAdapter } from './benchmarks/t2-ragbench' import { createTau2BenchAdapter } from './benchmarks/tau2-bench' import { createTau3BankingAdapter } from './benchmarks/tau3-banking' import { createTerminalBenchAdapter } from './benchmarks/terminal-bench' @@ -57,6 +62,11 @@ export const ADAPTERS: Record BenchmarkAdapter> = { cadbench: createCadBenchAdapter, cadgenbench: createCadGenBenchAdapter, frames: createFramesAdapter, + ragbench: createRagBenchAdapter, + crag: createCragAdapter, + nomiracl: createNoMiraclAdapter, + 'open-rag-bench': createOpenRagBenchAdapter, + 't2-ragbench': createT2RagBenchAdapter, finresearchbench: createFinResearchBenchAdapter, finsearchcomp: createFinsearchcompAdapter, simpleqa: createSimpleQaAdapter, diff --git a/bench/src/benchmarks/appworld.test.mts b/bench/src/benchmarks/appworld.test.mts index 54bc5ff9..685a1ece 100644 --- a/bench/src/benchmarks/appworld.test.mts +++ b/bench/src/benchmarks/appworld.test.mts @@ -27,19 +27,25 @@ test('goldArtifact is undefined — reference solution ships only inside the eng assert.equal(await a.goldArtifact({ id: 't', prompt: '', metadata: { taskId: 't', split: 'dev' } }), undefined) }) -test('preflight FAILS LOUD with the install + download-data fix when the engine is absent', async () => { +test('preflight passes when installed or FAILS LOUD with the install + download-data fix', async () => { const a = createAppWorldAdapter() - await assert.rejects(a.preflight(), (e: Error) => { + try { + await a.preflight() + } catch (err) { + const e = err as Error assert.match(e.message, /pip install appworld/) assert.match(e.message, /appworld download data/) - return true - }) + } }) -test('loadTasks FAILS LOUD (engine enumeration) rather than fabricating tasks offline', async () => { +test('loadTasks either enumerates live engine rows or FAILS LOUD without fabricating tasks', async () => { const a = createAppWorldAdapter() - await assert.rejects(a.loadTasks({ limit: 1 }), (e: Error) => { - assert.match(e.message, /appworld driver failed|appworld import failed/) - return true - }) + try { + const tasks = await a.loadTasks({ limit: 1 }) + assert.equal(tasks.length, 1) + assert.ok(tasks[0].id.length > 0) + assert.match(tasks[0].prompt, /Solve this by writing Python/) + } catch (err) { + assert.match((err as Error).message, /appworld driver failed|appworld import failed/) + } }) diff --git a/bench/src/benchmarks/crag.ts b/bench/src/benchmarks/crag.ts new file mode 100644 index 00000000..91cde193 --- /dev/null +++ b/bench/src/benchmarks/crag.ts @@ -0,0 +1,137 @@ +/** + * CRAG adapter (Comprehensive RAG Benchmark). + * + * Live mode expects an official or compatible CRAG JSON/JSONL export. The + * adapter preserves CRAG domain/type/dynamism tags in metadata and scores final + * answers deterministically against the provided gold answer list. + */ + +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { benchRoot } from './_harness' +import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' +import { + FINAL_ANSWER_SENTINEL, + allStrings, + answerScoreToBenchScore, + firstString, + isObject, + ragAnswerOutput, + readJsonRows, + scoreAnswerArtifact, + selectTasks, + stringFrom, +} from './rag-shared' + +const FIXTURES = join(benchRoot, 'fixtures', 'crag.json') + +interface CragMeta { + benchmark: 'crag' + query: string + goldAnswers: string[] + domain: string + questionType: string + dynamism: string +} + +const dataFile = (): string | undefined => process.env.CRAG_DATA_FILE + +function rowToTask(raw: unknown, index: number): BenchTask { + if (!isObject(raw)) throw new Error(`CRAG row ${index} must be an object`) + const query = firstString(raw, ['query', 'question', 'prompt']) + const goldAnswers = allStrings(raw, ['answer', 'answers', 'gold', 'gold_answer', 'expected_answer']) + if (!query) throw new Error(`CRAG row ${index} missing query`) + if (goldAnswers.length === 0) throw new Error(`CRAG row ${index} missing gold answer`) + const domain = stringFrom(raw.domain) ?? 'unknown' + const questionType = stringFrom(raw.question_type) ?? stringFrom(raw.questionType) ?? 'unknown' + const dynamism = stringFrom(raw.static_or_dynamic) ?? stringFrom(raw.dynamism) ?? 'unknown' + const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `crag-${index}` + const meta: CragMeta = { + benchmark: 'crag', + query, + goldAnswers, + domain, + questionType, + dynamism, + } + return { + id, + split: stringFrom(raw.split) ?? domain, + prompt: [ + 'Answer this CRAG factual question.', + 'Return a concise answer and do not guess when the evidence is insufficient.', + 'End with a single final line: `FINAL ANSWER: `.', + '', + `Question: ${query}`, + `Domain: ${domain}`, + `Question type: ${questionType}`, + `Dynamism: ${dynamism}`, + ].join('\n'), + metadata: meta as unknown as Record, + } +} + +function readMeta(task: BenchTask): CragMeta { + const md = task.metadata + if (!md || !Array.isArray(md.goldAnswers)) { + throw new Error(`CRAG task ${task.id} missing metadata — loadTasks did not populate it`) + } + return md as unknown as CragMeta +} + +async function loadRows(path: string): Promise { + const rows = await readJsonRows(path) + if (rows.length === 0) throw new Error(`CRAG: no rows in ${path}`) + return rows +} + +async function loadFixtures(opts: LoadOptions): Promise { + const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[] + console.warn(`[crag] CRAG_FIXTURES=1 — loading ${rows.length} adapter fixtures`) + return selectTasks(rows.map(rowToTask), opts, 'CRAG') +} + +export function createCragAdapter(): BenchmarkAdapter { + const fixturesMode = process.env.CRAG_FIXTURES === '1' + + return { + name: 'crag', + output: ragAnswerOutput, + + async preflight() { + if (fixturesMode) { + await readFile(FIXTURES, 'utf8') + return + } + const path = dataFile() + if (!path) { + throw new Error( + 'CRAG_DATA_FILE is required. Fix: export facebookresearch/CRAG rows to JSONL and set CRAG_DATA_FILE=/path/to/crag.jsonl, or set CRAG_FIXTURES=1 for adapter plumbing.', + ) + } + await loadRows(path) + }, + + async loadTasks(opts: LoadOptions = {}) { + if (fixturesMode) return loadFixtures(opts) + const path = dataFile() + if (!path) throw new Error('CRAG_DATA_FILE is required to load CRAG tasks') + return selectTasks((await loadRows(path)).map(rowToTask), opts, 'CRAG') + }, + + async goldArtifact(task: BenchTask) { + return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}` + }, + + async judge(task: BenchTask, artifact: string): Promise { + const meta = readMeta(task) + const score = scoreAnswerArtifact(artifact, meta.goldAnswers) + return answerScoreToBenchScore(score, { + benchmark: meta.benchmark, + domain: meta.domain, + questionType: meta.questionType, + dynamism: meta.dynamism, + }) + }, + } +} diff --git a/bench/src/benchmarks/nomiracl.ts b/bench/src/benchmarks/nomiracl.ts new file mode 100644 index 00000000..f74e5e18 --- /dev/null +++ b/bench/src/benchmarks/nomiracl.ts @@ -0,0 +1,180 @@ +/** + * NoMIRACL adapter. + * + * NoMIRACL tests robustness to irrelevant retrieved passages. The worker does + * not generate an answer here; it classifies whether the supplied passages + * contain enough evidence to answer the query. This directly measures false + * positive / false negative behavior for RAG abstention. + */ + +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { benchRoot } from './_harness' +import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' +import { + contextBlock, + contextsFrom, + firstString, + isObject, + ragAnswerOutput, + readJsonRows, + selectTasks, + stringFrom, + type RagContext, +} from './rag-shared' + +const FIXTURES = join(benchRoot, 'fixtures', 'nomiracl.json') + +interface NoMiraclMeta { + benchmark: 'nomiracl' + query: string + answerable: boolean + language: string + subset: string + contexts: RagContext[] +} + +const dataFile = (): string | undefined => process.env.NOMIRACL_DATA_FILE + +function passagesFrom(value: unknown, relevant: boolean): RagContext[] { + return contextsFrom(value).map((context) => ({ ...context, relevant })) +} + +function answerableFrom(raw: Record, contexts: readonly RagContext[]): boolean { + const direct = raw.answerable ?? raw.is_answerable ?? raw.has_answer ?? raw.label ?? raw.relevant + if (typeof direct === 'boolean') return direct + if (typeof direct === 'number') return direct > 0 + if (typeof direct === 'string') { + const normalized = direct.toLowerCase() + if (['true', 't', 'relevant', 'answerable', 'positive', '1', 'yes'].includes(normalized)) return true + if (['false', 'f', 'non-relevant', 'non_relevant', 'unanswerable', 'negative', '0', 'no'].includes(normalized)) { + return false + } + } + return contexts.some((context) => context.relevant === true) +} + +function rowToTask(raw: unknown, index: number): BenchTask { + if (!isObject(raw)) throw new Error(`NoMIRACL row ${index} must be an object`) + const query = firstString(raw, ['query', 'question']) + if (!query) throw new Error(`NoMIRACL row ${index} missing query`) + const labelledPassages = [ + ...passagesFrom(raw.positive_passages, true), + ...passagesFrom(raw.negative_passages, false), + ] + const contexts = + labelledPassages.length > 0 + ? labelledPassages + : contextsFrom(raw.passages).length > 0 + ? contextsFrom(raw.passages) + : contextsFrom(raw.contexts).length > 0 + ? contextsFrom(raw.contexts) + : contextsFrom(raw.documents) + const answerable = answerableFrom(raw, contexts) + const language = stringFrom(raw.lang) ?? stringFrom(raw.language) ?? 'unknown' + const subset = stringFrom(raw.subset) ?? (answerable ? 'relevant' : 'non-relevant') + const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `nomiracl-${index}` + const meta: NoMiraclMeta = { + benchmark: 'nomiracl', + query, + answerable, + language, + subset, + contexts, + } + return { + id, + split: stringFrom(raw.split) ?? language, + prompt: [ + 'Classify this NoMIRACL RAG relevance case.', + 'Return exactly one word: ANSWERABLE if at least one supplied passage contains enough evidence to answer the query, otherwise UNANSWERABLE.', + '', + `Query: ${query}`, + contexts.length > 0 ? `\nPassages:\n${contextBlock(contexts)}` : undefined, + ] + .filter(Boolean) + .join('\n'), + metadata: meta as unknown as Record, + } +} + +function readMeta(task: BenchTask): NoMiraclMeta { + const md = task.metadata + if (!md || typeof md.answerable !== 'boolean') { + throw new Error(`NoMIRACL task ${task.id} missing metadata — loadTasks did not populate it`) + } + return md as unknown as NoMiraclMeta +} + +function parseClassification(artifact: string): boolean | null { + const normalized = artifact.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim() + if (/\bunanswerable\b|\bnot answerable\b|\bno answer\b|\bnegative\b|\bfalse\b/.test(normalized)) { + return false + } + if (/\banswerable\b|\brelevant\b|\bpositive\b|\btrue\b|\byes\b/.test(normalized)) return true + return null +} + +async function loadRows(path: string): Promise { + const rows = await readJsonRows(path) + if (rows.length === 0) throw new Error(`NoMIRACL: no rows in ${path}`) + return rows +} + +async function loadFixtures(opts: LoadOptions): Promise { + const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[] + console.warn(`[nomiracl] NOMIRACL_FIXTURES=1 — loading ${rows.length} adapter fixtures`) + return selectTasks(rows.map(rowToTask), opts, 'NoMIRACL') +} + +export function createNoMiraclAdapter(): BenchmarkAdapter { + const fixturesMode = process.env.NOMIRACL_FIXTURES === '1' + + return { + name: 'nomiracl', + output: ragAnswerOutput, + + async preflight() { + if (fixturesMode) { + await readFile(FIXTURES, 'utf8') + return + } + const path = dataFile() + if (!path) { + throw new Error( + 'NOMIRACL_DATA_FILE is required. Fix: export project-miracl/nomiracl rows to JSONL and set NOMIRACL_DATA_FILE=/path/to/nomiracl.jsonl, or set NOMIRACL_FIXTURES=1 for adapter plumbing.', + ) + } + await loadRows(path) + }, + + async loadTasks(opts: LoadOptions = {}) { + if (fixturesMode) return loadFixtures(opts) + const path = dataFile() + if (!path) throw new Error('NOMIRACL_DATA_FILE is required to load NoMIRACL tasks') + return selectTasks((await loadRows(path)).map(rowToTask), opts, 'NoMIRACL') + }, + + async goldArtifact(task: BenchTask) { + return readMeta(task).answerable ? 'ANSWERABLE' : 'UNANSWERABLE' + }, + + async judge(task: BenchTask, artifact: string): Promise { + const meta = readMeta(task) + const got = parseClassification(artifact) + const resolved = got === meta.answerable + return { + resolved, + score: resolved ? 1 : 0, + detail: JSON.stringify({ + benchmark: meta.benchmark, + language: meta.language, + subset: meta.subset, + expected: meta.answerable ? 'ANSWERABLE' : 'UNANSWERABLE', + got: got === null ? null : got ? 'ANSWERABLE' : 'UNANSWERABLE', + contextCount: meta.contexts.length, + }), + } + }, + } +} diff --git a/bench/src/benchmarks/open-rag-bench.ts b/bench/src/benchmarks/open-rag-bench.ts new file mode 100644 index 00000000..16185140 --- /dev/null +++ b/bench/src/benchmarks/open-rag-bench.ts @@ -0,0 +1,153 @@ +/** + * Open RAG Bench adapter. + * + * This targets Vectara-style Open RAG Bench exports over PDF-derived text, + * table, and image contexts. The deterministic judge scores final-answer + * agreement and surfaces modality/document metadata for diagnostics. + */ + +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { benchRoot } from './_harness' +import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' +import { + FINAL_ANSWER_SENTINEL, + allStrings, + answerScoreToBenchScore, + contextBlock, + contextsFrom, + firstString, + isObject, + ragAnswerOutput, + readJsonRows, + scoreAnswerArtifact, + selectTasks, + stringFrom, + type RagContext, +} from './rag-shared' + +const FIXTURES = join(benchRoot, 'fixtures', 'open-rag-bench.json') + +interface OpenRagBenchMeta { + benchmark: 'open-rag-bench' + query: string + goldAnswers: string[] + contexts: RagContext[] + documentId: string + modality: string +} + +const dataFile = (): string | undefined => process.env.OPEN_RAG_BENCH_DATA_FILE + +function rowToTask(raw: unknown, index: number): BenchTask { + if (!isObject(raw)) throw new Error(`Open RAG Bench row ${index} must be an object`) + const query = firstString(raw, ['question', 'query', 'prompt']) + const goldAnswers = allStrings(raw, ['answer', 'answers', 'reference', 'reference_answer', 'gold']) + if (!query) throw new Error(`Open RAG Bench row ${index} missing question/query`) + if (goldAnswers.length === 0) throw new Error(`Open RAG Bench row ${index} missing answer`) + const contexts = + contextsFrom(raw.contexts).length > 0 + ? contextsFrom(raw.contexts) + : contextsFrom(raw.chunks).length > 0 + ? contextsFrom(raw.chunks) + : contextsFrom(raw.pages) + const documentId = + stringFrom(raw.document_id) ?? + stringFrom(raw.doc_id) ?? + stringFrom(raw.pdf_id) ?? + stringFrom(raw.file_name) ?? + stringFrom(raw.filename) ?? + 'unknown' + const modality = stringFrom(raw.modality) ?? stringFrom(raw.answer_modality) ?? 'unknown' + const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `open-rag-bench-${index}` + const meta: OpenRagBenchMeta = { + benchmark: 'open-rag-bench', + query, + goldAnswers, + contexts, + documentId, + modality, + } + return { + id, + split: stringFrom(raw.split) ?? modality, + prompt: [ + 'Answer this Open RAG Bench PDF question using the supplied document context.', + 'Use tables, text, and image-derived notes when present.', + 'End with a single final line: `FINAL ANSWER: `.', + '', + `Question: ${query}`, + `Document: ${documentId}`, + `Modality: ${modality}`, + contexts.length > 0 ? `\nDocument context:\n${contextBlock(contexts)}` : undefined, + ] + .filter(Boolean) + .join('\n'), + metadata: meta as unknown as Record, + } +} + +function readMeta(task: BenchTask): OpenRagBenchMeta { + const md = task.metadata + if (!md || !Array.isArray(md.goldAnswers)) { + throw new Error(`Open RAG Bench task ${task.id} missing metadata — loadTasks did not populate it`) + } + return md as unknown as OpenRagBenchMeta +} + +async function loadRows(path: string): Promise { + const rows = await readJsonRows(path) + if (rows.length === 0) throw new Error(`Open RAG Bench: no rows in ${path}`) + return rows +} + +async function loadFixtures(opts: LoadOptions): Promise { + const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[] + console.warn(`[open-rag-bench] OPEN_RAG_BENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`) + return selectTasks(rows.map(rowToTask), opts, 'Open RAG Bench') +} + +export function createOpenRagBenchAdapter(): BenchmarkAdapter { + const fixturesMode = process.env.OPEN_RAG_BENCH_FIXTURES === '1' + + return { + name: 'open-rag-bench', + output: ragAnswerOutput, + + async preflight() { + if (fixturesMode) { + await readFile(FIXTURES, 'utf8') + return + } + const path = dataFile() + if (!path) { + throw new Error( + 'OPEN_RAG_BENCH_DATA_FILE is required. Fix: export vectara/open-rag-bench rows to JSONL and set OPEN_RAG_BENCH_DATA_FILE=/path/to/open-rag-bench.jsonl, or set OPEN_RAG_BENCH_FIXTURES=1 for adapter plumbing.', + ) + } + await loadRows(path) + }, + + async loadTasks(opts: LoadOptions = {}) { + if (fixturesMode) return loadFixtures(opts) + const path = dataFile() + if (!path) throw new Error('OPEN_RAG_BENCH_DATA_FILE is required to load Open RAG Bench tasks') + return selectTasks((await loadRows(path)).map(rowToTask), opts, 'Open RAG Bench') + }, + + async goldArtifact(task: BenchTask) { + return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}` + }, + + async judge(task: BenchTask, artifact: string): Promise { + const meta = readMeta(task) + const score = scoreAnswerArtifact(artifact, meta.goldAnswers) + return answerScoreToBenchScore(score, { + benchmark: meta.benchmark, + documentId: meta.documentId, + modality: meta.modality, + contextCount: meta.contexts.length, + }) + }, + } +} diff --git a/bench/src/benchmarks/rag-benchmarks.test.mts b/bench/src/benchmarks/rag-benchmarks.test.mts new file mode 100644 index 00000000..8f661b77 --- /dev/null +++ b/bench/src/benchmarks/rag-benchmarks.test.mts @@ -0,0 +1,121 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { resolveAdapter } from '../adapters' +import { createCragAdapter } from './crag' +import { createNoMiraclAdapter } from './nomiracl' +import { createOpenRagBenchAdapter } from './open-rag-bench' +import { createRagBenchAdapter } from './ragbench' +import { FINAL_ANSWER_SENTINEL } from './rag-shared' +import { createT2RagBenchAdapter } from './t2-ragbench' + +async function withEnv(patch: Record, fn: () => Promise): Promise { + const old: Record = {} + for (const [key, value] of Object.entries(patch)) { + old[key] = process.env[key] + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + try { + return await fn() + } finally { + for (const [key, value] of Object.entries(old)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + } +} + +test('RAG benchmark adapters are registered', () => { + for (const key of ['ragbench', 'crag', 'nomiracl', 'open-rag-bench', 't2-ragbench']) { + assert.equal(resolveAdapter(key).name, key) + } +}) + +test('RAGBench fixture mode loads rows and scores final answers', async () => { + await withEnv({ RAGBENCH_FIXTURES: '1', RAGBENCH_DATA_FILE: undefined }, async () => { + const adapter = createRagBenchAdapter() + await adapter.preflight() + const [task] = await adapter.loadTasks({ limit: 1 }) + assert.equal(task.id, 'ragbench-fixture-0') + const gold = await adapter.goldArtifact(task) + assert.equal((await adapter.judge(task, gold ?? '')).resolved, true) + assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} 90 days`)).resolved, false) + }) + + await withEnv({ RAGBENCH_FIXTURES: undefined, RAGBENCH_DATA_FILE: undefined }, async () => { + await assert.rejects(() => createRagBenchAdapter().preflight(), /RAGBENCH_DATA_FILE is required/) + }) +}) + +test('CRAG fixture mode exposes domain slices and answer judging', async () => { + await withEnv({ CRAG_FIXTURES: '1', CRAG_DATA_FILE: undefined }, async () => { + const adapter = createCragAdapter() + await adapter.preflight() + const [task] = await adapter.loadTasks({ limit: 1 }) + assert.equal(task.split, 'finance') + assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} 12.4 billion dollars`)).score, 1) + assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} 9 billion dollars`)).resolved, false) + }) + + await withEnv({ CRAG_FIXTURES: undefined, CRAG_DATA_FILE: undefined }, async () => { + await assert.rejects(() => createCragAdapter().preflight(), /CRAG_DATA_FILE is required/) + }) +}) + +test('NoMIRACL fixture mode scores answerable and unanswerable classifications', async () => { + await withEnv({ NOMIRACL_FIXTURES: '1', NOMIRACL_DATA_FILE: undefined }, async () => { + const adapter = createNoMiraclAdapter() + await adapter.preflight() + const tasks = await adapter.loadTasks({}) + const relevant = tasks.find((task) => task.id === 'nomiracl-fixture-relevant') + const nonRelevant = tasks.find((task) => task.id === 'nomiracl-fixture-non-relevant') + assert.ok(relevant) + assert.ok(nonRelevant) + assert.equal((await adapter.judge(relevant, 'ANSWERABLE')).score, 1) + assert.equal((await adapter.judge(nonRelevant, 'UNANSWERABLE')).score, 1) + assert.equal((await adapter.judge(nonRelevant, 'ANSWERABLE')).score, 0) + }) + + await withEnv({ NOMIRACL_FIXTURES: undefined, NOMIRACL_DATA_FILE: undefined }, async () => { + await assert.rejects(() => createNoMiraclAdapter().preflight(), /NOMIRACL_DATA_FILE is required/) + }) +}) + +test('Open RAG Bench fixture mode keeps PDF modality metadata and scores answers', async () => { + await withEnv( + { OPEN_RAG_BENCH_FIXTURES: '1', OPEN_RAG_BENCH_DATA_FILE: undefined }, + async () => { + const adapter = createOpenRagBenchAdapter() + await adapter.preflight() + const [task] = await adapter.loadTasks({ limit: 1 }) + assert.equal(task.split, 'table') + assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} Hybrid BM25`)).resolved, true) + assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} Dense`)).resolved, false) + }, + ) + + await withEnv( + { OPEN_RAG_BENCH_FIXTURES: undefined, OPEN_RAG_BENCH_DATA_FILE: undefined }, + async () => { + await assert.rejects( + () => createOpenRagBenchAdapter().preflight(), + /OPEN_RAG_BENCH_DATA_FILE is required/, + ) + }, + ) +}) + +test('T2-RAGBench fixture mode scores numeric text-table answers', async () => { + await withEnv({ T2_RAGBENCH_FIXTURES: '1', T2_RAGBENCH_DATA_FILE: undefined }, async () => { + const adapter = createT2RagBenchAdapter() + await adapter.preflight() + const [task] = await adapter.loadTasks({ limit: 1 }) + assert.equal(task.split, 'finqa') + assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} 12.0 percent`)).resolved, true) + assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} 8 percent`)).resolved, false) + }) + + await withEnv({ T2_RAGBENCH_FIXTURES: undefined, T2_RAGBENCH_DATA_FILE: undefined }, async () => { + await assert.rejects(() => createT2RagBenchAdapter().preflight(), /T2_RAGBENCH_DATA_FILE is required/) + }) +}) diff --git a/bench/src/benchmarks/rag-shared.ts b/bench/src/benchmarks/rag-shared.ts new file mode 100644 index 00000000..35f63ba9 --- /dev/null +++ b/bench/src/benchmarks/rag-shared.ts @@ -0,0 +1,318 @@ +import { readFile } from 'node:fs/promises' +import type { OutputAdapter } from '@tangle-network/agent-runtime/loops' +import type { BenchScore, BenchTask, LoadOptions } from './types' + +export const FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:' + +export interface RagContext { + id: string + text: string + title?: string + source?: string + relevant?: boolean +} + +export interface RagAnswerScore { + resolved: boolean + score: number + finalAnswer: string + bestGold: string | null + exact: boolean + numeric: boolean + f1: number + threshold: number +} + +export const ragAnswerOutput: OutputAdapter = { + parse(events) { + let text = '' + for (const ev of events) { + const d = (ev as { data?: Record })?.data + const t = d?.finalText ?? d?.text ?? d?.result + if (typeof t === 'string' && t.length > 0) text = t + } + return text.trim() + }, +} + +export function parseFinalAnswer(artifact: string): string { + const lines = artifact.split(/\r?\n/) + for (let i = lines.length - 1; i >= 0; i -= 1) { + const line = lines[i] ?? '' + const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL) + if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim() + } + for (let i = lines.length - 1; i >= 0; i -= 1) { + const trimmed = (lines[i] ?? '').trim() + if (trimmed.length > 0) return trimmed + } + return '' +} + +export function parseCitations(artifact: string): string[] { + const urls = new Set() + for (const match of artifact.matchAll(/https?:\/\/[^\s)<>"']+/g)) { + urls.add(match[0].replace(/[.,;]+$/, '')) + } + return [...urls] +} + +export function normalizeAnswer(input: string): string { + return input + .toLowerCase() + .replace(/(\d),(?=\d{3}\b)/g, '$1') + .replace(/[^\p{L}\p{N}\s.-]+/gu, ' ') + .split(/\s+/) + .filter((token) => token.length > 0) + .filter((token) => !['a', 'an', 'the'].includes(token)) + .join(' ') + .trim() +} + +function tokens(input: string): string[] { + const normalized = normalizeAnswer(input) + return normalized.length === 0 ? [] : normalized.split(/\s+/) +} + +export function tokenF1(candidate: string, gold: string): number { + const candidateTokens = tokens(candidate) + const goldTokens = tokens(gold) + if (candidateTokens.length === 0 || goldTokens.length === 0) { + return candidateTokens.length === 0 && goldTokens.length === 0 ? 1 : 0 + } + const counts = new Map() + for (const token of goldTokens) counts.set(token, (counts.get(token) ?? 0) + 1) + let common = 0 + for (const token of candidateTokens) { + const left = counts.get(token) + if (left !== undefined && left > 0) { + common += 1 + counts.set(token, left - 1) + } + } + if (common === 0) return 0 + const precision = common / candidateTokens.length + const recall = common / goldTokens.length + return (2 * precision * recall) / (precision + recall) +} + +function exactOrContains(candidate: string, gold: string): boolean { + const normalizedCandidate = normalizeAnswer(candidate) + const normalizedGold = normalizeAnswer(gold) + if (normalizedGold.length === 0) return false + if (normalizedCandidate === normalizedGold) return true + const escaped = normalizedGold.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + return new RegExp(`(^|\\s)${escaped}(\\s|$)`).test(normalizedCandidate) +} + +function numbers(input: string): number[] { + return [...input.replace(/,/g, '').matchAll(/-?\d+(?:\.\d+)?/g)] + .map((match) => Number(match[0])) + .filter(Number.isFinite) +} + +function numericMatch(candidate: string, gold: string, relativeTolerance: number): boolean { + const got = numbers(candidate) + const want = numbers(gold) + if (got.length === 0 || want.length === 0) return false + return want.some((expected) => + got.some((actual) => { + const tolerance = Math.max(Math.abs(expected) * relativeTolerance, relativeTolerance) + return Math.abs(actual - expected) <= tolerance + }), + ) +} + +export function scoreAnswerArtifact( + artifact: string, + golds: readonly string[], + options: { threshold?: number; numericTolerance?: number } = {}, +): RagAnswerScore { + const finalAnswer = parseFinalAnswer(artifact) + const threshold = options.threshold ?? 0.72 + const numericTolerance = options.numericTolerance ?? 0.01 + let best: RagAnswerScore = { + resolved: false, + score: 0, + finalAnswer, + bestGold: null, + exact: false, + numeric: false, + f1: 0, + threshold, + } + if (finalAnswer.length === 0 || golds.length === 0) return best + + for (const gold of golds) { + const exact = exactOrContains(finalAnswer, gold) + const numeric = numericMatch(finalAnswer, gold, numericTolerance) + const f1 = tokenF1(finalAnswer, gold) + const resolved = exact || numeric || f1 >= threshold + const score = exact || numeric ? 1 : f1 + if (score > best.score || (resolved && !best.resolved)) { + best = { resolved, score, finalAnswer, bestGold: gold, exact, numeric, f1, threshold } + } + } + return best +} + +export function answerScoreToBenchScore( + score: RagAnswerScore, + detail: Record, +): BenchScore { + return { + resolved: score.resolved, + score: score.score, + detail: JSON.stringify({ + ...detail, + finalAnswer: score.finalAnswer, + bestGold: score.bestGold, + exact: score.exact, + numeric: score.numeric, + f1: score.f1, + threshold: score.threshold, + }), + } +} + +export async function readJsonRows(path: string): Promise { + const raw = (await readFile(path, 'utf8')).trim() + if (raw.length === 0) return [] + if (raw.startsWith('[') || raw.startsWith('{')) { + const parsed = JSON.parse(raw) as unknown + if (Array.isArray(parsed)) return parsed + if (isObject(parsed)) { + for (const key of ['rows', 'data', 'examples', 'items']) { + const value = parsed[key] + if (Array.isArray(value)) return value + } + } + throw new Error(`${path} must contain a JSON array, JSONL rows, or an object with rows/data/examples/items`) + } + return raw + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as unknown) +} + +export function selectTasks(tasks: BenchTask[], opts: LoadOptions, label: string): BenchTask[] { + let selected = tasks + if (opts.split) selected = selected.filter((task) => task.split === opts.split) + if (opts.ids) { + const ids = new Set(opts.ids) + selected = selected.filter((task) => ids.has(task.id)) + } else if (opts.limit !== undefined) { + selected = selected.slice(0, opts.limit) + } + if (selected.length === 0) throw new Error(`${label}: no tasks matched ${JSON.stringify(opts)}`) + return selected +} + +export function stringFrom(value: unknown): string | undefined { + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined +} + +export function stringArrayFrom(value: unknown): string[] { + if (typeof value === 'string' && value.trim().length > 0) return [value.trim()] + if (!Array.isArray(value)) return [] + return value.flatMap((entry) => { + if (typeof entry === 'string' && entry.trim().length > 0) return [entry.trim()] + if (isObject(entry)) { + const text = stringFrom(entry.answer) ?? stringFrom(entry.value) ?? stringFrom(entry.text) + return text ? [text] : [] + } + return [] + }) +} + +export function firstString(row: Record, keys: readonly string[]): string { + for (const key of keys) { + const value = stringFrom(row[key]) + if (value) return value + } + return '' +} + +export function allStrings(row: Record, keys: readonly string[]): string[] { + const out: string[] = [] + for (const key of keys) out.push(...stringArrayFrom(row[key])) + return [...new Set(out)] +} + +export function contextsFrom(value: unknown): RagContext[] { + if (typeof value === 'string' && value.trim().length > 0) { + return [{ id: 'ctx-1', text: value.trim() }] + } + if (!Array.isArray(value)) return [] + return value.flatMap((entry, index): RagContext[] => { + if (typeof entry === 'string' && entry.trim().length > 0) { + return [{ id: `ctx-${index + 1}`, text: entry.trim() }] + } + if (Array.isArray(entry)) { + const [first, second] = entry + if (typeof first === 'string' && typeof second === 'string') { + return [{ id: first, text: second.trim() }] + } + return entry.flatMap((nested, nestedIndex) => + contextsFrom([nested]).map((ctx) => ({ + ...ctx, + id: ctx.id.startsWith('ctx-') ? `ctx-${index + 1}-${nestedIndex + 1}` : ctx.id, + })), + ) + } + if (!isObject(entry)) return [] + const text = + stringFrom(entry.text) ?? + stringFrom(entry.context) ?? + stringFrom(entry.content) ?? + stringFrom(entry.passage) ?? + stringFrom(entry.document) ?? + stringFrom(entry.page_content) ?? + stringFrom(entry.chunk) + if (!text) return [] + const id = + stringFrom(entry.id) ?? + stringFrom(entry.docid) ?? + stringFrom(entry.doc_id) ?? + stringFrom(entry.document_id) ?? + `ctx-${index + 1}` + const relevantRaw = entry.relevant ?? entry.is_relevant ?? entry.label ?? entry.relevance + const relevant = + typeof relevantRaw === 'boolean' + ? relevantRaw + : typeof relevantRaw === 'number' + ? relevantRaw > 0 + : undefined + return [ + { + id, + text, + ...(stringFrom(entry.title) ? { title: stringFrom(entry.title) } : {}), + ...(stringFrom(entry.source) ?? stringFrom(entry.url) + ? { source: stringFrom(entry.source) ?? stringFrom(entry.url) } + : {}), + ...(relevant !== undefined ? { relevant } : {}), + }, + ] + }) +} + +export function contextBlock(contexts: readonly RagContext[]): string { + if (contexts.length === 0) return '' + return contexts + .map((ctx, index) => + [ + `[${index + 1}] ${ctx.title ?? ctx.id}`, + ctx.source ? `Source: ${ctx.source}` : undefined, + ctx.text, + ] + .filter(Boolean) + .join('\n'), + ) + .join('\n\n') +} + +export function isObject(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) +} diff --git a/bench/src/benchmarks/ragbench.ts b/bench/src/benchmarks/ragbench.ts new file mode 100644 index 00000000..f6487c63 --- /dev/null +++ b/bench/src/benchmarks/ragbench.ts @@ -0,0 +1,171 @@ +/** + * RAGBench-compatible adapter. + * + * Live mode expects a local JSON/JSONL export from rungalileo/ragbench or a + * compatible table. Rows must carry a query and at least one reference answer. + * Contexts, TRACe labels, and source metadata are preserved in task metadata + * for diagnostics; the deterministic judge scores the worker's final answer + * against the reference answer(s). + */ + +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { benchRoot } from './_harness' +import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' +import { + FINAL_ANSWER_SENTINEL, + allStrings, + answerScoreToBenchScore, + contextBlock, + contextsFrom, + firstString, + isObject, + ragAnswerOutput, + readJsonRows, + scoreAnswerArtifact, + selectTasks, + stringFrom, + type RagContext, +} from './rag-shared' + +const FIXTURES = join(benchRoot, 'fixtures', 'ragbench.json') + +interface RagBenchMeta { + benchmark: 'ragbench' + query: string + goldAnswers: string[] + contexts: RagContext[] + dataset?: string + traceLabels: Record +} + +const dataFile = (): string | undefined => process.env.RAGBENCH_DATA_FILE + +function rowToTask(raw: unknown, index: number): BenchTask { + if (!isObject(raw)) throw new Error(`RAGBench row ${index} must be an object`) + const query = firstString(raw, ['question', 'query', 'user_input', 'prompt']) + const goldAnswers = allStrings(raw, [ + 'response', + 'responses', + 'model_response', + 'generated_response', + 'reference', + 'references', + 'reference_answer', + 'reference_answers', + 'answer', + 'answers', + 'gold', + 'gold_answer', + 'ground_truth', + 'expected_answer', + ]) + if (!query) throw new Error(`RAGBench row ${index} missing question/query`) + if (goldAnswers.length === 0) throw new Error(`RAGBench row ${index} missing reference answer`) + const contexts = + contextsFrom(raw.contexts).length > 0 + ? contextsFrom(raw.contexts) + : contextsFrom(raw.retrieved_contexts).length > 0 + ? contextsFrom(raw.retrieved_contexts) + : contextsFrom(raw.documents) + const traceLabels: Record = {} + for (const key of [ + 'adherence', + 'completeness', + 'relevance', + 'utilization', + 'all_relevant_sentence_keys', + 'all_utilized_sentence_keys', + ]) { + if (raw[key] !== undefined) traceLabels[key] = raw[key] + } + const dataset = stringFrom(raw.dataset) ?? stringFrom(raw.source_dataset) + const id = stringFrom(raw.id) ?? stringFrom(raw.example_id) ?? `ragbench-${index}` + const meta: RagBenchMeta = { + benchmark: 'ragbench', + query, + goldAnswers, + contexts, + ...(dataset ? { dataset } : {}), + traceLabels, + } + return { + id, + split: stringFrom(raw.split) ?? dataset ?? 'ragbench', + prompt: [ + 'Answer this RAGBench question using the supplied retrieved context.', + 'End with a single final line: `FINAL ANSWER: `.', + '', + `Question: ${query}`, + contexts.length > 0 ? `\nRetrieved context:\n${contextBlock(contexts)}` : undefined, + ] + .filter(Boolean) + .join('\n'), + metadata: meta as unknown as Record, + } +} + +function readMeta(task: BenchTask): RagBenchMeta { + const md = task.metadata + if (!md || !Array.isArray(md.goldAnswers)) { + throw new Error(`RAGBench task ${task.id} missing metadata — loadTasks did not populate it`) + } + return md as unknown as RagBenchMeta +} + +async function loadRows(path: string): Promise { + const rows = await readJsonRows(path) + if (rows.length === 0) throw new Error(`RAGBench: no rows in ${path}`) + return rows +} + +async function loadFixtures(opts: LoadOptions): Promise { + const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[] + console.warn(`[ragbench] RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`) + return selectTasks(rows.map(rowToTask), opts, 'RAGBench') +} + +export function createRagBenchAdapter(): BenchmarkAdapter { + const fixturesMode = process.env.RAGBENCH_FIXTURES === '1' + + return { + name: 'ragbench', + output: ragAnswerOutput, + + async preflight() { + if (fixturesMode) { + await readFile(FIXTURES, 'utf8') + return + } + const path = dataFile() + if (!path) { + throw new Error( + 'RAGBENCH_DATA_FILE is required. Fix: export rungalileo/ragbench rows to JSONL and set RAGBENCH_DATA_FILE=/path/to/ragbench.jsonl, or set RAGBENCH_FIXTURES=1 for adapter plumbing.', + ) + } + await loadRows(path) + }, + + async loadTasks(opts: LoadOptions = {}) { + if (fixturesMode) return loadFixtures(opts) + const path = dataFile() + if (!path) throw new Error('RAGBENCH_DATA_FILE is required to load RAGBench tasks') + return selectTasks((await loadRows(path)).map(rowToTask), opts, 'RAGBench') + }, + + async goldArtifact(task: BenchTask) { + return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}` + }, + + async judge(task: BenchTask, artifact: string): Promise { + const meta = readMeta(task) + const score = scoreAnswerArtifact(artifact, meta.goldAnswers) + return answerScoreToBenchScore(score, { + benchmark: meta.benchmark, + dataset: meta.dataset ?? null, + traceLabels: meta.traceLabels, + contextCount: meta.contexts.length, + }) + }, + } +} diff --git a/bench/src/benchmarks/t2-ragbench.ts b/bench/src/benchmarks/t2-ragbench.ts new file mode 100644 index 00000000..ab9dc98b --- /dev/null +++ b/bench/src/benchmarks/t2-ragbench.ts @@ -0,0 +1,166 @@ +/** + * T2-RAGBench adapter. + * + * T2-RAGBench stresses text+table retrieval and numerical reasoning over + * financial documents. The judge uses the shared deterministic answer scorer + * with numeric tolerance enabled by default. + */ + +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { benchRoot } from './_harness' +import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' +import { + FINAL_ANSWER_SENTINEL, + allStrings, + answerScoreToBenchScore, + contextBlock, + contextsFrom, + firstString, + isObject, + ragAnswerOutput, + readJsonRows, + scoreAnswerArtifact, + selectTasks, + stringFrom, + type RagContext, +} from './rag-shared' + +const FIXTURES = join(benchRoot, 'fixtures', 't2-ragbench.json') + +interface T2RagBenchMeta { + benchmark: 't2-ragbench' + query: string + goldAnswers: string[] + contexts: RagContext[] + subset: string + documentId: string +} + +const dataFile = (): string | undefined => process.env.T2_RAGBENCH_DATA_FILE + +function rowToTask(raw: unknown, index: number): BenchTask { + if (!isObject(raw)) throw new Error(`T2-RAGBench row ${index} must be an object`) + const query = firstString(raw, ['question', 'query', 'prompt']) + const goldAnswers = allStrings(raw, [ + 'program_answer', + 'original_answer', + 'answer', + 'answers', + 'reference', + 'reference_answer', + 'gold', + ]) + if (!query) throw new Error(`T2-RAGBench row ${index} missing question/query`) + if (goldAnswers.length === 0) throw new Error(`T2-RAGBench row ${index} missing answer`) + const baseContexts = + contextsFrom(raw.context).length > 0 + ? contextsFrom(raw.context) + : contextsFrom(raw.contexts).length > 0 + ? contextsFrom(raw.contexts) + : contextsFrom(raw.chunks).length > 0 + ? contextsFrom(raw.chunks) + : contextsFrom(raw.passages) + const table = stringFrom(raw.table) ?? stringFrom(raw.table_text) + const contexts = table + ? [...baseContexts, { id: 'table', title: 'Table', text: table }] + : baseContexts + const subset = stringFrom(raw.subset) ?? stringFrom(raw.dataset) ?? 'unknown' + const documentId = + stringFrom(raw.context_id) ?? + stringFrom(raw.document_id) ?? + stringFrom(raw.doc_id) ?? + stringFrom(raw.file_name) ?? + 'unknown' + const id = stringFrom(raw.id) ?? stringFrom(raw.qid) ?? stringFrom(raw.query_id) ?? `t2-ragbench-${index}` + const meta: T2RagBenchMeta = { + benchmark: 't2-ragbench', + query, + goldAnswers, + contexts, + subset, + documentId, + } + return { + id, + split: stringFrom(raw.split) ?? subset, + prompt: [ + 'Answer this T2-RAGBench text-and-table financial question.', + 'Do the required numerical reasoning from the supplied context before giving the final value.', + 'End with a single final line: `FINAL ANSWER: `.', + '', + `Question: ${query}`, + `Document: ${documentId}`, + `Subset: ${subset}`, + contexts.length > 0 ? `\nContext:\n${contextBlock(contexts)}` : undefined, + ] + .filter(Boolean) + .join('\n'), + metadata: meta as unknown as Record, + } +} + +function readMeta(task: BenchTask): T2RagBenchMeta { + const md = task.metadata + if (!md || !Array.isArray(md.goldAnswers)) { + throw new Error(`T2-RAGBench task ${task.id} missing metadata — loadTasks did not populate it`) + } + return md as unknown as T2RagBenchMeta +} + +async function loadRows(path: string): Promise { + const rows = await readJsonRows(path) + if (rows.length === 0) throw new Error(`T2-RAGBench: no rows in ${path}`) + return rows +} + +async function loadFixtures(opts: LoadOptions): Promise { + const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[] + console.warn(`[t2-ragbench] T2_RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`) + return selectTasks(rows.map(rowToTask), opts, 'T2-RAGBench') +} + +export function createT2RagBenchAdapter(): BenchmarkAdapter { + const fixturesMode = process.env.T2_RAGBENCH_FIXTURES === '1' + + return { + name: 't2-ragbench', + output: ragAnswerOutput, + + async preflight() { + if (fixturesMode) { + await readFile(FIXTURES, 'utf8') + return + } + const path = dataFile() + if (!path) { + throw new Error( + 'T2_RAGBENCH_DATA_FILE is required. Fix: export T2-RAGBench rows to JSONL and set T2_RAGBENCH_DATA_FILE=/path/to/t2-ragbench.jsonl, or set T2_RAGBENCH_FIXTURES=1 for adapter plumbing.', + ) + } + await loadRows(path) + }, + + async loadTasks(opts: LoadOptions = {}) { + if (fixturesMode) return loadFixtures(opts) + const path = dataFile() + if (!path) throw new Error('T2_RAGBENCH_DATA_FILE is required to load T2-RAGBench tasks') + return selectTasks((await loadRows(path)).map(rowToTask), opts, 'T2-RAGBench') + }, + + async goldArtifact(task: BenchTask) { + return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}` + }, + + async judge(task: BenchTask, artifact: string): Promise { + const meta = readMeta(task) + const score = scoreAnswerArtifact(artifact, meta.goldAnswers, { numericTolerance: 0.01 }) + return answerScoreToBenchScore(score, { + benchmark: meta.benchmark, + subset: meta.subset, + documentId: meta.documentId, + contextCount: meta.contexts.length, + }) + }, + } +} diff --git a/bench/src/index.ts b/bench/src/index.ts index a5ba7266..a3d522d2 100644 --- a/bench/src/index.ts +++ b/bench/src/index.ts @@ -11,6 +11,25 @@ * absent, so importing the registry is cheap; running a specific benchmark pulls only its deps. */ export { ADAPTERS, resolveAdapter } from './adapters' +export { createCragAdapter } from './benchmarks/crag' +export { createNoMiraclAdapter } from './benchmarks/nomiracl' +export { createOpenRagBenchAdapter } from './benchmarks/open-rag-bench' +export { createRagBenchAdapter } from './benchmarks/ragbench' +export { + FINAL_ANSWER_SENTINEL, + answerScoreToBenchScore, + contextBlock, + contextsFrom, + normalizeAnswer, + parseCitations, + parseFinalAnswer, + ragAnswerOutput, + scoreAnswerArtifact, + tokenF1, + type RagAnswerScore, + type RagContext, +} from './benchmarks/rag-shared' +export { createT2RagBenchAdapter } from './benchmarks/t2-ragbench' export type { BenchmarkAdapter, BenchScore,