feat(bench): add RAG benchmark adapters

bench/HARNESS.md

@@ -3,7 +3,7 @@
 If you're an agent picking this up: read this page, then run `pnpm help` + `pnpm gate` —
 do NOT re-derive the harness from source. This map is SHORT on purpose; if it disagrees
 with the code, the code wins — fix this page in the same turn (the anti-rediscovery law).
-Verified against source 2026-06-10 · agent-eval pinned `^0.83.0`. The CANONICAL surface is now
+Verified against source 2026-07-07 · agent-eval pinned `^0.106.1`. The CANONICAL surface is now
 the published optimization suite (`@tangle-network/agent-runtime/loops`): `Environment` +
 `Strategy`/`defineStrategy` + `runBenchmark` — see the section below FIRST. The recursive
 diverse-vs-blind gate runs through the keystone (`gate-cli.mts` → `runGate`);
@@ -202,11 +202,11 @@ The code-benches share `benchmarks/_harness.ts` (stage artifact → run the benc
 in a `.venv`/Docker subprocess → parse its JSON report → `{resolved,score}`). No per-adapter
 copy of the process/venv/Docker/temp/report plumbing; commit0+appworld also share its
 stdin-piping runner (`runVenvScriptStdin`).
-- **Real, runnable with ZERO extra deps:** finsearchcomp (GitHub dataset + fixtures + LLM judge — the gate bench), hotpotqa + simpleqa + frames (HF/web QA + F1/LLM judge; `*_FIXTURES=1` offline), **aec-bench** (real GitHub task tree + fixtures; judge = the task's own `tests/verify.py` over python3 stdlib — **deterministic, graded per-field partial credit, no Docker, no LLM** → the candidate non-oracle correctable-middle-band bench for the open gate).
+- **Real, runnable with ZERO extra deps:** finsearchcomp (GitHub dataset + fixtures + LLM judge — the gate bench), hotpotqa + simpleqa + frames (HF/web QA + F1/LLM judge; `*_FIXTURES=1` offline), **ragbench**, **crag**, **nomiracl**, **open-rag-bench**, **t2-ragbench** (SOTA RAG/knowledge benchmarks with committed fixtures and deterministic answer/relevance judges; live mode reads explicit `*_DATA_FILE` JSON/JSONL exports), **aec-bench** (real GitHub task tree + fixtures; judge = the task's own `tests/verify.py` over python3 stdlib — **deterministic, graded per-field partial credit, no Docker, no LLM** → the candidate non-oracle correctable-middle-band bench for the open gate).
 - **Real code, needs an external harness/tools to run (fail loud with the exact install/Docker fix; never a fabricated score):** swe-bench + terminal-bench (`bench/.venv` + Docker), **commit0** (ISOLATED `bench/.venv-commit0` via `python3 -m venv bench/.venv-commit0 && bench/.venv-commit0/bin/pip install commit0 datasets` — its deps conflict with the shared `.venv`; override dir with `COMMIT0_VENV` — plus Docker; judge = official pytest harness, graded (passed+xfail)/total; the rollout prompt stages in-box (clones `commit-0/<repo>` @ `base_commit`, emits `git diff`); `COMMIT0_FIXTURES=1` for offline listing), **programbench** (`pip install programbench` + Docker on linux/amd64 + HF blobs; judge = official cleanroom eval, graded passed/total; `PROGRAMBENCH_FIXTURES=1` offline), **appworld** (`pip install appworld` + `appworld install` + `appworld download data`; judge = AppWorld's own `world.evaluate()`, graded passes/num_tests — NO committed fixture: task data exists only after `download data`, so loadTasks fails loud rather than fabricate a task), **dabstep** (`DABSTEP_DIR=/path/to/EnvCommons/DABStep` with the released `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`; judge delegates to official `grade.py`; `DABSTEP_FIXTURES=1` only tests adapter plumbing and does not fabricate benchmark scores), **webarena-verified** (`WEBARENA_VERIFIED_DIR=/path/to/webarena-verified`; judge delegates to official `eval-tasks` over a run output directory), **tau2-bench** (`TAU2_BENCH_DIR=/path/to/tau2-bench`; judge recomputes tau2 trajectory rewards), **tau3-banking** (`TAU3_BENCH_DIR=/path/to/tau2-bench`; default domain `banking_knowledge`; judge recomputes tau trajectory rewards through the upstream tau3 package), **agentbench** DBBench subset (`AGENTBENCH_DIR=/path/to/AgentBench`; exact-match deterministic label judge), **bfcl** deterministic function-call subset (`BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard`; loads official BFCL JSONL + `possible_answer`; score = structured call/argument match, not the full BFCL leaderboard evaluator), **toollm** API-selection subset (`TOOLBENCH_DIR=/path/to/ToolBench`; score = recall of ToolBench `relevant APIs` labels, resolved only when the worker emits the requested structured JSON call list; official ToolEval pass rate remains LLM-judged/stochastic), **finresearchbench** (`FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl`; rows must carry official `judge_system_prompt` + `judge_prompt_template`; no self-authored live judge), mind2web, cad-design + cadbench + cadgenbench (openscad/blender/build123d).
 - **goldArtifact:** aec-bench returns the task's real `golden_pass.md` (verify-judge works fully offline). commit0 / programbench / appworld return `undefined` — the oracle is a git ref / stripped source / engine-bundled solution, not a portable string; judge correctness is proven by a real solve through the harness, not a synthetic gold (documented + fail-loud, not a fake).
 - **Absent (not built):** swe-gym, swe-bench-multimodal, and the rest of the survey set.
-Every unbuilt/scaffold adapter fails LOUD (throws with the integration step) rather than faking a score — no silent zeros in any corpus. Offline fixture tests: `benchmarks/{aec-bench,commit0,programbench,appworld}.test.mts` (`tsx --test`).
+Every unbuilt/scaffold adapter fails LOUD (throws with the integration step) rather than faking a score — no silent zeros in any corpus. Offline fixture tests: `benchmarks/{aec-bench,commit0,programbench,appworld,rag-benchmarks}.test.mts` (`tsx --test`).
 
 ## Is it runnable RIGHT NOW? (verify the map, don't trust it blindly)
 ```

bench/fixtures/crag.json

@@ -0,0 +1,10 @@
+[
+  {
+    "id": "crag-fixture-0",
+    "domain": "finance",
+    "question_type": "simple",
+    "static_or_dynamic": "slow-changing",
+    "query": "What was Contoso's reported revenue in 2025?",
+    "answer": ["$12.4 billion", "12.4 billion dollars"]
+  }
+]

bench/fixtures/nomiracl.json

@@ -0,0 +1,26 @@
+[
+  {
+    "id": "nomiracl-fixture-relevant",
+    "language": "en",
+    "query": "What is the refund window?",
+    "positive_passages": [
+      {
+        "docid": "p1",
+        "title": "Refund policy",
+        "text": "The refund window is 30 days."
+      }
+    ]
+  },
+  {
+    "id": "nomiracl-fixture-non-relevant",
+    "language": "en",
+    "query": "What is the refund window?",
+    "negative_passages": [
+      {
+        "docid": "p2",
+        "title": "Shipping policy",
+        "text": "Shipping labels expire after 7 days."
+      }
+    ]
+  }
+]

bench/fixtures/open-rag-bench.json

@@ -0,0 +1,16 @@
+[
+  {
+    "id": "open-rag-bench-fixture-0",
+    "file_name": "arxiv-0001.pdf",
+    "modality": "table",
+    "question": "Which method has the highest accuracy in the table?",
+    "answer": "Hybrid BM25",
+    "contexts": [
+      {
+        "id": "table-1",
+        "title": "Accuracy table",
+        "text": "Method | Accuracy\nDense | 82.1\nHybrid BM25 | 87.4\nSparse | 79.0"
+      }
+    ]
+  }
+]

bench/fixtures/ragbench.json

@@ -0,0 +1,21 @@
+[
+  {
+    "id": "ragbench-fixture-0",
+    "dataset": "support",
+    "question": "How long is the refund window?",
+    "response": "30 days",
+    "documents": [
+      [
+        ["0a", "Title: Refund policy"],
+        ["0b", "Customers can request a refund within 30 days of purchase."]
+      ],
+      [
+        ["1a", "Title: Shipping policy"],
+        ["1b", "Shipping labels expire after 7 days."]
+      ]
+    ],
+    "adherence": 1,
+    "completeness": 1,
+    "relevance": 1
+  }
+]

bench/fixtures/t2-ragbench.json

@@ -0,0 +1,13 @@
+[
+  {
+    "id": "t2-ragbench-fixture-0",
+    "subset": "finqa",
+    "context_id": "10k-0001",
+    "question": "What is the percentage increase from 100 to 112?",
+    "program_answer": "12%",
+    "original_answer": "12.0 percent",
+    "context": "Revenue increased from 100 in 2024 to 112 in 2025.",
+    "table": "Year | Revenue\n2024 | 100\n2025 | 112",
+    "file_name": "10k-0001.pdf"
+  }
+]

bench/package.json

@@ -2,7 +2,7 @@
   "name": "@tangle-network/agent-bench",
   "version": "0.1.0",
   "type": "module",
-  "description": "The unified benchmark suite for agent-runtime agents: 26 adapters (commit0, enterpriseops-gym, tau3-banking, bfcl, finresearchbench, trata-hedge, finsearchcomp, dabstep, webarena-verified, tau2-bench, agentbench, swe-bench, humaneval, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
+  "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
   "main": "src/index.ts",
   "types": "src/index.ts",
   "exports": {
@@ -18,9 +18,9 @@
     "terminal-compare": "tsx src/terminal-compare.ts"
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "^0.103.2",
-    "@tangle-network/agent-runtime": "^0.88.0",
-    "@tangle-network/sandbox": "^0.9.5"
+    "@tangle-network/agent-eval": "^0.106.3",
+    "@tangle-network/agent-runtime": "^0.89.0",
+    "@tangle-network/sandbox": "^0.9.7"
   },
   "devDependencies": {
     "tsx": "^4.19.0",

bench/src/adapters.ts

@@ -13,6 +13,7 @@ import { createCadBenchAdapter } from './benchmarks/cadbench'
 import { createCadDesignAdapter } from './benchmarks/cad-design'
 import { createCadGenBenchAdapter } from './benchmarks/cadgenbench'
 import { createCommit0Adapter } from './benchmarks/commit0'
+import { createCragAdapter } from './benchmarks/crag'
 import { createDabstepAdapter } from './benchmarks/dabstep'
 import { createEnterpriseOpsGymAdapter } from './benchmarks/enterpriseops-gym'
 import { createFinResearchBenchAdapter } from './benchmarks/finresearchbench'
@@ -21,9 +22,13 @@ import { createFramesAdapter } from './benchmarks/frames'
 import { createHotpotqaAdapter } from './benchmarks/hotpotqa'
 import { createHumanEvalAdapter } from './benchmarks/humaneval'
 import { createMind2WebAdapter } from './benchmarks/mind2web'
+import { createNoMiraclAdapter } from './benchmarks/nomiracl'
+import { createOpenRagBenchAdapter } from './benchmarks/open-rag-bench'
 import { createProgrambenchAdapter } from './benchmarks/programbench'
+import { createRagBenchAdapter } from './benchmarks/ragbench'
 import { createSimpleQaAdapter } from './benchmarks/simpleqa'
 import { createSweBenchAdapter } from './benchmarks/swe-bench'
+import { createT2RagBenchAdapter } from './benchmarks/t2-ragbench'
 import { createTau2BenchAdapter } from './benchmarks/tau2-bench'
 import { createTau3BankingAdapter } from './benchmarks/tau3-banking'
 import { createTerminalBenchAdapter } from './benchmarks/terminal-bench'
@@ -57,6 +62,11 @@ export const ADAPTERS: Record<string, () => BenchmarkAdapter> = {
   cadbench: createCadBenchAdapter,
   cadgenbench: createCadGenBenchAdapter,
   frames: createFramesAdapter,
+  ragbench: createRagBenchAdapter,
+  crag: createCragAdapter,
+  nomiracl: createNoMiraclAdapter,
+  'open-rag-bench': createOpenRagBenchAdapter,
+  't2-ragbench': createT2RagBenchAdapter,
   finresearchbench: createFinResearchBenchAdapter,
   finsearchcomp: createFinsearchcompAdapter,
   simpleqa: createSimpleQaAdapter,

bench/src/benchmarks/appworld.test.mts

@@ -27,19 +27,25 @@ test('goldArtifact is undefined — reference solution ships only inside the eng
   assert.equal(await a.goldArtifact({ id: 't', prompt: '', metadata: { taskId: 't', split: 'dev' } }), undefined)
 })
 
-test('preflight FAILS LOUD with the install + download-data fix when the engine is absent', async () => {
+test('preflight passes when installed or FAILS LOUD with the install + download-data fix', async () => {
   const a = createAppWorldAdapter()
-  await assert.rejects(a.preflight(), (e: Error) => {
+  try {
+    await a.preflight()
+  } catch (err) {
+    const e = err as Error
     assert.match(e.message, /pip install appworld/)
     assert.match(e.message, /appworld download data/)
-    return true
-  })
+  }
 })
 
-test('loadTasks FAILS LOUD (engine enumeration) rather than fabricating tasks offline', async () => {
+test('loadTasks either enumerates live engine rows or FAILS LOUD without fabricating tasks', async () => {
   const a = createAppWorldAdapter()
-  await assert.rejects(a.loadTasks({ limit: 1 }), (e: Error) => {
-    assert.match(e.message, /appworld driver failed|appworld import failed/)
-    return true
-  })
+  try {
+    const tasks = await a.loadTasks({ limit: 1 })
+    assert.equal(tasks.length, 1)
+    assert.ok(tasks[0].id.length > 0)
+    assert.match(tasks[0].prompt, /Solve this by writing Python/)
+  } catch (err) {
+    assert.match((err as Error).message, /appworld driver failed|appworld import failed/)
+  }
 })