From b7fa6ebf900014480912b4dcb513a8ed3af9f4a3 Mon Sep 17 00:00:00 2001
From: Drew Stone <drewstone329@gmail.com>
Date: Tue, 7 Jul 2026 17:23:43 -0600
Subject: [PATCH] fix(bench): load object-starting RAG JSONL exports

---
 bench/src/benchmarks/rag-benchmarks.test.mts | 19 +++++++++++++++++-
 bench/src/benchmarks/rag-shared.ts           | 21 ++++++++++++++------
 2 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/bench/src/benchmarks/rag-benchmarks.test.mts b/bench/src/benchmarks/rag-benchmarks.test.mts
index 8f661b77..6705666d 100644
--- a/bench/src/benchmarks/rag-benchmarks.test.mts
+++ b/bench/src/benchmarks/rag-benchmarks.test.mts
@@ -1,11 +1,14 @@
 import assert from 'node:assert/strict'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
 import { test } from 'node:test'
 import { resolveAdapter } from '../adapters'
 import { createCragAdapter } from './crag'
 import { createNoMiraclAdapter } from './nomiracl'
 import { createOpenRagBenchAdapter } from './open-rag-bench'
 import { createRagBenchAdapter } from './ragbench'
-import { FINAL_ANSWER_SENTINEL } from './rag-shared'
+import { FINAL_ANSWER_SENTINEL, readJsonRows } from './rag-shared'
 import { createT2RagBenchAdapter } from './t2-ragbench'
 
 async function withEnv<T>(patch: Record<string, string | undefined>, fn: () => Promise<T>): Promise<T> {
@@ -31,6 +34,20 @@ test('RAG benchmark adapters are registered', () => {
   }
 })
 
+test('RAG JSON reader accepts object-starting JSONL exports', async () => {
+  const dir = await mkdtemp(join(tmpdir(), 'rag-jsonl-'))
+  try {
+    const file = join(dir, 'rows.jsonl')
+    await writeFile(file, '{"id":"a","question":"q1"}\n{"id":"b","question":"q2"}\n')
+    assert.deepEqual(await readJsonRows(file), [
+      { id: 'a', question: 'q1' },
+      { id: 'b', question: 'q2' },
+    ])
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+})
+
 test('RAGBench fixture mode loads rows and scores final answers', async () => {
   await withEnv({ RAGBENCH_FIXTURES: '1', RAGBENCH_DATA_FILE: undefined }, async () => {
     const adapter = createRagBenchAdapter()
diff --git a/bench/src/benchmarks/rag-shared.ts b/bench/src/benchmarks/rag-shared.ts
index 35f63ba9..882557a0 100644
--- a/bench/src/benchmarks/rag-shared.ts
+++ b/bench/src/benchmarks/rag-shared.ts
@@ -178,16 +178,25 @@ export function answerScoreToBenchScore(
 export async function readJsonRows(path: string): Promise<unknown[]> {
   const raw = (await readFile(path, 'utf8')).trim()
   if (raw.length === 0) return []
-  if (raw.startsWith('[') || raw.startsWith('{')) {
+  if (raw.startsWith('[')) {
     const parsed = JSON.parse(raw) as unknown
     if (Array.isArray(parsed)) return parsed
-    if (isObject(parsed)) {
-      for (const key of ['rows', 'data', 'examples', 'items']) {
-        const value = parsed[key]
-        if (Array.isArray(value)) return value
+    throw new Error(`${path} must contain a JSON array when it starts with [`)
+  }
+  if (raw.startsWith('{')) {
+    try {
+      const parsed = JSON.parse(raw) as unknown
+      if (isObject(parsed)) {
+        for (const key of ['rows', 'data', 'examples', 'items']) {
+          const value = parsed[key]
+          if (Array.isArray(value)) return value
+        }
+        return [parsed]
       }
+      throw new Error(`${path} must contain a JSON object, JSON array, JSONL rows, or an object with rows/data/examples/items`)
+    } catch (err) {
+      if (!raw.includes('\n')) throw err
     }
-    throw new Error(`${path} must contain a JSON array, JSONL rows, or an object with rows/data/examples/items`)
   }
   return raw
     .split(/\r?\n/)