From d40538a4229bc0cd39421492728704f3d927d3b6 Mon Sep 17 00:00:00 2001 From: Ankit Bari Date: Tue, 28 Jul 2026 17:27:31 +0000 Subject: [PATCH 1/3] feat(benchmark): Add InfiniteBench benchmark adapter --- src/benchmarks/infinitebench/index.ts | 204 ++++++++++++++++++++++++++ src/benchmarks/infinitebench/types.ts | 7 + 2 files changed, 211 insertions(+) create mode 100644 src/benchmarks/infinitebench/index.ts create mode 100644 src/benchmarks/infinitebench/types.ts diff --git a/src/benchmarks/infinitebench/index.ts b/src/benchmarks/infinitebench/index.ts new file mode 100644 index 0000000..8413333 --- /dev/null +++ b/src/benchmarks/infinitebench/index.ts @@ -0,0 +1,204 @@ +import { existsSync, readFileSync } from "fs" +import { join } from "path" + +import type { Benchmark, BenchmarkConfig, QuestionFilter } from "../../types/benchmark" +import type { + UnifiedMessage, + UnifiedQuestion, + UnifiedSession, + QuestionTypeRegistry, +} from "../../types/unified" + +import type { InfiniteBenchItem } from "./types" +import { logger } from "../../utils/logger" + +const DEFAULT_DATA_PATH = "./data/benchmarks/infinitebench" + +const TASKS = [ + "passkey", + "kv_retrieval", + "number_string", + "code_run", + "code_debug", + "math_find", + "math_calc", + "longdialogue_qa_eng", + "longbook_qa_eng", + "longbook_sum_eng", + "longbook_choice_eng", + "longbook_qa_chn", +] as const + +export const INFINITEBENCH_QUESTION_TYPES: QuestionTypeRegistry = { + passkey: { + id: "passkey", + alias: "passkey", + description: "Synthetic passkey retrieval", + }, + kv_retrieval: { + id: "kv_retrieval", + alias: "kv", + description: "Key-value retrieval", + }, + number_string: { + id: "number_string", + alias: "number-string", + description: "Number string retrieval", + }, + code_run: { + id: "code_run", + alias: "code-run", + description: "Code execution", + }, + code_debug: { + id: "code_debug", + alias: "code-debug", + description: "Code debugging", + }, + math_find: { + id: "math_find", + alias: "math-find", + description: "Mathematical search", + }, + math_calc: { + id: "math_calc", + alias: "math-calc", + description: "Mathematical calculation", + }, + longdialogue_qa_eng: { + id: "longdialogue_qa_eng", + alias: "dialogue", + description: "Long dialogue QA", + }, + longbook_qa_eng: { + id: "longbook_qa_eng", + alias: "book-qa", + description: "Long book question answering", + }, + longbook_sum_eng: { + id: "longbook_sum_eng", + alias: "book-summary", + description: "Long book summarization", + }, + longbook_choice_eng: { + id: "longbook_choice_eng", + alias: "book-choice", + description: "Long book multiple choice", + }, + longbook_qa_chn: { + id: "longbook_qa_chn", + alias: "book-qa-zh", + description: "Chinese long book QA", + }, +} + +export class InfiniteBenchBenchmark implements Benchmark { + name = "infinitebench" + + private questions: UnifiedQuestion[] = [] + private sessionsMap: Map = new Map() + private dataPath = "" + + async load(config?: BenchmarkConfig): Promise { + this.dataPath = config?.dataPath || DEFAULT_DATA_PATH + + const fullPath = join(process.cwd(), this.dataPath) + + if (!existsSync(fullPath)) { + throw new Error( + `InfiniteBench dataset not found at ${fullPath}. Download it from https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench and place the JSONL files under ${this.dataPath}.` + ) + } + + this.loadQuestions(fullPath) + } + + private loadQuestions(fullPath: string): void { + for (const task of TASKS) { + const filePath = join(fullPath, `${task}.jsonl`) + + if (!existsSync(filePath)) { + logger.warn(`Missing file for ${task}: ${filePath}`) + continue + } + + try { + const lines = readFileSync(filePath, "utf8").split("\n").filter(Boolean) + + for (const line of lines) { + const item: InfiniteBenchItem = JSON.parse(line) + this.processItem(item, task) + } + } catch (error) { + logger.error(`Failed to load ${task}: ${error}`) + } + } + + logger.info(`Loaded ${this.questions.length} questions from InfiniteBench`) + } + + private processItem(item: InfiniteBenchItem, task: string): void { + const questionId = `infinitebench-${task}-${item.id}` + + const session = this.createSession(item, questionId) + + this.questions.push({ + questionId, + question: item.input, + questionType: task, + groundTruth: String(item.answer[0]), + haystackSessionIds: [session.sessionId], + metadata: { + task, + options: item.options, + }, + }) + + this.sessionsMap.set(questionId, [session]) + } + + private createSession(item: InfiniteBenchItem, questionId: string): UnifiedSession { + const message: UnifiedMessage = { + role: "user", + content: item.context, + } + + return { + sessionId: `${questionId}-session-0`, + messages: [message], + } + } + + getQuestions(filter?: QuestionFilter): UnifiedQuestion[] { + let result = [...this.questions] + + if (filter?.questionTypes?.length) { + result = result.filter((q) => filter.questionTypes!.includes(q.questionType)) + } + + if (filter?.offset) { + result = result.slice(filter.offset) + } + + if (filter?.limit) { + result = result.slice(0, filter.limit) + } + + return result + } + + getHaystackSessions(questionId: string): UnifiedSession[] { + return this.sessionsMap.get(questionId) || [] + } + + getGroundTruth(questionId: string): string { + const question = this.questions.find((q) => q.questionId === questionId) + return question?.groundTruth ?? "" + } + + getQuestionTypes(): QuestionTypeRegistry { + return INFINITEBENCH_QUESTION_TYPES + } +} + +export default InfiniteBenchBenchmark \ No newline at end of file diff --git a/src/benchmarks/infinitebench/types.ts b/src/benchmarks/infinitebench/types.ts new file mode 100644 index 0000000..03fe12d --- /dev/null +++ b/src/benchmarks/infinitebench/types.ts @@ -0,0 +1,7 @@ +export interface InfiniteBenchItem { + id: number + context: string + input: string + answer: Array + options: string[] +} \ No newline at end of file From 3a5541688b69ff757281f543d4d8be06f093a5fc Mon Sep 17 00:00:00 2001 From: Ankit Bari Date: Tue, 28 Jul 2026 17:38:50 +0000 Subject: [PATCH 2/3] feat(benchmark): Add cli for InfiniteBench benchmark adapter --- src/benchmarks/README.md | 1 + src/benchmarks/index.ts | 4 +++- src/cli/index.ts | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/benchmarks/README.md b/src/benchmarks/README.md index 8d43cf9..a0b4ae4 100644 --- a/src/benchmarks/README.md +++ b/src/benchmarks/README.md @@ -36,6 +36,7 @@ interface Benchmark { | `locomo` | GitHub snap-research/locomo | Long context memory benchmark | | `longmemeval` | HuggingFace xiaowu0162/longmemeval-cleaned | Long-term memory evaluation | | `convomem` | HuggingFace Salesforce/ConvoMem | Conversational memory benchmark | +| `infinitebench` | HuggingFace xiaowu0162/infinitebench | Long-context understanding benchmark for testing retrieval, reasoning, and comprehension over extremely long inputs | ## Question Types diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index b790e87..f05e06c 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -2,11 +2,13 @@ import type { Benchmark, BenchmarkName } from "../types/benchmark" import { LoCoMoBenchmark } from "./locomo" import { LongMemEvalBenchmark } from "./longmemeval" import { ConvoMemBenchmark } from "./convomem" +import { InfiniteBenchBenchmark } from "./infinitebench" const benchmarks: Record Benchmark> = { locomo: LoCoMoBenchmark, longmemeval: LongMemEvalBenchmark, convomem: ConvoMemBenchmark, + infinitebench: InfiniteBenchBenchmark, } export function createBenchmark(name: BenchmarkName): Benchmark { @@ -21,4 +23,4 @@ export function getAvailableBenchmarks(): BenchmarkName[] { return Object.keys(benchmarks) as BenchmarkName[] } -export { LoCoMoBenchmark, LongMemEvalBenchmark, ConvoMemBenchmark } +export { LoCoMoBenchmark, LongMemEvalBenchmark, ConvoMemBenchmark, InfiniteBenchBenchmark } diff --git a/src/cli/index.ts b/src/cli/index.ts index b3c29d6..4fb20eb 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -150,11 +150,16 @@ Available benchmark datasets for evaluation: convomem ConvoMem - Conversational memory benchmark Tests: user facts, assistant facts, preferences, implicit connections Source: HuggingFace Salesforce/ConvoMem (downloaded on first use) + + infinitebench InfiniteBench - Ultra-long context understanding benchmark + Tests: document QA, long-context retrieval, code understanding, multi-hop reasoning + Source: GitHub OpenBMB/InfiniteBench (downloaded on first use) Usage: -b locomo Run LoCoMo benchmark -b longmemeval Run LongMemEval benchmark -b convomem Run ConvoMem benchmark + -b infinitebench Run InfiniteBench benchmark `) } From c0585bbc97b4ae3a0d7376f05e03bf34199e186a Mon Sep 17 00:00:00 2001 From: Ankit Bari Date: Tue, 28 Jul 2026 17:58:46 +0000 Subject: [PATCH 3/3] feat(benchmark): Use Map in benchmark adapter --- src/benchmarks/README.md | 2 +- src/benchmarks/infinitebench/index.ts | 150 ++++++++++---------------- src/benchmarks/infinitebench/types.ts | 2 +- src/types/benchmark.ts | 2 +- 4 files changed, 57 insertions(+), 99 deletions(-) diff --git a/src/benchmarks/README.md b/src/benchmarks/README.md index a0b4ae4..46d634e 100644 --- a/src/benchmarks/README.md +++ b/src/benchmarks/README.md @@ -36,7 +36,7 @@ interface Benchmark { | `locomo` | GitHub snap-research/locomo | Long context memory benchmark | | `longmemeval` | HuggingFace xiaowu0162/longmemeval-cleaned | Long-term memory evaluation | | `convomem` | HuggingFace Salesforce/ConvoMem | Conversational memory benchmark | -| `infinitebench` | HuggingFace xiaowu0162/infinitebench | Long-context understanding benchmark for testing retrieval, reasoning, and comprehension over extremely long inputs | +| `infinitebench` | HuggingFace xinrongzhang2022/InfiniteBench | Long-context understanding benchmark for testing retrieval, reasoning, and comprehension over extremely long inputs | ## Question Types diff --git a/src/benchmarks/infinitebench/index.ts b/src/benchmarks/infinitebench/index.ts index 8413333..56a5680 100644 --- a/src/benchmarks/infinitebench/index.ts +++ b/src/benchmarks/infinitebench/index.ts @@ -29,148 +29,107 @@ const TASKS = [ "longbook_qa_chn", ] as const -export const INFINITEBENCH_QUESTION_TYPES: QuestionTypeRegistry = { - passkey: { - id: "passkey", - alias: "passkey", - description: "Synthetic passkey retrieval", - }, - kv_retrieval: { - id: "kv_retrieval", - alias: "kv", - description: "Key-value retrieval", - }, - number_string: { - id: "number_string", - alias: "number-string", - description: "Number string retrieval", - }, - code_run: { - id: "code_run", - alias: "code-run", - description: "Code execution", - }, - code_debug: { - id: "code_debug", - alias: "code-debug", - description: "Code debugging", - }, - math_find: { - id: "math_find", - alias: "math-find", - description: "Mathematical search", - }, - math_calc: { - id: "math_calc", - alias: "math-calc", - description: "Mathematical calculation", - }, - longdialogue_qa_eng: { - id: "longdialogue_qa_eng", - alias: "dialogue", - description: "Long dialogue QA", - }, - longbook_qa_eng: { - id: "longbook_qa_eng", - alias: "book-qa", - description: "Long book question answering", - }, - longbook_sum_eng: { - id: "longbook_sum_eng", - alias: "book-summary", - description: "Long book summarization", - }, - longbook_choice_eng: { - id: "longbook_choice_eng", - alias: "book-choice", - description: "Long book multiple choice", - }, - longbook_qa_chn: { - id: "longbook_qa_chn", - alias: "book-qa-zh", - description: "Chinese long book QA", - }, -} +type InfiniteBenchTask = (typeof TASKS)[number] + +export const INFINITEBENCH_QUESTION_TYPES: QuestionTypeRegistry = Object.fromEntries( + TASKS.map((task) => [ + task, + { + id: task, + alias: task, + description: `InfiniteBench ${task} task`, + }, + ]) +) export class InfiniteBenchBenchmark implements Benchmark { name = "infinitebench" private questions: UnifiedQuestion[] = [] - private sessionsMap: Map = new Map() - private dataPath = "" + private sessions: Map = new Map() + + private dataPath = DEFAULT_DATA_PATH async load(config?: BenchmarkConfig): Promise { - this.dataPath = config?.dataPath || DEFAULT_DATA_PATH + this.dataPath = config?.dataPath ?? DEFAULT_DATA_PATH - const fullPath = join(process.cwd(), this.dataPath) + const datasetPath = join(process.cwd(), this.dataPath) - if (!existsSync(fullPath)) { - throw new Error( - `InfiniteBench dataset not found at ${fullPath}. Download it from https://huggingface.co/datasets/xinrongzhang2022/InfiniteBench and place the JSONL files under ${this.dataPath}.` - ) + if (!existsSync(datasetPath)) { + throw new Error(`InfiniteBench dataset missing: ${datasetPath}`) } - this.loadQuestions(fullPath) + this.loadDataset(datasetPath) } - private loadQuestions(fullPath: string): void { + private loadDataset(datasetPath: string): void { for (const task of TASKS) { - const filePath = join(fullPath, `${task}.jsonl`) + const file = join(datasetPath, `${task}.jsonl`) - if (!existsSync(filePath)) { - logger.warn(`Missing file for ${task}: ${filePath}`) + if (!existsSync(file)) { + logger.warn(`Skipping missing task file: ${file}`) continue } - try { - const lines = readFileSync(filePath, "utf8").split("\n").filter(Boolean) + const rows = readFileSync(file, "utf-8").split("\n").filter(Boolean) + + for (const row of rows) { + try { + const item: InfiniteBenchItem = JSON.parse(row) - for (const line of lines) { - const item: InfiniteBenchItem = JSON.parse(line) - this.processItem(item, task) + this.addQuestion(item, task) + } catch (error) { + logger.error(`Failed parsing ${file}: ${error}`) } - } catch (error) { - logger.error(`Failed to load ${task}: ${error}`) } } - logger.info(`Loaded ${this.questions.length} questions from InfiniteBench`) + logger.info(`Loaded ${this.questions.length} InfiniteBench questions`) } - private processItem(item: InfiniteBenchItem, task: string): void { + private addQuestion(item: InfiniteBenchItem, task: InfiniteBenchTask): void { const questionId = `infinitebench-${task}-${item.id}` - const session = this.createSession(item, questionId) + const session = this.createSession(item.context, questionId) - this.questions.push({ + const question: UnifiedQuestion = { questionId, + question: item.input, + questionType: task, - groundTruth: String(item.answer[0]), + + groundTruth: item.answer.at(0) ?? "", + haystackSessionIds: [session.sessionId], + metadata: { task, options: item.options, }, - }) + } - this.sessionsMap.set(questionId, [session]) + this.questions.push(question) + + this.sessions.set(questionId, [session]) } - private createSession(item: InfiniteBenchItem, questionId: string): UnifiedSession { + private createSession(context: string, questionId: string): UnifiedSession { const message: UnifiedMessage = { role: "user", - content: item.context, + + content: context, } return { - sessionId: `${questionId}-session-0`, + sessionId: `${questionId}-session`, + messages: [message], } } getQuestions(filter?: QuestionFilter): UnifiedQuestion[] { - let result = [...this.questions] + let result = this.questions if (filter?.questionTypes?.length) { result = result.filter((q) => filter.questionTypes!.includes(q.questionType)) @@ -188,12 +147,11 @@ export class InfiniteBenchBenchmark implements Benchmark { } getHaystackSessions(questionId: string): UnifiedSession[] { - return this.sessionsMap.get(questionId) || [] + return this.sessions.get(questionId) ?? [] } getGroundTruth(questionId: string): string { - const question = this.questions.find((q) => q.questionId === questionId) - return question?.groundTruth ?? "" + return this.questions.find((q) => q.questionId === questionId)?.groundTruth ?? "" } getQuestionTypes(): QuestionTypeRegistry { diff --git a/src/benchmarks/infinitebench/types.ts b/src/benchmarks/infinitebench/types.ts index 03fe12d..12139f8 100644 --- a/src/benchmarks/infinitebench/types.ts +++ b/src/benchmarks/infinitebench/types.ts @@ -2,6 +2,6 @@ export interface InfiniteBenchItem { id: number context: string input: string - answer: Array + answer: string[] options: string[] } \ No newline at end of file diff --git a/src/types/benchmark.ts b/src/types/benchmark.ts index 07961e4..c73b384 100644 --- a/src/types/benchmark.ts +++ b/src/types/benchmark.ts @@ -20,4 +20,4 @@ export interface Benchmark { getQuestionTypes(): QuestionTypeRegistry } -export type BenchmarkName = "locomo" | "longmemeval" | "convomem" +export type BenchmarkName = "locomo" | "longmemeval" | "convomem" | "infinitebench"