diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4b42eb4..4d57455b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,31 @@ jobs: - name: Run unit tests with coverage run: | - uv run pytest -m "not integration" --cov=openbench --cov-report=term-missing + uv run pytest -m "not integration and not docker" --cov=openbench --cov-report=term-missing + + docker-sandbox-test: + name: Docker Sandbox Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + version: "latest" + enable-cache: true + cache-dependency-glob: | + **/pyproject.toml + **/uv.lock + + - name: Set up Python + run: uv python install 3.12 + + - name: Install dependencies + run: uv sync --group dev + + - name: Build and exercise hardened sandboxes + run: uv run pytest -m docker -v integration-test: name: Integration Tests @@ -140,14 +164,15 @@ jobs: all-checks: name: All Checks Pass runs-on: ubuntu-latest - needs: [quality-checks, test, integration-test, security] + needs: [quality-checks, test, docker-sandbox-test, integration-test, security] if: always() steps: - name: Verify all checks passed run: | # Quality checks and test must always pass if [[ "${{ needs.quality-checks.result }}" != "success" || - "${{ needs.test.result }}" != "success" ]]; then + "${{ needs.test.result }}" != "success" || + "${{ needs.docker-sandbox-test.result }}" != "success" ]]; then echo "One or more required checks failed" exit 1 fi diff --git a/benchmark/bfcl-v4/README.md b/benchmark/bfcl-v4/README.md new file mode 100644 index 00000000..81de0bfc --- /dev/null +++ b/benchmark/bfcl-v4/README.md @@ -0,0 +1,61 @@ +# BFCL v4 integration contract + +OpenBench exposes three independently named BFCL v4 sections and one offline +aggregate: + +- `bfcl_v4_single_turn`: all 13 official single-turn categories; +- `bfcl_v4_multi_turn`: all four official stateful categories; +- `bfcl_v4_agentic_offline`: three memory backends and two frozen web-search + configurations; +- `bfcl_v4_offline`: all 5,106 samples with BFCL's 10/10/10/30/40 weights. + +## Provenance + +- Upstream: `ShishirPatil/gorilla` +- Revision: `6ea57973c7a6097fd7c5915698c54c17c5b1b6c8` +- License: Apache-2.0 +- Every downloaded question and answer file is protected by a pinned SHA-256. + +## Included categories + +- Non-live: Python, Java, JavaScript, multiple, parallel, + parallel-multiple, and irrelevance. +- Live: simple, multiple, parallel, parallel-multiple, irrelevance, and + relevance. + +The integration preserves BFCL's category aggregation: simple non-live is a +macro-average across languages, non-live is a macro-average across task shapes, +live is sample-weighted, and irrelevance is averaged across live/non-live. + +## Common function-calling layer + +The implementation supplies reusable primitives for: + +- JSON Schema to provider-safe Inspect tool definitions; +- reversible normalization of names such as `weather.get`; +- provider-native, JSON, and Python-style call parsing without `eval`; +- exact type/value validation, optional arguments, nested containers, and + one-to-one parallel matching; +- per-category and official-style aggregate metrics. + +## Explicit boundary + +The multi-turn task runs BFCL's pinned official state and response checker in a +network-disabled, read-only Docker sandbox. It covers base, missing-function, +missing-parameter, and long-context categories. + +The agentic task is intentionally an offline adaptation. Memory retrieval is +initialized from the public BFCL source record, while web search uses a frozen +corpus derived from BFCL's cited evidence. The no-snippet mode exposes URLs only +until the model calls `fetch_url_content`. This removes SERPAPI credentials and +web drift from CI, but it is not numerically interchangeable with the live +leaderboard environment. + +Consequently, `bfcl_v4_offline` applies the official section weights but does +not claim the official leaderboard score. The unqualified `bfcl_v4` alias stays +reserved until a versioned live-search snapshot and the exact official memory +prerequisite pipeline can run reproducibly. + +The older “Berkeley Function Calling” label in the Phi model card is not aliased +to this task: without a harness/version citation it may refer to BFCL v1 rather +than the v4 composition. diff --git a/benchmark/bfcl-v4/matrix.tsv b/benchmark/bfcl-v4/matrix.tsv new file mode 100644 index 00000000..a2cbacae --- /dev/null +++ b/benchmark/bfcl-v4/matrix.tsv @@ -0,0 +1,12 @@ +candidate case rep metric resource status notes +bfcl_v4_single_turn pinned_dataset 1 3641 13 done All official single-turn categories loaded at revision 6ea57973c7a6097fd7c5915698c54c17c5b1b6c8 with per-file SHA-256 verification +function_calling_core unit_tests 1 9 1.98 done Native JSON and Python parsing, schema conversion, optional/nested arguments, parallel matching, scorer, task, and registry covered +bfcl_v4_single_turn canonical_answer_sweep 1 2501 0 done All scored AST ground truths exercised; five inconsistent upstream records remain fail-closed under official-style validation +phase_0_1 full_unit_tests 1 441 50.40 done 441 passed, 2 environment-dependent skips, 11 integration/docker tests deselected +phase_0_1 docker_sandboxes 1 2 10.31 done LiveCodeBench and EvalPlus hardened images built and executed end to end +phase_0_1 pre_commit 1 5 0 done Ruff check, Ruff format, pyproject sync, mypy, and registry imports all passed +bfcl_v4_multi_turn pinned_dataset 1 800 4 done All official multi-turn categories loaded and checked by the pinned upstream state/response evaluator in Docker +bfcl_v4_agentic_offline pinned_dataset 1 665 5 done Three memory backends and two frozen web-search modes loaded without external credentials +bfcl_v4_offline weighted_aggregate 1 5106 5 done Official 10/10/10/30/40 section weights applied; explicitly not labeled as the live official score +bfcl_v4_multi_turn mock_transport 1 1 4.41 done End-to-end Inspect tool transport, Docker startup, solver, and scorer completed +bfcl_v4_agentic_offline mock_transport 1 1 0.02 done End-to-end frozen agentic solver and scorer completed diff --git a/benchmark/model-card-eval-audit/contract.md b/benchmark/model-card-eval-audit/contract.md new file mode 100644 index 00000000..da38b0be --- /dev/null +++ b/benchmark/model-card-eval-audit/contract.md @@ -0,0 +1,19 @@ +# Model-card eval coverage audit + +- Candidates: Qwen/Qwen3.6-27B, LiquidAI/LFM2.5-2.6B, + Qwen/Qwen3.5-4B, Nanbeige/Nanbeige4.2-3B, + microsoft/Phi-4-mini-instruct, google/gemma-4-E4B-it. +- Sources: the official Hugging Face model-card README for each candidate. +- Task matrix: every benchmark explicitly reported in a result table or + evaluation-results passage. Training-only dataset mentions are excluded. +- Primary classification: exact, alias-backed, partial/variant, or missing in + the current openbench2 registry and implementation. +- Preserved qualifiers: benchmark spelling, suite/subtask, metric, shot count, + prompting/reasoning mode, modality, language, and model-card context. +- Fairness budget: official card content only; no third-party leaderboard + substitutions. Every card receives the same extraction and classification + treatment. +- Stopping condition: every extracted model-card eval has a source and an + openbench2 support classification, with unresolved ambiguities called out. +- Baseline: clean `main` worktree at audit start; registry in + `src/openbench/config.py` and implementations under `src/openbench/evals/`. diff --git a/benchmark/model-card-eval-audit/coverage.md b/benchmark/model-card-eval-audit/coverage.md new file mode 100644 index 00000000..320fad87 --- /dev/null +++ b/benchmark/model-card-eval-audit/coverage.md @@ -0,0 +1,137 @@ +# openbench2 coverage of model-card evaluations + +## Implementation progress + +Wave 1 adds C-Eval, the complete BBH aggregate, OCRBench v1, AIME 2026, +HMMT November 2025, HMMT February 2026, GSM-Hard, HumanEval+, MBPP+, and +LiveCodeBench v6. Global PIQA generation is also available, but remains partial +for card-reproduction purposes because cloze `acc_bytes` is unsupported. + +Current exact scored-row coverage is **66/186 (35.5%)**, with 6 partial and 114 +missing rows. This is an increase of 16 exact rows over the 50/186 audit +baseline. LiveCodeBench accounts for five rows; C-Eval for two; current +competition math for six; and completing BBH plus adding OCRBench v1 for three. +The detailed per-card tables below are retained as the pre-implementation +baseline so the original gap analysis remains auditable. Current Wave 1 protocol +status is recorded in [`wave1_status.md`](wave1_status.md). + +BFCL v4 single-turn is now runnable across all 13 official single-turn +categories through provider-native tools or safely parsed prompted calls. It is +kept explicitly partial: the model-card `BFCLv4` overall score also weights +multi-turn and agentic web-search/memory sections, which are not represented by +the `bfcl_v4_single_turn` registry ID. + +The four official multi-turn categories are now available as +`bfcl_v4_multi_turn`, backed by the pinned upstream state checker in Docker. +Memory and web-search are available as `bfcl_v4_agentic_offline`, and the +10/10/10/30/40 weighted composition as `bfcl_v4_offline`. These remain partial +for historical-score reproduction because the agentic environment uses frozen +evidence rather than live SERPAPI results and precomputed memory prerequisites. + +## Classification rule + +- **Have**: the same public dataset/subset/version is runnable in openbench2, + possibly under a normalized registry ID. +- **Partial**: openbench2 has a related family, but the card's subset, version, + metric, tool condition, or aggregate cannot currently be reproduced. +- **Missing**: no matching implementation is registered. +- These labels measure task coverage, not score reproducibility. Most Qwen, + LiquidAI, Nanbeige, and Gemma rows omit at least one of metric, shots, prompt, + harness revision, or sampling details. + +The registry evidence is in [`src/openbench/config.py`](../../src/openbench/config.py), +with implementations under [`src/openbench/evals`](../../src/openbench/evals). + +At the pre-implementation audit baseline, across 186 scored-row occurrences, +50 map to the same benchmark +dataset/variant, 9 map only to a related or incompatible variant, and 127 are +missing. These counts retain duplicate rows and distinct configurations such as +VideoMME with versus without subtitles. + +## Coverage by card + +| Card | Have | Partial | Missing | +|---|---|---|---| +| Qwen3.6-27B | MMLU-Pro; MMLU-Redux; SuperGPQA; GPQA Diamond; HLE (no tools); HMMT Feb 25; MMMU; MMMU-Pro; MathVista mini; MMStar | OCRBench (only OCRBenchV2 exists) | SWE-bench Verified/Pro/Multilingual; Terminal-Bench 2.0; SkillsBench; QwenWebBench; NL2Repo; both Claw-Eval aggregates; QwenClawBench; C-Eval; LiveCodeBench v6; HMMT Nov 25/Feb 26; IMOAnswerBench; AIME26; DynaMath; VlmsAreBlind; RealWorldQA; MMBench; SimpleVQA; CharXiv; CC-OCR; ERQA; CountBench; RefCOCO; EmbSpatialBench; RefSpatialBench; VideoMME; VideoMMMU; MLVU; MVBench; V*; AndroidWorld | +| LFM2.5-2.6B | AIME25; IFBench | BrowseComp+ (base BrowseComp only) | AA Omniscience; LiveCodeBench v6; Multi-IF; IFStruct; BFCLv4; ToolSandbox; τ³-Bench Banking; Claw-Eval average (EN); PinchBench | +| Qwen3.5-4B | MMLU-Pro; MMLU-Redux; SuperGPQA; GPQA Diamond; IFEval; IFBench; MultiChallenge; HMMT Feb 25; TAU2-Bench; MMMLU; MMMU; MMMU-Pro; MathVista mini; MMStar | Global PIQA (English PIQA only); OCRBench (V2 only) | C-Eval; AA-LCR; LongBench v2; HMMT Nov 25; LiveCodeBench v6; OJBench; BFCL-V4; VITA-Bench; DeepPlanning; MMLU-ProX; NOVA-63; INCLUDE; PolyMATH; WMT24++; MAXIFE; MathVision; We-Math; DynaMath; both ZEROBench rows; VlmsAreBlind; BabyVision; RealWorldQA; MMBench; SimpleVQA; HallusionBench; OmniDocBench1.5; CharXiv; MMLongBench-Doc; CC-OCR; AI2D_TEST; ERQA; CountBench; RefCOCO; EmbSpatialBench; RefSpatialBench; LingoQA; Hypersim; Nuscene; both VideoMME modes; VideoMMMU; MLVU; MVBench; LVBench; MMVU; ScreenSpot Pro; OSWorld-Verified; AndroidWorld; TIR-Bench; V*; SLAKE; PMC-VQA; MedXpertQA-MM | +| Nanbeige4.2-3B | HLE without search; SciCode; GPQA-Diamond; IF-Bench | DeepResearch Bench II (openbench has the original DeepResearch Bench, not Bench II) | GDPval/rubrics; Agent-IF-Oneday; Office-QA-Pro; Pinch-Bench-V2; Claw-Gym; Claw-Eval pass^3; MCP-Atlas; SWE-Bench Verified/Pro; Terminal-Bench 2.0; HMMT-Feb-2026; IMO-Answer-Bench; LiveCodeBench-V6; AA-LCR; Recruit-Bench; ResearchRubrics | +| Phi-4-mini-instruct scored table | MMLU; MMLU-Pro; ARC Challenge; BoolQ; GPQA; HellaSwag; OpenBookQA; PIQA; Social IQA; Winogrande; Multilingual MMLU/MMMLU; MGSM; GSM8K; MATH | BigBench Hard (openbench has 18 core tasks, not the complete 23-task aggregate); TruthfulQA MC2 (openbench currently implements MC1) | Arena Hard | +| Phi appendix additions | MedQA; ANLI; TriviaQA; HumanEval; MBPP; IFEval; AGI Eval; Toxigen | none | Berkeley Function Calling; GSM8K Hard; HumanEval+; MBPP+; LiveCodeBench (`LiveCodeBenh` typo in source); LiveBench; BigCode Bench; Spider; MEGA; DecodingTrust; XSTest; unnamed/internal evals | +| Gemma-4-E4B-it | MMLU Pro; GPQA Diamond; Tau2 average over retail/airline/telecom; HLE no tools; MMMLU; MMMU Pro | HLE with search (dataset exists, search-enabled harness does not); MRCR v2 8-needle 128k (8-needle/context controls exist, but no explicit v2 identity) | AIME 2026; LiveCodeBench v6; Codeforces ELO; BigBench Extra Hard; OmniDocBench 1.5; MATH-Vision; MedXPertQA MM; CoVoST; FLEURS | + +## Existing implementations that cover the cards + +| Card name/family | openbench2 registry ID(s) | Important caveat | +|---|---|---| +| MMLU / MMLU-Pro / MMLU-Redux | `mmlu`, `mmlu-pro`, `mmlu-redux` | Prompt/shot settings must be matched per card. | +| Multilingual MMLU / MMMLU | `mmmlu` and language subtasks | Alias-backed. | +| GPQA / GPQA Diamond | `gpqa`, `gpqa_diamond` | Diamond is separate and available. | +| SuperGPQA | `supergpqa` | Available. | +| IFEval / IFBench / MultiChallenge | `ifeval`, `ifbench`, `multichallenge` | Available. | +| HLE without tools | `hle`, `hle_text` | No search-enabled solver matching Gemma's HLE-with-search row. | +| AIME25 / HMMT Feb 25 | `aime_2025` or `gpt_oss_aime25`; `hmmt_feb_2025` | Newer 2026/November variants are absent. | +| Tau2 | `tau_bench_retail`, `tau_bench_airline`, `tau_bench_telecom` | Implementation downloads the official `sierra-research/tau2-bench`; reproducing Qwen3.5's score still requires confirming its stated airline fix. | +| MMMU / MMMU-Pro | `mmmu`, `mmmu_pro` | Available. | +| MathVista mini | `mathvista` | The implementation supports the testmini split. | +| MMStar | `mmstar` | Available. | +| MRCR 8 needle | `openai_mrcr_8n` | Related support only: `max_context_size` can constrain to 128k, but the task does not explicitly identify or pin Gemma's reported v2. | +| SciCode | `scicode` | Optional dependency group may be required. | +| Classic Phi tasks | `arc_challenge`, `boolq`, `hellaswag`, `openbookqa`, `piqa`, `social_iqa`, `winogrande`, `mgsm`, `gsm8k`, `math` | Dataset coverage exists; reproduce card shots/CoT separately. | +| Phi appendix tasks | `medqa`, `anli`, `triviaqa`, `humaneval`, `mbpp`, `ifeval`, `agieval`, `toxigen` | `HumanEval+` and `MBPP+` are distinct and absent. | + +## What to implement, in order + +### P0 — highest reuse across these cards + +1. ~~**LiveCodeBench v6**~~ — implemented as `livecodebench_v6` with the + official cumulative release composition and scoring protocol. +2. **SWE-bench + Terminal-Bench 2.0** — add Verified, Pro, Multilingual, and + Terminal-Bench as versioned agent tasks with explicit scaffold selection. + Do not bake Qwen's corrected SWE-bench Pro set into the public canonical ID. +3. **Current competition math** — AIME 2026, HMMT Nov 2025, HMMT Feb 2026, + and IMOAnswerBench. Keep year/month in IDs and dataset metadata. +4. **BFCL v4** — shared by LFM and Qwen3.5 and strategically useful for tool + calling. Preserve AST/function-call scoring and category aggregates. +5. **C-Eval** — a straightforward academic coverage gap appearing on both + Qwen cards. +6. **TruthfulQA MC2** — extend the existing task rather than creating an + unrelated implementation; current code explicitly scores MC1. +7. **Full BBH aggregate** — add the five missing tasks and a canonical + 23-task aggregate before claiming Phi's BigBench Hard row as reproducible. + +### P1 — shared multimodal and agent coverage + +1. Build shared multimodal primitives, then add the evals repeated across both + Qwen cards: DynaMath, VlmsAreBlind, RealWorldQA, MMBench, SimpleVQA, + CharXiv, OCRBench v1, ERQA, CountBench, RefCOCO, EmbSpatialBench, + RefSpatialBench, VideoMME, VideoMMMU, MLVU, MVBench, V*, and AndroidWorld. +2. Add agent/tool suites as separately versioned packages: Claw-Eval, + PinchBench/Pinch-Bench-V2, ToolSandbox, SkillsBench, and τ³-Bench. Do not + alias τ³-Bench to the existing Tau2 implementation. +3. Add HLE-with-search as a solver/configuration over the existing HLE dataset, + with search provider and cost captured in run metadata. +4. Evaluate whether DeepResearch Bench II can share the existing original + DeepResearch Bench scorer; it must remain a separate registry ID unless the + datasets and metric definitions are proven identical. + +### P2 — breadth and card-specific tails + +- Gemma: BigBench Extra Hard, OmniDocBench 1.5, MATH-Vision, MedXPertQA-MM, + CoVoST, and FLEURS. +- Phi: Berkeley Function Calling, GSM8K Hard, HumanEval+, MBPP+, LiveBench, + BigCode Bench, Spider, MEGA, DecodingTrust, and XSTest. +- Qwen-specific long tail: LongBench v2, OJBench, WMT24++, MAXIFE, + MMLU-ProX, MathVision/We-Math/PolyMATH, document/UI/navigation, medical VQA, + and the remaining proprietary or newly introduced suites. + +## Findings that affect implementation design + +- **Versioned IDs are mandatory.** The cards mix AIME25/AIME26, multiple HMMT + dates, OCRBench/OCRBenchV2, and DeepResearch Bench/Bench II. +- **Harness is part of the eval.** SWE-bench, Terminal-Bench, Tau2, Claw, and + OpenClaw scores depend on agent scaffold and tool environment. +- **Metric/configuration is part of identity.** TruthfulQA MC1 is not MC2; + HLE no-tools is not HLE-with-search; VideoMME with subtitles is not without. +- **Store provenance.** Nanbeige's HMMT-Feb-2026 README and generated YAML + disagree (82.8 vs 82.1), and several cards omit metrics entirely. diff --git a/benchmark/model-card-eval-audit/matrix.tsv b/benchmark/model-card-eval-audit/matrix.tsv new file mode 100644 index 00000000..6e42d1e6 --- /dev/null +++ b/benchmark/model-card-eval-audit/matrix.tsv @@ -0,0 +1,7 @@ +candidate case rep metric resource status notes +Qwen/Qwen3.6-27B scored_eval_exact_coverage 1 0.2222 45 done 10 exact; 1 partial; 34 missing +LiquidAI/LFM2.5-2.6B scored_eval_exact_coverage 1 0.1667 12 done 2 exact; 1 partial; 9 missing +Qwen/Qwen3.5-4B scored_eval_exact_coverage 1 0.2000 70 done 14 exact; 2 partial; 54 missing +Nanbeige/Nanbeige4.2-3B scored_eval_exact_coverage 1 0.1600 25 done 4 exact; 1 partial; 20 missing +microsoft/Phi-4-mini-instruct scored_eval_exact_coverage 1 0.8235 17 done 14 exact; 2 partial; 1 missing +google/gemma-4-E4B-it scored_eval_exact_coverage 1 0.3529 17 done 6 exact; 2 partial; 9 missing diff --git a/benchmark/model-card-eval-audit/model_card_inventory.md b/benchmark/model-card-eval-audit/model_card_inventory.md new file mode 100644 index 00000000..5eed8668 --- /dev/null +++ b/benchmark/model-card-eval-audit/model_card_inventory.md @@ -0,0 +1,268 @@ +# Model-card evaluation inventory + +This inventory preserves the model-card spellings. Parenthetical/subscript +qualifiers are part of the reported evaluation configuration, not silently +collapsed aliases. A name in the Phi appendix means the card says the model was +evaluated on it, even where no score is published. + +## Qwen/Qwen3.6-27B + +Source: [revision 6a9e13b](https://huggingface.co/Qwen/Qwen3.6-27B/blob/6a9e13bd6fc8f0983b9b99948120bc37f49c13e9/README.md#L56) + +45 scored rows: + +1. SWE-bench Verified +2. SWE-bench Pro +3. SWE-bench Multilingual +4. Terminal-Bench 2.0 +5. SkillsBench (Avg5) +6. QwenWebBench +7. NL2Repo +8. Claw-Eval (Avg) +9. Claw-Eval (Pass^3) +10. QwenClawBench +11. MMLU-Pro +12. MMLU-Redux +13. SuperGPQA +14. C-Eval +15. GPQA Diamond +16. HLE +17. LiveCodeBench v6 +18. HMMT Feb 25 +19. HMMT Nov 25 +20. HMMT Feb 26 +21. IMOAnswerBench +22. AIME26 +23. MMMU +24. MMMU-Pro +25. MathVista (mini) +26. DynaMath +27. VlmsAreBlind +28. RealWorldQA +29. MMStar +30. MMBench (EN-DEV-v1.1) +31. SimpleVQA +32. CharXiv (RQ) +33. CC-OCR +34. OCRBench +35. ERQA +36. CountBench +37. RefCOCO (avg) +38. EmbSpatialBench +39. RefSpatialBench +40. VideoMME (w sub.) +41. VideoMMMU +42. MLVU +43. MVBench +44. V* +45. AndroidWorld + +Protocol notes: SWE-bench uses Qwen's internal agent scaffold; the card says +its SWE-bench Pro set contains corrections to problematic public tasks. +SkillsBench is a 78-task self-contained subset averaged across five runs. +NL2Repo uses Claude Code for comparator models. No uniform shot count or metric +name is supplied for the complete table. + +## LiquidAI/LFM2.5-2.6B + +Source: [revision a4e00e8](https://huggingface.co/LiquidAI/LFM2.5-2.6B/blob/a4e00e83c0979ee9deb88d04b6360599fa956656/README.md#L203-L222) + +12 scored rows: + +1. AA Omniscience +2. AIME25 +3. LiveCodeBenchv6 +4. IFBench +5. Multi-IF +6. IFStruct +7. BFCLv4 +8. ToolSandbox +9. τ³-Bench Banking +10. Claw-Eval average (EN) +11. PinchBench +12. BrowseComp+ (OpenClaw) + +The card supplies scores but no metric names, shot counts, prompting protocol, +dataset revisions, or confidence intervals. + +## Qwen/Qwen3.5-4B + +Source: [revision 851bf6e](https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md#L66) + +70 scored rows (69 benchmark names; VideoMME has two configurations): + +1. MMLU-Pro +2. MMLU-Redux +3. C-Eval +4. SuperGPQA +5. GPQA Diamond +6. IFEval +7. IFBench +8. MultiChallenge +9. AA-LCR +10. LongBench v2 +11. HMMT Feb 25 +12. HMMT Nov 25 +13. LiveCodeBench v6 +14. OJBench +15. BFCL-V4 +16. TAU2-Bench +17. VITA-Bench +18. DeepPlanning +19. MMMLU +20. MMLU-ProX +21. NOVA-63 +22. INCLUDE +23. Global PIQA +24. PolyMATH +25. WMT24++ +26. MAXIFE +27. MMMU +28. MMMU-Pro +29. MathVision +30. Mathvista(mini) +31. We-Math +32. DynaMath +33. ZEROBench +34. ZEROBench_sub +35. VlmsAreBlind +36. BabyVision +37. RealWorldQA +38. MMStar +39. MMBench (EN-DEV-v1.1) +40. SimpleVQA +41. HallusionBench +42. OmniDocBench1.5 +43. CharXiv(RQ) +44. MMLongBench-Doc +45. CC-OCR +46. AI2D_TEST +47. OCRBench +48. ERQA +49. CountBench +50. RefCOCO(avg) +51. EmbSpatialBench +52. RefSpatialBench +53. LingoQA +54. Hypersim +55. Nuscene +56. VideoMME (w sub.) +57. VideoMME (w/o sub.) +58. VideoMMMU +59. MLVU +60. MVBench +61. LVBench +62. MMVU +63. ScreenSpot Pro +64. OSWorld-Verified +65. AndroidWorld +66. TIR-Bench +67. V* +68. SLAKE +69. PMC-VQA +70. MedXpertQA-MM + +The TAU2 result follows the official setup except for an airline-domain fix. +MathVision uses a fixed boxed-answer prompt for Qwen; comparator scores select +the better result with or without boxed formatting. No uniform shot-count or +metric specification is given for the full table. + +## Nanbeige/Nanbeige4.2-3B + +Sources: [general/agentic table](https://huggingface.co/Nanbeige/Nanbeige4.2-3B/blob/5d54321e9e01e0d026f8e371046678fc384dca39/README.md#L45-L109), [local-assistant table](https://huggingface.co/Nanbeige/Nanbeige4.2-3B/blob/5d54321e9e01e0d026f8e371046678fc384dca39/README.md#L114-L147) + +25 scored row occurrences, 23 distinct labels/configurations: + +1. GDPval rubrics +2. Agent-IF-Oneday (in-house scaffold) +3. Office-QA-Pro +4. Pinch-Bench-V2 +5. Claw-Gym +6. Claw-Eval (pass^3) +7. MCP-Atlas +8. SWE-Bench Verified +9. SWE-Bench Pro +10. Terminal-Bench 2.0 +11. HLE w/o Search +12. SciCode +13. GPQA-Diamond +14. HMMT-Feb-2026 +15. IMO-Answer-Bench +16. LiveCodeBench-V6 +17. AA-LCR +18. IF-Bench +19. Recruit-Bench +20. GDPval (OpenClaw) +21. Agent-IF-Oneday (OpenClaw) +22. DeepResearch Bench II +23. ResearchRubrics + +Pinch-Bench-V2 and Claw-Gym are repeated unchanged in the second table; +Agent-IF-Oneday is repeated under a different scaffold and score. All README +evaluations are stated to use thinking mode with `preserve_thinking=true`. +Hugging Face's generated metadata duplicates seven rows and conflicts with the +README on HMMT-Feb-2026: 82.1 in YAML versus 82.8 in the README. + +## microsoft/Phi-4-mini-instruct + +Sources: [scored table](https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md#L83), [evaluated-datasets appendix](https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/cfbefacb99257ffa30c83adab238a50856ac3083/README.md#L328-L374) + +17 scored rows: + +1. Arena Hard +2. BigBench Hard (0-shot, CoT) +3. MMLU (5-shot) +4. MMLU-Pro (0-shot, CoT) +5. ARC Challenge (10-shot) +6. BoolQ (2-shot) +7. GPQA (0-shot, CoT) +8. HellaSwag (5-shot) +9. OpenBookQA (10-shot) +10. PIQA (5-shot) +11. Social IQA (5-shot) +12. TruthfulQA (MC2) (10-shot) +13. Winogrande (5-shot) +14. Multilingual MMLU (5-shot) +15. MGSM (0-shot, CoT) +16. GSM8K (8-shot, CoT) +17. MATH (0-shot, CoT) + +Additional public named evals in the appendix (no result published there): +MedQA, ANLI, Berkeley function calling, TriviaQA, GSM8K Hard, HumanEval, +HumanEval+, MBPP, MBPP+, LiveCodeBenh (source typo), LiveBench, BigCode Bench, +Spider, IFEval, MEGA, AGI Eval, DecodingTrust, XSTest, and Toxigen. The appendix +also repeats several scored-table evals and mentions unnamed internal function +calling, coding, instruction-following, multilingual-safety, multi-turn, and red +team evaluations; those unnamed/internal sources are not implementable from the +card alone. + +Important mismatch: the card reports TruthfulQA **MC2**, while openbench2's +current task is explicitly **MC1**. + +## google/gemma-4-E4B-it + +Source: [revision ee0ef60](https://huggingface.co/google/gemma-4-E4B-it/blob/ee0ef6023621cff504d758262d4e04895a5af4a2/README.md#L84-L113) + +17 scored rows: + +1. MMLU Pro +2. AIME 2026 no tools +3. LiveCodeBench v6 +4. Codeforces ELO +5. GPQA Diamond +6. Tau2 (average over 3) +7. HLE no tools +8. HLE with search +9. BigBench Extra Hard +10. MMMLU +11. MMMU Pro +12. OmniDocBench 1.5 (average edit distance, lower is better) +13. MATH-Vision +14. MedXPertQA MM +15. CoVoST +16. FLEURS (lower is better) +17. MRCR v2 8 needle 128k (average) + +The card names no shots. It explicitly gives ELO for Codeforces and average edit +distance for OmniDocBench; other underlying metric names should not be inferred. +The later safety section names content categories but no safety benchmark. diff --git a/benchmark/model-card-eval-audit/research_log.tsv b/benchmark/model-card-eval-audit/research_log.tsv new file mode 100644 index 00000000..2dc19c8b --- /dev/null +++ b/benchmark/model-card-eval-audit/research_log.tsv @@ -0,0 +1,14 @@ +timestamp candidate_or_scope action status notes +2026-08-06T00:00:00Z workspace baseline completed Clean main worktree; inspected config registry and eval/dataset/scorer file inventory +2026-08-06T00:00:01Z all six cards raw README access smoke test completed All six official raw README URLs returned successfully +2026-08-06T00:00:02Z workspace tooling check inconclusive rg is unavailable; continued with find and grep +2026-08-06T00:00:03Z Qwen/Qwen3.6-27B extract 45 scored rows completed Revision 6a9e13bd6fc8f0983b9b99948120bc37f49c13e9 +2026-08-06T00:00:04Z LiquidAI/LFM2.5-2.6B extract 12 scored rows completed All rows omit metric, shots, and benchmark protocol +2026-08-06T00:00:05Z Qwen/Qwen3.5-4B extract 70 scored rows completed 69 names; VideoMME has with/without subtitle configurations +2026-08-06T00:00:06Z Nanbeige/Nanbeige4.2-3B extract 25 scored occurrences completed README/YAML HMMT-Feb-2026 conflict retained +2026-08-06T00:00:07Z microsoft/Phi-4-mini-instruct extract scored table and appendix completed 17 scored rows plus 19 public named appendix additions +2026-08-06T00:00:08Z google/gemma-4-E4B-it extract 17 scored rows completed URL available and ungated +2026-08-06T00:00:09Z workspace map registry and implementation coverage completed Exact, partial/version mismatch, missing, and internal buckets recorded in coverage.md +2026-08-06T00:00:10Z workspace broad repository actor scan dead-end Actor stalled; replaced with targeted config/eval inspection +2026-08-06T00:00:11Z workspace delayed strict registry cross-check completed 594 built-in IDs confirmed; MRCR v2 reclassified exact-to-partial because v2 identity is not explicit +2026-08-06T00:00:12Z LiveCodeBench v6 implement highest-priority gap completed Pinned cumulative release_v6; official prompt and pass@k protocol; network-disabled Docker execution diff --git a/benchmark/model-card-eval-audit/wave1_status.md b/benchmark/model-card-eval-audit/wave1_status.md new file mode 100644 index 00000000..e11e8f62 --- /dev/null +++ b/benchmark/model-card-eval-audit/wave1_status.md @@ -0,0 +1,31 @@ +# Wave 1 implementation status + +Status after the source/protocol audit and the current OpenBench integration pass. +"Implemented" means the public artifact is pinned and the local protocol is tested; +"adapted" means the local score is useful but is not directly comparable with the +historical leaderboard protocol. + +| Evaluation | Status | Reproducibility note | +|---|---|---| +| TruthfulQA MC2 | Blocked | MC2 requires probability mass over every true/false continuation. Inspect's provider-neutral generation interface does not expose portable forced-continuation log-likelihoods; generated-choice accuracy would be a different metric. | +| C-Eval / C-Eval Hard | Implemented | Pinned `ceval/ceval-exam`; all 52 subjects, official five-shot Chinese prompt, eight-subject hard subset, grouped subject/category metrics. | +| BIG-Bench Hard | Implemented | Registry now covers all 23 conceptual tasks / 27 physical configurations. The nine formerly absent configurations use full free-response targets instead of letter-only truncation. | +| OCRBench v1 | Implemented | Pinned 1,000-example `echo840/OCRBench`; multimodal transport and historical substring scorer, kept separate from OCRBenchV2. | +| Global PIQA v1 | Implemented (generation) | Both pinned parallel/non-parallel corpora, official 2-choice/4-choice prompts and sampling, and hierarchical macro aggregation. Cloze `acc_bytes` remains unsupported because it needs continuation log-likelihoods. | +| BrowseComp-Plus | Blocked | Requires the fixed 100,195-document corpus, a versioned retriever/search tool, multi-step agent traces, evidence recall, and a Qwen3-32B judge. It is not a BrowseComp dataset alias. LiquidAI's OpenClaw scaffold/configuration is unpublished. | +| AIME 2026 | Implemented | Pinned 30-problem MathArena artifact, four runs, boxed final answers. This is a third-party republication; original MAA redistribution rights were not independently established. | +| HMMT November 2025 | Implemented subset | Pinned 30-problem MathArena subset, not the complete 66-problem human contest. | +| HMMT February 2026 | Implemented subset | Pinned 33-problem MathArena subset, not the complete 76-problem contest. Fraction/power answers now use a non-AIME scorer. | +| IMO-AnswerBench | Blocked for faithful integration | The maintained v2 CSV changes 29 rows and still has a malformed row; candidate-generation settings and the Gemini 2.5 Pro judge snapshot are missing. HF mirrors contain deprecated v1. | +| GSM8K Hard / PAL GSM-Hard | Implemented (adapted) | Pinned canonical 1,319-row artifact and strict `<1e-3` numeric scoring. The local task is direct-answer generation; canonical PAL instead generates and securely executes Python with an eight-shot prompt. | +| HumanEval+ | Implemented | Pinned EvalPlus HumanEval+ v0.1.10; base and augmented differential tests run in a fail-closed, network-disabled Docker sandbox. Hidden tests stay out of Inspect logs and are unlinked before candidate execution. | +| MBPP+ | Implemented | Pinned EvalPlus MBPP+ v0.2.0 with the same isolated base/plus execution, adaptive limits, special oracles, and hidden-test protections. | + +## Remaining architecture work + +1. Add a provider capability for forced-continuation log-likelihoods; then implement + TruthfulQA MC2 and Global PIQA cloze/`acc_bytes` without metric substitution. +2. Add a versioned corpus/retriever/tool-trace layer; then implement BrowseComp-Plus. +3. Revisit IMO-AnswerBench only after upstream repairs the v2 row and a reproducible + judge snapshot/protocol is available, or expose an explicitly named OpenBench + adaptation rather than claiming historical-score equivalence. diff --git a/benchmark/wave1-integration/contract.md b/benchmark/wave1-integration/contract.md new file mode 100644 index 00000000..56c78607 --- /dev/null +++ b/benchmark/wave1-integration/contract.md @@ -0,0 +1,56 @@ +# Wave 1 benchmark integration contract + +## Goal + +Integrate the 13 Wave 1 evaluations requested on 2026-08-06, in the supplied +order, while preserving official dataset, prompt, sampling, and scoring +semantics. + +## Frozen candidates + +1. TruthfulQA MC2 +2. C-Eval +3. BigBench Hard (all 23 tasks and aggregate) +4. OCRBench v1 +5. Global PIQA +6. BrowseComp+ +7. AIME 2026 +8. HMMT November 2025 +9. HMMT February 2026 +10. IMOAnswerBench +11. GSM8K Hard +12. HumanEval+ +13. MBPP+ + +## Acceptance matrix + +Every candidate is checked against the same requirements: authoritative public +source, redistributable or remotely loadable data, immutable revision when the +host supports it, faithful prompt and scoring, distinct registry identity, +offline unit tests, import smoke test, and documented limitations. + +The primary metric is the number of candidates that satisfy every applicable +requirement and pass their focused tests. Secondary metrics are unsupported or +blocked candidates, protocol deviations, network-bound smoke tests, and added +dependencies. + +## Fairness and evidence rules + +- No benchmark is represented by a similarly named substitute. +- A missing or non-public 2025/2026 dataset is recorded as blocked rather than + reconstructed from memory or unofficial questions. +- Existing implementations are reused only when semantics remain identical. +- Generated code is never executed directly on the host as part of scoring. +- Research failures and implementation failures remain visible in `matrix.tsv`. + +## Environment + +- Repository: `openbench2` +- Date frozen: 2026-08-06 +- Python commands run after `source .venv/bin/activate` +- Existing uncommitted LiveCodeBench work is out of scope and must be preserved. + +## Stopping condition + +Stop after all 13 candidates are either implemented and verified or have a +source-backed blocking rationale. Do not begin Wave 2. diff --git a/benchmark/wave1-integration/matrix.tsv b/benchmark/wave1-integration/matrix.tsv new file mode 100644 index 00000000..6bba339a --- /dev/null +++ b/benchmark/wave1-integration/matrix.tsv @@ -0,0 +1,21 @@ +candidate case rep metric resource status notes +truthfulqa_mc2 baseline_registry 1 0 0 unsupported Only the existing TruthfulQA task was found before Wave 1 work +ceval baseline_registry 1 0 0 unsupported No existing integration found before Wave 1 work +bigbench_hard_23 baseline_registry 1 18 18 unsupported Existing implementation exposes 18 of 23 tasks and no 23-task aggregate +ocrbench_v1 baseline_registry 1 0 0 unsupported Only OCRBench v2 exists; it must remain distinct +global_piqa baseline_registry 1 0 0 unsupported Only PIQA exists +browsecomp_plus baseline_registry 1 0 0 unsupported Only BrowseComp exists +aime_2026 focused_tests 1 1 0 done MathArena/aime_2026 pinned at d2de22f3c656b4f56cf8981212186377d1e23bc3; registry and group tested +hmmt_nov_2025 focused_tests 1 1 0 done MathArena/hmmt_nov_2025 pinned at 118dbfb45c4c9467c672268ed55166642897aa46; third-party CC-BY-NC-SA-4.0 transcription avoids redistributing copyrighted PDFs +hmmt_feb_2026 focused_tests 1 1 0 done MathArena/hmmt_feb_2026 pinned at 02fba4f74d8e68e73e66a02d540fd979c05c274c; registry and group tested +imoanswerbench baseline_registry 1 0 0 unsupported No existing integration found before Wave 1 work +gsm8k_hard baseline_registry 1 0 0 unsupported Only GSM8K and GSM-Plus were found +humaneval_plus baseline_registry 1 0 0 unsupported Only HumanEval exists +mbpp_plus baseline_registry 1 0 0 unsupported Only MBPP exists +wave1 baseline_unit_tests 1 399 15.52 done 399 passed; 3 unrelated pre-existing failures; 2 skipped; 9 integration tests deselected +wave1 baseline_export_cli 1 0 0 error Two pre-existing export command assertions fail +wave1 baseline_dependency_resolution 1 0 0 error inspect-ai==0.3.141 is absent from the current package index +matharena_new focused_tests 1 18 5.74 done 18 passed for new factories, immutable revisions, registry, and aggregate group +wave1 docker_sandbox_smoke 1 2 3.71 done LiveCodeBench and EvalPlus images built; network/read-only/capability/PID controls and payload unlinking verified end to end +wave1 mypy_full 1 453 0 done No type errors after adding Python-3.10-compatible pandas stubs +wave1 full_unit_tests 1 432 49.70 done 432 passed and 11 environment-dependent tests skipped before dedicated Docker execution diff --git a/docs/snippets/benchmarks.data.mdx b/docs/snippets/benchmarks.data.mdx index d734fc77..ccc2c222 100644 --- a/docs/snippets/benchmarks.data.mdx +++ b/docs/snippets/benchmarks.data.mdx @@ -342,6 +342,20 @@ export const benchmarksData = [ "function_name": "aime_2025_II", "is_alpha": false }, + { + "name": "AIME 2026", + "description": "Combined American Invitational Mathematics Examination 2026", + "category": "math", + "tags": [ + "math", + "competition", + "aime", + "2026", + "combined" + ], + "function_name": "aime_2026", + "is_alpha": false + }, { "name": "ANLI (All Rounds)", "description": "Adversarial Natural Language Inference - challenging NLI benchmark", @@ -1021,6 +1035,18 @@ export const benchmarksData = [ "function_name": "arabic_exams_social_science_primary_school", "is_alpha": false }, + { + "name": "BBH: Boolean Expressions", + "description": "BIG-Bench Hard boolean expression evaluation", + "category": "core", + "tags": [ + "reasoning", + "bigbench", + "boolean" + ], + "function_name": "bbh_boolean_expressions", + "is_alpha": false + }, { "name": "BBH: Causal Judgment", "description": "BigBench Hard - Causal judgment reasoning", @@ -1060,6 +1086,30 @@ export const benchmarksData = [ "function_name": "bbh_disambiguation_qa", "is_alpha": false }, + { + "name": "BBH: Dyck Languages", + "description": "BIG-Bench Hard balanced-parentheses completion", + "category": "core", + "tags": [ + "reasoning", + "bigbench", + "formal-language" + ], + "function_name": "bbh_dyck_languages", + "is_alpha": false + }, + { + "name": "BBH: Formal Fallacies", + "description": "BIG-Bench Hard formal fallacy detection", + "category": "core", + "tags": [ + "reasoning", + "bigbench", + "logic" + ], + "function_name": "bbh_formal_fallacies", + "is_alpha": false + }, { "name": "BBH: Geometric Shapes", "description": "BigBench Hard - Reasoning about geometric shapes", @@ -1074,6 +1124,18 @@ export const benchmarksData = [ "function_name": "bbh_geometric_shapes", "is_alpha": false }, + { + "name": "BBH: Hyperbaton", + "description": "BIG-Bench Hard adjective-order reasoning", + "category": "core", + "tags": [ + "reasoning", + "bigbench", + "language" + ], + "function_name": "bbh_hyperbaton", + "is_alpha": false + }, { "name": "BBH: Logical Deduction (3 Objects)", "description": "BigBench Hard - Logical deduction with three objects", @@ -1129,6 +1191,18 @@ export const benchmarksData = [ "function_name": "bbh_movie_recommendation", "is_alpha": false }, + { + "name": "BBH: Multistep Arithmetic Two", + "description": "BIG-Bench Hard multistep arithmetic", + "category": "core", + "tags": [ + "reasoning", + "bigbench", + "math" + ], + "function_name": "bbh_multistep_arithmetic_two", + "is_alpha": false + }, { "name": "BBH: Navigate", "description": "BigBench Hard - Spatial navigation reasoning", @@ -1143,6 +1217,30 @@ export const benchmarksData = [ "function_name": "bbh_navigate", "is_alpha": false }, + { + "name": "BBH: Object Counting", + "description": "BIG-Bench Hard object counting", + "category": "core", + "tags": [ + "reasoning", + "bigbench", + "counting" + ], + "function_name": "bbh_object_counting", + "is_alpha": false + }, + { + "name": "BBH: Penguins in a Table", + "description": "BIG-Bench Hard tabular reasoning", + "category": "core", + "tags": [ + "reasoning", + "bigbench", + "tables" + ], + "function_name": "bbh_penguins_in_a_table", + "is_alpha": false + }, { "name": "BBH: Reasoning About Colored Objects", "description": "BigBench Hard - Reasoning about colored objects", @@ -1268,6 +1366,30 @@ export const benchmarksData = [ "function_name": "bbh_tracking_shuffled_objects_seven_objects", "is_alpha": false }, + { + "name": "BBH: Web of Lies", + "description": "BIG-Bench Hard truth-teller reasoning", + "category": "core", + "tags": [ + "reasoning", + "bigbench", + "logic" + ], + "function_name": "bbh_web_of_lies", + "is_alpha": false + }, + { + "name": "BBH: Word Sorting", + "description": "BIG-Bench Hard ordered word sorting", + "category": "core", + "tags": [ + "reasoning", + "bigbench", + "sorting" + ], + "function_name": "bbh_word_sorting", + "is_alpha": false + }, { "name": "BBQ (Main Function)", "description": "BBQ bias evaluation for a specific category - use individual category tasks instead", @@ -1449,6 +1571,62 @@ export const benchmarksData = [ "function_name": "bbq_ses", "is_alpha": false }, + { + "name": "BFCL v4 Agentic Offline", + "description": "Reproducible offline BFCL v4 memory and frozen web-search adaptation", + "category": "core", + "tags": [ + "function-calling", + "tools", + "bfcl", + "agentic", + "offline" + ], + "function_name": "bfcl_v4_agentic_offline", + "is_alpha": false + }, + { + "name": "BFCL v4 Multi-Turn", + "description": "Official four-category BFCL v4 stateful multi-turn evaluation", + "category": "core", + "tags": [ + "function-calling", + "tools", + "bfcl", + "multi-turn", + "docker" + ], + "function_name": "bfcl_v4_multi_turn", + "is_alpha": false + }, + { + "name": "BFCL v4 Offline Aggregate", + "description": "All BFCL v4 sections with official weights and frozen agentic evidence", + "category": "core", + "tags": [ + "function-calling", + "tools", + "bfcl", + "aggregate", + "offline" + ], + "function_name": "bfcl_v4_offline", + "is_alpha": false + }, + { + "name": "BFCL v4 Single-Turn", + "description": "Berkeley Function Calling Leaderboard v4 single-turn AST and relevance evaluation", + "category": "core", + "tags": [ + "function-calling", + "tools", + "bfcl", + "ast", + "parallel" + ], + "function_name": "bfcl_v4_single_turn", + "is_alpha": false + }, { "name": "BLiMP (67 Linguistic Phenomena)", "description": "Benchmark of Linguistic Minimal Pairs testing grammatical knowledge through minimal pair comparisons", @@ -3890,6 +4068,33 @@ export const benchmarksData = [ "function_name": "browsecomp", "is_alpha": false }, + { + "name": "C-Eval", + "description": "Chinese academic multiple-choice benchmark across 52 subjects", + "category": "knowledge-qa", + "tags": [ + "multiple-choice", + "chinese", + "academic", + "multidisciplinary" + ], + "function_name": "ceval", + "is_alpha": false + }, + { + "name": "C-Eval Hard", + "description": "Official eight-subject hard subset of C-Eval", + "category": "knowledge-qa", + "tags": [ + "multiple-choice", + "chinese", + "math", + "science", + "reasoning" + ], + "function_name": "ceval_hard", + "is_alpha": false + }, { "name": "COPA", "description": "Choice of Plausible Alternatives for causal reasoning", @@ -4401,6 +4606,32 @@ export const benchmarksData = [ "function_name": "gsm8k", "is_alpha": false }, + { + "name": "GSM8K Hard (GSM-Hard)", + "description": "Numerically perturbed GSM8K evaluation set introduced by PAL", + "category": "math", + "tags": [ + "math", + "reasoning", + "gsm8k", + "robustness" + ], + "function_name": "gsm8k_hard", + "is_alpha": false + }, + { + "name": "Global PIQA v1", + "description": "Multilingual and multicultural physical commonsense reasoning", + "category": "core", + "tags": [ + "commonsense", + "multilingual", + "multicultural", + "multiple-choice" + ], + "function_name": "global_piqa", + "is_alpha": false + }, { "name": "Global-MMLU (42 Languages)", "description": "Culturally adapted multilingual MMLU with 42 languages", @@ -5040,6 +5271,32 @@ export const benchmarksData = [ "function_name": "hmmt_feb_2025", "is_alpha": false }, + { + "name": "HMMT February 2026", + "description": "Harvard-MIT Mathematics Tournament February 2026", + "category": "math", + "tags": [ + "math", + "competition", + "hmmt", + "2026" + ], + "function_name": "hmmt_feb_2026", + "is_alpha": false + }, + { + "name": "HMMT November 2025", + "description": "Harvard-MIT Mathematics Tournament November 2025", + "category": "math", + "tags": [ + "math", + "competition", + "hmmt", + "2025" + ], + "function_name": "hmmt_nov_2025", + "is_alpha": false + }, { "name": "HeadQA", "description": "Spanish healthcare specialization exam questions (Spanish and English)", @@ -5144,6 +5401,19 @@ export const benchmarksData = [ "function_name": "humaneval", "is_alpha": false }, + { + "name": "HumanEval+", + "description": "EvalPlus HumanEval with base and augmented differential tests", + "category": "core", + "tags": [ + "code", + "generation", + "evalplus", + "sandbox" + ], + "function_name": "humaneval_plus", + "is_alpha": false + }, { "name": "Humanity's Last Exam", "description": "Multi-modal benchmark at the frontier of human knowledge - 2,500 questions across mathematics, humanities, and natural sciences designed by subject-matter experts globally", @@ -5223,6 +5493,20 @@ export const benchmarksData = [ "function_name": "legalsupport", "is_alpha": false }, + { + "name": "LiveCodeBench v6", + "description": "Contamination-aware Python code generation through the April 2025 release", + "category": "core", + "tags": [ + "coding", + "generation", + "execution", + "docker", + "live" + ], + "function_name": "livecodebench_v6", + "is_alpha": false + }, { "name": "LiveMCPBench", "description": "Benchmark for evaluating LLM agents on real-world tasks using the Model Context Protocol (MCP) - 95 tasks across different categories", @@ -5292,6 +5576,19 @@ export const benchmarksData = [ "function_name": "mbpp", "is_alpha": false }, + { + "name": "MBPP+", + "description": "EvalPlus MBPP with base and augmented differential tests", + "category": "core", + "tags": [ + "code", + "generation", + "evalplus", + "sandbox" + ], + "function_name": "mbpp_plus", + "is_alpha": false + }, { "name": "MGSM", "description": "Multilingual Grade School Math benchmark across 11 languages for testing mathematical reasoning", @@ -6399,6 +6696,19 @@ export const benchmarksData = [ "function_name": "natural_questions", "is_alpha": false }, + { + "name": "OCRBench v1", + "description": "1,000-example OCR benchmark for large multimodal models", + "category": "core", + "tags": [ + "multimodal", + "vision-language", + "ocr", + "images" + ], + "function_name": "ocrbench", + "is_alpha": false + }, { "name": "OCRBench v2", "description": "Visual text localization and reasoning benchmark across 31 diverse OCR and document understanding scenarios", @@ -8060,23 +8370,30 @@ export const evalGroupsData = [ }, { "name": "BIG-Bench Hard", - "description": "Aggregate of 18 challenging BIG-Bench tasks that require multi-step reasoning", + "description": "Complete aggregate of 23 conceptual BBH tasks (27 dataset configurations)", "category": "eval-group", "tags": [ "eval-group" ], "id": "bbh", - "benchmark_count": 18, + "benchmark_count": 27, "benchmarks": [ + "bbh_boolean_expressions", "bbh_causal_judgment", "bbh_date_understanding", "bbh_disambiguation_qa", + "bbh_dyck_languages", + "bbh_formal_fallacies", "bbh_geometric_shapes", + "bbh_hyperbaton", "bbh_logical_deduction_five_objects", "bbh_logical_deduction_seven_objects", "bbh_logical_deduction_three_objects", "bbh_movie_recommendation", + "bbh_multistep_arithmetic_two", "bbh_navigate", + "bbh_object_counting", + "bbh_penguins_in_a_table", "bbh_reasoning_about_colored_objects", "bbh_ruin_names", "bbh_salient_translation_error_detection", @@ -8085,7 +8402,9 @@ export const evalGroupsData = [ "bbh_temporal_sequences", "bbh_tracking_shuffled_objects_five_objects", "bbh_tracking_shuffled_objects_seven_objects", - "bbh_tracking_shuffled_objects_three_objects" + "bbh_tracking_shuffled_objects_three_objects", + "bbh_web_of_lies", + "bbh_word_sorting" ] }, { @@ -8388,13 +8707,13 @@ export const evalGroupsData = [ }, { "name": "MathArena", - "description": "Aggregate of 11 math competition tasks", + "description": "Aggregate of 14 math competition tasks", "category": "eval-group", "tags": [ "eval-group" ], "id": "matharena", - "benchmark_count": 11, + "benchmark_count": 14, "benchmarks": [ "aime_2023_I", "aime_2023_II", @@ -8403,10 +8722,13 @@ export const evalGroupsData = [ "aime_2024_II", "aime_2025", "aime_2025_II", + "aime_2026", "brumo_2025", "hmmt_feb_2023", "hmmt_feb_2024", - "hmmt_feb_2025" + "hmmt_feb_2025", + "hmmt_nov_2025", + "hmmt_feb_2026" ] }, { diff --git a/packages/openbench-core/pyproject.toml b/packages/openbench-core/pyproject.toml index a9861b7e..060c6562 100644 --- a/packages/openbench-core/pyproject.toml +++ b/packages/openbench-core/pyproject.toml @@ -15,7 +15,7 @@ authors = [ dependencies = [ "datasets>=3.6.0", "groq>=0.33.0", - "inspect-ai==0.3.141", + "inspect-ai==0.3.142", "inspect_swe>=0.2.26", "anthropic>=0.69.0", "openai>=2.0.0", @@ -28,6 +28,8 @@ dependencies = [ "tiktoken>=0.11.0", "typer>=0.15.3", "numpy==2.2.6", + "tree-sitter>=0.25.2", + "tree-sitter-python>=0.25.0", ] [project.urls] @@ -46,6 +48,14 @@ package-dir = {"" = "../../src"} where = ["../../src"] include = ["openbench*"] +[tool.setuptools.package-data] +"openbench.evals.livecodebench" = ["Dockerfile", "compose.yaml"] +"openbench.evals.evalplus" = ["Dockerfile", "compose.yaml"] +"openbench.evals.bfcl" = ["Dockerfile", "compose.yaml", "runner.py"] + +[tool.setuptools.exclude-package-data] +"*" = ["__pycache__/*", "*.pyc"] + [tool.pytest.ini_options] minversion = "8.0" addopts = "-ra -q --strict-markers" @@ -55,6 +65,7 @@ asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" markers = [ "integration: marks tests that require external services (deselect with '-m \"not integration\"')", + "docker: marks tests that build and execute Docker sandboxes", ] filterwarnings = [ "ignore:install \"ipywidgets\" for Jupyter support:UserWarning:rich.live", @@ -85,6 +96,7 @@ tau2 = { git = "https://github.com/sierra-research/tau2-bench.git", rev = "558e6 [dependency-groups] dev = [ "mypy>=1.15.0", + "pandas-stubs>=2.3.3.260113,<3", "pre-commit>=4.2.0", "pytest>=8.3.5", "pytest-asyncio==0.24.0", diff --git a/pyproject.toml b/pyproject.toml index a904018a..57e97c54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ authors = [ dependencies = [ "datasets>=3.6.0", "groq>=0.33.0", - "inspect-ai==0.3.141", + "inspect-ai==0.3.142", "inspect_swe>=0.2.26", "anthropic>=0.69.0", "openai>=2.0.0", @@ -29,6 +29,8 @@ dependencies = [ "tiktoken>=0.11.0", "typer>=0.15.3", "numpy==2.2.6", + "tree-sitter>=0.25.2", + "tree-sitter-python>=0.25.0", ] [project.urls] @@ -51,6 +53,14 @@ package-dir = {"" = "src"} where = ["src"] include = ["openbench*"] +[tool.setuptools.package-data] +"openbench.evals.livecodebench" = ["Dockerfile", "compose.yaml"] +"openbench.evals.evalplus" = ["Dockerfile", "compose.yaml"] +"openbench.evals.bfcl" = ["Dockerfile", "compose.yaml", "runner.py"] + +[tool.setuptools.exclude-package-data] +"*" = ["__pycache__/*", "*.pyc"] + [tool.pytest.ini_options] minversion = "8.0" addopts = "-ra -q --strict-markers" @@ -60,6 +70,7 @@ asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" markers = [ "integration: marks tests that require external services (deselect with '-m \"not integration\"')", + "docker: marks tests that build and execute Docker sandboxes", ] filterwarnings = [ "ignore:install \"ipywidgets\" for Jupyter support:UserWarning:rich.live", @@ -90,6 +101,7 @@ tau2 = { git = "https://github.com/sierra-research/tau2-bench.git", rev = "558e6 [dependency-groups] dev = [ "mypy>=1.15.0", + "pandas-stubs>=2.3.3.260113,<3", "pre-commit>=4.2.0", "pytest>=8.3.5", "pytest-asyncio==0.24.0", diff --git a/src/openbench/_registry.py b/src/openbench/_registry.py index 600b427b..3a12f9c7 100644 --- a/src/openbench/_registry.py +++ b/src/openbench/_registry.py @@ -265,6 +265,7 @@ def openbench_vllm_override(): from .evals.hellaswag import hellaswag # noqa: F401, E402 from .evals.hle import hle, hle_text # noqa: F401, E402 from .evals.humaneval import humaneval # noqa: F401, E402 +from .evals.livecodebench import livecodebench_v6 # noqa: F401, E402 from .evals.ifeval import ifeval # noqa: F401, E402 from .evals.ifbench import ifbench # noqa: F401, E402 from .evals.exercism.exercism import ( # noqa: F401, E402 @@ -415,6 +416,12 @@ def openbench_vllm_override(): from .evals import mathqa # noqa: F401, E402 from .evals import sciq # noqa: F401, E402 from .evals import truthfulqa # noqa: F401, E402 +from .evals import bfcl # noqa: F401, E402 +from .evals import gsm8k_hard # noqa: F401, E402 +from .evals import ceval # noqa: F401, E402 +from .evals import ocrbench # noqa: F401, E402 +from .evals import global_piqa # noqa: F401, E402 +from .evals import evalplus # noqa: F401, E402 from .evals import factscore # noqa: F401, E402 # Linguistic Phenomena benchmarks @@ -428,10 +435,13 @@ def openbench_vllm_override(): from .evals.matharena.aime_2024.aime_2024 import aime_2024 # noqa: F401, E402 from .evals.matharena.aime_2025.aime_2025 import aime_2025 # noqa: F401, E402 from .evals.matharena.aime_2025_II.aime_2025_II import aime_2025_II # noqa: F401, E402 +from .evals.matharena.aime_2026.aime_2026 import aime_2026 # noqa: F401, E402 from .evals.matharena.brumo_2025.brumo_2025 import brumo_2025 # noqa: F401, E402 from .evals.matharena.hmmt_feb_2023.hmmt_feb_2023 import hmmt_feb_2023 # noqa: F401, E402 from .evals.matharena.hmmt_feb_2024.hmmt_feb_2024 import hmmt_feb_2024 # noqa: F401, E402 from .evals.matharena.hmmt_feb_2025.hmmt_feb_2025 import hmmt_feb_2025 # noqa: F401, E402 +from .evals.matharena.hmmt_nov_2025.hmmt_nov_2025 import hmmt_nov_2025 # noqa: F401, E402 +from .evals.matharena.hmmt_feb_2026.hmmt_feb_2026 import hmmt_feb_2026 # noqa: F401, E402 # Domain-Specific benchmarks from .evals.arabic_exams import ( # noqa: F401, E402 diff --git a/src/openbench/config.py b/src/openbench/config.py index 9070ee7d..8119f6b2 100644 --- a/src/openbench/config.py +++ b/src/openbench/config.py @@ -97,6 +97,42 @@ class EvalGroup: # Built-in benchmark metadata - minimal, no duplication _BUILTIN_BENCHMARKS = { + "bfcl_v4_single_turn": BenchmarkMetadata( + name="BFCL v4 Single-Turn", + description="Berkeley Function Calling Leaderboard v4 single-turn AST and relevance evaluation", + category="core", + tags=["function-calling", "tools", "bfcl", "ast", "parallel"], + module_path="openbench.evals.bfcl", + function_name="bfcl_v4_single_turn", + is_alpha=False, + ), + "bfcl_v4_multi_turn": BenchmarkMetadata( + name="BFCL v4 Multi-Turn", + description="Official four-category BFCL v4 stateful multi-turn evaluation", + category="core", + tags=["function-calling", "tools", "bfcl", "multi-turn", "docker"], + module_path="openbench.evals.bfcl", + function_name="bfcl_v4_multi_turn", + is_alpha=False, + ), + "bfcl_v4_agentic_offline": BenchmarkMetadata( + name="BFCL v4 Agentic Offline", + description="Reproducible offline BFCL v4 memory and frozen web-search adaptation", + category="core", + tags=["function-calling", "tools", "bfcl", "agentic", "offline"], + module_path="openbench.evals.bfcl", + function_name="bfcl_v4_agentic_offline", + is_alpha=False, + ), + "bfcl_v4_offline": BenchmarkMetadata( + name="BFCL v4 Offline Aggregate", + description="All BFCL v4 sections with official weights and frozen agentic evidence", + category="core", + tags=["function-calling", "tools", "bfcl", "aggregate", "offline"], + module_path="openbench.evals.bfcl", + function_name="bfcl_v4_offline", + is_alpha=False, + ), "mbpp": BenchmarkMetadata( name="MBPP", description="Mostly Basic Python Problems — code generation tasks with unit test verification", @@ -106,6 +142,24 @@ class EvalGroup: function_name="mbpp", is_alpha=False, ), + "humaneval_plus": BenchmarkMetadata( + name="HumanEval+", + description="EvalPlus HumanEval with base and augmented differential tests", + category="core", + tags=["code", "generation", "evalplus", "sandbox"], + module_path="openbench.evals.evalplus", + function_name="humaneval_plus", + is_alpha=False, + ), + "mbpp_plus": BenchmarkMetadata( + name="MBPP+", + description="EvalPlus MBPP with base and augmented differential tests", + category="core", + tags=["code", "generation", "evalplus", "sandbox"], + module_path="openbench.evals.evalplus", + function_name="mbpp_plus", + is_alpha=False, + ), # Graphwalks benchmarks (alpha) "clockbench": BenchmarkMetadata( name="ClockBench", @@ -370,6 +424,14 @@ class EvalGroup: module_path="openbench.evals.humaneval", function_name="humaneval", ), + "livecodebench_v6": BenchmarkMetadata( + name="LiveCodeBench v6", + description="Contamination-aware Python code generation through the April 2025 release", + category="core", + tags=["coding", "generation", "execution", "docker", "live"], + module_path="openbench.evals.livecodebench", + function_name="livecodebench_v6", + ), # Exercism benchmarks "exercism": BenchmarkMetadata( name="Exercism", @@ -988,6 +1050,15 @@ class EvalGroup: function_name="aime_2025_II", subtask=True, ), + "aime_2026": BenchmarkMetadata( + name="AIME 2026", + description="Combined American Invitational Mathematics Examination 2026", + category="math", + tags=["math", "competition", "aime", "2026", "combined"], + module_path="openbench.evals.matharena.aime_2026.aime_2026", + function_name="aime_2026", + subtask=True, + ), "brumo_2025": BenchmarkMetadata( name="BRUMO 2025", description="Bruno Mathematical Olympiad 2025", @@ -1024,6 +1095,24 @@ class EvalGroup: function_name="hmmt_feb_2025", subtask=True, ), + "hmmt_nov_2025": BenchmarkMetadata( + name="HMMT November 2025", + description="Harvard-MIT Mathematics Tournament November 2025", + category="math", + tags=["math", "competition", "hmmt", "2025"], + module_path="openbench.evals.matharena.hmmt_nov_2025.hmmt_nov_2025", + function_name="hmmt_nov_2025", + subtask=True, + ), + "hmmt_feb_2026": BenchmarkMetadata( + name="HMMT February 2026", + description="Harvard-MIT Mathematics Tournament February 2026", + category="math", + tags=["math", "competition", "hmmt", "2026"], + module_path="openbench.evals.matharena.hmmt_feb_2026.hmmt_feb_2026", + function_name="hmmt_feb_2026", + subtask=True, + ), "global_mmlu": BenchmarkMetadata( name="Global-MMLU (42 Languages)", description="Culturally adapted multilingual MMLU with 42 languages", @@ -2720,6 +2809,87 @@ class EvalGroup: function_name="bbh_tracking_shuffled_objects_three_objects", subtask=True, ), + "bbh_boolean_expressions": BenchmarkMetadata( + name="BBH: Boolean Expressions", + description="BIG-Bench Hard boolean expression evaluation", + category="core", + tags=["reasoning", "bigbench", "boolean"], + module_path="openbench.evals.bigbench_hard", + function_name="bbh_boolean_expressions", + subtask=True, + ), + "bbh_dyck_languages": BenchmarkMetadata( + name="BBH: Dyck Languages", + description="BIG-Bench Hard balanced-parentheses completion", + category="core", + tags=["reasoning", "bigbench", "formal-language"], + module_path="openbench.evals.bigbench_hard", + function_name="bbh_dyck_languages", + subtask=True, + ), + "bbh_formal_fallacies": BenchmarkMetadata( + name="BBH: Formal Fallacies", + description="BIG-Bench Hard formal fallacy detection", + category="core", + tags=["reasoning", "bigbench", "logic"], + module_path="openbench.evals.bigbench_hard", + function_name="bbh_formal_fallacies", + subtask=True, + ), + "bbh_hyperbaton": BenchmarkMetadata( + name="BBH: Hyperbaton", + description="BIG-Bench Hard adjective-order reasoning", + category="core", + tags=["reasoning", "bigbench", "language"], + module_path="openbench.evals.bigbench_hard", + function_name="bbh_hyperbaton", + subtask=True, + ), + "bbh_multistep_arithmetic_two": BenchmarkMetadata( + name="BBH: Multistep Arithmetic Two", + description="BIG-Bench Hard multistep arithmetic", + category="core", + tags=["reasoning", "bigbench", "math"], + module_path="openbench.evals.bigbench_hard", + function_name="bbh_multistep_arithmetic_two", + subtask=True, + ), + "bbh_object_counting": BenchmarkMetadata( + name="BBH: Object Counting", + description="BIG-Bench Hard object counting", + category="core", + tags=["reasoning", "bigbench", "counting"], + module_path="openbench.evals.bigbench_hard", + function_name="bbh_object_counting", + subtask=True, + ), + "bbh_penguins_in_a_table": BenchmarkMetadata( + name="BBH: Penguins in a Table", + description="BIG-Bench Hard tabular reasoning", + category="core", + tags=["reasoning", "bigbench", "tables"], + module_path="openbench.evals.bigbench_hard", + function_name="bbh_penguins_in_a_table", + subtask=True, + ), + "bbh_web_of_lies": BenchmarkMetadata( + name="BBH: Web of Lies", + description="BIG-Bench Hard truth-teller reasoning", + category="core", + tags=["reasoning", "bigbench", "logic"], + module_path="openbench.evals.bigbench_hard", + function_name="bbh_web_of_lies", + subtask=True, + ), + "bbh_word_sorting": BenchmarkMetadata( + name="BBH: Word Sorting", + description="BIG-Bench Hard ordered word sorting", + category="core", + tags=["reasoning", "bigbench", "sorting"], + module_path="openbench.evals.bigbench_hard", + function_name="bbh_word_sorting", + subtask=True, + ), # BIG-Bench Lite (alias for BBH) "medmcqa": BenchmarkMetadata( name="MedMCQA", @@ -2814,6 +2984,14 @@ class EvalGroup: module_path="openbench.evals.piqa", function_name="piqa", ), + "global_piqa": BenchmarkMetadata( + name="Global PIQA v1", + description="Multilingual and multicultural physical commonsense reasoning", + category="core", + tags=["commonsense", "multilingual", "multicultural", "multiple-choice"], + module_path="openbench.evals.global_piqa", + function_name="global_piqa", + ), "prost": BenchmarkMetadata( name="PROST", description="Physical Reasoning about Objects through Space and Time", @@ -4064,6 +4242,31 @@ class EvalGroup: module_path="openbench.evals.truthfulqa", function_name="truthfulqa", ), + "gsm8k_hard": BenchmarkMetadata( + name="GSM8K Hard (GSM-Hard)", + description="Numerically perturbed GSM8K evaluation set introduced by PAL", + category="math", + tags=["math", "reasoning", "gsm8k", "robustness"], + module_path="openbench.evals.gsm8k_hard", + function_name="gsm8k_hard", + ), + "ceval": BenchmarkMetadata( + name="C-Eval", + description="Chinese academic multiple-choice benchmark across 52 subjects", + category="knowledge-qa", + tags=["multiple-choice", "chinese", "academic", "multidisciplinary"], + module_path="openbench.evals.ceval", + function_name="ceval", + ), + "ceval_hard": BenchmarkMetadata( + name="C-Eval Hard", + description="Official eight-subject hard subset of C-Eval", + category="knowledge-qa", + tags=["multiple-choice", "chinese", "math", "science", "reasoning"], + module_path="openbench.evals.ceval", + function_name="ceval_hard", + subtask=True, + ), # BLiMP: Benchmark of Linguistic Minimal Pairs "blimp": BenchmarkMetadata( name="BLiMP (67 Linguistic Phenomena)", @@ -5921,6 +6124,15 @@ class EvalGroup: function_name="ocrbenchv2", is_alpha=False, ), + "ocrbench": BenchmarkMetadata( + name="OCRBench v1", + description="1,000-example OCR benchmark for large multimodal models", + category="core", + tags=["multimodal", "vision-language", "ocr", "images"], + module_path="openbench.evals.ocrbench", + function_name="ocrbench", + is_alpha=False, + ), "deep_research_bench": BenchmarkMetadata( name="DeepResearch Bench", description="A comprehensive benchmark for evaluating Deep Research Agents", @@ -6398,17 +6610,24 @@ def get_eval_metadata(path_like: str) -> BenchmarkMetadata | None: ), "bbh": EvalGroup( name="BIG-Bench Hard", - description="Aggregate of 18 challenging BIG-Bench tasks that require multi-step reasoning", + description="Complete aggregate of 23 conceptual BBH tasks (27 dataset configurations)", benchmarks=[ + "bbh_boolean_expressions", "bbh_causal_judgment", "bbh_date_understanding", "bbh_disambiguation_qa", + "bbh_dyck_languages", + "bbh_formal_fallacies", "bbh_geometric_shapes", + "bbh_hyperbaton", "bbh_logical_deduction_five_objects", "bbh_logical_deduction_seven_objects", "bbh_logical_deduction_three_objects", "bbh_movie_recommendation", + "bbh_multistep_arithmetic_two", "bbh_navigate", + "bbh_object_counting", + "bbh_penguins_in_a_table", "bbh_reasoning_about_colored_objects", "bbh_ruin_names", "bbh_salient_translation_error_detection", @@ -6418,6 +6637,8 @@ def get_eval_metadata(path_like: str) -> BenchmarkMetadata | None: "bbh_tracking_shuffled_objects_five_objects", "bbh_tracking_shuffled_objects_seven_objects", "bbh_tracking_shuffled_objects_three_objects", + "bbh_web_of_lies", + "bbh_word_sorting", ], ), "agieval": EvalGroup( @@ -6731,7 +6952,7 @@ def get_eval_metadata(path_like: str) -> BenchmarkMetadata | None: ), "matharena": EvalGroup( name="MathArena", - description="Aggregate of 11 math competition tasks", + description="Aggregate of 14 math competition tasks", benchmarks=[ "aime_2023_I", "aime_2023_II", @@ -6740,10 +6961,13 @@ def get_eval_metadata(path_like: str) -> BenchmarkMetadata | None: "aime_2024_II", "aime_2025", "aime_2025_II", + "aime_2026", "brumo_2025", "hmmt_feb_2023", "hmmt_feb_2024", "hmmt_feb_2025", + "hmmt_nov_2025", + "hmmt_feb_2026", ], ), "qa4mre": EvalGroup( diff --git a/src/openbench/datasets/bfcl.py b/src/openbench/datasets/bfcl.py new file mode 100644 index 00000000..bcbc129c --- /dev/null +++ b/src/openbench/datasets/bfcl.py @@ -0,0 +1,349 @@ +"""Pinned BFCL v4 single-turn dataset loader.""" + +from __future__ import annotations + +import hashlib +import json +import urllib.request +from pathlib import Path +from typing import Any + +from inspect_ai.dataset import MemoryDataset, Sample +from inspect_ai.model import ChatMessageSystem, ChatMessageUser +from platformdirs import user_cache_dir + +BFCL_REVISION = "6ea57973c7a6097fd7c5915698c54c17c5b1b6c8" +BFCL_LICENSE = "Apache-2.0" +BFCL_BASE_URL = ( + "https://raw.githubusercontent.com/ShishirPatil/gorilla/" + f"{BFCL_REVISION}/berkeley-function-call-leaderboard/bfcl_eval/data" +) + +SINGLE_TURN_CATEGORIES = ( + "simple_python", + "simple_java", + "simple_javascript", + "multiple", + "parallel", + "parallel_multiple", + "irrelevance", + "live_simple", + "live_multiple", + "live_parallel", + "live_parallel_multiple", + "live_irrelevance", + "live_relevance", +) + +MULTI_TURN_CATEGORIES = ( + "multi_turn_base", + "multi_turn_miss_func", + "multi_turn_miss_param", + "multi_turn_long_context", +) + +AGENTIC_CATEGORIES = ( + "memory_kv", + "memory_vector", + "memory_rec_sum", + "web_search_base", + "web_search_no_snippet", +) + +_MULTI_TURN_COUNT = 200 + +_FUNCTION_DOCS = { + "GorillaFileSystem": "gorilla_file_system.json", + "MathAPI": "math_api.json", + "MessageAPI": "message_api.json", + "TwitterAPI": "posting_api.json", + "TicketAPI": "ticket_api.json", + "TradingBot": "trading_bot.json", + "TravelAPI": "travel_booking.json", + "VehicleControlAPI": "vehicle_control.json", + "WebSearchAPI": "web_search.json", + "MemoryAPI_kv": "memory_kv.json", + "MemoryAPI_vector": "memory_vector.json", + "MemoryAPI_rec_sum": "memory_rec_sum.json", +} + +_FILE_SHA256 = { + "BFCL_v4_simple_python.json": "82dd63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991", + "BFCL_v4_simple_java.json": "13d2303a125b08754f0e41995b9273b5005fa8ed8ebfaa24ef53b4d83c4b5c6e", + "BFCL_v4_simple_javascript.json": "329e67fedf79a6243d93dbda4b388d12bd2d31f1f2163d92cb6ef676d1764f44", + "BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a", + "BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84", + "BFCL_v4_parallel_multiple.json": "8863ea8433239f55c5f016154cf0830853c89f693c6ea270396a2fa121960579", + "BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6", + "BFCL_v4_live_simple.json": "1af2ac87dca47556db7b7e37e51e28b459a38b594e3c7b3c792b4903598ca0c4", + "BFCL_v4_live_multiple.json": "fd8ccfad4d911420d0e3341dbe2fff77d1d341da934248b9bb2bda24ab3a10c8", + "BFCL_v4_live_parallel.json": "6c26e9fdc3350cf596e6d1ea9c179cbff834761bccf562f4141ed29a839ca421", + "BFCL_v4_live_parallel_multiple.json": "21d4b9319c1faac431e22757b367ea28917fe467364c3a4b17f16ec06d4f6e79", + "BFCL_v4_live_irrelevance.json": "6559fda2beaceb609a2cd2e504c65b4a56cb448e1ef88fddfd199e163d163349", + "BFCL_v4_live_relevance.json": "e03f9e241657a137cba48a89ee12f47bf3fcb7e4f6274263e9c699a0c974203a", + "possible_answer/BFCL_v4_simple_python.json": "90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbf795bb51b7dc581f8c86dc", + "possible_answer/BFCL_v4_simple_java.json": "78f25616084044fa05bbfcee68e03f6ececb222bdd5cb3b7783a675fb3366e35", + "possible_answer/BFCL_v4_simple_javascript.json": "e2f9f2e51d88e0c8056ffbf1a3dd3d02eb032532d2b5d98c9cc9003385bdd56b", + "possible_answer/BFCL_v4_multiple.json": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047", + "possible_answer/BFCL_v4_parallel.json": "8a6aa19c1adddc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0", + "possible_answer/BFCL_v4_parallel_multiple.json": "5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3", + "possible_answer/BFCL_v4_live_simple.json": "fec9cfa9744a936f9126981e85a2023da1e63e273eafebc81923a1162fad70ce", + "possible_answer/BFCL_v4_live_multiple.json": "97e90d59c5bd76c55a2920ce93e5566e9046307d3f558578f085f9d3a56c3084", + "possible_answer/BFCL_v4_live_parallel.json": "8a9f189ff0e832ebbbbdade1fd95a7dbcc67406e9177df3f0aad76f59ab00350", + "possible_answer/BFCL_v4_live_parallel_multiple.json": "f5b5f360556c5feb51db46fb9f56ee4b304f4b45b161599bbb14161c98a2873f", + "BFCL_v4_multi_turn_base.json": "1a21a995d06fd6f20ba55de7bced30ef953ec35e998f502ec2ecf4d66ef1c43a", + "BFCL_v4_multi_turn_miss_func.json": "87d28ce10e37d864b72de85d5732eef2a867b241d6c1c99b4ae682c9e3ea921c", + "BFCL_v4_multi_turn_miss_param.json": "f0c66dda3795f5f53e3e1c0cc8ba0246b6761c8f58bdba8317203bf451ab8838", + "BFCL_v4_multi_turn_long_context.json": "78c3268c5cc8e97c0f4ec6c811b3b9a2bba14323b1830b7a874b06d822749324", + "possible_answer/BFCL_v4_multi_turn_base.json": "1fee67823b317571649177dd89d63969feaae4e810cc7448ee55ba797fb7c8fc", + "possible_answer/BFCL_v4_multi_turn_miss_func.json": "69e679b806d1c871b05393a4b95583bb973248e5b8d96c2d7f4ca05e29fc32e6", + "possible_answer/BFCL_v4_multi_turn_miss_param.json": "59c442901779e2c31c33abcd566d032e03736e5ad8069de2fe05489873046ecf", + "possible_answer/BFCL_v4_multi_turn_long_context.json": "e82aa0e839c39d23c64a05834f7ae024d7c9738ec21738ecee78dc876e3d0d18", + "BFCL_v4_memory.json": "40fc21d4528af53c6b44204def89e81515d0101654229e2c2e82bbf5f047b14f", + "BFCL_v4_web_search.json": "6fc41d96d003dc849028966a782923560d2fc127ed2088aa967f06daaafa4268", + "possible_answer/BFCL_v4_memory.json": "2355cf8d842f94af6bcb7bfa6ad2f9e472bc6d825d6ecd45702cfc41e27d7e5d", + "possible_answer/BFCL_v4_web_search.json": "771cab45fdad5744563456801d4623a42c5a358f10514b9b9105d2f6052b4999", + "multi_turn_func_doc/gorilla_file_system.json": "c4c1b741c71e2a17c97a5dc9c4a91d89978c4eaece56494d14272d5df6c650e9", + "multi_turn_func_doc/math_api.json": "83fa31708c89442bdcf12ac4dfbe3be8663ec9183a1b1524a9fda98164bc7e0b", + "multi_turn_func_doc/memory_kv.json": "96480cd9cbd3d4a34cd8f78879bc6622731768a26e080f7a34782e36e3402287", + "multi_turn_func_doc/memory_rec_sum.json": "4ceff946df00983c7f0b95d1b70ae402247f3312fb898bac9ecfa1de52f5a783", + "multi_turn_func_doc/memory_vector.json": "917908ca99fdd01e203274b7ec6eb2347fa91d57f3c6341e20945cc9c9d746cf", + "multi_turn_func_doc/message_api.json": "4d58ea933a5d2b280d7a52366617fb47fecd333d7b0e08e724db6fa12fb5f847", + "multi_turn_func_doc/posting_api.json": "87f9fc404a06e4107d7c366f1399c596440e84c16cb119faf374b2f532d86e8b", + "multi_turn_func_doc/ticket_api.json": "31324e0380782664fe82ba05bd23517808ad55692b45a5fe7c3d4d395fe4f0f8", + "multi_turn_func_doc/trading_bot.json": "1a7933fd8f0cb8fbec38aeae05cb8c24132747cad4904fe1c13ac3d01fc22c2d", + "multi_turn_func_doc/travel_booking.json": "f17b950c13adddf41d0848077df58788252e4c2e7cad5cfa71c8c4bf04f57b26", + "multi_turn_func_doc/vehicle_control.json": "0c8a66292844874ef7b168f343bc394d8615d2d9e1f4387999a9ee23011eac78", + "multi_turn_func_doc/web_search.json": "61fcee411e35f7ff67415e18cd67276615cf06e1c8841a683d2d997dbb46eac5", +} + +_COUNTS = { + "simple_python": 400, + "simple_java": 100, + "simple_javascript": 50, + "multiple": 200, + "parallel": 200, + "parallel_multiple": 200, + "irrelevance": 240, + "live_simple": 258, + "live_multiple": 1053, + "live_parallel": 16, + "live_parallel_multiple": 24, + "live_irrelevance": 884, + "live_relevance": 16, +} + + +def _cache_dir() -> Path: + return Path(user_cache_dir("openbench")) / "bfcl" / BFCL_REVISION + + +def _ensure_file(relative_path: str) -> Path: + expected = _FILE_SHA256[relative_path] + path = _cache_dir() / relative_path + if path.exists() and hashlib.sha256(path.read_bytes()).hexdigest() == expected: + return path + + path.parent.mkdir(parents=True, exist_ok=True) + with urllib.request.urlopen( + f"{BFCL_BASE_URL}/{relative_path}", timeout=120 + ) as response: + content = response.read() + digest = hashlib.sha256(content).hexdigest() + if digest != expected: + raise ValueError( + f"BFCL checksum mismatch for {relative_path}: expected {expected}, got {digest}" + ) + path.write_bytes(content) + return path + + +def _load_jsonl(relative_path: str) -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in _ensure_file(relative_path).read_text().splitlines() + ] + + +def _messages(question: list[list[dict[str, str]]]) -> list[Any]: + messages: list[Any] = [] + for message in question[0]: + content = str(message["content"]) + if message["role"] == "system": + messages.append(ChatMessageSystem(content=content)) + else: + messages.append(ChatMessageUser(content=content)) + return messages + + +def get_bfcl_v4_single_turn_dataset( + categories: list[str] | tuple[str, ...] | None = None, +) -> MemoryDataset: + """Load pinned BFCL v4 single-turn categories and public ground truths.""" + + selected = tuple(categories or SINGLE_TURN_CATEGORIES) + unknown = set(selected) - set(SINGLE_TURN_CATEGORIES) + if unknown: + raise ValueError( + f"Unsupported BFCL v4 single-turn categories: {sorted(unknown)}" + ) + + samples: list[Sample] = [] + for category in selected: + questions = _load_jsonl(f"BFCL_v4_{category}.json") + if category in {"irrelevance", "live_irrelevance", "live_relevance"}: + answers_by_id: dict[str, list[dict[str, Any]]] = {} + else: + answers = _load_jsonl(f"possible_answer/BFCL_v4_{category}.json") + answers_by_id = { + str(answer["id"]): list(answer["ground_truth"]) for answer in answers + } + if len(questions) != _COUNTS[category]: + raise ValueError( + f"BFCL {category} expected {_COUNTS[category]} rows, got {len(questions)}" + ) + for question in questions: + sample_id = str(question["id"]) + expected = answers_by_id.get(sample_id, []) + samples.append( + Sample( + id=sample_id, + input=_messages(question["question"]), + target=json.dumps(expected), + metadata={ + "category": category, + "functions": question["function"], + "expected_calls": expected, + "bfcl_revision": BFCL_REVISION, + "license": BFCL_LICENSE, + }, + ) + ) + return MemoryDataset(samples=samples, name="bfcl_v4_single_turn") + + +def _load_function_docs(class_names: list[str]) -> list[dict[str, Any]]: + functions: list[dict[str, Any]] = [] + for class_name in class_names: + functions.extend( + _load_jsonl(f"multi_turn_func_doc/{_FUNCTION_DOCS[class_name]}") + ) + return functions + + +def get_bfcl_v4_multi_turn_dataset( + categories: list[str] | tuple[str, ...] | None = None, +) -> MemoryDataset: + """Load the four pinned BFCL v4 multi-turn categories.""" + + selected = tuple(categories or MULTI_TURN_CATEGORIES) + unknown = set(selected) - set(MULTI_TURN_CATEGORIES) + if unknown: + raise ValueError(f"Unsupported BFCL multi-turn categories: {sorted(unknown)}") + + samples: list[Sample] = [] + for category in selected: + questions = _load_jsonl(f"BFCL_v4_{category}.json") + answers = _load_jsonl(f"possible_answer/BFCL_v4_{category}.json") + answers_by_id = {str(row["id"]): row["ground_truth"] for row in answers} + if len(questions) != _MULTI_TURN_COUNT: + raise ValueError( + f"BFCL {category} expected {_MULTI_TURN_COUNT} rows, got {len(questions)}" + ) + for question in questions: + sample_id = str(question["id"]) + all_functions = _load_function_docs(question["involved_classes"]) + missed: dict[str, list[dict[str, Any]]] = {} + for turn, names in question.get("missed_function", {}).items(): + missed[str(turn)] = [ + function for function in all_functions if function["name"] in names + ] + missed_names = { + function["name"] + for functions in missed.values() + for function in functions + } + initial_functions = [ + function + for function in all_functions + if function["name"] not in missed_names + and function["name"] not in question.get("excluded_function", []) + ] + turns = question["question"] + expected = answers_by_id[sample_id] + samples.append( + Sample( + id=sample_id, + input=_messages([turns[0]]), + target=json.dumps(expected), + metadata={ + "category": category, + "turns": turns, + "functions": initial_functions, + "missed_functions": missed, + "initial_config": question.get("initial_config", {}), + "involved_classes": question["involved_classes"], + "ground_truth": expected, + "bfcl_revision": BFCL_REVISION, + "license": BFCL_LICENSE, + }, + ) + ) + return MemoryDataset(samples=samples, name="bfcl_v4_multi_turn") + + +def get_bfcl_v4_agentic_dataset( + categories: list[str] | tuple[str, ...] | None = None, +) -> MemoryDataset: + """Load reproducible offline variants of BFCL v4 memory and web-search.""" + + selected = tuple(categories or AGENTIC_CATEGORIES) + unknown = set(selected) - set(AGENTIC_CATEGORIES) + if unknown: + raise ValueError(f"Unsupported BFCL agentic categories: {sorted(unknown)}") + + memory_questions = _load_jsonl("BFCL_v4_memory.json") + memory_answers = { + str(row["id"]): row + for row in _load_jsonl("possible_answer/BFCL_v4_memory.json") + } + web_questions = _load_jsonl("BFCL_v4_web_search.json") + web_answers = { + str(row["id"]): row + for row in _load_jsonl("possible_answer/BFCL_v4_web_search.json") + } + samples: list[Sample] = [] + for category in selected: + is_memory = category.startswith("memory_") + questions = memory_questions if is_memory else web_questions + answers = memory_answers if is_memory else web_answers + class_name = ( + f"MemoryAPI_{category.removeprefix('memory_')}" + if is_memory + else "WebSearchAPI" + ) + functions = _load_function_docs([class_name]) + for question in questions: + source_id = str(question["id"]) + answer = answers[source_id] + sample_id = source_id.replace( + "memory" if is_memory else "web_search", category + ) + samples.append( + Sample( + id=sample_id, + input=_messages(question["question"]), + target=json.dumps(answer["ground_truth"]), + metadata={ + "category": category, + "functions": functions, + "expected_answers": answer["ground_truth"], + "frozen_source": answer["source"], + "show_snippet": category != "web_search_no_snippet", + "bfcl_revision": BFCL_REVISION, + "license": BFCL_LICENSE, + "offline_adaptation": True, + }, + ) + ) + return MemoryDataset(samples=samples, name="bfcl_v4_agentic_offline") diff --git a/src/openbench/datasets/ceval.py b/src/openbench/datasets/ceval.py new file mode 100644 index 00000000..8044fe8b --- /dev/null +++ b/src/openbench/datasets/ceval.py @@ -0,0 +1,144 @@ +"""Pinned C-Eval dataset loader with official Chinese prompts.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from datasets import load_dataset # type: ignore[import-untyped] +from inspect_ai.dataset import MemoryDataset, Sample + +DATASET_PATH = "ceval/ceval-exam" +DATASET_REVISION = "617524a00b307ff6f9933702f724131fe12ca7ce" + +# subject: (Chinese display name, official super-category) +SUBJECTS = { + "computer_network": ("计算机网络", "STEM"), + "operating_system": ("操作系统", "STEM"), + "computer_architecture": ("计算机组成", "STEM"), + "college_programming": ("大学编程", "STEM"), + "college_physics": ("大学物理", "STEM"), + "college_chemistry": ("大学化学", "STEM"), + "advanced_mathematics": ("高等数学", "STEM"), + "probability_and_statistics": ("概率统计", "STEM"), + "discrete_mathematics": ("离散数学", "STEM"), + "electrical_engineer": ("注册电气工程师", "STEM"), + "metrology_engineer": ("注册计量师", "STEM"), + "high_school_mathematics": ("高中数学", "STEM"), + "high_school_physics": ("高中物理", "STEM"), + "high_school_chemistry": ("高中化学", "STEM"), + "high_school_biology": ("高中生物", "STEM"), + "middle_school_mathematics": ("初中数学", "STEM"), + "middle_school_biology": ("初中生物", "STEM"), + "middle_school_physics": ("初中物理", "STEM"), + "middle_school_chemistry": ("初中化学", "STEM"), + "veterinary_medicine": ("兽医学", "STEM"), + "college_economics": ("大学经济学", "Social Science"), + "business_administration": ("工商管理", "Social Science"), + "marxism": ("马克思主义基本原理", "Social Science"), + "mao_zedong_thought": ( + "毛泽东思想和中国特色社会主义理论体系概论", + "Social Science", + ), + "education_science": ("教育学", "Social Science"), + "teacher_qualification": ("教师资格", "Social Science"), + "high_school_politics": ("高中政治", "Social Science"), + "high_school_geography": ("高中地理", "Social Science"), + "middle_school_politics": ("初中政治", "Social Science"), + "middle_school_geography": ("初中地理", "Social Science"), + "modern_chinese_history": ("近代史纲要", "Humanities"), + "ideological_and_moral_cultivation": ("思想道德修养与法律基础", "Humanities"), + "logic": ("逻辑学", "Humanities"), + "law": ("法学", "Humanities"), + "chinese_language_and_literature": ("中国语言文学", "Humanities"), + "art_studies": ("艺术学", "Humanities"), + "professional_tour_guide": ("导游资格", "Humanities"), + "legal_professional": ("法律职业资格", "Humanities"), + "high_school_chinese": ("高中语文", "Humanities"), + "high_school_history": ("高中历史", "Humanities"), + "middle_school_history": ("初中历史", "Humanities"), + "civil_servant": ("公务员", "Other"), + "sports_science": ("体育学", "Other"), + "plant_protection": ("植物保护", "Other"), + "basic_medicine": ("基础医学", "Other"), + "clinical_medicine": ("临床医学", "Other"), + "urban_and_rural_planner": ("注册城乡规划师", "Other"), + "accountant": ("注册会计师", "Other"), + "fire_engineer": ("注册消防工程师", "Other"), + "environmental_impact_assessment_engineer": ("环境影响评价工程师", "Other"), + "tax_accountant": ("税务师", "Other"), + "physician": ("医师资格", "Other"), +} + +HARD_SUBJECTS = ( + "advanced_mathematics", + "discrete_mathematics", + "probability_and_statistics", + "college_chemistry", + "college_physics", + "high_school_mathematics", + "high_school_chemistry", + "high_school_physics", +) + + +def _format_question(record: dict, answer: str | None = None) -> str: + text = ( + f"{record['question']}\n" + f"A. {record['A']}\nB. {record['B']}\n" + f"C. {record['C']}\nD. {record['D']}\n答案：" + ) + return text + (answer if answer is not None else "") + + +def get_ceval_dataset( + *, + subjects: Iterable[str] | None = None, + split: str = "val", + shots: int = 5, +) -> MemoryDataset: + """Load C-Eval subjects and apply the official answer-only prompt.""" + if split not in {"val", "test"}: + raise ValueError("C-Eval evaluation split must be 'val' or 'test'") + if shots not in {0, 5}: + raise ValueError("C-Eval officially supports 0-shot or 5-shot prompting") + + selected = tuple(subjects) if subjects is not None else tuple(SUBJECTS) + unknown = sorted(set(selected) - SUBJECTS.keys()) + if unknown: + raise ValueError(f"Unknown C-Eval subjects: {unknown}") + + samples: list[Sample] = [] + for subject in selected: + chinese_name, category = SUBJECTS[subject] + prefix = ( + f"以下是中国关于{chinese_name}考试的单项选择题，请选出其中的正确答案。\n\n" + ) + if shots: + dev = load_dataset( + DATASET_PATH, + name=subject, + split="dev", + revision=DATASET_REVISION, + ) + prefix += "\n\n".join( + _format_question(record, str(record["answer"])) for record in dev + ) + prefix += "\n\n" + + records = load_dataset( + DATASET_PATH, + name=subject, + split=split, + revision=DATASET_REVISION, + ) + for record in records: + samples.append( + Sample( + id=f"{subject}-{record['id']}", + input=prefix + _format_question(record), + target=str(record["answer"]), + metadata={"subject": subject, "category": category}, + ) + ) + + return MemoryDataset(samples=samples, name="ceval") diff --git a/src/openbench/datasets/evalplus.py b/src/openbench/datasets/evalplus.py new file mode 100644 index 00000000..49f985e7 --- /dev/null +++ b/src/openbench/datasets/evalplus.py @@ -0,0 +1,123 @@ +"""Pinned loaders for the official EvalPlus release artifacts.""" + +from __future__ import annotations + +import gzip +import hashlib +import json +import urllib.request +from pathlib import Path + +from inspect_ai.dataset import MemoryDataset, Sample +from platformdirs import user_cache_dir + +RELEASES = { + "humaneval": { + "version": "v0.1.10", + "url": "https://raw.githubusercontent.com/evalplus/humanevalplus_release/200defce9e3429d28ca215b6dd061c0f7f31c18b/HumanEvalPlus.jsonl.gz", + "sha256": "272720b90ac375502c8ed23cd791c2a93dfb22a911641a494da74a426c09f101", + "expanded_sha256": "42526ec0e7d5f3ee0b06d6ced98f8c8bae3d76519151bfb3d36f79010645bd7f", + "count": 164, + }, + "mbpp": { + "version": "v0.2.0", + "url": "https://raw.githubusercontent.com/evalplus/mbppplus_release/64fc4195b858a17cdfdb3324f0baf37939144e14/MbppPlus.jsonl.gz", + "sha256": "af43697e8791c4c149bdfd6b489d8b5412507551ac20e28a439f650b8225db63", + "expanded_sha256": "b54e762755248ca411b523c917fa9f93c07b5ff2966bf60b3917b853926a3dad", + "count": 378, + }, +} + + +def _cache_dir() -> Path: + return Path(user_cache_dir("openbench")) / "evalplus" + + +def _ensure_release(dataset: str) -> Path: + release = RELEASES[dataset] + cache_dir = _cache_dir() + cache_dir.mkdir(parents=True, exist_ok=True) + compressed = cache_dir / f"{dataset}-{release['version']}.jsonl.gz" + expanded = cache_dir / f"{dataset}-{release['version']}.jsonl" + + valid_cache = ( + compressed.exists() + and hashlib.sha256(compressed.read_bytes()).hexdigest() == release["sha256"] + ) + if not valid_cache: + with urllib.request.urlopen(str(release["url"]), timeout=120) as response: + content = response.read() + digest = hashlib.sha256(content).hexdigest() + if digest != release["sha256"]: + raise ValueError( + f"EvalPlus {dataset} checksum mismatch: " + f"expected {release['sha256']}, got {digest}" + ) + compressed.write_bytes(content) + + expanded_valid = ( + expanded.exists() + and hashlib.sha256(expanded.read_bytes()).hexdigest() + == release["expanded_sha256"] + ) + if not expanded_valid: + content = gzip.decompress(compressed.read_bytes()) + digest = hashlib.sha256(content).hexdigest() + if digest != release["expanded_sha256"]: + raise ValueError( + f"Expanded EvalPlus {dataset} checksum mismatch: " + f"expected {release['expanded_sha256']}, got {digest}" + ) + expanded.write_bytes(content) + return expanded + + +def load_evalplus_record(metadata: dict) -> dict: + """Reload one record without putting hidden tests in Inspect's sample log.""" + path = Path(str(metadata["source_file"])) + with path.open("rb") as source: + source.seek(int(metadata["source_offset"])) + line = source.read(int(metadata["source_length"])) + return json.loads(line) + + +def _instruction(prompt: str) -> str: + return ( + "Please provide a self-contained Python script that solves the following " + f"problem in a markdown code block:\n```python\n{prompt.strip()}\n```" + ) + + +def get_evalplus_dataset(dataset: str) -> MemoryDataset: + """Create samples whose metadata references, but never embeds, hidden tests.""" + if dataset not in RELEASES: + raise ValueError(f"Unknown EvalPlus dataset: {dataset}") + path = _ensure_release(dataset) + samples: list[Sample] = [] + with path.open("rb") as source: + while line := source.readline(): + offset = source.tell() - len(line) + record = json.loads(line) + samples.append( + Sample( + id=str(record["task_id"]), + input=_instruction(str(record["prompt"])), + target=str(record["entry_point"]), + metadata={ + "dataset": dataset, + "entry_point": str(record["entry_point"]), + "prompt": str(record["prompt"]), + "source_file": str(path), + "source_offset": offset, + "source_length": len(line), + "release_version": RELEASES[dataset]["version"], + "release_sha256": RELEASES[dataset]["sha256"], + }, + ) + ) + if len(samples) != RELEASES[dataset]["count"]: + raise ValueError( + f"EvalPlus {dataset} expected {RELEASES[dataset]['count']} records, " + f"got {len(samples)}" + ) + return MemoryDataset(samples=samples, name=f"{dataset}plus") diff --git a/src/openbench/datasets/global_piqa.py b/src/openbench/datasets/global_piqa.py new file mode 100644 index 00000000..1e91a44b --- /dev/null +++ b/src/openbench/datasets/global_piqa.py @@ -0,0 +1,83 @@ +"""Global PIQA v1 loader for parallel and non-parallel components.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from datasets import get_dataset_config_names, load_dataset # type: ignore[import-untyped] +from inspect_ai.dataset import MemoryDataset, Sample + +DATASETS = { + "nonparallel": ( + "mrlbenchmarks/global-piqa-nonparallel", + "6777742fa3634c0583cda3b7f8a482ea7b1b0937", + ), + "parallel": ( + "mrlbenchmarks/global-piqa-parallel", + "b0b18516a8bc2cb1106bce3dd4db32848ca715ea", + ), +} + + +def _prompt(record: dict, component: str) -> str: + choices = [ + record[f"solution{index}"] + for index in range(2 if component == "nonparallel" else 4) + ] + options = "\n\n".join( + f"Option {chr(65 + index)}: {choice}" for index, choice in enumerate(choices) + ) + if component == "nonparallel": + return ( + "Given the following situation, which option is more likely to be correct?\n\n" + f"Situation:\n{record['prompt']}\n\n{options}\n\n" + 'Your response should end with "The best answer is: [answer_letter]" ' + "where [answer_letter] is one of A or B." + ) + return ( + f"{record['prompt']}\n\n{options}\n\n" + 'Your response should end with "The best answer is: [answer_letter]" ' + "where [answer_letter] is one of A, B, C, or D." + ) + + +def get_global_piqa_dataset( + *, + components: Iterable[str] = ("nonparallel", "parallel"), + languages: Iterable[str] | None = None, +) -> MemoryDataset: + """Load Global PIQA generation mode with immutable source revisions.""" + selected_components = tuple(components) + unknown = sorted(set(selected_components) - DATASETS.keys()) + if unknown: + raise ValueError(f"Unknown Global PIQA components: {unknown}") + + selected_languages = set(languages) if languages is not None else None + samples: list[Sample] = [] + for component in selected_components: + path, revision = DATASETS[component] + configs = get_dataset_config_names(path, revision=revision) + if selected_languages is not None: + configs = [config for config in configs if config in selected_languages] + for language in configs: + records = load_dataset( + path, + name=language, + split="test", + revision=revision, + ) + for index, record in enumerate(records): + target = chr(65 + int(record["label"])) + samples.append( + Sample( + id=f"{component}-{language}-{record.get('example_id', index)}", + input=_prompt(record, component), + target=target, + metadata={ + "component": component, + "language": language, + "example_id": record.get("example_id"), + }, + ) + ) + return MemoryDataset(samples=samples, name="global_piqa") diff --git a/src/openbench/datasets/livecodebench.py b/src/openbench/datasets/livecodebench.py new file mode 100644 index 00000000..2e234221 --- /dev/null +++ b/src/openbench/datasets/livecodebench.py @@ -0,0 +1,206 @@ +"""Dataset loader for LiveCodeBench code generation release v6.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from datetime import datetime +from pathlib import Path +from typing import Any + +from huggingface_hub import hf_hub_download +from inspect_ai.dataset import Dataset, MemoryDataset, Sample +from inspect_ai.model import ChatMessageSystem, ChatMessageUser + +DATASET_REPOSITORY = "livecodebench/code_generation_lite" +DATASET_REVISION = "819a13a5c0347c1bd2f5600a35bc9ac9695d461b" +RELEASE_VERSION = "release_v6" +RELEASE_FILES = ( + "test.jsonl", + "test2.jsonl", + "test3.jsonl", + "test4.jsonl", + "test5.jsonl", + "test6.jsonl", +) + +SYSTEM_PROMPT = ( + "You are an expert Python programmer. You will be given a question " + "(problem specification) and will generate a correct Python program that " + "matches the specification and passes all tests." +) + +STDIN_FORMAT = ( + "Read the inputs from stdin solve the problem and write the answer to " + "stdout (do not directly test on the sample inputs). Enclose your code " + "within delimiters as follows. Ensure that when the python program runs, " + "it reads the inputs, runs the algorithm and writes output to STDOUT." +) + +STARTER_FORMAT = ( + "You will use the following starter code to write the solution to the " + "problem and enclose your code within delimiters." +) + + +def _release_path(filename: str, *, local_files_only: bool = False) -> Path: + if filename not in RELEASE_FILES: + raise ValueError(f"Unknown LiveCodeBench release file: {filename}") + return Path( + hf_hub_download( + repo_id=DATASET_REPOSITORY, + filename=filename, + repo_type="dataset", + revision=DATASET_REVISION, + local_files_only=local_files_only, + ) + ) + + +def _release_paths() -> list[tuple[str, Path]]: + """Cache and return the immutable release-v6 source shards.""" + + return [(filename, _release_path(filename)) for filename in RELEASE_FILES] + + +def load_livecodebench_test_fields( + metadata: dict[str, Any], +) -> tuple[str, str, str]: + """Load one sample's test payloads from its immutable shard reference.""" + + filename = metadata["source_file"] + offset = metadata["source_offset"] + length = metadata["source_length"] + if not isinstance(filename, str): + raise TypeError("LiveCodeBench source_file must be a string") + if not isinstance(offset, int) or not isinstance(length, int): + raise TypeError("LiveCodeBench source offsets must be integers") + + path = _release_path(filename, local_files_only=True) + with path.open("rb") as shard: + shard.seek(offset) + record = json.loads(shard.read(length)) + if record["question_id"] != metadata["source_question_id"]: + raise ValueError("LiveCodeBench source reference resolved to the wrong record") + return ( + record["public_test_cases"], + record["private_test_cases"], + record["metadata"], + ) + + +def _parse_date(value: str | datetime) -> datetime: + """Parse the ISO date format used by LiveCodeBench records.""" + + return value if isinstance(value, datetime) else datetime.fromisoformat(value) + + +def _format_prompt(question: str, starter_code: str) -> str: + """Format the generic official LiveCodeBench code-generation prompt.""" + + prompt = f"### Question:\n{question}\n\n" + if starter_code: + prompt += f"### Format: {STARTER_FORMAT}\n" + prompt += f"```python\n{starter_code}\n```\n\n" + else: + prompt += f"### Format: {STDIN_FORMAT}\n" + prompt += "```python\n# YOUR CODE HERE\n```\n\n" + return prompt + "### Answer: (use the provided format with backticks)\n\n" + + +def record_to_sample( + start_date: str | None = None, + end_date: str | None = None, +) -> Callable[[dict[str, Any]], Sample | list[Sample]]: + """Create a converter for official LiveCodeBench release-v6 records. + + Date boundaries are inclusive, matching the official runner. Returning an + empty list allows Inspect to filter records before constructing the dataset. + """ + + parsed_start = _parse_date(start_date) if start_date else None + parsed_end = _parse_date(end_date) if end_date else None + if ( + parsed_start is not None + and parsed_end is not None + and parsed_start > parsed_end + ): + raise ValueError("start_date must not be after end_date") + + def _record_to_sample(record: dict[str, Any]) -> Sample | list[Sample]: + contest_date = _parse_date(record["contest_date"]) + if parsed_start is not None and contest_date < parsed_start: + return [] + if parsed_end is not None and contest_date > parsed_end: + return [] + + starter_code = record.get("starter_code") or "" + return Sample( + id=record["question_id"], + input=[ + ChatMessageSystem(content=SYSTEM_PROMPT), + ChatMessageUser( + content=_format_prompt(record["question_content"], starter_code) + ), + ], + target="", + metadata={ + "question_title": record["question_title"], + "platform": record["platform"], + "contest_id": record["contest_id"], + "contest_date": contest_date.isoformat(), + "difficulty": record["difficulty"], + "starter_code": starter_code, + "public_test_cases": record["public_test_cases"], + "private_test_cases": record["private_test_cases"], + "test_metadata": record["metadata"], + "release_version": RELEASE_VERSION, + "dataset_revision": DATASET_REVISION, + }, + ) + + return _record_to_sample + + +def get_livecodebench_v6_dataset( + start_date: str | None = None, + end_date: str | None = None, +) -> Dataset: + """Load the cumulative 1,055-problem LiveCodeBench release v6. + + The official Hugging Face repository uses an executable loading script. + Loading the six immutable JSONL shards through the generic JSON builder + avoids remote-code execution while preserving release-v6 composition. + """ + + converter = record_to_sample(start_date=start_date, end_date=end_date) + samples: list[Sample] = [] + for filename, path in _release_paths(): + with path.open("rb") as shard: + while line := shard.readline(): + offset = shard.tell() - len(line) + converted = converter(json.loads(line)) + if isinstance(converted, Sample): + metadata = converted.metadata + if metadata is None: + raise ValueError("LiveCodeBench sample metadata is required") + metadata.pop("public_test_cases") + metadata.pop("private_test_cases") + metadata.pop("test_metadata") + metadata.update( + { + "source_file": filename, + "source_offset": offset, + "source_length": len(line), + "source_question_id": converted.id, + } + ) + samples.append(converted) + else: + samples.extend(converted) + samples.sort(key=lambda sample: str(sample.id)) + return MemoryDataset( + samples=samples, + name="livecodebench_v6", + location=DATASET_REPOSITORY, + ) diff --git a/src/openbench/datasets/ocrbench.py b/src/openbench/datasets/ocrbench.py new file mode 100644 index 00000000..1c83bb25 --- /dev/null +++ b/src/openbench/datasets/ocrbench.py @@ -0,0 +1,61 @@ +"""OCRBench v1 dataset loader.""" + +from __future__ import annotations + +from datasets import load_dataset # type: ignore[import-untyped] +from inspect_ai.dataset import MemoryDataset, Sample +from inspect_ai.model import ChatMessageUser, ContentImage, ContentText + +from openbench.utils.image import image_bytes_to_data_uri, pil_image_to_bytes + +DATASET_REVISION = "92a54bd1384387c178d5a07140a2d85e0a3d12e1" + + +def _component(question_type: str) -> str: + if "Recognition" in question_type and "Expression" not in question_type: + return "text_recognition" + if "Scene Text" in question_type: + return "scene_text_vqa" + if "Doc" in question_type: + return "document_vqa" + if "Information Extraction" in question_type: + return "key_information_extraction" + if "Expression" in question_type: + return "handwritten_math_expression" + return question_type + + +def get_ocrbench_dataset() -> MemoryDataset: + """Load the official 1,000-example OCRBench v1 test set.""" + records = load_dataset( + "echo840/OCRBench", + split="test", + revision=DATASET_REVISION, + ) + samples: list[Sample] = [] + for index, record in enumerate(records): + image = record["image"] + if image.mode in ("RGBA", "LA", "P"): + image = image.convert("RGB") + image_uri = image_bytes_to_data_uri(pil_image_to_bytes(image, format="JPEG")) + question_type = str(record["question_type"]) + samples.append( + Sample( + id=f"ocrbench-{index}", + input=[ + ChatMessageUser( + content=[ + ContentImage(image=image_uri), + ContentText(text=str(record["question"])), + ] + ) + ], + target=list(record["answer"]), + metadata={ + "dataset_name": str(record["dataset"]), + "question_type": question_type, + "component": _component(question_type), + }, + ) + ) + return MemoryDataset(samples=samples, name="OCRBench") diff --git a/src/openbench/evals/bfcl/Dockerfile b/src/openbench/evals/bfcl/Dockerfile new file mode 100644 index 00000000..3cc16467 --- /dev/null +++ b/src/openbench/evals/bfcl/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.11-slim@sha256:94c50be2dc994b873b55bc123e95e6dbade08095b3dfd790f51c34de3f08cbb7 + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends git ca-certificates \ + && pip install --no-cache-dir --no-deps \ + "bfcl_eval @ git+https://github.com/ShishirPatil/gorilla.git@6ea57973c7a6097fd7c5915698c54c17c5b1b6c8#subdirectory=berkeley-function-call-leaderboard" \ + && pip install --no-cache-dir mpmath==1.3.0 \ + && apt-get purge --yes --auto-remove git \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd --create-home --uid 1000 sandbox \ + && mkdir --parents /workspace \ + && chown sandbox:sandbox /workspace + +COPY runner.py /opt/openbench/bfcl_runner.py + +USER sandbox +WORKDIR /workspace + +CMD ["tail", "-f", "/dev/null"] diff --git a/src/openbench/evals/bfcl/__init__.py b/src/openbench/evals/bfcl/__init__.py new file mode 100644 index 00000000..f976f40d --- /dev/null +++ b/src/openbench/evals/bfcl/__init__.py @@ -0,0 +1,295 @@ +"""Berkeley Function Calling Leaderboard v4 tasks.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from inspect_ai import Task, task +from inspect_ai.model import ( + ChatMessageSystem, + ChatMessageTool, + ChatMessageUser, + GenerateConfig, +) +from inspect_ai.solver import Generate, Solver, TaskState, solver +from inspect_ai.util import sandbox + +from openbench.datasets.bfcl import ( + AGENTIC_CATEGORIES, + MULTI_TURN_CATEGORIES, + SINGLE_TURN_CATEGORIES, + get_bfcl_v4_agentic_dataset, + get_bfcl_v4_multi_turn_dataset, + get_bfcl_v4_single_turn_dataset, +) +from openbench.function_calling import parse_function_calls +from openbench.function_calling.schema import build_tool_definitions +from openbench.function_calling.stateful import ( + format_python_call, + frozen_agentic_result, +) +from openbench.scorers.bfcl import ( + bfcl_v4_agentic_scorer, + bfcl_v4_multi_turn_scorer, + bfcl_v4_offline_scorer, + bfcl_v4_scorer, +) + +COMPOSE_PATH = (Path(__file__).parent / "compose.yaml").resolve() +MAX_STEPS = 20 + + +@solver +def bfcl_generate() -> Solver: + async def solve(state: TaskState, generate: Generate) -> TaskState: + functions: list[dict[str, Any]] = list(state.metadata["functions"]) + tools, mapping = build_tool_definitions(functions) + state.tools = [definition.as_tool() for definition in tools] + state.tool_choice = "auto" + state.metadata["tool_name_mapping"] = mapping + return await generate(state, tool_calls="none") + + return solve + + +def _append_messages(state: TaskState, messages: list[dict[str, str]]) -> None: + for message in messages: + content = str(message.get("content", "")) + if message.get("role") == "system": + state.messages.append(ChatMessageSystem(content=content)) + else: + state.messages.append(ChatMessageUser(content=content)) + + +async def _sandbox_request(state: TaskState, payload: dict[str, Any]) -> dict[str, Any]: + environment = sandbox() + payload_path = f".openbench_bfcl_{state.uuid}.json" + await environment.write_file(payload_path, json.dumps(payload)) + result = await environment.exec( + ["python", "/opt/openbench/bfcl_runner.py", payload_path], + timeout=120, + timeout_retry=False, + ) + if not result.success: + raise RuntimeError("BFCL sandbox runner failed") + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def _append_tool_results( + state: TaskState, + outputs: list[str], + native_calls: list[Any] | None, +) -> None: + if native_calls: + for index, call in enumerate(native_calls): + state.messages.append( + ChatMessageTool( + content=outputs[index] if index < len(outputs) else "No result", + tool_call_id=call.id, + function=call.function, + ) + ) + else: + state.messages.append( + ChatMessageUser(content=f"Tool execution results: {json.dumps(outputs)}") + ) + + +@solver +def bfcl_multi_turn_generate() -> Solver: + async def solve(state: TaskState, generate: Generate) -> TaskState: + functions = list(state.metadata["functions"]) + missed = dict(state.metadata.get("missed_functions", {})) + turns = list(state.metadata["turns"]) + model_turn_calls: list[list[list[str]]] = [] + all_calls: list[str] = [] + + for turn_index, turn_messages in enumerate(turns): + if turn_index > 0: + if str(turn_index) in missed: + functions.extend(missed[str(turn_index)]) + _append_messages( + state, + [ + { + "role": "user", + "content": "You now have additional tools available. Continue the previous request.", + } + ], + ) + else: + _append_messages(state, turn_messages) + + tools, mapping = build_tool_definitions(functions) + state.tools = [definition.as_tool() for definition in tools] + state.tool_choice = "auto" + state.metadata["tool_name_mapping"] = mapping + turn_steps: list[list[str]] = [] + + for _ in range(MAX_STEPS): + state = await generate(state, tool_calls="none") + message = state.output.message + calls = parse_function_calls( + message.tool_calls, state.output.completion, mapping + ) + if not calls: + break + encoded = [format_python_call(call) for call in calls] + previous_count = len(all_calls) + all_calls.extend(encoded) + turn_steps.append(encoded) + execution = await _sandbox_request( + state, + { + "operation": "execute", + "calls": all_calls, + "previous_count": previous_count, + "initial_config": state.metadata["initial_config"], + "involved_classes": state.metadata["involved_classes"], + "id": state.sample_id, + "category": state.metadata["category"], + }, + ) + _append_tool_results( + state, list(execution["outputs"]), message.tool_calls + ) + model_turn_calls.append(turn_steps) + + state.metadata["model_turn_calls"] = model_turn_calls + return state + + return solve + + +@solver +def bfcl_agentic_generate() -> Solver: + async def solve(state: TaskState, generate: Generate) -> TaskState: + tools, mapping = build_tool_definitions(list(state.metadata["functions"])) + state.tools = [definition.as_tool() for definition in tools] + state.tool_choice = "auto" + state.metadata["tool_name_mapping"] = mapping + trace: list[dict[str, Any]] = [] + for _ in range(MAX_STEPS): + state = await generate(state, tool_calls="none") + message = state.output.message + calls = parse_function_calls( + message.tool_calls, state.output.completion, mapping + ) + if not calls: + break + outputs = [ + frozen_agentic_result( + call, + source=state.metadata["frozen_source"], + show_snippet=bool(state.metadata["show_snippet"]), + ) + for call in calls + ] + trace.extend( + {"name": call.name, "arguments": call.arguments, "result": result} + for call, result in zip(calls, outputs) + ) + _append_tool_results(state, outputs, message.tool_calls) + state.metadata["agentic_trace"] = trace + return state + + return solve + + +@task +def bfcl_v4_single_turn( + categories: list[str] | None = None, +) -> Task: + """Evaluate BFCL v4 single-turn native or prompted function calls.""" + + return Task( + dataset=get_bfcl_v4_single_turn_dataset( + categories or list(SINGLE_TURN_CATEGORIES) + ), + solver=bfcl_generate(), + scorer=bfcl_v4_scorer(), + config=GenerateConfig( + temperature=0, + max_tokens=1024, + parallel_tool_calls=True, + ), + metadata={ + "benchmark": "BFCL v4", + "scope": "single-turn", + "official_overall_score": False, + }, + ) + + +@task +def bfcl_v4_multi_turn(categories: list[str] | None = None) -> Task: + """Evaluate BFCL v4 multi-turn with the pinned official state checker.""" + + return Task( + dataset=get_bfcl_v4_multi_turn_dataset( + categories or list(MULTI_TURN_CATEGORIES) + ), + solver=bfcl_multi_turn_generate(), + scorer=bfcl_v4_multi_turn_scorer(), + sandbox=("docker", str(COMPOSE_PATH)), + config=GenerateConfig(temperature=0, max_tokens=2048, parallel_tool_calls=True), + message_limit=200, + metadata={"benchmark": "BFCL v4", "scope": "multi-turn"}, + ) + + +@task +def bfcl_v4_agentic_offline(categories: list[str] | None = None) -> Task: + """Evaluate BFCL v4 agentic tasks against immutable offline evidence.""" + + return Task( + dataset=get_bfcl_v4_agentic_dataset(categories or list(AGENTIC_CATEGORIES)), + solver=bfcl_agentic_generate(), + scorer=bfcl_v4_agentic_scorer(), + config=GenerateConfig(temperature=0, max_tokens=2048, parallel_tool_calls=True), + message_limit=100, + metadata={ + "benchmark": "BFCL v4", + "scope": "agentic-offline", + "official_overall_score": False, + }, + ) + + +@solver +def bfcl_offline_generate() -> Solver: + async def solve(state: TaskState, generate: Generate) -> TaskState: + category = str(state.metadata["category"]) + if category in MULTI_TURN_CATEGORIES: + return await bfcl_multi_turn_generate()(state, generate) + if category in AGENTIC_CATEGORIES: + return await bfcl_agentic_generate()(state, generate) + return await bfcl_generate()(state, generate) + + return solve + + +@task +def bfcl_v4_offline() -> Task: + """Run all BFCL v4 sections with frozen offline agentic evidence.""" + + samples = [ + *list(get_bfcl_v4_single_turn_dataset()), + *list(get_bfcl_v4_multi_turn_dataset()), + *list(get_bfcl_v4_agentic_dataset()), + ] + return Task( + dataset=samples, + solver=bfcl_offline_generate(), + scorer=bfcl_v4_offline_scorer(), + sandbox=("docker", str(COMPOSE_PATH)), + config=GenerateConfig(temperature=0, max_tokens=2048, parallel_tool_calls=True), + message_limit=200, + metadata={ + "benchmark": "BFCL v4", + "scope": "offline-complete", + "official_overall_score": False, + }, + ) diff --git a/src/openbench/evals/bfcl/compose.yaml b/src/openbench/evals/bfcl/compose.yaml new file mode 100644 index 00000000..f6dec57f --- /dev/null +++ b/src/openbench/evals/bfcl/compose.yaml @@ -0,0 +1,17 @@ +services: + default: + build: + context: . + init: true + command: tail -f /dev/null + network_mode: none + read_only: true + tmpfs: + - /workspace:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=1073741824 + - /tmp:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=536870912 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + pids_limit: 64 + mem_limit: 2g diff --git a/src/openbench/evals/bfcl/runner.py b/src/openbench/evals/bfcl/runner.py new file mode 100644 index 00000000..4c4948f9 --- /dev/null +++ b/src/openbench/evals/bfcl/runner.py @@ -0,0 +1,61 @@ +"""Container-side adapter around BFCL's pinned official multi-turn checker.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_checker import ( # type: ignore[import-not-found] + multi_turn_checker, + multi_turn_irrelevance_checker, +) +from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_utils import ( # type: ignore[import-not-found] + execute_multi_turn_func_call, +) + + +def execute(payload: dict[str, Any]) -> dict[str, Any]: + calls = [str(value) for value in payload["calls"]] + previous_count = int(payload.get("previous_count", 0)) + outputs, _ = execute_multi_turn_func_call( + func_call_list=calls, + initial_config=payload["initial_config"], + involved_classes=payload["involved_classes"], + model_name="openbench_generation", + test_entry_id=payload["id"], + long_context="long_context" in payload["category"], + is_evaL_run=False, + ) + return {"outputs": outputs[previous_count:]} + + +def score(payload: dict[str, Any]) -> dict[str, Any]: + result = multi_turn_checker( + payload["model_turn_calls"], + payload["ground_truth"], + payload["test_entry"], + payload["category"], + "openbench", + ) + if result.get("valid"): + result = multi_turn_irrelevance_checker( + payload["model_turn_calls"], payload["ground_truth"] + ) + return result + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("usage: bfcl_runner PAYLOAD.json") + payload_path = Path(sys.argv[1]) + payload = json.loads(payload_path.read_text()) + payload_path.unlink() + operation = payload.pop("operation") + result = execute(payload) if operation == "execute" else score(payload) + print(json.dumps(result, default=str)) + + +if __name__ == "__main__": + main() diff --git a/src/openbench/evals/bigbench_hard.py b/src/openbench/evals/bigbench_hard.py index 0093665d..7b9d9319 100644 --- a/src/openbench/evals/bigbench_hard.py +++ b/src/openbench/evals/bigbench_hard.py @@ -39,10 +39,123 @@ } """ +import re + from inspect_ai import Task, task +from inspect_ai.dataset import Sample, hf_dataset +from inspect_ai.model import GenerateConfig +from inspect_ai.scorer import ( + CORRECT, + INCORRECT, + Score, + Target, + accuracy, + scorer, + stderr, +) +from inspect_ai.solver import TaskState, generate from openbench.utils.mcq import MCQEval, MCQSample from openbench.utils.text import create_dynamic_multiple_choice_prompt +BBH_DATASET_REVISION = "982bb89fd79532a8ac676a61fc42eb1aeec63f99" + + +def _free_response_record(record: dict) -> Sample: + return Sample( + input=( + f"{record['input']}\n\nSolve the problem step by step. End your response " + "with `So the answer is `." + ), + target=str(record["target"]), + ) + + +def _normalize_bbh_answer(value: str) -> str: + value = value.strip().rstrip(".").strip() + return re.sub(r"\s+", " ", value).casefold() + + +@scorer(metrics=[accuracy(), stderr()]) +def bbh_exact_match_scorer(): + """Extract the final BBH answer without truncating free responses.""" + + async def score(state: TaskState, target: Target) -> Score: + completion = state.output.completion.strip() + matches = re.findall( + r"(?is)so\s+the\s+answer\s+is\s*:?\s*(.+?)(?:\n|$)", completion + ) + answer = matches[-1].strip() if matches else completion.splitlines()[-1].strip() + correct = _normalize_bbh_answer(answer) == _normalize_bbh_answer(target.text) + return Score( + value=CORRECT if correct else INCORRECT, + answer=answer, + explanation="Normalized exact match against the BBH target", + ) + + return score + + +def _bbh_free_response_task(subset: str) -> Task: + return Task( + dataset=hf_dataset( + path="lukaemon/bbh", + name=subset, + split="test", + revision=BBH_DATASET_REVISION, + sample_fields=_free_response_record, + auto_id=True, + ), + solver=generate(), + scorer=bbh_exact_match_scorer(), + config=GenerateConfig(temperature=0.0, max_tokens=2048), + name=f"bbh_{subset}", + ) + + +@task +def bbh_boolean_expressions() -> Task: + return _bbh_free_response_task("boolean_expressions") + + +@task +def bbh_dyck_languages() -> Task: + return _bbh_free_response_task("dyck_languages") + + +@task +def bbh_formal_fallacies() -> Task: + return _bbh_free_response_task("formal_fallacies") + + +@task +def bbh_hyperbaton() -> Task: + return _bbh_free_response_task("hyperbaton") + + +@task +def bbh_multistep_arithmetic_two() -> Task: + return _bbh_free_response_task("multistep_arithmetic_two") + + +@task +def bbh_object_counting() -> Task: + return _bbh_free_response_task("object_counting") + + +@task +def bbh_penguins_in_a_table() -> Task: + return _bbh_free_response_task("penguins_in_a_table") + + +@task +def bbh_web_of_lies() -> Task: + return _bbh_free_response_task("web_of_lies") + + +@task +def bbh_word_sorting() -> Task: + return _bbh_free_response_task("word_sorting") + def record_to_mcq_sample(record: dict) -> MCQSample: """ diff --git a/src/openbench/evals/ceval.py b/src/openbench/evals/ceval.py new file mode 100644 index 00000000..73571072 --- /dev/null +++ b/src/openbench/evals/ceval.py @@ -0,0 +1,34 @@ +"""C-Eval Chinese academic multiple-choice benchmark.""" + +from inspect_ai import Task, task +from inspect_ai.model import GenerateConfig +from inspect_ai.solver import generate + +from openbench.datasets.ceval import HARD_SUBJECTS, get_ceval_dataset +from openbench.scorers.mcq import create_mcq_scorer + + +def _ceval_task(*, hard: bool, split: str, shots: int) -> Task: + return Task( + dataset=get_ceval_dataset( + subjects=HARD_SUBJECTS if hard else None, + split=split, + shots=shots, + ), + solver=generate(), + scorer=create_mcq_scorer(group_keys=["category", "subject"])(), + config=GenerateConfig(temperature=0.0, max_tokens=32), + name="ceval_hard" if hard else "ceval", + ) + + +@task +def ceval(split: str = "val", shots: int = 5) -> Task: + """Evaluate all 52 C-Eval subjects with official 0/5-shot prompting.""" + return _ceval_task(hard=False, split=split, shots=shots) + + +@task +def ceval_hard(split: str = "val", shots: int = 5) -> Task: + """Evaluate the official eight-subject C-Eval Hard subset.""" + return _ceval_task(hard=True, split=split, shots=shots) diff --git a/src/openbench/evals/evalplus/Dockerfile b/src/openbench/evals/evalplus/Dockerfile new file mode 100644 index 00000000..6a6ea763 --- /dev/null +++ b/src/openbench/evals/evalplus/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim@sha256:94c50be2dc994b873b55bc123e95e6dbade08095b3dfd790f51c34de3f08cbb7 + +RUN pip install --no-cache-dir numpy==2.2.6 + +RUN useradd --create-home --uid 1000 sandbox \ + && mkdir --parents /workspace \ + && chown sandbox:sandbox /workspace + +USER sandbox +WORKDIR /workspace + +CMD ["tail", "-f", "/dev/null"] diff --git a/src/openbench/evals/evalplus/__init__.py b/src/openbench/evals/evalplus/__init__.py new file mode 100644 index 00000000..8ec464fa --- /dev/null +++ b/src/openbench/evals/evalplus/__init__.py @@ -0,0 +1,5 @@ +"""EvalPlus benchmark tasks.""" + +from .evalplus import humaneval_plus, mbpp_plus + +__all__ = ["humaneval_plus", "mbpp_plus"] diff --git a/src/openbench/evals/evalplus/compose.yaml b/src/openbench/evals/evalplus/compose.yaml new file mode 100644 index 00000000..951eb4c2 --- /dev/null +++ b/src/openbench/evals/evalplus/compose.yaml @@ -0,0 +1,17 @@ +services: + default: + build: + context: . + init: true + command: tail -f /dev/null + network_mode: none + read_only: true + tmpfs: + - /workspace:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=1073741824 + - /tmp:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=536870912 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + pids_limit: 64 + mem_limit: 4g diff --git a/src/openbench/evals/evalplus/evalplus.py b/src/openbench/evals/evalplus/evalplus.py new file mode 100644 index 00000000..f9163966 --- /dev/null +++ b/src/openbench/evals/evalplus/evalplus.py @@ -0,0 +1,43 @@ +"""HumanEval+ and MBPP+ with isolated differential testing.""" + +from pathlib import Path + +from inspect_ai import Epochs, Task, task +from inspect_ai.model import GenerateConfig +from inspect_ai.solver import generate + +from openbench.datasets.evalplus import get_evalplus_dataset +from openbench.scorers.evalplus import evalplus_scorer + +COMPOSE_PATH = (Path(__file__).parent / "compose.yaml").resolve() + + +def _evalplus_task(dataset: str, epochs: int, total_timeout: int) -> Task: + if epochs <= 0: + raise ValueError("epochs must be positive") + reducers = ["mean", "pass_at_1"] + if epochs >= 10: + reducers.append("pass_at_10") + if epochs >= 100: + reducers.append("pass_at_100") + return Task( + name=f"{dataset}_plus", + dataset=get_evalplus_dataset(dataset), + solver=generate(), + scorer=evalplus_scorer(total_timeout=total_timeout), + sandbox=("docker", str(COMPOSE_PATH)), + epochs=Epochs(epochs, reducer=reducers), + config=GenerateConfig(temperature=0.2, top_p=0.95, max_tokens=2048), + ) + + +@task +def humaneval_plus(epochs: int = 1, total_timeout: int = 900) -> Task: + """Evaluate HumanEval+ v0.1.10 base and augmented tests.""" + return _evalplus_task("humaneval", epochs, total_timeout) + + +@task +def mbpp_plus(epochs: int = 1, total_timeout: int = 900) -> Task: + """Evaluate MBPP+ v0.2.0 base and augmented tests.""" + return _evalplus_task("mbpp", epochs, total_timeout) diff --git a/src/openbench/evals/global_piqa.py b/src/openbench/evals/global_piqa.py new file mode 100644 index 00000000..c17d824b --- /dev/null +++ b/src/openbench/evals/global_piqa.py @@ -0,0 +1,25 @@ +"""Global PIQA generation-mode evaluation.""" + +from collections.abc import Iterable + +from inspect_ai import Task, task +from inspect_ai.model import GenerateConfig +from inspect_ai.solver import generate + +from openbench.datasets.global_piqa import get_global_piqa_dataset +from openbench.scorers.global_piqa import global_piqa_scorer + + +@task +def global_piqa( + components: Iterable[str] = ("nonparallel", "parallel"), + languages: Iterable[str] | None = None, +) -> Task: + """Evaluate both Global PIQA components using the official generation mode.""" + return Task( + dataset=get_global_piqa_dataset(components=components, languages=languages), + solver=generate(), + scorer=global_piqa_scorer(), + config=GenerateConfig(temperature=0.8, top_p=0.95, max_tokens=2048), + name="global_piqa", + ) diff --git a/src/openbench/evals/gsm8k_hard.py b/src/openbench/evals/gsm8k_hard.py new file mode 100644 index 00000000..2ffa86b6 --- /dev/null +++ b/src/openbench/evals/gsm8k_hard.py @@ -0,0 +1,40 @@ +"""GSM-Hard, the numerically perturbed GSM8K evaluation set.""" + +from inspect_ai import Task, task +from inspect_ai.dataset import Sample, hf_dataset +from inspect_ai.model import GenerateConfig +from inspect_ai.solver import generate + +from openbench.scorers.grade_school_math import numeric_tolerance_scorer + +DATASET_REVISION = "960448f73503112d4226baeb8eb41d3fb5ae2506" +PROMPT_TEMPLATE = """Solve this math problem. Show your reasoning, then put only the +numeric result after `Answer:` on the final line. + +{question}""" + + +def record_to_sample(record: dict) -> Sample: + """Convert a canonical GSM-Hard record to an Inspect sample.""" + return Sample( + input=PROMPT_TEMPLATE.format(question=record["input"]), + target=str(record["target"]), + metadata={"answer_prefix": "Answer"}, + ) + + +@task +def gsm8k_hard() -> Task: + """Evaluate the 1,319-example GSM-Hard dataset from the PAL authors.""" + return Task( + dataset=hf_dataset( + path="reasoning-machines/gsm-hard", + revision=DATASET_REVISION, + split="train", + sample_fields=record_to_sample, + ), + solver=generate(), + scorer=numeric_tolerance_scorer(tolerance=1e-3), + config=GenerateConfig(temperature=0.0, max_tokens=2048), + name="gsm8k_hard", + ) diff --git a/src/openbench/evals/livecodebench/Dockerfile b/src/openbench/evals/livecodebench/Dockerfile new file mode 100644 index 00000000..94839669 --- /dev/null +++ b/src/openbench/evals/livecodebench/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12-slim@sha256:646fb0bca3dd3ea1bcc6feb72c17ed16eed6e10cffc732fcc1478bd3e7f02d7b + +RUN useradd --create-home --uid 1000 sandbox \ + && mkdir --parents /workspace \ + && chown sandbox:sandbox /workspace + +USER sandbox +WORKDIR /workspace + +CMD ["tail", "-f", "/dev/null"] diff --git a/src/openbench/evals/livecodebench/__init__.py b/src/openbench/evals/livecodebench/__init__.py new file mode 100644 index 00000000..b078dbb4 --- /dev/null +++ b/src/openbench/evals/livecodebench/__init__.py @@ -0,0 +1,5 @@ +"""LiveCodeBench evaluation package.""" + +from .livecodebench import livecodebench_v6 + +__all__ = ["livecodebench_v6"] diff --git a/src/openbench/evals/livecodebench/compose.yaml b/src/openbench/evals/livecodebench/compose.yaml new file mode 100644 index 00000000..f6dec57f --- /dev/null +++ b/src/openbench/evals/livecodebench/compose.yaml @@ -0,0 +1,17 @@ +services: + default: + build: + context: . + init: true + command: tail -f /dev/null + network_mode: none + read_only: true + tmpfs: + - /workspace:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=1073741824 + - /tmp:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=536870912 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + pids_limit: 64 + mem_limit: 2g diff --git a/src/openbench/evals/livecodebench/livecodebench.py b/src/openbench/evals/livecodebench/livecodebench.py new file mode 100644 index 00000000..da2b33c5 --- /dev/null +++ b/src/openbench/evals/livecodebench/livecodebench.py @@ -0,0 +1,48 @@ +"""LiveCodeBench v6 code-generation evaluation.""" + +from pathlib import Path + +from inspect_ai import Epochs, Task, task +from inspect_ai.model import GenerateConfig +from inspect_ai.solver import generate + +from openbench.datasets.livecodebench import get_livecodebench_v6_dataset +from openbench.scorers.livecodebench import livecodebench_scorer + +TASK_DIR = Path(__file__).parent +COMPOSE_PATH = (TASK_DIR / "compose.yaml").resolve() + + +@task +def livecodebench_v6( + start_date: str | None = None, + end_date: str | None = None, + test_timeout: int = 6, + total_timeout: int = 600, +) -> Task: + """Run the pinned cumulative LiveCodeBench release-v6 benchmark. + + The official protocol samples 10 completions at temperature 0.2 and reports + pass@1 and pass@5. Optional date boundaries are inclusive. + """ + + return Task( + name="livecodebench_v6", + dataset=get_livecodebench_v6_dataset( + start_date=start_date, + end_date=end_date, + ), + solver=generate(), + scorer=livecodebench_scorer( + test_timeout=test_timeout, + total_timeout=total_timeout, + ), + sandbox=("docker", str(COMPOSE_PATH)), + epochs=Epochs(10, reducer=["mean", "pass_at_1", "pass_at_5"]), + config=GenerateConfig( + temperature=0.2, + top_p=0.95, + max_tokens=2000, + stop_seqs=["###"], + ), + ) diff --git a/src/openbench/evals/matharena/aime_2026/__init__.py b/src/openbench/evals/matharena/aime_2026/__init__.py new file mode 100644 index 00000000..f438435b --- /dev/null +++ b/src/openbench/evals/matharena/aime_2026/__init__.py @@ -0,0 +1 @@ +"""AIME 2026 evaluation.""" diff --git a/src/openbench/evals/matharena/aime_2026/aime_2026.py b/src/openbench/evals/matharena/aime_2026/aime_2026.py new file mode 100644 index 00000000..34e8da47 --- /dev/null +++ b/src/openbench/evals/matharena/aime_2026/aime_2026.py @@ -0,0 +1,20 @@ +from inspect_ai import Task, task + +from openbench.evals.matharena.matharena import matharena_task + + +@task +def aime_2026() -> Task: + """Evaluate the combined AIME I and II 2026 dataset.""" + return matharena_task( + dataset_path="MathArena/aime_2026", + revision="d2de22f3c656b4f56cf8981212186377d1e23bc3", + instruction=( + "Please reason step by step, and put your final answer within " + "\\boxed{{}}.\nThe answer is an integer between 0 and 999 inclusive." + ), + default_temperature=0.6, + default_max_tokens=8000, + default_epochs=4, + name="aime_2026", + ) diff --git a/src/openbench/evals/matharena/hmmt_feb_2026/__init__.py b/src/openbench/evals/matharena/hmmt_feb_2026/__init__.py new file mode 100644 index 00000000..d9159fac --- /dev/null +++ b/src/openbench/evals/matharena/hmmt_feb_2026/__init__.py @@ -0,0 +1 @@ +"""HMMT February 2026 evaluation.""" diff --git a/src/openbench/evals/matharena/hmmt_feb_2026/hmmt_feb_2026.py b/src/openbench/evals/matharena/hmmt_feb_2026/hmmt_feb_2026.py new file mode 100644 index 00000000..21e28646 --- /dev/null +++ b/src/openbench/evals/matharena/hmmt_feb_2026/hmmt_feb_2026.py @@ -0,0 +1,19 @@ +from inspect_ai import Task, task + +from openbench.evals.matharena.matharena import matharena_task + + +@task +def hmmt_feb_2026() -> Task: + """Evaluate HMMT February 2026.""" + return matharena_task( + dataset_path="MathArena/hmmt_feb_2026", + revision="02fba4f74d8e68e73e66a02d540fd979c05c274c", + instruction=( + "Please reason step by step, and put your final answer within \\boxed{{}}." + ), + default_temperature=0.6, + default_max_tokens=16000, + default_epochs=4, + name="hmmt_feb_2026", + ) diff --git a/src/openbench/evals/matharena/hmmt_nov_2025/__init__.py b/src/openbench/evals/matharena/hmmt_nov_2025/__init__.py new file mode 100644 index 00000000..f60cbeff --- /dev/null +++ b/src/openbench/evals/matharena/hmmt_nov_2025/__init__.py @@ -0,0 +1 @@ +"""HMMT November 2025 evaluation.""" diff --git a/src/openbench/evals/matharena/hmmt_nov_2025/hmmt_nov_2025.py b/src/openbench/evals/matharena/hmmt_nov_2025/hmmt_nov_2025.py new file mode 100644 index 00000000..2c5687bc --- /dev/null +++ b/src/openbench/evals/matharena/hmmt_nov_2025/hmmt_nov_2025.py @@ -0,0 +1,19 @@ +from inspect_ai import Task, task + +from openbench.evals.matharena.matharena import matharena_task + + +@task +def hmmt_nov_2025() -> Task: + """Evaluate HMMT November 2025.""" + return matharena_task( + dataset_path="MathArena/hmmt_nov_2025", + revision="118dbfb45c4c9467c672268ed55166642897aa46", + instruction=( + "Please reason step by step, and put your final answer within \\boxed{{}}." + ), + default_temperature=0.6, + default_max_tokens=16000, + default_epochs=4, + name="hmmt_nov_2025", + ) diff --git a/src/openbench/evals/matharena/matharena.py b/src/openbench/evals/matharena/matharena.py index f9ebeaf3..c7448202 100644 --- a/src/openbench/evals/matharena/matharena.py +++ b/src/openbench/evals/matharena/matharena.py @@ -1,7 +1,7 @@ from inspect_ai.dataset import hf_dataset, Sample from inspect_ai import Task from inspect_ai.model import GenerateConfig -from openbench.scorers import aime_scorer +from openbench.scorers import matharena_answer_scorer from inspect_ai.solver import generate, prompt_template @@ -23,6 +23,7 @@ def matharena_task( instruction: str, name: str, default_max_tokens: int, + revision: str | None = None, default_temperature: float = 0.6, default_epochs: int = 4, ) -> Task: @@ -30,13 +31,14 @@ def matharena_task( path=dataset_path, split="train", sample_fields=matharena_record_to_sample, + revision=revision, ) TEMPLATE = instruction + "\n\n" + "{prompt}" return Task( dataset=dataset, solver=[prompt_template(TEMPLATE), generate()], - scorer=aime_scorer(), # Use specialized AIME scorer with robust extraction + scorer=matharena_answer_scorer(), name=name, config=GenerateConfig( temperature=default_temperature, diff --git a/src/openbench/evals/ocrbench.py b/src/openbench/evals/ocrbench.py new file mode 100644 index 00000000..1165ce22 --- /dev/null +++ b/src/openbench/evals/ocrbench.py @@ -0,0 +1,20 @@ +"""OCRBench v1 evaluation.""" + +from inspect_ai import Task, task +from inspect_ai.model import GenerateConfig +from inspect_ai.solver import generate + +from openbench.datasets.ocrbench import get_ocrbench_dataset +from openbench.scorers.ocrbench import ocrbench_scorer + + +@task +def ocrbench() -> Task: + """Evaluate OCRBench v1, kept distinct from OCRBench v2.""" + return Task( + dataset=get_ocrbench_dataset(), + solver=generate(), + scorer=ocrbench_scorer(), + config=GenerateConfig(temperature=0.0, max_tokens=100), + name="ocrbench", + ) diff --git a/src/openbench/function_calling/__init__.py b/src/openbench/function_calling/__init__.py new file mode 100644 index 00000000..05063105 --- /dev/null +++ b/src/openbench/function_calling/__init__.py @@ -0,0 +1,17 @@ +"""Provider-neutral primitives for function-calling evaluations.""" + +from openbench.function_calling.matching import ( + FunctionCall, + FunctionMatch, + match_function_calls, +) +from openbench.function_calling.parsing import parse_function_calls +from openbench.function_calling.schema import build_tool_definitions + +__all__ = [ + "FunctionCall", + "FunctionMatch", + "build_tool_definitions", + "match_function_calls", + "parse_function_calls", +] diff --git a/src/openbench/function_calling/matching.py b/src/openbench/function_calling/matching.py new file mode 100644 index 00000000..7bf0c333 --- /dev/null +++ b/src/openbench/function_calling/matching.py @@ -0,0 +1,208 @@ +"""Deterministic, provider-neutral function-call matching.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class FunctionCall: + name: str + arguments: dict[str, Any] + parse_error: str | None = None + + +@dataclass(frozen=True) +class FunctionMatch: + matched: bool + error: str | None = None + + +_PYTHON_TYPES: dict[str, type[Any] | tuple[type[Any], ...]] = { + "string": str, + "String": str, + "char": str, + "integer": int, + "byte": int, + "short": int, + "long": int, + "Bigint": int, + "number": float, + "float": float, + "double": float, + "boolean": bool, + "bool": bool, + "Boolean": bool, + "array": list, + "Array": list, + "ArrayList": list, + "list": list, + "Queue": list, + "Stack": list, + "tuple": (list, tuple), + "dict": dict, + "HashMap": dict, + "Hashtable": dict, + "object": dict, + "Any": object, + "any": object, + "": object, +} + + +def _standardize_string(value: str) -> str: + return re.sub(r"[ ,./\-_*^]", "", value).lower().replace("'", '"') + + +def _value_matches(actual: Any, allowed: list[Any]) -> bool: + for expected in allowed: + if expected == "": + continue + if isinstance(actual, str) and isinstance(expected, str): + if _standardize_string(actual) == _standardize_string(expected): + return True + elif isinstance(actual, (list, tuple)) and isinstance(expected, list): + if len(actual) == len(expected) and all( + _value_matches(item, [target]) for item, target in zip(actual, expected) + ): + return True + elif isinstance(actual, dict) and isinstance(expected, dict): + if any(key not in expected for key in actual): + continue + if not all( + _value_matches( + actual[key], + expected[key] + if isinstance(expected[key], list) + else [expected[key]], + ) + for key in actual + ): + continue + if all( + key in actual or (isinstance(options, list) and "" in options) + for key, options in expected.items() + ): + return True + elif type(actual) is type(expected) and actual == expected: + return True + elif ( + isinstance(actual, (int, float)) + and not isinstance(actual, bool) + and isinstance(expected, (int, float)) + and not isinstance(expected, bool) + and float(actual) == float(expected) + ): + return True + return False + + +def _type_matches(value: Any, schema: dict[str, Any], allowed: list[Any]) -> bool: + expected = _PYTHON_TYPES.get(str(schema.get("type", "")), object) + if expected is object: + return True + if expected is int and isinstance(value, bool): + return False + if expected is float: + return (isinstance(value, (int, float)) and not isinstance(value, bool)) or any( + item != "" and type(value) is type(item) for item in allowed + ) + if not isinstance(value, expected): + # BFCL uses symbolic variable names in the Java/JavaScript sets and a + # few Python records. The official checker accepts the public answer's + # concrete type when it intentionally differs from the schema type. + return any(item != "" and type(value) is type(item) for item in allowed) + if isinstance(value, (list, tuple)) and isinstance(schema.get("items"), dict): + allowed_items = [ + item + for candidate in allowed + if isinstance(candidate, (list, tuple)) + for item in candidate + ] + return all( + _type_matches(item, schema["items"], allowed_items) for item in value + ) + return True + + +def _single_call_matches( + actual: FunctionCall, + expected: dict[str, dict[str, list[Any]]], + function: dict[str, Any], +) -> FunctionMatch: + if actual.parse_error: + return FunctionMatch(False, f"parse error: {actual.parse_error}") + expected_name, expected_arguments = next(iter(expected.items())) + if actual.name != expected_name: + return FunctionMatch(False, f"expected {expected_name}, got {actual.name}") + + properties = function.get("parameters", {}).get("properties", {}) + required = set(function.get("parameters", {}).get("required", [])) + unexpected = set(actual.arguments) - set(properties) + unexpected.update(set(actual.arguments) - set(expected_arguments)) + if unexpected: + return FunctionMatch(False, f"unexpected arguments: {sorted(unexpected)}") + + for name, allowed in expected_arguments.items(): + if name not in actual.arguments: + if name in required: + return FunctionMatch(False, f"missing required argument: {name}") + if "" not in allowed: + return FunctionMatch(False, f"missing expected argument: {name}") + continue + value = actual.arguments[name] + schema = properties.get(name, {}) + if not _type_matches(value, schema, allowed): + return FunctionMatch(False, f"invalid type for argument: {name}") + if not _value_matches(value, allowed): + return FunctionMatch(False, f"invalid value for argument: {name}") + + missing_required = required - set(actual.arguments) + for name in missing_required: + if "" not in expected_arguments.get(name, []): + return FunctionMatch(False, f"missing required argument: {name}") + return FunctionMatch(True) + + +def match_function_calls( + actual: list[FunctionCall], + expected: list[dict[str, dict[str, list[Any]]]], + functions: list[dict[str, Any]], + *, + order_sensitive: bool = False, +) -> FunctionMatch: + """Match calls one-to-one, allowing parallel calls in any order.""" + + if len(actual) != len(expected): + return FunctionMatch( + False, f"expected {len(expected)} call(s), got {len(actual)}" + ) + functions_by_name = {str(function["name"]): function for function in functions} + + if order_sensitive: + pairs = zip(actual, expected) + for actual_call, expected_call in pairs: + name = next(iter(expected_call)) + result = _single_call_matches( + actual_call, expected_call, functions_by_name[name] + ) + if not result.matched: + return result + return FunctionMatch(True) + + unmatched = list(range(len(actual))) + for expected_call in expected: + name = next(iter(expected_call)) + for index in unmatched: + actual_call = actual[index] + result = _single_call_matches( + actual_call, expected_call, functions_by_name[name] + ) + if result.matched: + unmatched.remove(index) + break + else: + return FunctionMatch(False, f"no actual call matched {name}") + return FunctionMatch(True) diff --git a/src/openbench/function_calling/parsing.py b/src/openbench/function_calling/parsing.py new file mode 100644 index 00000000..fb55a982 --- /dev/null +++ b/src/openbench/function_calling/parsing.py @@ -0,0 +1,101 @@ +"""Safe parsing for native, JSON, and Python-style function calls.""" + +from __future__ import annotations + +import ast +import json +import re +from typing import Any + +from inspect_ai.tool import ToolCall + +from openbench.function_calling.matching import FunctionCall + + +def _from_json(value: Any) -> list[FunctionCall]: + if isinstance(value, dict): + if "function" in value and isinstance(value["function"], dict): + value = value["function"] + name = value.get("name") or value.get("function") + arguments = value.get("arguments", value.get("parameters", {})) + if isinstance(arguments, str): + arguments = json.loads(arguments) + if isinstance(name, str) and isinstance(arguments, dict): + return [FunctionCall(name=name, arguments=arguments)] + if len(value) == 1: + name, arguments = next(iter(value.items())) + if isinstance(name, str) and isinstance(arguments, dict): + return [FunctionCall(name=name, arguments=arguments)] + return [] + if isinstance(value, list): + calls: list[FunctionCall] = [] + for item in value: + calls.extend(_from_json(item)) + return calls + return [] + + +def _call_name(node: ast.expr) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + parent = _call_name(node.value) + return f"{parent}.{node.attr}" if parent else None + return None + + +def _from_python(text: str) -> list[FunctionCall]: + parsed = ast.parse(text.strip(), mode="eval").body + nodes = parsed.elts if isinstance(parsed, (ast.List, ast.Tuple)) else [parsed] + calls: list[FunctionCall] = [] + for node in nodes: + if not isinstance(node, ast.Call) or node.args: + return [] + name = _call_name(node.func) + if name is None or any(keyword.arg is None for keyword in node.keywords): + return [] + arguments = { + str(keyword.arg): ast.literal_eval(keyword.value) + for keyword in node.keywords + } + calls.append(FunctionCall(name=name, arguments=arguments)) + return calls + + +def parse_function_calls( + native_calls: list[ToolCall] | None, + completion: str, + name_mapping: dict[str, str] | None = None, +) -> list[FunctionCall]: + """Normalize provider-native calls, then fall back to safe text parsing.""" + + mapping = name_mapping or {} + if native_calls: + return [ + FunctionCall( + name=mapping.get(call.function, call.function), + arguments=dict(call.arguments), + parse_error=call.parse_error, + ) + for call in native_calls + ] + + text = completion.strip() + fenced = re.findall(r"```(?:json|python)?\s*(.*?)```", text, flags=re.DOTALL) + if fenced: + text = fenced[-1].strip() + for parser in (lambda value: _from_json(json.loads(value)), _from_python): + try: + calls = parser(text) + except (SyntaxError, ValueError, TypeError, json.JSONDecodeError): + continue + if calls: + return [ + FunctionCall( + name=mapping.get(call.name, call.name), + arguments=call.arguments, + parse_error=call.parse_error, + ) + for call in calls + ] + return [] diff --git a/src/openbench/function_calling/schema.py b/src/openbench/function_calling/schema.py new file mode 100644 index 00000000..d7abcad5 --- /dev/null +++ b/src/openbench/function_calling/schema.py @@ -0,0 +1,115 @@ +"""Conversion from benchmark JSON schemas to Inspect tool definitions.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from typing import Any + +from inspect_ai.tool import ToolDef, ToolParams +from inspect_ai.util import JSONSchema + +_TYPE_MAPPING = { + "": None, + "Any": None, + "any": None, + "array": "array", + "Array": "array", + "ArrayList": "array", + "list": "array", + "Queue": "array", + "Stack": "array", + "Bigint": "integer", + "bool": "boolean", + "boolean": "boolean", + "Boolean": "boolean", + "char": "string", + "dict": "object", + "HashMap": "object", + "Hashtable": "object", + "object": "object", + "number": "number", + "byte": "integer", + "short": "integer", + "float": "number", + "double": "number", + "integer": "integer", + "long": "integer", + "String": "string", + "string": "string", + "tuple": "array", +} + + +def safe_tool_name(name: str) -> str: + """Return a provider-compatible function name.""" + + normalized = re.sub(r"[^a-zA-Z0-9_-]", "_", name) + return normalized[:64] or "tool" + + +def _json_schema(schema: Mapping[str, Any]) -> JSONSchema: + raw_type = str(schema.get("type", "")) + mapped_type = _TYPE_MAPPING.get(raw_type, "string") + properties = schema.get("properties") + items = schema.get("items") + return JSONSchema( + type=mapped_type, # type: ignore[arg-type] + description=(str(schema["description"]) if schema.get("description") else None), + default=schema.get("default"), + enum=list(schema["enum"]) if isinstance(schema.get("enum"), list) else None, + items=_json_schema(items) if isinstance(items, Mapping) else None, + properties=( + {str(key): _json_schema(value) for key, value in properties.items()} + if isinstance(properties, Mapping) + else None + ), + required=( + [str(value) for value in schema.get("required", [])] + if mapped_type == "object" + else None + ), + additionalProperties=( + bool(schema.get("additionalProperties", False)) + if mapped_type == "object" + else None + ), + ) + + +def build_tool_definitions( + functions: list[dict[str, Any]], +) -> tuple[list[ToolDef], dict[str, str]]: + """Build non-executing Inspect tools and a safe-to-original name mapping.""" + + definitions: list[ToolDef] = [] + name_mapping: dict[str, str] = {} + + for index, function in enumerate(functions): + original_name = str(function["name"]) + safe_name = safe_tool_name(original_name) + if safe_name in name_mapping and name_mapping[safe_name] != original_name: + suffix = f"_{index}" + safe_name = f"{safe_name[: 64 - len(suffix)]}{suffix}" + name_mapping[safe_name] = original_name + + async def unavailable_tool(**_: Any) -> str: + return "Tool execution is disabled for this evaluation." + + parameters = function.get("parameters", {}) + converted = _json_schema(parameters) + definitions.append( + ToolDef( + unavailable_tool, + name=safe_name, + description=str(function.get("description", "")), + parameters=ToolParams( + properties=converted.properties or {}, + required=converted.required or [], + additionalProperties=bool(converted.additionalProperties), + ), + parallel=True, + ) + ) + + return definitions, name_mapping diff --git a/src/openbench/function_calling/stateful.py b/src/openbench/function_calling/stateful.py new file mode 100644 index 00000000..5a638cc7 --- /dev/null +++ b/src/openbench/function_calling/stateful.py @@ -0,0 +1,56 @@ +"""Reusable stateful execution helpers for function-calling benchmarks.""" + +from __future__ import annotations + +import json +from typing import Any + +from openbench.function_calling.matching import FunctionCall + + +def format_python_call(call: FunctionCall) -> str: + """Serialize a normalized call without evaluating model-authored source.""" + + arguments = ", ".join(f"{name}={value!r}" for name, value in call.arguments.items()) + return f"{call.name}({arguments})" + + +def frozen_agentic_result( + call: FunctionCall, + *, + source: str | list[dict[str, Any]], + show_snippet: bool, +) -> str: + """Execute BFCL agentic tools against an immutable offline evidence set.""" + + name = call.name + if name == "search_engine_query" and isinstance(source, list): + results = [] + for index, item in enumerate(source): + result = { + "title": str(item.get("subquestion", f"Result {index + 1}")), + "url": str(item.get("source", "")), + } + if show_snippet: + result["snippet"] = str(item.get("answer", "")) + results.append(result) + limit = int(call.arguments.get("max_results", 10)) + return json.dumps(results[:limit]) + + if name == "fetch_url_content" and isinstance(source, list): + url = str(call.arguments.get("url", "")) + matches = [item for item in source if str(item.get("source", "")) == url] + content = "\n".join( + f"{item.get('subquestion', '')}: {item.get('answer', '')}" + for item in matches + ) + return json.dumps({"content": content or "URL not found in frozen corpus"}) + + memory = str(source) + if name == "memory_retrieve": + return json.dumps({"memory_content": memory}) + if "retrieve" in name or "search" in name: + return json.dumps({"results": [{"id": 0, "score": 1.0, "text": memory}]}) + if "list_keys" in name: + return json.dumps({"keys": ["profile"]}) + return json.dumps({"status": "ok"}) diff --git a/src/openbench/metrics/bfcl.py b/src/openbench/metrics/bfcl.py new file mode 100644 index 00000000..adf33f8e --- /dev/null +++ b/src/openbench/metrics/bfcl.py @@ -0,0 +1,195 @@ +"""Official-style category aggregation for BFCL v4 single-turn tasks.""" + +from __future__ import annotations + +from collections import defaultdict + +from inspect_ai.scorer import Metric, SampleScore, Value, metric + + +def _category_accuracy(scores: list[SampleScore]) -> dict[str, float]: + grouped: dict[str, list[float]] = defaultdict(list) + for sample in scores: + if sample.sample_metadata is None: + continue + grouped[str(sample.sample_metadata["category"])].append(sample.score.as_float()) + return { + category: sum(values) / len(values) + for category, values in grouped.items() + if values + } + + +@metric +def bfcl_v4_single_turn_metrics() -> Metric: + def calculate(scores: list[SampleScore]) -> Value: + grouped: dict[str, list[float]] = defaultdict(list) + for sample in scores: + if sample.sample_metadata is not None: + grouped[str(sample.sample_metadata["category"])].append( + sample.score.as_float() + ) + accuracy = _category_accuracy(scores) + + def mean(categories: list[str]) -> float: + values = [ + accuracy[category] for category in categories if category in accuracy + ] + return sum(values) / len(values) if values else 0.0 + + simple_categories = ["simple_python", "simple_java", "simple_javascript"] + has_simple = any(name in accuracy for name in simple_categories) + simple_non_live = mean(simple_categories) + non_live_values = [ + *([simple_non_live] if has_simple else []), + *( + accuracy[name] + for name in ["multiple", "parallel", "parallel_multiple"] + if name in accuracy + ), + ] + non_live = ( + sum(non_live_values) / len(non_live_values) if non_live_values else 0.0 + ) + + live_categories = [ + "live_simple", + "live_multiple", + "live_parallel", + "live_parallel_multiple", + ] + live_total = sum(len(grouped[name]) for name in live_categories) + live = ( + sum( + accuracy.get(name, 0.0) * len(grouped[name]) for name in live_categories + ) + / live_total + if live_total + else 0.0 + ) + irrelevance = mean(["irrelevance", "live_irrelevance"]) + sections = [ + *([non_live] if non_live_values else []), + *([live] if live_total else []), + *( + [irrelevance] + if "irrelevance" in accuracy or "live_irrelevance" in accuracy + else [] + ), + ] + single_turn = sum(sections) / len(sections) if sections else 0.0 + return { + "single_turn": single_turn, + "non_live": non_live, + "live": live, + "irrelevance": irrelevance, + "relevance": accuracy.get("live_relevance", 0.0), + **accuracy, + } + + return calculate + + +@metric +def bfcl_v4_multi_turn_metrics() -> Metric: + def calculate(scores: list[SampleScore]) -> Value: + accuracy = _category_accuracy(scores) + values = list(accuracy.values()) + return { + "multi_turn": sum(values) / len(values) if values else 0.0, + **accuracy, + } + + return calculate + + +@metric +def bfcl_v4_agentic_metrics() -> Metric: + def calculate(scores: list[SampleScore]) -> Value: + accuracy = _category_accuracy(scores) + + def mean(names: list[str]) -> float: + values = [accuracy[name] for name in names if name in accuracy] + return sum(values) / len(values) if values else 0.0 + + web = mean(["web_search_base", "web_search_no_snippet"]) + memory = mean(["memory_kv", "memory_vector", "memory_rec_sum"]) + sections = [ + *([web] if any(name.startswith("web_search") for name in accuracy) else []), + *([memory] if any(name.startswith("memory_") for name in accuracy) else []), + ] + return { + "agentic": sum(sections) / len(sections) if sections else 0.0, + "web_search": web, + "memory": memory, + **accuracy, + } + + return calculate + + +@metric +def bfcl_v4_offline_metrics() -> Metric: + def calculate(scores: list[SampleScore]) -> Value: + accuracy = _category_accuracy(scores) + + def mean(names: list[str]) -> float: + values = [accuracy[name] for name in names if name in accuracy] + return sum(values) / len(values) if values else 0.0 + + simple = mean(["simple_python", "simple_java", "simple_javascript"]) + non_live = ( + sum( + [ + simple, + accuracy["multiple"], + accuracy["parallel"], + accuracy["parallel_multiple"], + ] + ) + / 4 + ) + live_names = [ + "live_simple", + "live_multiple", + "live_parallel", + "live_parallel_multiple", + ] + counts: dict[str, int] = defaultdict(int) + for sample in scores: + if sample.sample_metadata is not None: + counts[str(sample.sample_metadata["category"])] += 1 + live_count = sum(counts[name] for name in live_names) + live = sum(accuracy[name] * counts[name] for name in live_names) / live_count + irrelevance = mean(["irrelevance", "live_irrelevance"]) + multi_turn = mean( + [ + "multi_turn_base", + "multi_turn_miss_func", + "multi_turn_miss_param", + "multi_turn_long_context", + ] + ) + web = mean(["web_search_base", "web_search_no_snippet"]) + memory = mean(["memory_kv", "memory_vector", "memory_rec_sum"]) + agentic = (web + memory) / 2 + overall = ( + 0.1 * non_live + + 0.1 * live + + 0.1 * irrelevance + + 0.3 * multi_turn + + 0.4 * agentic + ) + return { + "overall_offline": overall, + "non_live": non_live, + "live": live, + "irrelevance": irrelevance, + "multi_turn": multi_turn, + "agentic": agentic, + "web_search": web, + "memory": memory, + **accuracy, + } + + return calculate diff --git a/src/openbench/scorers/__init__.py b/src/openbench/scorers/__init__.py index b0d2c036..94053fc3 100644 --- a/src/openbench/scorers/__init__.py +++ b/src/openbench/scorers/__init__.py @@ -14,7 +14,12 @@ from .score_boxed import score_boxed from .fallback_scorer import fallback_scorer from .mcq import robust_mcq_scorer, extract_mcq_answer -from .robust_boxed import robust_boxed_scorer, aime_scorer, extract_boxed_answer +from .robust_boxed import ( + aime_scorer, + extract_boxed_answer, + matharena_answer_scorer, + robust_boxed_scorer, +) from .open_answer import create_open_answer_scorer, simple_open_answer_scorer from .mmmu import mmmu_mixed_scorer from .exercism import exercism_scorer @@ -27,6 +32,7 @@ "score_boxed", "robust_boxed_scorer", "aime_scorer", + "matharena_answer_scorer", # Multiple choice scoring "robust_mcq_scorer", "extract_mcq_answer", diff --git a/src/openbench/scorers/bfcl.py b/src/openbench/scorers/bfcl.py new file mode 100644 index 00000000..b91b56a4 --- /dev/null +++ b/src/openbench/scorers/bfcl.py @@ -0,0 +1,149 @@ +"""Scoring for BFCL v4 function calls.""" + +from __future__ import annotations + +import json +import re + +from inspect_ai.scorer import CORRECT, INCORRECT, Score, Scorer, Target, scorer +from inspect_ai.solver import TaskState +from inspect_ai.util import sandbox + +from openbench.function_calling import match_function_calls, parse_function_calls +from openbench.metrics.bfcl import ( + bfcl_v4_agentic_metrics, + bfcl_v4_multi_turn_metrics, + bfcl_v4_offline_metrics, + bfcl_v4_single_turn_metrics, +) + + +@scorer(metrics=[bfcl_v4_single_turn_metrics()]) +def bfcl_v4_scorer() -> Scorer: + async def score(state: TaskState, target: Target) -> Score: + del target + message = state.output.message + calls = parse_function_calls( + message.tool_calls, + state.output.completion, + state.metadata.get("tool_name_mapping", {}), + ) + category = str(state.metadata["category"]) + + if category in {"irrelevance", "live_irrelevance"}: + matched = not calls + error = None if matched else "A tool was called for an irrelevant request" + elif category == "live_relevance": + matched = bool(calls) + error = None if matched else "No tool was called for a relevant request" + else: + result = match_function_calls( + calls, + list(state.metadata["expected_calls"]), + list(state.metadata["functions"]), + order_sensitive=False, + ) + matched, error = result.matched, result.error + + answer = [{"name": call.name, "arguments": call.arguments} for call in calls] + return Score( + value=CORRECT if matched else INCORRECT, + answer=str(answer), + explanation=error, + metadata={"category": category, "call_count": len(calls)}, + ) + + return score + + +async def _multi_turn_score(state: TaskState) -> Score: + payload_path = f".openbench_bfcl_score_{state.uuid}.json" + payload = { + "operation": "score", + "model_turn_calls": state.metadata.get("model_turn_calls", []), + "ground_truth": state.metadata["ground_truth"], + "category": state.metadata["category"], + "test_entry": { + "id": state.sample_id, + "initial_config": state.metadata["initial_config"], + "involved_classes": state.metadata["involved_classes"], + }, + } + environment = sandbox() + await environment.write_file(payload_path, json.dumps(payload)) + result = await environment.exec( + ["python", "/opt/openbench/bfcl_runner.py", payload_path], + timeout=180, + timeout_retry=False, + ) + if not result.success: + return Score(value=INCORRECT, explanation="BFCL official checker failed") + evaluation = json.loads(result.stdout.strip().splitlines()[-1]) + matched = evaluation.get("valid") is True + return Score( + value=CORRECT if matched else INCORRECT, + answer=str(state.metadata.get("model_turn_calls", [])), + explanation=None + if matched + else str(evaluation.get("error_message", evaluation)), + metadata={"category": state.metadata["category"]}, + ) + + +@scorer(metrics=[bfcl_v4_multi_turn_metrics()]) +def bfcl_v4_multi_turn_scorer() -> Scorer: + async def score(state: TaskState, target: Target) -> Score: + del target + return await _multi_turn_score(state) + + return score + + +def _standardize_answer(value: str) -> str: + return re.sub(r"[,./\-_*^()]", "", value).lower().replace("'", '"') + + +def _agentic_score(state: TaskState) -> Score: + completion = state.output.completion + standardized = _standardize_answer(completion) + expected = [str(value) for value in state.metadata["expected_answers"]] + matched = any( + re.search(rf"\b{re.escape(_standardize_answer(answer))}\b", standardized) + for answer in expected + ) + return Score( + value=CORRECT if matched else INCORRECT, + answer=completion, + explanation=None if matched else f"Expected one of {expected}", + metadata={ + "category": state.metadata["category"], + "offline_adaptation": True, + }, + ) + + +@scorer(metrics=[bfcl_v4_agentic_metrics()]) +def bfcl_v4_agentic_scorer() -> Scorer: + async def score(state: TaskState, target: Target) -> Score: + del target + return _agentic_score(state) + + return score + + +@scorer(metrics=[bfcl_v4_offline_metrics()]) +def bfcl_v4_offline_scorer() -> Scorer: + single = bfcl_v4_scorer() + + async def score(state: TaskState, target: Target) -> Score: + category = str(state.metadata["category"]) + if category.startswith("multi_turn_"): + return await _multi_turn_score(state) + if category.startswith("memory_") or category.startswith("web_search_"): + return _agentic_score(state) + result = await single(state, target) + if result is None: + return Score(value=INCORRECT, explanation="BFCL scorer returned no score") + return result + + return score diff --git a/src/openbench/scorers/evalplus.py b/src/openbench/scorers/evalplus.py new file mode 100644 index 00000000..129f4456 --- /dev/null +++ b/src/openbench/scorers/evalplus.py @@ -0,0 +1,115 @@ +"""EvalPlus scorer backed by a fail-closed Docker sandbox.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from inspect_ai.scorer import ( + CORRECT, + INCORRECT, + Score, + Scorer, + Target, + accuracy, + scorer, + stderr, +) +from inspect_ai.solver import TaskState +from inspect_ai.util import sandbox + +from openbench.datasets.evalplus import load_evalplus_record +from openbench.scorers.evalplus_sanitize import sanitize + + +def extract_python(completion: str) -> str: + """Compatibility wrapper around the canonical EvalPlus sanitizer.""" + return sanitize(completion) + + +@scorer( + metrics=[ + { + "base": [accuracy(), stderr()], + "plus": [accuracy(), stderr()], + } + ] +) +def evalplus_scorer(total_timeout: int = 900) -> Scorer: + """Evaluate base and plus tests; both must pass for a correct sample.""" + if total_timeout <= 0: + raise ValueError("total_timeout must be positive") + + async def score(state: TaskState, target: Target) -> Score: + del target + record = load_evalplus_record(state.metadata) + completion = state.output.completion + prompt = str(record["prompt"]) + entry_point = str(record["entry_point"]) + code = sanitize(completion, entry_point) + payload = { + "dataset": state.metadata["dataset"], + "task_id": record["task_id"], + "entry_point": entry_point, + "prompt": prompt, + "canonical_solution": record["canonical_solution"], + "base_input": record["base_input"], + "plus_input": record["plus_input"], + "atol": record["atol"], + "code": code, + } + environment = sandbox() + payload_path = ".openbench_evalplus_payload.json" + runner_path = ".openbench_evalplus_runner.py" + await environment.write_file(payload_path, json.dumps(payload, allow_nan=True)) + await environment.write_file( + runner_path, + Path(__file__).with_name("evalplus_runner.py").read_text(), + ) + try: + result = await environment.exec( + ["python", runner_path, payload_path], + timeout=total_timeout, + timeout_retry=False, + ) + except TimeoutError: + return Score( + value={"base": INCORRECT, "plus": INCORRECT}, + answer=completion, + explanation="EvalPlus runner timed out", + ) + if not result.success: + return Score( + value={"base": INCORRECT, "plus": INCORRECT}, + answer=completion, + explanation="EvalPlus sandbox failed", + ) + try: + evaluation = json.loads(result.stdout.strip().splitlines()[-1]) + except (IndexError, json.JSONDecodeError): + return Score( + value={"base": INCORRECT, "plus": INCORRECT}, + answer=completion, + explanation="Invalid EvalPlus result", + ) + base_passed = evaluation.get("base_passed") is True + passed = evaluation.get("passed") is True + return Score( + value={ + "base": CORRECT if base_passed else INCORRECT, + "plus": CORRECT if passed else INCORRECT, + }, + answer=completion, + explanation=( + f"base={evaluation.get('base_passed')}, " + f"plus={evaluation.get('plus_passed')}, " + f"tests={evaluation.get('tests_run', 0)}, " + f"error={evaluation.get('error')}" + ), + metadata={ + "base_passed": evaluation.get("base_passed", False), + "plus_passed": evaluation.get("plus_passed", False), + }, + ) + + return score diff --git a/src/openbench/scorers/evalplus_runner.py b/src/openbench/scorers/evalplus_runner.py new file mode 100644 index 00000000..cbbdf94e --- /dev/null +++ b/src/openbench/scorers/evalplus_runner.py @@ -0,0 +1,445 @@ +"""Container-side differential runner for EvalPlus. + +The payload containing canonical code and hidden outputs is unlinked before any +candidate process starts. Candidates receive only invocation inputs and time limits. +""" + +from __future__ import annotations + +import json +import math +import os +import signal +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any + +import numpy as np + +MBPP_OUTPUT_NOT_NONE_TASKS = {"check_str", "text_match_three", "text_starta_endb"} +MBPP_OUTPUT_SET_EQ_TASKS = { + "similar_elements", + "find_char_long", + "common_in_nested_lists", + "extract_singly", + "larg_nnum", + "intersection_array", + "find_dissimilar", + "Diff", +} + +CHILD_RUNNER = r""" +import json as _json +import os as _os +import signal as _signal +import sys as _sys +import time as _time + +def _decode(value): + if isinstance(value, list): + return [_decode(item) for item in value] + if not isinstance(value, dict) or "__type__" not in value: + return value + kind = value["__type__"] + if kind == "tuple": + return tuple(_decode(item) for item in value["items"]) + if kind == "set": + return set(_decode(item) for item in value["items"]) + if kind == "complex": + return complex(value["real"], value["imag"]) + if kind == "dict": + return {_decode(key): _decode(item) for key, item in value["items"]} + raise ValueError("Unknown wire type") + +def _encode(value): + if value is None or isinstance(value, (bool, int, float, str)): + return {"type": "scalar", "value": value, "pytype": type(value).__module__ + "." + type(value).__qualname__} + if isinstance(value, complex): + return {"type": "complex", "real": value.real, "imag": value.imag, "pytype": type(value).__module__ + "." + type(value).__qualname__} + if isinstance(value, tuple): + return {"type": "tuple", "items": [_encode(item) for item in value]} + if isinstance(value, list): + return {"type": "list", "items": [_encode(item) for item in value]} + if isinstance(value, (set, frozenset)): + items = [_encode(item) for item in value] + return {"type": "set", "items": sorted(items, key=repr)} + if isinstance(value, dict): + items = [(_encode(key), _encode(item)) for key, item in value.items()] + return {"type": "dict", "items": sorted(items, key=lambda item: repr(item[0]))} + if hasattr(value, "tolist"): + return {"type": "array", "value": _encode(value.tolist()), "pytype": type(value).__module__ + "." + type(value).__qualname__} + return {"type": "non_none"} + +def _timeout(signum, frame): + raise TimeoutError() + +request = _json.loads(_sys.stdin.read()) +inputs = _decode(request["inputs"]) +namespace = {} +try: + exec(request["code"], namespace) + function = namespace[request["entry_point"]] +except BaseException: + result = {"error": "compile_error", "outputs": [], "times": []} + _os.write(1, (_json.dumps(result) + "\n").encode()) + raise SystemExit(0) + +outputs = [] +times = [] +for args, limit in zip(inputs, request["time_limits"]): + _signal.signal(_signal.SIGALRM, _timeout) + _signal.setitimer(_signal.ITIMER_REAL, limit) + started = _time.perf_counter() + try: + outputs.append({"ok": True, "value": _encode(function(*args))}) + except TimeoutError: + outputs.append({"ok": False, "error": "timeout"}) + except BaseException: + outputs.append({"ok": False, "error": "runtime_error"}) + finally: + _signal.setitimer(_signal.ITIMER_REAL, 0) + times.append(_time.perf_counter() - started) + +result = {"error": None, "outputs": outputs, "times": times} +_os.write(1, (_json.dumps(result, allow_nan=True) + "\n").encode()) +""" + + +def _deserialize_mbpp_inputs(task_id: str, inputs: list) -> list: + number = int(task_id.split("/")[-1]) + tuple_lists = { + 2, + 116, + 132, + 143, + 222, + 261, + 273, + 394, + 399, + 421, + 424, + 429, + 470, + 560, + 579, + 596, + 616, + 630, + 726, + 740, + 744, + 809, + } + nested_tuple_lists = { + 63, + 64, + 70, + 94, + 120, + 237, + 272, + 299, + 400, + 409, + 417, + 438, + 473, + 614, + 780, + } + if number in tuple_lists: + return [[tuple(item) for item in args] for args in inputs] + if number in nested_tuple_lists: + return [[[tuple(item) for item in group] for group in args] for args in inputs] + if number in {75, 413, 444, 753}: + return [[[tuple(item) for item in args[0]], args[1]] for args in inputs] + if number in {106, 750}: + return [[args[0], tuple(args[1])] for args in inputs] + if number == 115: + return [[[set(item) if item else {} for item in args[0]]] for args in inputs] + if number == 124: + return [[float(args[0]), complex(args[1])] for args in inputs] + if number in {250, 405, 446, 617, 720, 763, 808}: + return [[tuple(args[0]), args[1]] for args in inputs] + if number in {259, 401, 445}: + converted = [ + [[tuple(item) for item in group] for group in args] for args in inputs + ] + return [[tuple(group) for group in args] for args in converted] + if number == 278: + return [ + [tuple(tuple(item) if isinstance(item, list) else item for item in args[0])] + for args in inputs + ] + if number == 307: + return [[tuple(args[0]), args[1], args[2]] for args in inputs] + if number == 722: + return [ + [{key: tuple(value) for key, value in args[0].items()}, *args[1:]] + for args in inputs + ] + if number == 252: + return [[complex(args[0])] for args in inputs] + if number in {580, 615, 791}: + + def tuples(value: Any) -> Any: + return ( + tuple(tuples(item) for item in value) + if isinstance(value, list) + else value + ) + + return [tuples(args) for args in inputs] + return inputs + + +def _wire(value: Any) -> Any: + if isinstance(value, tuple): + return {"__type__": "tuple", "items": [_wire(item) for item in value]} + if isinstance(value, set): + return {"__type__": "set", "items": [_wire(item) for item in value]} + if isinstance(value, complex): + return {"__type__": "complex", "real": value.real, "imag": value.imag} + if isinstance(value, list): + return [_wire(item) for item in value] + if isinstance(value, dict): + return { + "__type__": "dict", + "items": [[_wire(key), _wire(item)] for key, item in value.items()], + } + return value + + +def _run_code( + code: str, + entry_point: str, + inputs: list, + limits: list[float], + timeout: float, +) -> dict: + with tempfile.TemporaryDirectory(prefix="openbench-evalplus-") as workdir: + process = subprocess.Popen( + [sys.executable, "-c", CHILD_RUNNER], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=workdir, + start_new_session=True, + ) + request = json.dumps( + { + "code": code, + "entry_point": entry_point, + "inputs": _wire(inputs), + "time_limits": limits, + }, + allow_nan=True, + ) + try: + stdout, _ = process.communicate(request, timeout=timeout) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + return {"error": "task_timeout", "outputs": [], "times": []} + if process.returncode != 0: + return {"error": "runner_error", "outputs": [], "times": []} + try: + return json.loads(stdout.strip().splitlines()[-1]) + except (IndexError, json.JSONDecodeError): + return {"error": "invalid_result", "outputs": [], "times": []} + + +def _value(encoded: dict) -> Any: + if encoded.get("type") == "scalar": + return encoded.get("value") + if encoded.get("type") == "complex": + return complex(encoded["real"], encoded["imag"]) + if encoded.get("type") == "tuple": + return tuple(_value(item) for item in encoded.get("items", [])) + if encoded.get("type") == "list": + return [_value(item) for item in encoded.get("items", [])] + if encoded.get("type") == "set": + return set(_value(item) for item in encoded.get("items", [])) + if encoded.get("type") == "dict": + return {_value(key): _value(item) for key, item in encoded.get("items", [])} + if encoded.get("type") == "array": + return np.asarray(_value(encoded["value"])) + return object() + + +def _scalar(encoded: dict) -> Any: + value = _value(encoded) + return value if isinstance(value, (bool, int, float, complex, str)) else None + + +def _allclose(actual: dict, expected: dict, atol: float) -> bool: + left, right = _value(actual), _value(expected) + try: + if bool(left == right): + return True + except (TypeError, ValueError): + return False + expected_is_floats = ( + isinstance(right, float) + or ( + isinstance(right, (list, tuple)) + and bool(right) + and all(isinstance(item, float) for item in right) + ) + or (isinstance(right, np.ndarray) and right.dtype in {np.float32, np.float64}) + ) + if atol == 0 and expected_is_floats: + atol = 1e-6 + if atol == 0 or type(left) is not type(right): + return False + if isinstance(right, (list, tuple)) and len(left) != len(right): + return False + try: + return bool(np.allclose(left, right, rtol=1e-7, atol=atol)) + except (TypeError, ValueError): + return False + + +def _poly(coefficients: list, x: float) -> float: + return sum( + coefficient * math.pow(x, index) + for index, coefficient in enumerate(coefficients) + ) + + +def _matches( + dataset: str, + entry_point: str, + args: list, + actual: dict, + expected: dict, + atol: float, +) -> bool: + try: + if actual == expected: + return True + if dataset == "mbpp": + if entry_point == "are_equivalent": + return True + if entry_point == "sum_div" and _scalar(actual) == 0: + return True + if entry_point == "surface_Area": + base_edge, height = args + slant = math.sqrt((base_edge / 2) ** 2 + height**2) + reference = round(base_edge**2 + 2 * base_edge * slant) + return abs(_scalar(actual) - reference) <= atol + if entry_point == "digit_distance_nums": + one, two = str(args[0]), str(args[1]) + width = max(len(one), len(two)) + reference = sum( + abs(int(a) - int(b)) + for a, b in zip(one.zfill(width), two.zfill(width)) + ) + return _scalar(actual) == reference + if entry_point in MBPP_OUTPUT_SET_EQ_TASKS: + return set(_value(actual)) == set(_value(expected)) + if entry_point in MBPP_OUTPUT_NOT_NONE_TASKS: + value, expected_value = _scalar(actual), _scalar(expected) + if isinstance(value, bool): + return value == expected_value + return expected_value == ( + actual.get("type") != "scalar" or value is not None + ) + if dataset == "humaneval" and entry_point == "find_zero": + root = _scalar(actual) + return isinstance(root, (int, float)) and abs(_poly(args[0], root)) <= atol + return _allclose(actual, expected, atol) + except (ArithmeticError, TypeError, ValueError): + return False + + +def evaluate(payload: dict[str, Any]) -> dict[str, Any]: + dataset = str(payload["dataset"]) + task_id = str(payload["task_id"]) + entry_point = str(payload["entry_point"]) + base_inputs, plus_inputs = payload["base_input"], payload["plus_input"] + if dataset == "mbpp": + base_inputs = _deserialize_mbpp_inputs(task_id, base_inputs) + plus_inputs = _deserialize_mbpp_inputs(task_id, plus_inputs) + canonical = str(payload["prompt"]) + str(payload["canonical_solution"]) + atol = float(payload.get("atol", 0.0)) + + def run_suite(inputs: list) -> tuple[bool, int, str | None]: + if not inputs: + return True, 0, None + oracle = _run_code( + canonical, + entry_point, + inputs, + [30.0] * len(inputs), + max(60.0, len(inputs) * 30.0), + ) + if oracle.get("error") or len(oracle.get("outputs", [])) != len(inputs): + return False, 0, "oracle_failure" + if dataset == "mbpp" and entry_point in MBPP_OUTPUT_NOT_NONE_TASKS: + for output in oracle["outputs"]: + if output.get("ok"): + is_not_none = not ( + output["value"].get("type") == "scalar" + and output["value"].get("value") is None + ) + output["value"] = { + "type": "scalar", + "value": is_not_none, + "pytype": "builtins.bool", + } + limits = [max(4.0, 4.0 * elapsed) for elapsed in oracle["times"]] + candidate = _run_code( + str(payload["code"]), + entry_point, + inputs, + limits, + min(60.0, sum(limits)) + 2.0, + ) + if candidate.get("error") or len(candidate.get("outputs", [])) != len(inputs): + return False, 0, candidate.get("error", "candidate_failure") + for args, actual, expected in zip( + inputs, candidate["outputs"], oracle["outputs"] + ): + if not ( + actual.get("ok") is True + and expected.get("ok") is True + and _matches( + dataset, + entry_point, + args, + actual["value"], + expected["value"], + atol, + ) + ): + return False, len(inputs), "wrong_answer" + return True, len(inputs), None + + base_passed, base_count, base_error = run_suite(base_inputs) + plus_passed, plus_count, plus_error = run_suite(plus_inputs) + return { + "passed": base_passed and plus_passed, + "base_passed": base_passed, + "plus_passed": plus_passed, + "tests_run": base_count + plus_count, + "error": base_error or plus_error, + } + + +def main() -> None: + payload_path = Path(sys.argv[1]) + payload = json.loads(payload_path.read_text()) + payload_path.unlink() + print(json.dumps(evaluate(payload))) + + +if __name__ == "__main__": + main() diff --git a/src/openbench/scorers/evalplus_sanitize.py b/src/openbench/scorers/evalplus_sanitize.py new file mode 100644 index 00000000..cadf4d2a --- /dev/null +++ b/src/openbench/scorers/evalplus_sanitize.py @@ -0,0 +1,162 @@ +"""EvalPlus's tree-sitter Python solution sanitizer. + +Adapted under Apache-2.0 from ``evalplus/sanitize.py`` at commit +26d6d00bb1fd0fa37f39c99d5290da67891d1c5e. Copyright EvalPlus contributors. +""" + +from __future__ import annotations + +import ast +import re +from collections.abc import Generator + +import tree_sitter_python +from tree_sitter import Language, Node, Parser + +CLASS_TYPE = "class_definition" +FUNCTION_TYPE = "function_definition" +IMPORT_TYPES = {"import_statement", "import_from_statement"} +IDENTIFIER_TYPE = "identifier" +RETURN_TYPE = "return_statement" +EXPRESSION_TYPE = "expression_statement" +ASSIGNMENT_TYPE = "assignment" + + +def _syntax_check(code: str) -> bool: + try: + ast.parse(code) + return True + except (SyntaxError, MemoryError): + return False + + +def code_extract(text: str) -> str: + """Return the longest contiguous syntactically valid part of a response.""" + ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") + lines = ansi_escape.sub("", text).split("\n") + longest_pair = (0, 0) + longest_so_far = 0 + for start in range(len(lines)): + for end in range(start + 1, len(lines)): + current = lines[start : end + 1] + if _syntax_check("\n".join(current)): + length = sum(1 for line in current if line.strip()) + if length > longest_so_far: + longest_so_far = length + longest_pair = (start, end) + return "\n".join(lines[longest_pair[0] : longest_pair[1] + 1]) + + +def _traverse_tree(node: Node) -> Generator[Node, None, None]: + cursor = node.walk() + depth = 0 + visited_children = False + while True: + if not visited_children: + current_node = cursor.node + if current_node is not None: + yield current_node + if not cursor.goto_first_child(): + depth += 1 + visited_children = True + elif cursor.goto_next_sibling(): + visited_children = False + elif not cursor.goto_parent() or depth == 0: + break + else: + depth -= 1 + + +def _definition_name(node: Node) -> str: + for child in node.children: + if child.type == IDENTIFIER_TYPE: + assert child.text is not None + return child.text.decode("utf8") + raise ValueError("Definition has no identifier") + + +def _has_return(node: Node) -> bool: + return any(item.type == RETURN_TYPE for item in _traverse_tree(node)) + + +def _dependencies(nodes: list[tuple[str, Node]]) -> dict[str, set[str]]: + def visit(node: Node, result: set[str]) -> None: + for child in node.children: + if child.type == IDENTIFIER_TYPE: + assert child.text is not None + result.add(child.text.decode("utf8")) + else: + visit(child, result) + + dependencies: dict[str, set[str]] = {} + for name, node in nodes: + dependencies[name] = set() + visit(node, dependencies[name]) + return dependencies + + +def _reachable(entrypoint: str, graph: dict[str, set[str]]) -> set[str]: + queue = [entrypoint] + visited = {entrypoint} + while queue: + current = queue.pop(0) + for neighbour in graph.get(current, set()): + if neighbour not in visited: + visited.add(neighbour) + queue.append(neighbour) + return visited + + +def extract_target_code_or_empty(code: str, entrypoint: str | None = None) -> str: + code = code_extract(code) + code_bytes = code.encode("utf8") + tree = Parser(Language(tree_sitter_python.language())).parse(code_bytes) + class_names: set[str] = set() + function_names: set[str] = set() + variable_names: set[str] = set() + import_nodes: list[Node] = [] + definition_nodes: list[tuple[str, Node]] = [] + + for child in tree.root_node.children: + if child.type in IMPORT_TYPES: + import_nodes.append(child) + elif child.type == CLASS_TYPE: + name = _definition_name(child) + if name not in class_names | variable_names | function_names: + definition_nodes.append((name, child)) + class_names.add(name) + elif child.type == FUNCTION_TYPE: + name = _definition_name(child) + if ( + name not in function_names | variable_names | class_names + and _has_return(child) + ): + definition_nodes.append((name, child)) + function_names.add(name) + elif ( + child.type == EXPRESSION_TYPE + and child.children + and child.children[0].type == ASSIGNMENT_TYPE + ): + assignment = child.children[0] + name = _definition_name(assignment) + if name not in variable_names | function_names | class_names: + definition_nodes.append((name, assignment)) + variable_names.add(name) + + reachable = ( + _reachable(entrypoint, _dependencies(definition_nodes)) if entrypoint else None + ) + output = b"" + for node in import_nodes: + output += code_bytes[node.start_byte : node.end_byte] + b"\n" + for name, node in definition_nodes: + if reachable is not None and name not in reachable: + continue + output += code_bytes[node.start_byte : node.end_byte] + b"\n" + return output[:-1].decode("utf8") + + +def sanitize(code: str, entrypoint: str | None = None) -> str: + sanitized = extract_target_code_or_empty(code, entrypoint).strip() + return sanitized if sanitized else code_extract(code) diff --git a/src/openbench/scorers/global_piqa.py b/src/openbench/scorers/global_piqa.py new file mode 100644 index 00000000..800c5911 --- /dev/null +++ b/src/openbench/scorers/global_piqa.py @@ -0,0 +1,68 @@ +"""Generation scorer and hierarchical macro metric for Global PIQA.""" + +import re +from collections import defaultdict +from collections.abc import Callable + +from inspect_ai.scorer import ( + CORRECT, + INCORRECT, + Metric, + SampleScore, + Score, + Target, + Value, + metric, + scorer, +) +from inspect_ai.solver import TaskState + + +@metric +def global_piqa_macro_accuracy() -> Metric: + """Average examples by language, then languages by component, then components.""" + + def calculate(scores: list[SampleScore]) -> Value: + grouped: dict[tuple[str, str], list[float]] = defaultdict(list) + for sample in scores: + metadata = sample.score.metadata or {} + value = 1.0 if sample.score.value == CORRECT else 0.0 + grouped[(metadata["component"], metadata["language"])].append(value) + + components: dict[str, list[float]] = defaultdict(list) + for (component, _), values in grouped.items(): + components[component].append(sum(values) / len(values)) + component_scores = [sum(values) / len(values) for values in components.values()] + return ( + sum(component_scores) / len(component_scores) if component_scores else 0.0 + ) + + return calculate + + +@scorer(metrics=[global_piqa_macro_accuracy()]) +def global_piqa_scorer() -> Callable: + """Extract the answer letter using the official generation-mode patterns.""" + + async def score(state: TaskState, target: Target) -> Score: + patterns = ( + r"[Tt]he (?:[Bb]est [Aa]nswer|[Ff]inal [Aa]nswer|[Aa]nswer)[^A-D]*([A-D])", + r"[Aa]nswer\s*:[^A-D]*([A-D])", + r"\\boxed\{([A-D])\}", + ) + matches = [ + match + for pattern in patterns + for match in re.findall(pattern, state.output.completion) + ] + answer = matches[-1].upper() if matches else "" + return Score( + value=CORRECT if answer == target.text.upper() else INCORRECT, + answer=answer, + metadata={ + "component": state.metadata["component"], + "language": state.metadata["language"], + }, + ) + + return score diff --git a/src/openbench/scorers/grade_school_math.py b/src/openbench/scorers/grade_school_math.py index 56c92a5c..9514ef64 100644 --- a/src/openbench/scorers/grade_school_math.py +++ b/src/openbench/scorers/grade_school_math.py @@ -36,3 +36,27 @@ async def score_numeric_answer(state: TaskState, target: Target) -> Score: def grade_school_math_scorer() -> Callable: """Scorer for grade school math problems using numeric answer extraction.""" return score_numeric_answer + + +@scorer(metrics=[accuracy(), stderr()]) +def numeric_tolerance_scorer(tolerance: float = 1e-3) -> Callable: + """Score a parsed numeric answer with a strict absolute tolerance.""" + + async def score(state: TaskState, target: Target) -> Score: + extracted = parse_numeric_answer( + state.output.completion, state.metadata.get("answer_prefix", "Answer") + ) + try: + prediction = float(extracted.replace(",", "")) + expected = float(target.text) + correct = abs(prediction - expected) < tolerance + except (AttributeError, TypeError, ValueError): + correct = False + + return Score( + value=1.0 if correct else 0.0, + answer=extracted or "[No answer found]", + explanation=f"Absolute tolerance: {tolerance}", + ) + + return score diff --git a/src/openbench/scorers/livecodebench.py b/src/openbench/scorers/livecodebench.py new file mode 100644 index 00000000..427b4f13 --- /dev/null +++ b/src/openbench/scorers/livecodebench.py @@ -0,0 +1,154 @@ +"""LiveCodeBench v6 scorer using isolated, sandboxed program execution.""" + +from __future__ import annotations + +import base64 +import json +import pickle +import zlib +from io import BytesIO +from pathlib import Path + +from inspect_ai.scorer import ( + CORRECT, + INCORRECT, + Score, + Scorer, + Target, + accuracy, + scorer, + stderr, +) +from inspect_ai.solver import TaskState +from inspect_ai.util import sandbox + +from openbench.datasets.livecodebench import load_livecodebench_test_fields + + +class _DataOnlyUnpickler(pickle.Unpickler): + """Decode upstream's compressed JSON string without loading classes.""" + + def find_class(self, module: str, name: str) -> None: + raise pickle.UnpicklingError(f"Disallowed pickle global: {module}.{name}") + + +def extract_code(completion: str) -> str: + """Extract the last fenced code block, matching LiveCodeBench parsing.""" + + lines = completion.splitlines() + fence_lines = [index for index, line in enumerate(lines) if "```" in line] + if len(fence_lines) < 2: + return "" + return "\n".join(lines[fence_lines[-2] + 1 : fence_lines[-1]]) + + +def decode_test_cases(value: str | list[dict[str, object]]) -> list[dict[str, object]]: + """Decode plain JSON or the pinned dataset's compressed hidden tests.""" + + if isinstance(value, list): + return value + try: + decoded = json.loads(value) + except json.JSONDecodeError: + packed = zlib.decompress(base64.b64decode(value.encode("utf-8"))) + serialized_json = _DataOnlyUnpickler(BytesIO(packed)).load() + decoded = json.loads(serialized_json) + if not isinstance(decoded, list): + raise TypeError("LiveCodeBench test cases must decode to a list") + return decoded + + +def decode_metadata(value: str | dict[str, object]) -> dict[str, object]: + if isinstance(value, dict): + return value + decoded = json.loads(value) + if not isinstance(decoded, dict): + raise TypeError("LiveCodeBench metadata must decode to an object") + return decoded + + +@scorer(metrics=[accuracy(), stderr()]) +def livecodebench_scorer( + test_timeout: int = 6, + total_timeout: int = 600, +) -> Scorer: + """Run a completion against every public and private test in the sandbox.""" + + if test_timeout <= 0: + raise ValueError("test_timeout must be positive") + if total_timeout <= 0: + raise ValueError("total_timeout must be positive") + + async def score(state: TaskState, target: Target) -> Score: + del target + code = extract_code(state.output.completion) + if "source_file" in state.metadata: + public_tests, private_tests, metadata = load_livecodebench_test_fields( + state.metadata + ) + else: + public_tests = state.metadata["public_test_cases"] + private_tests = state.metadata["private_test_cases"] + metadata = state.metadata["test_metadata"] + tests = decode_test_cases(public_tests) + tests += decode_test_cases(private_tests) + test_metadata = decode_metadata(metadata) + + payload = { + "code": code, + "tests": tests, + "function_name": test_metadata.get("func_name"), + "timeout": test_timeout, + } + environment = sandbox() + payload_path = ".openbench_livecodebench_payload.json" + runner_path = ".openbench_livecodebench_runner.py" + runner_source = Path(__file__).with_name("livecodebench_runner.py").read_text() + await environment.write_file(payload_path, json.dumps(payload)) + await environment.write_file(runner_path, runner_source) + + try: + result = await environment.exec( + ["python", runner_path, payload_path], + timeout=total_timeout, + timeout_retry=False, + ) + except TimeoutError: + return Score( + value=INCORRECT, + answer=code, + explanation="LiveCodeBench evaluation exceeded its total timeout.", + ) + + if not result.success: + return Score( + value=INCORRECT, + answer=code, + explanation="LiveCodeBench runner failed inside the sandbox.", + ) + + try: + evaluation = json.loads(result.stdout.strip().splitlines()[-1]) + except (IndexError, json.JSONDecodeError): + return Score( + value=INCORRECT, + answer=code, + explanation="LiveCodeBench runner returned an invalid result.", + ) + + passed = evaluation.get("passed") is True + explanation = ( + f"Passed all {evaluation['tests_run']} tests." + if passed + else ( + f"Failed after {evaluation.get('tests_run', 0)} test(s): " + f"{evaluation.get('error', 'unknown error')}." + ) + ) + return Score( + value=CORRECT if passed else INCORRECT, + answer=code, + explanation=explanation, + ) + + return score diff --git a/src/openbench/scorers/livecodebench_runner.py b/src/openbench/scorers/livecodebench_runner.py new file mode 100644 index 00000000..1ac79798 --- /dev/null +++ b/src/openbench/scorers/livecodebench_runner.py @@ -0,0 +1,365 @@ +"""Child-process runner for LiveCodeBench Python submissions. + +The module deliberately has no Inspect imports so it can run inside the task's +sandbox. Each individual test executes in another bounded child process. +""" + +from __future__ import annotations + +import json +import os +import signal +import subprocess +import sys +import tempfile +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any + +RESULT_MARKER = "__OPENBENCH_LCB_RESULT__=" +_IS_CHILD_SUBREAPER = False +IMPORT_PRELUDE = """from string import * +from re import * +from datetime import * +from collections import * +from heapq import * +from bisect import * +from copy import * +from math import * +from random import * +from statistics import * +from itertools import * +from functools import * +from operator import * +from io import * +from sys import * +from json import * +from builtins import * +from typing import * +import string +import re +import datetime +import collections +import heapq +import bisect +import copy +import math +import random +import statistics +import itertools +import functools +import operator +import io +import json +import signal +import sys +sys.setrecursionlimit(50000) +sys.set_int_max_str_digits(50000) +""" + + +def _execute_script( + script: str, + input_text: str, + timeout: int, +) -> tuple[int, str] | None: + """Run a submission in an empty directory and kill its process group on timeout.""" + + with tempfile.TemporaryDirectory(prefix="openbench-lcb-") as workdir: + process = subprocess.Popen( + [sys.executable, "-c", script], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=workdir, + start_new_session=True, + ) + try: + stdout, _ = process.communicate(input=input_text, timeout=timeout) + except subprocess.TimeoutExpired: + _terminate_process_tree(process) + return None + return process.returncode, stdout + + +def _enable_child_subreaper() -> None: + """Adopt daemonized submission descendants inside the Linux sandbox.""" + + global _IS_CHILD_SUBREAPER + if not sys.platform.startswith("linux"): + return + + import ctypes + + pr_set_child_subreaper = 36 + libc = ctypes.CDLL(None, use_errno=True) + if libc.prctl(pr_set_child_subreaper, 1, 0, 0, 0) != 0: + error_number = ctypes.get_errno() + raise OSError(error_number, os.strerror(error_number)) + _IS_CHILD_SUBREAPER = True + + +def _linux_descendants(root_pid: int) -> set[int]: + """Return descendants from procfs, including children in new sessions.""" + + proc = Path("/proc") + if not proc.is_dir(): + return set() + + parents: dict[int, int] = {} + for entry in proc.iterdir(): + if not entry.name.isdigit(): + continue + try: + stat = (entry / "stat").read_text() + fields = stat[stat.rfind(")") + 2 :].split() + parents[int(entry.name)] = int(fields[1]) + except (FileNotFoundError, IndexError, PermissionError, ValueError): + continue + + descendants: set[int] = set() + frontier = {root_pid} + while frontier: + children = {pid for pid, parent in parents.items() if parent in frontier} + children -= descendants + descendants.update(children) + frontier = children + return descendants + + +def _terminate_process_tree(process: subprocess.Popen[str]) -> None: + """Stop and kill the submission tree, then reap the direct child.""" + + root_pid = os.getpid() if _IS_CHILD_SUBREAPER else process.pid + descendants: set[int] = set() + for _ in range(3): + newly_found = _linux_descendants(root_pid) - descendants + if not newly_found: + break + descendants.update(newly_found) + for pid in newly_found: + try: + os.kill(pid, signal.SIGSTOP) + except ProcessLookupError: + pass + + for pid in descendants: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.wait(timeout=1) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=1) + if _IS_CHILD_SUBREAPER: + while True: + try: + reaped_pid, _ = os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + break + if reaped_pid == 0: + break + for stream in (process.stdin, process.stdout, process.stderr): + if stream is not None: + stream.close() + + +def _decimal_tokens(value: str) -> list[Decimal] | None: + try: + return [Decimal(token) for token in value.split()] + except InvalidOperation: + return None + + +def outputs_match(actual: str, expected: str) -> bool: + """Compare stdout using LiveCodeBench's stripped-line semantics.""" + + actual_lines = [line.strip() for line in actual.strip().splitlines()] + expected_lines = [line.strip() for line in expected.strip().splitlines()] + if len(actual_lines) != len(expected_lines): + return False + + for actual_line, expected_line in zip(actual_lines, expected_lines): + if actual_line == expected_line: + continue + actual_numbers = _decimal_tokens(actual_line) + expected_numbers = _decimal_tokens(expected_line) + if actual_numbers is None or expected_numbers is None: + return False + if actual_numbers != expected_numbers: + return False + return True + + +def _run_stdio(code: str, test: dict[str, Any], timeout: int) -> tuple[bool, str]: + result = _execute_script( + IMPORT_PRELUDE + "\n" + code, + str(test["input"]), + timeout, + ) + if result is None: + return False, "timeout" + + returncode, stdout = result + if returncode != 0: + return False, "runtime_error" + if not outputs_match(stdout, str(test["output"])): + return False, "wrong_answer" + return True, "" + + +def _functional_harness(code: str) -> str: + return ( + IMPORT_PRELUDE + + "\n" + + code + + "\n" + + """ +def __openbench_lcb_main(): + import builtins as __ob_builtins + import json as __ob_json + import signal as __ob_signal + import sys as __ob_sys + namespace = __ob_builtins.globals() + request = __ob_json.loads(__ob_sys.stdin.read()) + solution_class = namespace.get("Solution") + target = ( + solution_class() + if __ob_builtins.isinstance(solution_class, __ob_builtins.type) + else namespace + ) + function = ( + __ob_builtins.getattr(target, request["fn_name"]) + if __ob_builtins.isinstance(solution_class, __ob_builtins.type) + else target[request["fn_name"]] + ) + results = [] + error = None + for args in request["args_list"]: + def timeout_handler(signum, frame): + raise __ob_builtins.TimeoutError() + __ob_signal.signal(__ob_signal.SIGALRM, timeout_handler) + __ob_signal.alarm(request["timeout"]) + try: + result = function(*args) + if __ob_builtins.isinstance(result, __ob_builtins.tuple): + result = __ob_builtins.list(result) + results.append(result) + except __ob_builtins.TimeoutError: + error = "timeout" + break + except __ob_builtins.Exception: + error = "runtime_error" + break + finally: + __ob_signal.alarm(0) + __ob_sys.stdout.write(""" + + repr(RESULT_MARKER) + + ' + __ob_json.dumps({"results": results, "error": error}, ensure_ascii=False) + "\\n")\n' + + "__openbench_lcb_main()\n" + ) + + +def _run_functional_tests( + code: str, + tests: list[dict[str, Any]], + function_name: str, + timeout: int, +) -> tuple[bool, str, int]: + try: + arguments = [ + [json.loads(line) for line in str(test["input"]).splitlines()] + for test in tests + ] + expected = [json.loads(str(test["output"])) for test in tests] + except (json.JSONDecodeError, TypeError): + return False, "invalid_test", 0 + + result = _execute_script( + _functional_harness(code), + json.dumps( + { + "fn_name": function_name, + "args_list": arguments, + "timeout": timeout, + } + ), + timeout * max(len(tests), 1) + 5, + ) + if result is None: + return False, "timeout", 0 + + returncode, stdout = result + if returncode != 0: + return False, "runtime_error", 0 + marker_position = stdout.rfind(RESULT_MARKER) + if marker_position < 0: + return False, "missing_result", 0 + encoded_result = stdout[marker_position + len(RESULT_MARKER) :].strip() + try: + evaluation = json.loads(encoded_result) + except json.JSONDecodeError: + return False, "invalid_result", 0 + + actual = evaluation.get("results", []) + for index, expected_result in enumerate(expected): + if index >= len(actual): + return False, evaluation.get("error") or "missing_result", index + 1 + if actual[index] != expected_result: + return False, "wrong_answer", index + 1 + return True, "", len(tests) + + +def evaluate_submission(payload: dict[str, Any]) -> dict[str, Any]: + """Evaluate one generated program against all public and hidden tests.""" + + code = payload["code"] + tests = payload["tests"] + function_name = payload.get("function_name") + timeout = int(payload.get("timeout", 6)) + + if function_name: + passed, error, tests_run = _run_functional_tests( + code, + tests, + function_name, + timeout, + ) + return { + "passed": passed, + "tests_run": tests_run, + "failed_test_index": None if passed else max(tests_run - 1, 0), + "error": None if passed else error, + } + + for index, test in enumerate(tests): + passed, error = _run_stdio(code, test, timeout) + if not passed: + return { + "passed": False, + "tests_run": index + 1, + "failed_test_index": index, + "error": error, + } + return {"passed": True, "tests_run": len(tests), "error": None} + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("usage: livecodebench_runner PAYLOAD.json") + _enable_child_subreaper() + payload_path = Path(sys.argv[1]) + payload = json.loads(payload_path.read_text()) + payload_path.unlink() + print(json.dumps(evaluate_submission(payload))) + + +if __name__ == "__main__": + main() diff --git a/src/openbench/scorers/ocrbench.py b/src/openbench/scorers/ocrbench.py new file mode 100644 index 00000000..b56d435b --- /dev/null +++ b/src/openbench/scorers/ocrbench.py @@ -0,0 +1,49 @@ +"""Official OCRBench v1 substring scorer.""" + +from collections.abc import Callable + +from inspect_ai.scorer import ( + CORRECT, + INCORRECT, + Score, + Target, + accuracy, + scorer, + stderr, +) +from inspect_ai.solver import TaskState + +from openbench.metrics.grouped import grouped + + +def normalize_ocrbench_text(text: str, *, hme: bool) -> str: + """Apply the normalization from OCRBench's reference evaluation script.""" + normalized = ( + text.replace("\n", "") if hme else text.lower().strip().replace("\n", " ") + ) + return normalized.replace(" ", "") if hme else normalized + + +@scorer( + metrics=[ + accuracy(), + stderr(), + grouped(group_key="component", metric=accuracy(), all=False), + grouped(group_key="question_type", metric=accuracy(), all=False), + ] +) +def ocrbench_scorer() -> Callable: + """Score each sample when any reference is a substring of the prediction.""" + + async def score(state: TaskState, target: Target) -> Score: + hme = state.metadata.get("dataset_name") == "HME100k" + prediction = normalize_ocrbench_text(state.output.completion, hme=hme) + references = [normalize_ocrbench_text(value, hme=hme) for value in target] + correct = any(reference in prediction for reference in references) + return Score( + value=CORRECT if correct else INCORRECT, + answer=state.output.completion, + explanation="OCRBench v1 normalized substring match", + ) + + return score diff --git a/src/openbench/scorers/robust_boxed.py b/src/openbench/scorers/robust_boxed.py index cf85a127..80e2fb7d 100644 --- a/src/openbench/scorers/robust_boxed.py +++ b/src/openbench/scorers/robust_boxed.py @@ -1,7 +1,10 @@ """Enhanced boxed answer extraction scorer with better fallback logic.""" # Adapted from https://github.com/openai/gpt-oss +import ast +import operator import re +from fractions import Fraction from typing import Optional from inspect_ai.scorer import ( @@ -35,18 +38,22 @@ def extract_boxed_answer( Returns: The extracted answer string, or None if not found """ - # Look for boxed, fbox, or framebox patterns - pattern = r"\\(?:boxed|fbox|framebox)\{([^}]*?)\}" - matches = re.findall(pattern, text, re.DOTALL) - - if matches: - # Get the last boxed answer (most likely to be final answer) - answer = matches[-1] - # If there are nested braces, extract innermost content - if "," in answer: - # Sometimes answers have extra formatting, take last part - answer = answer.split(",")[-1] - return answer.strip() + # Parse balanced braces so fractions such as \boxed{-\frac{1}{21}} survive. + answers: list[str] = [] + for match in re.finditer(r"\\(?:boxed|fbox|framebox)\{", text): + start = match.end() + depth = 1 + for index in range(start, len(text)): + if text[index] == "{": + depth += 1 + elif text[index] == "}": + depth -= 1 + if depth == 0: + answers.append(text[start:index].strip()) + break + if answers: + answer = answers[-1] + return answer.split(",")[-1].strip() if "," in answer else answer # Fallback to last number if enabled if fallback_to_last_number: @@ -58,6 +65,104 @@ def extract_boxed_answer( return None +def _replace_latex_fractions(expression: str) -> str: + token = "\\frac" + while token in expression: + start = expression.rfind(token) + cursor = start + len(token) + args: list[str] = [] + for _ in range(2): + if cursor >= len(expression) or expression[cursor] != "{": + return expression + depth = 1 + arg_start = cursor + 1 + cursor += 1 + while cursor < len(expression) and depth: + depth += expression[cursor] == "{" + depth -= expression[cursor] == "}" + cursor += 1 + if depth: + return expression + args.append(expression[arg_start : cursor - 1]) + expression = ( + expression[:start] + f"(({args[0]})/({args[1]}))" + expression[cursor:] + ) + return expression + + +def _exact_arithmetic_value(answer: str) -> Fraction | None: + expression = answer.strip().replace(",", "") + expression = expression.replace("\\dfrac", "\\frac").replace("\\tfrac", "\\frac") + expression = _replace_latex_fractions(expression) + expression = re.sub(r"\^\{([^{}]+)\}", r"**(\1)", expression) + expression = re.sub(r"\^(\-?\d+)", r"**\1", expression) + expression = expression.replace("\\cdot", "*").replace("\\times", "*") + expression = expression.replace(" ", "") + try: + tree = ast.parse(expression, mode="eval") + except SyntaxError: + return None + + binary = { + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.Div: operator.truediv, + ast.Pow: operator.pow, + } + + def evaluate(node: ast.AST) -> Fraction: + if isinstance(node, ast.Expression): + return evaluate(node.body) + if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)): + return Fraction(str(node.value)) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): + value = evaluate(node.operand) + return -value if isinstance(node.op, ast.USub) else value + if isinstance(node, ast.BinOp) and type(node.op) in binary: + left, right = evaluate(node.left), evaluate(node.right) + if isinstance(node.op, ast.Pow): + if right.denominator != 1 or abs(right.numerator) > 10000: + raise ValueError + return left**right.numerator + return binary[type(node.op)](left, right) + raise ValueError + + try: + return evaluate(tree) + except (ValueError, ZeroDivisionError, OverflowError): + return None + + +@scorer(metrics=[accuracy(), std(), stderr()]) +def matharena_answer_scorer() -> Scorer: + """Score MathArena final answers, including fractions and exact powers.""" + + async def score(state: TaskState, target: Target) -> Score: + extracted = extract_boxed_answer(state.output.completion, True) + if extracted is None: + return Score(value=INCORRECT, explanation="No final answer found") + + actual = _exact_arithmetic_value(extracted) + expected = _exact_arithmetic_value(target.text) + if actual is not None and expected is not None: + correct = actual == expected + else: + + def normalize(value: str) -> str: + value = value.replace("\\left", "").replace("\\right", "") + return re.sub(r"\s+", "", value) + + correct = normalize(extracted) == normalize(target.text) + return Score( + value=CORRECT if correct else INCORRECT, + answer=extracted, + explanation=f"Extracted '{extracted}', target was '{target.text}'", + ) + + return score + + def normalize_numeric_answer(answer: str) -> Optional[str]: """ Normalize a numeric answer for comparison. diff --git a/src/openbench/tools/livemcpbench/copilot/__init__.py b/src/openbench/tools/livemcpbench/copilot/__init__.py index d7deb722..7295e54f 100644 --- a/src/openbench/tools/livemcpbench/copilot/__init__.py +++ b/src/openbench/tools/livemcpbench/copilot/__init__.py @@ -13,6 +13,14 @@ https://github.com/icip-cas/LiveMCPBench/tree/main/baseline/mcp_copilot """ -from .server import serve as run_copilot_server +from typing import Any + + +def run_copilot_server(*args: Any, **kwargs: Any) -> Any: + """Start the optional MCP server without importing it at package load time.""" + from .server import serve + + return serve(*args, **kwargs) + __all__ = ["run_copilot_server"] diff --git a/src/openbench/tools/livemcpbench/copilot/prepare.py b/src/openbench/tools/livemcpbench/copilot/prepare.py index 3333def4..33f7316a 100644 --- a/src/openbench/tools/livemcpbench/copilot/prepare.py +++ b/src/openbench/tools/livemcpbench/copilot/prepare.py @@ -11,7 +11,6 @@ from typing import Optional import json as _json -from .server import _user_cache_dir, _ensure_parent_dir, _generate_embeddings_file from .upstream_cache import ( get_clean_config_cached, get_tools_json_cached, @@ -21,6 +20,8 @@ def _default_embeddings_path() -> Path: + from .server import _user_cache_dir + embedding_model = os.getenv("EMBEDDING_MODEL", "text-embedding-3-small") abstract_model = os.getenv("ABSTRACT_MODEL", "gpt-4.1-2025-04-14") return ( @@ -42,6 +43,8 @@ def prepare_copilot_cache( Returns: Path to the generated embeddings JSON. """ + from .server import _ensure_parent_dir, _generate_embeddings_file + # Ensure upstream JSONs are cached get_clean_config_cached(force_refresh) get_tools_json_cached(force_refresh) diff --git a/tests/integration/test_docker_sandboxes.py b/tests/integration/test_docker_sandboxes.py new file mode 100644 index 00000000..963e2650 --- /dev/null +++ b/tests/integration/test_docker_sandboxes.py @@ -0,0 +1,181 @@ +"""End-to-end checks for hardened code-execution sandboxes.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import uuid +from pathlib import Path + +import pytest + + +pytestmark = pytest.mark.docker + +ROOT = Path(__file__).parents[2] + + +def _docker(*args: str, timeout: int = 180) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["docker", *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + timeout=timeout, + ) + + +def _write_container_file(compose: list[str], destination: str, source: Path) -> None: + subprocess.run( + [ + "docker", + *compose, + "exec", + "-T", + "default", + "sh", + "-c", + f"cat > {destination}", + ], + cwd=ROOT, + input=source.read_text(), + check=True, + capture_output=True, + text=True, + timeout=30, + ) + + +def _exercise_sandbox( + tmp_path: Path, + *, + compose_path: Path, + runner_path: Path | None, + payload: dict[str, object], + payload_name: str, + image_runner_path: str | None = None, +) -> dict[str, object]: + if shutil.which("docker") is None: + pytest.skip("Docker CLI is unavailable") + try: + _docker("info", timeout=30) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + pytest.skip("Docker daemon is unavailable") + + project = f"openbench-sandbox-{uuid.uuid4().hex[:12]}" + compose = ["compose", "-p", project, "-f", str(compose_path)] + local_payload = tmp_path / payload_name + local_payload.write_text(json.dumps(payload)) + + try: + _docker(*compose, "build", timeout=600) + _docker(*compose, "up", "-d", timeout=120) + container_id = _docker(*compose, "ps", "-q", "default").stdout.strip() + assert container_id + + inspection = json.loads(_docker("inspect", container_id).stdout)[0] + host = inspection["HostConfig"] + assert host["NetworkMode"] == "none" + assert host["ReadonlyRootfs"] is True + assert host["PidsLimit"] == 64 + assert "ALL" in host["CapDrop"] + assert "no-new-privileges:true" in host["SecurityOpt"] + + container_runner = image_runner_path or ( + f"/workspace/{runner_path.name}" if runner_path is not None else "" + ) + container_payload = f"/workspace/{payload_name}" + if runner_path is not None: + _write_container_file(compose, container_runner, runner_path) + _write_container_file(compose, container_payload, local_payload) + completed = _docker( + *compose, + "exec", + "-T", + "default", + "python", + container_runner, + container_payload, + timeout=180, + ) + return json.loads(completed.stdout.strip().splitlines()[-1]) + finally: + subprocess.run( + ["docker", *compose, "down", "--volumes", "--remove-orphans"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + timeout=120, + ) + + +def test_livecodebench_sandbox_executes_without_payload_leak(tmp_path: Path) -> None: + result = _exercise_sandbox( + tmp_path, + compose_path=ROOT / "src/openbench/evals/livecodebench/compose.yaml", + runner_path=ROOT / "src/openbench/scorers/livecodebench_runner.py", + payload={ + "code": ( + "from pathlib import Path\n" + "print('leaked' if list(Path('/workspace').glob('*payload*')) " + "else 'safe')" + ), + "tests": [{"input": "", "output": "safe\n"}], + "function_name": None, + "timeout": 2, + }, + payload_name=".openbench_livecodebench_payload.json", + ) + assert result["passed"] is True + + +def test_evalplus_sandbox_executes_without_payload_leak(tmp_path: Path) -> None: + result = _exercise_sandbox( + tmp_path, + compose_path=ROOT / "src/openbench/evals/evalplus/compose.yaml", + runner_path=ROOT / "src/openbench/scorers/evalplus_runner.py", + payload={ + "dataset": "humaneval", + "task_id": "HumanEval/0", + "entry_point": "add", + "prompt": "def add(a, b):\n", + "canonical_solution": " return a + b\n", + "base_input": [[1, 2]], + "plus_input": [[-4, 9]], + "atol": 0, + "code": ( + "def add(a, b):\n" + " from pathlib import Path\n" + " leaked = list(Path('/workspace').glob('*payload*'))\n" + " return -1 if leaked else a + b\n" + ), + }, + payload_name=".openbench_evalplus_payload.json", + ) + assert result["passed"] is True + + +def test_bfcl_official_multi_turn_checker_runs_in_sandbox(tmp_path: Path) -> None: + ground_truth = [["add(a=1.0, b=2.0)"]] + result = _exercise_sandbox( + tmp_path, + compose_path=ROOT / "src/openbench/evals/bfcl/compose.yaml", + runner_path=None, + image_runner_path="/opt/openbench/bfcl_runner.py", + payload={ + "operation": "score", + "model_turn_calls": [[ground_truth[0]]], + "ground_truth": ground_truth, + "category": "multi_turn_base", + "test_entry": { + "id": "multi_turn_base_smoke", + "initial_config": {}, + "involved_classes": ["MathAPI"], + }, + }, + payload_name=".openbench_bfcl_payload.json", + ) + assert result["valid"] is True diff --git a/tests/test_bfcl.py b/tests/test_bfcl.py new file mode 100644 index 00000000..623a502d --- /dev/null +++ b/tests/test_bfcl.py @@ -0,0 +1,196 @@ +"""Tests for the BFCL v4 single-turn integration.""" + +from typing import Any, cast +from unittest.mock import patch + +import pytest +from inspect_ai.dataset import MemoryDataset, Sample +from inspect_ai.model import ChatMessageAssistant, ModelName, ModelOutput +from inspect_ai.scorer import CORRECT, Target +from inspect_ai.solver import TaskState +from inspect_ai.tool import ToolCall + +from openbench.config import BENCHMARKS +from openbench.datasets.bfcl import ( + BFCL_REVISION, + get_bfcl_v4_agentic_dataset, + get_bfcl_v4_multi_turn_dataset, + get_bfcl_v4_single_turn_dataset, +) +from openbench.evals.bfcl import bfcl_v4_single_turn +from openbench.scorers.bfcl import bfcl_v4_scorer + + +QUESTION = { + "id": "simple_python_0", + "question": [[{"role": "user", "content": "What is 1 + 2?"}]], + "function": [ + { + "name": "math.add", + "description": "Add two integers.", + "parameters": { + "type": "dict", + "properties": { + "a": {"type": "integer"}, + "b": {"type": "integer"}, + }, + "required": ["a", "b"], + }, + } + ], +} +ANSWER = { + "id": "simple_python_0", + "ground_truth": [{"math.add": {"a": [1], "b": [2]}}], +} + + +def test_bfcl_dataset_preserves_tools_answers_and_revision() -> None: + def fake_load(path: str): + return [ANSWER] if path.startswith("possible_answer/") else [QUESTION] + + with ( + patch("openbench.datasets.bfcl._load_jsonl", side_effect=fake_load), + patch.dict("openbench.datasets.bfcl._COUNTS", {"simple_python": 1}), + ): + sample = list(get_bfcl_v4_single_turn_dataset(["simple_python"]))[0] + + assert sample.metadata is not None + assert sample.id == "simple_python_0" + assert sample.metadata["category"] == "simple_python" + assert sample.metadata["bfcl_revision"] == BFCL_REVISION + assert sample.metadata["functions"][0]["name"] == "math.add" + assert sample.metadata["expected_calls"] == ANSWER["ground_truth"] + + +def test_bfcl_task_uses_deterministic_native_tool_generation() -> None: + dataset = MemoryDataset([Sample(input="question", target="")]) + with patch( + "openbench.evals.bfcl.get_bfcl_v4_single_turn_dataset", + return_value=dataset, + ): + task_factory = cast(Any, bfcl_v4_single_turn) + task = task_factory.__wrapped__(["simple_python"]) + + assert task.config.temperature == 0 + assert task.config.parallel_tool_calls is True + assert task.metadata["official_overall_score"] is False + + +def test_bfcl_registry_entry_is_explicitly_single_turn() -> None: + metadata = BENCHMARKS["bfcl_v4_single_turn"] + assert metadata.function_name == "bfcl_v4_single_turn" + assert "Single-Turn" in metadata.name + assert BENCHMARKS["bfcl_v4_multi_turn"].function_name == "bfcl_v4_multi_turn" + assert ( + BENCHMARKS["bfcl_v4_agentic_offline"].function_name == "bfcl_v4_agentic_offline" + ) + assert BENCHMARKS["bfcl_v4_offline"].function_name == "bfcl_v4_offline" + + +def test_bfcl_multi_turn_dataset_holds_out_functions() -> None: + question = { + "id": "multi_turn_miss_func_0", + "question": [[{"role": "user", "content": "Add numbers"}], []], + "initial_config": {}, + "involved_classes": ["MathAPI"], + "missed_function": {"1": ["add"]}, + } + answer = {"id": question["id"], "ground_truth": [[], ["add(a=1,b=2)"]]} + + def fake_load(path: str): + if path.startswith("possible_answer/"): + return [answer] + if path.startswith("multi_turn_func_doc/"): + function = cast(dict[str, Any], QUESTION["function"][0]) + return [{**function, "name": "add"}] + return [question] + + with ( + patch("openbench.datasets.bfcl._load_jsonl", side_effect=fake_load), + patch("openbench.datasets.bfcl._MULTI_TURN_COUNT", 1), + ): + sample = list(get_bfcl_v4_multi_turn_dataset(["multi_turn_miss_func"]))[0] + + assert sample.metadata is not None + assert sample.metadata["functions"] == [] + assert sample.metadata["missed_functions"]["1"][0]["name"] == "add" + + +def test_bfcl_agentic_dataset_expands_backends_and_web_modes() -> None: + memory_question = { + "id": "memory_0-customer-0", + "question": [[{"role": "user", "content": "My name?"}]], + } + memory_answer = { + "id": memory_question["id"], + "ground_truth": ["Michael"], + "source": "My name is Michael.", + } + web_question = { + "id": "web_search_0", + "question": [[{"role": "user", "content": "Who?"}]], + } + web_answer = { + "id": web_question["id"], + "ground_truth": ["Ada"], + "source": [{"subquestion": "Who?", "answer": "Ada", "source": "u"}], + } + + def fake_load(path: str): + if "multi_turn_func_doc" in path: + return QUESTION["function"] + if "memory" in path: + return ( + [memory_answer] + if path.startswith("possible_answer/") + else [memory_question] + ) + return [web_answer] if path.startswith("possible_answer/") else [web_question] + + with patch("openbench.datasets.bfcl._load_jsonl", side_effect=fake_load): + dataset = get_bfcl_v4_agentic_dataset( + ["memory_kv", "web_search_base", "web_search_no_snippet"] + ) + + assert len(dataset) == 3 + metadata = [cast(dict[str, Any], sample.metadata) for sample in dataset] + assert [item["category"] for item in metadata] == [ + "memory_kv", + "web_search_base", + "web_search_no_snippet", + ] + + +@pytest.mark.asyncio +async def test_bfcl_scorer_reads_native_tool_calls() -> None: + output = ModelOutput.from_message( + ChatMessageAssistant( + content="", + tool_calls=[ + ToolCall( + id="call-1", + function="math_add", + arguments={"a": 1, "b": 2}, + ) + ], + ), + stop_reason="tool_calls", + ) + state = TaskState( + model=ModelName("mock/test"), + sample_id="simple_python_0", + epoch=1, + input="What is 1 + 2?", + messages=[], + output=output, + metadata={ + "category": "simple_python", + "functions": QUESTION["function"], + "expected_calls": ANSWER["ground_truth"], + "tool_name_mapping": {"math_add": "math.add"}, + }, + ) + result = await bfcl_v4_scorer()(state, Target("")) + assert result is not None + assert result.value == CORRECT diff --git a/tests/test_bigbench_hard_complete.py b/tests/test_bigbench_hard_complete.py new file mode 100644 index 00000000..dc8857fa --- /dev/null +++ b/tests/test_bigbench_hard_complete.py @@ -0,0 +1,47 @@ +from unittest.mock import patch + +from openbench.config import BENCHMARKS, EVAL_GROUPS +from openbench.evals.bigbench_hard import ( + BBH_DATASET_REVISION, + _bbh_free_response_task, + _free_response_record, +) + + +MISSING_CONFIGS = { + "boolean_expressions", + "dyck_languages", + "formal_fallacies", + "hyperbaton", + "multistep_arithmetic_two", + "object_counting", + "penguins_in_a_table", + "web_of_lies", + "word_sorting", +} + + +def test_bbh_free_response_keeps_full_target(): + sample = _free_response_record({"input": "Sort words", "target": "a b c"}) + + assert sample.target == "a b c" + assert "So the answer is" in sample.input + + +def test_bbh_new_tasks_are_pinned(): + with patch( + "openbench.evals.bigbench_hard.hf_dataset", + return_value=[_free_response_record({"input": "x", "target": "y"})], + ) as load: + _bbh_free_response_task("word_sorting") + + assert load.call_args.kwargs["revision"] == BBH_DATASET_REVISION + assert load.call_args.kwargs["name"] == "word_sorting" + + +def test_bbh_complete_registry_and_group(): + expected_ids = {f"bbh_{name}" for name in MISSING_CONFIGS} + + assert expected_ids <= BENCHMARKS.keys() + assert expected_ids <= set(EVAL_GROUPS["bbh"].benchmarks) + assert len(EVAL_GROUPS["bbh"].benchmarks) == 27 diff --git a/tests/test_ceval.py b/tests/test_ceval.py new file mode 100644 index 00000000..67ad16f1 --- /dev/null +++ b/tests/test_ceval.py @@ -0,0 +1,58 @@ +from unittest.mock import patch + +import pytest + +from openbench.config import BENCHMARKS +from openbench.datasets.ceval import ( + DATASET_REVISION, + HARD_SUBJECTS, + SUBJECTS, + get_ceval_dataset, +) + + +def _record(record_id: int, answer: str = "B") -> dict: + return { + "id": record_id, + "question": "测试题", + "A": "甲", + "B": "乙", + "C": "丙", + "D": "丁", + "answer": answer, + "explanation": "", + } + + +def test_ceval_subject_inventory(): + assert len(SUBJECTS) == 52 + assert len(HARD_SUBJECTS) == 8 + assert set(HARD_SUBJECTS) <= SUBJECTS.keys() + + +def test_ceval_five_shot_prompt_and_pinning(): + def fake_load(path, *, name, split, revision): + assert path == "ceval/ceval-exam" + assert name == "computer_network" + assert revision == DATASET_REVISION + return [_record(i) for i in range(5)] if split == "dev" else [_record(9)] + + with patch("openbench.datasets.ceval.load_dataset", side_effect=fake_load): + dataset = get_ceval_dataset(subjects=["computer_network"], shots=5) + + sample = list(dataset)[0] + assert sample.input.count("答案：B") == 5 + assert sample.input.endswith("答案：") + assert sample.target == "B" + assert sample.metadata == {"subject": "computer_network", "category": "STEM"} + + +@pytest.mark.parametrize("shots", [1, 4, 10]) +def test_ceval_rejects_nonstandard_shot_counts(shots): + with pytest.raises(ValueError, match="0-shot or 5-shot"): + get_ceval_dataset(subjects=["computer_network"], shots=shots) + + +def test_ceval_registry_entries(): + assert BENCHMARKS["ceval"].function_name == "ceval" + assert BENCHMARKS["ceval_hard"].function_name == "ceval_hard" diff --git a/tests/test_evalplus.py b/tests/test_evalplus.py new file mode 100644 index 00000000..533f6b84 --- /dev/null +++ b/tests/test_evalplus.py @@ -0,0 +1,148 @@ +import gzip +import hashlib +import json +from pathlib import Path +from unittest.mock import patch + +from inspect_ai.dataset import MemoryDataset, Sample + +from openbench.config import BENCHMARKS +from openbench.datasets.evalplus import ( + RELEASES, + _ensure_release, + get_evalplus_dataset, + load_evalplus_record, +) +from openbench.evals.evalplus.evalplus import humaneval_plus, mbpp_plus +from openbench.scorers.evalplus import extract_python +from openbench.scorers.evalplus_runner import evaluate + + +def _record(task_id: str = "HumanEval/0") -> dict: + return { + "task_id": task_id, + "prompt": "def add(a, b):\n", + "entry_point": "add", + "canonical_solution": " return a + b\n", + "base_input": [[1, 2]], + "plus_input": [[-4, 9]], + "atol": 0, + } + + +def test_evalplus_dataset_keeps_tests_out_of_metadata(tmp_path: Path): + path = tmp_path / "human.jsonl" + path.write_text(json.dumps(_record()) + "\n") + with ( + patch("openbench.datasets.evalplus._ensure_release", return_value=path), + patch.dict(RELEASES["humaneval"], {"count": 1}), + ): + sample = list(get_evalplus_dataset("humaneval"))[0] + + assert sample.metadata is not None + assert "base_input" not in sample.metadata + assert "plus_input" not in sample.metadata + assert load_evalplus_record(sample.metadata)["plus_input"] == [[-4, 9]] + + +def test_evalplus_runner_accepts_correct_and_rejects_wrong(): + payload = { + "dataset": "humaneval", + **_record(), + "code": "def add(a, b):\n return a + b\n", + } + assert evaluate(payload)["passed"] is True + + payload["code"] = "def add(a, b):\n return a - b\n" + assert evaluate(payload)["passed"] is False + + +def test_evalplus_runs_base_and_plus_in_fresh_processes(): + payload = { + "dataset": "humaneval", + **_record(), + "code": ( + "calls = 0\n" + "def add(a, b):\n" + " global calls\n" + " calls += 1\n" + " return a + b if calls == 1 else None\n" + ), + } + assert evaluate(payload)["passed"] is True + + +def test_evalplus_mbpp_special_oracles(): + not_none = { + "dataset": "mbpp", + "task_id": "Mbpp/1", + "prompt": "", + "entry_point": "check_str", + "canonical_solution": ( + "import re\ndef check_str(value):\n return re.match(r'a', value)\n" + ), + "base_input": [["abc"]], + "plus_input": [], + "atol": 0, + "code": "def check_str(value):\n return 'accepted'\n", + } + assert evaluate(not_none)["passed"] is True + + set_equivalent = { + "dataset": "mbpp", + "task_id": "Mbpp/2", + "prompt": "", + "entry_point": "similar_elements", + "canonical_solution": "def similar_elements(value):\n return [1]\n", + "base_input": [[[0]]], + "plus_input": [], + "atol": 0, + "code": "def similar_elements(value):\n return [1.0]\n", + } + assert evaluate(set_equivalent)["passed"] is True + + +def test_evalplus_extracts_last_python_fence(): + assert ( + extract_python("text\n```python\ndef f():\n pass\n```") + == "def f():\n pass" + ) + + +def test_evalplus_repairs_corrupt_expanded_cache(tmp_path: Path): + content = (json.dumps(_record()) + "\n").encode() + compressed = gzip.compress(content) + release = { + "version": "test", + "url": "https://invalid.example/test.jsonl.gz", + "sha256": hashlib.sha256(compressed).hexdigest(), + "expanded_sha256": hashlib.sha256(content).hexdigest(), + "count": 1, + } + compressed_path = tmp_path / "humaneval-test.jsonl.gz" + expanded_path = tmp_path / "humaneval-test.jsonl" + compressed_path.write_bytes(compressed) + expanded_path.write_text("corrupt") + with ( + patch("openbench.datasets.evalplus._cache_dir", return_value=tmp_path), + patch.dict(RELEASES, {"humaneval": release}), + ): + assert _ensure_release("humaneval").read_bytes() == content + + +def test_evalplus_tasks_use_docker_sandbox(): + dataset = MemoryDataset([Sample(input="x", target="x")]) + with patch( + "openbench.evals.evalplus.evalplus.get_evalplus_dataset", + return_value=dataset, + ): + human = humaneval_plus.__wrapped__(epochs=1) + mbpp = mbpp_plus.__wrapped__(epochs=1) + + assert human.sandbox.type == "docker" + assert mbpp.sandbox.type == "docker" + + +def test_evalplus_registry_entries(): + assert BENCHMARKS["humaneval_plus"].function_name == "humaneval_plus" + assert BENCHMARKS["mbpp_plus"].function_name == "mbpp_plus" diff --git a/tests/test_function_calling.py b/tests/test_function_calling.py new file mode 100644 index 00000000..9043ca8e --- /dev/null +++ b/tests/test_function_calling.py @@ -0,0 +1,126 @@ +"""Tests for provider-neutral function-calling primitives.""" + +from typing import Any + +from inspect_ai.tool import ToolCall + +from openbench.function_calling import ( + FunctionCall, + build_tool_definitions, + match_function_calls, + parse_function_calls, +) +from openbench.function_calling.stateful import frozen_agentic_result + + +FUNCTIONS = [ + { + "name": "weather.get", + "description": "Get weather.", + "parameters": { + "type": "dict", + "properties": { + "city": {"type": "string"}, + "days": {"type": "integer"}, + "units": {"type": "string"}, + }, + "required": ["city", "days"], + }, + } +] + + +def test_build_tool_definitions_normalizes_names_and_schema() -> None: + definitions, mapping = build_tool_definitions(FUNCTIONS) + info = definitions[0] + + assert info.name == "weather_get" + assert mapping == {"weather_get": "weather.get"} + assert info.parameters.required == ["city", "days"] + assert info.parameters.properties["days"].type == "integer" + + +def test_parse_native_json_and_python_calls() -> None: + native = [ToolCall(id="1", function="weather_get", arguments={"city": "Paris"})] + assert parse_function_calls(native, "", {"weather_get": "weather.get"}) == [ + FunctionCall("weather.get", {"city": "Paris"}) + ] + + assert parse_function_calls( + None, '[{"name":"weather.get","arguments":{"city":"Paris"}}]' + ) == [FunctionCall("weather.get", {"city": "Paris"})] + assert parse_function_calls(None, "[weather.get(city='Paris', days=2)]") == [ + FunctionCall("weather.get", {"city": "Paris", "days": 2}) + ] + + +def test_match_calls_handles_optional_values_and_parallel_order() -> None: + expected: list[dict[str, dict[str, list[Any]]]] = [ + { + "weather.get": { + "city": ["Paris"], + "days": [2], + "units": ["", "metric"], + } + }, + { + "weather.get": { + "city": ["New York"], + "days": [1], + "units": ["", "metric"], + } + }, + ] + actual = [ + FunctionCall("weather.get", {"city": "New-York", "days": 1}), + FunctionCall("weather.get", {"city": "paris", "days": 2, "units": "metric"}), + ] + result = match_function_calls(actual, expected, FUNCTIONS) + assert result.matched is True + + +def test_match_calls_rejects_wrong_types_and_extra_arguments() -> None: + expected: list[dict[str, dict[str, list[Any]]]] = [ + {"weather.get": {"city": ["Paris"], "days": [2]}} + ] + wrong_type = match_function_calls( + [FunctionCall("weather.get", {"city": "Paris", "days": 2.0})], + expected, + FUNCTIONS, + ) + extra = match_function_calls( + [FunctionCall("weather.get", {"city": "Paris", "days": 2, "unknown": True})], + expected, + FUNCTIONS, + ) + assert wrong_type.matched is False + assert extra.matched is False + + +def test_match_calls_allows_python_int_for_float_schema() -> None: + functions = [ + { + "name": "scale", + "parameters": { + "type": "dict", + "properties": {"value": {"type": "float"}}, + "required": ["value"], + }, + } + ] + result = match_function_calls( + [FunctionCall("scale", {"value": 3})], + [{"scale": {"value": [3.0]}}], + functions, + ) + assert result.matched is True + + +def test_frozen_web_search_hides_or_exposes_snippets() -> None: + source = [{"subquestion": "Who?", "answer": "Ada", "source": "https://x"}] + call = FunctionCall("search_engine_query", {"keywords": "who"}) + assert "Ada" in frozen_agentic_result(call, source=source, show_snippet=True) + assert "Ada" not in frozen_agentic_result(call, source=source, show_snippet=False) + + fetch = FunctionCall("fetch_url_content", {"url": "https://x"}) + assert "Ada" in frozen_agentic_result(fetch, source=source, show_snippet=False) diff --git a/tests/test_global_piqa.py b/tests/test_global_piqa.py new file mode 100644 index 00000000..eb376917 --- /dev/null +++ b/tests/test_global_piqa.py @@ -0,0 +1,51 @@ +from unittest.mock import patch + +from openbench.config import BENCHMARKS +from openbench.datasets.global_piqa import DATASETS, get_global_piqa_dataset + + +def test_global_piqa_loads_both_pinned_components_and_prompts(): + nonparallel = { + "prompt": "Situation", + "solution0": "A0", + "solution1": "B0", + "label": 1, + "example_id": "n1", + } + parallel = { + **nonparallel, + "solution2": "C0", + "solution3": "D0", + "label": 2, + "example_id": "p1", + } + + def fake_load(path, *, name, split, revision): + assert name == "eng_latn" + assert split == "test" + assert ( + revision + == DATASETS["nonparallel" if "nonparallel" in path else "parallel"][1] + ) + return [nonparallel if "nonparallel" in path else parallel] + + with ( + patch( + "openbench.datasets.global_piqa.get_dataset_config_names", + return_value=["eng_latn"], + ), + patch("openbench.datasets.global_piqa.load_dataset", side_effect=fake_load), + ): + samples = list(get_global_piqa_dataset()) + + assert [sample.target for sample in samples] == ["B", "C"] + assert "one of A or B" in samples[0].input + assert "A, B, C, or D" in samples[1].input + assert {sample.metadata["component"] for sample in samples} == { + "parallel", + "nonparallel", + } + + +def test_global_piqa_registry_entry(): + assert BENCHMARKS["global_piqa"].function_name == "global_piqa" diff --git a/tests/test_gsm8k_hard.py b/tests/test_gsm8k_hard.py new file mode 100644 index 00000000..059ce491 --- /dev/null +++ b/tests/test_gsm8k_hard.py @@ -0,0 +1,41 @@ +import pytest +from inspect_ai.model import ModelOutput +from inspect_ai.scorer import Target +from inspect_ai.solver import TaskState + +from openbench.config import BENCHMARKS +from openbench.evals.gsm8k_hard import DATASET_REVISION, record_to_sample +from openbench.scorers.grade_school_math import numeric_tolerance_scorer + + +def test_gsm8k_hard_record_conversion(): + sample = record_to_sample({"input": "What is 2 + 2?", "target": 4.0}) + + assert "What is 2 + 2?" in sample.input + assert sample.target == "4.0" + assert sample.metadata == {"answer_prefix": "Answer"} + + +@pytest.mark.asyncio +async def test_gsm8k_hard_uses_official_strict_tolerance(): + scorer = numeric_tolerance_scorer(tolerance=1e-3) + state = TaskState( + model="mock", + sample_id="1", + epoch=1, + input="question", + messages=[], + output=ModelOutput.from_content("mock", "Answer: 1.0009"), + metadata={"answer_prefix": "Answer"}, + ) + + assert (await scorer(state, Target("1.0"))).value == 1.0 + # The canonical PAL scorer uses binary floats, where 1.001 - 1.0 is + # marginally below 0.001. Use a value unambiguously outside the threshold. + state.output = ModelOutput.from_content("mock", "Answer: 1.0011") + assert (await scorer(state, Target("1.0"))).value == 0.0 + + +def test_gsm8k_hard_registry_and_revision(): + assert BENCHMARKS["gsm8k_hard"].function_name == "gsm8k_hard" + assert DATASET_REVISION == "960448f73503112d4226baeb8eb41d3fb5ae2506" diff --git a/tests/test_livecodebench.py b/tests/test_livecodebench.py new file mode 100644 index 00000000..d423a975 --- /dev/null +++ b/tests/test_livecodebench.py @@ -0,0 +1,306 @@ +"""Tests for the LiveCodeBench v6 adapter and execution runner.""" + +import base64 +import json +import pickle +import subprocess +import sys +import time +import zlib +from pathlib import Path +from unittest.mock import patch + +import pytest +from inspect_ai.dataset import MemoryDataset, Sample + +from openbench.datasets.livecodebench import ( + DATASET_REVISION, + RELEASE_VERSION, + SYSTEM_PROMPT, + get_livecodebench_v6_dataset, + load_livecodebench_test_fields, + record_to_sample, +) +from openbench.evals.livecodebench import livecodebench_v6 +from openbench.scorers.livecodebench import decode_test_cases, extract_code +from openbench.scorers.livecodebench_runner import evaluate_submission, outputs_match + + +def _record(**overrides): + record = { + "question_title": "Add", + "question_content": "Read two integers and print their sum.", + "platform": "codeforces", + "question_id": "cf-add", + "contest_id": "1", + "contest_date": "2025-04-01T00:00:00", + "starter_code": "", + "difficulty": "easy", + "public_test_cases": json.dumps( + [{"input": "1 2\n", "output": "3\n", "testtype": "stdin"}] + ), + "private_test_cases": json.dumps([]), + "metadata": json.dumps({}), + } + record.update(overrides) + return record + + +def test_record_to_sample_preserves_release_and_hidden_tests(): + sample = record_to_sample()(_record()) + assert isinstance(sample, Sample) + assert sample.id == "cf-add" + assert "Read two integers" in str(sample.input) + assert sample.metadata["release_version"] == RELEASE_VERSION + assert sample.metadata["dataset_revision"] == DATASET_REVISION + assert sample.metadata["private_test_cases"] == "[]" + assert sample.input[0].content == SYSTEM_PROMPT + assert sample.input[1].content == ( + "### Question:\nRead two integers and print their sum.\n\n" + "### Format: Read the inputs from stdin solve the problem and write the " + "answer to stdout (do not directly test on the sample inputs). Enclose " + "your code within delimiters as follows. Ensure that when the python " + "program runs, it reads the inputs, runs the algorithm and writes output " + "to STDOUT.\n```python\n# YOUR CODE HERE\n```\n\n" + "### Answer: (use the provided format with backticks)\n\n" + ) + + +def test_record_to_sample_applies_inclusive_date_filter(): + converter = record_to_sample( + start_date="2025-04-01T00:00:00", + end_date="2025-04-01T00:00:00", + ) + assert converter(_record()) != [] + assert converter(_record(contest_date="2025-03-31T23:59:59")) == [] + + +def test_record_to_sample_includes_starter_code_contract(): + sample = record_to_sample()(_record(starter_code="class Solution:\n pass")) + assert isinstance(sample, Sample) + assert "starter code" in str(sample.input) + assert "class Solution" in str(sample.input) + + +def test_dataset_parses_cached_shards_and_sorts_without_arrow_cache(tmp_path): + later = _record(question_id="z", contest_date="2025-04-02T00:00:00") + earlier = _record(question_id="a") + shard = tmp_path / "test.jsonl" + shard.write_text("\n".join([json.dumps(later), json.dumps(earlier)])) + with patch( + "openbench.datasets.livecodebench._release_paths", + return_value=[("test.jsonl", shard)], + ) as release_paths: + dataset = get_livecodebench_v6_dataset() + + assert [sample.id for sample in dataset] == ["a", "z"] + assert dataset[1].metadata["contest_date"] == "2025-04-02T00:00:00" + assert "private_test_cases" not in dataset[1].metadata + release_paths.assert_called_once_with() + + with patch( + "openbench.datasets.livecodebench._release_path", + return_value=shard, + ): + public, private, metadata = load_livecodebench_test_fields(dataset[1].metadata) + assert json.loads(public)[0]["output"] == "3\n" + assert private == "[]" + assert metadata == "{}" + + +def test_task_uses_official_generic_sampling_configuration(): + dataset = MemoryDataset([Sample(input="question", target="")]) + with patch( + "openbench.evals.livecodebench.livecodebench.get_livecodebench_v6_dataset", + return_value=dataset, + ): + task = livecodebench_v6() + + assert task.epochs == 10 + assert task.config.temperature == 0.2 + assert task.config.top_p == 0.95 + assert task.config.max_tokens == 2000 + assert task.config.stop_seqs == ["###"] + + +def test_decode_test_cases_supports_plain_and_compressed_payloads(): + tests = [{"input": "1\n", "output": "1\n", "testtype": "stdin"}] + assert decode_test_cases(json.dumps(tests)) == tests + + compressed = base64.b64encode( + zlib.compress(pickle.dumps(json.dumps(tests))) + ).decode("utf-8") + assert decode_test_cases(compressed) == tests + + +def test_extract_code_uses_final_fence_pair_for_any_label(): + completion = "```python\nprint(1)\n```\nreasoning\n```Python\nprint(2)\n```" + assert extract_code(completion) == "print(2)" + assert extract_code("```py\nprint(3)\n```") == "print(3)" + assert extract_code("print(4)") == "" + + +def test_outputs_match_uses_exact_decimal_tokens(): + assert outputs_match("1.0 2\n", "1.00 2.0\n") + assert not outputs_match("1.0000000000000001\n", "1.0\n") + + +def test_runner_scores_standard_input_submission(): + result = evaluate_submission( + { + "code": "a, b = map(int, input().split())\nprint(a + b)", + "tests": [ + {"input": "1 2\n", "output": "3\n"}, + {"input": "-2 5\n", "output": "3\n"}, + ], + "function_name": None, + "timeout": 2, + } + ) + assert result == {"passed": True, "tests_run": 2, "error": None} + + +def test_runner_provides_official_module_import_prelude(): + result = evaluate_submission( + { + "code": "print(math.isqrt(int(input())))", + "tests": [{"input": "81\n", "output": "9\n"}], + "function_name": None, + "timeout": 2, + } + ) + assert result["passed"] is True + + +def test_runner_scores_functional_submission_and_failure(): + passing = evaluate_submission( + { + "code": "class Solution:\n def add(self, a, b):\n return a + b", + "tests": [{"input": "1\n2", "output": "3"}], + "function_name": "add", + "timeout": 2, + } + ) + assert passing["passed"] is True + + failing = evaluate_submission( + { + "code": "class Solution:\n def add(self, a, b):\n return a - b", + "tests": [{"input": "1\n2", "output": "3"}], + "function_name": "add", + "timeout": 2, + } + ) + assert failing["passed"] is False + assert failing["error"] == "wrong_answer" + + +def test_runner_preserves_functional_solution_state_across_tests(): + result = evaluate_submission( + { + "code": ( + "class Solution:\n" + " def __init__(self): self.total = 0\n" + " def add(self, value):\n" + " self.total += value\n" + " return self.total" + ), + "tests": [ + {"input": "1", "output": "1"}, + {"input": "2", "output": "3"}, + ], + "function_name": "add", + "timeout": 2, + } + ) + assert result["passed"] is True + + +def test_runner_kills_submission_process_group_on_timeout(): + started = time.monotonic() + result = evaluate_submission( + { + "code": ( + "import subprocess, sys, time\n" + "subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'])\n" + "time.sleep(30)" + ), + "tests": [{"input": "", "output": ""}], + "function_name": None, + "timeout": 1, + } + ) + assert result["error"] == "timeout" + assert time.monotonic() - started < 5 + + +def test_runner_deletes_payload_before_submission_starts(tmp_path): + payload_path = tmp_path / ".openbench_livecodebench_payload.json" + payload_path.write_text( + json.dumps( + { + "code": ( + "from pathlib import Path\n" + "print('leaked' if list(Path('.').glob('*payload*')) else 'safe')" + ), + "tests": [{"input": "", "output": "safe\n"}], + "function_name": None, + "timeout": 2, + } + ) + ) + runner = Path(__file__).parents[1] / "src/openbench/scorers/livecodebench_runner.py" + sandbox_runner = tmp_path / ".openbench_livecodebench_runner.py" + sandbox_runner.write_text(runner.read_text()) + completed = subprocess.run( + [sys.executable, str(sandbox_runner), str(payload_path)], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + timeout=10, + ) + assert completed.returncode == 0 + assert json.loads(completed.stdout)["passed"] is True + assert not payload_path.exists() + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="requires procfs") +def test_runner_reaps_double_forked_descendants(tmp_path): + payload_path = tmp_path / ".openbench_livecodebench_payload.json" + payload_path.write_text( + json.dumps( + { + "code": ( + "import os, time\n" + "child = os.fork()\n" + "if child == 0:\n" + " os.setsid()\n" + " if os.fork() > 0: os._exit(0)\n" + " time.sleep(30)\n" + " os._exit(0)\n" + "os.waitpid(child, 0)\n" + "time.sleep(30)" + ), + "tests": [{"input": "", "output": ""}], + "function_name": None, + "timeout": 1, + } + ) + ) + runner = Path(__file__).parents[1] / "src/openbench/scorers/livecodebench_runner.py" + sandbox_runner = tmp_path / ".openbench_livecodebench_runner.py" + sandbox_runner.write_text(runner.read_text()) + + started = time.monotonic() + completed = subprocess.run( + [sys.executable, str(sandbox_runner), str(payload_path)], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + timeout=10, + ) + assert completed.returncode == 0 + assert json.loads(completed.stdout)["error"] == "timeout" + assert time.monotonic() - started < 5 diff --git a/tests/test_ocrbench.py b/tests/test_ocrbench.py new file mode 100644 index 00000000..f6554247 --- /dev/null +++ b/tests/test_ocrbench.py @@ -0,0 +1,39 @@ +from unittest.mock import patch + +from PIL import Image + +from openbench.config import BENCHMARKS +from openbench.datasets.ocrbench import DATASET_REVISION, get_ocrbench_dataset +from openbench.scorers.ocrbench import normalize_ocrbench_text + + +def test_ocrbench_loader_is_v1_and_pinned(): + rows = [ + { + "dataset": "IIIT5K", + "question": "what is written in the image?", + "question_type": "Regular Text Recognition", + "answer": ["CENTRE"], + "image": Image.new("RGB", (2, 2), "white"), + } + ] + with patch("openbench.datasets.ocrbench.load_dataset", return_value=rows) as load: + sample = list(get_ocrbench_dataset())[0] + + assert load.call_args.kwargs == { + "split": "test", + "revision": DATASET_REVISION, + } + assert load.call_args.args == ("echo840/OCRBench",) + assert sample.target == ["CENTRE"] + assert sample.metadata["component"] == "text_recognition" + + +def test_ocrbench_normalization_modes(): + assert normalize_ocrbench_text(" CenTre\nText ", hme=False) == "centre text" + assert normalize_ocrbench_text(" x + y \n", hme=True) == "x+y" + + +def test_ocrbench_v1_and_v2_are_distinct_registry_entries(): + assert BENCHMARKS["ocrbench"].function_name == "ocrbench" + assert BENCHMARKS["ocrbenchv2"].function_name == "ocrbenchv2" diff --git a/tests/test_registry.py b/tests/test_registry.py index 070ecb51..abaf9de8 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -1,11 +1,13 @@ """Test the registry module functionality.""" -import pytest from unittest.mock import Mock, patch + +import pytest + from openbench.config import ( - load_task, TASK_REGISTRY, _load_entry_point_benchmarks, + load_task, ) from openbench.utils import BenchmarkMetadata @@ -14,6 +16,9 @@ def test_task_registry_contents(): """Test that the task registry contains expected benchmarks.""" assert "mmlu" in TASK_REGISTRY assert TASK_REGISTRY["mmlu"] == "openbench.evals.mmlu.mmlu" + assert TASK_REGISTRY["livecodebench_v6"] == ( + "openbench.evals.livecodebench.livecodebench_v6" + ) def test_load_task_valid(): diff --git a/tests/test_registry_imports.py b/tests/test_registry_imports.py index a3806083..bc892ca9 100644 --- a/tests/test_registry_imports.py +++ b/tests/test_registry_imports.py @@ -22,7 +22,10 @@ def test_imports_for_optional_dependencies(): # Create temp venv and install package with only base dependencies steps = [ (["uv", "venv", venv_path], "create venv"), - (["uv", "pip", "install", "-e", ".", "--python", python_exe], "install openbench with base deps"), + ( + ["uv", "pip", "install", "-e", ".", "--python", python_exe], + "install openbench with base deps", + ), ] for cmd, desc in steps: @@ -33,7 +36,8 @@ def test_imports_for_optional_dependencies(): # Test the import in the venv result = subprocess.run( [python_exe, "-c", "import openbench._registry"], - capture_output=True, text=True + capture_output=True, + text=True, ) if result.returncode != 0: @@ -45,4 +49,4 @@ def test_imports_for_optional_dependencies(): " import optional_package # type: ignore[import-untyped,import-not-found]\n" " except ImportError:\n" " optional_package = None" - ) \ No newline at end of file + ) diff --git a/tests/test_wave1_matharena.py b/tests/test_wave1_matharena.py new file mode 100644 index 00000000..1418fd43 --- /dev/null +++ b/tests/test_wave1_matharena.py @@ -0,0 +1,53 @@ +from unittest.mock import patch + +import pytest + +from openbench.config import BENCHMARKS, EVAL_GROUPS +from openbench.evals.matharena.aime_2026.aime_2026 import aime_2026 +from openbench.evals.matharena.hmmt_feb_2026.hmmt_feb_2026 import hmmt_feb_2026 +from openbench.evals.matharena.hmmt_nov_2025.hmmt_nov_2025 import hmmt_nov_2025 +from openbench.scorers.robust_boxed import _exact_arithmetic_value, extract_boxed_answer + + +@pytest.mark.parametrize( + ("factory", "path", "revision"), + [ + ( + aime_2026, + "MathArena/aime_2026", + "d2de22f3c656b4f56cf8981212186377d1e23bc3", + ), + ( + hmmt_nov_2025, + "MathArena/hmmt_nov_2025", + "118dbfb45c4c9467c672268ed55166642897aa46", + ), + ( + hmmt_feb_2026, + "MathArena/hmmt_feb_2026", + "02fba4f74d8e68e73e66a02d540fd979c05c274c", + ), + ], +) +def test_new_matharena_factories_are_pinned(factory, path, revision): + with patch( + f"{factory.__module__}.matharena_task", return_value=object() + ) as task_factory: + factory.__wrapped__() + + assert task_factory.call_args.kwargs["dataset_path"] == path + assert task_factory.call_args.kwargs["revision"] == revision + + +def test_new_matharena_registry_entries_and_group(): + expected = {"aime_2026", "hmmt_nov_2025", "hmmt_feb_2026"} + + assert expected <= BENCHMARKS.keys() + assert expected <= set(EVAL_GROUPS["matharena"].benchmarks) + + +def test_matharena_fraction_answer_parsing(): + answer = extract_boxed_answer(r"Reasoning... \boxed{-\frac{1}{21}}") + + assert answer == r"-\frac{1}{21}" + assert _exact_arithmetic_value(answer) == _exact_arithmetic_value("-1/21") diff --git a/uv.lock b/uv.lock index 1ac87d42..d841416d 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.13'", @@ -1423,7 +1423,7 @@ wheels = [ [[package]] name = "inspect-ai" -version = "0.3.141" +version = "0.3.142" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aioboto3" }, @@ -1462,9 +1462,9 @@ dependencies = [ { name = "universal-pathlib" }, { name = "zipp" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/69/d0/fcb7209b9e1b4a49a534de8bc7c12954830906756ff079782f0a2c80ae76/inspect_ai-0.3.141.tar.gz", hash = "sha256:4d82e289c8e4ea241a99c780a54ff5e3f546abdbe2a6c24dbbf4f53ffcf09631", size = 42787335, upload-time = "2025-10-27T11:57:05.748Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/5f/4eb973acb702e628067b5fc8ec6c6f97cf864c88c996efa18eef71184266/inspect_ai-0.3.142.tar.gz", hash = "sha256:9c0f5a0b128a26e297cc8781dcdabe0f60ab474dea0e2ef1918b9afc170e370c", size = 42790227, upload-time = "2025-10-27T13:36:53.623Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/43/afbed4ccb75c9864599ed571b5e0a06bb105e2d75f72e35d0f89e6761b12/inspect_ai-0.3.141-py3-none-any.whl", hash = "sha256:22339ee9619619770bca451274ff23af39a7e309ea412f2051adfdf2e7d7731d", size = 34133970, upload-time = "2025-10-27T11:56:56.99Z" }, + { url = "https://files.pythonhosted.org/packages/9c/96/2058c58fdeaf8a03b0bccd808a72b1529238875b5ecf2e31cbf715360e84/inspect_ai-0.3.142-py3-none-any.whl", hash = "sha256:d9b6499e06e0394fd264a854b3568f0975b081e1ced36f42c5a70264cc157401", size = 34135007, upload-time = "2025-10-27T13:36:45.135Z" }, ] [[package]] @@ -2541,7 +2541,7 @@ wheels = [ [[package]] name = "openbench" -version = "0.5.2" +version = "0.5.3" source = { editable = "." } dependencies = [ { name = "anthropic" }, @@ -2558,6 +2558,8 @@ dependencies = [ { name = "pydantic-settings" }, { name = "scipy" }, { name = "tiktoken" }, + { name = "tree-sitter" }, + { name = "tree-sitter-python" }, { name = "typer" }, ] @@ -2572,6 +2574,7 @@ deep-research-bench = [ ] dev = [ { name = "mypy" }, + { name = "pandas-stubs" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -2617,7 +2620,7 @@ requires-dist = [ { name = "anthropic", specifier = ">=0.69.0" }, { name = "datasets", specifier = ">=3.6.0" }, { name = "groq", specifier = ">=0.33.0" }, - { name = "inspect-ai", specifier = "==0.3.141" }, + { name = "inspect-ai", specifier = "==0.3.142" }, { name = "inspect-swe", specifier = ">=0.2.26" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "mcp", specifier = ">=1.13.1" }, @@ -2628,6 +2631,8 @@ requires-dist = [ { name = "pydantic-settings", specifier = ">=2.9.1" }, { name = "scipy", specifier = ">=1.15.3" }, { name = "tiktoken", specifier = ">=0.11.0" }, + { name = "tree-sitter", specifier = ">=0.25.2" }, + { name = "tree-sitter-python", specifier = ">=0.25.0" }, { name = "typer", specifier = ">=0.15.3" }, ] @@ -2642,6 +2647,7 @@ deep-research-bench = [ ] dev = [ { name = "mypy", specifier = ">=1.15.0" }, + { name = "pandas-stubs", specifier = ">=2.3.3.260113,<3" }, { name = "pre-commit", specifier = ">=4.2.0" }, { name = "pytest", specifier = ">=8.3.5" }, { name = "pytest-asyncio", specifier = "==0.24.0" }, @@ -2822,6 +2828,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/f9/07086f5b0f2a19872554abeea7658200824f5835c58a106fa8f2ae96a46c/pandas-2.3.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5db9637dbc24b631ff3707269ae4559bce4b7fd75c1c4d7e13f40edc42df4444", size = 13189044, upload-time = "2025-07-07T19:19:39.999Z" }, ] +[[package]] +name = "pandas-stubs" +version = "2.3.3.260113" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "types-pytz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/5d/be23854a73fda69f1dbdda7bc10fbd6f930bd1fa87aaec389f00c901c1e8/pandas_stubs-2.3.3.260113.tar.gz", hash = "sha256:076e3724bcaa73de78932b012ec64b3010463d377fa63116f4e6850643d93800", size = 116131, upload-time = "2026-01-13T22:30:16.704Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/c6/df1fe324248424f77b89371116dab5243db7f052c32cc9fe7442ad9c5f75/pandas_stubs-2.3.3.260113-py3-none-any.whl", hash = "sha256:ec070b5c576e1badf12544ae50385872f0631fc35d99d00dc598c2954ec564d3", size = 168246, upload-time = "2026-01-13T22:30:15.244Z" }, +] + [[package]] name = "pathlib-abc" version = "0.5.2" @@ -4250,6 +4269,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, ] +[[package]] +name = "tree-sitter" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/03/5600b84aff2e6c4fe80cfebb4063fe2f50299521befe5f6092ab8c082f4a/tree_sitter-0.26.0.tar.gz", hash = "sha256:b40c219edccc4564530c96f8f1556f6202b37cda964d1cbd7bd2b7e68b40a245", size = 191423, upload-time = "2026-06-30T12:14:27.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/2f/201c33ea65875d8e4ec73e4d1949718ec49780d84c0adf19793ef75d99a2/tree_sitter-0.26.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ff527388df14cb5009f9274faf78cc69a7393ae6acf3b04784b8acca249519c5", size = 148676, upload-time = "2026-06-30T12:13:42.718Z" }, + { url = "https://files.pythonhosted.org/packages/9e/db/05b9d45dd2b9827bf91b6819e749227ca6d686d58658292c0f149294b18e/tree_sitter-0.26.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7bcbadfa614326debef581957d5c780a9d7f66065c13deea61aa21d1dd36263f", size = 140757, upload-time = "2026-06-30T12:13:44.007Z" }, + { url = "https://files.pythonhosted.org/packages/b9/08/1e1da65c1585b8d70130b26d65b41a71737ab623c1fab1008479c2b95b50/tree_sitter-0.26.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f941cea06128c1f74f8937a8e2a90c7db49cf4be6647cd9e07d92a306d91517", size = 631526, upload-time = "2026-06-30T12:13:45.008Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b7/06353044a80ee58a71e884b4a9b2913705849d81025d87308abdfef8f883/tree_sitter-0.26.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e9e46b664887d8c1014f1fb33e09454bbdd9ec1fe29b7fd02dde7b46bc1bb81a", size = 658688, upload-time = "2026-06-30T12:13:46.491Z" }, + { url = "https://files.pythonhosted.org/packages/df/56/c4b22ccbc4f89ae507c0b76e29f363ad4f16eb38c43f7392b3eb9afec64e/tree_sitter-0.26.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:763627db05db34f12333081bd7422cc1c675893d373cc870b3e9249e200700e4", size = 644399, upload-time = "2026-06-30T12:13:47.719Z" }, + { url = "https://files.pythonhosted.org/packages/b5/b7/6b3f0192d5b9b49a199cb0dcd5e45dd1327a82c52c80a49edd790e3a2d9b/tree_sitter-0.26.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:17a1c5cfd3a05d5c7c86bf4282b6ef8092c91dc0a98390499669c3fedb7d1814", size = 655316, upload-time = "2026-06-30T12:13:49.03Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6b/f7475c8f8d699671c2a80c3ed16f5cddd161280c6ed5b845117179c66075/tree_sitter-0.26.0-cp310-cp310-win_amd64.whl", hash = "sha256:f289be0225ba2ace8e87d6c9639b2bc9ff2b5271afb7c5d39282a4a00e248682", size = 129494, upload-time = "2026-06-30T12:13:50.242Z" }, + { url = "https://files.pythonhosted.org/packages/f6/20/0df8dd708638cba7ef875fff4ce80122af7f604f1f0b566de2164108bc01/tree_sitter-0.26.0-cp310-cp310-win_arm64.whl", hash = "sha256:526a165a2cb1d1f79e247d400f0e0acd8d49a817d6f312d543513af200b1f886", size = 116486, upload-time = "2026-06-30T12:13:51.21Z" }, + { url = "https://files.pythonhosted.org/packages/41/18/78aae7e4b5a36daaebb0276e4b07d084d45298758000787838e89329e11f/tree_sitter-0.26.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1d6fe0e8fb4df77b5ee816228e2c4475a63d8cc1d4d3a7ffd7097b2b87fc3e95", size = 148679, upload-time = "2026-06-30T12:13:52.27Z" }, + { url = "https://files.pythonhosted.org/packages/24/e4/b371b9553b0e47d130fc2073e56cab94fecc868be04666bf5bbd1fcd1cc9/tree_sitter-0.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:514a9bf8993e5210e7970736aaf6020d1759b670e195ef17b1c48f586aa30736", size = 140759, upload-time = "2026-06-30T12:13:53.221Z" }, + { url = "https://files.pythonhosted.org/packages/22/7d/266fb0f2c41e6fb00b0f40e7a3338cdf99651e6a6511ca72bc78fc697636/tree_sitter-0.26.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10f0d4eb94aa7242dcb7f554bcd24dd7ba1c114f00d58759ba08c7a46c8ec51a", size = 637206, upload-time = "2026-06-30T12:13:54.334Z" }, + { url = "https://files.pythonhosted.org/packages/40/9f/47cf22febb47132d5b3a507a27bb99ef89fe5c8ec420a13c6daa9b64f782/tree_sitter-0.26.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:335294ce0504fcefde5245dff596778ffaf820205b98ae0b549c72e48855f1d8", size = 664758, upload-time = "2026-06-30T12:13:55.42Z" }, + { url = "https://files.pythonhosted.org/packages/4c/4d/8d144ca3beb46a62a5102b6deac76bb0da55235c2c7840faf3b12f2e9d97/tree_sitter-0.26.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9997ba61368c48ed54e715676afadf703947a1542464e39d047764fb3624b01", size = 647438, upload-time = "2026-06-30T12:13:56.523Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ed/ed1d6e78520c4fb64ed52fec3f2947bf8c1fbad7bc24e282c56193c9ba42/tree_sitter-0.26.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c56581ad256c4195a21bfe449fed5d44a02fe83a4a7d6e70e6ec302c881191c7", size = 661944, upload-time = "2026-06-30T12:13:57.82Z" }, + { url = "https://files.pythonhosted.org/packages/10/83/45f5bd43db1b8248d2fd08ef6cbe43e2725c539e09a2cfb8bc2818646788/tree_sitter-0.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:0f8793fd18ad7eec276ed4b51c097b4bf2002b357259b66b0d75db1f3f41c754", size = 129496, upload-time = "2026-06-30T12:13:59.216Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/be68e6c04563eb54145424cc83fe0aa8b0ba6c90d8989cf8a032671b5f16/tree_sitter-0.26.0-cp311-cp311-win_arm64.whl", hash = "sha256:dea4b4e27d49e9ec5b785d4f994da000e6726882fcc6ad05ec98478500c71aef", size = 116484, upload-time = "2026-06-30T12:14:00.147Z" }, + { url = "https://files.pythonhosted.org/packages/87/ca/565702c44815393e3a973552ad546db4e5ca081ca8698640b4e93d809f51/tree_sitter-0.26.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6cb2bd20efb2544c19ac54486ab7cb8ec7b36f913bbe1ce95df84acb96743d9c", size = 148934, upload-time = "2026-06-30T12:14:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/54/6f/8bb61957f16ec1b1d92410a006cdc84a952b6352a7313b2ad299f2d21484/tree_sitter-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:918d89529786873f0982a0f59c2a303cd065fbfd1b903d71a8e4e1584f67b42e", size = 140820, upload-time = "2026-06-30T12:14:02.087Z" }, + { url = "https://files.pythonhosted.org/packages/78/0a/8a6f08559182643a814a4ab559948ae817b2851890fd9b995a4fff6541ce/tree_sitter-0.26.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30a88be89ff1f2755297f81e8080d88b795dd98720c3f9fa2acf93873182cc95", size = 638844, upload-time = "2026-06-30T12:14:03.428Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2f/6e6781b31677231366cb3cf27bc8269157f6d4b03c9032865a4f5f2bbe7e/tree_sitter-0.26.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a6b333b0282d8bb0af741f9b018bd2523d4eecb2686bf6717066a625fecfaa4", size = 667487, upload-time = "2026-06-30T12:14:04.669Z" }, + { url = "https://files.pythonhosted.org/packages/02/0b/0483078c8567445557a7015b0e5b187f6d7d4fda73464df9c4bdea7f7f3c/tree_sitter-0.26.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f3c44339dd34fe8eb2b8d5aa7610660499a795f70376b130bbee7a437337280", size = 647975, upload-time = "2026-06-30T12:14:05.797Z" }, + { url = "https://files.pythonhosted.org/packages/27/68/da83ca72c984e96ab4eb3bee0db1a6ffb5de1c8c455f92bd9f420cde7f0e/tree_sitter-0.26.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:94550e13b6ae576969da40246f4c4abb206380b5375ad43f26dd9151d55438e3", size = 665018, upload-time = "2026-06-30T12:14:07.278Z" }, + { url = "https://files.pythonhosted.org/packages/d1/36/4d67927fd47b89af4a00f65f55a7370e28778cd50e972c2430487e3ecc27/tree_sitter-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca89e361a276dbc934b28a43dd881199e25d34ff5493ee0ce45f3c52a6124a37", size = 129619, upload-time = "2026-06-30T12:14:08.373Z" }, + { url = "https://files.pythonhosted.org/packages/ed/72/cdefad523eb78710679c6da6a79e3d90f5afd32b1c6aa5a17bac7eef99f6/tree_sitter-0.26.0-cp312-cp312-win_arm64.whl", hash = "sha256:bc6cb01d5ee75c85424aa1f1c72a82d8f07fd52539a0f3c4a6ed3e8721079b84", size = 116545, upload-time = "2026-06-30T12:14:09.273Z" }, + { url = "https://files.pythonhosted.org/packages/cb/b0/465257cf8f972ad9f9812ec1cbaa8ec210ebebb601ade9a15881aa2436b4/tree_sitter-0.26.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ed0889dbed843ce45ede9f5169c0b2dea2222f12685844a03fadb81f12705867", size = 148893, upload-time = "2026-06-30T12:14:10.541Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ec/19d093e854b45e807fecfdd26105c266f43aeecc39c4dc97992a7074ad5a/tree_sitter-0.26.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6189c6c340c7384357711e3d92645e96bfb79f7a502f86de1ebdb23eb43f7dab", size = 140829, upload-time = "2026-06-30T12:14:11.626Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ee/87e74671ed63a837e7a1f17ab94aa3913871e033b27523d8e7b83d6f7ad0/tree_sitter-0.26.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ff2e0750b7daa722302838356d7b65e303829b7eb73c915df127ddba115e1d1", size = 639334, upload-time = "2026-06-30T12:14:12.836Z" }, + { url = "https://files.pythonhosted.org/packages/66/e7/f7e04cd9dff6b6ac0adf23922796fbc76accd4cf4bcda50542748d485679/tree_sitter-0.26.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7075ef857ef86f327dbb72d1e2574dda78db5754b3a1fca6506acd7fe5d561a7", size = 668102, upload-time = "2026-06-30T12:14:14.035Z" }, + { url = "https://files.pythonhosted.org/packages/d3/90/0bfb16b7894fea728c774a89d5af421a9368a2f913bbd4e8dcab7caaecfb/tree_sitter-0.26.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:26c996c1edfee86e977bb3f5462e74fcec0d0b0db1e85a3c475875763caa03be", size = 648560, upload-time = "2026-06-30T12:14:15.302Z" }, + { url = "https://files.pythonhosted.org/packages/cd/e6/0fe05ba396e9623b0ae40ccf34171336b8701ec8d7bd0ee9f5224d638665/tree_sitter-0.26.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00289bfe7978f3e0dc0ce69813a20fa9f44ea4c100b3ec62043e5eb74ccfc3a2", size = 665121, upload-time = "2026-06-30T12:14:16.403Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/a944b1ca35bed6068dc84a9967aaf3049d8cc0b7a36179eea8787270a6ab/tree_sitter-0.26.0-cp313-cp313-win_amd64.whl", hash = "sha256:93e220cab7e6a823efeb2046c49171427de92ef71c7c681c01820d14d8d3721f", size = 129615, upload-time = "2026-06-30T12:14:17.463Z" }, + { url = "https://files.pythonhosted.org/packages/09/ef/c7ca48293580d2249f36940c4eed5b4ddeb9ce75baf9a4ef30621987e0c7/tree_sitter-0.26.0-cp313-cp313-win_arm64.whl", hash = "sha256:b31a8195d2f224224c530ac814632d98c1dcc123d227442c07c736e86b70d564", size = 116525, upload-time = "2026-06-30T12:14:18.53Z" }, + { url = "https://files.pythonhosted.org/packages/c5/7a/4d84e6f6ae2c3e757490dd84de251712c31e293dfe31f28da1ec019cefa2/tree_sitter-0.26.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5a3c93a352b7e6f70f73e121bbfa2d0117ba7478bd51114ed35c91b0b78814fa", size = 148901, upload-time = "2026-06-30T12:14:19.452Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d9/efe62ec65dc9d096e834d27b8c058127e2146e42ff3380b822a233f016a6/tree_sitter-0.26.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5fc2f41bf246ff2f70a9cc3690be35ec7580a4923151873d898c8bcb1a4503d3", size = 140805, upload-time = "2026-06-30T12:14:20.478Z" }, + { url = "https://files.pythonhosted.org/packages/c4/2c/c82326b7b97e3c485c18679883b16f89e5e913c639d3b219d3da70c9e67e/tree_sitter-0.26.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8ea92a255c91671a7ec4625aba3ab7bb5220c423630ffbf83c45d7312abe084", size = 640586, upload-time = "2026-06-30T12:14:21.527Z" }, + { url = "https://files.pythonhosted.org/packages/e2/7a/f56e7d8282859452611024c7cbc623bfba5b24b8cb9b8f8bc88c5219fe9a/tree_sitter-0.26.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f665510f0fcf4636fb9696f1f7853bed7a3bd764b7bb0cb8494e619c14ed5a0c", size = 668300, upload-time = "2026-06-30T12:14:22.728Z" }, + { url = "https://files.pythonhosted.org/packages/91/51/240ee81b9d5e9ca0a6cb1528e8605ffa70ab58c89ce126631be96d3e4bae/tree_sitter-0.26.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:253df7ab82cc0a9d311cd65f06e9f99fb3eac55996ae9fc94da22f123a861b90", size = 649627, upload-time = "2026-06-30T12:14:23.819Z" }, + { url = "https://files.pythonhosted.org/packages/6a/54/760035cefedf9eb44f0f84c4ac22f1322e73155853e272576ee876336312/tree_sitter-0.26.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ff80d4833d330a73184a3ac5132abe93c575d2dea31975c6f15c0d21fef238aa", size = 664885, upload-time = "2026-06-30T12:14:25.064Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1b/0b36fe2a984ecedc4ce6aefd5d56447a6626a8e9b595c4e48658510ce8f8/tree_sitter-0.26.0-cp314-cp314-win_amd64.whl", hash = "sha256:a4033fecc8f606c7f2e8b8014d0057b74668a7f0152763606f7bc25c5f9ec64c", size = 132688, upload-time = "2026-06-30T12:14:26.106Z" }, + { url = "https://files.pythonhosted.org/packages/4d/74/ebc041a13fbf40144afdb0d4b447e48e0b4012ca866c63de8b48f801f0c1/tree_sitter-0.26.0-cp314-cp314-win_arm64.whl", hash = "sha256:823251c4b6725a7c03ed497a339135ede7ae4bdde75bb8be7ef5e305aeb4ff52", size = 120287, upload-time = "2026-06-30T12:14:26.991Z" }, +] + +[[package]] +name = "tree-sitter-python" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b8/8b/c992ff0e768cb6768d5c96234579bf8842b3a633db641455d86dd30d5dac/tree_sitter_python-0.25.0.tar.gz", hash = "sha256:b13e090f725f5b9c86aa455a268553c65cadf325471ad5b65cd29cac8a1a68ac", size = 159845, upload-time = "2025-09-11T06:47:58.159Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/64/a4e503c78a4eb3ac46d8e72a29c1b1237fa85238d8e972b063e0751f5a94/tree_sitter_python-0.25.0-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:14a79a47ddef72f987d5a2c122d148a812169d7484ff5c75a3db9609d419f361", size = 73790, upload-time = "2025-09-11T06:47:47.652Z" }, + { url = "https://files.pythonhosted.org/packages/e6/1d/60d8c2a0cc63d6ec4ba4e99ce61b802d2e39ef9db799bdf2a8f932a6cd4b/tree_sitter_python-0.25.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:480c21dbd995b7fe44813e741d71fed10ba695e7caab627fb034e3828469d762", size = 76691, upload-time = "2025-09-11T06:47:49.038Z" }, + { url = "https://files.pythonhosted.org/packages/aa/cb/d9b0b67d037922d60cbe0359e0c86457c2da721bc714381a63e2c8e35eba/tree_sitter_python-0.25.0-cp310-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:86f118e5eecad616ecdb81d171a36dde9bef5a0b21ed71ea9c3e390813c3baf5", size = 108133, upload-time = "2025-09-11T06:47:50.499Z" }, + { url = "https://files.pythonhosted.org/packages/40/bd/bf4787f57e6b2860f3f1c8c62f045b39fb32d6bac4b53d7a9e66de968440/tree_sitter_python-0.25.0-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be71650ca2b93b6e9649e5d65c6811aad87a7614c8c1003246b303f6b150f61b", size = 110603, upload-time = "2025-09-11T06:47:51.985Z" }, + { url = "https://files.pythonhosted.org/packages/5d/25/feff09f5c2f32484fbce15db8b49455c7572346ce61a699a41972dea7318/tree_sitter_python-0.25.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6d5b5799628cc0f24691ab2a172a8e676f668fe90dc60468bee14084a35c16d", size = 108998, upload-time = "2025-09-11T06:47:53.046Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/4946da3d6c0df316ccb938316ce007fb565d08f89d02d854f2d308f0309f/tree_sitter_python-0.25.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:71959832fc5d9642e52c11f2f7d79ae520b461e63334927e93ca46cd61cd9683", size = 107268, upload-time = "2025-09-11T06:47:54.388Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a2/996fc2dfa1076dc460d3e2f3c75974ea4b8f02f6bc925383aaae519920e8/tree_sitter_python-0.25.0-cp310-abi3-win_amd64.whl", hash = "sha256:9bcde33f18792de54ee579b00e1b4fe186b7926825444766f849bf7181793a76", size = 76073, upload-time = "2025-09-11T06:47:55.773Z" }, + { url = "https://files.pythonhosted.org/packages/07/19/4b5569d9b1ebebb5907d11554a96ef3fa09364a30fcfabeff587495b512f/tree_sitter_python-0.25.0-cp310-abi3-win_arm64.whl", hash = "sha256:0fbf6a3774ad7e89ee891851204c2e2c47e12b63a5edbe2e9156997731c128bb", size = 74169, upload-time = "2025-09-11T06:47:56.747Z" }, +] + [[package]] name = "typer" version = "0.15.3" @@ -4277,6 +4360,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/96/b8/c92fa726786f503a95b02033f1739404c49db9d2a7b56ff50207a22cdb9a/types_jsonschema-4.25.1.20250821-py3-none-any.whl", hash = "sha256:73e22048556bfd097d58883b203ff298a2dbece0d76b35115dbee0dad9b02bab", size = 15798, upload-time = "2025-08-21T03:01:50.848Z" }, ] +[[package]] +name = "types-pytz" +version = "2026.3.1.20260727" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/cf/eae96172a036b942e0ad0ec49512108b4ef4b97cd6c2aed5677540a597e7/types_pytz-2026.3.1.20260727.tar.gz", hash = "sha256:4364075b6867dd15b210bb8c1d29727d609917129b45600defe1d4b3eda5ecb9", size = 10914, upload-time = "2026-07-27T05:36:32.389Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/24/a654176875944981c75516857cd8750600eeb9ff7fc07a2b82573619a57c/types_pytz-2026.3.1.20260727-py3-none-any.whl", hash = "sha256:42ac44e83645bfceb46597342c8fb8ec028a1407e2feae9c5c539986eab6b57a", size = 10132, upload-time = "2026-07-27T05:36:31.52Z" }, +] + [[package]] name = "types-pyyaml" version = "6.0.12.20250809"