diff --git a/benchmark/bfcl-v4/README.md b/benchmark/bfcl-v4/README.md index 81de0bfc..3a96a5b2 100644 --- a/benchmark/bfcl-v4/README.md +++ b/benchmark/bfcl-v4/README.md @@ -1,12 +1,14 @@ # BFCL v4 integration contract -OpenBench exposes three independently named BFCL v4 sections and one offline +OpenBench exposes four independently named BFCL v4 sections and one offline aggregate: - `bfcl_v4_single_turn`: all 13 official single-turn categories; - `bfcl_v4_multi_turn`: all four official stateful categories; - `bfcl_v4_agentic_offline`: three memory backends and two frozen web-search configurations; +- `bfcl_v4_agentic_live`: BFCL's model-generated memory prerequisites and live + SERPAPI web backends; - `bfcl_v4_offline`: all 5,106 samples with BFCL's 10/10/10/30/40 weights. ## Provenance @@ -44,17 +46,39 @@ The multi-turn task runs BFCL's pinned official state and response checker in a network-disabled, read-only Docker sandbox. It covers base, missing-function, missing-parameter, and long-context categories. -The agentic task is intentionally an offline adaptation. Memory retrieval is +The offline agentic task is intentionally an adaptation. Memory retrieval is initialized from the public BFCL source record, while web search uses a frozen corpus derived from BFCL's cited evidence. The no-snippet mode exposes URLs only until the model calls `fetch_url_content`. This removes SERPAPI credentials and web drift from CI, but it is not numerically interchangeable with the live leaderboard environment. +The separate `bfcl_v4_agentic_live` task preserves the official agentic +protocol: + +- each model generates all 37 prerequisite conversations once for each of the + three memory backends; +- the resulting KV, vector, or recursive-summary state is reused for all 155 + questions without leaking one target conversation into another; +- `all-MiniLM-L6-v2` is pinned at revision + `1110a243fdf4706b3f48f1d95db1a4f5529b4d41` and all Python dependencies are + locked; +- the 100 web questions run in both snippet and no-snippet modes against + BFCL's DuckDuckGo SerpAPI backend. + +The live task contains five workflow samples but reports 665 logical case +outcomes. It requires `SERPAPI_API_KEY` for web categories. A run can record +content-addressable, model-trace-specific web evidence with +`-T record_web_snapshot_dir=PATH`, then replay that exact trace without network +using `-T web_snapshot_dir=PATH`. Replay fails closed if a model changes any +tool call or step count; a recorded trace is therefore an audit artifact, not a +universal replacement for live search. + Consequently, `bfcl_v4_offline` applies the official section weights but does -not claim the official leaderboard score. The unqualified `bfcl_v4` alias stays -reserved until a versioned live-search snapshot and the exact official memory -prerequisite pipeline can run reproducibly. +not claim the official leaderboard score. The memory prerequisite pipeline and +live-search transport now exist, but the unqualified `bfcl_v4` alias remains +reserved until two credentialed provider runs and differential score comparison +against the upstream harness demonstrate parity. The older “Berkeley Function Calling” label in the Phi model card is not aliased to this task: without a harness/version citation it may refer to BFCL v1 rather diff --git a/benchmark/bfcl-v4/live_parity.md b/benchmark/bfcl-v4/live_parity.md new file mode 100644 index 00000000..d66918cd --- /dev/null +++ b/benchmark/bfcl-v4/live_parity.md @@ -0,0 +1,36 @@ +# BFCL v4 live parity decision + +## Implemented + +- Upstream code and datasets are pinned to + `6ea57973c7a6097fd7c5915698c54c17c5b1b6c8`. +- All five memory prerequisite conversations are checksum-verified. +- Prerequisite calls are generated once per model, scenario, and backend. +- Target questions start from an isolated replay of the prerequisite state. +- The official KV, vector, recursive-summary, SerpAPI, URL-fetch, and answer + checker implementations run inside a dedicated Docker boundary. +- The vector encoder and the complete Linux dependency graph are pinned. +- Live web traces can be recorded and replayed content-exactly with networking + disabled. + +## Evidence + +- 155 logical cases load for each memory backend and 100 for each web mode. +- KV, vector, and recursive-summary backends were executed in the live image; + each produced the official state-derived system prompt. +- A persistent KV prerequisite followed by a target retrieval scored 1.0 in an + end-to-end Inspect run. +- A recorded web-search trace replayed in the network-disabled image and scored + 1.0; mismatched calls and unused steps fail closed. + +## Alias decision + +Do not expose `bfcl_v4` yet. No real provider credentials were available in the +execution environment, so the required two-provider transport run and +model-output differential against the upstream harness could not be performed. +In addition, live SerpAPI results and fetched pages are intrinsically mutable; +recorded snapshots prove a particular run but are not a provider-independent +official environment. + +The supported live component is named `bfcl_v4_agentic_live`. The existing +`bfcl_v4_offline` remains the reproducible full-coverage diagnostic aggregate. diff --git a/benchmark/bfcl-v4/matrix.tsv b/benchmark/bfcl-v4/matrix.tsv index a2cbacae..f9e8d6d2 100644 --- a/benchmark/bfcl-v4/matrix.tsv +++ b/benchmark/bfcl-v4/matrix.tsv @@ -10,3 +10,9 @@ bfcl_v4_agentic_offline pinned_dataset 1 665 5 done Three memory backends and tw bfcl_v4_offline weighted_aggregate 1 5106 5 done Official 10/10/10/30/40 section weights applied; explicitly not labeled as the live official score bfcl_v4_multi_turn mock_transport 1 1 4.41 done End-to-end Inspect tool transport, Docker startup, solver, and scorer completed bfcl_v4_agentic_offline mock_transport 1 1 0.02 done End-to-end frozen agentic solver and scorer completed +bfcl_v4_agentic_live pinned_workflows 1 665 5 done 37 model-specific memory prerequisites, 465 memory targets, and 200 live web targets grouped into five race-free workflows +bfcl_v4_agentic_live memory_backend_smoke 1 3 1 done KV, all-MiniLM-L6-v2 vector, and recursive-summary backends produced official state-derived prompts in Docker +bfcl_v4_agentic_live persistent_memory_transport 1 1 1 done Inspect mock model wrote prerequisite state, retrieved it in an isolated target conversation, and scored 1.0 +bfcl_v4_agentic_live web_snapshot_replay 1 1 1 done Exact recorded tool trace replayed in a network-disabled image and scored 1.0 +bfcl_v4_agentic_live two_provider_transport 1 0 0 blocked No provider credentials or configured local model were available; no real-model score is claimed +bfcl_v4 official_alias_gate 1 0 0 blocked Alias withheld pending two-provider runs and differential output scoring against the upstream harness diff --git a/docs/snippets/benchmarks.data.mdx b/docs/snippets/benchmarks.data.mdx index ccc2c222..a4c2dd46 100644 --- a/docs/snippets/benchmarks.data.mdx +++ b/docs/snippets/benchmarks.data.mdx @@ -1571,6 +1571,21 @@ export const benchmarksData = [ "function_name": "bbq_ses", "is_alpha": false }, + { + "name": "BFCL v4 Agentic Live", + "description": "Official model-specific BFCL memory pipelines and live SERPAPI web search", + "category": "core", + "tags": [ + "function-calling", + "tools", + "bfcl", + "agentic", + "live", + "docker" + ], + "function_name": "bfcl_v4_agentic_live", + "is_alpha": false + }, { "name": "BFCL v4 Agentic Offline", "description": "Reproducible offline BFCL v4 memory and frozen web-search adaptation", diff --git a/packages/openbench-core/pyproject.toml b/packages/openbench-core/pyproject.toml index 060c6562..d2d13b5a 100644 --- a/packages/openbench-core/pyproject.toml +++ b/packages/openbench-core/pyproject.toml @@ -51,7 +51,16 @@ include = ["openbench*"] [tool.setuptools.package-data] "openbench.evals.livecodebench" = ["Dockerfile", "compose.yaml"] "openbench.evals.evalplus" = ["Dockerfile", "compose.yaml"] -"openbench.evals.bfcl" = ["Dockerfile", "compose.yaml", "runner.py"] +"openbench.evals.bfcl" = [ + "Dockerfile", + "Dockerfile.agentic", + "compose.yaml", + "compose.live.yaml", + "compose.replay.yaml", + "requirements-agentic.in", + "requirements-agentic.txt", + "runner.py", +] [tool.setuptools.exclude-package-data] "*" = ["__pycache__/*", "*.pyc"] diff --git a/pyproject.toml b/pyproject.toml index 57e97c54..3bb7a640 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,16 @@ include = ["openbench*"] [tool.setuptools.package-data] "openbench.evals.livecodebench" = ["Dockerfile", "compose.yaml"] "openbench.evals.evalplus" = ["Dockerfile", "compose.yaml"] -"openbench.evals.bfcl" = ["Dockerfile", "compose.yaml", "runner.py"] +"openbench.evals.bfcl" = [ + "Dockerfile", + "Dockerfile.agentic", + "compose.yaml", + "compose.live.yaml", + "compose.replay.yaml", + "requirements-agentic.in", + "requirements-agentic.txt", + "runner.py", +] [tool.setuptools.exclude-package-data] "*" = ["__pycache__/*", "*.pyc"] diff --git a/src/openbench/config.py b/src/openbench/config.py index 8119f6b2..6cd31b3d 100644 --- a/src/openbench/config.py +++ b/src/openbench/config.py @@ -124,6 +124,15 @@ class EvalGroup: function_name="bfcl_v4_agentic_offline", is_alpha=False, ), + "bfcl_v4_agentic_live": BenchmarkMetadata( + name="BFCL v4 Agentic Live", + description="Official model-specific BFCL memory pipelines and live SERPAPI web search", + category="core", + tags=["function-calling", "tools", "bfcl", "agentic", "live", "docker"], + module_path="openbench.evals.bfcl", + function_name="bfcl_v4_agentic_live", + is_alpha=False, + ), "bfcl_v4_offline": BenchmarkMetadata( name="BFCL v4 Offline Aggregate", description="All BFCL v4 sections with official weights and frozen agentic evidence", diff --git a/src/openbench/datasets/bfcl.py b/src/openbench/datasets/bfcl.py index bcbc129c..e4b0d059 100644 --- a/src/openbench/datasets/bfcl.py +++ b/src/openbench/datasets/bfcl.py @@ -50,6 +50,8 @@ "web_search_no_snippet", ) +MEMORY_SCENARIOS = ("customer", "finance", "healthcare", "notetaker", "student") + _MULTI_TURN_COUNT = 200 _FUNCTION_DOCS = { @@ -115,6 +117,11 @@ "multi_turn_func_doc/travel_booking.json": "f17b950c13adddf41d0848077df58788252e4c2e7cad5cfa71c8c4bf04f57b26", "multi_turn_func_doc/vehicle_control.json": "0c8a66292844874ef7b168f343bc394d8615d2d9e1f4387999a9ee23011eac78", "multi_turn_func_doc/web_search.json": "61fcee411e35f7ff67415e18cd67276615cf06e1c8841a683d2d997dbb46eac5", + "memory_prereq_conversation/memory_customer.json": "806ac91e558e3d933e526ecc0c286e417657f76a3c9b9073eef830a532eb4eb2", + "memory_prereq_conversation/memory_finance.json": "3cef1f235667f479b8f45c7082247454b2129ee88fe4fbcdbc53fd2cdf495b75", + "memory_prereq_conversation/memory_healthcare.json": "56e7bd7b2e7c01efe81f1e0be85149ea3886bb2d9cb59f90e9d9e27b91f4604d", + "memory_prereq_conversation/memory_notetaker.json": "07f3239ef2e9e1421901f05d3c8192469fa129324e124a47bcb35ff9c7785c91", + "memory_prereq_conversation/memory_student.json": "e2d6c32e9e2671c61f682e845dcdafd9eee7c9f3e2d5da1615d3afd2aeecf2f0", } _COUNTS = { @@ -347,3 +354,99 @@ def get_bfcl_v4_agentic_dataset( ) ) return MemoryDataset(samples=samples, name="bfcl_v4_agentic_offline") + + +def get_bfcl_v4_agentic_live_dataset( + categories: list[str] | tuple[str, ...] | None = None, +) -> MemoryDataset: + """Load BFCL's live agentic workflows, grouped by backend. + + Each dataset sample represents one complete backend evaluation. Grouping keeps + the official memory prerequisite chain model-specific and prevents Inspect's + sample parallelism from racing the shared scenario snapshots. + """ + + selected = tuple(categories or AGENTIC_CATEGORIES) + unknown = set(selected) - set(AGENTIC_CATEGORIES) + if unknown: + raise ValueError(f"Unsupported BFCL live agentic categories: {sorted(unknown)}") + + memory_questions = _load_jsonl("BFCL_v4_memory.json") + memory_answers = { + str(row["id"]): list(row["ground_truth"]) + for row in _load_jsonl("possible_answer/BFCL_v4_memory.json") + } + web_questions = _load_jsonl("BFCL_v4_web_search.json") + web_answers = { + str(row["id"]): list(row["ground_truth"]) + for row in _load_jsonl("possible_answer/BFCL_v4_web_search.json") + } + prerequisites = { + scenario: _load_jsonl(f"memory_prereq_conversation/memory_{scenario}.json") + for scenario in MEMORY_SCENARIOS + } + + samples: list[Sample] = [] + for category in selected: + is_memory = category.startswith("memory_") + if is_memory: + backend = category.removeprefix("memory_") + class_name = f"MemoryAPI_{backend}" + scenarios = [] + for scenario in MEMORY_SCENARIOS: + cases = [ + { + "id": str(question["id"]).replace("memory", category), + "question": question["question"], + "expected_answers": memory_answers[str(question["id"])], + } + for question in memory_questions + if question["scenario"] == scenario + ] + scenarios.append( + { + "name": scenario, + "prerequisites": prerequisites[scenario], + "cases": cases, + } + ) + workflow = {"scenarios": scenarios} + case_count = len(memory_questions) + else: + class_name = "WebSearchAPI" + workflow = { + "cases": [ + { + "id": str(question["id"]).replace("web_search", category), + "question": question["question"], + "expected_answers": web_answers[str(question["id"])], + } + for question in web_questions + ] + } + case_count = len(web_questions) + + samples.append( + Sample( + id=f"{category}_workflow", + input=[ + ChatMessageUser( + content=f"Run the pinned BFCL v4 {category} workflow." + ) + ], + target="", + metadata={ + "category": category, + "functions": _load_function_docs([class_name]), + "involved_classes": [class_name], + "workflow": workflow, + "case_count": case_count, + "show_snippet": category != "web_search_no_snippet", + "bfcl_revision": BFCL_REVISION, + "license": BFCL_LICENSE, + "live_agentic": True, + }, + ) + ) + + return MemoryDataset(samples=samples, name="bfcl_v4_agentic_live") diff --git a/src/openbench/evals/bfcl/Dockerfile b/src/openbench/evals/bfcl/Dockerfile index 3cc16467..89c2a19b 100644 --- a/src/openbench/evals/bfcl/Dockerfile +++ b/src/openbench/evals/bfcl/Dockerfile @@ -2,9 +2,13 @@ FROM python:3.11-slim@sha256:94c50be2dc994b873b55bc123e95e6dbade08095b3dfd790f51 RUN apt-get update \ && apt-get install --yes --no-install-recommends git ca-certificates \ - && pip install --no-cache-dir --no-deps \ + && pip install --no-cache-dir \ + build==1.5.0 setuptools==83.0.0 setuptools-scm==10.2.1 wheel==0.47.0 \ + && pip install --no-cache-dir --no-build-isolation --no-deps \ "bfcl_eval @ git+https://github.com/ShishirPatil/gorilla.git@6ea57973c7a6097fd7c5915698c54c17c5b1b6c8#subdirectory=berkeley-function-call-leaderboard" \ - && pip install --no-cache-dir mpmath==1.3.0 \ + && pip install --no-cache-dir \ + filelock==3.20.0 mpmath==1.3.0 numpy==1.26.4 overrides==7.7.0 \ + rank-bm25==0.2.2 \ && apt-get purge --yes --auto-remove git \ && rm -rf /var/lib/apt/lists/* diff --git a/src/openbench/evals/bfcl/Dockerfile.agentic b/src/openbench/evals/bfcl/Dockerfile.agentic new file mode 100644 index 00000000..dfb94c48 --- /dev/null +++ b/src/openbench/evals/bfcl/Dockerfile.agentic @@ -0,0 +1,37 @@ +FROM python:3.11-slim@sha256:94c50be2dc994b873b55bc123e95e6dbade08095b3dfd790f51c34de3f08cbb7 + +ARG BFCL_REVISION=6ea57973c7a6097fd7c5915698c54c17c5b1b6c8 +ARG EMBEDDING_REVISION=1110a243fdf4706b3f48f1d95db1a4f5529b4d41 + +ENV HF_HOME=/opt/huggingface \ + TOKENIZERS_PARALLELISM=false \ + OMP_NUM_THREADS=1 \ + MKL_NUM_THREADS=1 + +COPY requirements-agentic.txt /tmp/requirements-agentic.txt + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends git ca-certificates \ + && pip install --no-cache-dir \ + --extra-index-url https://download.pytorch.org/whl/cpu \ + -r /tmp/requirements-agentic.txt \ + && pip install --no-cache-dir --no-build-isolation --no-deps \ + "bfcl_eval @ git+https://github.com/ShishirPatil/gorilla.git@${BFCL_REVISION}#subdirectory=berkeley-function-call-leaderboard" \ + && python -c "from huggingface_hub import snapshot_download; snapshot_download('sentence-transformers/all-MiniLM-L6-v2', revision='${EMBEDDING_REVISION}')" \ + && mkdir -p /opt/huggingface/hub/models--sentence-transformers--all-MiniLM-L6-v2/refs \ + && printf '%s' "${EMBEDDING_REVISION}" > /opt/huggingface/hub/models--sentence-transformers--all-MiniLM-L6-v2/refs/main \ + && apt-get purge --yes --auto-remove git \ + && rm -rf /var/lib/apt/lists/* /tmp/requirements-agentic.txt + +ENV HF_HUB_DISABLE_XET=1 \ + HF_HUB_OFFLINE=1 + +RUN useradd --create-home --uid 1000 sandbox \ + && mkdir --parents /workspace \ + && chown sandbox:sandbox /workspace + +COPY runner.py /opt/openbench/bfcl_runner.py + +USER sandbox +WORKDIR /workspace +CMD ["sleep", "infinity"] diff --git a/src/openbench/evals/bfcl/__init__.py b/src/openbench/evals/bfcl/__init__.py index f976f40d..e2892d27 100644 --- a/src/openbench/evals/bfcl/__init__.py +++ b/src/openbench/evals/bfcl/__init__.py @@ -2,9 +2,11 @@ from __future__ import annotations +import hashlib import json +import os from pathlib import Path -from typing import Any +from typing import Any, Awaitable, Callable, cast from inspect_ai import Task, task from inspect_ai.model import ( @@ -21,6 +23,7 @@ MULTI_TURN_CATEGORIES, SINGLE_TURN_CATEGORIES, get_bfcl_v4_agentic_dataset, + get_bfcl_v4_agentic_live_dataset, get_bfcl_v4_multi_turn_dataset, get_bfcl_v4_single_turn_dataset, ) @@ -31,15 +34,22 @@ frozen_agentic_result, ) from openbench.scorers.bfcl import ( + agentic_answer_matches, bfcl_v4_agentic_scorer, + bfcl_v4_agentic_live_scorer, bfcl_v4_multi_turn_scorer, bfcl_v4_offline_scorer, bfcl_v4_scorer, ) COMPOSE_PATH = (Path(__file__).parent / "compose.yaml").resolve() +LIVE_COMPOSE_PATH = (Path(__file__).parent / "compose.live.yaml").resolve() +REPLAY_COMPOSE_PATH = (Path(__file__).parent / "compose.replay.yaml").resolve() MAX_STEPS = 20 +AGENTIC_RESPONSE_PROMPT = """For your final answer to the user, you must respond in this format: {'answer': A short and precise answer to the question, 'context': A brief explanation of how you arrived at this answer or why it is correct}. If you do not know the answer, respond with {'answer': 'I do not know', 'context': 'I do not know'}. If you think the question cannot be properly answered, response with {'answer': 'I cannot answer this question', 'context': A short reason explaining why this question cannot be answered}. +""" + @solver def bfcl_generate() -> Solver: @@ -63,6 +73,45 @@ def _append_messages(state: TaskState, messages: list[dict[str, str]]) -> None: state.messages.append(ChatMessageUser(content=content)) +def _configure_tools(state: TaskState) -> dict[str, str]: + tools, mapping = build_tool_definitions(list(state.metadata["functions"])) + state.tools = [definition.as_tool() for definition in tools] + state.tool_choice = "auto" + state.metadata["tool_name_mapping"] = mapping + return mapping + + +async def _run_agentic_conversation( + state: TaskState, + generate: Generate, + *, + turns: list[list[dict[str, str]]], + system_prompt: str, + execute_calls: Callable[[list[str], list[Any] | None], Awaitable[list[str]]], +) -> str: + """Run one isolated BFCL conversation through its complete tool loop.""" + + state.messages = [ChatMessageSystem(content=system_prompt)] + mapping = _configure_tools(state) + for turn in turns: + _append_messages(state, turn) + for _ in range(MAX_STEPS): + state = await generate(state, tool_calls="none") + message = state.output.message + calls = parse_function_calls( + message.tool_calls, state.output.completion, mapping + ) + if not calls: + break + outputs = await execute_calls( + [format_python_call(call) for call in calls], message.tool_calls + ) + _append_tool_results(state, outputs, message.tool_calls) + else: + raise RuntimeError(f"BFCL agentic workflow exceeded {MAX_STEPS} tool steps") + return state.output.completion + + async def _sandbox_request(state: TaskState, payload: dict[str, Any]) -> dict[str, Any]: environment = sandbox() payload_path = f".openbench_bfcl_{state.uuid}.json" @@ -77,6 +126,55 @@ async def _sandbox_request(state: TaskState, payload: dict[str, Any]) -> dict[st return json.loads(result.stdout.strip().splitlines()[-1]) +async def _start_live_runner(state: TaskState) -> str: + environment = sandbox() + socket_path = f"/workspace/.openbench_bfcl_{state.uuid}.sock" + log_path = f"/workspace/.openbench_bfcl_{state.uuid}.log" + command = ( + f"rm -f {socket_path}; " + f"nohup python /opt/openbench/bfcl_runner.py --serve {socket_path} " + f">{log_path} 2>&1 & " + f"for i in $(seq 1 300); do [ -S {socket_path} ] && exit 0; sleep 0.1; done; " + f"cat {log_path}; exit 1" + ) + result = await environment.exec(["sh", "-c", command], timeout=60) + if not result.success: + raise RuntimeError(f"BFCL live runner failed to start: {result.stderr}") + return socket_path + + +async def _live_runner_request( + state: TaskState, socket_path: str, payload: dict[str, Any] +) -> dict[str, Any]: + environment = sandbox() + payload_path = f".openbench_bfcl_live_{state.uuid}.json" + await environment.write_file(payload_path, json.dumps(payload)) + result = await environment.exec( + [ + "python", + "/opt/openbench/bfcl_runner.py", + "--client", + socket_path, + payload_path, + ], + timeout=180, + timeout_retry=False, + ) + if not result.success: + raise RuntimeError("BFCL live runner request failed") + response = json.loads(result.stdout.strip().splitlines()[-1]) + if "runner_error" in response: + raise RuntimeError(f"BFCL live backend failed: {response['runner_error']}") + return response + + +async def _stop_live_runner(state: TaskState, socket_path: str) -> None: + try: + await _live_runner_request(state, socket_path, {"operation": "shutdown"}) + except Exception: + pass + + def _append_tool_results( state: TaskState, outputs: list[str], @@ -198,6 +296,276 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: return solve +def _memory_request( + state: TaskState, + *, + operation: str, + scenario: str, + session_id: str, + calls: list[str], +) -> dict[str, Any]: + class_name = str(state.metadata["involved_classes"][0]) + category = str(state.metadata["category"]) + return { + "operation": operation, + "calls": calls, + "previous_count": 0, + "initial_config": { + class_name: { + "model_result_dir": "/workspace/memory_snapshots", + "scenario": scenario, + "test_id": session_id, + "test_category": category, + } + }, + "involved_classes": [class_name], + "id": session_id, + "category": category, + "scenario": scenario, + } + + +@solver +def bfcl_agentic_live_generate() -> Solver: + """Run BFCL's model-specific memory prerequisites and live web tools.""" + + async def solve(state: TaskState, generate: Generate) -> TaskState: + category = str(state.metadata["category"]) + snapshot_dir_value = state.metadata.get("web_snapshot_dir") + record_dir_value = state.metadata.get("record_web_snapshot_dir") + if snapshot_dir_value and record_dir_value: + raise ValueError( + "BFCL web snapshot replay and recording are mutually exclusive" + ) + if ( + category.startswith("web_search_") + and not snapshot_dir_value + and not os.getenv("SERPAPI_API_KEY") + ): + raise RuntimeError( + "bfcl_v4_agentic_live requires SERPAPI_API_KEY for web search" + ) + + socket_path = await _start_live_runner(state) + results: list[dict[str, Any]] = [] + try: + if category.startswith("memory_"): + for scenario_data in state.metadata["workflow"]["scenarios"]: + scenario = str(scenario_data["name"]) + prerequisite_calls: list[str] = [] + prerequisite_session = f"{category}_prereq_{scenario}_session-0" + + for prerequisite in scenario_data["prerequisites"]: + context = await _live_runner_request( + state, + socket_path, + _memory_request( + state, + operation="memory_context", + scenario=scenario, + session_id=prerequisite_session, + calls=[], + ), + ) + + async def execute_prerequisite( + encoded: list[str], native_calls: list[Any] | None + ) -> list[str]: + del native_calls + prerequisite_calls.extend(encoded) + response = await _live_runner_request( + state, + socket_path, + _memory_request( + state, + operation="execute", + scenario=scenario, + session_id=prerequisite_session, + calls=encoded, + ), + ) + return list(response["outputs"]) + + await _run_agentic_conversation( + state, + generate, + turns=list(prerequisite["question"]), + system_prompt=str(context["system_prompt"]), + execute_calls=execute_prerequisite, + ) + + for index, case in enumerate(scenario_data["cases"]): + target_session = ( + f"{category}_prereq_{scenario}_target_{index}-0" + ) + context = await _live_runner_request( + state, + socket_path, + _memory_request( + state, + operation="memory_context", + scenario=scenario, + session_id=target_session, + calls=prerequisite_calls, + ), + ) + + async def execute_target( + encoded: list[str], native_calls: list[Any] | None + ) -> list[str]: + del native_calls + response = await _live_runner_request( + state, + socket_path, + _memory_request( + state, + operation="execute", + scenario=scenario, + session_id=target_session, + calls=encoded, + ), + ) + return list(response["outputs"]) + + completion = await _run_agentic_conversation( + state, + generate, + turns=list(case["question"]), + system_prompt=( + f"{context['system_prompt']}\n\n" + f"{AGENTIC_RESPONSE_PROMPT}" + ), + execute_calls=execute_target, + ) + expected = [str(value) for value in case["expected_answers"]] + results.append( + { + "id": case["id"], + "correct": agentic_answer_matches(completion, expected), + "completion": completion, + } + ) + else: + class_name = str(state.metadata["involved_classes"][0]) + snapshot: dict[str, Any] | None = None + if snapshot_dir_value: + snapshot_path = Path(str(snapshot_dir_value)) / f"{category}.json" + loaded_snapshot = cast( + dict[str, Any], json.loads(snapshot_path.read_text()) + ) + if ( + loaded_snapshot.get("bfcl_revision") + != state.metadata["bfcl_revision"] + ): + raise ValueError( + "BFCL web snapshot revision does not match dataset" + ) + if loaded_snapshot.get("category") != category: + raise ValueError( + "BFCL web snapshot category does not match sample" + ) + snapshot = loaded_snapshot + recorded_cases: dict[str, list[dict[str, Any]]] = {} + initial_config = { + class_name: {"show_snippet": bool(state.metadata["show_snippet"])} + } + for case in state.metadata["workflow"]["cases"]: + session_id = str(case["id"]) + step_index = 0 + case_records: list[dict[str, Any]] = [] + + async def execute_web( + encoded: list[str], native_calls: list[Any] | None + ) -> list[str]: + nonlocal step_index + del native_calls + if snapshot is not None: + expected_steps = snapshot["cases"].get(session_id, []) + if step_index >= len(expected_steps): + raise ValueError( + f"BFCL snapshot has no step {step_index} for {session_id}" + ) + expected_step = expected_steps[step_index] + if expected_step["calls"] != encoded: + raise ValueError( + f"BFCL snapshot call mismatch for {session_id} step {step_index}" + ) + outputs = [str(value) for value in expected_step["outputs"]] + else: + response = await _live_runner_request( + state, + socket_path, + { + "operation": "execute", + "calls": encoded, + "previous_count": 0, + "initial_config": initial_config, + "involved_classes": [class_name], + "id": session_id, + "category": category, + }, + ) + outputs = list(response["outputs"]) + case_records.append({"calls": encoded, "outputs": outputs}) + step_index += 1 + return outputs + + completion = await _run_agentic_conversation( + state, + generate, + turns=list(case["question"]), + system_prompt=AGENTIC_RESPONSE_PROMPT, + execute_calls=execute_web, + ) + if snapshot is not None and step_index != len( + snapshot["cases"].get(session_id, []) + ): + raise ValueError( + f"BFCL snapshot has unused steps for {session_id}" + ) + expected = [str(value) for value in case["expected_answers"]] + results.append( + { + "id": case["id"], + "correct": agentic_answer_matches(completion, expected), + "completion": completion, + } + ) + recorded_cases[session_id] = case_records + + if record_dir_value: + record_dir = Path(str(record_dir_value)) + record_dir.mkdir(parents=True, exist_ok=True) + snapshot_payload = { + "schema_version": 1, + "bfcl_revision": state.metadata["bfcl_revision"], + "category": category, + "cases": recorded_cases, + } + serialized = json.dumps( + snapshot_payload, ensure_ascii=False, indent=2, sort_keys=True + ) + destination = record_dir / f"{category}.json" + temporary = destination.with_suffix(".json.tmp") + temporary.write_text(serialized) + temporary.replace(destination) + state.metadata["web_snapshot_sha256"] = hashlib.sha256( + serialized.encode() + ).hexdigest() + finally: + await _stop_live_runner(state, socket_path) + + correct_count = sum(bool(result["correct"]) for result in results) + state.metadata["agentic_results"] = results + state.metadata["agentic_correct_count"] = correct_count + state.metadata["agentic_accuracy"] = ( + correct_count / len(results) if results else 0.0 + ) + return state + + return solve + + @task def bfcl_v4_single_turn( categories: list[str] | None = None, @@ -258,6 +626,45 @@ def bfcl_v4_agentic_offline(categories: list[str] | None = None) -> Task: ) +@task +def bfcl_v4_agentic_live( + categories: list[str] | None = None, + web_snapshot_dir: str | None = None, + record_web_snapshot_dir: str | None = None, +) -> Task: + """Run BFCL v4 agentic workflows with official model-specific state and web.""" + + dataset = get_bfcl_v4_agentic_live_dataset(categories or list(AGENTIC_CATEGORIES)) + for sample in dataset: + if sample.metadata is not None: + sample.metadata["web_snapshot_dir"] = web_snapshot_dir + sample.metadata["record_web_snapshot_dir"] = record_web_snapshot_dir + + return Task( + dataset=dataset, + solver=bfcl_agentic_live_generate(), + scorer=bfcl_v4_agentic_live_scorer(), + sandbox=( + "docker", + str(REPLAY_COMPOSE_PATH if web_snapshot_dir else LIVE_COMPOSE_PATH), + ), + config=GenerateConfig( + temperature=0.001, + max_tokens=2048, + parallel_tool_calls=True, + ), + message_limit=200, + metadata={ + "benchmark": "BFCL v4", + "scope": "agentic-live", + "logical_case_count": 665, + "official_overall_score": False, + "web_snapshot_dir": web_snapshot_dir, + "record_web_snapshot_dir": record_web_snapshot_dir, + }, + ) + + @solver def bfcl_offline_generate() -> Solver: async def solve(state: TaskState, generate: Generate) -> TaskState: diff --git a/src/openbench/evals/bfcl/compose.live.yaml b/src/openbench/evals/bfcl/compose.live.yaml new file mode 100644 index 00000000..9ca9feb3 --- /dev/null +++ b/src/openbench/evals/bfcl/compose.live.yaml @@ -0,0 +1,20 @@ +services: + default: + build: + context: . + dockerfile: Dockerfile.agentic + environment: + BFCL_PROJECT_ROOT: /workspace + SERPAPI_API_KEY: ${SERPAPI_API_KEY:-} + init: true + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + pids_limit: 256 + mem_limit: 6g + cpus: 2 + tmpfs: + - /workspace:size=256m,mode=0700,uid=1000,gid=1000 + - /opt/huggingface/hub/models--sentence-transformers--all-MiniLM-L6-v2/trees:size=1m,mode=0700,uid=1000,gid=1000 diff --git a/src/openbench/evals/bfcl/compose.replay.yaml b/src/openbench/evals/bfcl/compose.replay.yaml new file mode 100644 index 00000000..e0bc4dd7 --- /dev/null +++ b/src/openbench/evals/bfcl/compose.replay.yaml @@ -0,0 +1,20 @@ +services: + default: + build: + context: . + dockerfile: Dockerfile.agentic + environment: + BFCL_PROJECT_ROOT: /workspace + init: true + network_mode: none + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + pids_limit: 256 + mem_limit: 6g + cpus: 2 + tmpfs: + - /workspace:size=256m,mode=0700,uid=1000,gid=1000 + - /opt/huggingface/hub/models--sentence-transformers--all-MiniLM-L6-v2/trees:size=1m,mode=0700,uid=1000,gid=1000 diff --git a/src/openbench/evals/bfcl/compose.yaml b/src/openbench/evals/bfcl/compose.yaml index f6dec57f..7f39a653 100644 --- a/src/openbench/evals/bfcl/compose.yaml +++ b/src/openbench/evals/bfcl/compose.yaml @@ -5,6 +5,8 @@ services: init: true command: tail -f /dev/null network_mode: none + environment: + BFCL_PROJECT_ROOT: /workspace read_only: true tmpfs: - /workspace:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=1073741824 diff --git a/src/openbench/evals/bfcl/requirements-agentic.in b/src/openbench/evals/bfcl/requirements-agentic.in new file mode 100644 index 00000000..7028008f --- /dev/null +++ b/src/openbench/evals/bfcl/requirements-agentic.in @@ -0,0 +1,14 @@ +beautifulsoup4 +build +faiss-cpu==1.11.0 +filelock==3.20.0 +google-search-results +html2text +huggingface-hub +numpy==1.26.4 +overrides +rank-bm25==0.2.2 +requests +sentence-transformers>=2.7.0 +setuptools-scm[toml]>=8.0 +wheel diff --git a/src/openbench/evals/bfcl/requirements-agentic.txt b/src/openbench/evals/bfcl/requirements-agentic.txt new file mode 100644 index 00000000..bc10f8e1 --- /dev/null +++ b/src/openbench/evals/bfcl/requirements-agentic.txt @@ -0,0 +1,157 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --python-version 3.11 --python-platform x86_64-manylinux_2_28 --torch-backend cpu --output-file src/openbench/evals/bfcl/requirements-agentic.txt src/openbench/evals/bfcl/requirements-agentic.in +annotated-doc==0.0.5 + # via typer +anyio==4.14.2 + # via httpx +beautifulsoup4==4.15.0 + # via -r src/openbench/evals/bfcl/requirements-agentic.in +build==1.5.0 + # via -r src/openbench/evals/bfcl/requirements-agentic.in +certifi==2026.7.22 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.9 + # via requests +click==8.4.2 + # via huggingface-hub +faiss-cpu==1.11.0 + # via -r src/openbench/evals/bfcl/requirements-agentic.in +filelock==3.20.0 + # via + # -r src/openbench/evals/bfcl/requirements-agentic.in + # huggingface-hub + # torch +fsspec==2026.7.0 + # via + # huggingface-hub + # torch +google-search-results==2.4.2 + # via -r src/openbench/evals/bfcl/requirements-agentic.in +h11==0.16.0 + # via httpcore +hf-xet==1.6.0 + # via huggingface-hub +html2text==2025.4.15 + # via -r src/openbench/evals/bfcl/requirements-agentic.in +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via huggingface-hub +huggingface-hub==1.26.1 + # via + # -r src/openbench/evals/bfcl/requirements-agentic.in + # sentence-transformers + # tokenizers + # transformers +idna==3.18 + # via + # anyio + # httpx + # requests +jinja2==3.1.6 + # via torch +joblib==1.5.3 + # via scikit-learn +markdown-it-py==4.2.0 + # via rich +markupsafe==3.0.3 + # via jinja2 +mdurl==0.1.2 + # via markdown-it-py +mpmath==1.3.0 + # via sympy +narwhals==2.24.0 + # via scikit-learn +networkx==3.6.1 + # via torch +numpy==1.26.4 + # via + # -r src/openbench/evals/bfcl/requirements-agentic.in + # faiss-cpu + # rank-bm25 + # scikit-learn + # scipy + # sentence-transformers + # transformers +overrides==7.7.0 + # via -r src/openbench/evals/bfcl/requirements-agentic.in +packaging==26.3 + # via + # build + # faiss-cpu + # huggingface-hub + # setuptools-scm + # transformers + # vcs-versioning + # wheel +pygments==2.20.0 + # via rich +pyproject-hooks==1.2.0 + # via build +pyyaml==6.0.3 + # via + # huggingface-hub + # transformers +rank-bm25==0.2.2 + # via -r src/openbench/evals/bfcl/requirements-agentic.in +regex==2026.7.19 + # via transformers +requests==2.34.2 + # via + # -r src/openbench/evals/bfcl/requirements-agentic.in + # google-search-results +rich==15.0.0 + # via typer +safetensors==0.8.0 + # via transformers +scikit-learn==1.9.0 + # via sentence-transformers +scipy==1.17.1 + # via + # scikit-learn + # sentence-transformers +sentence-transformers==5.6.1 + # via -r src/openbench/evals/bfcl/requirements-agentic.in +setuptools==83.0.0 + # via + # setuptools-scm + # torch +setuptools-scm==10.2.1 + # via -r src/openbench/evals/bfcl/requirements-agentic.in +shellingham==1.5.4 + # via typer +soupsieve==2.9.1 + # via beautifulsoup4 +sympy==1.14.0 + # via torch +threadpoolctl==3.6.0 + # via scikit-learn +tokenizers==0.22.2 + # via transformers +torch==2.13.0+cpu + # via sentence-transformers +tqdm==4.70.0 + # via + # huggingface-hub + # sentence-transformers + # transformers +transformers==5.14.1 + # via sentence-transformers +typer==0.27.1 + # via transformers +typing-extensions==4.16.0 + # via + # anyio + # beautifulsoup4 + # huggingface-hub + # sentence-transformers + # torch +urllib3==2.7.0 + # via requests +vcs-versioning==2.2.3 + # via setuptools-scm +wheel==0.47.0 + # via -r src/openbench/evals/bfcl/requirements-agentic.in diff --git a/src/openbench/evals/bfcl/runner.py b/src/openbench/evals/bfcl/runner.py index 4c4948f9..c15f9bfe 100644 --- a/src/openbench/evals/bfcl/runner.py +++ b/src/openbench/evals/bfcl/runner.py @@ -2,11 +2,20 @@ from __future__ import annotations +import ast +import ipaddress import json +import socket import sys from pathlib import Path from typing import Any +from urllib.parse import urlparse +from bfcl_eval.constants.default_prompts import ( # type: ignore[import-not-found] + MEMORY_AGENT_SETTINGS, + MEMORY_BACKEND_INSTRUCTION_CORE_ARCHIVAL, + MEMORY_BACKEND_INSTRUCTION_UNIFIED, +) from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_checker import ( # type: ignore[import-not-found] multi_turn_checker, multi_turn_irrelevance_checker, @@ -16,21 +25,85 @@ ) -def execute(payload: dict[str, Any]) -> dict[str, Any]: +def _execute_calls(payload: dict[str, Any]) -> tuple[list[str], dict[str, Any]]: calls = [str(value) for value in payload["calls"]] - previous_count = int(payload.get("previous_count", 0)) - outputs, _ = execute_multi_turn_func_call( + _validate_web_fetches(calls) + initial_config = payload["initial_config"] + for config in initial_config.values(): + if "model_result_dir" in config: + config["model_result_dir"] = Path(config["model_result_dir"]) + return execute_multi_turn_func_call( func_call_list=calls, - initial_config=payload["initial_config"], + initial_config=initial_config, involved_classes=payload["involved_classes"], model_name="openbench_generation", test_entry_id=payload["id"], long_context="long_context" in payload["category"], is_evaL_run=False, ) + + +def _validate_web_fetches(calls: list[str]) -> None: + """Prevent model-authored web calls from reaching private network services.""" + + for call in calls: + expression = ast.parse(call, mode="eval").body + if not isinstance(expression, ast.Call): + continue + function_name = ( + expression.func.id if isinstance(expression.func, ast.Name) else "" + ) + if function_name != "fetch_url_content": + continue + url_node = next( + (keyword.value for keyword in expression.keywords if keyword.arg == "url"), + expression.args[0] if expression.args else None, + ) + if url_node is None: + continue + url = str(ast.literal_eval(url_node)) + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("BFCL web fetch requires an HTTP(S) URL") + addresses = { + item[4][0] + for item in socket.getaddrinfo(parsed.hostname, parsed.port or 443) + } + if not addresses or any( + not ipaddress.ip_address(address).is_global for address in addresses + ): + raise ValueError("BFCL web fetch blocked a non-public destination") + + +def execute(payload: dict[str, Any]) -> dict[str, Any]: + previous_count = int(payload.get("previous_count", 0)) + outputs, _ = _execute_calls(payload) return {"outputs": outputs[previous_count:]} +def memory_context(payload: dict[str, Any]) -> dict[str, Any]: + """Replay a model's prerequisite calls and render BFCL's exact memory prompt.""" + + outputs, involved_instances = _execute_calls(payload) + if len(involved_instances) != 1: + raise ValueError("BFCL memory workflows require exactly one backend instance") + instance = next(iter(involved_instances.values())) + memory_content = instance._dump_core_memory_to_context() + scenario = str(payload["scenario"]) + template = ( + MEMORY_BACKEND_INSTRUCTION_UNIFIED + if "rec_sum" in str(payload["category"]) + else MEMORY_BACKEND_INSTRUCTION_CORE_ARCHIVAL + ) + return { + "outputs": outputs[int(payload.get("previous_count", 0)) :], + "system_prompt": template.format( + scenario_setting=MEMORY_AGENT_SETTINGS[scenario], + memory_content=memory_content, + ), + } + + def score(payload: dict[str, Any]) -> dict[str, Any]: result = multi_turn_checker( payload["model_turn_calls"], @@ -46,14 +119,72 @@ def score(payload: dict[str, Any]) -> dict[str, Any]: return result +def dispatch(payload: dict[str, Any]) -> dict[str, Any]: + operation = payload.pop("operation") + if operation == "execute": + return execute(payload) + if operation == "memory_context": + return memory_context(payload) + if operation == "score": + return score(payload) + raise ValueError(f"Unsupported BFCL runner operation: {operation}") + + +def serve(socket_path: Path) -> None: + """Serve requests while retaining official backend instances in memory.""" + + socket_path.unlink(missing_ok=True) + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server.bind(str(socket_path)) + server.listen() + try: + while True: + connection, _ = server.accept() + with connection: + chunks = [] + while chunk := connection.recv(1024 * 1024): + chunks.append(chunk) + request = json.loads(b"".join(chunks).decode()) + if request.get("operation") == "shutdown": + connection.sendall(b'{"stopped": true}') + return + try: + response = dispatch(request) + except Exception as error: + response = {"runner_error": str(error)} + connection.sendall(json.dumps(response, default=str).encode()) + finally: + server.close() + socket_path.unlink(missing_ok=True) + + +def client(socket_path: Path, payload_path: Path) -> None: + payload = payload_path.read_bytes() + payload_path.unlink() + connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.connect(str(socket_path)) + with connection: + connection.sendall(payload) + connection.shutdown(socket.SHUT_WR) + chunks = [] + while chunk := connection.recv(1024 * 1024): + chunks.append(chunk) + print(b"".join(chunks).decode()) + + def main() -> None: + if len(sys.argv) == 3 and sys.argv[1] == "--serve": + serve(Path(sys.argv[2])) + return + if len(sys.argv) == 4 and sys.argv[1] == "--client": + client(Path(sys.argv[2]), Path(sys.argv[3])) + return if len(sys.argv) != 2: raise SystemExit("usage: bfcl_runner PAYLOAD.json") payload_path = Path(sys.argv[1]) payload = json.loads(payload_path.read_text()) payload_path.unlink() - operation = payload.pop("operation") - result = execute(payload) if operation == "execute" else score(payload) + result = dispatch(payload) print(json.dumps(result, default=str)) diff --git a/src/openbench/scorers/bfcl.py b/src/openbench/scorers/bfcl.py index b91b56a4..8e3ffcbb 100644 --- a/src/openbench/scorers/bfcl.py +++ b/src/openbench/scorers/bfcl.py @@ -103,14 +103,20 @@ def _standardize_answer(value: str) -> str: return re.sub(r"[,./\-_*^()]", "", value).lower().replace("'", '"') -def _agentic_score(state: TaskState) -> Score: - completion = state.output.completion +def agentic_answer_matches(completion: str, expected: list[str]) -> bool: + """Apply BFCL's official normalized answer-containment rule.""" + standardized = _standardize_answer(completion) - expected = [str(value) for value in state.metadata["expected_answers"]] - matched = any( + return any( re.search(rf"\b{re.escape(_standardize_answer(answer))}\b", standardized) for answer in expected ) + + +def _agentic_score(state: TaskState) -> Score: + completion = state.output.completion + expected = [str(value) for value in state.metadata["expected_answers"]] + matched = agentic_answer_matches(completion, expected) return Score( value=CORRECT if matched else INCORRECT, answer=completion, @@ -131,6 +137,26 @@ async def score(state: TaskState, target: Target) -> Score: return score +@scorer(metrics=[bfcl_v4_agentic_metrics()]) +def bfcl_v4_agentic_live_scorer() -> Scorer: + async def score(state: TaskState, target: Target) -> Score: + del target + accuracy = float(state.metadata.get("agentic_accuracy", 0.0)) + case_count = int(state.metadata.get("case_count", 0)) + correct_count = int(state.metadata.get("agentic_correct_count", 0)) + return Score( + value=accuracy, + answer=f"{correct_count}/{case_count}", + metadata={ + "category": state.metadata["category"], + "live_agentic": True, + "case_count": case_count, + }, + ) + + return score + + @scorer(metrics=[bfcl_v4_offline_metrics()]) def bfcl_v4_offline_scorer() -> Scorer: single = bfcl_v4_scorer() diff --git a/tests/integration/test_docker_sandboxes.py b/tests/integration/test_docker_sandboxes.py index 963e2650..17dec293 100644 --- a/tests/integration/test_docker_sandboxes.py +++ b/tests/integration/test_docker_sandboxes.py @@ -179,3 +179,93 @@ def test_bfcl_official_multi_turn_checker_runs_in_sandbox(tmp_path: Path) -> Non payload_name=".openbench_bfcl_payload.json", ) assert result["valid"] is True + + +def test_bfcl_memory_runner_preserves_prerequisite_state(tmp_path: Path) -> None: + if shutil.which("docker") is None: + pytest.skip("Docker CLI is unavailable") + project = f"openbench-bfcl-memory-{uuid.uuid4().hex[:12]}" + compose = [ + "compose", + "-p", + project, + "-f", + str(ROOT / "src/openbench/evals/bfcl/compose.yaml"), + ] + socket_path = "/workspace/bfcl-memory.sock" + class_name = "MemoryAPI_kv" + session_id = "memory_kv_prereq_integration-0" + common: dict[str, object] = { + "previous_count": 0, + "initial_config": { + class_name: { + "model_result_dir": "/workspace/memory_snapshots", + "scenario": "customer", + "test_id": session_id, + "test_category": "memory_kv", + } + }, + "involved_classes": [class_name], + "id": session_id, + "category": "memory_kv", + "scenario": "customer", + } + + try: + _docker(*compose, "build", timeout=600) + _docker(*compose, "up", "-d", timeout=120) + _docker( + *compose, + "exec", + "-T", + "default", + "sh", + "-c", + "nohup python /opt/openbench/bfcl_runner.py --serve " + f"{socket_path} >/workspace/server.log 2>&1 & " + f"for i in $(seq 1 100); do [ -S {socket_path} ] && exit 0; " + "sleep 0.1; done; exit 1", + timeout=60, + ) + + responses = [] + for index, payload in enumerate( + [ + { + "operation": "execute", + "calls": ["core_memory_add(key='first_name', value='Michael')"], + **common, + }, + {"operation": "memory_context", "calls": [], **common}, + {"operation": "shutdown"}, + ] + ): + local_payload = tmp_path / f"memory-{index}.json" + local_payload.write_text(json.dumps(payload)) + container_payload = f"/workspace/memory-{index}.json" + _write_container_file(compose, container_payload, local_payload) + completed = _docker( + *compose, + "exec", + "-T", + "default", + "python", + "/opt/openbench/bfcl_runner.py", + "--client", + socket_path, + container_payload, + timeout=180, + ) + responses.append(json.loads(completed.stdout.strip().splitlines()[-1])) + + assert "Michael" in responses[1]["system_prompt"] + assert responses[2]["stopped"] is True + finally: + subprocess.run( + ["docker", *compose, "down", "--volumes", "--remove-orphans"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + timeout=120, + ) diff --git a/tests/test_bfcl.py b/tests/test_bfcl.py index 623a502d..57ae9f9c 100644 --- a/tests/test_bfcl.py +++ b/tests/test_bfcl.py @@ -14,10 +14,11 @@ from openbench.datasets.bfcl import ( BFCL_REVISION, get_bfcl_v4_agentic_dataset, + get_bfcl_v4_agentic_live_dataset, get_bfcl_v4_multi_turn_dataset, get_bfcl_v4_single_turn_dataset, ) -from openbench.evals.bfcl import bfcl_v4_single_turn +from openbench.evals.bfcl import bfcl_v4_agentic_live, bfcl_v4_single_turn from openbench.scorers.bfcl import bfcl_v4_scorer @@ -77,6 +78,22 @@ def test_bfcl_task_uses_deterministic_native_tool_generation() -> None: assert task.metadata["official_overall_score"] is False +def test_bfcl_live_task_selects_network_disabled_snapshot_replay() -> None: + dataset = MemoryDataset( + [Sample(input="workflow", target="", metadata={"category": "memory_kv"})] + ) + with patch( + "openbench.evals.bfcl.get_bfcl_v4_agentic_live_dataset", + return_value=dataset, + ): + task_factory = cast(Any, bfcl_v4_agentic_live) + task = task_factory.__wrapped__(["memory_kv"], web_snapshot_dir="snapshots") + + assert str(task.sandbox.config).endswith("compose.replay.yaml") + assert dataset[0].metadata is not None + assert dataset[0].metadata["web_snapshot_dir"] == "snapshots" + + def test_bfcl_registry_entry_is_explicitly_single_turn() -> None: metadata = BENCHMARKS["bfcl_v4_single_turn"] assert metadata.function_name == "bfcl_v4_single_turn" @@ -85,6 +102,7 @@ def test_bfcl_registry_entry_is_explicitly_single_turn() -> None: assert ( BENCHMARKS["bfcl_v4_agentic_offline"].function_name == "bfcl_v4_agentic_offline" ) + assert BENCHMARKS["bfcl_v4_agentic_live"].function_name == "bfcl_v4_agentic_live" assert BENCHMARKS["bfcl_v4_offline"].function_name == "bfcl_v4_offline" @@ -162,6 +180,56 @@ def fake_load(path: str): ] +def test_bfcl_live_agentic_dataset_groups_model_dependent_workflows() -> None: + memory_question = { + "id": "memory_0-customer-0", + "scenario": "customer", + "question": [[{"role": "user", "content": "My name?"}]], + } + memory_answer = { + "id": memory_question["id"], + "ground_truth": ["Michael"], + } + prerequisite = { + "id": "memory_prereq_0-customer-0", + "scenario": "customer", + "question": [[{"role": "user", "content": "My name is Michael."}]], + } + web_question = { + "id": "web_search_0", + "question": [[{"role": "user", "content": "Who?"}]], + } + web_answer = {"id": web_question["id"], "ground_truth": ["Ada"]} + + def fake_load(path: str): + if "multi_turn_func_doc" in path: + return QUESTION["function"] + if "memory_prereq_conversation" in path: + return [prerequisite] + if path == "BFCL_v4_memory.json": + return [memory_question] + if path == "possible_answer/BFCL_v4_memory.json": + return [memory_answer] + if path == "BFCL_v4_web_search.json": + return [web_question] + return [web_answer] + + with ( + patch("openbench.datasets.bfcl._load_jsonl", side_effect=fake_load), + patch("openbench.datasets.bfcl.MEMORY_SCENARIOS", ("customer",)), + ): + dataset = get_bfcl_v4_agentic_live_dataset(["memory_kv", "web_search_base"]) + + assert len(dataset) == 2 + memory_metadata = cast(dict[str, Any], dataset[0].metadata) + web_metadata = cast(dict[str, Any], dataset[1].metadata) + assert memory_metadata["case_count"] == 1 + assert memory_metadata["workflow"]["scenarios"][0]["prerequisites"] == [ + prerequisite + ] + assert web_metadata["case_count"] == 1 + + @pytest.mark.asyncio async def test_bfcl_scorer_reads_native_tool_calls() -> None: output = ModelOutput.from_message(