From c8db558b63348ee70e8d41e98cbdedbe7217c83f Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 19:39:50 -0700 Subject: [PATCH 01/12] feat(evals): passive/interactive eval framework over memory2 EvalCase/PassiveEval/InteractiveEval with EvalRig protocol dispatch, EvalRunner implementing the rig (model call / mcp skill / agent loop / live-store sampling), scorers as plain functions wrapping openevals, generated + hand VQA suites over go2 replays, dimsim go-to-bed interactive suite, dimos evals CLI + EvalModule MCP skills. extracts _init_model to dimos/agents/model.py for shared use. --- dimos/agents/mcp/mcp_client.py | 19 +- dimos/agents/mcp/test_mcp_client_unit.py | 2 +- dimos/agents/model.py | 44 +++ dimos/cli/dimos.py | 4 + dimos/evals/__init__.py | 48 +++ dimos/evals/cli.py | 68 ++++ dimos/evals/generate.py | 88 +++++ dimos/evals/module.py | 64 ++++ dimos/evals/runner.py | 396 +++++++++++++++++++++++ dimos/evals/scorers.py | 106 ++++++ dimos/evals/suites/__init__.py | 15 + dimos/evals/suites/dimsim_house.py | 101 ++++++ dimos/evals/suites/examples.py | 62 ++++ dimos/evals/suites/go2_smoke.py | 81 +++++ dimos/evals/suites/go2_vqa.json | 86 +++++ dimos/evals/suites/go2_vqa.py | 73 +++++ dimos/evals/test_evals.py | 312 ++++++++++++++++++ dimos/evals/test_smoke.py | 38 +++ dimos/evals/types.py | 198 ++++++++++++ pyproject.toml | 1 + uv.lock | 21 ++ 21 files changed, 1809 insertions(+), 18 deletions(-) create mode 100644 dimos/agents/model.py create mode 100644 dimos/evals/__init__.py create mode 100644 dimos/evals/cli.py create mode 100644 dimos/evals/generate.py create mode 100644 dimos/evals/module.py create mode 100644 dimos/evals/runner.py create mode 100644 dimos/evals/scorers.py create mode 100644 dimos/evals/suites/__init__.py create mode 100644 dimos/evals/suites/dimsim_house.py create mode 100644 dimos/evals/suites/examples.py create mode 100644 dimos/evals/suites/go2_smoke.py create mode 100644 dimos/evals/suites/go2_vqa.json create mode 100644 dimos/evals/suites/go2_vqa.py create mode 100644 dimos/evals/test_evals.py create mode 100644 dimos/evals/test_smoke.py create mode 100644 dimos/evals/types.py diff --git a/dimos/agents/mcp/mcp_client.py b/dimos/agents/mcp/mcp_client.py index 859b15451b..6beddb6b07 100644 --- a/dimos/agents/mcp/mcp_client.py +++ b/dimos/agents/mcp/mcp_client.py @@ -20,16 +20,15 @@ import uuid from langchain.agents import create_agent -from langchain.chat_models import init_chat_model from langchain_core.messages import HumanMessage from langchain_core.messages.base import BaseMessage from langchain_core.tools import StructuredTool -from langchain_openai import ChatOpenAI from langgraph.graph.state import CompiledStateGraph from reactivex.disposable import Disposable import requests from dimos.agents.mcp import tool_stream +from dimos.agents.model import init_model from dimos.agents.system_prompt import SYSTEM_PROMPT from dimos.agents.utils import pretty_print_langchain_message from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT @@ -42,20 +41,6 @@ logger = setup_logger() -_RESPONSES_REASONING_MODEL_PREFIXES = ("gpt-5", "o1", "o3", "o4") - - -def _init_model(model_name: str) -> Any: - """Initialize a model while preserving LangChain provider resolution.""" - if ":" in model_name or not model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES): - return init_chat_model(model=model_name) - - return ChatOpenAI( - model=model_name, - use_responses_api=True, - reasoning={"effort": "medium", "summary": "auto"}, - ) - class McpClientConfig(ModuleConfig): system_prompt: str | None = SYSTEM_PROMPT @@ -233,7 +218,7 @@ def on_system_modules(self, _modules: list[RPCClient]) -> None: model = MockModel(json_path=self.config.model_fixture) else: - model = _init_model(self.config.model) + model = init_model(self.config.model) with self._lock: self._state_graph = create_agent( diff --git a/dimos/agents/mcp/test_mcp_client_unit.py b/dimos/agents/mcp/test_mcp_client_unit.py index a49df130ff..dc1af78dbf 100644 --- a/dimos/agents/mcp/test_mcp_client_unit.py +++ b/dimos/agents/mcp/test_mcp_client_unit.py @@ -260,7 +260,7 @@ def test_on_system_modules_resolves_non_reasoning_models( with ( patch("dimos.agents.mcp.mcp_client.create_agent"), - patch("dimos.agents.mcp.mcp_client.init_chat_model", return_value=resolved_model) as init, + patch("dimos.agents.model.init_chat_model", return_value=resolved_model) as init, ): configured_mcp_client.on_system_modules([]) diff --git a/dimos/agents/model.py b/dimos/agents/model.py new file mode 100644 index 0000000000..b71471c01f --- /dev/null +++ b/dimos/agents/model.py @@ -0,0 +1,44 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared chat-model construction for agents and evals.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from langchain.chat_models import init_chat_model +from langchain_openai import ChatOpenAI + +if TYPE_CHECKING: + from langchain_core.language_models.chat_models import BaseChatModel + +_RESPONSES_REASONING_MODEL_PREFIXES = ("gpt-5", "o1", "o3", "o4") + + +def init_model(model_name: str) -> BaseChatModel: + """Initialize a model while preserving LangChain provider resolution. + + OpenAI reasoning models (gpt-5*/o*) without an explicit ``provider:`` prefix + go through the Responses API with reasoning enabled — the same configuration + the production ``McpClient`` runs, so evals measure what deploys. + """ + if ":" in model_name or not model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES): + return init_chat_model(model=model_name) + + return ChatOpenAI( + model=model_name, + use_responses_api=True, + reasoning={"effort": "medium", "summary": "auto"}, + ) diff --git a/dimos/cli/dimos.py b/dimos/cli/dimos.py index ee6a92f457..f4c83f3315 100644 --- a/dimos/cli/dimos.py +++ b/dimos/cli/dimos.py @@ -798,6 +798,10 @@ def dataprep_inspect( main.add_typer(mem_app, name="mem") +from dimos.evals.cli import app as evals_app + +main.add_typer(evals_app, name="evals") + @main.command() def cameracalibrate( diff --git a/dimos/evals/__init__.py b/dimos/evals/__init__.py new file mode 100644 index 0000000000..f43b4f11cc --- /dev/null +++ b/dimos/evals/__init__.py @@ -0,0 +1,48 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DimOS evals: passive (frozen mem2 recordings) and interactive (live robot/sim).""" + +from dimos.evals.runner import EvalRunner, EvalRunnerConfig, RunSummary, summarize +from dimos.evals.scorers import exact, final, floor, judge, mean, ramp, within +from dimos.evals.types import ( + EvalCase, + EvalResult, + EvalRig, + InteractiveEval, + PassiveEval, + Select, + Suite, +) + +__all__ = [ + "EvalCase", + "EvalResult", + "EvalRig", + "EvalRunner", + "EvalRunnerConfig", + "InteractiveEval", + "PassiveEval", + "RunSummary", + "Select", + "Suite", + "exact", + "final", + "floor", + "judge", + "mean", + "ramp", + "summarize", + "within", +] diff --git a/dimos/evals/cli.py b/dimos/evals/cli.py new file mode 100644 index 0000000000..7df7b7618e --- /dev/null +++ b/dimos/evals/cli.py @@ -0,0 +1,68 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""``dimos evals`` — run and list eval suites. Heavy imports stay inside +command bodies (test_cli_startup budget).""" + +from __future__ import annotations + +import importlib + +import typer + +app = typer.Typer(help="Run agent evals on recordings, sim, or a live robot.") + + +@app.command("run") +def run( + suite: str = typer.Argument( + help="Dotted suite module exporting SUITE, e.g. dimos.evals.suites.go2_smoke" + ), + tags: str = typer.Option("", help="Comma-separated tag filter"), + model: str = typer.Option("", help="Override chat model"), + blind: bool = typer.Option(False, help="Withhold observations (guessing ablation)"), + attach: bool = typer.Option(False, help="Drive an already-running dimos (interactive cases)"), + limit: int = typer.Option(0, help="Run at most N cases"), + live_db: str = typer.Option("recording.db", help="Live Recorder db (interactive cases)"), +) -> None: + from dimos.evals.runner import EvalRunner, summarize + + cases = importlib.import_module(suite).SUITE + overrides: dict[str, object] = {"blind": blind, "attach": attach, "live_db": live_db} + if model: + overrides["model"] = model + runner = EvalRunner(**overrides) + results = runner.run( + cases, + tags=frozenset(t for t in tags.split(",") if t) if tags else frozenset(), + limit=limit, + ) + + for r in results: + status = "ERROR" if r.error else ("PASS" if r.passed else "fail") + detail = r.error or f"score={r.score:.2f} answer={r.outputs[:60]!r}" + typer.echo(f"{status:5} {r.case_id:30} {detail} ({r.duration_s:.1f}s)") + s = summarize(results) + typer.echo( + f"\n{s.n} cases | mean {s.mean_score:.2f} | pass {s.pass_rate:.0%} " + f"| errors {s.errors} | {s.duration_s:.0f}s | {runner.run_dir}" + ) + + +@app.command("list") +def list_() -> None: + from dimos.evals.module import list_suites + + for name in list_suites(): + typer.echo(name) diff --git a/dimos/evals/generate.py b/dimos/evals/generate.py new file mode 100644 index 0000000000..45bf0c4bee --- /dev/null +++ b/dimos/evals/generate.py @@ -0,0 +1,88 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Eval-row generators — deferred feature (PRD: low priority), kept minimal. + +Ground truth is computed analytically from a *privileged* modality; the emitted +case quizzes a different (or lossily-encoded) surface. Rows are pure data — +a suite module maps them onto typed :class:`PassiveEval` cases. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from dimos.memory2.cli.dataset import open_dataset + +Row = dict[str, object] + + +def displacement_rows(dataset: str, windows: Sequence[tuple[float, float]]) -> list[Row]: + """Straight-line displacement over each window (odom is the privileged truth; + the case quizzes the encoded odom summary). Sampling-invariant ground truth.""" + store = open_dataset(dataset) + try: + rows: list[Row] = [] + for t1, t2 in windows: + obs = store.streams.odom.range_time(t1, t2).to_list() + if len(obs) < 2: + continue + d = (obs[-1].data.position - obs[0].data.position).length() + rows.append( + { + "id": f"{dataset}_disp_{t1:g}_{t2:g}", + "q": "How far in a straight line is your final position from your " + "position at the first shown observation, in meters?", + "a": round(d, 1), + "band": max(1.0, d * 0.4), + "stream": "odom", + "window": [t1, t2], + "dataset": dataset, + } + ) + return rows + finally: + store.stop() + + +def path_length_rows(dataset: str, windows: Sequence[tuple[float, float]]) -> list[Row]: + """Integrated path length per window. Deliberately hard on a downsampled + encoding — expect partial credit; that gap is the finding.""" + store = open_dataset(dataset) + try: + rows: list[Row] = [] + for t1, t2 in windows: + path, prev = 0.0, None + for obs in store.streams.odom.range_time(t1, t2): + p = obs.data.position + if prev is not None: + path += (p - prev).length() + prev = p + if prev is None: + continue + rows.append( + { + "id": f"{dataset}_path_{t1:g}_{t2:g}", + "q": "Roughly how many meters did you travel in total over these " + "observations (path length, not displacement)?", + "a": round(path, 1), + "band": max(2.0, path * 0.5), + "stream": "odom", + "window": [t1, t2], + "dataset": dataset, + } + ) + return rows + finally: + store.stop() diff --git a/dimos/evals/module.py b/dimos/evals/module.py new file mode 100644 index 0000000000..8849d9bc4a --- /dev/null +++ b/dimos/evals/module.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MCP surface for evals — lets a coding agent spin runs and grep the run dir.""" + +from __future__ import annotations + +import importlib +import pkgutil + +from dimos.agents.annotation import skill +from dimos.agents.skill_result import SkillResult +from dimos.core.module import Module + + +def list_suites() -> list[str]: + """Dotted module paths under dimos.evals.suites exporting ``SUITE``.""" + from dimos.evals import suites + + return [ + name for _, name, _ in pkgutil.iter_modules(suites.__path__, prefix=f"{suites.__name__}.") + ] + + +class EvalModule(Module): + """Expose eval runs as skills so agents can iterate: run, read the summary, + grep transcripts in the returned run_dir, edit code/prompts, run again.""" + + @skill + def run_evals(self, suite: str, tags: str = "") -> SkillResult: + """Run an eval suite by dotted module path (see list_eval_suites). + + Args: + suite: e.g. "dimos.evals.suites.go2_smoke" (must export SUITE). + tags: optional comma-separated tag filter. + """ + from dimos.evals.runner import EvalRunner, summarize + + cases = importlib.import_module(suite).SUITE + runner = EvalRunner(attach=True) + results = runner.run( + cases, tags=frozenset(t for t in tags.split(",") if t) if tags else frozenset() + ) + s = summarize(results) + return SkillResult.ok( + f"{s.n} cases: mean={s.mean_score:.2f} pass={s.pass_rate:.0%} errors={s.errors}", + run_dir=str(runner.run_dir), + ) + + @skill + def list_eval_suites(self) -> SkillResult: + """List available eval suite module paths.""" + return SkillResult.ok(", ".join(list_suites())) diff --git a/dimos/evals/runner.py b/dimos/evals/runner.py new file mode 100644 index 0000000000..9fdfec96ff --- /dev/null +++ b/dimos/evals/runner.py @@ -0,0 +1,396 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""EvalRunner — the one engine behind CLI, MCP skill, and pytest. + +Implements the :class:`~dimos.evals.types.EvalRig` protocol structurally. +Cases own their evaluation flow (``case.evaluate(rig)``); the runner owns +resources (model client, MCP adapter, sim process, live store) plus run +lifecycle: preflight, timing, error isolation, artifacts. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, dataclass, replace +import json +from pathlib import Path +import subprocess +import time +from typing import TYPE_CHECKING, Any + +from dimos.core.resource import CompositeResource +from dimos.evals.types import EvalCase, EvalResult, InteractiveEval, Suite +from dimos.protocol.service.spec import BaseConfig, Configurable +from dimos.utils.logging_config import setup_logger + +if TYPE_CHECKING: + from langchain_core.language_models.chat_models import BaseChatModel + + from dimos.e2e_tests.dimos_cli_call import DimosCliCall + from dimos.memory2.store.base import Store + from dimos.memory2.stream import Stream + +logger = setup_logger() + +EVAL_SYSTEM_PROMPT = ( + "You are evaluating a robot's perception and memory. Answer the question " + "using only the provided observations. Reply with the answer value only — " + "a bare number or a short phrase. No explanation, no units unless asked." +) + +BLIND_BLOCK: dict[str, str] = { + "type": "text", + "text": "[observations withheld — answer anyway]", +} + + +class EvalRunnerConfig(BaseConfig): + model: str = "gpt-5.6-luna" # mirrors McpClientConfig.model + # House convention (StoreConfig): pass an instance to inject, e.g. a fake + # chat model in tests. None -> built from `model` via init_model(). + chat_model: Any | None = None + mcp_url: str = "http://localhost:9990/mcp" + live_db: str = "recording.db" # store the Recorder writes (interactive) + blind: bool = False # ablation: context withheld (SPACE guessing check) + threshold: float = 1.0 # passed = score >= threshold + strict: bool = False # preflight failure aborts the whole run + context_budget: int = 8 # max observations encoded per context Select + attach: bool = False # True: drive an already-running dimos + launch_timeout_s: float = 1200.0 # blueprint + MCP readiness (e2e parity) + out_dir: Path = Path("~/.local/state/dimos/evals").expanduser() + + +@dataclass(frozen=True, kw_only=True) +class RunSummary: + n: int + mean_score: float + pass_rate: float + errors: int + duration_s: float + + +def summarize(results: Sequence[EvalResult]) -> RunSummary: + scored = [r for r in results if not r.error] + return RunSummary( + n=len(results), + mean_score=sum(r.score for r in scored) / len(scored) if scored else 0.0, + pass_rate=sum(r.passed for r in scored) / len(scored) if scored else 0.0, + errors=sum(1 for r in results if r.error), + duration_s=sum(r.duration_s for r in results), + ) + + +class EvalRunner(Configurable, CompositeResource): + config: EvalRunnerConfig + + def __init__(self, **kwargs: Any) -> None: + Configurable.__init__(self, **kwargs) + CompositeResource.__init__(self) + self._model: BaseChatModel | None = None + self._proc: DimosCliCall | None = None + self._run_dir: Path | None = None + + # -- run lifecycle ----------------------------------------------------------- + + def run( + self, + cases: Suite, + *, + tags: frozenset[str] = frozenset(), + limit: int = 0, + ) -> list[EvalResult]: + selected = [c for c in cases if not tags or tags & c.tags] + if limit: + selected = selected[:limit] + self._run_dir = self._new_run_dir() + + results: list[EvalResult] = [] + runnable: list[EvalCase] = [] + for case in selected: + try: + case.preflight(self) + runnable.append(case) + except Exception as e: + if self.config.strict: + raise + logger.warning("preflight failed", case=case.id, error=str(e)) + results.append(EvalResult(case_id=case.id, error=f"preflight: {e}")) + + for case in runnable: + result = self._guarded(case) + logger.info( + "eval case done", + case=case.id, + score=round(result.score, 3), + error=result.error or None, + ) + results.append(result) + + self._write_artifacts(results) + self.stop() + return results + + def _guarded(self, case: EvalCase) -> EvalResult: + t0 = time.monotonic() + try: + result = case.evaluate(self) + transcript = self.run_dir / f"{case.id}.jsonl" + return replace( + result, + duration_s=time.monotonic() - t0, + passed=result.score >= self.config.threshold and not result.error, + transcript=str(transcript) if transcript.exists() else result.transcript, + ) + except Exception as e: + return EvalResult(case_id=case.id, error=repr(e), duration_s=time.monotonic() - t0) + + @property + def run_dir(self) -> Path: + assert self._run_dir is not None, "run_dir is available only during run()" + return self._run_dir + + def _new_run_dir(self) -> Path: + run_dir = self.config.out_dir / time.strftime("run-%Y%m%d-%H%M%S") + run_dir.mkdir(parents=True, exist_ok=True) + return run_dir + + def _write_artifacts(self, results: list[EvalResult]) -> None: + lines = [json.dumps(asdict(r)) for r in results] + (self.run_dir / "results.jsonl").write_text("\n".join(lines) + "\n") + summary: dict[str, Any] = asdict(summarize(results)) + summary |= {"model": self.config.model, "blind": self.config.blind, "git": _git_sha()} + (self.run_dir / "summary.json").write_text(json.dumps(summary, indent=2)) + + def stop(self) -> None: + if self._proc is not None: + self._proc.stop() + self._proc = None + super().stop() + + # -- EvalRig: shared ------------------------------------------------------------ + + @property + def blind(self) -> bool: + return self.config.blind + + @property + def mcp_url(self) -> str: + return self.config.mcp_url + + def open_dataset(self, name: str) -> Store: + from dimos.memory2.cli.dataset import open_dataset + + return open_dataset(name) + + def live_store(self) -> Store: + from dimos.memory2.store.sqlite import SqliteStore + + return SqliteStore(path=self.config.live_db, must_exist=True) + + def encode(self, stream: Stream[Any, Any]) -> list[dict[str, Any]]: + """mem2 Stream -> model-legible content blocks (the surface under test). + + Metadata iterates lazily; blobs load only for the <= context_budget + observations actually encoded. ``agent_encode()`` is used where a type + provides it; ``str(data)`` otherwise (an encoding gap the eval will + surface, by design). + """ + observations = list(stream) + if not observations: + return [{"type": "text", "text": f"stream {stream.name!r}: no observations"}] + + budget = self.config.context_budget + if len(observations) > budget: + step = (len(observations) - 1) / (budget - 1) + observations = [observations[round(i * step)] for i in range(budget)] + + t0 = observations[0].ts + blocks: list[dict[str, Any]] = [ + { + "type": "text", + "text": f"observations from stream {stream.name!r} " + f"(t is seconds from the first shown):", + } + ] + for obs in observations: + data = obs.data + encoded = data.agent_encode() if hasattr(data, "agent_encode") else None + stamp = f"[t={obs.ts - t0:.1f}s]" + if isinstance(encoded, list): # e.g. Image -> image_url blocks + blocks.append({"type": "text", "text": stamp}) + blocks.extend(encoded) + elif encoded is not None: + blocks.append( + {"type": "text", "text": f"{stamp} {json.dumps(encoded, default=str)}"} + ) + else: + blocks.append({"type": "text", "text": f"{stamp} {data}"}) + return blocks + + def ask(self, context: Sequence[dict[str, Any]], question: str) -> str: + from langchain_core.messages import HumanMessage, SystemMessage + + blocks = list(context) if context else [BLIND_BLOCK] + message = HumanMessage(content=[*blocks, {"type": "text", "text": question}]) + response = self.model.invoke([SystemMessage(EVAL_SYSTEM_PROMPT), message]) + return str(response.text) + + @property + def model(self) -> BaseChatModel: + if self.config.chat_model is not None: + return self.config.chat_model # type: ignore[no-any-return] + if self._model is None: + from dimos.agents.model import init_model + + self._model = init_model(self.config.model) + return self._model + + def call_skill(self, name: str, args: Mapping[str, object]) -> str: + from dimos.agents.mcp.mcp_adapter import McpAdapter + + return McpAdapter(self.config.mcp_url).call_tool_text(name, dict(args)) + + def mcp_ready(self) -> bool: + from dimos.agents.mcp.mcp_adapter import McpAdapter + + return McpAdapter(self.config.mcp_url).wait_for_ready(timeout=2.0) + + def agent_loop(self, case: EvalCase) -> str: + """Fresh create_agent per case over the MCP toolset — the McpClient loop + minus its queue/thread shell. Transcript -> /.jsonl.""" + from langchain.agents import create_agent + from langchain_core.messages import HumanMessage, SystemMessage + from langchain_core.tools import StructuredTool + + from dimos.agents.mcp.mcp_adapter import McpAdapter + + adapter = McpAdapter(self.config.mcp_url) + tools = [ + StructuredTool( + name=t["name"], + description=t.get("description", ""), + args_schema=t.get("inputSchema", {}), + func=lambda _name=t["name"], **kwargs: adapter.call_tool_text(_name, kwargs), + ) + for t in adapter.list_tools() + ] + graph: Any = create_agent(self.model, tools) + messages: list[Any] = [SystemMessage(EVAL_SYSTEM_PROMPT), HumanMessage(case.inputs)] + transcript = self.run_dir / f"{case.id}.jsonl" + final_text = "" + with transcript.open("w") as fh: + for update in graph.stream({"messages": messages}, stream_mode="updates"): + for _node, payload in update.items(): + for msg in payload.get("messages", []): + fh.write( + json.dumps({"type": type(msg).__name__, "content": str(msg.content)}) + + "\n" + ) + final_text = str(msg.content) + return final_text + + # -- EvalRig: interactive ---------------------------------------------------------- + + def setup_env(self, case: InteractiveEval) -> None: + from dimos.evals.types import _no_setup + + if case.simulator and not self.config.attach: + from dimos.e2e_tests.dimos_cli_call import DimosCliCall + + proc = DimosCliCall() + proc.simulator = case.simulator + proc.global_args = ["--dimsim-scene", case.scene] + proc.demo_args = ["run", *case.blueprint.split()] + proc.start() + self._proc = proc + if not self._wait_mcp(self.config.launch_timeout_s): + raise RuntimeError(f"MCP at {self.config.mcp_url} not ready — is dimos up?") + if case.setup is not _no_setup: + from dimos.e2e_tests.dim_sim_client import DimSimClient + + sim = DimSimClient() + sim.start() + case.setup(sim) + + def check_env(self, case: InteractiveEval) -> None: + if self.config.attach or not case.simulator: + if not self.mcp_ready(): + raise RuntimeError( + f"{case.id}: attach mode needs a running dimos at {self.config.mcp_url}" + ) + return + import shutil + + if case.simulator == "dimsim" and shutil.which("deno") is None: + raise RuntimeError(f"{case.id}: dimsim requires deno on PATH") + + def _wait_mcp(self, timeout: float) -> bool: + from dimos.agents.mcp.mcp_adapter import McpAdapter + + return McpAdapter(self.config.mcp_url).wait_for_ready(timeout=timeout, interval=2.0) + + def instruct(self, text: str) -> None: + from dimos.core.transport import pLCMTransport + + transport: pLCMTransport[str] = pLCMTransport("/human_input") + transport.lcm.start() + try: + transport.publish(text) + time.sleep(0.5) # let LCM flush before teardown + finally: + transport.lcm.stop() + + def sample( + self, score: Callable[[Store], float], interval_s: float, timeout_s: float + ) -> list[tuple[float, float]]: + """Score the live Recorder store on an interval — the mem2 analogue of + lcm_spy.wait_until_odom_position, but it returns a graded series.""" + deadline = time.monotonic() + timeout_s + t0 = time.monotonic() + series: list[tuple[float, float]] = [] + store = self._wait_live_store(deadline) + try: + while time.monotonic() < deadline: + try: + value = score(store) + except LookupError: + value = None # stream not written yet — keep waiting + if value is not None: + series.append((time.monotonic() - t0, value)) + if value >= 0.999: # ponytail: early exit on success; drop if + break # aggregates ever need the full window + time.sleep(interval_s) + finally: + store.stop() + return series + + def _wait_live_store(self, deadline: float) -> Store: + path = Path(self.config.live_db) + while not path.exists() and time.monotonic() < deadline: + time.sleep(1.0) + return self.live_store() + + +def _git_sha() -> str: + try: + return subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + timeout=5, + check=False, + ).stdout.strip() + except OSError: + return "" diff --git a/dimos/evals/scorers.py b/dimos/evals/scorers.py new file mode 100644 index 0000000000..49d6c0d369 --- /dev/null +++ b/dimos/evals/scorers.py @@ -0,0 +1,106 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Scoring helpers: plain functions over typed values, graded credit in one line. + +Scores are floats in ``[0, 1]``. Msg types support arithmetic, so physical +scorers stay one-liners:: + + lambda s: ramp((GOAL - s.streams.odom.last().data.position).length(), band=0.5) + +LLM-based scoring wraps ``openevals`` — a function library (nothing to +subclass): factories return evaluators called with +``inputs/outputs/reference_outputs`` returning ``{"key", "score", "comment"}``. +""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from typing import TypeVar + +T = TypeVar("T") + + +def exact(expected: T, got: T) -> float: + return float(expected == got) + + +# -- parsers (model text -> typed answer) ----------------------------------------- + + +def first_number(text: str) -> float: + """Pull the first number out of a model reply ("about 12.5 meters" -> 12.5).""" + import re + + match = re.search(r"-?\d+(?:\.\d+)?", text) + if match is None: + raise ValueError(f"no number in reply: {text[:80]!r}") + return float(match.group()) + + +def yes_no(text: str) -> str: + """Normalize a reply to "yes"/"no".""" + t = text.strip().lower() + if t.startswith(("yes", "no")): + return "yes" if t.startswith("yes") else "no" + raise ValueError(f"not a yes/no reply: {text[:80]!r}") + + +def choice(text: str) -> str: + """Normalize a multiple-choice reply for exact comparison.""" + return text.strip().lower().rstrip(".") + + +def within(band: float) -> Callable[[float, float], float]: + """1.0 at exact, linear to 0.0 at ``band`` away.""" + return lambda expected, got: max(0.0, 1.0 - abs(got - expected) / band) + + +def ramp(distance: float, band: float) -> float: + """Distance (meters) -> [0, 1] credit inside ``band``.""" + return max(0.0, 1.0 - distance / band) + + +def judge(rubric: str, *, model: str = "openai:gpt-5.6-luna") -> Callable[[str, str], float]: + """LLM-as-judge with partial credit via openevals ``continuous=True``. + + ``rubric`` may reference ``{inputs}``, ``{outputs}``, ``{reference_outputs}``. + """ + from openevals.llm import create_llm_as_judge + + evaluator = create_llm_as_judge(prompt=rubric, model=model, continuous=True) + + def _score(expected: str, got: str) -> float: + result = evaluator(inputs="", outputs=got, reference_outputs=expected) + if isinstance(result, list): + result = result[0] + return float(result["score"]) + + return _score + + +# -- aggregates for interactive score series ------------------------------------- + + +def final(scores: Sequence[float]) -> float: + return scores[-1] + + +def floor(scores: Sequence[float]) -> float: + """Worst moment wins — "never left the zone".""" + return min(scores) + + +def mean(scores: Sequence[float]) -> float: + return sum(scores) / len(scores) diff --git a/dimos/evals/suites/__init__.py b/dimos/evals/suites/__init__.py new file mode 100644 index 0000000000..f02f56de83 --- /dev/null +++ b/dimos/evals/suites/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Eval suites. Each module exports ``SUITE: Suite``.""" diff --git a/dimos/evals/suites/dimsim_house.py b/dimos/evals/suites/dimsim_house.py new file mode 100644 index 0000000000..3e685f7c89 --- /dev/null +++ b/dimos/evals/suites/dimsim_house.py @@ -0,0 +1,101 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Interactive suite — dimsim apartment, parity with test_dimsim_spatial_memory. + +The e2e test asserts ``wait_until_odom_position(-3.567, -1.332, threshold=2)`` +after "go to the bed"; here the same success condition is a graded ramp scored +against the live mem2 store written by the ``go2-memory`` Recorder. + +Exploration (the e2e ``explore_house`` fixture) is the case's ``setup`` — the +agent needs spatial memory of the apartment before it can navigate it. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING + +from dimos.evals.scorers import final, ramp +from dimos.evals.types import InteractiveEval, Suite +from dimos.msgs.geometry_msgs.Vector3 import Vector3 + +if TYPE_CHECKING: + from dimos.e2e_tests.dim_sim_client import DimSimClient + from dimos.memory2.store.base import Store + +BED = Vector3(-3.567, -1.332, 0.0) + +_HOUSE_TOUR = [ + (3.881, 4.803), + (4.160, 1.615), + (1.596, 1.505), + (1.649, 0.137), + (-3.644, -0.064), + (-3.759, -2.661), + (-4.186, -4.830), + (-3.759, -2.661), + (-1.070, -3.285), + (-2.504, -2.452), + (-2.647, 5.243), + (-3.663, 3.591), + (-1.178, 1.974), + (-2.416, 2.629), + (-2.581, 0.164), + (1.834, 0.072), + (3.010, -3.883), + (1.756, -3.742), + (6.336, -4.077), + (8.264, -5.119), + (6.258, -0.964), + (6.453, 5.327), +] + + +def _explore_house(sim: DimSimClient) -> None: + from dimos.simulation.mujoco.direct_cmd_vel_explorer import DirectCmdVelExplorer + + explorer = DirectCmdVelExplorer() + explorer.linear_speed = 0.5 + explorer.start() + try: + explorer.follow_points(_HOUSE_TOUR) + finally: + explorer.stop() + + +def _xy_distance_to(target: Vector3) -> Callable[[Store], float]: + def distance(store: Store) -> float: + p = store.streams.odom.last().data.position + d = Vector3(p.x - target.x, p.y - target.y, 0.0).length() + return ramp(d, band=2.0) # e2e parity: threshold=2 -> full credit inside 2m + + return distance + + +go_to_bed = InteractiveEval( + id="dimsim_go_to_bed", + inputs="go to the bed", + score=_xy_distance_to(BED), + aggregate=final, + interval_s=2.0, + timeout_s=180.0, # e2e parity + blueprint="unitree-go2-agentic go2-memory", + simulator="dimsim", + scene="apartment", + setup=_explore_house, + tags=frozenset({"nav", "system"}), +) + +SUITE: Suite = [go_to_bed] diff --git a/dimos/evals/suites/examples.py b/dimos/evals/suites/examples.py new file mode 100644 index 0000000000..4dda943a9e --- /dev/null +++ b/dimos/evals/suites/examples.py @@ -0,0 +1,62 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Documentation examples — the smallest useful evals, one concept each. + +Run them:: + + dimos evals run dimos.evals.suites.examples + +Or from pytest / a notebook:: + + from dimos.evals import EvalRunner + from dimos.evals.suites.examples import SUITE + + results = EvalRunner().run(SUITE) +""" + +from __future__ import annotations + +from dimos.evals.scorers import exact, first_number, within, yes_no +from dimos.evals.types import PassiveEval, Suite + +# One lidar frame from the unitree go2 replay. The context selector returns the +# real mem2 Stream — `.limit(1)` keeps exactly the first PointCloud2. The str() +# fallback encoding exposes `num_points`, so the answer is verifiable. +single_lidar_frame = PassiveEval( + id="example_single_lidar_frame", + inputs="How many points does the shown pointcloud contain?", + expected=20834.0, + parse=first_number, + score=within(5000.0), + context=(lambda s: s.streams.lidar.limit(1),), + dataset="go2_short", + tags=frozenset({"example", "encoding", "pointcloud"}), +) + +# A range of 10 image frames from the same replay: seconds 45..61 of the +# recording, capped to 10 observations. Image.agent_encode() turns each into an +# image content block; the person at the table appears near the end. +ten_image_range = PassiveEval( + id="example_ten_image_range", + inputs="Is a person visible in any of these images?", + expected="yes", + parse=yes_no, + score=exact, + context=(lambda s: s.streams.color_image.range_time(45, 61).limit(10),), + dataset="go2_short", + tags=frozenset({"example", "encoding", "image"}), +) + +SUITE: Suite = [single_lidar_frame, ten_image_range] diff --git a/dimos/evals/suites/go2_smoke.py b/dimos/evals/suites/go2_smoke.py new file mode 100644 index 0000000000..6c1a5168b4 --- /dev/null +++ b/dimos/evals/suites/go2_smoke.py @@ -0,0 +1,81 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hand-written passive smoke suite over the go2 replays. + +Ground truth verified against the recordings (contact sheets + odom math): +go2_short is 60s — chairs room, store shelves, robot kiosk, glass booths, a +person at a table at the end; path 37.9m, displacement 1.7m. +""" + +from __future__ import annotations + +from dimos.evals.scorers import choice, exact, first_number, within, yes_no +from dimos.evals.types import PassiveEval, Suite + +SUITE: Suite = [ + PassiveEval( + id="short_person_visible", + inputs="Is a person visible in any of these images?", + expected="yes", + parse=yes_no, + score=exact, + context=(lambda s: s.streams.color_image.range_time(40, 61),), + dataset="go2_short", + tags=frozenset({"image", "presence"}), + ), + PassiveEval( + id="short_start_furniture", + inputs="At the start of these observations, which furniture is most numerous? " + "Answer with one of: chairs, sofas, beds, desks.", + expected="chairs", + parse=choice, + score=exact, + context=(lambda s: s.streams.color_image.range_time(0, 8),), + dataset="go2_short", + tags=frozenset({"image", "mcq"}), + ), + PassiveEval( + id="short_displacement", + inputs="How far in a straight line is your final position from your first " + "shown position, in meters?", + expected=1.7, + parse=first_number, + score=within(1.5), + context=(lambda s: s.streams.odom,), + dataset="go2_short", + tags=frozenset({"odom", "numeric"}), + ), + PassiveEval( + id="short_lidar_points", + inputs="How many points does the shown pointcloud contain?", + expected=20834.0, + parse=first_number, + score=within(5000.0), + context=(lambda s: s.streams.lidar.limit(1),), + dataset="go2_short", + tags=frozenset({"pointcloud", "numeric"}), + ), + PassiveEval( + id="hk_not_seen", + inputs="Which of these did you NOT see anywhere in the observations? " + "Answer with one of: a couch, store shelves, a swimming pool, an office chair.", + expected="a swimming pool", + parse=choice, + score=exact, + context=(lambda s: s.streams.color_image,), + dataset="go2_hongkong_office", + tags=frozenset({"image", "mcq"}), + ), +] diff --git a/dimos/evals/suites/go2_vqa.json b/dimos/evals/suites/go2_vqa.json new file mode 100644 index 0000000000..aa2262f25d --- /dev/null +++ b/dimos/evals/suites/go2_vqa.json @@ -0,0 +1,86 @@ +[ + { + "id": "go2_short_disp_0_60", + "q": "How far in a straight line is your final position from your position at the first shown observation, in meters?", + "a": 1.7, + "band": 1.0, + "stream": "odom", + "window": [ + 0, + 60 + ], + "dataset": "go2_short" + }, + { + "id": "go2_short_disp_0_30", + "q": "How far in a straight line is your final position from your position at the first shown observation, in meters?", + "a": 11.4, + "band": 4.543036677935427, + "stream": "odom", + "window": [ + 0, + 30 + ], + "dataset": "go2_short" + }, + { + "id": "go2_short_disp_30_60", + "q": "How far in a straight line is your final position from your position at the first shown observation, in meters?", + "a": 11.8, + "band": 4.738980277235515, + "stream": "odom", + "window": [ + 30, + 60 + ], + "dataset": "go2_short" + }, + { + "id": "go2_short_path_0_60", + "q": "Roughly how many meters did you travel in total over these observations (path length, not displacement)?", + "a": 37.9, + "band": 18.963056711366217, + "stream": "odom", + "window": [ + 0, + 60 + ], + "dataset": "go2_short" + }, + { + "id": "go2_hongkong_office_disp_0_558", + "q": "How far in a straight line is your final position from your position at the first shown observation, in meters?", + "a": 9.6, + "band": 3.8486174236058126, + "stream": "odom", + "window": [ + 0, + 558 + ], + "dataset": "go2_hongkong_office" + }, + { + "id": "go2_hongkong_office_disp_100_300", + "q": "How far in a straight line is your final position from your position at the first shown observation, in meters?", + "a": 27.5, + "band": 11.016777380908813, + "stream": "odom", + "window": [ + 100, + 300 + ], + "dataset": "go2_hongkong_office" + }, + { + "id": "go2_hongkong_office_path_0_558", + "q": "Roughly how many meters did you travel in total over these observations (path length, not displacement)?", + "a": 192.5, + "band": 96.26788393075581, + "stream": "odom", + "window": [ + 0, + 558 + ], + "dataset": "go2_hongkong_office" + } +] diff --git a/dimos/evals/suites/go2_vqa.py b/dimos/evals/suites/go2_vqa.py new file mode 100644 index 0000000000..8377434e78 --- /dev/null +++ b/dimos/evals/suites/go2_vqa.py @@ -0,0 +1,73 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generated VQA suite over the go2 replays. + +Rows (``go2_vqa.json``) are pure data emitted by :mod:`dimos.evals.generate` — +ground truth computed analytically from odom, quizzing the encoded odom +summary. Typing and scoring live here; the JSON stays behavior-free. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from dimos.evals.scorers import exact, first_number, within, yes_no +from dimos.evals.types import PassiveEval, Suite + +_ROWS = json.loads((Path(__file__).parent / "go2_vqa.json").read_text()) + +_generated: list[PassiveEval[float]] = [ + PassiveEval( + id=str(row["id"]), + inputs=str(row["q"]), + expected=float(row["a"]), # type: ignore[arg-type] + parse=first_number, + score=within(float(row["band"])), # type: ignore[arg-type] + context=( + lambda s, name=str(row["stream"]), w=tuple(row["window"]): # type: ignore[misc] + s.streams[name].range_time(*w), + ), + dataset=str(row["dataset"]), + tags=frozenset({"generated", "odom", "numeric"}), + ) + for row in _ROWS +] + +# Hand-labeled presence questions, verified against the recording imagery. +_hand: list[PassiveEval[str]] = [ + PassiveEval( + id="hk_couch_seen", + inputs="Did you see a couch or sofa at any point?", + expected="yes", + parse=yes_no, + score=exact, + context=(lambda s: s.streams.color_image.range_time(150, 250),), + dataset="go2_hongkong_office", + tags=frozenset({"image", "presence"}), + ), + PassiveEval( + id="hk_plants_seen", + inputs="Did you see any potted plants?", + expected="yes", + parse=yes_no, + score=exact, + context=(lambda s: s.streams.color_image.range_time(0, 60),), + dataset="go2_hongkong_office", + tags=frozenset({"image", "presence"}), + ), +] + +SUITE: Suite = [*_generated, *_hand] diff --git a/dimos/evals/test_evals.py b/dimos/evals/test_evals.py new file mode 100644 index 0000000000..3dd97c8289 --- /dev/null +++ b/dimos/evals/test_evals.py @@ -0,0 +1,312 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline unit tests: scorers, case dispatch, preflight, runner artifacts. + +No network, no robot, no LLM — the chat model is a fake and the rig in case +tests is a plain object satisfying the EvalRig protocol structurally. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +import json +from pathlib import Path +from typing import Any + +import pytest + +from dimos.evals.scorers import ( + choice, + exact, + final, + first_number, + floor, + mean, + ramp, + within, + yes_no, +) +from dimos.evals.types import EvalCase, InteractiveEval, PassiveEval +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Vector3 import make_vector3 + + +def _pose(x: float, y: float) -> PoseStamped: + return PoseStamped( + position=make_vector3(x, y, 0.0), + orientation=Quaternion(0.0, 0.0, 0.0, 1.0), + frame_id="world", + ) + + +@pytest.fixture +def dataset(tmp_path: Path) -> str: + """A tiny on-disk mem2 dataset: 5 odom poses walking 4m in +x over 4s.""" + from dimos.memory2.store.sqlite import SqliteStore + + path = tmp_path / "tiny.db" + try: + store = SqliteStore(path=str(path)) + except Exception as e: # pragma: no cover — sqlite-vec unavailable platforms + pytest.skip(f"SqliteStore unavailable: {e}") + stream = store.stream("odom", PoseStamped) + for i in range(5): + stream.append(_pose(float(i), 0.0), ts=1000.0 + i) + store.stop() + return str(path) + + +class FakeRig: + """Structural EvalRig for case-level tests.""" + + blind = False + mcp_url = "http://localhost:9990/mcp" + + def __init__(self, answer: str = "", series: list[tuple[float, float]] | None = None): + self.answer = answer + self.series = series or [] + self.calls: list[str] = [] + + def open_dataset(self, name: str) -> Any: + from dimos.memory2.cli.dataset import open_dataset + + return open_dataset(name) + + def live_store(self) -> Any: + raise NotImplementedError + + def encode(self, stream: Any) -> list[dict[str, Any]]: + return [{"type": "text", "text": f"{len(list(stream))} observations"}] + + def ask(self, context: Sequence[dict[str, Any]], question: str) -> str: + self.calls.append("ask") + return self.answer + + def call_skill(self, name: str, args: Mapping[str, object]) -> str: + self.calls.append(f"skill:{name}") + return self.answer + + def agent_loop(self, case: EvalCase) -> str: + self.calls.append("agent_loop") + return self.answer + + def mcp_ready(self) -> bool: + return False + + def setup_env(self, case: InteractiveEval) -> None: + self.calls.append("setup_env") + + def check_env(self, case: InteractiveEval) -> None: + pass + + def instruct(self, text: str) -> None: + self.calls.append(f"instruct:{text}") + + def sample( + self, score: Callable[[Any], float], interval_s: float, timeout_s: float + ) -> list[tuple[float, float]]: + return self.series + + +# -- scorers ------------------------------------------------------------------------ + + +def test_scorer_math() -> None: + assert exact(6, 6) == 1.0 + assert exact("yes", "no") == 0.0 + assert within(2.0)(10.0, 10.0) == 1.0 + assert within(2.0)(10.0, 11.0) == 0.5 + assert within(2.0)(10.0, 13.0) == 0.0 + assert ramp(0.0, band=2.0) == 1.0 + assert ramp(1.0, band=2.0) == 0.5 + assert ramp(5.0, band=2.0) == 0.0 + assert final([0.1, 0.9]) == 0.9 + assert floor([0.4, 0.2, 0.8]) == 0.2 + assert mean([0.0, 1.0]) == 0.5 + + +def test_parsers() -> None: + assert first_number("about 12.5 meters") == 12.5 + assert first_number("-3") == -3.0 + with pytest.raises(ValueError): + first_number("none") + assert yes_no("Yes, there is.") == "yes" + assert yes_no("no") == "no" + with pytest.raises(ValueError): + yes_no("maybe") + assert choice(" Chairs. ") == "chairs" + + +# -- case dispatch --------------------------------------------------------------------- + + +def test_passive_model_path(dataset: str) -> None: + case = PassiveEval( + id="disp", + inputs="how far?", + expected=4.0, + parse=first_number, + score=within(1.0), + context=(lambda s: s.streams.odom,), + dataset=dataset, + ) + rig = FakeRig(answer="about 4 meters") + result = case.evaluate(rig) + assert result.score == 1.0 + assert rig.calls == ["ask"] + + +def test_passive_skill_path(dataset: str) -> None: + case = PassiveEval( + id="sk", + inputs="", + skill="detect", + skill_args={"query": "person"}, + expected="yes", + parse=yes_no, + dataset=dataset, + ) + rig = FakeRig(answer="yes") + assert case.evaluate(rig).score == 1.0 + assert rig.calls == ["skill:detect"] + + +def test_interactive_dispatch() -> None: + case = InteractiveEval( + id="nav", + inputs="go to the bed", + score=lambda store: 1.0, + aggregate=floor, + simulator="", + ) + rig = FakeRig(series=[(0.0, 0.2), (1.0, 0.6), (2.0, 0.9)]) + result = case.evaluate(rig) + assert result.score == 0.2 # floor aggregate + assert result.series == ((0.0, 0.2), (1.0, 0.6), (2.0, 0.9)) + assert rig.calls == ["setup_env", "instruct:go to the bed"] + + +def test_interactive_no_samples_is_error() -> None: + case = InteractiveEval(id="n", inputs="x", score=lambda s: 1.0, simulator="") + assert "no samples" in case.evaluate(FakeRig()).error + + +# -- preflight ---------------------------------------------------------------------- + + +def test_preflight_missing_stream(dataset: str) -> None: + case = PassiveEval( + id="bad", + inputs="?", + expected=1.0, + parse=first_number, + context=(lambda s: s.streams.lidar.limit(1),), + dataset=dataset, + ) + with pytest.raises(AttributeError, match="No stream 'lidar'"): + case.preflight(FakeRig()) + + +def test_preflight_needs_mcp(dataset: str) -> None: + case = PassiveEval( + id="needs_mcp", + inputs="?", + expected="yes", + parse=yes_no, + tools=True, + dataset=dataset, + ) + with pytest.raises(RuntimeError, match="needs MCP"): + case.preflight(FakeRig()) + + +# -- runner ------------------------------------------------------------------------ + + +def test_runner_end_to_end_offline(dataset: str, tmp_path: Path) -> None: + from langchain_core.language_models.fake_chat_models import FakeListChatModel + + from dimos.evals.runner import EvalRunner, summarize + + cases = [ + PassiveEval( + id="disp", + inputs="straight-line distance in meters?", + expected=4.0, + parse=first_number, + score=within(1.0), + context=(lambda s: s.streams.odom,), + dataset=dataset, + ), + PassiveEval( # parse failure -> error result, run survives + id="unparseable", + inputs="?", + expected=1.0, + parse=first_number, + context=(lambda s: s.streams.odom.limit(1),), + dataset=dataset, + ), + PassiveEval( # preflight failure -> error result, run survives + id="missing_stream", + inputs="?", + expected=1.0, + parse=first_number, + context=(lambda s: s.streams.lidar,), + dataset=dataset, + ), + ] + runner = EvalRunner( + chat_model=FakeListChatModel(responses=["4.0", "no numbers here"]), + out_dir=tmp_path / "evals", + ) + results = runner.run(cases) + + by_id = {r.case_id: r for r in results} + assert by_id["disp"].passed and by_id["disp"].score == 1.0 + assert "ValueError" in by_id["unparseable"].error + assert by_id["missing_stream"].error.startswith("preflight:") + + s = summarize(results) + assert s.n == 3 and s.errors == 2 + + run_dir = runner.run_dir + lines = (run_dir / "results.jsonl").read_text().strip().splitlines() + assert len(lines) == 3 + summary = json.loads((run_dir / "summary.json").read_text()) + assert summary["n"] == 3 and "model" in summary + + +def test_runner_encode_budget(dataset: str, tmp_path: Path) -> None: + from dimos.evals.runner import EvalRunner + + runner = EvalRunner(context_budget=3, out_dir=tmp_path) + store = runner.open_dataset(dataset) + try: + blocks = runner.encode(store.streams.odom) + finally: + store.stop() + # 1 header + 3 sampled observations, all text (PoseStamped str fallback) + assert len(blocks) == 4 + assert all(b["type"] == "text" for b in blocks) + assert "pos=" in blocks[1]["text"] + + +def test_suites_importable() -> None: + """Suite modules construct without data or network (lambdas stay lazy).""" + from dimos.evals.suites import dimsim_house, examples, go2_smoke, go2_vqa + + for module in (examples, go2_smoke, go2_vqa, dimsim_house): + assert module.SUITE, module.__name__ diff --git a/dimos/evals/test_smoke.py b/dimos/evals/test_smoke.py new file mode 100644 index 0000000000..891d3db11d --- /dev/null +++ b/dimos/evals/test_smoke.py @@ -0,0 +1,38 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Live passive smoke: real model, real LFS recordings. Self-hosted + API key.""" + +from __future__ import annotations + +import pytest + +pytestmark = [pytest.mark.self_hosted, pytest.mark.skipif_no_openai] + + +def test_passive_smoke(tmp_path) -> None: # type: ignore[no-untyped-def] + from dimos.evals.runner import EvalRunner, summarize + from dimos.evals.suites.examples import SUITE + from dimos.utils.data import get_data + + get_data("go2_short.db") + + runner = EvalRunner(model="gpt-4o-mini", out_dir=tmp_path / "evals") + results = runner.run(SUITE) + + assert not any(r.error for r in results), [r.error for r in results] + s = summarize(results) + # The lidar-points case reads a number embedded in the str() encoding and + # the image case is unambiguous — a competent VLM should clear both. + assert s.mean_score >= 0.5 diff --git a/dimos/evals/types.py b/dimos/evals/types.py new file mode 100644 index 0000000000..be54495621 --- /dev/null +++ b/dimos/evals/types.py @@ -0,0 +1,198 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Eval case primitives. + +Two taxonomies, orthogonal: + +- **Passive** evals: world state is immutable (a frozen frame or replay, any + time window). The model's output never feeds back into its input. Cheap, + deterministic, repeatable. +- **Interactive** evals: actions feed back into observations; state is + mutable. Needs sim or a real robot, scored by sampling the live memory2 + store the robot's Recorder writes. + +Suites are Python modules exporting ``SUITE: Suite`` (behavior is typed code; +JSON holds only data rows). memory2 is the source of truth for all input and +perception: context selectors return real :class:`~dimos.memory2.stream.Stream` +objects and scoring reads :class:`~dimos.memory2.store.base.Store`. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Generic, Protocol, TypeVar + +from dimos.evals.scorers import exact, final + +if TYPE_CHECKING: + from dimos.e2e_tests.dim_sim_client import DimSimClient + from dimos.memory2.store.base import Store + from dimos.memory2.stream import Stream + +T = TypeVar("T") + +Select = Callable[["Store"], "Stream[Any, Any]"] +"""Context selector — hands the model real mem2 streams, whole or windowed:: + + lambda s: s.streams.lidar.limit(1) + lambda s: s.streams.odom.range_time(0, 600) +""" + + +@dataclass(frozen=True, kw_only=True) +class EvalResult: + case_id: str + outputs: str = "" + score: float = 0.0 + passed: bool = False + duration_s: float = 0.0 + error: str = "" + series: tuple[tuple[float, float], ...] = () # (t, score) — interactive only + transcript: str = "" # path within the run dir, when an agent loop ran + + +class EvalRig(Protocol): + """What a case may ask of the runner. :class:`EvalRunner` implements this + structurally — no import cycle, mypy-checked at call sites, and a fake rig + in tests is any object with these methods.""" + + @property + def blind(self) -> bool: ... + @property + def mcp_url(self) -> str: ... + + def open_dataset(self, name: str) -> Store: ... + def live_store(self) -> Store: ... + def encode(self, stream: Stream[Any, Any]) -> list[dict[str, Any]]: ... + def ask(self, context: Sequence[dict[str, Any]], question: str) -> str: ... + def call_skill(self, name: str, args: Mapping[str, object]) -> str: ... + def agent_loop(self, case: EvalCase) -> str: ... + def mcp_ready(self) -> bool: ... + def setup_env(self, case: InteractiveEval) -> None: ... + def check_env(self, case: InteractiveEval) -> None: ... + def instruct(self, text: str) -> None: ... + def sample( + self, score: Callable[[Store], float], interval_s: float, timeout_s: float + ) -> list[tuple[float, float]]: ... + + +@dataclass(frozen=True, kw_only=True) +class EvalCase(ABC): + """Common surface the runner, report, and filters operate on. + + ``skill`` set -> score one tool call (no agent loop): ``detect()`` on a + replay, ``grasp()`` in sim. ``skill`` empty -> subclass decides. + """ + + id: str + inputs: str + skill: str = "" + skill_args: Mapping[str, object] = field(default_factory=dict) + tags: frozenset[str] = frozenset() + timeout_s: float = 60.0 + + @abstractmethod + def evaluate(self, rig: EvalRig) -> EvalResult: + """Produce this case's result using the rig's resources.""" + + def preflight(self, rig: EvalRig) -> None: + """Raise with a precise message if this case cannot run on this rig. + + Cheap: resolves resources, reads no data, starts no processes. + """ + if self.skill and not rig.mcp_ready(): + raise RuntimeError(f"{self.id}: needs MCP at {rig.mcp_url}, nothing listening") + + +@dataclass(frozen=True, kw_only=True) +class PassiveEval(EvalCase, Generic[T]): + """World state immutable; ``T`` ties ``expected``/``parse``/``score`` + together so mypy checks the triple agrees per case.""" + + expected: T + parse: Callable[[str], T] + score: Callable[[T, T], float] = exact + context: tuple[Select, ...] = () + dataset: str = "go2_short" + tools: bool = False # True: full agent loop over the frozen store + + def evaluate(self, rig: EvalRig) -> EvalResult: + store = rig.open_dataset(self.dataset) + try: + if self.skill: + outputs = rig.call_skill(self.skill, self.skill_args) + elif self.tools: + outputs = rig.agent_loop(self) + else: + blocks = ( + [] if rig.blind else [b for sel in self.context for b in rig.encode(sel(store))] + ) + outputs = rig.ask(blocks, self.inputs) + finally: + store.stop() + got = self.parse(outputs) + return EvalResult(case_id=self.id, outputs=outputs, score=self.score(self.expected, got)) + + def preflight(self, rig: EvalRig) -> None: + store = rig.open_dataset(self.dataset) # raises: dataset unresolvable + try: + for sel in self.context: + sel(store) # raises: "No stream 'x'. Available: [...]" — no data read + finally: + store.stop() + if (self.skill or self.tools) and not rig.mcp_ready(): + raise RuntimeError(f"{self.id}: needs MCP at {rig.mcp_url}, nothing listening") + + +def _no_setup(sim: DimSimClient) -> None: + return None + + +@dataclass(frozen=True, kw_only=True) +class InteractiveEval(EvalCase): + """Actions feed back into observations. The case names its environment so + the eval is reproducible; the runner only decides attach-vs-launch.""" + + score: Callable[[Store], float] # sampled every interval_s against live mem2 + aggregate: Callable[[Sequence[float]], float] = final + interval_s: float = 1.0 + timeout_s: float = 300.0 + blueprint: str = "unitree-go2-agentic" + simulator: str = "dimsim" # "" = attach to a running dimos / real robot + scene: str = "apartment" # --dimsim-scene name (ScenePackage name later) + setup: Callable[[DimSimClient], None] = _no_setup + + def evaluate(self, rig: EvalRig) -> EvalResult: + rig.setup_env(self) + if self.skill: + rig.call_skill(self.skill, self.skill_args) + else: + rig.instruct(self.inputs) + series = rig.sample(self.score, self.interval_s, self.timeout_s) + if not series: + return EvalResult(case_id=self.id, error=f"{self.id}: no samples collected") + return EvalResult( + case_id=self.id, + score=self.aggregate([s for _, s in series]), + series=tuple(series), + ) + + def preflight(self, rig: EvalRig) -> None: + rig.check_env(self) + + +Suite = Sequence[EvalCase] diff --git a/pyproject.toml b/pyproject.toml index 1fc0193bab..a2bc303efd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -226,6 +226,7 @@ agents = [ "langchain-huggingface>=1,<2", "langchain-ollama>=1,<2", "ollama>=0.6.0", + "openevals>=0.1", # eval scorers (LLM-as-judge etc.) — dimos/evals # Audio "openai", diff --git a/uv.lock b/uv.lock index e2313b7058..0eac32ef6b 100644 --- a/uv.lock +++ b/uv.lock @@ -1640,6 +1640,7 @@ agents = [ { name = "langchain-openai" }, { name = "ollama" }, { name = "openai" }, + { name = "openevals" }, { name = "sounddevice" }, ] all = [ @@ -1680,6 +1681,7 @@ all = [ { name = "onnxruntime-gpu", marker = "platform_machine == 'x86_64'" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "piper-sdk" }, { name = "playground" }, @@ -1735,6 +1737,7 @@ base = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "rerun-sdk" }, { name = "sounddevice" }, @@ -1845,6 +1848,7 @@ unitree = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "rerun-sdk" }, { name = "sounddevice" }, @@ -1879,6 +1883,7 @@ unitree-dds = [ { name = "ollama" }, { name = "omegaconf" }, { name = "openai" }, + { name = "openevals" }, { name = "pillow" }, { name = "rerun-sdk" }, { name = "sounddevice" }, @@ -2145,6 +2150,7 @@ requires-dist = [ { name = "open3d-unofficial-arm", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'", specifier = ">=0.19.0.post9" }, { name = "openai", marker = "extra == 'agents'" }, { name = "opencv-contrib-python", specifier = ">=4.8,<5" }, + { name = "openevals", marker = "extra == 'agents'", specifier = ">=0.1" }, { name = "packaging", specifier = ">=24.0" }, { name = "pandas", marker = "extra == 'learning'" }, { name = "pillow", marker = "extra == 'perception'" }, @@ -5908,6 +5914,21 @@ dependencies = [ { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, ] +[[package]] +name = "openevals" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain" }, + { name = "langchain-openai" }, + { name = "langsmith" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/b1/028a05846136805b29b7a3afb58a940c2d213fa2c3d0a7d7003c7fbaa115/openevals-0.2.0.tar.gz", hash = "sha256:7e95fa64625be53eaa8c657d7f69b842a52bda10bdf3bb91781c7d09a385b069", size = 140711, upload-time = "2026-04-07T19:45:22.749Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/8b/00f402b7f3475e235c339a9bc82d2eaf46cc08b53ecd85d4a117850170d3/openevals-0.2.0-py3-none-any.whl", hash = "sha256:2bce5964be9d162e3d38c2dfd026739156e1ac521536ade6b8e2f0a89b632f2c", size = 106958, upload-time = "2026-04-07T19:45:21.575Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.42.1" From 4ea607ab548096c7c29f327ecdb092387a83bf3b Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 19:56:41 -0700 Subject: [PATCH 02/12] feat(evals): keyless local eval model (moondream2 chat adapter) --- dimos/evals/local.py | 105 +++++++++++++++++++++++++++++++++ dimos/evals/suites/examples.py | 9 ++- 2 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 dimos/evals/local.py diff --git a/dimos/evals/local.py b/dimos/evals/local.py new file mode 100644 index 0000000000..2281b11ae1 --- /dev/null +++ b/dimos/evals/local.py @@ -0,0 +1,105 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Keyless local eval model: MoondreamVlModel behind the chat-model interface. + +Lets ``EvalRunner(chat_model=MoondreamChat())`` run passive evals with zero API +keys on any GPU box. Moondream is single-image, so multi-image contexts are +tiled into one contact sheet; text blocks concatenate into the question. +""" + +from __future__ import annotations + +import base64 +from functools import cached_property +from typing import Any + +from langchain_core.callbacks import CallbackManagerForLLMRun +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import AIMessage, BaseMessage +from langchain_core.outputs import ChatGeneration, ChatResult +import numpy as np + +from dimos.msgs.sensor_msgs.Image import Image + + +class MoondreamChat(BaseChatModel): + """Chat-model adapter over the local moondream2 VLM (dimos MoondreamVlModel).""" + + tile_columns: int = 3 + + @property + def _llm_type(self) -> str: + return "moondream-local" + + @cached_property + def _vl(self) -> Any: + from dimos.models.vl.moondream import MoondreamVlModel + + model = MoondreamVlModel() + model.start() + return model + + def _generate( + self, + messages: list[BaseMessage], + stop: list[str] | None = None, + run_manager: CallbackManagerForLLMRun | None = None, + **kwargs: Any, + ) -> ChatResult: + texts: list[str] = [] + frames: list[np.ndarray[Any, Any]] = [] + for message in messages: + content = message.content + if isinstance(content, str): + texts.append(content) + continue + for block in content: + if not isinstance(block, dict): + texts.append(str(block)) + elif block.get("type") == "text": + texts.append(str(block["text"])) + elif block.get("type") == "image_url": + frames.append(_decode_data_uri(str(block["image_url"]["url"]))) + + image = Image.from_numpy(_tile(frames) if frames else _BLANK) + answer = self._vl.query(image, "\n".join(texts)) + return ChatResult(generations=[ChatGeneration(message=AIMessage(content=str(answer)))]) + + +_BLANK = np.full((64, 64, 3), 128, dtype=np.uint8) + + +def _decode_data_uri(uri: str) -> np.ndarray[Any, Any]: + import cv2 + + payload = uri.split(",", 1)[1] + buffer = np.frombuffer(base64.b64decode(payload), dtype=np.uint8) + frame: np.ndarray[Any, Any] = cv2.imdecode(buffer, cv2.IMREAD_COLOR) + return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + +def _tile(frames: list[np.ndarray[Any, Any]], columns: int = 3) -> np.ndarray[Any, Any]: + """Grid-tile frames into one contact sheet (moondream is single-image).""" + if len(frames) == 1: + return frames[0] + height = min(f.shape[0] for f in frames) + width = min(f.shape[1] for f in frames) + import cv2 + + resized = [cv2.resize(f, (width, height)) for f in frames] + rows = [np.hstack(resized[i : i + columns]) for i in range(0, len(resized), columns)] + max_w = max(r.shape[1] for r in rows) + rows = [np.pad(r, ((0, 0), (0, max_w - r.shape[1]), (0, 0))) for r in rows] + return np.vstack(rows) diff --git a/dimos/evals/suites/examples.py b/dimos/evals/suites/examples.py index 4dda943a9e..c4f5f700d3 100644 --- a/dimos/evals/suites/examples.py +++ b/dimos/evals/suites/examples.py @@ -45,16 +45,19 @@ tags=frozenset({"example", "encoding", "pointcloud"}), ) -# A range of 10 image frames from the same replay: seconds 45..61 of the +# A range of 10 image frames from the same replay: seconds 58..61 of the # recording, capped to 10 observations. Image.agent_encode() turns each into an -# image content block; the person at the table appears near the end. +# image content block; a person sits at a table in this stretch. +# Note: `.limit(n)` keeps the *first* n observations of the window — for a +# spread across a long window, give the runner the whole range and let its +# context budget downsample evenly instead. ten_image_range = PassiveEval( id="example_ten_image_range", inputs="Is a person visible in any of these images?", expected="yes", parse=yes_no, score=exact, - context=(lambda s: s.streams.color_image.range_time(45, 61).limit(10),), + context=(lambda s: s.streams.color_image.range_time(58, 61).limit(10),), dataset="go2_short", tags=frozenset({"example", "encoding", "image"}), ) From f99a4931d4d540da2c2bdd29f9d3ddcf655e2c15 Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 20:57:56 -0700 Subject: [PATCH 03/12] test(evals): mem2 wiring integration tests (passive prompt path + live-store sampling) --- dimos/evals/local.py | 4 +- dimos/evals/test_mem2_wiring.py | 208 ++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 dimos/evals/test_mem2_wiring.py diff --git a/dimos/evals/local.py b/dimos/evals/local.py index 2281b11ae1..d0537db1f6 100644 --- a/dimos/evals/local.py +++ b/dimos/evals/local.py @@ -86,7 +86,9 @@ def _decode_data_uri(uri: str) -> np.ndarray[Any, Any]: payload = uri.split(",", 1)[1] buffer = np.frombuffer(base64.b64decode(payload), dtype=np.uint8) - frame: np.ndarray[Any, Any] = cv2.imdecode(buffer, cv2.IMREAD_COLOR) + frame = cv2.imdecode(buffer, cv2.IMREAD_COLOR) + if frame is None: + raise ValueError("undecodable image data URI") return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) diff --git a/dimos/evals/test_mem2_wiring.py b/dimos/evals/test_mem2_wiring.py new file mode 100644 index 0000000000..47b9d7b91e --- /dev/null +++ b/dimos/evals/test_mem2_wiring.py @@ -0,0 +1,208 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Integration tests for the memory2 <-> EvalCase connection. + +Passive: a case's context Selects pull real Streams from a recording, the +runner encodes them, and the *actual observation data* (image blocks, pose +text) reaches the model prompt. + +Interactive: a case's score callable reads the *live* store while a writer is +appending — the mem2 analogue of a robot's Recorder running mid-task. +""" + +from __future__ import annotations + +from pathlib import Path +import threading +import time +from typing import Any + +from langchain_core.callbacks import CallbackManagerForLLMRun +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import AIMessage, BaseMessage +from langchain_core.outputs import ChatGeneration, ChatResult +import numpy as np +import pytest + +from dimos.evals.runner import EvalRunner +from dimos.evals.scorers import final, first_number, ramp, within +from dimos.evals.types import InteractiveEval, PassiveEval +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Vector3 import make_vector3 +from dimos.msgs.sensor_msgs.Image import Image + + +def _pose(x: float, y: float) -> PoseStamped: + return PoseStamped( + position=make_vector3(x, y, 0.0), + orientation=Quaternion(0.0, 0.0, 0.0, 1.0), + frame_id="world", + ) + + +def _open_store(path: Path) -> Any: + from dimos.memory2.store.sqlite import SqliteStore + + try: + return SqliteStore(path=str(path)) + except Exception as e: # pragma: no cover — sqlite-vec unavailable platforms + pytest.skip(f"SqliteStore unavailable: {e}") + + +class SpyChat(BaseChatModel): + """Captures the exact messages the runner sends; replies with a constant.""" + + reply: str = "42" + seen: list[list[BaseMessage]] = [] + + @property + def _llm_type(self) -> str: + return "spy" + + def _generate( + self, + messages: list[BaseMessage], + stop: list[str] | None = None, + run_manager: CallbackManagerForLLMRun | None = None, + **kwargs: Any, + ) -> ChatResult: + self.seen.append(messages) + return ChatResult(generations=[ChatGeneration(message=AIMessage(content=self.reply))]) + + +# -- passive: recording -> Select -> encode -> prompt -------------------------------- + + +def test_passive_streams_reach_the_prompt(tmp_path: Path) -> None: + store = _open_store(tmp_path / "rec.db") + odom = store.stream("odom", PoseStamped) + for i in range(20): + odom.append(_pose(float(i), 2.5), ts=1000.0 + i) + frame = np.full((16, 16, 3), 200, dtype=np.uint8) + images = store.stream("color_image", Image) + for i in range(3): + images.append(Image.from_numpy(frame, frame_id="cam", ts=1000.0 + i), ts=1000.0 + i) + store.stop() + + case = PassiveEval( + id="wiring", + inputs="how far along x did you travel?", + expected=19.0, + parse=first_number, + score=within(1.0), + context=( + lambda s: s.streams.odom.range_time(0, 100), + lambda s: s.streams.color_image.limit(2), + ), + dataset=str(tmp_path / "rec.db"), + ) + + spy = SpyChat(reply="19") + spy.seen.clear() + runner = EvalRunner(chat_model=spy, out_dir=tmp_path / "evals") + results = runner.run([case]) + + assert results[0].passed, results[0] + blocks = [b for m in spy.seen[0] for b in (m.content if isinstance(m.content, list) else [])] + image_blocks = [b for b in blocks if b.get("type") == "image_url"] + text = " ".join(b["text"] for b in blocks if b.get("type") == "text") + # the actual observation data crossed from mem2 into the prompt: + assert len(image_blocks) == 2, "both selected image observations should be encoded" + assert image_blocks[0]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert "pos=[0.000, 2.500" in text.replace(" ", " ") or "0.000" in text + assert "19.000" in text or "19.0" in text, "last odom pose must reach the prompt" + assert case.inputs in text + + +def test_passive_context_budget_downsamples_not_truncates(tmp_path: Path) -> None: + store = _open_store(tmp_path / "rec.db") + odom = store.stream("odom", PoseStamped) + for i in range(100): + odom.append(_pose(float(i), 0.0), ts=1000.0 + i) + store.stop() + + runner = EvalRunner(context_budget=5, out_dir=tmp_path / "evals") + reopened = runner.open_dataset(str(tmp_path / "rec.db")) + try: + blocks = runner.encode(reopened.streams.odom) + finally: + reopened.stop() + texts = [b["text"] for b in blocks[1:]] # skip header + assert len(texts) == 5 + assert "0.000" in texts[0] and "99.000" in texts[-1], "spread must cover the whole window" + + +# -- interactive: live store -> score sampling ---------------------------------------- + + +def test_interactive_scores_live_store_while_writing(tmp_path: Path) -> None: + """A writer thread plays the Recorder role: the case's score callable must + see fresh observations appear in the live store as they are appended.""" + db = tmp_path / "live.db" + store = _open_store(db) + odom = store.stream("odom", PoseStamped) + odom.append(_pose(5.0, 0.0), ts=time.time()) # robot starts 5m from goal + + stop = threading.Event() + + def writer() -> None: + for i in range(1, 26): + if stop.is_set(): + return + odom.append(_pose(max(0.0, 5.0 - i * 0.2), 0.0), ts=time.time()) + time.sleep(0.05) + + thread = threading.Thread(target=writer) + + class NoEnvRunner(EvalRunner): + """Rig with the sim/MCP environment stubbed out — mem2 path stays real.""" + + def check_env(self, case: InteractiveEval) -> None: + pass + + def setup_env(self, case: InteractiveEval) -> None: + thread.start() + + def instruct(self, text: str) -> None: + pass + + case = InteractiveEval( + id="live_wiring", + inputs="go to the goal", + score=lambda s: ramp(abs(s.streams.odom.last().data.position.x), band=2.0), + aggregate=final, + interval_s=0.1, + timeout_s=10.0, + simulator="", + ) + + runner = NoEnvRunner(live_db=str(db), out_dir=tmp_path / "evals") + try: + results = runner.run([case]) + finally: + stop.set() + if thread.ident is not None: + thread.join(timeout=5.0) + store.stop() + + r = results[0] + assert not r.error, r.error + assert len(r.series) >= 3, "sampler must observe multiple live states" + scores = [s for _, s in r.series] + assert scores[0] < 0.9, "first sample sees the robot far from the goal" + assert scores[-1] >= 0.99, "last sample sees the robot arrive (live data flowed)" + assert r.score >= 0.99 # aggregate=final + assert scores == sorted(scores), "monotonic approach must be visible in the series" From 09d0f5db801577b67f759554200f8abfc15ed562 Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 21:15:56 -0700 Subject: [PATCH 04/12] fix(evals): wait for odom before dimsim house exploration --- dimos/evals/suites/dimsim_house.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dimos/evals/suites/dimsim_house.py b/dimos/evals/suites/dimsim_house.py index 3e685f7c89..975264a564 100644 --- a/dimos/evals/suites/dimsim_house.py +++ b/dimos/evals/suites/dimsim_house.py @@ -64,8 +64,22 @@ def _explore_house(sim: DimSimClient) -> None: + import time + + from dimos.core.transport import LCMTransport + from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.simulation.mujoco.direct_cmd_vel_explorer import DirectCmdVelExplorer + # dimsim spawns the robot well after MCP is up — wait for odom before driving + seen: list[PoseStamped] = [] + probe: LCMTransport[PoseStamped] = LCMTransport("/odom", PoseStamped) + probe.subscribe(lambda msg, *args: seen.append(msg)) + deadline = time.time() + 180.0 + while not seen and time.time() < deadline: + time.sleep(1.0) + if not seen: + raise TimeoutError("no /odom within 180s — robot never spawned") + explorer = DirectCmdVelExplorer() explorer.linear_speed = 0.5 explorer.start() From a6e809a409382b37025dfa19fdda8378d5a50c95 Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 21:55:26 -0700 Subject: [PATCH 05/12] chore(evals): drop __init__.py files per repo convention (namespace packages, no __all__) --- dimos/evals/__init__.py | 48 ---------------------------------- dimos/evals/suites/__init__.py | 15 ----------- dimos/evals/suites/examples.py | 2 +- 3 files changed, 1 insertion(+), 64 deletions(-) delete mode 100644 dimos/evals/__init__.py delete mode 100644 dimos/evals/suites/__init__.py diff --git a/dimos/evals/__init__.py b/dimos/evals/__init__.py deleted file mode 100644 index f43b4f11cc..0000000000 --- a/dimos/evals/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""DimOS evals: passive (frozen mem2 recordings) and interactive (live robot/sim).""" - -from dimos.evals.runner import EvalRunner, EvalRunnerConfig, RunSummary, summarize -from dimos.evals.scorers import exact, final, floor, judge, mean, ramp, within -from dimos.evals.types import ( - EvalCase, - EvalResult, - EvalRig, - InteractiveEval, - PassiveEval, - Select, - Suite, -) - -__all__ = [ - "EvalCase", - "EvalResult", - "EvalRig", - "EvalRunner", - "EvalRunnerConfig", - "InteractiveEval", - "PassiveEval", - "RunSummary", - "Select", - "Suite", - "exact", - "final", - "floor", - "judge", - "mean", - "ramp", - "summarize", - "within", -] diff --git a/dimos/evals/suites/__init__.py b/dimos/evals/suites/__init__.py deleted file mode 100644 index f02f56de83..0000000000 --- a/dimos/evals/suites/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Eval suites. Each module exports ``SUITE: Suite``.""" diff --git a/dimos/evals/suites/examples.py b/dimos/evals/suites/examples.py index c4f5f700d3..288bf2665e 100644 --- a/dimos/evals/suites/examples.py +++ b/dimos/evals/suites/examples.py @@ -20,7 +20,7 @@ Or from pytest / a notebook:: - from dimos.evals import EvalRunner + from dimos.evals.runner import EvalRunner from dimos.evals.suites.examples import SUITE results = EvalRunner().run(SUITE) From 48c36de60acb9cbe1dbf317e0d5c1280ea1acea3 Mon Sep 17 00:00:00 2001 From: stash Date: Sat, 8 Aug 2026 22:42:15 -0700 Subject: [PATCH 06/12] chore(evals): keep model construction private to evals, no agents/ changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reuse mcp_client._init_model lazily instead of extracting it — keeps this PR scoped to dimos/evals (+ cli registration). extraction can be its own PR if we want it shared properly. --- dimos/agents/mcp/mcp_client.py | 19 ++++++++-- dimos/agents/mcp/test_mcp_client_unit.py | 2 +- dimos/agents/model.py | 44 ------------------------ dimos/evals/runner.py | 8 +++-- 4 files changed, 23 insertions(+), 50 deletions(-) delete mode 100644 dimos/agents/model.py diff --git a/dimos/agents/mcp/mcp_client.py b/dimos/agents/mcp/mcp_client.py index 6beddb6b07..859b15451b 100644 --- a/dimos/agents/mcp/mcp_client.py +++ b/dimos/agents/mcp/mcp_client.py @@ -20,15 +20,16 @@ import uuid from langchain.agents import create_agent +from langchain.chat_models import init_chat_model from langchain_core.messages import HumanMessage from langchain_core.messages.base import BaseMessage from langchain_core.tools import StructuredTool +from langchain_openai import ChatOpenAI from langgraph.graph.state import CompiledStateGraph from reactivex.disposable import Disposable import requests from dimos.agents.mcp import tool_stream -from dimos.agents.model import init_model from dimos.agents.system_prompt import SYSTEM_PROMPT from dimos.agents.utils import pretty_print_langchain_message from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT @@ -41,6 +42,20 @@ logger = setup_logger() +_RESPONSES_REASONING_MODEL_PREFIXES = ("gpt-5", "o1", "o3", "o4") + + +def _init_model(model_name: str) -> Any: + """Initialize a model while preserving LangChain provider resolution.""" + if ":" in model_name or not model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES): + return init_chat_model(model=model_name) + + return ChatOpenAI( + model=model_name, + use_responses_api=True, + reasoning={"effort": "medium", "summary": "auto"}, + ) + class McpClientConfig(ModuleConfig): system_prompt: str | None = SYSTEM_PROMPT @@ -218,7 +233,7 @@ def on_system_modules(self, _modules: list[RPCClient]) -> None: model = MockModel(json_path=self.config.model_fixture) else: - model = init_model(self.config.model) + model = _init_model(self.config.model) with self._lock: self._state_graph = create_agent( diff --git a/dimos/agents/mcp/test_mcp_client_unit.py b/dimos/agents/mcp/test_mcp_client_unit.py index dc1af78dbf..a49df130ff 100644 --- a/dimos/agents/mcp/test_mcp_client_unit.py +++ b/dimos/agents/mcp/test_mcp_client_unit.py @@ -260,7 +260,7 @@ def test_on_system_modules_resolves_non_reasoning_models( with ( patch("dimos.agents.mcp.mcp_client.create_agent"), - patch("dimos.agents.model.init_chat_model", return_value=resolved_model) as init, + patch("dimos.agents.mcp.mcp_client.init_chat_model", return_value=resolved_model) as init, ): configured_mcp_client.on_system_modules([]) diff --git a/dimos/agents/model.py b/dimos/agents/model.py deleted file mode 100644 index b71471c01f..0000000000 --- a/dimos/agents/model.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Shared chat-model construction for agents and evals.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from langchain.chat_models import init_chat_model -from langchain_openai import ChatOpenAI - -if TYPE_CHECKING: - from langchain_core.language_models.chat_models import BaseChatModel - -_RESPONSES_REASONING_MODEL_PREFIXES = ("gpt-5", "o1", "o3", "o4") - - -def init_model(model_name: str) -> BaseChatModel: - """Initialize a model while preserving LangChain provider resolution. - - OpenAI reasoning models (gpt-5*/o*) without an explicit ``provider:`` prefix - go through the Responses API with reasoning enabled — the same configuration - the production ``McpClient`` runs, so evals measure what deploys. - """ - if ":" in model_name or not model_name.startswith(_RESPONSES_REASONING_MODEL_PREFIXES): - return init_chat_model(model=model_name) - - return ChatOpenAI( - model=model_name, - use_responses_api=True, - reasoning={"effort": "medium", "summary": "auto"}, - ) diff --git a/dimos/evals/runner.py b/dimos/evals/runner.py index 9fdfec96ff..489ce11b25 100644 --- a/dimos/evals/runner.py +++ b/dimos/evals/runner.py @@ -59,7 +59,7 @@ class EvalRunnerConfig(BaseConfig): model: str = "gpt-5.6-luna" # mirrors McpClientConfig.model # House convention (StoreConfig): pass an instance to inject, e.g. a fake - # chat model in tests. None -> built from `model` via init_model(). + # chat model in tests. None -> built from `model` like McpClient does. chat_model: Any | None = None mcp_url: str = "http://localhost:9990/mcp" live_db: str = "recording.db" # store the Recorder writes (interactive) @@ -252,9 +252,11 @@ def model(self) -> BaseChatModel: if self.config.chat_model is not None: return self.config.chat_model # type: ignore[no-any-return] if self._model is None: - from dimos.agents.model import init_model + # Same construction as the production agent (Responses-API branch + # for gpt-5.x) so evals measure the deployed model config. + from dimos.agents.mcp.mcp_client import _init_model - self._model = init_model(self.config.model) + self._model = _init_model(self.config.model) return self._model def call_skill(self, name: str, args: Mapping[str, object]) -> str: From c67a2da2809617b60ea2e47556172de112c75636 Mon Sep 17 00:00:00 2001 From: stash Date: Sun, 9 Aug 2026 00:34:11 -0700 Subject: [PATCH 07/12] fix blueprint test --- dimos/robot/all_blueprints.py | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 58b941ae79..f6f7738ae8 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -192,6 +192,7 @@ "drone-tracking-module": "dimos.robot.drone.drone_tracking_module.DroneTrackingModule", "emitter-module": "dimos.utils.demo_image_encoding.EmitterModule", "episode-monitor-module": "dimos.imitation.collection.episode_monitor.EpisodeMonitorModule", + "eval-module": "dimos.evals.module.EvalModule", "evaluator": "dimos.navigation.nav_3d.evaluator.evaluator.Evaluator", "far-planner": "dimos.navigation.cmu_nav.modules.far_planner.far_planner.FarPlanner", "fast-lio2": "dimos.hardware.sensors.lidar.fastlio2.module.FastLio2", diff --git a/pyproject.toml b/pyproject.toml index a2bc303efd..31d62180e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -613,6 +613,7 @@ module = [ "mujoco_playground.*", "nav_msgs.*", "open_clip", + "openevals.*", "pinocchio", "pink", "pink.*", From 0ba6f9264b2e7e4513c1e0dc94a1b7323aa559d3 Mon Sep 17 00:00:00 2001 From: stash Date: Sun, 9 Aug 2026 00:36:15 -0700 Subject: [PATCH 08/12] docs(evals): runnable intro (memory2 doc conventions) --- dimos/evals/intro.md | 199 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 dimos/evals/intro.md diff --git a/dimos/evals/intro.md b/dimos/evals/intro.md new file mode 100644 index 0000000000..a8ed863180 --- /dev/null +++ b/dimos/evals/intro.md @@ -0,0 +1,199 @@ +# Evals Intro + +Evals measure what an agent (or a bare model, or a single skill) can do with +the robot's memory. Two kinds: + +- **Passive** — the world is a frozen memory2 recording. Deterministic, cheap, + repeatable. Run these constantly. +- **Interactive** — a live robot or sim; actions change the world; scoring + samples the live memory2 store while the agent works. + +memory2 is the source of truth for everything an eval sees: context selectors +return real `Stream`s, and interactive scoring reads a real `Store`. + +## Quick start (CLI) + +```bash +# two documentation cases against the go2_short recording (needs OPENAI_API_KEY) +dimos evals run dimos.evals.suites.examples + +# same questions with observations withheld — the guessing ablation +dimos evals run dimos.evals.suites.examples --blind + +# list available suites +dimos evals list +``` + +Each run prints a per-case table and writes `results.jsonl`, `summary.json`, +and per-case transcripts to `~/.local/state/dimos/evals/run-*/`. + +## Your first eval, end to end + +Build a tiny recording (any memory2 store works — this is the same API the +robot's Recorder uses; see `dimos/memory2/intro.md` for the full Stream API): + +```python session=evals ansi=false no-result +from dimos.memory2.store.sqlite import SqliteStore +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Vector3 import make_vector3 + +store = SqliteStore(path="/tmp/evals_intro.db") +odom = store.stream("odom", PoseStamped) +for i in range(20): + odom.append( + PoseStamped(position=make_vector3(float(i), 2.5, 0.0), + orientation=Quaternion(0, 0, 0, 1), frame_id="world"), + ts=1000.0 + i, + ) +``` + +```python session=evals ansi=false +print(odom.summary()) +``` + + +``` +Stream("odom"): 20 items, 1970-01-01 00:16:40 — 1970-01-01 00:16:59 (19.0s, 1.00 Hz, 1.68 KiB) +``` + +A passive eval is one Python literal. `context` is a tuple of callables that +receive the opened `Store` and return the mem2 `Stream`s the model may see — +anything the Stream API expresses (windows, filters, single frames) works, and +the runner evenly downsamples each selected stream to `context_budget` +observations before encoding: + +```python session=evals ansi=false no-result +from dimos.evals.scorers import first_number, within +from dimos.evals.types import PassiveEval + +case = PassiveEval( + id="how_far", + inputs="How far along x did you travel, in meters?", + expected=19.0, + parse=first_number, # model text -> float + score=within(1.0), # graded: 1.0 exact, linear to 0 at ±1m + context=(lambda s: s.streams.odom,), + dataset="/tmp/evals_intro.db", # a mem2 name ("go2_short") or a path +) +``` + +Run it. `chat_model=` injects any LangChain chat model — here a canned fake so +this document runs offline; drop the argument to use the production model +config (`gpt-5.6-luna`, same construction as the deployed `McpClient`): + +```python session=evals ansi=false +from langchain_core.language_models.fake_chat_models import FakeListChatModel +from dimos.evals.runner import EvalRunner, summarize + +runner = EvalRunner(chat_model=FakeListChatModel(responses=["about 19 meters"])) +result = runner.run([case])[0] +print(f"score={result.score} passed={result.passed} outputs={result.outputs!r}") +print(summarize([result])) +``` + + +``` +score=1.0 passed=True outputs='about 19 meters' +RunSummary(n=1, mean_score=1.0, pass_rate=1.0, errors=0, duration_s=0.31) +``` + +That's the whole loop: dataset -> context streams -> encoded prompt -> model +-> parse -> score -> artifacts. + +## Scoring + +Scores are floats in `[0, 1]`; `passed = score >= threshold`. Scorers are +plain functions `(expected, got) -> float` — a custom heuristic is a lambda, +not a class: + +```python session=evals ansi=false +from dimos.evals.scorers import choice, exact, first_number, ramp, within, yes_no + +print(exact("yes", "yes"), within(2.0)(10.0, 11.0), ramp(1.0, band=2.0)) +print(first_number("around 12.5 m"), yes_no("Yes, clearly."), choice(" Chairs. ")) +``` + + +``` +1.0 0.5 0.5 +12.5 yes chairs +``` + +- `exact` — equality (the default). Pair with a parser (`yes_no`, `choice`, + `int`) so formatting noise doesn't fail a correct answer. +- `within(band)` — graded numeric credit: 1.0 exact, 0.5 halfway, 0 outside. +- `ramp(distance, band)` — same ramp over meters; msg types support + arithmetic, so physical scorers stay one-liners: + `lambda s: ramp((GOAL - s.streams.odom.last().data.position).length(), band=0.5)` +- `judge(rubric)` — LLM-as-judge with partial credit, wrapping the + langchain/openevals standard (`inputs`/`reference_outputs` convention, so + external VQA benchmarks map on natively). + +Interactive evals score a *series* (one sample per `interval_s`); `aggregate` +reduces it: + +```python session=evals ansi=false +from dimos.evals.scorers import final, floor, mean + +print(final([0.2, 0.9]), floor([0.4, 0.2, 0.8]), mean([0.0, 1.0])) +``` + + +``` +0.9 0.2 0.5 +``` + +`final` = "where did it end up", `floor` = "never left the zone", +`mean` = "how good was it throughout". + +## Interactive evals + +The case names its environment (reproducibility); `score` reads the **live** +store the robot's Recorder writes, sampled every `interval_s`: + +```python session=evals ansi=false no-result +from dimos.evals.scorers import final, ramp +from dimos.evals.types import InteractiveEval +from dimos.msgs.geometry_msgs.Vector3 import Vector3 + +BED = Vector3(-3.567, -1.332, 0.0) + +go_to_bed = InteractiveEval( + id="go_to_bed", + inputs="go to the bed", + score=lambda s: ramp((BED - s.streams.odom.last().data.position).length(), band=2.0), + aggregate=final, + interval_s=2.0, + timeout_s=180.0, + blueprint="unitree-go2-agentic go2-memory", + simulator="dimsim", + scene="apartment", +) +``` + +```bash +dimos evals run dimos.evals.suites.dimsim_house --live-db recording_go2.db +``` + +The result carries the full `(t, score)` series — "reached the bed at t=50s +and stayed" and "grazed it at the deadline" score differently under `floor` +vs `final`. + +## Running + +- **CLI**: `dimos evals run [--tags nav --blind --limit 5 --model gpt-4o]` +- **Python**: `EvalRunner(...).run(SUITE, tags=frozenset({"encoding"}))` +- **pytest**: suites are importable lists — + `@pytest.mark.parametrize("case", SUITE)` and assert on `passed` + (gate live-model tests with `skipif_no_openai`). +- **MCP**: the `EvalModule` skills `run_evals` / `list_eval_suites` return the + summary + run dir, so a coding agent can run evals, grep transcripts, edit + prompts/encodings, and run again. +- **Blind ablation**: `EvalRunner(blind=True)` withholds all observations. A + case that still passes blind is guessable — fix its distractors. Run every + new suite sighted and blind once before trusting it. +- **Preflight**: before anything runs, every case is checked against the rig — + a missing stream fails with `"No stream 'lidar'. Available: [...]"`, a case + needing MCP/sim fails with what's missing. Errors are per-case; one broken + case never kills a run. From c250cd1a1edbbc691d2175122514ececd12ae802 Mon Sep 17 00:00:00 2001 From: stash Date: Sun, 9 Aug 2026 00:43:35 -0700 Subject: [PATCH 09/12] docs(evals): md-babel executable intro, stable results --- dimos/evals/intro.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/dimos/evals/intro.md b/dimos/evals/intro.md index a8ed863180..771b57f366 100644 --- a/dimos/evals/intro.md +++ b/dimos/evals/intro.md @@ -33,11 +33,17 @@ Build a tiny recording (any memory2 store works — this is the same API the robot's Recorder uses; see `dimos/memory2/intro.md` for the full Stream API): ```python session=evals ansi=false no-result +import os +from pathlib import Path + +os.environ["DIMOS_LOG_LEVEL"] = "WARNING" # keep doc output stable + from dimos.memory2.store.sqlite import SqliteStore from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.msgs.geometry_msgs.Quaternion import Quaternion from dimos.msgs.geometry_msgs.Vector3 import make_vector3 +Path("/tmp/evals_intro.db").unlink(missing_ok=True) store = SqliteStore(path="/tmp/evals_intro.db") odom = store.stream("odom", PoseStamped) for i in range(20): @@ -52,8 +58,7 @@ for i in range(20): print(odom.summary()) ``` - -``` +```results Stream("odom"): 20 items, 1970-01-01 00:16:40 — 1970-01-01 00:16:59 (19.0s, 1.00 Hz, 1.68 KiB) ``` @@ -89,13 +94,13 @@ from dimos.evals.runner import EvalRunner, summarize runner = EvalRunner(chat_model=FakeListChatModel(responses=["about 19 meters"])) result = runner.run([case])[0] print(f"score={result.score} passed={result.passed} outputs={result.outputs!r}") -print(summarize([result])) +s = summarize([result]) +print(f"n={s.n} mean={s.mean_score} pass_rate={s.pass_rate} errors={s.errors}") ``` - -``` +```results score=1.0 passed=True outputs='about 19 meters' -RunSummary(n=1, mean_score=1.0, pass_rate=1.0, errors=0, duration_s=0.31) +n=1 mean=1.0 pass_rate=1.0 errors=0 ``` That's the whole loop: dataset -> context streams -> encoded prompt -> model @@ -114,8 +119,7 @@ print(exact("yes", "yes"), within(2.0)(10.0, 11.0), ramp(1.0, band=2.0)) print(first_number("around 12.5 m"), yes_no("Yes, clearly."), choice(" Chairs. ")) ``` - -``` +```results 1.0 0.5 0.5 12.5 yes chairs ``` @@ -139,8 +143,7 @@ from dimos.evals.scorers import final, floor, mean print(final([0.2, 0.9]), floor([0.4, 0.2, 0.8]), mean([0.0, 1.0])) ``` - -``` +```results 0.9 0.2 0.5 ``` From 0ddf8b493ac0fc50b9e5d1c263e6725fdeaecbe9 Mon Sep 17 00:00:00 2001 From: stash Date: Sun, 9 Aug 2026 01:23:59 -0700 Subject: [PATCH 10/12] =?UTF-8?q?fix(evals):=20smoke=20test=20skips=20jpeg?= =?UTF-8?q?=20cases=20=E2=80=94=20ros-dev=20container=20lacks=20libturbojp?= =?UTF-8?q?eg?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dimos/evals/test_smoke.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/dimos/evals/test_smoke.py b/dimos/evals/test_smoke.py index 891d3db11d..c9ffa3ee69 100644 --- a/dimos/evals/test_smoke.py +++ b/dimos/evals/test_smoke.py @@ -12,27 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Live passive smoke: real model, real LFS recordings. Self-hosted + API key.""" +"""Live passive smoke: real model, real LFS recordings. Self-hosted + API key. + +Runs only the ``numeric`` cases (odom + pointcloud str encodings) — the +self-hosted ros-dev container has no libturbojpeg, so image-encoding cases +are exercised locally via ``dimos evals run dimos.evals.suites.examples``. +""" from __future__ import annotations +from pathlib import Path + import pytest pytestmark = [pytest.mark.self_hosted, pytest.mark.skipif_no_openai] -def test_passive_smoke(tmp_path) -> None: # type: ignore[no-untyped-def] +def test_passive_smoke(tmp_path: Path) -> None: from dimos.evals.runner import EvalRunner, summarize - from dimos.evals.suites.examples import SUITE + from dimos.evals.suites.go2_smoke import SUITE from dimos.utils.data import get_data get_data("go2_short.db") runner = EvalRunner(model="gpt-4o-mini", out_dir=tmp_path / "evals") - results = runner.run(SUITE) + results = runner.run(SUITE, tags=frozenset({"numeric"})) assert not any(r.error for r in results), [r.error for r in results] s = summarize(results) - # The lidar-points case reads a number embedded in the str() encoding and - # the image case is unambiguous — a competent VLM should clear both. + # The lidar-points case reads a number embedded in the str() encoding — + # a competent model clears it outright; displacement earns graded credit. assert s.mean_score >= 0.5 From f12f64e1489adfd53a2ecd0b3f786ea6b05b9ee9 Mon Sep 17 00:00:00 2001 From: stash Date: Tue, 11 Aug 2026 16:27:13 -0700 Subject: [PATCH 11/12] chore(blueprints): regenerate all_blueprints.py after nav evaluator merge --- dimos/robot/all_blueprints.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 88ef6889d8..d4a8d851ae 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -192,7 +192,6 @@ "emitter-module": "dimos.utils.demo_image_encoding.EmitterModule", "episode-monitor-module": "dimos.imitation.collection.episode_monitor.EpisodeMonitorModule", "eval-module": "dimos.evals.module.EvalModule", - "evaluator": "dimos.navigation.nav_3d.evaluator.evaluator.Evaluator", "far-planner": "dimos.navigation.cmu_nav.modules.far_planner.far_planner.FarPlanner", "fast-lio2": "dimos.hardware.sensors.lidar.fastlio2.module.FastLio2", "fast-lio2-recorder": "dimos.hardware.sensors.lidar.fastlio2.recorder.FastLio2Recorder", From fe02470bccc53960a7a5368fa62009066293c8c8 Mon Sep 17 00:00:00 2001 From: stash Date: Tue, 11 Aug 2026 19:13:16 -0700 Subject: [PATCH 12/12] docs(evals): move evals intro to docs/usage per review --- docs/docs.json | 5 +++++ dimos/evals/intro.md => docs/usage/evals.md | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) rename dimos/evals/intro.md => docs/usage/evals.md (99%) diff --git a/docs/docs.json b/docs/docs.json index 9b4246f2be..ec9988d663 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -54,6 +54,7 @@ "usage/camera_calibration", "usage/native_modules", "usage/tool_streams", + "usage/evals", { "group": "Data streams", "pages": [ @@ -285,6 +286,10 @@ "source": "/docs/usage/tool_streams.md", "destination": "/usage/tool_streams" }, + { + "source": "/docs/usage/evals.md", + "destination": "/usage/evals" + }, { "source": "/docs/usage/data_streams/index.md", "destination": "/usage/data_streams/index" diff --git a/dimos/evals/intro.md b/docs/usage/evals.md similarity index 99% rename from dimos/evals/intro.md rename to docs/usage/evals.md index 771b57f366..c30e471613 100644 --- a/dimos/evals/intro.md +++ b/docs/usage/evals.md @@ -1,4 +1,6 @@ -# Evals Intro +--- +title: "Evals" +--- Evals measure what an agent (or a bare model, or a single skill) can do with the robot's memory. Two kinds: