feat(evals): passive/interactive agent eval framework over memory2

dimos/cli/dimos.py

@@ -803,6 +803,10 @@ def dataprep_inspect(
 
 main.add_typer(mem_app, name="mem")
 
+from dimos.evals.cli import app as evals_app
+
+main.add_typer(evals_app, name="evals")
+
 
 @main.command()
 def cameracalibrate(

dimos/evals/cli.py

@@ -0,0 +1,68 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""``dimos evals`` — run and list eval suites. Heavy imports stay inside
+command bodies (test_cli_startup budget)."""
+
+from __future__ import annotations
+
+import importlib
+
+import typer
+
+app = typer.Typer(help="Run agent evals on recordings, sim, or a live robot.")
+
+
+@app.command("run")
+def run(
+    suite: str = typer.Argument(
+        help="Dotted suite module exporting SUITE, e.g. dimos.evals.suites.go2_smoke"
+    ),
+    tags: str = typer.Option("", help="Comma-separated tag filter"),
+    model: str = typer.Option("", help="Override chat model"),
+    blind: bool = typer.Option(False, help="Withhold observations (guessing ablation)"),
+    attach: bool = typer.Option(False, help="Drive an already-running dimos (interactive cases)"),
+    limit: int = typer.Option(0, help="Run at most N cases"),
+    live_db: str = typer.Option("recording.db", help="Live Recorder db (interactive cases)"),
+) -> None:
+    from dimos.evals.runner import EvalRunner, summarize
+
+    cases = importlib.import_module(suite).SUITE
+    overrides: dict[str, object] = {"blind": blind, "attach": attach, "live_db": live_db}
+    if model:
+        overrides["model"] = model
+    runner = EvalRunner(**overrides)
+    results = runner.run(
+        cases,
+        tags=frozenset(t for t in tags.split(",") if t) if tags else frozenset(),
+        limit=limit,
+    )
+
+    for r in results:
+        status = "ERROR" if r.error else ("PASS" if r.passed else "fail")
+        detail = r.error or f"score={r.score:.2f} answer={r.outputs[:60]!r}"
+        typer.echo(f"{status:5} {r.case_id:30} {detail} ({r.duration_s:.1f}s)")
+    s = summarize(results)
+    typer.echo(
+        f"\n{s.n} cases | mean {s.mean_score:.2f} | pass {s.pass_rate:.0%} "
+        f"| errors {s.errors} | {s.duration_s:.0f}s | {runner.run_dir}"
+    )
+
+
+@app.command("list")
+def list_() -> None:
+    from dimos.evals.module import list_suites
+
+    for name in list_suites():
+        typer.echo(name)

dimos/evals/generate.py

@@ -0,0 +1,88 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Eval-row generators — deferred feature (PRD: low priority), kept minimal.
+
+Ground truth is computed analytically from a *privileged* modality; the emitted
+case quizzes a different (or lossily-encoded) surface. Rows are pure data —
+a suite module maps them onto typed :class:`PassiveEval` cases.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from dimos.memory2.cli.dataset import open_dataset
+
+Row = dict[str, object]
+
+
+def displacement_rows(dataset: str, windows: Sequence[tuple[float, float]]) -> list[Row]:
+    """Straight-line displacement over each window (odom is the privileged truth;
+    the case quizzes the encoded odom summary). Sampling-invariant ground truth."""
+    store = open_dataset(dataset)
+    try:
+        rows: list[Row] = []
+        for t1, t2 in windows:
+            obs = store.streams.odom.range_time(t1, t2).to_list()
+            if len(obs) < 2:
+                continue
+            d = (obs[-1].data.position - obs[0].data.position).length()
+            rows.append(
+                {
+                    "id": f"{dataset}_disp_{t1:g}_{t2:g}",
+                    "q": "How far in a straight line is your final position from your "
+                    "position at the first shown observation, in meters?",
+                    "a": round(d, 1),
+                    "band": max(1.0, d * 0.4),
+                    "stream": "odom",
+                    "window": [t1, t2],
+                    "dataset": dataset,
+                }
+            )
+        return rows
+    finally:
+        store.stop()
+
+
+def path_length_rows(dataset: str, windows: Sequence[tuple[float, float]]) -> list[Row]:
+    """Integrated path length per window. Deliberately hard on a downsampled
+    encoding — expect partial credit; that gap is the finding."""
+    store = open_dataset(dataset)
+    try:
+        rows: list[Row] = []
+        for t1, t2 in windows:
+            path, prev = 0.0, None
+            for obs in store.streams.odom.range_time(t1, t2):
+                p = obs.data.position
+                if prev is not None:
+                    path += (p - prev).length()
+                prev = p
+            if prev is None:
+                continue
+            rows.append(
+                {
+                    "id": f"{dataset}_path_{t1:g}_{t2:g}",
+                    "q": "Roughly how many meters did you travel in total over these "
+                    "observations (path length, not displacement)?",
+                    "a": round(path, 1),
+                    "band": max(2.0, path * 0.5),
+                    "stream": "odom",
+                    "window": [t1, t2],
+                    "dataset": dataset,
+                }
+            )
+        return rows
+    finally:
+        store.stop()

dimos/evals/local.py

@@ -0,0 +1,107 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Keyless local eval model: MoondreamVlModel behind the chat-model interface.
+
+Lets ``EvalRunner(chat_model=MoondreamChat())`` run passive evals with zero API
+keys on any GPU box. Moondream is single-image, so multi-image contexts are
+tiled into one contact sheet; text blocks concatenate into the question.
+"""
+
+from __future__ import annotations
+
+import base64
+from functools import cached_property
+from typing import Any
+
+from langchain_core.callbacks import CallbackManagerForLLMRun
+from langchain_core.language_models.chat_models import BaseChatModel
+from langchain_core.messages import AIMessage, BaseMessage
+from langchain_core.outputs import ChatGeneration, ChatResult
+import numpy as np
+
+from dimos.msgs.sensor_msgs.Image import Image
+
+
+class MoondreamChat(BaseChatModel):
+    """Chat-model adapter over the local moondream2 VLM (dimos MoondreamVlModel)."""
+
+    tile_columns: int = 3
+
+    @property
+    def _llm_type(self) -> str:
+        return "moondream-local"
+
+    @cached_property
+    def _vl(self) -> Any:
+        from dimos.models.vl.moondream import MoondreamVlModel
+
+        model = MoondreamVlModel()
+        model.start()
+        return model
+
+    def _generate(
+        self,
+        messages: list[BaseMessage],
+        stop: list[str] | None = None,
+        run_manager: CallbackManagerForLLMRun | None = None,
+        **kwargs: Any,
+    ) -> ChatResult:
+        texts: list[str] = []
+        frames: list[np.ndarray[Any, Any]] = []
+        for message in messages:
+            content = message.content
+            if isinstance(content, str):
+                texts.append(content)
+                continue
+            for block in content:
+                if not isinstance(block, dict):
+                    texts.append(str(block))
+                elif block.get("type") == "text":
+                    texts.append(str(block["text"]))
+                elif block.get("type") == "image_url":
+                    frames.append(_decode_data_uri(str(block["image_url"]["url"])))
+
+        image = Image.from_numpy(_tile(frames) if frames else _BLANK)
+        answer = self._vl.query(image, "\n".join(texts))
+        return ChatResult(generations=[ChatGeneration(message=AIMessage(content=str(answer)))])
+
+
+_BLANK = np.full((64, 64, 3), 128, dtype=np.uint8)
+
+
+def _decode_data_uri(uri: str) -> np.ndarray[Any, Any]:
+    import cv2
+
+    payload = uri.split(",", 1)[1]
+    buffer = np.frombuffer(base64.b64decode(payload), dtype=np.uint8)
+    frame = cv2.imdecode(buffer, cv2.IMREAD_COLOR)
+    if frame is None:
+        raise ValueError("undecodable image data URI")
+    return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+
+def _tile(frames: list[np.ndarray[Any, Any]], columns: int = 3) -> np.ndarray[Any, Any]:
+    """Grid-tile frames into one contact sheet (moondream is single-image)."""
+    if len(frames) == 1:
+        return frames[0]
+    height = min(f.shape[0] for f in frames)
+    width = min(f.shape[1] for f in frames)
+    import cv2
+
+    resized = [cv2.resize(f, (width, height)) for f in frames]
+    rows = [np.hstack(resized[i : i + columns]) for i in range(0, len(resized), columns)]
+    max_w = max(r.shape[1] for r in rows)
+    rows = [np.pad(r, ((0, 0), (0, max_w - r.shape[1]), (0, 0))) for r in rows]
+    return np.vstack(rows)

dimos/evals/module.py

@@ -0,0 +1,64 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""MCP surface for evals — lets a coding agent spin runs and grep the run dir."""
+
+from __future__ import annotations
+
+import importlib
+import pkgutil
+
+from dimos.agents.annotation import skill
+from dimos.agents.skill_result import SkillResult
+from dimos.core.module import Module
+
+
+def list_suites() -> list[str]:
+    """Dotted module paths under dimos.evals.suites exporting ``SUITE``."""
+    from dimos.evals import suites
+
+    return [
+        name for _, name, _ in pkgutil.iter_modules(suites.__path__, prefix=f"{suites.__name__}.")
+    ]
+
+
+class EvalModule(Module):
+    """Expose eval runs as skills so agents can iterate: run, read the summary,
+    grep transcripts in the returned run_dir, edit code/prompts, run again."""
+
+    @skill
+    def run_evals(self, suite: str, tags: str = "") -> SkillResult:
+        """Run an eval suite by dotted module path (see list_eval_suites).
+
+        Args:
+            suite: e.g. "dimos.evals.suites.go2_smoke" (must export SUITE).
+            tags: optional comma-separated tag filter.
+        """
+        from dimos.evals.runner import EvalRunner, summarize
+
+        cases = importlib.import_module(suite).SUITE
+        runner = EvalRunner(attach=True)
+        results = runner.run(
+            cases, tags=frozenset(t for t in tags.split(",") if t) if tags else frozenset()
+        )
+        s = summarize(results)
+        return SkillResult.ok(
+            f"{s.n} cases: mean={s.mean_score:.2f} pass={s.pass_rate:.0%} errors={s.errors}",
+            run_dir=str(runner.run_dir),
+        )
+
+    @skill
+    def list_eval_suites(self) -> SkillResult:
+        """List available eval suite module paths."""
+        return SkillResult.ok(", ".join(list_suites()))