From 570a2b32191d1860ebb176cee485a8222dedb32d Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Thu, 6 Aug 2026 15:46:50 -0700 Subject: [PATCH 01/12] feat(perception): add VQA grounding core --- dimos/models/segmentation/edge_tam.py | 32 +++ dimos/perception/vqa/adapters.py | 61 +++++ dimos/perception/vqa/evaluate.py | 47 ++++ dimos/perception/vqa/geometry.py | 85 +++++++ dimos/perception/vqa/ground_truth_agent.py | 278 +++++++++++++++++++++ dimos/perception/vqa/grounding.py | 72 ++++++ dimos/perception/vqa/models.py | 161 ++++++++++++ dimos/perception/vqa/pipeline.py | 55 ++++ dimos/perception/vqa/question_agent.py | 87 +++++++ dimos/perception/vqa/questions.py | 64 +++++ dimos/perception/vqa/test_agents.py | 242 ++++++++++++++++++ dimos/perception/vqa/test_geometry.py | 61 +++++ dimos/perception/vqa/test_single_frame.py | 77 ++++++ 13 files changed, 1322 insertions(+) create mode 100644 dimos/perception/vqa/adapters.py create mode 100644 dimos/perception/vqa/evaluate.py create mode 100644 dimos/perception/vqa/geometry.py create mode 100644 dimos/perception/vqa/ground_truth_agent.py create mode 100644 dimos/perception/vqa/grounding.py create mode 100644 dimos/perception/vqa/models.py create mode 100644 dimos/perception/vqa/pipeline.py create mode 100644 dimos/perception/vqa/question_agent.py create mode 100644 dimos/perception/vqa/questions.py create mode 100644 dimos/perception/vqa/test_agents.py create mode 100644 dimos/perception/vqa/test_geometry.py create mode 100644 dimos/perception/vqa/test_single_frame.py diff --git a/dimos/models/segmentation/edge_tam.py b/dimos/models/segmentation/edge_tam.py index 72f1484af8..3f644bd0a0 100644 --- a/dimos/models/segmentation/edge_tam.py +++ b/dimos/models/segmentation/edge_tam.py @@ -30,6 +30,7 @@ from dimos.perception.detection.detectors.base import Detector from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.point import Detection2DPoint from dimos.perception.detection.type.detection2d.seg import Detection2DSeg from dimos.utils.data import get_data from dimos.utils.logging_config import setup_logger @@ -132,6 +133,37 @@ def segment(self, detections: ImageDetections2D) -> ImageDetections2D: ] return ImageDetections2D(image, segmented) + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: + """Create one foreground mask for every positive point prompt.""" + import cv2 + + if not len(points): + return points + image = points.image + rgb = cv2.cvtColor(image.to_opencv(), cv2.COLOR_BGR2RGB) + segmented: list[Detection2DBBox] = [] + with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16): + self._predictor.set_image(rgb) + for point in points: + if not isinstance(point, Detection2DPoint): + continue + mask, _, _ = self._predictor.predict( + point_coords=np.array([[point.x, point.y]], dtype=np.float32), + point_labels=np.array([1], dtype=np.int32), + multimask_output=False, + ) + segmented.append( + Detection2DSeg.from_sam2_result( + mask.squeeze(), + point.track_id, + image, + class_id=point.class_id, + name=point.name, + confidence=point.confidence, + ) + ) + return ImageDetections2D(image, segmented) + class EdgeTAMProcessor(Detector): _predictor: "SAM2VideoPredictor" diff --git a/dimos/perception/vqa/adapters.py b/dimos/perception/vqa/adapters.py new file mode 100644 index 0000000000..e9ff36a9b0 --- /dev/null +++ b/dimos/perception/vqa/adapters.py @@ -0,0 +1,61 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MoonDream and EdgeTAM adapters for the single-frame VQA pipeline.""" + +from __future__ import annotations + +from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter +from dimos.models.vl.moondream import MoondreamVlModel +from dimos.msgs.sensor_msgs.Image import Image +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D + + +class MoondreamObjectDetector: + """Adapt MoonDream's query detection API to the grounding interface.""" + + def __init__(self, model: MoondreamVlModel) -> None: + self._model = model + + def detect(self, image: Image, query: str) -> ImageDetections2D: + return self._model.query_detections(image, query) + + def locate(self, image: Image, query: str) -> ImageDetections2D: + points = self._model.query_points(image, f"center of the {query}") + for point in points: + point.name = query + return points + + +class EdgeTamObjectSegmenter: + """Adapt EdgeTAM single-image segmentation to the grounding interface.""" + + def __init__(self, segmenter: EdgeTAMImageSegmenter) -> None: + self._segmenter = segmenter + + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: + return self._segmenter.segment(detections) + + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: + return self._segmenter.segment_points(points) + + +class MoondreamQuestionAnswerer: + """Ask MoonDream a question using only the supplied image.""" + + def __init__(self, model: MoondreamVlModel) -> None: + self._model = model + + def answer(self, image: Image, question: str) -> str: + return self._model.query(image, question) diff --git a/dimos/perception/vqa/evaluate.py b/dimos/perception/vqa/evaluate.py new file mode 100644 index 0000000000..1874527a98 --- /dev/null +++ b/dimos/perception/vqa/evaluate.py @@ -0,0 +1,47 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Image-only visual-question-answering evaluation.""" + +from __future__ import annotations + +import re + +from dimos.msgs.sensor_msgs.Image import Image +from dimos.perception.vqa.models import VisualQuestionAnswerer, VqaEvaluation, VqaExample + + +def evaluate_examples( + image: Image, examples: list[VqaExample], answerer: VisualQuestionAnswerer +) -> list[VqaEvaluation]: + """Ask an agent questions from an image and compare closed answers.""" + evaluations: list[VqaEvaluation] = [] + for example in examples: + raw_response = answerer.answer(image, example.question) + normalized = _normalize_response(raw_response, example.expected_answer) + evaluations.append( + VqaEvaluation( + example_id=example.id, + expected_answer=example.expected_answer, + raw_response=raw_response, + normalized_response=normalized, + passed=normalized == example.expected_answer, + ) + ) + return evaluations + + +def _normalize_response(response: str, expected: str) -> str | None: + tokens = re.findall(r"[a-z]+", response.lower()) + return expected if expected.lower() in tokens else None diff --git a/dimos/perception/vqa/geometry.py b/dimos/perception/vqa/geometry.py new file mode 100644 index 0000000000..d25d4364fc --- /dev/null +++ b/dimos/perception/vqa/geometry.py @@ -0,0 +1,85 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Static calibrated point-cloud projection for single-frame VQA.""" + +from __future__ import annotations + +import numpy as np + +from dimos.perception.vqa.models import CalibratedFrame, ProjectedPoints, ProjectionConfig + + +def project_visible_points( + frame: CalibratedFrame, config: ProjectionConfig = ProjectionConfig() +) -> ProjectedPoints: + """Project the nearest point at each image pixel using static calibration. + + The supplied transform maps point-cloud coordinates to the camera optical + frame. This function intentionally has no time or TF dependency. + """ + if not frame.image_is_rectified: + raise ValueError("VQA projection requires a rectified pinhole image") + if config.min_depth_m <= 0: + raise ValueError("min_depth_m must be positive") + + camera_info = frame.camera_info + if camera_info.width != frame.image.width or camera_info.height != frame.image.height: + raise ValueError("camera intrinsics dimensions must match the image") + if len(camera_info.K) != 9: + raise ValueError("camera intrinsics must contain a 3x3 matrix") + + points, _ = frame.pointcloud.as_numpy() + if len(points) == 0: + return ProjectedPoints([], [], []) + + homogeneous = np.column_stack((points, np.ones(len(points), dtype=points.dtype))) + camera_points = (frame.pointcloud_to_camera.to_matrix() @ homogeneous.T).T[:, :3] + source_indices = np.arange(len(points)) + + depth_mask = camera_points[:, 2] >= config.min_depth_m + camera_points = camera_points[depth_mask] + source_indices = source_indices[depth_mask] + if len(camera_points) == 0: + return ProjectedPoints([], [], []) + + fx, fy = camera_info.K[0], camera_info.K[4] + cx, cy = camera_info.K[2], camera_info.K[5] + if fx <= 0 or fy <= 0: + raise ValueError("camera focal lengths must be positive") + + u = np.floor(fx * camera_points[:, 0] / camera_points[:, 2] + cx).astype(np.int64) + v = np.floor(fy * camera_points[:, 1] / camera_points[:, 2] + cy).astype(np.int64) + in_image = (u >= 0) & (u < frame.image.width) & (v >= 0) & (v < frame.image.height) + camera_points = camera_points[in_image] + source_indices = source_indices[in_image] + u = u[in_image] + v = v[in_image] + if len(camera_points) == 0: + return ProjectedPoints([], [], []) + + pixel_ids = v * frame.image.width + u + nearest_first = np.lexsort((camera_points[:, 2], pixel_ids)) + first_per_pixel = np.concatenate( + ([True], pixel_ids[nearest_first][1:] != pixel_ids[nearest_first][:-1]) + ) + visible = nearest_first[first_per_pixel] + + return ProjectedPoints( + camera_points=[ + (float(point[0]), float(point[1]), float(point[2])) for point in camera_points[visible] + ], + pixels=[(int(x), int(y)) for x, y in zip(u[visible], v[visible], strict=True)], + source_indices=[int(index) for index in source_indices[visible]], + ) diff --git a/dimos/perception/vqa/ground_truth_agent.py b/dimos/perception/vqa/ground_truth_agent.py new file mode 100644 index 0000000000..65024972e2 --- /dev/null +++ b/dimos/perception/vqa/ground_truth_agent.py @@ -0,0 +1,278 @@ +# Copyright 2026 Dimensional Inc. +"""Tool-driven private answer generation for single-frame VQA.""" + +from __future__ import annotations + +from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox +from dimos.perception.detection.type.detection2d.point import Detection2DPoint +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg +from dimos.perception.vqa.grounding import ground_segmented_objects +from dimos.perception.vqa.models import ( + CalibratedFrame, + GroundedObject, + GroundingConfig, + GroundTruthResult, + ObjectDetector, + ObjectPointLocalizer, + ObjectSegmenter, + PointObjectSegmenter, + QuestionIntent, + ToolTrace, + VqaExample, +) +from dimos.perception.vqa.questions import generate_questions + + +class GroundTruthPerceptionAgent: + """Answer constrained questions by calling detection, segmentation, and geometry tools.""" + + def __init__( + self, + detector: ObjectDetector, + segmenter: ObjectSegmenter, + localizer: ObjectPointLocalizer | None = None, + point_segmenter: PointObjectSegmenter | None = None, + config: GroundingConfig = GroundingConfig(), + ) -> None: + self._detector = detector + self._segmenter = segmenter + self._localizer = localizer + self._point_segmenter = point_segmenter + if config.min_mask_area_px < 1 or config.min_foreground_points < 1: + raise ValueError("grounding thresholds must be positive") + self._config = config + self._groundings: dict[str, tuple[list[GroundedObject], tuple[ToolTrace, ...]]] = {} + self._masks: dict[str, list[Detection2DSeg]] = {} + self._detections: dict[str, list[Detection2DBBox]] = {} + self._points: dict[str, list[Detection2DPoint]] = {} + self._overlay_results: list[GroundTruthResult] = [] + + def answer(self, frame: CalibratedFrame, intent: QuestionIntent) -> GroundTruthResult: + cached = self._groundings.get(intent.object_query) + if cached is not None: + objects, prior_trace = cached + result = self._answer_from_objects( + frame, + intent, + objects, + (ToolTrace("reuse_grounding", intent.object_query), *prior_trace), + ) + self._overlay_results.append(result) + return result + trace: list[ToolTrace] = [ToolTrace("detect_objects", intent.object_query)] + detections = self._detector.detect(frame.image, intent.object_query) + self._detections[intent.object_query] = [ + item for item in detections if isinstance(item, Detection2DBBox) + ] + if len(detections): + trace.append(ToolTrace("segment_objects", f"count={len(detections)}")) + segmented = self._segmenter.segment(detections) + elif self._localizer is not None and self._point_segmenter is not None: + trace.append(ToolTrace("locate_object_point", intent.object_query)) + points = self._localizer.locate(frame.image, intent.object_query) + self._points[intent.object_query] = [ + item for item in points if isinstance(item, Detection2DPoint) + ] + trace.append(ToolTrace("segment_object_point", f"count={len(points)}")) + segmented = self._point_segmenter.segment_points(points) + else: + segmented = detections + masks = [ + item + for item in segmented + if isinstance(item, Detection2DSeg) + and int((item.mask > 0).sum()) >= self._config.min_mask_area_px + ] + trace.append(ToolTrace("get_foreground_geometry", f"masks={len(masks)}")) + self._masks[intent.object_query] = masks + objects = ground_segmented_objects( + frame, masks, min_foreground_points=self._config.min_foreground_points + ) + stored_trace = tuple(trace) + self._groundings[intent.object_query] = (objects, stored_trace) + result = self._answer_from_objects(frame, intent, objects, stored_trace) + self._overlay_results.append(result) + return result + + def _answer_from_objects( + self, + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + ) -> GroundTruthResult: + if not objects: + rejected = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.kind}", + _render_question(intent), + "", + "", + (), + ) + return GroundTruthResult( + intent, rejected, "rejected", None, "no_grounded_object", (), trace + ) + if intent.kind == "compare_nearest_by_side": + return _compare_nearest_by_side(frame, intent, objects, trace) + examples = generate_questions( + frame.id, objects, [intent.object_query], distance_m=intent.threshold_m or 3.0 + ) + suffix = { + "presence": "presence", + "horizontal_direction": "direction", + "within_distance": "range", + }[intent.kind] + example = next((item for item in examples if item.id.endswith(f"-{suffix}")), None) + if example is not None: + return GroundTruthResult( + intent, + example, + "answered", + example.expected_answer, + None, + tuple(objects), + trace, + ) + rejected = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.kind}", _render_question(intent), "", "", () + ) + return GroundTruthResult( + intent, rejected, "rejected", None, "no_grounded_object", tuple(objects), trace + ) + + def write_overlay(self, frame: CalibratedFrame, path: str) -> None: + """Save private masks, detector prompts, and a per-question audit legend.""" + import cv2 + import numpy as np + + overlay = frame.image.data.copy() + colors = ((255, 128, 0), (0, 200, 255), (180, 0, 255), (0, 200, 80)) + question_labels: dict[str, list[str]] = {} + for index, result in enumerate(self._overlay_results, start=1): + question_labels.setdefault(result.intent.object_query, []).append(f"Q{index}") + for index, query in enumerate(question_labels): + color = colors[index % len(colors)] + label = ",".join(question_labels[query]) + for mask in self._masks.get(query, []): + overlay[mask.mask > 0] = color + x1, y1, _, _ = map(int, mask.bbox) + cv2.putText( + overlay, + f"{label} {query}", + (x1, max(y1 - 6, 12)), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + color, + 1, + ) + for detection in self._detections.get(query, []): + x1, y1, x2, y2 = map(int, detection.bbox) + cv2.rectangle(overlay, (x1, y1), (x2, y2), color, 2) + cv2.putText( + overlay, + f"{label} box", + (x1, min(y2 + 16, frame.image.height - 4)), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + color, + 1, + ) + for point in self._points.get(query, []): + position = (int(point.x), int(point.y)) + cv2.drawMarker(overlay, position, color, cv2.MARKER_CROSS, 14, 2) + cv2.putText( + overlay, + f"{label} point", + (position[0] + 8, max(position[1] - 8, 12)), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + color, + 1, + ) + rendered = cv2.addWeighted(frame.image.data, 0.55, overlay, 0.45, 0) + legend_lines = ["Private grounding audit"] + for index, result in enumerate(self._overlay_results, start=1): + status = result.answer if result.status == "answered" else f"rejected: {result.reason}" + legend_lines.extend(_wrap_overlay_text(f"Q{index}: {result.question.question}", 52)) + legend_lines.extend(_wrap_overlay_text(f" {status}", 52)) + line_height = 20 + panel_width = 440 + panel_height = max(rendered.shape[0], 16 + line_height * len(legend_lines)) + audit = np.full((panel_height, rendered.shape[1] + panel_width, 3), 32, dtype=np.uint8) + audit[: rendered.shape[0], : rendered.shape[1]] = rendered + for index, line in enumerate(legend_lines): + cv2.putText( + audit, + line, + (rendered.shape[1] + 12, 20 + index * line_height), + cv2.FONT_HERSHEY_SIMPLEX, + 0.45, + (255, 255, 255), + 1, + ) + if not cv2.imwrite(path, audit): + raise RuntimeError(f"failed to write {path}") + + +def _render_question(intent: QuestionIntent) -> str: + if intent.kind == "presence": + return f"Is there a {intent.object_query} in the image? Answer yes or no." + if intent.kind == "horizontal_direction": + return f"Where is the nearest {intent.object_query}: left, center, or right?" + if intent.kind == "compare_nearest_by_side": + return f"Which {intent.object_query} is closer: the left one or the right one?" + return f"Is the nearest {intent.object_query} within {intent.threshold_m or 3:g} meters? Answer yes or no." + + +def _wrap_overlay_text(text: str, width: int) -> list[str]: + words = text.split() + lines: list[str] = [] + line = "" + for word in words: + candidate = f"{line} {word}".strip() + if line and len(candidate) > width: + lines.append(line) + line = word + else: + line = candidate + if line: + lines.append(line) + return lines + + +def _compare_nearest_by_side( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + left_matches = [item for item in objects if item.horizontal_direction == "left"] + right_matches = [item for item in objects if item.horizontal_direction == "right"] + if not left_matches or not right_matches: + return _rejected_result(frame, intent, objects, trace, "missing_grounded_side") + left = min(left_matches, key=lambda item: item.range_m) + right = min(right_matches, key=lambda item: item.range_m) + if left.range_m == right.range_m: + return _rejected_result(frame, intent, objects, trace, "ambiguous_nearest_by_side") + answer = "left" if left.range_m < right.range_m else "right" + example = VqaExample( + f"{frame.id}-{intent.object_query}-nearest-by-side", + _render_question(intent), + answer, + "choice", + (left.id, right.id), + ) + return GroundTruthResult(intent, example, "answered", answer, None, (left, right), trace) + + +def _rejected_result( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + reason: str, +) -> GroundTruthResult: + rejected = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.kind}", _render_question(intent), "", "", () + ) + return GroundTruthResult(intent, rejected, "rejected", None, reason, tuple(objects), trace) diff --git a/dimos/perception/vqa/grounding.py b/dimos/perception/vqa/grounding.py new file mode 100644 index 0000000000..d1f5ba67b3 --- /dev/null +++ b/dimos/perception/vqa/grounding.py @@ -0,0 +1,72 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Foreground-mask point-cloud grounding for one calibrated frame.""" + +from __future__ import annotations + +import numpy as np + +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg +from dimos.perception.vqa.geometry import project_visible_points +from dimos.perception.vqa.models import CalibratedFrame, GroundedObject, ProjectionConfig + + +def ground_segmented_objects( + frame: CalibratedFrame, + detections: list[Detection2DSeg], + *, + min_foreground_points: int = 3, + projection: ProjectionConfig = ProjectionConfig(), +) -> list[GroundedObject]: + """Create object geometry from visible points covered by each foreground mask.""" + if min_foreground_points < 1: + raise ValueError("min_foreground_points must be positive") + + projected = project_visible_points(frame, projection) + grounded: list[GroundedObject] = [] + for index, detection in enumerate(detections): + mask = detection.mask + if mask.shape != (frame.image.height, frame.image.width): + raise ValueError("segmentation mask dimensions must match the image") + selected = [ + point + for point, (x, y) in zip(projected.camera_points, projected.pixels, strict=True) + if mask[y, x] > 0 + ] + if len(selected) < min_foreground_points: + continue + + ranges = np.linalg.norm(np.asarray(selected), axis=1) + image_x = [x for x, y in projected.pixels if mask[y, x] > 0] + median_x = float(np.median(image_x)) + direction = _horizontal_direction(median_x, frame.image.width) + grounded.append( + GroundedObject( + id=f"{frame.id}-{detection.name}-{index}", + label=detection.name, + point_count=len(selected), + range_m=float(np.median(ranges)), + horizontal_direction=direction, + ) + ) + return grounded + + +def _horizontal_direction(x: float, width: int) -> str: + if x < width / 3: + return "left" + if x >= 2 * width / 3: + return "right" + return "center" diff --git a/dimos/perception/vqa/models.py b/dimos/perception/vqa/models.py new file mode 100644 index 0000000000..75af168686 --- /dev/null +++ b/dimos/perception/vqa/models.py @@ -0,0 +1,161 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contracts shared by the single-frame perception VQA pipeline.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal, Protocol + +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D + + +@dataclass(frozen=True) +class CalibratedFrame: + """One self-contained image and point-cloud pair for VQA generation.""" + + id: str + image: Image + pointcloud: PointCloud2 + camera_info: CameraInfo + pointcloud_to_camera: Transform + image_is_rectified: bool + original_image: Image | None = None + + +@dataclass(frozen=True) +class ProjectionConfig: + """Controls static pinhole projection of a frame's point cloud.""" + + min_depth_m: float = 0.01 + + +@dataclass(frozen=True) +class GroundingConfig: + """Quality thresholds for accepting an image mask as a grounded object.""" + + min_mask_area_px: int = 128 + min_foreground_points: int = 3 + + +@dataclass(frozen=True) +class ProjectedPoints: + """Visible point-cloud samples represented in the camera image.""" + + camera_points: list[tuple[float, float, float]] + pixels: list[tuple[int, int]] + source_indices: list[int] + + +@dataclass(frozen=True) +class GroundedObject: + """One semantic object with foreground point-cloud support.""" + + id: str + label: str + point_count: int + range_m: float + horizontal_direction: str + + +@dataclass(frozen=True) +class VqaExample: + """A closed-answer question generated from grounded objects.""" + + id: str + question: str + expected_answer: str + answer_type: str + object_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class VqaEvaluation: + """The result of asking an image-only agent one generated question.""" + + example_id: str + expected_answer: str + raw_response: str + normalized_response: str | None + passed: bool + + +QuestionKind = Literal[ + "presence", "horizontal_direction", "within_distance", "compare_nearest_by_side" +] + + +@dataclass(frozen=True) +class QuestionIntent: + """A constrained question proposed from an image.""" + + kind: QuestionKind + object_query: str + threshold_m: float | None = None + + +@dataclass(frozen=True) +class ToolTrace: + """One perception operation used to establish a ground-truth answer.""" + + tool: str + detail: str + + +@dataclass(frozen=True) +class GroundTruthResult: + """An answered or rejected question with private perception evidence.""" + + intent: QuestionIntent + question: VqaExample + status: Literal["answered", "rejected"] + answer: str | None + reason: str | None + evidence: tuple[GroundedObject, ...] + trace: tuple[ToolTrace, ...] + + +class ObjectDetector(Protocol): + """Produces object-query detections from one image.""" + + def detect(self, image: Image, query: str) -> ImageDetections2D: ... + + +class ObjectSegmenter(Protocol): + """Refines object detections into foreground masks in one image.""" + + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: ... + + +class ObjectPointLocalizer(Protocol): + """Locates a queried object with positive image points.""" + + def locate(self, image: Image, query: str) -> ImageDetections2D: ... + + +class PointObjectSegmenter(Protocol): + """Creates foreground masks from positive image-point prompts.""" + + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: ... + + +class VisualQuestionAnswerer(Protocol): + """Answers a question from an image without access to ground truth.""" + + def answer(self, image: Image, question: str) -> str: ... diff --git a/dimos/perception/vqa/pipeline.py b/dimos/perception/vqa/pipeline.py new file mode 100644 index 0000000000..834e32836c --- /dev/null +++ b/dimos/perception/vqa/pipeline.py @@ -0,0 +1,55 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Single-frame VQA ground-truth generation and image-only evaluation.""" + +from __future__ import annotations + +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg +from dimos.perception.vqa.evaluate import evaluate_examples +from dimos.perception.vqa.grounding import ground_segmented_objects +from dimos.perception.vqa.models import ( + CalibratedFrame, + ObjectDetector, + ObjectSegmenter, + VisualQuestionAnswerer, + VqaEvaluation, + VqaExample, +) +from dimos.perception.vqa.questions import generate_questions + + +def generate_ground_truth( + frame: CalibratedFrame, + queries: list[str], + detector: ObjectDetector, + segmenter: ObjectSegmenter, +) -> list[VqaExample]: + """Generate VQA examples from MoonDream/EdgeTAM-style image models and LiDAR.""" + segmented: list[Detection2DSeg] = [] + for query in queries: + detections = detector.detect(frame.image, query) + result = segmenter.segment(detections) + segmented.extend(detection for detection in result if isinstance(detection, Detection2DSeg)) + objects = ground_segmented_objects(frame, segmented) + return generate_questions(frame.id, objects, queries) + + +def evaluate_ground_truth( + frame: CalibratedFrame, + examples: list[VqaExample], + answerer: VisualQuestionAnswerer, +) -> list[VqaEvaluation]: + """Evaluate an answerer without passing it geometry or expected answers.""" + return evaluate_examples(frame.image, examples, answerer) diff --git a/dimos/perception/vqa/question_agent.py b/dimos/perception/vqa/question_agent.py new file mode 100644 index 0000000000..6c317637a9 --- /dev/null +++ b/dimos/perception/vqa/question_agent.py @@ -0,0 +1,87 @@ +# Copyright 2026 Dimensional Inc. +"""Image-only constrained VQA question proposal.""" + +from __future__ import annotations + +import json +import re +from typing import Any + +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.sensor_msgs.Image import Image +from dimos.perception.vqa.models import QuestionIntent + +QUESTION_PROMPT = """You generate challenging but visually answerable single-frame VQA questions. +Inspect only this image. Do not assume depth, point clouds, metadata, or temporal context. +Return JSON only: an array of at most 5 visible, salient object names as strings. +Do not return floors, walls, ceilings, background surfaces, questions, explanations, Markdown, +or information not visible in the image.""" + +_UNSUPPORTED_QUERIES = {"background", "ceiling", "floor", "ground", "room", "wall"} + + +class OpenAIQuestionAgent: + """Propose constrained VQA intents from an image without geometry access.""" + + def __init__(self, model: OpenAIVlModel) -> None: + self._model = model + + def propose(self, image: Image) -> list[QuestionIntent]: + try: + payload: Any = _parse_json_array(self._model.query(image, QUESTION_PROMPT)) + except json.JSONDecodeError as exc: + raise ValueError("question agent did not return JSON") from exc + if not isinstance(payload, list) or len(payload) > 5: + raise ValueError("question agent must return an array of at most five intents") + intents: list[QuestionIntent] = [] + if all(isinstance(item, str) for item in payload): + return [ + intent + for query in dict.fromkeys(item.strip() for item in payload if item.strip()) + if query.lower() not in _UNSUPPORTED_QUERIES + for intent in _intents_for_query(query) + ] + for item in payload: + if not isinstance(item, dict): + raise ValueError("question intent must be an object") + kind, query, threshold = ( + item.get("kind"), + item.get("object_query"), + item.get("threshold_m"), + ) + if kind not in ( + "presence", + "horizontal_direction", + "within_distance", + "compare_nearest_by_side", + ): + raise ValueError(f"unsupported question kind: {kind!r}") + if not isinstance(query, str) or not query: + raise ValueError("question intent requires object_query") + if kind == "within_distance" and ( + not isinstance(threshold, (int, float)) or threshold <= 0 + ): + raise ValueError("within_distance requires a positive threshold_m") + if kind != "within_distance": + threshold = None + intents.append(QuestionIntent(kind=kind, object_query=query, threshold_m=threshold)) + return intents + + +def _intents_for_query(query: str) -> list[QuestionIntent]: + return [ + QuestionIntent(kind="presence", object_query=query), + QuestionIntent(kind="horizontal_direction", object_query=query), + QuestionIntent(kind="within_distance", object_query=query, threshold_m=3.0), + QuestionIntent(kind="compare_nearest_by_side", object_query=query), + ] + + +def _parse_json_array(response: str) -> Any: + stripped = response.strip() + if stripped.startswith("```"): + stripped = re.sub(r"^```(?:json)?\s*|\s*```$", "", stripped, flags=re.IGNORECASE) + start, end = stripped.find("["), stripped.rfind("]") + if start < 0 or end < start: + raise json.JSONDecodeError("expected JSON array", stripped, 0) + return json.loads(stripped[start : end + 1]) diff --git a/dimos/perception/vqa/questions.py b/dimos/perception/vqa/questions.py new file mode 100644 index 0000000000..354a8a0119 --- /dev/null +++ b/dimos/perception/vqa/questions.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Deterministic closed-answer questions from grounded objects.""" + +from __future__ import annotations + +from dimos.perception.vqa.models import GroundedObject, VqaExample + + +def generate_questions( + frame_id: str, objects: list[GroundedObject], queries: list[str], *, distance_m: float = 3.0 +) -> list[VqaExample]: + """Generate presence, range, and direction questions for one frame.""" + if distance_m <= 0: + raise ValueError("distance_m must be positive") + + examples: list[VqaExample] = [] + for query in queries: + matches = sorted( + (item for item in objects if item.label == query), key=lambda item: item.range_m + ) + nearest = matches[0] if matches else None + examples.append( + VqaExample( + id=f"{frame_id}-{query}-presence", + question=f"Is there a {query} in the image? Answer yes or no.", + expected_answer="yes" if nearest is not None else "no", + answer_type="boolean", + object_ids=(nearest.id,) if nearest is not None else (), + ) + ) + if nearest is None: + continue + examples.extend( + [ + VqaExample( + id=f"{frame_id}-{query}-direction", + question=f"Where is the nearest {query}: left, center, or right?", + expected_answer=nearest.horizontal_direction, + answer_type="choice", + object_ids=(nearest.id,), + ), + VqaExample( + id=f"{frame_id}-{query}-range", + question=f"Is the nearest {query} within {distance_m:g} meters? Answer yes or no.", + expected_answer="yes" if nearest.range_m <= distance_m else "no", + answer_type="boolean", + object_ids=(nearest.id,), + ), + ] + ) + return examples diff --git a/dimos/perception/vqa/test_agents.py b/dimos/perception/vqa/test_agents.py new file mode 100644 index 0000000000..2bdd1462b5 --- /dev/null +++ b/dimos/perception/vqa/test_agents.py @@ -0,0 +1,242 @@ +# Copyright 2026 Dimensional Inc. + +from __future__ import annotations + +from pathlib import Path +from typing import cast + +import cv2 +import numpy as np + +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.point import Detection2DPoint +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg +from dimos.perception.vqa.ground_truth_agent import GroundTruthPerceptionAgent +from dimos.perception.vqa.models import CalibratedFrame, GroundingConfig, QuestionIntent +from dimos.perception.vqa.question_agent import OpenAIQuestionAgent + + +class _QuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert image.width == 6 + assert "point clouds" in prompt + return '```json\n["chair"]\n```' + + +class _Detector: + def __init__(self, image: Image, detection: Detection2DSeg) -> None: + self._image = image + self._detection = detection + + def detect(self, image: Image, query: str) -> ImageDetections2D: + return ImageDetections2D(image, [self._detection] if query == "chair" else []) + + +class _MultiDetector: + def __init__(self, image: Image, detections: list[Detection2DSeg]) -> None: + self._image = image + self._detections = detections + + def detect(self, image: Image, query: str) -> ImageDetections2D: + return ImageDetections2D(self._image, self._detections if query == "chair" else []) + + +class _Segmenter: + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: + return detections + + +class _PointLocalizer: + def locate(self, image: Image, query: str) -> ImageDetections2D: + return ImageDetections2D(image, [Detection2DPoint(3.0, 3.0, query, 0.0, image)]) + + +class _PointSegmenter: + def __init__(self, detection: Detection2DSeg) -> None: + self._detection = detection + + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: + return ImageDetections2D(points.image, [self._detection]) + + +def _frame_and_detection() -> tuple[CalibratedFrame, Detection2DSeg]: + image = Image.from_numpy(np.zeros((6, 6, 3), dtype=np.uint8)) + frame = CalibratedFrame( + id="frame-1", + image=image, + pointcloud=PointCloud2.from_numpy( + np.array([[0.0, 0.0, 1.0], [0.4, 0.0, 1.0], [-0.4, 0.0, 1.0]], dtype=np.float32) + ), + camera_info=CameraInfo.from_intrinsics(3.0, 3.0, 3.0, 3.0, 6, 6), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + detection = Detection2DSeg( + (0.0, 0.0, 5.0, 5.0), 0, -1, 1.0, "chair", 0.0, image, np.full((6, 6), 255, dtype=np.uint8) + ) + return frame, detection + + +def test_question_agent_returns_constrained_intents() -> None: + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _QuestionModel())).propose(frame.image) + + assert intents == [ + QuestionIntent(kind="presence", object_query="chair"), + QuestionIntent(kind="horizontal_direction", object_query="chair"), + QuestionIntent(kind="within_distance", object_query="chair", threshold_m=3.0), + QuestionIntent(kind="compare_nearest_by_side", object_query="chair"), + ] + + +def test_ground_truth_agent_records_tools_and_rejects_unsupported_question() -> None: + frame, detection = _frame_and_detection() + agent = GroundTruthPerceptionAgent( + _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) + ) + + answered = agent.answer( + frame, QuestionIntent(kind="within_distance", object_query="chair", threshold_m=3.0) + ) + rejected = agent.answer( + frame, QuestionIntent(kind="horizontal_direction", object_query="table") + ) + absent = agent.answer(frame, QuestionIntent(kind="presence", object_query="table")) + + assert answered.status == "answered" + assert answered.answer == "yes" + assert answered.evidence[0].point_count == 3 + assert [item.tool for item in answered.trace] == [ + "detect_objects", + "segment_objects", + "get_foreground_geometry", + ] + assert rejected.status == "rejected" + assert rejected.reason == "no_grounded_object" + assert absent.status == "rejected" + + +def test_ground_truth_agent_rejects_small_masks() -> None: + frame, detection = _frame_and_detection() + agent = GroundTruthPerceptionAgent( + _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=37) + ) + + result = agent.answer(frame, QuestionIntent(kind="presence", object_query="chair")) + + assert result.status == "rejected" + + +def test_ground_truth_agent_falls_back_to_point_prompt() -> None: + frame, detection = _frame_and_detection() + point_detection = Detection2DSeg( + detection.bbox, + detection.track_id, + detection.class_id, + detection.confidence, + "plant", + detection.ts, + detection.image, + detection.mask, + ) + agent = GroundTruthPerceptionAgent( + _Detector(frame.image, detection), + _Segmenter(), + localizer=_PointLocalizer(), + point_segmenter=_PointSegmenter(point_detection), + config=GroundingConfig(min_mask_area_px=1), + ) + + result = agent.answer(frame, QuestionIntent(kind="presence", object_query="plant")) + + assert result.answer == "yes" + assert [item.tool for item in result.trace] == [ + "detect_objects", + "locate_object_point", + "segment_object_point", + "get_foreground_geometry", + ] + + +def test_ground_truth_agent_compares_nearest_objects_by_side() -> None: + image = Image.from_numpy(np.zeros((6, 6, 3), dtype=np.uint8)) + frame = CalibratedFrame( + id="frame-1", + image=image, + pointcloud=PointCloud2.from_numpy( + np.array( + [ + [-0.5, -0.6, 1.0], + [-0.5, 0.0, 1.0], + [-0.5, 0.6, 1.0], + [1.0, -1.0, 2.0], + [1.0, 0.0, 2.0], + [1.0, 1.0, 2.0], + ], + dtype=np.float32, + ) + ), + camera_info=CameraInfo.from_intrinsics(3.0, 3.0, 3.0, 3.0, 6, 6), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + left_mask = np.zeros((6, 6), dtype=np.uint8) + left_mask[:, :3] = 255 + right_mask = np.zeros((6, 6), dtype=np.uint8) + right_mask[:, 3:] = 255 + detections = [ + Detection2DSeg((0.0, 0.0, 2.0, 5.0), 0, -1, 1.0, "chair", 0.0, image, left_mask), + Detection2DSeg((3.0, 0.0, 5.0, 5.0), 1, -1, 1.0, "chair", 0.0, image, right_mask), + ] + agent = GroundTruthPerceptionAgent( + _MultiDetector(image, detections), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) + ) + + result = agent.answer( + frame, QuestionIntent(kind="compare_nearest_by_side", object_query="chair") + ) + + assert result.status == "answered" + assert result.answer == "left" + assert result.question.object_ids == ("frame-1-chair-0", "frame-1-chair-1") + + +def test_ground_truth_agent_rejects_side_comparison_without_both_sides() -> None: + frame, detection = _frame_and_detection() + agent = GroundTruthPerceptionAgent( + _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) + ) + + result = agent.answer( + frame, QuestionIntent(kind="compare_nearest_by_side", object_query="chair") + ) + + assert result.status == "rejected" + assert result.reason == "missing_grounded_side" + + +def test_ground_truth_agent_overlay_includes_detector_prompts_and_question_legend( + tmp_path: Path, +) -> None: + frame, detection = _frame_and_detection() + agent = GroundTruthPerceptionAgent( + _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) + ) + agent.answer(frame, QuestionIntent(kind="presence", object_query="chair")) + agent.answer( + frame, QuestionIntent(kind="within_distance", object_query="chair", threshold_m=3.0) + ) + path = tmp_path / "overlay.jpg" + + agent.write_overlay(frame, str(path)) + + rendered = cv2.imread(str(path)) + assert rendered is not None + assert rendered.shape[0] >= frame.image.height + assert rendered.shape[1] > frame.image.width diff --git a/dimos/perception/vqa/test_geometry.py b/dimos/perception/vqa/test_geometry.py new file mode 100644 index 0000000000..6d9e9fe43e --- /dev/null +++ b/dimos/perception/vqa/test_geometry.py @@ -0,0 +1,61 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import numpy as np +import pytest + +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.vqa.geometry import project_visible_points +from dimos.perception.vqa.models import CalibratedFrame + + +def _frame(points: np.ndarray, *, rectified: bool = True) -> CalibratedFrame: + return CalibratedFrame( + id="frame-1", + image=Image.from_numpy(np.zeros((4, 4, 3), dtype=np.uint8)), + pointcloud=PointCloud2.from_numpy(points), + camera_info=CameraInfo.from_intrinsics(2.0, 2.0, 2.0, 2.0, 4, 4), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=rectified, + ) + + +def test_project_visible_points_keeps_nearest_point_per_pixel() -> None: + frame = _frame( + np.array( + [ + [0.0, 0.0, 2.0], + [0.0, 0.0, 1.0], + [0.5, 0.0, 1.0], + [0.0, 0.0, -1.0], + ], + dtype=np.float32, + ) + ) + + projected = project_visible_points(frame) + + assert projected.pixels == [(2, 2), (3, 2)] + assert projected.source_indices == [1, 2] + assert projected.camera_points == [(0.0, 0.0, 1.0), (0.5, 0.0, 1.0)] + + +def test_project_visible_points_rejects_unrectified_images() -> None: + with pytest.raises(ValueError, match="rectified"): + project_visible_points(_frame(np.zeros((1, 3), dtype=np.float32), rectified=False)) diff --git a/dimos/perception/vqa/test_single_frame.py b/dimos/perception/vqa/test_single_frame.py new file mode 100644 index 0000000000..a68addc09c --- /dev/null +++ b/dimos/perception/vqa/test_single_frame.py @@ -0,0 +1,77 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import numpy as np + +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg +from dimos.perception.vqa.models import CalibratedFrame +from dimos.perception.vqa.pipeline import evaluate_ground_truth, generate_ground_truth + + +class _Answerer: + def __init__(self) -> None: + self.calls: list[tuple[Image, str]] = [] + + def answer(self, image: Image, question: str) -> str: + self.calls.append((image, question)) + return "Yes." + + +class _Detector: + def __init__(self, image: Image, detection: Detection2DSeg) -> None: + self._image = image + self._detection = detection + + def detect(self, image: Image, query: str) -> ImageDetections2D: + assert image is self._image + return ImageDetections2D(image, [self._detection] if query == "chair" else []) + + +class _Segmenter: + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: + return detections + + +def test_single_frame_ground_truth_and_image_only_evaluation() -> None: + image = Image.from_numpy(np.zeros((6, 6, 3), dtype=np.uint8)) + frame = CalibratedFrame( + id="frame-1", + image=image, + pointcloud=PointCloud2.from_numpy( + np.array([[0.0, 0.0, 1.0], [0.4, 0.0, 1.0], [-0.4, 0.0, 1.0]], dtype=np.float32) + ), + camera_info=CameraInfo.from_intrinsics(3.0, 3.0, 3.0, 3.0, 6, 6), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + mask = np.full((6, 6), 255, dtype=np.uint8) + detection = Detection2DSeg((0.0, 0.0, 5.0, 5.0), 0, -1, 1.0, "chair", 0.0, image, mask) + + examples = generate_ground_truth( + frame, ["chair", "table"], _Detector(image, detection), _Segmenter() + ) + answerer = _Answerer() + evaluations = evaluate_ground_truth(frame, examples, answerer) + + assert {example.expected_answer for example in examples} == {"yes", "no", "center"} + assert all(image_arg is image for image_arg, _ in answerer.calls) + assert evaluations[0].passed + assert not evaluations[1].passed From 30e98069819d4b9d89dc652c085919e479fe0395 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Thu, 6 Aug 2026 15:47:02 -0700 Subject: [PATCH 02/12] feat(cli): add VQA dataset generation commands --- dimos/cli/dimos.py | 2 + dimos/cli/vqa.py | 230 ++++++++++++++++++++++++++++++ dimos/perception/vqa/dataset.py | 94 ++++++++++++ dimos/perception/vqa/recording.py | 64 +++++++++ 4 files changed, 390 insertions(+) create mode 100644 dimos/cli/vqa.py create mode 100644 dimos/perception/vqa/dataset.py create mode 100644 dimos/perception/vqa/recording.py diff --git a/dimos/cli/dimos.py b/dimos/cli/dimos.py index 0f50caaaea..9137621c5f 100644 --- a/dimos/cli/dimos.py +++ b/dimos/cli/dimos.py @@ -53,6 +53,7 @@ from dimos.cli.eval import app as eval_app from dimos.cli.hardware_cli import app as hardware_app from dimos.cli.shell import shell +from dimos.cli.vqa import app as vqa_app from dimos.constants import CONFIG_DIR, LOG_DIR from dimos.core.daemon import daemonize, install_signal_handlers from dimos.core.global_config import GlobalConfig, global_config @@ -77,6 +78,7 @@ help="Dimensional CLI", no_args_is_help=True, ) +main.add_typer(vqa_app, name="vqa") load_dotenv() diff --git a/dimos/cli/vqa.py b/dimos/cli/vqa.py new file mode 100644 index 0000000000..a3ab974f90 --- /dev/null +++ b/dimos/cli/vqa.py @@ -0,0 +1,230 @@ +# Copyright 2026 Dimensional Inc. +"""Single-frame point-cloud-grounded VQA commands.""" + +from __future__ import annotations + +from dataclasses import asdict +import json +from pathlib import Path + +import cv2 +import typer + +from dimos.constants import STATE_DIR +from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter +from dimos.models.vl.moondream import MoondreamVlModel +from dimos.models.vl.openai import OpenAIVlModel +from dimos.perception.vqa.adapters import ( + EdgeTamObjectSegmenter, + MoondreamObjectDetector, + MoondreamQuestionAnswerer, +) +from dimos.perception.vqa.dataset import write_dataset_manifest, write_frame_record +from dimos.perception.vqa.ground_truth_agent import GroundTruthPerceptionAgent +from dimos.perception.vqa.models import GroundingConfig, QuestionIntent +from dimos.perception.vqa.pipeline import evaluate_ground_truth +from dimos.perception.vqa.question_agent import OpenAIQuestionAgent +from dimos.perception.vqa.recording import load_go2_frame +from dimos.utils.data import resolve_named_path + +app = typer.Typer(help="Generate and evaluate point-cloud-grounded VQA examples") + + +@app.command("single-frame") +def single_frame( + recording: str = typer.Option(..., "--recording"), + frame_index: int = typer.Option(0, "--frame-index"), + query: list[str] = typer.Option([], "--query"), + propose_questions: bool = typer.Option(False, "--propose-questions"), + question_model: str = typer.Option("gpt-4o-mini", "--question-model"), + min_mask_area_px: int = typer.Option(128, "--min-mask-area-px"), + min_foreground_points: int = typer.Option(3, "--min-foreground-points"), + output: Path | None = typer.Option(None, "--output"), +) -> None: + """Generate and evaluate questions for one Go2 recording frame.""" + output = output or ( + STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frame-{frame_index:06d}" + ) + if output.exists() or (not query and not propose_questions): + raise typer.BadParameter("output must not already exist") + frame = load_go2_frame(str(resolve_named_path(recording, ".db")), frame_index) + model = MoondreamVlModel() + model.start() + question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=question_model)) + try: + intents = ( + question_agent.propose(frame.image) + if propose_questions + else [ + QuestionIntent( + kind=kind, + object_query=item, + threshold_m=3.0 if kind == "within_distance" else None, + ) + for item in query + for kind in ( + "presence", + "horizontal_direction", + "within_distance", + "compare_nearest_by_side", + ) + ] + ) + ground_truth = GroundTruthPerceptionAgent( + detector := MoondreamObjectDetector(model), + segmenter := EdgeTamObjectSegmenter(EdgeTAMImageSegmenter()), + localizer=detector, + point_segmenter=segmenter, + config=GroundingConfig( + min_mask_area_px=min_mask_area_px, + min_foreground_points=min_foreground_points, + ), + ) + results = [ground_truth.answer(frame, intent) for intent in intents] + examples = [result.question for result in results if result.status == "answered"] + evaluations = evaluate_ground_truth(frame, examples, MoondreamQuestionAnswerer(model)) + finally: + model.stop() + output.mkdir(parents=True) + image_path = output / "image.jpg" + if not cv2.imwrite(str(image_path), frame.image.data): + raise RuntimeError(f"failed to write {image_path}") + original_image_path = output / "original_image.jpg" + if frame.original_image is not None and not cv2.imwrite( + str(original_image_path), frame.original_image.data + ): + raise RuntimeError(f"failed to write {original_image_path}") + overlay_path = output / "grounding_overlay.jpg" + ground_truth.write_overlay(frame, str(overlay_path)) + (output / "frame.json").write_text( + json.dumps( + { + "schema_version": "1.0", + "frame_id": frame.id, + "recording": recording, + "frame_index": frame_index, + "image": image_path.name, + "original_image": original_image_path.name + if frame.original_image is not None + else None, + "grounding_overlay": overlay_path.name, + "question_count": len(intents), + "accepted_question_count": len(examples), + "rejected_question_count": len(results) - len(examples), + "question_source": "image_agent" if propose_questions else "explicit_queries", + "question_model": question_model if propose_questions else None, + "grounding": { + "min_mask_area_px": min_mask_area_px, + "min_foreground_points": min_foreground_points, + }, + }, + indent=2, + ) + + "\n" + ) + (output / "intents.json").write_text( + json.dumps([asdict(item) for item in intents], indent=2) + "\n" + ) + (output / "examples.json").write_text( + json.dumps([asdict(item) for item in examples], indent=2) + "\n" + ) + (output / "ground_truth.json").write_text( + json.dumps([asdict(item) for item in results], indent=2) + "\n" + ) + (output / "evaluations.json").write_text( + json.dumps([asdict(item) for item in evaluations], indent=2) + "\n" + ) + typer.echo(f"Wrote {len(examples)} examples and {len(evaluations)} evaluations to {output}") + + +@app.command("generate") +def generate( + recording: str = typer.Option(..., "--recording"), + start_index: int = typer.Option(0, "--start-index"), + stop_index: int = typer.Option(..., "--stop-index"), + stride: int = typer.Option(1, "--stride"), + query: list[str] = typer.Option([], "--query"), + propose_questions: bool = typer.Option(False, "--propose-questions"), + question_model: str = typer.Option("gpt-4o-mini", "--question-model"), + min_mask_area_px: int = typer.Option(128, "--min-mask-area-px"), + min_foreground_points: int = typer.Option(3, "--min-foreground-points"), + output: Path | None = typer.Option(None, "--output"), +) -> None: + """Generate a resumable VQA dataset from sampled Go2 recording frames.""" + if ( + start_index < 0 + or stop_index <= start_index + or stride < 1 + or (not query and not propose_questions) + ): + raise typer.BadParameter( + "provide valid frame bounds and either --query or --propose-questions" + ) + output = output or (STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frames") + output.mkdir(parents=True, exist_ok=True) + model = MoondreamVlModel() + model.start() + try: + detector = MoondreamObjectDetector(model) + segmenter = EdgeTamObjectSegmenter(EdgeTAMImageSegmenter()) + question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=question_model)) + for frame_index in range(start_index, stop_index, stride): + frame_output = output / f"frame-{frame_index:06d}" + if (frame_output / "frame.json").is_file(): + continue + frame = load_go2_frame(str(resolve_named_path(recording, ".db")), frame_index) + intents = ( + question_agent.propose(frame.image) + if propose_questions + else [ + QuestionIntent( + kind=kind, + object_query=item, + threshold_m=3.0 if kind == "within_distance" else None, + ) + for item in query + for kind in ( + "presence", + "horizontal_direction", + "within_distance", + "compare_nearest_by_side", + ) + ] + ) + ground_truth = GroundTruthPerceptionAgent( + detector, + segmenter, + localizer=detector, + point_segmenter=segmenter, + config=GroundingConfig( + min_mask_area_px=min_mask_area_px, min_foreground_points=min_foreground_points + ), + ) + results = [ground_truth.answer(frame, intent) for intent in intents] + examples = [result.question for result in results if result.status == "answered"] + evaluations = evaluate_ground_truth(frame, examples, MoondreamQuestionAnswerer(model)) + write_frame_record( + frame_output, + frame, + recording, + frame_index, + intents, + results, + evaluations, + ground_truth, + { + "question_source": "openai_image_agent" + if propose_questions + else "explicit_queries", + "question_model": question_model if propose_questions else None, + "grounding": { + "min_mask_area_px": min_mask_area_px, + "min_foreground_points": min_foreground_points, + }, + }, + ) + typer.echo(f"Generated {frame_output}") + finally: + model.stop() + summary = write_dataset_manifest(output) + typer.echo(f"Dataset manifest: {summary}") diff --git a/dimos/perception/vqa/dataset.py b/dimos/perception/vqa/dataset.py new file mode 100644 index 0000000000..66eef98d89 --- /dev/null +++ b/dimos/perception/vqa/dataset.py @@ -0,0 +1,94 @@ +# Copyright 2026 Dimensional Inc. +"""Persist generated single-frame VQA records and dataset manifests.""" + +from __future__ import annotations + +from dataclasses import asdict +import json +from pathlib import Path +from typing import Any + +import cv2 + +from dimos.perception.vqa.ground_truth_agent import GroundTruthPerceptionAgent +from dimos.perception.vqa.models import ( + CalibratedFrame, + GroundTruthResult, + QuestionIntent, + VqaEvaluation, +) + + +def write_frame_record( + output: Path, + frame: CalibratedFrame, + recording: str, + frame_index: int, + intents: list[QuestionIntent], + results: list[GroundTruthResult], + evaluations: list[VqaEvaluation], + ground_truth: GroundTruthPerceptionAgent, + metadata: dict[str, Any], +) -> None: + """Write one self-contained frame record and its private evidence.""" + output.mkdir(parents=True, exist_ok=False) + image_path = output / "image.jpg" + if not cv2.imwrite(str(image_path), frame.image.data): + raise RuntimeError(f"failed to write {image_path}") + original_image_path = output / "original_image.jpg" + if frame.original_image is not None and not cv2.imwrite( + str(original_image_path), frame.original_image.data + ): + raise RuntimeError(f"failed to write {original_image_path}") + overlay_path = output / "grounding_overlay.jpg" + ground_truth.write_overlay(frame, str(overlay_path)) + examples = [result.question for result in results if result.status == "answered"] + frame_meta = { + "schema_version": "1.0", + "frame_id": frame.id, + "recording": recording, + "frame_index": frame_index, + "image": image_path.name, + "original_image": original_image_path.name if frame.original_image is not None else None, + "grounding_overlay": overlay_path.name, + "question_count": len(intents), + "accepted_question_count": len(examples), + "rejected_question_count": len(results) - len(examples), + **metadata, + } + _write_json(output / "frame.json", frame_meta) + _write_json(output / "intents.json", [asdict(item) for item in intents]) + _write_json(output / "examples.json", [asdict(item) for item in examples]) + _write_json(output / "ground_truth.json", [asdict(item) for item in results]) + _write_json(output / "evaluations.json", [asdict(item) for item in evaluations]) + + +def write_dataset_manifest(output: Path) -> dict[str, int]: + """Rebuild aggregate manifests from completed frame record directories.""" + frames = sorted(path for path in output.glob("frame-*") if (path / "frame.json").is_file()) + accepted = 0 + rejected = 0 + with ( + (output / "frames.jsonl").open("w") as frame_file, + (output / "ground_truth.jsonl").open("w") as gt_file, + ): + for path in frames: + frame = json.loads((path / "frame.json").read_text()) + frame_file.write(json.dumps(frame) + "\n") + for result in json.loads((path / "ground_truth.json").read_text()): + gt_file.write(json.dumps({"frame_id": frame["frame_id"], **result}) + "\n") + if result["status"] == "answered": + accepted += 1 + else: + rejected += 1 + summary = { + "frame_count": len(frames), + "accepted_question_count": accepted, + "rejected_question_count": rejected, + } + _write_json(output / "manifest.json", summary) + return summary + + +def _write_json(path: Path, payload: Any) -> None: + path.write_text(json.dumps(payload, indent=2) + "\n") diff --git a/dimos/perception/vqa/recording.py b/dimos/perception/vqa/recording.py new file mode 100644 index 0000000000..62c7165189 --- /dev/null +++ b/dimos/perception/vqa/recording.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +"""Build self-contained VQA frames from Go2 Memory2 recordings.""" + +from __future__ import annotations + +import numpy as np + +from dimos.memory2.store.sqlite import SqliteStore +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.perception.vqa.models import CalibratedFrame +from dimos.robot.unitree.go2.connection import BASE_TO_OPTICAL, GO2Connection + + +def load_go2_frame(recording: str, frame_index: int, tolerance_s: float = 0.25) -> CalibratedFrame: + """Load one image with its nearest LiDAR and odometry observations.""" + if frame_index < 0 or tolerance_s <= 0: + raise ValueError("frame_index must be non-negative and tolerance_s must be positive") + store = SqliteStore(path=recording, must_exist=True) + store.start() + try: + image_obs = store.streams.color_image.offset(frame_index).first() + lidar_obs = store.streams.lidar.at(image_obs.ts, tolerance_s).first() + odom_obs = store.streams.odom.at(image_obs.ts, tolerance_s).first() + image_data = image_obs.data + lidar_data = lidar_obs.data + odom_data = odom_obs.data + finally: + store.stop() + image, camera_info = _rectify_go2_image(image_data) + world_to_camera = -(Transform.from_pose("base_link", odom_data) + BASE_TO_OPTICAL) + return CalibratedFrame( + id=f"go2-{frame_index}", + image=image, + pointcloud=lidar_data, + camera_info=camera_info, + pointcloud_to_camera=world_to_camera, + image_is_rectified=True, + original_image=image_data, + ) + + +def _rectify_go2_image(image: Image) -> tuple[Image, CameraInfo]: + import cv2 + + source = GO2Connection.camera_info_static + matrix = np.asarray(source.K, dtype=np.float64).reshape(3, 3) + distortion = np.asarray(source.D, dtype=np.float64) + size = (image.width, image.height) + map_x, map_y = cv2.fisheye.initUndistortRectifyMap( + matrix, distortion, np.eye(3), matrix, size, cv2.CV_32FC1 + ) + data = cv2.remap(image.data, map_x, map_y, interpolation=cv2.INTER_LINEAR) + camera_info = CameraInfo.from_intrinsics( + matrix[0, 0], + matrix[1, 1], + matrix[0, 2], + matrix[1, 2], + image.width, + image.height, + frame_id="camera_optical", + ) + return Image(data=data, format=image.format, frame_id=image.frame_id, ts=image.ts), camera_info From 5e8d629d05a2f45d3a222fad2aa2844ca2bc06b5 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Fri, 7 Aug 2026 11:01:23 -0700 Subject: [PATCH 03/12] refactor(benchmark): move VQA generation package --- .../vqa/evaluation/scoring.py} | 21 ++++++++++- .../vqa/generation}/adapters.py | 10 ------ .../vqa/generation}/dataset.py | 9 ++--- .../vqa/generation}/geometry.py | 2 +- .../vqa/generation/ground_truth_generator.py} | 14 ++++---- .../vqa/generation}/grounding.py | 4 +-- .../vqa/generation}/pipeline.py | 22 +++--------- .../vqa/generation}/question_agent.py | 2 +- .../vqa/generation}/questions.py | 2 +- .../vqa/generation}/recording.py | 2 +- .../vqa/generation}/test_agents.py | 18 +++++----- .../vqa/generation}/test_geometry.py | 4 +-- .../vqa/generation}/test_single_frame.py | 7 ++-- dimos/{perception => benchmark}/vqa/models.py | 17 --------- dimos/cli/vqa.py | 35 +++++++------------ 15 files changed, 69 insertions(+), 100 deletions(-) rename dimos/{perception/vqa/evaluate.py => benchmark/vqa/evaluation/scoring.py} (75%) rename dimos/{perception/vqa => benchmark/vqa/generation}/adapters.py (86%) rename dimos/{perception/vqa => benchmark/vqa/generation}/dataset.py (91%) rename dimos/{perception/vqa => benchmark/vqa/generation}/geometry.py (97%) rename dimos/{perception/vqa/ground_truth_agent.py => benchmark/vqa/generation/ground_truth_generator.py} (98%) rename dimos/{perception/vqa => benchmark/vqa/generation}/grounding.py (93%) rename dimos/{perception/vqa => benchmark/vqa/generation}/pipeline.py (68%) rename dimos/{perception/vqa => benchmark/vqa/generation}/question_agent.py (98%) rename dimos/{perception/vqa => benchmark/vqa/generation}/questions.py (97%) rename dimos/{perception/vqa => benchmark/vqa/generation}/recording.py (97%) rename dimos/{perception/vqa => benchmark/vqa/generation}/test_agents.py (94%) rename dimos/{perception/vqa => benchmark/vqa/generation}/test_geometry.py (94%) rename dimos/{perception/vqa => benchmark/vqa/generation}/test_single_frame.py (91%) rename dimos/{perception => benchmark}/vqa/models.py (90%) diff --git a/dimos/perception/vqa/evaluate.py b/dimos/benchmark/vqa/evaluation/scoring.py similarity index 75% rename from dimos/perception/vqa/evaluate.py rename to dimos/benchmark/vqa/evaluation/scoring.py index 1874527a98..fb024fbabd 100644 --- a/dimos/perception/vqa/evaluate.py +++ b/dimos/benchmark/vqa/evaluation/scoring.py @@ -16,10 +16,29 @@ from __future__ import annotations +from dataclasses import dataclass import re +from typing import Protocol +from dimos.benchmark.vqa.models import VqaExample from dimos.msgs.sensor_msgs.Image import Image -from dimos.perception.vqa.models import VisualQuestionAnswerer, VqaEvaluation, VqaExample + + +@dataclass(frozen=True) +class VqaEvaluation: + """The result of asking an image-only agent one generated question.""" + + example_id: str + expected_answer: str + raw_response: str + normalized_response: str | None + passed: bool + + +class VisualQuestionAnswerer(Protocol): + """Answers a question from an image without access to ground truth.""" + + def answer(self, image: Image, question: str) -> str: ... def evaluate_examples( diff --git a/dimos/perception/vqa/adapters.py b/dimos/benchmark/vqa/generation/adapters.py similarity index 86% rename from dimos/perception/vqa/adapters.py rename to dimos/benchmark/vqa/generation/adapters.py index e9ff36a9b0..ebe4323599 100644 --- a/dimos/perception/vqa/adapters.py +++ b/dimos/benchmark/vqa/generation/adapters.py @@ -49,13 +49,3 @@ def segment(self, detections: ImageDetections2D) -> ImageDetections2D: def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: return self._segmenter.segment_points(points) - - -class MoondreamQuestionAnswerer: - """Ask MoonDream a question using only the supplied image.""" - - def __init__(self, model: MoondreamVlModel) -> None: - self._model = model - - def answer(self, image: Image, question: str) -> str: - return self._model.query(image, question) diff --git a/dimos/perception/vqa/dataset.py b/dimos/benchmark/vqa/generation/dataset.py similarity index 91% rename from dimos/perception/vqa/dataset.py rename to dimos/benchmark/vqa/generation/dataset.py index 66eef98d89..aa15bc62cf 100644 --- a/dimos/perception/vqa/dataset.py +++ b/dimos/benchmark/vqa/generation/dataset.py @@ -10,12 +10,11 @@ import cv2 -from dimos.perception.vqa.ground_truth_agent import GroundTruthPerceptionAgent -from dimos.perception.vqa.models import ( +from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator +from dimos.benchmark.vqa.models import ( CalibratedFrame, GroundTruthResult, QuestionIntent, - VqaEvaluation, ) @@ -26,8 +25,7 @@ def write_frame_record( frame_index: int, intents: list[QuestionIntent], results: list[GroundTruthResult], - evaluations: list[VqaEvaluation], - ground_truth: GroundTruthPerceptionAgent, + ground_truth: VqaGroundTruthGenerator, metadata: dict[str, Any], ) -> None: """Write one self-contained frame record and its private evidence.""" @@ -60,7 +58,6 @@ def write_frame_record( _write_json(output / "intents.json", [asdict(item) for item in intents]) _write_json(output / "examples.json", [asdict(item) for item in examples]) _write_json(output / "ground_truth.json", [asdict(item) for item in results]) - _write_json(output / "evaluations.json", [asdict(item) for item in evaluations]) def write_dataset_manifest(output: Path) -> dict[str, int]: diff --git a/dimos/perception/vqa/geometry.py b/dimos/benchmark/vqa/generation/geometry.py similarity index 97% rename from dimos/perception/vqa/geometry.py rename to dimos/benchmark/vqa/generation/geometry.py index d25d4364fc..db094027b6 100644 --- a/dimos/perception/vqa/geometry.py +++ b/dimos/benchmark/vqa/generation/geometry.py @@ -18,7 +18,7 @@ import numpy as np -from dimos.perception.vqa.models import CalibratedFrame, ProjectedPoints, ProjectionConfig +from dimos.benchmark.vqa.models import CalibratedFrame, ProjectedPoints, ProjectionConfig def project_visible_points( diff --git a/dimos/perception/vqa/ground_truth_agent.py b/dimos/benchmark/vqa/generation/ground_truth_generator.py similarity index 98% rename from dimos/perception/vqa/ground_truth_agent.py rename to dimos/benchmark/vqa/generation/ground_truth_generator.py index 65024972e2..7dd4d0c880 100644 --- a/dimos/perception/vqa/ground_truth_agent.py +++ b/dimos/benchmark/vqa/generation/ground_truth_generator.py @@ -3,11 +3,9 @@ from __future__ import annotations -from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox -from dimos.perception.detection.type.detection2d.point import Detection2DPoint -from dimos.perception.detection.type.detection2d.seg import Detection2DSeg -from dimos.perception.vqa.grounding import ground_segmented_objects -from dimos.perception.vqa.models import ( +from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects +from dimos.benchmark.vqa.generation.questions import generate_questions +from dimos.benchmark.vqa.models import ( CalibratedFrame, GroundedObject, GroundingConfig, @@ -20,10 +18,12 @@ ToolTrace, VqaExample, ) -from dimos.perception.vqa.questions import generate_questions +from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox +from dimos.perception.detection.type.detection2d.point import Detection2DPoint +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg -class GroundTruthPerceptionAgent: +class VqaGroundTruthGenerator: """Answer constrained questions by calling detection, segmentation, and geometry tools.""" def __init__( diff --git a/dimos/perception/vqa/grounding.py b/dimos/benchmark/vqa/generation/grounding.py similarity index 93% rename from dimos/perception/vqa/grounding.py rename to dimos/benchmark/vqa/generation/grounding.py index d1f5ba67b3..b6b6aa6c61 100644 --- a/dimos/perception/vqa/grounding.py +++ b/dimos/benchmark/vqa/generation/grounding.py @@ -18,9 +18,9 @@ import numpy as np +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.models import CalibratedFrame, GroundedObject, ProjectionConfig from dimos.perception.detection.type.detection2d.seg import Detection2DSeg -from dimos.perception.vqa.geometry import project_visible_points -from dimos.perception.vqa.models import CalibratedFrame, GroundedObject, ProjectionConfig def ground_segmented_objects( diff --git a/dimos/perception/vqa/pipeline.py b/dimos/benchmark/vqa/generation/pipeline.py similarity index 68% rename from dimos/perception/vqa/pipeline.py rename to dimos/benchmark/vqa/generation/pipeline.py index 834e32836c..19476557f6 100644 --- a/dimos/perception/vqa/pipeline.py +++ b/dimos/benchmark/vqa/generation/pipeline.py @@ -12,22 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Single-frame VQA ground-truth generation and image-only evaluation.""" +"""Single-frame VQA private ground-truth generation.""" from __future__ import annotations -from dimos.perception.detection.type.detection2d.seg import Detection2DSeg -from dimos.perception.vqa.evaluate import evaluate_examples -from dimos.perception.vqa.grounding import ground_segmented_objects -from dimos.perception.vqa.models import ( +from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects +from dimos.benchmark.vqa.generation.questions import generate_questions +from dimos.benchmark.vqa.models import ( CalibratedFrame, ObjectDetector, ObjectSegmenter, - VisualQuestionAnswerer, - VqaEvaluation, VqaExample, ) -from dimos.perception.vqa.questions import generate_questions +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg def generate_ground_truth( @@ -44,12 +41,3 @@ def generate_ground_truth( segmented.extend(detection for detection in result if isinstance(detection, Detection2DSeg)) objects = ground_segmented_objects(frame, segmented) return generate_questions(frame.id, objects, queries) - - -def evaluate_ground_truth( - frame: CalibratedFrame, - examples: list[VqaExample], - answerer: VisualQuestionAnswerer, -) -> list[VqaEvaluation]: - """Evaluate an answerer without passing it geometry or expected answers.""" - return evaluate_examples(frame.image, examples, answerer) diff --git a/dimos/perception/vqa/question_agent.py b/dimos/benchmark/vqa/generation/question_agent.py similarity index 98% rename from dimos/perception/vqa/question_agent.py rename to dimos/benchmark/vqa/generation/question_agent.py index 6c317637a9..b6290137a1 100644 --- a/dimos/perception/vqa/question_agent.py +++ b/dimos/benchmark/vqa/generation/question_agent.py @@ -7,9 +7,9 @@ import re from typing import Any +from dimos.benchmark.vqa.models import QuestionIntent from dimos.models.vl.openai import OpenAIVlModel from dimos.msgs.sensor_msgs.Image import Image -from dimos.perception.vqa.models import QuestionIntent QUESTION_PROMPT = """You generate challenging but visually answerable single-frame VQA questions. Inspect only this image. Do not assume depth, point clouds, metadata, or temporal context. diff --git a/dimos/perception/vqa/questions.py b/dimos/benchmark/vqa/generation/questions.py similarity index 97% rename from dimos/perception/vqa/questions.py rename to dimos/benchmark/vqa/generation/questions.py index 354a8a0119..08efe2c4c3 100644 --- a/dimos/perception/vqa/questions.py +++ b/dimos/benchmark/vqa/generation/questions.py @@ -16,7 +16,7 @@ from __future__ import annotations -from dimos.perception.vqa.models import GroundedObject, VqaExample +from dimos.benchmark.vqa.models import GroundedObject, VqaExample def generate_questions( diff --git a/dimos/perception/vqa/recording.py b/dimos/benchmark/vqa/generation/recording.py similarity index 97% rename from dimos/perception/vqa/recording.py rename to dimos/benchmark/vqa/generation/recording.py index 62c7165189..ff3cd6d49e 100644 --- a/dimos/perception/vqa/recording.py +++ b/dimos/benchmark/vqa/generation/recording.py @@ -5,11 +5,11 @@ import numpy as np +from dimos.benchmark.vqa.models import CalibratedFrame from dimos.memory2.store.sqlite import SqliteStore from dimos.msgs.geometry_msgs.Transform import Transform from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo from dimos.msgs.sensor_msgs.Image import Image -from dimos.perception.vqa.models import CalibratedFrame from dimos.robot.unitree.go2.connection import BASE_TO_OPTICAL, GO2Connection diff --git a/dimos/perception/vqa/test_agents.py b/dimos/benchmark/vqa/generation/test_agents.py similarity index 94% rename from dimos/perception/vqa/test_agents.py rename to dimos/benchmark/vqa/generation/test_agents.py index 2bdd1462b5..7503c3510c 100644 --- a/dimos/perception/vqa/test_agents.py +++ b/dimos/benchmark/vqa/generation/test_agents.py @@ -8,6 +8,9 @@ import cv2 import numpy as np +from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator +from dimos.benchmark.vqa.generation.question_agent import OpenAIQuestionAgent +from dimos.benchmark.vqa.models import CalibratedFrame, GroundingConfig, QuestionIntent from dimos.models.vl.openai import OpenAIVlModel from dimos.msgs.geometry_msgs.Transform import Transform from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo @@ -16,9 +19,6 @@ from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D from dimos.perception.detection.type.detection2d.point import Detection2DPoint from dimos.perception.detection.type.detection2d.seg import Detection2DSeg -from dimos.perception.vqa.ground_truth_agent import GroundTruthPerceptionAgent -from dimos.perception.vqa.models import CalibratedFrame, GroundingConfig, QuestionIntent -from dimos.perception.vqa.question_agent import OpenAIQuestionAgent class _QuestionModel: @@ -97,7 +97,7 @@ def test_question_agent_returns_constrained_intents() -> None: def test_ground_truth_agent_records_tools_and_rejects_unsupported_question() -> None: frame, detection = _frame_and_detection() - agent = GroundTruthPerceptionAgent( + agent = VqaGroundTruthGenerator( _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) ) @@ -124,7 +124,7 @@ def test_ground_truth_agent_records_tools_and_rejects_unsupported_question() -> def test_ground_truth_agent_rejects_small_masks() -> None: frame, detection = _frame_and_detection() - agent = GroundTruthPerceptionAgent( + agent = VqaGroundTruthGenerator( _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=37) ) @@ -145,7 +145,7 @@ def test_ground_truth_agent_falls_back_to_point_prompt() -> None: detection.image, detection.mask, ) - agent = GroundTruthPerceptionAgent( + agent = VqaGroundTruthGenerator( _Detector(frame.image, detection), _Segmenter(), localizer=_PointLocalizer(), @@ -194,7 +194,7 @@ def test_ground_truth_agent_compares_nearest_objects_by_side() -> None: Detection2DSeg((0.0, 0.0, 2.0, 5.0), 0, -1, 1.0, "chair", 0.0, image, left_mask), Detection2DSeg((3.0, 0.0, 5.0, 5.0), 1, -1, 1.0, "chair", 0.0, image, right_mask), ] - agent = GroundTruthPerceptionAgent( + agent = VqaGroundTruthGenerator( _MultiDetector(image, detections), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) ) @@ -209,7 +209,7 @@ def test_ground_truth_agent_compares_nearest_objects_by_side() -> None: def test_ground_truth_agent_rejects_side_comparison_without_both_sides() -> None: frame, detection = _frame_and_detection() - agent = GroundTruthPerceptionAgent( + agent = VqaGroundTruthGenerator( _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) ) @@ -225,7 +225,7 @@ def test_ground_truth_agent_overlay_includes_detector_prompts_and_question_legen tmp_path: Path, ) -> None: frame, detection = _frame_and_detection() - agent = GroundTruthPerceptionAgent( + agent = VqaGroundTruthGenerator( _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) ) agent.answer(frame, QuestionIntent(kind="presence", object_query="chair")) diff --git a/dimos/perception/vqa/test_geometry.py b/dimos/benchmark/vqa/generation/test_geometry.py similarity index 94% rename from dimos/perception/vqa/test_geometry.py rename to dimos/benchmark/vqa/generation/test_geometry.py index 6d9e9fe43e..d4977c0adb 100644 --- a/dimos/perception/vqa/test_geometry.py +++ b/dimos/benchmark/vqa/generation/test_geometry.py @@ -17,12 +17,12 @@ import numpy as np import pytest +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.models import CalibratedFrame from dimos.msgs.geometry_msgs.Transform import Transform from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo from dimos.msgs.sensor_msgs.Image import Image from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 -from dimos.perception.vqa.geometry import project_visible_points -from dimos.perception.vqa.models import CalibratedFrame def _frame(points: np.ndarray, *, rectified: bool = True) -> CalibratedFrame: diff --git a/dimos/perception/vqa/test_single_frame.py b/dimos/benchmark/vqa/generation/test_single_frame.py similarity index 91% rename from dimos/perception/vqa/test_single_frame.py rename to dimos/benchmark/vqa/generation/test_single_frame.py index a68addc09c..3865913384 100644 --- a/dimos/perception/vqa/test_single_frame.py +++ b/dimos/benchmark/vqa/generation/test_single_frame.py @@ -16,14 +16,15 @@ import numpy as np +from dimos.benchmark.vqa.evaluation.scoring import evaluate_examples +from dimos.benchmark.vqa.generation.pipeline import generate_ground_truth +from dimos.benchmark.vqa.models import CalibratedFrame from dimos.msgs.geometry_msgs.Transform import Transform from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo from dimos.msgs.sensor_msgs.Image import Image from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D from dimos.perception.detection.type.detection2d.seg import Detection2DSeg -from dimos.perception.vqa.models import CalibratedFrame -from dimos.perception.vqa.pipeline import evaluate_ground_truth, generate_ground_truth class _Answerer: @@ -69,7 +70,7 @@ def test_single_frame_ground_truth_and_image_only_evaluation() -> None: frame, ["chair", "table"], _Detector(image, detection), _Segmenter() ) answerer = _Answerer() - evaluations = evaluate_ground_truth(frame, examples, answerer) + evaluations = evaluate_examples(frame.image, examples, answerer) assert {example.expected_answer for example in examples} == {"yes", "no", "center"} assert all(image_arg is image for image_arg, _ in answerer.calls) diff --git a/dimos/perception/vqa/models.py b/dimos/benchmark/vqa/models.py similarity index 90% rename from dimos/perception/vqa/models.py rename to dimos/benchmark/vqa/models.py index 75af168686..4e4e5d2133 100644 --- a/dimos/perception/vqa/models.py +++ b/dimos/benchmark/vqa/models.py @@ -85,17 +85,6 @@ class VqaExample: object_ids: tuple[str, ...] -@dataclass(frozen=True) -class VqaEvaluation: - """The result of asking an image-only agent one generated question.""" - - example_id: str - expected_answer: str - raw_response: str - normalized_response: str | None - passed: bool - - QuestionKind = Literal[ "presence", "horizontal_direction", "within_distance", "compare_nearest_by_side" ] @@ -153,9 +142,3 @@ class PointObjectSegmenter(Protocol): """Creates foreground masks from positive image-point prompts.""" def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: ... - - -class VisualQuestionAnswerer(Protocol): - """Answers a question from an image without access to ground truth.""" - - def answer(self, image: Image, question: str) -> str: ... diff --git a/dimos/cli/vqa.py b/dimos/cli/vqa.py index a3ab974f90..a20621b4f0 100644 --- a/dimos/cli/vqa.py +++ b/dimos/cli/vqa.py @@ -10,24 +10,22 @@ import cv2 import typer +from dimos.benchmark.vqa.generation.adapters import ( + EdgeTamObjectSegmenter, + MoondreamObjectDetector, +) +from dimos.benchmark.vqa.generation.dataset import write_dataset_manifest, write_frame_record +from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator +from dimos.benchmark.vqa.generation.question_agent import OpenAIQuestionAgent +from dimos.benchmark.vqa.generation.recording import load_go2_frame +from dimos.benchmark.vqa.models import GroundingConfig, QuestionIntent from dimos.constants import STATE_DIR from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter from dimos.models.vl.moondream import MoondreamVlModel from dimos.models.vl.openai import OpenAIVlModel -from dimos.perception.vqa.adapters import ( - EdgeTamObjectSegmenter, - MoondreamObjectDetector, - MoondreamQuestionAnswerer, -) -from dimos.perception.vqa.dataset import write_dataset_manifest, write_frame_record -from dimos.perception.vqa.ground_truth_agent import GroundTruthPerceptionAgent -from dimos.perception.vqa.models import GroundingConfig, QuestionIntent -from dimos.perception.vqa.pipeline import evaluate_ground_truth -from dimos.perception.vqa.question_agent import OpenAIQuestionAgent -from dimos.perception.vqa.recording import load_go2_frame from dimos.utils.data import resolve_named_path -app = typer.Typer(help="Generate and evaluate point-cloud-grounded VQA examples") +app = typer.Typer(help="Generate point-cloud-grounded VQA benchmark examples") @app.command("single-frame") @@ -70,7 +68,7 @@ def single_frame( ) ] ) - ground_truth = GroundTruthPerceptionAgent( + ground_truth = VqaGroundTruthGenerator( detector := MoondreamObjectDetector(model), segmenter := EdgeTamObjectSegmenter(EdgeTAMImageSegmenter()), localizer=detector, @@ -82,7 +80,6 @@ def single_frame( ) results = [ground_truth.answer(frame, intent) for intent in intents] examples = [result.question for result in results if result.status == "answered"] - evaluations = evaluate_ground_truth(frame, examples, MoondreamQuestionAnswerer(model)) finally: model.stop() output.mkdir(parents=True) @@ -131,10 +128,7 @@ def single_frame( (output / "ground_truth.json").write_text( json.dumps([asdict(item) for item in results], indent=2) + "\n" ) - (output / "evaluations.json").write_text( - json.dumps([asdict(item) for item in evaluations], indent=2) + "\n" - ) - typer.echo(f"Wrote {len(examples)} examples and {len(evaluations)} evaluations to {output}") + typer.echo(f"Wrote {len(examples)} examples to {output}") @app.command("generate") @@ -191,7 +185,7 @@ def generate( ) ] ) - ground_truth = GroundTruthPerceptionAgent( + ground_truth = VqaGroundTruthGenerator( detector, segmenter, localizer=detector, @@ -201,8 +195,6 @@ def generate( ), ) results = [ground_truth.answer(frame, intent) for intent in intents] - examples = [result.question for result in results if result.status == "answered"] - evaluations = evaluate_ground_truth(frame, examples, MoondreamQuestionAnswerer(model)) write_frame_record( frame_output, frame, @@ -210,7 +202,6 @@ def generate( frame_index, intents, results, - evaluations, ground_truth, { "question_source": "openai_image_agent" From fe84603f699d9b58e75ca2275b88ee583a731ac2 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Sat, 8 Aug 2026 16:12:10 -0700 Subject: [PATCH 04/12] refactor(benchmark): migrate VQA to shared evaluations --- dimos/benchmark/evaluation/registry.py | 1 + dimos/benchmark/vqa/evaluation.py | 144 +++++++ dimos/benchmark/vqa/evaluation/scoring.py | 66 ---- dimos/benchmark/vqa/generation/dataset.py | 172 +++++++-- .../vqa/generation/ground_truth_generator.py | 130 ++----- .../benchmark/vqa/generation/measurements.py | 93 +++++ dimos/benchmark/vqa/generation/oracle.py | 287 ++++++++++++++ .../benchmark/vqa/generation/oracle_tools.py | 281 ++++++++++++++ .../vqa/generation/question_agent.py | 81 +++- dimos/benchmark/vqa/generation/questions.py | 3 + dimos/benchmark/vqa/generation/test_agents.py | 5 +- .../benchmark/vqa/generation/test_dataset.py | 76 ++++ dimos/benchmark/vqa/generation/test_oracle.py | 360 ++++++++++++++++++ .../vqa/generation/test_single_frame.py | 18 +- dimos/benchmark/vqa/models.py | 110 ++++++ dimos/benchmark/vqa/test_evaluation.py | 47 +++ dimos/cli/vqa.py | 263 +++++++++---- dimos/models/base.py | 14 +- docs/docs.json | 3 +- docs/vqa-benchmark.md | 79 ++++ 20 files changed, 1932 insertions(+), 301 deletions(-) create mode 100644 dimos/benchmark/vqa/evaluation.py delete mode 100644 dimos/benchmark/vqa/evaluation/scoring.py create mode 100644 dimos/benchmark/vqa/generation/measurements.py create mode 100644 dimos/benchmark/vqa/generation/oracle.py create mode 100644 dimos/benchmark/vqa/generation/oracle_tools.py create mode 100644 dimos/benchmark/vqa/generation/test_dataset.py create mode 100644 dimos/benchmark/vqa/generation/test_oracle.py create mode 100644 dimos/benchmark/vqa/test_evaluation.py create mode 100644 docs/vqa-benchmark.md diff --git a/dimos/benchmark/evaluation/registry.py b/dimos/benchmark/evaluation/registry.py index a6608daf67..14e929d911 100644 --- a/dimos/benchmark/evaluation/registry.py +++ b/dimos/benchmark/evaluation/registry.py @@ -31,6 +31,7 @@ LOCAL_NAME_PATTERN = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") BUILTIN_EVALUATIONS = { "frozen-integer-qa": ("dimos.benchmark.short_horizon_qa.evaluation:frozen_integer_qa"), + "point-cloud-vqa": ("dimos.benchmark.vqa.evaluation:point_cloud_vqa"), } diff --git a/dimos/benchmark/vqa/evaluation.py b/dimos/benchmark/vqa/evaluation.py new file mode 100644 index 0000000000..c7ae259156 --- /dev/null +++ b/dimos/benchmark/vqa/evaluation.py @@ -0,0 +1,144 @@ +# Copyright 2026 Dimensional Inc. +"""Multiple-choice VQA evaluation plugin using public image artifacts only.""" + +from __future__ import annotations + +from collections.abc import Callable +import json +from pathlib import Path +import re + +import cv2 +from pydantic import BaseModel, ConfigDict, Field + +from dimos.benchmark.evaluation.models import ( + ArtifactNativeResult, + ArtifactReference, + EvaluationReport, + SummaryItem, +) +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.sensor_msgs.Image import Image + + +class VqaEvaluationConfig(BaseModel): + """Location and vision model for a generated VQA evaluation dataset.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + dataset: str = Field(min_length=1) + model: str = Field(default="gpt-4o-mini", min_length=1) + + +class MultipleChoiceVqaEvaluation: + """Score a vision model against generated image-question-choice VQA cases.""" + + name = "point-cloud-vqa" + config_model: type[BaseModel] = VqaEvaluationConfig + + def __init__(self, vision_factory: Callable[[str], OpenAIVlModel] | None = None) -> None: + self._vision_factory = vision_factory or (lambda model: OpenAIVlModel(model_name=model)) + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + if not isinstance(config, VqaEvaluationConfig): + raise TypeError("point-cloud-vqa received the wrong configuration type") + dataset = Path(config.dataset).expanduser() + if not dataset.is_absolute(): + dataset = context.spec_dir / dataset + dataset = dataset.resolve() + cases = _load_jsonl(dataset / "cases.jsonl") + labels = {item["id"]: item["answer"] for item in _load_jsonl(dataset / "labels.jsonl")} + model = self._vision_factory(config.model) + results = [_evaluate_case(dataset, model, case, labels) for case in cases] + artifact = context.workspace / "vqa-results.json" + artifact.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + passed = sum(item["passed"] for item in results) + total = len(results) + return EvaluationReport( + summary=( + SummaryItem(key="cases", label="Cases", value=total), + SummaryItem(key="passed", label="Passed", value=passed), + SummaryItem( + key="accuracy", label="Accuracy", value=passed / total if total else 0.0 + ), + ), + native_result=ArtifactNativeResult( + artifact=ArtifactReference( + path=artifact.relative_to(context.workspace).as_posix(), + label="VQA case results", + media_type="application/json", + ) + ), + artifacts=( + ArtifactReference( + path=artifact.relative_to(context.workspace).as_posix(), + label="VQA case results", + media_type="application/json", + ), + ), + ) + + +def _evaluate_case( + dataset: Path, model: OpenAIVlModel, case: dict[str, object], labels: dict[str, str] +) -> dict[str, object]: + case_id = _required_string(case, "id") + choices = _choices(case) + expected = labels.get(case_id) + if expected is None: + raise ValueError(f"missing private label for case {case_id}") + if expected not in choices: + raise ValueError(f"private label for {case_id} is not an allowed choice") + image = cv2.imread(str(dataset / _required_string(case, "image"))) + if image is None: + raise ValueError(f"unable to load public image for case {case_id}") + prompt = ( + f"{_required_string(case, 'question')}\n\n" + f"Choices: {', '.join(choices)}.\n" + "Use only the supplied image. End with exactly `ANSWER: `." + ) + response = model.query(Image.from_numpy(image), prompt) + answer = _parse_choice(response, choices) + return { + "id": case_id, + "expected": expected, + "answer": answer, + "passed": answer == expected, + "raw_response": response, + } + + +def _load_jsonl(path: Path) -> list[dict[str, object]]: + if not path.is_file(): + raise ValueError(f"missing VQA dataset file: {path}") + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] + + +def _required_string(item: dict[str, object], key: str) -> str: + value = item.get(key) + if not isinstance(value, str) or not value: + raise ValueError(f"VQA case requires non-empty {key}") + return value + + +def _choices(case: dict[str, object]) -> tuple[str, ...]: + value = case.get("choices") + if ( + not isinstance(value, list) + or len(value) < 2 + or not all(isinstance(item, str) for item in value) + ): + raise ValueError("VQA case requires at least two string choices") + return tuple(value) + + +def _parse_choice(response: str, choices: tuple[str, ...]) -> str | None: + match = re.search(r"^ANSWER:\s*(.+?)\s*$", response, re.MULTILINE) + if match is None: + return None + answer = match.group(1) + return answer if answer in choices else None + + +point_cloud_vqa = MultipleChoiceVqaEvaluation() diff --git a/dimos/benchmark/vqa/evaluation/scoring.py b/dimos/benchmark/vqa/evaluation/scoring.py deleted file mode 100644 index fb024fbabd..0000000000 --- a/dimos/benchmark/vqa/evaluation/scoring.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Image-only visual-question-answering evaluation.""" - -from __future__ import annotations - -from dataclasses import dataclass -import re -from typing import Protocol - -from dimos.benchmark.vqa.models import VqaExample -from dimos.msgs.sensor_msgs.Image import Image - - -@dataclass(frozen=True) -class VqaEvaluation: - """The result of asking an image-only agent one generated question.""" - - example_id: str - expected_answer: str - raw_response: str - normalized_response: str | None - passed: bool - - -class VisualQuestionAnswerer(Protocol): - """Answers a question from an image without access to ground truth.""" - - def answer(self, image: Image, question: str) -> str: ... - - -def evaluate_examples( - image: Image, examples: list[VqaExample], answerer: VisualQuestionAnswerer -) -> list[VqaEvaluation]: - """Ask an agent questions from an image and compare closed answers.""" - evaluations: list[VqaEvaluation] = [] - for example in examples: - raw_response = answerer.answer(image, example.question) - normalized = _normalize_response(raw_response, example.expected_answer) - evaluations.append( - VqaEvaluation( - example_id=example.id, - expected_answer=example.expected_answer, - raw_response=raw_response, - normalized_response=normalized, - passed=normalized == example.expected_answer, - ) - ) - return evaluations - - -def _normalize_response(response: str, expected: str) -> str | None: - tokens = re.findall(r"[a-z]+", response.lower()) - return expected if expected.lower() in tokens else None diff --git a/dimos/benchmark/vqa/generation/dataset.py b/dimos/benchmark/vqa/generation/dataset.py index aa15bc62cf..ed2bf905a5 100644 --- a/dimos/benchmark/vqa/generation/dataset.py +++ b/dimos/benchmark/vqa/generation/dataset.py @@ -1,5 +1,5 @@ # Copyright 2026 Dimensional Inc. -"""Persist generated single-frame VQA records and dataset manifests.""" +"""Persist VQA generation evidence and a simple multiple-choice evaluation export.""" from __future__ import annotations @@ -12,9 +12,13 @@ from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + BooleanAnswerContract, CalibratedFrame, GroundTruthResult, QuestionIntent, + QuestionProposal, + RejectedOracleResult, ) @@ -23,12 +27,12 @@ def write_frame_record( frame: CalibratedFrame, recording: str, frame_index: int, - intents: list[QuestionIntent], - results: list[GroundTruthResult], + intents: list[QuestionIntent | QuestionProposal], + results: list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult], ground_truth: VqaGroundTruthGenerator, metadata: dict[str, Any], ) -> None: - """Write one self-contained frame record and its private evidence.""" + """Write one frame's public cases alongside its private generation audit record.""" output.mkdir(parents=True, exist_ok=False) image_path = output / "image.jpg" if not cv2.imwrite(str(image_path), frame.image.data): @@ -40,52 +44,144 @@ def write_frame_record( raise RuntimeError(f"failed to write {original_image_path}") overlay_path = output / "grounding_overlay.jpg" ground_truth.write_overlay(frame, str(overlay_path)) - examples = [result.question for result in results if result.status == "answered"] - frame_meta = { - "schema_version": "1.0", - "frame_id": frame.id, - "recording": recording, - "frame_index": frame_index, - "image": image_path.name, - "original_image": original_image_path.name if frame.original_image is not None else None, - "grounding_overlay": overlay_path.name, - "question_count": len(intents), - "accepted_question_count": len(examples), - "rejected_question_count": len(results) - len(examples), - **metadata, - } - _write_json(output / "frame.json", frame_meta) + accepted = [result for result in results if _is_accepted(result)] + cases, labels = _evaluation_rows(frame.id, accepted) + _write_json( + output / "frame.json", + { + "schema_version": "1.0", + "frame_id": frame.id, + "recording": recording, + "frame_index": frame_index, + "image": image_path.name, + "original_image": original_image_path.name + if frame.original_image is not None + else None, + "grounding_overlay": overlay_path.name, + "question_count": len(intents), + "accepted_question_count": len(accepted), + "rejected_question_count": len(results) - len(accepted), + **metadata, + }, + ) _write_json(output / "intents.json", [asdict(item) for item in intents]) - _write_json(output / "examples.json", [asdict(item) for item in examples]) - _write_json(output / "ground_truth.json", [asdict(item) for item in results]) + _write_json(output / "examples.json", [_public_example(item, frame.id) for item in accepted]) + _write_json(output / "ground_truth.json", [_private_result(item) for item in results]) + _write_json(output / "cases.json", cases) + _write_json(output / "labels.json", labels) def write_dataset_manifest(output: Path) -> dict[str, int]: - """Rebuild aggregate manifests from completed frame record directories.""" + """Build aggregate public cases and private labels from completed frame records.""" frames = sorted(path for path in output.glob("frame-*") if (path / "frame.json").is_file()) + frame_rows: list[dict[str, Any]] = [] + case_rows: list[dict[str, Any]] = [] + label_rows: list[dict[str, Any]] = [] accepted = 0 rejected = 0 - with ( - (output / "frames.jsonl").open("w") as frame_file, - (output / "ground_truth.jsonl").open("w") as gt_file, - ): - for path in frames: - frame = json.loads((path / "frame.json").read_text()) - frame_file.write(json.dumps(frame) + "\n") - for result in json.loads((path / "ground_truth.json").read_text()): - gt_file.write(json.dumps({"frame_id": frame["frame_id"], **result}) + "\n") - if result["status"] == "answered": - accepted += 1 - else: - rejected += 1 - summary = { + for path in frames: + frame = json.loads((path / "frame.json").read_text()) + frame_rows.append(frame) + case_rows.extend( + {**case, "image": f"{path.name}/{case['image']}"} + for case in json.loads((path / "cases.json").read_text()) + ) + label_rows.extend(json.loads((path / "labels.json").read_text())) + accepted += frame["accepted_question_count"] + rejected += frame["rejected_question_count"] + _write_jsonl(output / "frames.jsonl", frame_rows) + _write_jsonl(output / "cases.jsonl", case_rows) + _write_jsonl(output / "labels.jsonl", label_rows) + _write_json( + output / "manifest.json", + { + "frame_count": len(frames), + "accepted_question_count": accepted, + "rejected_question_count": rejected, + }, + ) + return { "frame_count": len(frames), "accepted_question_count": accepted, "rejected_question_count": rejected, } - _write_json(output / "manifest.json", summary) - return summary + + +def _is_accepted(result: GroundTruthResult | AcceptedOracleResult | RejectedOracleResult) -> bool: + return isinstance(result, AcceptedOracleResult) or ( + isinstance(result, GroundTruthResult) and result.status == "answered" + ) + + +def _evaluation_rows( + frame_id: str, results: list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult] +) -> tuple[list[dict[str, Any]], list[dict[str, str]]]: + cases: list[dict[str, Any]] = [] + labels: list[dict[str, str]] = [] + for result in results: + if isinstance(result, RejectedOracleResult): + continue + if isinstance(result, AcceptedOracleResult): + contract = result.proposal.answer_contract + choices = ( + ("yes", "no") if isinstance(contract, BooleanAnswerContract) else contract.choices + ) + case_id = f"{frame_id}-{result.proposal.id}" + question = result.proposal.question + answer = result.answer + else: + case_id = result.question.id + question = result.question.question + choices = result.question.allowed_answers + answer = result.answer + if answer is None or answer not in choices: + raise ValueError(f"accepted VQA case {case_id} must have a choice answer") + cases.append( + {"id": case_id, "image": "image.jpg", "question": question, "choices": choices} + ) + labels.append({"id": case_id, "answer": answer}) + return cases, labels + + +def _public_example( + result: GroundTruthResult | AcceptedOracleResult, frame_id: str +) -> dict[str, Any]: + if isinstance(result, AcceptedOracleResult): + return { + "case_id": f"{frame_id}-{result.proposal.id}", + "question": result.proposal.question, + "answer_contract": asdict(result.proposal.answer_contract), + "object_queries": result.proposal.object_queries, + } + return asdict(result.question) + + +def _private_result( + result: GroundTruthResult | AcceptedOracleResult | RejectedOracleResult, +) -> dict[str, Any]: + if isinstance(result, AcceptedOracleResult): + return { + "status": "answered", + "answer": result.answer, + "proposal": asdict(result.proposal), + "evidence_ids": result.evidence_ids, + "tool_results": [asdict(item) for item in result.tool_results], + "trace": [asdict(item) for item in result.trace], + } + if isinstance(result, RejectedOracleResult): + return { + "status": "rejected", + "reason": result.reason, + "proposal": asdict(result.proposal), + "tool_results": [asdict(item) for item in result.tool_results], + "trace": [asdict(item) for item in result.trace], + } + return asdict(result) def _write_json(path: Path, payload: Any) -> None: path.write_text(json.dumps(payload, indent=2) + "\n") + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.write_text("".join(f"{json.dumps(row, sort_keys=True)}\n" for row in rows)) diff --git a/dimos/benchmark/vqa/generation/ground_truth_generator.py b/dimos/benchmark/vqa/generation/ground_truth_generator.py index 7dd4d0c880..1ea0dbe0b4 100644 --- a/dimos/benchmark/vqa/generation/ground_truth_generator.py +++ b/dimos/benchmark/vqa/generation/ground_truth_generator.py @@ -48,29 +48,35 @@ def __init__( self._overlay_results: list[GroundTruthResult] = [] def answer(self, frame: CalibratedFrame, intent: QuestionIntent) -> GroundTruthResult: - cached = self._groundings.get(intent.object_query) + objects, trace = self.ground(frame, intent.object_query) + result = self._answer_from_objects(frame, intent, objects, trace) + self._overlay_results.append(result) + return result + + def ground( + self, frame: CalibratedFrame, object_query: str + ) -> tuple[list[GroundedObject], tuple[ToolTrace, ...]]: + """Ground one semantic query for direct local oracle tools. + + The method operates solely on the supplied calibrated frame and exposes no + model or recording state to the caller. + """ + cached = self._groundings.get(object_query) if cached is not None: objects, prior_trace = cached - result = self._answer_from_objects( - frame, - intent, - objects, - (ToolTrace("reuse_grounding", intent.object_query), *prior_trace), - ) - self._overlay_results.append(result) - return result - trace: list[ToolTrace] = [ToolTrace("detect_objects", intent.object_query)] - detections = self._detector.detect(frame.image, intent.object_query) - self._detections[intent.object_query] = [ + return objects, (ToolTrace("reuse_grounding", object_query), *prior_trace) + trace: list[ToolTrace] = [ToolTrace("detect_objects", object_query)] + detections = self._detector.detect(frame.image, object_query) + self._detections[object_query] = [ item for item in detections if isinstance(item, Detection2DBBox) ] if len(detections): trace.append(ToolTrace("segment_objects", f"count={len(detections)}")) segmented = self._segmenter.segment(detections) elif self._localizer is not None and self._point_segmenter is not None: - trace.append(ToolTrace("locate_object_point", intent.object_query)) - points = self._localizer.locate(frame.image, intent.object_query) - self._points[intent.object_query] = [ + trace.append(ToolTrace("locate_object_point", object_query)) + points = self._localizer.locate(frame.image, object_query) + self._points[object_query] = [ item for item in points if isinstance(item, Detection2DPoint) ] trace.append(ToolTrace("segment_object_point", f"count={len(points)}")) @@ -84,15 +90,17 @@ def answer(self, frame: CalibratedFrame, intent: QuestionIntent) -> GroundTruthR and int((item.mask > 0).sum()) >= self._config.min_mask_area_px ] trace.append(ToolTrace("get_foreground_geometry", f"masks={len(masks)}")) - self._masks[intent.object_query] = masks + self._masks[object_query] = masks objects = ground_segmented_objects( frame, masks, min_foreground_points=self._config.min_foreground_points ) stored_trace = tuple(trace) - self._groundings[intent.object_query] = (objects, stored_trace) - result = self._answer_from_objects(frame, intent, objects, stored_trace) - self._overlay_results.append(result) - return result + self._groundings[object_query] = (objects, stored_trace) + return objects, stored_trace + + def masks_for_query(self, object_query: str) -> tuple[Detection2DSeg, ...]: + """Return masks produced by the most recent local grounding for a query.""" + return tuple(self._masks.get(object_query, [])) def _answer_from_objects( self, @@ -141,76 +149,17 @@ def _answer_from_objects( ) def write_overlay(self, frame: CalibratedFrame, path: str) -> None: - """Save private masks, detector prompts, and a per-question audit legend.""" + """Save private masks and detector prompts over the rectified image.""" import cv2 - import numpy as np overlay = frame.image.data.copy() colors = ((255, 128, 0), (0, 200, 255), (180, 0, 255), (0, 200, 80)) - question_labels: dict[str, list[str]] = {} - for index, result in enumerate(self._overlay_results, start=1): - question_labels.setdefault(result.intent.object_query, []).append(f"Q{index}") - for index, query in enumerate(question_labels): + for index, query in enumerate(self._masks): color = colors[index % len(colors)] - label = ",".join(question_labels[query]) for mask in self._masks.get(query, []): overlay[mask.mask > 0] = color - x1, y1, _, _ = map(int, mask.bbox) - cv2.putText( - overlay, - f"{label} {query}", - (x1, max(y1 - 6, 12)), - cv2.FONT_HERSHEY_SIMPLEX, - 0.5, - color, - 1, - ) - for detection in self._detections.get(query, []): - x1, y1, x2, y2 = map(int, detection.bbox) - cv2.rectangle(overlay, (x1, y1), (x2, y2), color, 2) - cv2.putText( - overlay, - f"{label} box", - (x1, min(y2 + 16, frame.image.height - 4)), - cv2.FONT_HERSHEY_SIMPLEX, - 0.5, - color, - 1, - ) - for point in self._points.get(query, []): - position = (int(point.x), int(point.y)) - cv2.drawMarker(overlay, position, color, cv2.MARKER_CROSS, 14, 2) - cv2.putText( - overlay, - f"{label} point", - (position[0] + 8, max(position[1] - 8, 12)), - cv2.FONT_HERSHEY_SIMPLEX, - 0.5, - color, - 1, - ) rendered = cv2.addWeighted(frame.image.data, 0.55, overlay, 0.45, 0) - legend_lines = ["Private grounding audit"] - for index, result in enumerate(self._overlay_results, start=1): - status = result.answer if result.status == "answered" else f"rejected: {result.reason}" - legend_lines.extend(_wrap_overlay_text(f"Q{index}: {result.question.question}", 52)) - legend_lines.extend(_wrap_overlay_text(f" {status}", 52)) - line_height = 20 - panel_width = 440 - panel_height = max(rendered.shape[0], 16 + line_height * len(legend_lines)) - audit = np.full((panel_height, rendered.shape[1] + panel_width, 3), 32, dtype=np.uint8) - audit[: rendered.shape[0], : rendered.shape[1]] = rendered - for index, line in enumerate(legend_lines): - cv2.putText( - audit, - line, - (rendered.shape[1] + 12, 20 + index * line_height), - cv2.FONT_HERSHEY_SIMPLEX, - 0.45, - (255, 255, 255), - 1, - ) - if not cv2.imwrite(path, audit): + if not cv2.imwrite(path, rendered): raise RuntimeError(f"failed to write {path}") @@ -224,22 +173,6 @@ def _render_question(intent: QuestionIntent) -> str: return f"Is the nearest {intent.object_query} within {intent.threshold_m or 3:g} meters? Answer yes or no." -def _wrap_overlay_text(text: str, width: int) -> list[str]: - words = text.split() - lines: list[str] = [] - line = "" - for word in words: - candidate = f"{line} {word}".strip() - if line and len(candidate) > width: - lines.append(line) - line = word - else: - line = candidate - if line: - lines.append(line) - return lines - - def _compare_nearest_by_side( frame: CalibratedFrame, intent: QuestionIntent, @@ -261,6 +194,7 @@ def _compare_nearest_by_side( answer, "choice", (left.id, right.id), + ("left", "right"), ) return GroundTruthResult(intent, example, "answered", answer, None, (left, right), trace) diff --git a/dimos/benchmark/vqa/generation/measurements.py b/dimos/benchmark/vqa/generation/measurements.py new file mode 100644 index 0000000000..b85872e7a3 --- /dev/null +++ b/dimos/benchmark/vqa/generation/measurements.py @@ -0,0 +1,93 @@ +# Copyright 2026 Dimensional Inc. +"""Deterministic point-cloud measurements for private VQA oracle tools.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.models import CalibratedFrame, GroundPlaneEstimate + + +@dataclass(frozen=True) +class PlaneFitResult: + """Accepted plane or explicit quality-gated rejection.""" + + estimate: GroundPlaneEstimate | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +def estimate_ground_plane(frame: CalibratedFrame) -> PlaneFitResult: + """Fit a robust plane to visible points in the lower image ground band.""" + projected = project_visible_points(frame) + candidates = np.asarray( + [ + point + for point, (_, y) in zip(projected.camera_points, projected.pixels, strict=True) + if y >= int(frame.image.height * 0.6) + ], + dtype=np.float64, + ) + min_points, min_inliers, threshold = 12, 10, 0.06 + if len(candidates) < min_points: + return PlaneFitResult(None, ("insufficient_ground_band_points",), "insufficient_support") + + # Open3D owns the randomized consensus search; seed its process-global RNG + # so repeated generation runs are stable for a fixed Open3D release. + import open3d as o3d + + o3d.utility.random.seed(0) + point_cloud = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(candidates)) + _, inlier_indices = point_cloud.segment_plane( + distance_threshold=threshold, + ransac_n=3, + num_iterations=816, + ) + if len(inlier_indices) < min_inliers: + return PlaneFitResult(None, ("insufficient_plane_inliers",), "insufficient_inliers") + + inlier_points = candidates[np.asarray(inlier_indices, dtype=int)] + center = np.mean(inlier_points, axis=0) + _, _, right = np.linalg.svd(inlier_points - center, full_matrices=False) + normal = right[-1] + offset = -float(normal @ center) + # The camera is normally above the support plane; this fixes signed heights. + if offset < 0: + normal, offset = -normal, -offset + residuals = np.abs(candidates @ normal + offset) + inliers = residuals <= threshold + residual_m = ( + float(np.sqrt(np.mean(np.square(residuals[inliers])))) if inliers.any() else float("inf") + ) + if int(inliers.sum()) < min_inliers: + return PlaneFitResult(None, ("insufficient_refined_inliers",), "insufficient_inliers") + if residual_m > 0.035: + return PlaneFitResult(None, ("high_plane_residual",), "residual_too_high") + return PlaneFitResult( + GroundPlaneEstimate( + tuple(float(value) for value in normal), + float(offset), + len(candidates), + int(inliers.sum()), + residual_m, + ), + ("ground_band_visible", "ransac_inliers_accepted"), + ) + + +def points_in_mask(frame: CalibratedFrame, mask: np.ndarray) -> np.ndarray: + """Return nearest visible camera points covered by one foreground mask.""" + if mask.shape != (frame.image.height, frame.image.width): + raise ValueError("segmentation mask dimensions must match the image") + projected = project_visible_points(frame) + return np.asarray( + [ + point + for point, (x, y) in zip(projected.camera_points, projected.pixels, strict=True) + if mask[y, x] > 0 + ], + dtype=np.float64, + ) diff --git a/dimos/benchmark/vqa/generation/oracle.py b/dimos/benchmark/vqa/generation/oracle.py new file mode 100644 index 0000000000..0a174a4cb2 --- /dev/null +++ b/dimos/benchmark/vqa/generation/oracle.py @@ -0,0 +1,287 @@ +# Copyright 2026 Dimensional Inc. +"""Private bounded LangChain oracle for generic VQA question proposals.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, replace +import json +import re +from typing import TYPE_CHECKING, Any, Protocol + +from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + AnswerContract, + BooleanAnswerContract, + ChoiceAnswerContract, + OracleToolResult, + OracleTrace, + QuestionProposal, + RejectedOracleResult, +) + +if TYPE_CHECKING: + from langchain_core.language_models.chat_models import BaseChatModel + + +@dataclass(frozen=True) +class SemanticEvidenceValidation: + """Private verdict on whether cited tool evidence supports an oracle answer.""" + + accepted: bool + reason: str + + +class SemanticEvidenceValidator(Protocol): + """Validate an answer only against the frozen question and cited local evidence.""" + + def validate( + self, + proposal: QuestionProposal, + answer: str, + cited_results: tuple[OracleToolResult, ...], + ) -> SemanticEvidenceValidation: ... + + +class OpenAISemanticEvidenceValidator: + """Private no-tools model judge for semantic grounding of a proposed answer.""" + + def __init__(self, model: BaseChatModel) -> None: + self._model = model + + def validate( + self, + proposal: QuestionProposal, + answer: str, + cited_results: tuple[OracleToolResult, ...], + ) -> SemanticEvidenceValidation: + from langchain_core.messages import HumanMessage, SystemMessage + + response = self._model.invoke( + [ + SystemMessage( + "You validate a private VQA oracle answer. Decide only whether the cited " + "structured local-tool evidence supports the answer to the frozen question. " + "Reject claims requiring measurements not present in the evidence; for example, " + "height is not supported by range or side alone. A cited measurement bucket must " + "exactly match the selected choice. Return strict JSON only: " + '{"accepted": true|false, "reason": "concise reason"}. Do not call tools.' + ), + HumanMessage( + json.dumps( + { + "question": proposal.question, + "answer": answer, + "answer_contract": asdict(proposal.answer_contract), + "cited_evidence": [asdict(result) for result in cited_results], + } + ) + ), + ] + ) + try: + payload = _parse_strict_json_object(_response_text(response.content)) + accepted, reason = payload.get("accepted"), payload.get("reason") + if not isinstance(accepted, bool) or not isinstance(reason, str) or not reason: + raise ValueError( + "validator response requires boolean accepted and non-empty reason" + ) + except (ValueError, json.JSONDecodeError, AttributeError) as exc: + return SemanticEvidenceValidation(False, f"invalid_validator_response:{exc}") + return SemanticEvidenceValidation(accepted, reason) + + +class PrivateToolCallingOracle: + """Use direct local tools only, then validate a model's final JSON response.""" + + def __init__( + self, + model: BaseChatModel, + max_tool_calls: int = 4, + semantic_validator: SemanticEvidenceValidator | None = None, + ) -> None: + if max_tool_calls < 1: + raise ValueError("max_tool_calls must be positive") + self._model = model + self._max_tool_calls = max_tool_calls + self._semantic_validator = semantic_validator + + def answer( + self, proposal: QuestionProposal, registry: LocalOracleToolRegistry + ) -> AcceptedOracleResult | RejectedOracleResult: + from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage + + tools = registry.tools() + model = self._model.bind_tools(tools) + messages: list[Any] = [ + SystemMessage( + "You are a private VQA oracle. Use only supplied local tools. Do not invent " + 'evidence. Finish with JSON only: {"answer": value, "evidence_ids": [..]}.' + ), + HumanMessage(_proposal_prompt(proposal)), + ] + trace: list[OracleTrace] = [] + calls = 0 + while calls < self._max_tool_calls: + response = model.invoke(messages) + messages.append(response) + tool_calls = getattr(response, "tool_calls", []) + if not tool_calls: + return _validated_result( + proposal, + _response_text(response.content), + registry.results, + trace, + self._semantic_validator, + ) + for call in tool_calls: + if calls >= self._max_tool_calls: + break + name = call.get("name") + tool = next((item for item in tools if item.name == name), None) + if tool is None: + return _rejected(proposal, "unsupported_tool", registry.results, trace) + try: + output = tool.invoke(call.get("args", {})) + except (TypeError, ValueError) as exc: + return _rejected(proposal, f"tool_error:{exc}", registry.results, trace) + calls += 1 + trace.append(OracleTrace("tool", str(name))) + messages.append(ToolMessage(content=str(output), tool_call_id=call["id"])) + return _rejected(proposal, "tool_call_limit", registry.results, trace) + + +def create_openai_oracle(model: str, max_tool_calls: int = 4) -> PrivateToolCallingOracle: + """Construct the private-only OpenAI tool-calling oracle.""" + from langchain_openai import ChatOpenAI + + return PrivateToolCallingOracle( + ChatOpenAI(model=model), + max_tool_calls=max_tool_calls, + semantic_validator=OpenAISemanticEvidenceValidator(ChatOpenAI(model=model)), + ) + + +def validate_oracle_answer( + proposal: QuestionProposal, + answer: Any, + evidence_ids: Any, + results: tuple[OracleToolResult, ...], +) -> str: + """Deterministically validate an answer format and citations.""" + if ( + not isinstance(evidence_ids, list) + or not evidence_ids + or not all(isinstance(item, str) for item in evidence_ids) + ): + raise ValueError("answer requires non-empty evidence_ids") + known_ids = {item.id for result in results for item in result.evidence} + if not set(evidence_ids).issubset(known_ids): + raise ValueError("answer cites unknown evidence") + contract = proposal.answer_contract + if isinstance(contract, BooleanAnswerContract): + if answer not in ("yes", "no"): + raise ValueError("boolean answer must be yes or no") + return str(answer) + if isinstance(contract, ChoiceAnswerContract): + if not isinstance(answer, str) or answer not in contract.choices: + raise ValueError("choice answer is not allowed") + measured_choices = {result.choice for result in results if result.choice is not None} + if measured_choices and answer not in measured_choices: + raise ValueError("choice answer does not match cited measurement bucket") + return answer + raise ValueError("unsupported answer contract") + + +def _validated_result( + proposal: QuestionProposal, + response: str, + results: tuple[OracleToolResult, ...], + trace: list[OracleTrace], + semantic_validator: SemanticEvidenceValidator | None, +) -> AcceptedOracleResult | RejectedOracleResult: + try: + payload = _parse_json_object(response) + answer = validate_oracle_answer( + proposal, payload.get("answer"), payload.get("evidence_ids"), results + ) + evidence_ids = tuple(payload["evidence_ids"]) + except (ValueError, json.JSONDecodeError, AttributeError) as exc: + return _rejected(proposal, f"invalid_final_answer:{exc}", results, trace) + cited_results = _cited_results(evidence_ids, results) + if semantic_validator is None: + return _rejected(proposal, "semantic_validator_not_configured", results, trace) + verdict = semantic_validator.validate(proposal, answer, cited_results) + trace.append( + OracleTrace( + "semantic_validation", + f"{'accepted' if verdict.accepted else 'rejected'}:{verdict.reason}", + ) + ) + if not verdict.accepted: + return _rejected(proposal, f"unsupported_evidence:{verdict.reason}", results, trace) + return AcceptedOracleResult(proposal, answer, evidence_ids, results, tuple(trace)) + + +def _cited_results( + evidence_ids: tuple[str, ...], results: tuple[OracleToolResult, ...] +) -> tuple[OracleToolResult, ...]: + cited = set(evidence_ids) + return tuple( + replace( + result, + evidence=tuple(evidence for evidence in result.evidence if evidence.id in cited), + ) + for result in results + if any(evidence.id in cited for evidence in result.evidence) + ) + + +def _rejected( + proposal: QuestionProposal, + reason: str, + results: tuple[OracleToolResult, ...], + trace: list[OracleTrace], +) -> RejectedOracleResult: + return RejectedOracleResult(proposal, reason, results, tuple(trace)) + + +def _proposal_prompt(proposal: QuestionProposal) -> str: + return ( + f"Question: {proposal.question}\nAnswer contract: {_contract_prompt(proposal.answer_contract)}\n" + f"Suggested object queries: {', '.join(proposal.object_queries) or 'none'}" + ) + + +def _contract_prompt(contract: AnswerContract) -> str: + if isinstance(contract, BooleanAnswerContract): + return "boolean: yes or no" + if isinstance(contract, ChoiceAnswerContract): + return f"choice: {', '.join(contract.choices)}" + raise ValueError("unsupported answer contract") + + +def _parse_json_object(response: str) -> dict[str, Any]: + stripped = re.sub(r"^```(?:json)?\s*|\s*```$", "", response.strip(), flags=re.IGNORECASE) + start, end = stripped.find("{"), stripped.rfind("}") + if start < 0 or end < start: + raise json.JSONDecodeError("expected JSON object", stripped, 0) + payload: Any = json.loads(stripped[start : end + 1]) + if not isinstance(payload, dict): + raise ValueError("final response must be an object") + return payload + + +def _parse_strict_json_object(response: str) -> dict[str, Any]: + payload: Any = json.loads(response) + if not isinstance(payload, dict): + raise ValueError("validator response must be an object") + return payload + + +def _response_text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return "".join(str(item.get("text", "")) for item in content if isinstance(item, dict)) + return str(content) diff --git a/dimos/benchmark/vqa/generation/oracle_tools.py b/dimos/benchmark/vqa/generation/oracle_tools.py new file mode 100644 index 0000000000..ff74e785f9 --- /dev/null +++ b/dimos/benchmark/vqa/generation/oracle_tools.py @@ -0,0 +1,281 @@ +# Copyright 2026 Dimensional Inc. +"""Direct local LangChain tools over one frozen VQA frame.""" + +from __future__ import annotations + +import json +from typing import Any + +from langchain_core.tools import StructuredTool +import numpy as np + +from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator +from dimos.benchmark.vqa.generation.measurements import estimate_ground_plane, points_in_mask +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + OracleEvidence, + OracleMeasurement, + OracleToolResult, +) + + +class LocalOracleToolRegistry: + """Expose only named, side-effect-free perception operations to an oracle.""" + + def __init__(self, frame: CalibratedFrame, grounding: VqaGroundTruthGenerator) -> None: + self._frame = frame + self._grounding = grounding + self._results: list[OracleToolResult] = [] + + @property + def results(self) -> tuple[OracleToolResult, ...]: + return tuple(self._results) + + def tools(self) -> list[StructuredTool]: + return [ + StructuredTool.from_function( + self.ground_semantic_object, + name="ground_semantic_object", + description=( + "Ground a visible semantic object query with private MoonDream, EdgeTAM, " + "and calibrated point-cloud geometry. Returns object geometry and evidence IDs." + ), + ), + StructuredTool.from_function( + self.estimate_ground_plane, + name="estimate_ground_plane", + description=( + "Estimate a visible ground plane from the frozen calibrated point cloud. " + "Returns a quality-gated local geometric result." + ), + ), + StructuredTool.from_function( + self.measure_object_height, + name="measure_object_height", + description=( + "Ground one visible semantic object with local MoonDream, EdgeTAM, and LiDAR " + "then measure its visible point-cloud height above the estimated ground plane." + ), + ), + StructuredTool.from_function( + self.measure_object_height_bucket, + name="measure_object_height_bucket", + description=( + "Measure one visible object's height, then return its public choice: under 0.5 m, " + "0.5-1.0 m, 1.0-1.5 m, or over 1.5 m." + ), + ), + ] + + def ground_semantic_object(self, query: str) -> str: + """Return grounded objects for a visible object query in the frozen frame.""" + objects, _ = self._grounding.ground(self._frame, query) + evidence = tuple( + OracleEvidence( + id=f"grounding:v1:{item.id}", + version="v1", + object_id=item.id, + label=item.label, + range_m=item.range_m, + side=item.horizontal_direction, + point_count=item.point_count, + ) + for item in objects + ) + result = OracleToolResult("ground_semantic_object", query, evidence) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def estimate_ground_plane(self) -> str: + """Estimate the lower-image visible ground plane without fabricating a result.""" + fit = estimate_ground_plane(self._frame) + if fit.estimate is None: + result = OracleToolResult( + "estimate_ground_plane", + "", + (), + quality_flags=fit.quality_flags, + rejection_reason=fit.rejection_reason, + ) + else: + measurement = OracleMeasurement( + fit.estimate.offset_m, + "m", + max(fit.estimate.residual_m, 0.01), + fit.quality_flags, + (f"frame:{self._frame.id}",), + ) + evidence = OracleEvidence( + f"ground-plane:v1:{self._frame.id}", + "v1", + "ground-plane", + "ground", + 0.0, + "n/a", + fit.estimate.inlier_count, + measurement, + ) + result = OracleToolResult( + "estimate_ground_plane", + "", + (evidence,), + measurement=measurement, + plane=fit.estimate, + quality_flags=fit.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def measure_object_height(self, query: str) -> str: + """Measure one unambiguous grounded object's visible height above local ground.""" + objects, _ = self._grounding.ground(self._frame, query) + masks_for_query = getattr(self._grounding, "masks_for_query", None) + masks = tuple(masks_for_query(query)) if callable(masks_for_query) else () + fit = estimate_ground_plane(self._frame) + flags = list(fit.quality_flags) + if fit.estimate is None: + return self._record_rejection( + "measure_object_height", query, flags, fit.rejection_reason + ) + if len(objects) != 1 or len(masks) != 1: + flags.append("ambiguous_or_missing_object_mask") + return self._record_rejection( + "measure_object_height", query, flags, "ambiguous_object_evidence" + ) + selected = points_in_mask(self._frame, masks[0].mask) + if len(selected) < 6: + flags.append("sparse_object_point_support") + return self._record_rejection( + "measure_object_height", query, flags, "insufficient_object_support" + ) + normal = np.asarray(fit.estimate.normal) + distances = selected @ normal + fit.estimate.offset_m + positive = distances[distances > 0.02] + if len(positive) < 4 or len(positive) / len(selected) < 0.6: + flags.append("partial_or_non_elevated_object_support") + return self._record_rejection( + "measure_object_height", query, flags, "ambiguous_object_extent" + ) + height = float(np.percentile(positive, 85)) + tolerance = float(max(0.05, fit.estimate.residual_m + np.std(positive) * 0.25)) + flags.extend(("visible_point_cloud_height", "conservative_upper_percentile")) + measurement = OracleMeasurement( + height, + "m", + tolerance, + tuple(flags), + ( + f"frame:{self._frame.id}", + f"ground-plane:v1:{self._frame.id}", + f"grounding:v1:{objects[0].id}", + ), + ) + evidence = OracleEvidence( + f"height:v1:{objects[0].id}", + "v1", + objects[0].id, + objects[0].label, + objects[0].range_m, + objects[0].horizontal_direction, + len(selected), + measurement, + ) + result = OracleToolResult( + "measure_object_height", + query, + (evidence,), + measurement=measurement, + plane=fit.estimate, + quality_flags=tuple(flags), + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def measure_object_height_bucket(self, query: str) -> str: + """Measure an object, then map its private height to a fixed public choice.""" + self.measure_object_height(query) + height_result = self._results[-1] + if height_result.measurement is None: + return json.dumps(_tool_payload(height_result)) + result = OracleToolResult( + "measure_object_height_bucket", + query, + height_result.evidence, + measurement=height_result.measurement, + choice=_height_bucket(height_result.measurement.value), + plane=height_result.plane, + quality_flags=height_result.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def _record_rejection(self, tool: str, query: str, flags: list[str], reason: str | None) -> str: + result = OracleToolResult( + tool, query, (), quality_flags=tuple(flags), rejection_reason=reason + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + +def _tool_payload(result: OracleToolResult) -> dict[str, Any]: + return { + "tool": result.tool, + "query": result.query, + "version": result.version, + "measurement": ( + { + "value": result.measurement.value, + "unit": result.measurement.unit, + "tolerance": result.measurement.tolerance, + "quality_flags": result.measurement.quality_flags, + "provenance_ids": result.measurement.provenance_ids, + } + if result.measurement is not None + else None + ), + "choice": result.choice, + "quality_flags": result.quality_flags, + "rejection_reason": result.rejection_reason, + "plane": ( + { + "normal": result.plane.normal, + "offset_m": result.plane.offset_m, + "sample_count": result.plane.sample_count, + "inlier_count": result.plane.inlier_count, + "residual_m": result.plane.residual_m, + } + if result.plane is not None + else None + ), + "objects": [_evidence_payload(item) for item in result.evidence], + } + + +def _evidence_payload(item: OracleEvidence) -> dict[str, Any]: + payload: dict[str, Any] = { + "evidence_id": item.id, + "id": item.object_id, + "label": item.label, + "range_m": item.range_m, + "side": item.side, + "point_count": item.point_count, + } + if item.measurement is not None: + payload["measurement"] = { + "value": item.measurement.value, + "unit": item.measurement.unit, + "tolerance": item.measurement.tolerance, + "quality_flags": item.measurement.quality_flags, + "provenance_ids": item.measurement.provenance_ids, + } + return payload + + +def _height_bucket(height_m: float) -> str: + if height_m < 0.5: + return "under 0.5 m" + if height_m < 1.0: + return "0.5-1.0 m" + if height_m < 1.5: + return "1.0-1.5 m" + return "over 1.5 m" diff --git a/dimos/benchmark/vqa/generation/question_agent.py b/dimos/benchmark/vqa/generation/question_agent.py index b6290137a1..cf1b21daee 100644 --- a/dimos/benchmark/vqa/generation/question_agent.py +++ b/dimos/benchmark/vqa/generation/question_agent.py @@ -7,7 +7,13 @@ import re from typing import Any -from dimos.benchmark.vqa.models import QuestionIntent +from dimos.benchmark.vqa.models import ( + AnswerContract, + BooleanAnswerContract, + ChoiceAnswerContract, + QuestionIntent, + QuestionProposal, +) from dimos.models.vl.openai import OpenAIVlModel from dimos.msgs.sensor_msgs.Image import Image @@ -17,6 +23,21 @@ Do not return floors, walls, ceilings, background surfaces, questions, explanations, Markdown, or information not visible in the image.""" +AGENTIC_QUESTION_PROMPT = """Author up to 5 challenging, visually answerable single-frame VQA questions. +Inspect only this image. Do not use or infer depth, point clouds, calibration, metadata, or answers. +Return JSON only: an array of objects with question, answer_contract, optional object_queries, +and optional tool_hints. answer_contract is {"kind":"boolean"} or {"kind":"choice","choices":[...]}. +Prioritize questions that a private point-cloud oracle can validate. For the height of one upright +object resting on visible ground, use exactly these choices: ["under 0.5 m", "0.5-1.0 m", +"1.0-1.5 m", "over 1.5 m"] and tool_hints ["measure_object_height_bucket"]. Also prefer which of +two named objects is closer (choice, with those object names as choices), object count, left/right +spatial relation, and distance-threshold questions. Use object_queries for every referenced object +and tool_hints from "measure_object_height_bucket", "measure_object_height", "estimate_ground_plane", +or "ground_semantic_object" when applicable. +Use visibility/presence questions only when no stronger geometric question is available. Do not ask +about color, material, text, intent, full physical size, hidden parts, or terrain. Use only visible +objects. Do not include answers, explanations, Markdown, or background surfaces.""" + _UNSUPPORTED_QUERIES = {"background", "ceiling", "floor", "ground", "room", "wall"} @@ -68,6 +89,64 @@ def propose(self, image: Image) -> list[QuestionIntent]: return intents +class OpenAIFreeformQuestionAuthor: + """Image-only author for generic public questions and answer contracts.""" + + def __init__(self, model: OpenAIVlModel, max_questions: int = 5) -> None: + self._model = model + self._max_questions = max_questions + + def propose(self, image: Image) -> list[QuestionProposal]: + try: + payload: Any = _parse_json_array(self._model.query(image, AGENTIC_QUESTION_PROMPT)) + except json.JSONDecodeError as exc: + raise ValueError("question author did not return JSON") from exc + if not isinstance(payload, list) or len(payload) > self._max_questions: + raise ValueError("question author returned too many questions") + proposals = [ + _proposal_from_json(item, index) for index, item in enumerate(payload, start=1) + ] + if len({item.id for item in proposals}) != len(proposals): + raise ValueError("question ids must be unique") + return proposals + + +def _proposal_from_json(item: Any, index: int) -> QuestionProposal: + if not isinstance(item, dict): + raise ValueError("question proposal must be an object") + identifier, question = item.get("id"), item.get("question") + if not isinstance(question, str) or not question: + raise ValueError("question proposal requires question") + if not isinstance(identifier, str) or not identifier: + identifier = f"proposal-{index:02d}" + queries = _string_tuple(item.get("object_queries", []), "object_queries") + hints = _string_tuple(item.get("tool_hints", []), "tool_hints") + contract = item.get("answer_contract") + if not isinstance(contract, dict): + raise ValueError("question proposal requires answer_contract") + kind = contract.get("kind") + if kind == "boolean": + answer_contract: AnswerContract = BooleanAnswerContract() + elif kind == "choice": + choices = _string_tuple(contract.get("choices"), "choices") + if len(choices) < 2: + raise ValueError("choice contract requires at least two choices") + answer_contract = ChoiceAnswerContract(choices) + else: + raise ValueError("unsupported answer contract") + return QuestionProposal(identifier, question, answer_contract, queries, hints) + + +def _string_tuple(value: Any, name: str) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str): + value = [value] + if not isinstance(value, list): + return () + return tuple(item.strip() for item in value if isinstance(item, str) and item.strip()) + + def _intents_for_query(query: str) -> list[QuestionIntent]: return [ QuestionIntent(kind="presence", object_query=query), diff --git a/dimos/benchmark/vqa/generation/questions.py b/dimos/benchmark/vqa/generation/questions.py index 08efe2c4c3..56b425e261 100644 --- a/dimos/benchmark/vqa/generation/questions.py +++ b/dimos/benchmark/vqa/generation/questions.py @@ -39,6 +39,7 @@ def generate_questions( expected_answer="yes" if nearest is not None else "no", answer_type="boolean", object_ids=(nearest.id,) if nearest is not None else (), + allowed_answers=("yes", "no"), ) ) if nearest is None: @@ -51,6 +52,7 @@ def generate_questions( expected_answer=nearest.horizontal_direction, answer_type="choice", object_ids=(nearest.id,), + allowed_answers=("left", "center", "right"), ), VqaExample( id=f"{frame_id}-{query}-range", @@ -58,6 +60,7 @@ def generate_questions( expected_answer="yes" if nearest.range_m <= distance_m else "no", answer_type="boolean", object_ids=(nearest.id,), + allowed_answers=("yes", "no"), ), ] ) diff --git a/dimos/benchmark/vqa/generation/test_agents.py b/dimos/benchmark/vqa/generation/test_agents.py index 7503c3510c..f1fbe106e6 100644 --- a/dimos/benchmark/vqa/generation/test_agents.py +++ b/dimos/benchmark/vqa/generation/test_agents.py @@ -221,7 +221,7 @@ def test_ground_truth_agent_rejects_side_comparison_without_both_sides() -> None assert result.reason == "missing_grounded_side" -def test_ground_truth_agent_overlay_includes_detector_prompts_and_question_legend( +def test_ground_truth_agent_overlay_contains_only_rectified_mask_view( tmp_path: Path, ) -> None: frame, detection = _frame_and_detection() @@ -238,5 +238,4 @@ def test_ground_truth_agent_overlay_includes_detector_prompts_and_question_legen rendered = cv2.imread(str(path)) assert rendered is not None - assert rendered.shape[0] >= frame.image.height - assert rendered.shape[1] > frame.image.width + assert rendered.shape[:2] == frame.image.data.shape[:2] diff --git a/dimos/benchmark/vqa/generation/test_dataset.py b/dimos/benchmark/vqa/generation/test_dataset.py new file mode 100644 index 0000000000..cda4830c33 --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_dataset.py @@ -0,0 +1,76 @@ +# Copyright 2026 Dimensional Inc. + +import json +from pathlib import Path + +from dimos.benchmark.vqa.generation.dataset import _evaluation_rows, write_dataset_manifest +from dimos.benchmark.vqa.models import GroundTruthResult, QuestionIntent, ToolTrace, VqaExample + + +def test_constrained_results_export_simple_multiple_choice_rows() -> None: + result = GroundTruthResult( + intent=QuestionIntent("presence", "chair"), + question=VqaExample( + "frame-chair-presence", + "Is there a chair?", + "yes", + "boolean", + (), + ("yes", "no"), + ), + status="answered", + answer="yes", + reason=None, + evidence=(), + trace=(ToolTrace("ground", "chair"),), + ) + + cases, labels = _evaluation_rows("frame", [result]) + + assert cases == [ + { + "id": "frame-chair-presence", + "image": "image.jpg", + "question": "Is there a chair?", + "choices": ("yes", "no"), + } + ] + assert labels == [{"id": "frame-chair-presence", "answer": "yes"}] + + +def test_dataset_manifest_exports_public_cases_and_private_labels(tmp_path: Path) -> None: + frame = tmp_path / "frame-000040" + frame.mkdir() + (frame / "frame.json").write_text( + json.dumps( + { + "frame_id": "frame-40", + "accepted_question_count": 1, + "rejected_question_count": 2, + } + ) + ) + (frame / "cases.json").write_text( + json.dumps( + [ + { + "id": "case-1", + "image": "image.jpg", + "question": "Is it visible?", + "choices": ["yes", "no"], + } + ] + ) + ) + (frame / "labels.json").write_text(json.dumps([{"id": "case-1", "answer": "yes"}])) + + summary = write_dataset_manifest(tmp_path) + + assert summary == {"frame_count": 1, "accepted_question_count": 1, "rejected_question_count": 2} + assert json.loads((tmp_path / "cases.jsonl").read_text()) == { + "id": "case-1", + "image": "frame-000040/image.jpg", + "question": "Is it visible?", + "choices": ["yes", "no"], + } + assert json.loads((tmp_path / "labels.jsonl").read_text()) == {"id": "case-1", "answer": "yes"} diff --git a/dimos/benchmark/vqa/generation/test_oracle.py b/dimos/benchmark/vqa/generation/test_oracle.py new file mode 100644 index 0000000000..143f35a554 --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_oracle.py @@ -0,0 +1,360 @@ +# Copyright 2026 Dimensional Inc. + +from __future__ import annotations + +import json +from typing import Any, cast + +from langchain_core.messages import AIMessage +import numpy as np + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.generation.measurements import estimate_ground_plane +from dimos.benchmark.vqa.generation.oracle import ( + PrivateToolCallingOracle, + SemanticEvidenceValidation, + validate_oracle_answer, +) +from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.generation.question_agent import ( + AGENTIC_QUESTION_PROMPT, + OpenAIFreeformQuestionAuthor, +) +from dimos.benchmark.vqa.models import ( + BooleanAnswerContract, + CalibratedFrame, + ChoiceAnswerContract, + OracleEvidence, + OracleToolResult, + QuestionProposal, +) +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 + + +class _QuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "answer_contract" in prompt + return '[{"id":"chair-presence","question":"Is there a chair?","answer_contract":{"kind":"boolean"},"object_queries":["chair"]}]' + + +class _Grounding: + def ground(self, frame: Any, query: str) -> tuple[list[Any], tuple[Any, ...]]: + return [ + type( + "Object", + (), + { + "id": "chair-1", + "label": query, + "range_m": 1.0, + "horizontal_direction": "left", + "point_count": 4, + }, + )() + ], () + + +class _GroundingWithMask(_Grounding): + def __init__(self, mask: np.ndarray) -> None: + self._mask = mask + + def masks_for_query(self, query: str) -> tuple[Any, ...]: + return (type("Mask", (), {"mask": self._mask})(),) + + +def _measurement_frame(points: np.ndarray | None = None) -> CalibratedFrame: + ground = [[x, 1.0, z] for z in (3.0, 4.0, 5.0) for x in (-1.0, -0.5, 0.0, 0.5, 1.0)] + object_points = [[2.0, 1.0 - height, 4.0] for height in np.linspace(0.2, 1.0, 9)] + image = Image.from_numpy(np.zeros((100, 100, 3), dtype=np.uint8)) + return CalibratedFrame( + id="synthetic", + image=image, + pointcloud=PointCloud2.from_numpy( + points + if points is not None + else np.asarray([*ground, *object_points], dtype=np.float32) + ), + camera_info=CameraInfo.from_intrinsics(50.0, 50.0, 50.0, 50.0, 100, 100), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + + +class _BoundModel: + def __init__(self) -> None: + self._calls = 0 + + def bind_tools(self, tools: Any) -> _BoundModel: + return self + + def invoke(self, messages: Any) -> AIMessage: + self._calls += 1 + if self._calls == 1: + return AIMessage( + content="", + tool_calls=[ + {"name": "ground_semantic_object", "args": {"query": "chair"}, "id": "call-1"} + ], + ) + return AIMessage(content='{"answer":"yes","evidence_ids":["grounding:v1:chair-1"]}') + + +class _SemanticValidator: + def __init__(self, verdict: SemanticEvidenceValidation) -> None: + self.verdict = verdict + self.calls: list[tuple[QuestionProposal, str | float, tuple[OracleToolResult, ...]]] = [] + + def validate( + self, + proposal: QuestionProposal, + answer: str | float, + cited_results: tuple[OracleToolResult, ...], + ) -> SemanticEvidenceValidation: + self.calls.append((proposal, answer, cited_results)) + return self.verdict + + +def test_freeform_question_author_parses_public_contract() -> None: + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposals = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _QuestionModel())).propose(image) + + assert proposals == [ + QuestionProposal( + "chair-presence", "Is there a chair?", BooleanAnswerContract(), ("chair",), () + ) + ] + + +def test_freeform_author_prompt_prioritizes_geometric_questions() -> None: + assert "measure_object_height" in AGENTIC_QUESTION_PROMPT + assert "two named objects is closer" in AGENTIC_QUESTION_PROMPT + assert "closer (choice" in AGENTIC_QUESTION_PROMPT + assert "Use visibility/presence questions only" in AGENTIC_QUESTION_PROMPT + + +def test_freeform_question_author_assigns_missing_ids() -> None: + class _ModelWithoutId: + def query(self, image: Image, prompt: str) -> str: + return '[{"question":"Is there a chair?","answer_contract":{"kind":"boolean"}}]' + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposals = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _ModelWithoutId())).propose( + image + ) + + assert proposals[0].id == "proposal-01" + + +def test_freeform_question_author_rejects_numeric_contracts() -> None: + class _NumericContractModel: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"numeric","unit":"m","tolerance":0.1}}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + try: + OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _NumericContractModel())).propose(image) + except ValueError as exc: + assert "unsupported answer contract" in str(exc) + else: + raise AssertionError("numeric answer contract was accepted") + + +def test_freeform_question_author_normalizes_optional_query_hints() -> None: + class _ModelWithStringHints: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"choice","choices":["under 0.5 m","0.5-1.0 m"]},' + '"object_queries":"chair","tool_hints":null}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposal = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _ModelWithStringHints())).propose( + image + )[0] + + assert proposal.object_queries == ("chair",) + assert proposal.tool_hints == () + + +def test_freeform_question_author_ignores_malformed_optional_query_hints() -> None: + class _ModelWithMalformedHints: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"choice","choices":["under 0.5 m","0.5-1.0 m"]},' + '"object_queries":{"query":"chair"}}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposal = OpenAIFreeformQuestionAuthor( + cast("OpenAIVlModel", _ModelWithMalformedHints()) + ).propose(image)[0] + + assert proposal.object_queries == () + + +def test_local_tool_returns_geometry_and_evidence_ids() -> None: + registry = LocalOracleToolRegistry(cast("Any", object()), cast("Any", _Grounding())) + + payload = json.loads(registry.ground_semantic_object("chair")) + + assert payload["objects"][0] == { + "evidence_id": "grounding:v1:chair-1", + "id": "chair-1", + "label": "chair", + "range_m": 1.0, + "side": "left", + "point_count": 4, + } + assert registry.results[0].version == "v1" + + +def test_ground_plane_estimator_fits_visible_lower_band() -> None: + fit = estimate_ground_plane(_measurement_frame()) + + assert fit.rejection_reason is None + assert fit.estimate is not None + assert fit.estimate.inlier_count >= 12 + assert fit.estimate.residual_m < 0.001 + assert np.allclose(fit.estimate.normal, (0.0, -1.0, 0.0), atol=0.001) + assert fit.estimate.offset_m == 1.0 + + +def test_ground_plane_tool_returns_quality_gated_rejection() -> None: + frame = _measurement_frame(np.asarray([[0.0, 1.0, 3.0]], dtype=np.float32)) + registry = LocalOracleToolRegistry(frame, cast("Any", _Grounding())) + + payload = json.loads(registry.estimate_ground_plane()) + + assert payload["measurement"] is None + assert payload["rejection_reason"] == "insufficient_support" + assert "insufficient_ground_band_points" in payload["quality_flags"] + + +def test_height_tool_measures_visible_object_points_above_plane() -> None: + frame = _measurement_frame() + mask = np.zeros((100, 100), dtype=np.uint8) + projected = project_visible_points(frame) + for (x, y), point in zip(projected.pixels, projected.camera_points, strict=True): + if point[0] > 1.5: + mask[y, x] = 255 + registry = LocalOracleToolRegistry(frame, cast("Any", _GroundingWithMask(mask))) + + payload = json.loads(registry.measure_object_height("chair")) + + assert payload["measurement"]["unit"] == "m" + assert 0.8 < payload["measurement"]["value"] < 1.0 + assert payload["measurement"]["tolerance"] >= 0.05 + assert payload["objects"][0]["evidence_id"] == "height:v1:chair-1" + assert "visible_point_cloud_height" in payload["quality_flags"] + + +def test_local_registry_exposes_geometry_tools() -> None: + registry = LocalOracleToolRegistry(cast("Any", object()), cast("Any", _Grounding())) + + assert {tool.name for tool in registry.tools()} == { + "ground_semantic_object", + "estimate_ground_plane", + "measure_object_height", + "measure_object_height_bucket", + } + + +def test_oracle_validates_evidence_and_answer_contract() -> None: + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract()) + result = OracleToolResult( + "ground", "chair", (OracleEvidence("e1", "v1", "o1", "chair", 1.0, "left", 3),) + ) + + assert validate_oracle_answer(proposal, "yes", ["e1"], (result,)) == "yes" + try: + validate_oracle_answer(proposal, "maybe", ["e1"], (result,)) + except ValueError as exc: + assert "boolean" in str(exc) + else: + raise AssertionError("invalid boolean answer was accepted") + try: + validate_oracle_answer(proposal, "yes", ["unknown"], (result,)) + except ValueError as exc: + assert "unknown evidence" in str(exc) + else: + raise AssertionError("unknown evidence was accepted") + + +def test_private_oracle_runs_direct_structured_tool() -> None: + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) + registry = LocalOracleToolRegistry(cast("Any", object()), cast("Any", _Grounding())) + validator = _SemanticValidator(SemanticEvidenceValidation(True, "chair grounding supports yes")) + + result = PrivateToolCallingOracle( + cast("Any", _BoundModel()), semantic_validator=validator + ).answer(proposal, registry) + + assert result.answer == "yes" + assert result.evidence_ids == ("grounding:v1:chair-1",) + assert validator.calls[0][2][0].evidence[0].id == "grounding:v1:chair-1" + assert result.trace[-1].detail == "accepted:chair grounding supports yes" + + +def test_private_oracle_rejects_unsupported_measurement_claim() -> None: + class _ChoiceModel(_BoundModel): + def invoke(self, messages: Any) -> AIMessage: + self._calls += 1 + if self._calls == 1: + return AIMessage( + content="", + tool_calls=[ + { + "name": "ground_semantic_object", + "args": {"query": "chair"}, + "id": "call-1", + } + ], + ) + return AIMessage( + content='{"answer":"0.5-1.0 m","evidence_ids":["grounding:v1:chair-1"]}' + ) + + proposal = QuestionProposal( + "q", "How tall is the chair?", ChoiceAnswerContract(("under 0.5 m", "0.5-1.0 m")) + ) + registry = LocalOracleToolRegistry(cast("Any", object()), cast("Any", _Grounding())) + validator = _SemanticValidator( + SemanticEvidenceValidation(False, "range and side do not measure height") + ) + + result = PrivateToolCallingOracle( + cast("Any", _ChoiceModel()), semantic_validator=validator + ).answer(proposal, registry) + + assert result.reason == "unsupported_evidence:range and side do not measure height" + assert result.trace[-1].detail == "rejected:range and side do not measure height" + + +def test_agentic_oracle_never_uses_legacy_answer_program() -> None: + class _GroundingWithoutLegacyAnswer(_Grounding): + def answer(self, frame: Any, intent: Any) -> None: + raise AssertionError("agentic oracle must not call legacy answer") + + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) + registry = LocalOracleToolRegistry(cast("Any", object()), _GroundingWithoutLegacyAnswer()) + validator = _SemanticValidator(SemanticEvidenceValidation(True, "chair grounding supports yes")) + + result = PrivateToolCallingOracle( + cast("Any", _BoundModel()), semantic_validator=validator + ).answer(proposal, registry) + + assert result.answer == "yes" diff --git a/dimos/benchmark/vqa/generation/test_single_frame.py b/dimos/benchmark/vqa/generation/test_single_frame.py index 3865913384..be6ef78c29 100644 --- a/dimos/benchmark/vqa/generation/test_single_frame.py +++ b/dimos/benchmark/vqa/generation/test_single_frame.py @@ -16,7 +16,6 @@ import numpy as np -from dimos.benchmark.vqa.evaluation.scoring import evaluate_examples from dimos.benchmark.vqa.generation.pipeline import generate_ground_truth from dimos.benchmark.vqa.models import CalibratedFrame from dimos.msgs.geometry_msgs.Transform import Transform @@ -27,15 +26,6 @@ from dimos.perception.detection.type.detection2d.seg import Detection2DSeg -class _Answerer: - def __init__(self) -> None: - self.calls: list[tuple[Image, str]] = [] - - def answer(self, image: Image, question: str) -> str: - self.calls.append((image, question)) - return "Yes." - - class _Detector: def __init__(self, image: Image, detection: Detection2DSeg) -> None: self._image = image @@ -51,7 +41,7 @@ def segment(self, detections: ImageDetections2D) -> ImageDetections2D: return detections -def test_single_frame_ground_truth_and_image_only_evaluation() -> None: +def test_single_frame_ground_truth_generates_multiple_choice_cases() -> None: image = Image.from_numpy(np.zeros((6, 6, 3), dtype=np.uint8)) frame = CalibratedFrame( id="frame-1", @@ -69,10 +59,6 @@ def test_single_frame_ground_truth_and_image_only_evaluation() -> None: examples = generate_ground_truth( frame, ["chair", "table"], _Detector(image, detection), _Segmenter() ) - answerer = _Answerer() - evaluations = evaluate_examples(frame.image, examples, answerer) assert {example.expected_answer for example in examples} == {"yes", "no", "center"} - assert all(image_arg is image for image_arg, _ in answerer.calls) - assert evaluations[0].passed - assert not evaluations[1].passed + assert all(example.expected_answer in example.allowed_answers for example in examples) diff --git a/dimos/benchmark/vqa/models.py b/dimos/benchmark/vqa/models.py index 4e4e5d2133..f66150d8c8 100644 --- a/dimos/benchmark/vqa/models.py +++ b/dimos/benchmark/vqa/models.py @@ -83,6 +83,7 @@ class VqaExample: expected_answer: str answer_type: str object_ids: tuple[str, ...] + allowed_answers: tuple[str, ...] = () QuestionKind = Literal[ @@ -120,6 +121,115 @@ class GroundTruthResult: trace: tuple[ToolTrace, ...] +@dataclass(frozen=True) +class BooleanAnswerContract: + """A yes/no answer required from a private oracle.""" + + kind: Literal["boolean"] = "boolean" + + +@dataclass(frozen=True) +class ChoiceAnswerContract: + """An answer selected exactly from the supplied choices.""" + + choices: tuple[str, ...] + kind: Literal["choice"] = "choice" + + +AnswerContract = BooleanAnswerContract | ChoiceAnswerContract + + +@dataclass(frozen=True) +class QuestionProposal: + """Public image-only question frozen before private oracle execution.""" + + id: str + question: str + answer_contract: AnswerContract + object_queries: tuple[str, ...] = () + tool_hints: tuple[str, ...] = () + + +@dataclass(frozen=True) +class OracleEvidence: + """A private evidence item emitted by a registered local tool.""" + + id: str + version: str + object_id: str + label: str + range_m: float + side: str + point_count: int + measurement: OracleMeasurement | None = None + + +@dataclass(frozen=True) +class OracleMeasurement: + """A private scalar measurement with its uncertainty and provenance.""" + + value: float + unit: str + tolerance: float + quality_flags: tuple[str, ...] + provenance_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class GroundPlaneEstimate: + """A robust ground-plane fit in frozen camera coordinates.""" + + normal: tuple[float, float, float] + offset_m: float + sample_count: int + inlier_count: int + residual_m: float + + +@dataclass(frozen=True) +class OracleToolResult: + """Structured result and evidence IDs from one local tool invocation.""" + + tool: str + query: str + evidence: tuple[OracleEvidence, ...] + version: str = "v1" + measurement: OracleMeasurement | None = None + choice: str | None = None + plane: GroundPlaneEstimate | None = None + quality_flags: tuple[str, ...] = () + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class OracleTrace: + """Audit record for a private oracle model or tool operation.""" + + operation: str + detail: str + + +@dataclass(frozen=True) +class AcceptedOracleResult: + """Validated private answer for a frozen public proposal.""" + + proposal: QuestionProposal + answer: str + evidence_ids: tuple[str, ...] + tool_results: tuple[OracleToolResult, ...] + trace: tuple[OracleTrace, ...] + + +@dataclass(frozen=True) +class RejectedOracleResult: + """Private oracle attempt that cannot be safely exported as a case.""" + + proposal: QuestionProposal + reason: str + tool_results: tuple[OracleToolResult, ...] + trace: tuple[OracleTrace, ...] + + class ObjectDetector(Protocol): """Produces object-query detections from one image.""" diff --git a/dimos/benchmark/vqa/test_evaluation.py b/dimos/benchmark/vqa/test_evaluation.py new file mode 100644 index 0000000000..2fab9b85c8 --- /dev/null +++ b/dimos/benchmark/vqa/test_evaluation.py @@ -0,0 +1,47 @@ +# Copyright 2026 Dimensional Inc. + +import json +from pathlib import Path +from typing import Any, cast + +import cv2 +import numpy as np + +from dimos.benchmark.evaluation.models import ArtifactNativeResult +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.benchmark.vqa.evaluation import MultipleChoiceVqaEvaluation, VqaEvaluationConfig + + +class _VisionModel: + def query(self, image: Any, prompt: str) -> str: + assert "Choices: left, right." in prompt + return "ANSWER: left" + + +def test_vqa_evaluation_uses_only_public_case_image_and_private_label(tmp_path: Path) -> None: + dataset = tmp_path / "dataset" + dataset.mkdir() + assert cv2.imwrite(str(dataset / "image.jpg"), np.zeros((1, 1, 3), dtype=np.uint8)) + (dataset / "cases.jsonl").write_text( + json.dumps( + { + "id": "case-1", + "image": "image.jpg", + "question": "Which side?", + "choices": ["left", "right"], + } + ) + + "\n" + ) + (dataset / "labels.jsonl").write_text(json.dumps({"id": "case-1", "answer": "left"}) + "\n") + workspace = tmp_path / "workspace" + workspace.mkdir() + + report = MultipleChoiceVqaEvaluation(lambda _: cast("Any", _VisionModel())).run( + VqaEvaluationConfig(dataset=str(dataset)), + EvaluationContext("run", tmp_path, workspace, cast("Any", None), None), + ) + + assert report.summary[1].value == 1 + assert isinstance(report.native_result, ArtifactNativeResult) + assert json.loads((workspace / "vqa-results.json").read_text())[0]["passed"] is True diff --git a/dimos/cli/vqa.py b/dimos/cli/vqa.py index a20621b4f0..7c652067a1 100644 --- a/dimos/cli/vqa.py +++ b/dimos/cli/vqa.py @@ -3,11 +3,10 @@ from __future__ import annotations -from dataclasses import asdict -import json +import os from pathlib import Path +from typing import Any, cast -import cv2 import typer from dimos.benchmark.vqa.generation.adapters import ( @@ -16,10 +15,24 @@ ) from dimos.benchmark.vqa.generation.dataset import write_dataset_manifest, write_frame_record from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator -from dimos.benchmark.vqa.generation.question_agent import OpenAIQuestionAgent +from dimos.benchmark.vqa.generation.oracle import create_openai_oracle +from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.generation.question_agent import ( + OpenAIFreeformQuestionAuthor, + OpenAIQuestionAgent, +) from dimos.benchmark.vqa.generation.recording import load_go2_frame -from dimos.benchmark.vqa.models import GroundingConfig, QuestionIntent +from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + CalibratedFrame, + GroundingConfig, + GroundTruthResult, + QuestionIntent, + QuestionProposal, + RejectedOracleResult, +) from dimos.constants import STATE_DIR +from dimos.models.base import default_local_model_device from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter from dimos.models.vl.moondream import MoondreamVlModel from dimos.models.vl.openai import OpenAIVlModel @@ -34,28 +47,42 @@ def single_frame( frame_index: int = typer.Option(0, "--frame-index"), query: list[str] = typer.Option([], "--query"), propose_questions: bool = typer.Option(False, "--propose-questions"), + question_mode: str = typer.Option("constrained", "--question-mode"), question_model: str = typer.Option("gpt-4o-mini", "--question-model"), + oracle_model: str = typer.Option("gpt-4o-mini", "--oracle-model"), min_mask_area_px: int = typer.Option(128, "--min-mask-area-px"), min_foreground_points: int = typer.Option(3, "--min-foreground-points"), output: Path | None = typer.Option(None, "--output"), ) -> None: - """Generate and evaluate questions for one Go2 recording frame.""" + """Generate private-grounded questions for one Go2 recording frame.""" output = output or ( STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frame-{frame_index:06d}" ) - if output.exists() or (not query and not propose_questions): + if output.exists(): raise typer.BadParameter("output must not already exist") + _validate_question_mode(question_mode) + uses_image_author = _uses_image_question_author(question_mode, query, propose_questions) + _require_openai_for_question_author(uses_image_author) + _require_edgetam_cuda() + typer.echo(f"Loading frame {frame_index} from {recording}") frame = load_go2_frame(str(resolve_named_path(recording, ".db")), frame_index) model = MoondreamVlModel() + typer.echo("Loading private MoonDream model") model.start() question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=question_model)) try: - intents = ( - question_agent.propose(frame.image) - if propose_questions + if uses_image_author: + typer.echo(f"Proposing questions with {question_model}") + intents: list[QuestionIntent] | list[QuestionProposal] = ( + OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=question_model)).propose( + frame.image + ) + if question_mode == "agentic" + else question_agent.propose(frame.image) + if uses_image_author else [ QuestionIntent( - kind=kind, + kind=cast("Any", kind), object_query=item, threshold_m=3.0 if kind == "within_distance" else None, ) @@ -68,6 +95,7 @@ def single_frame( ) ] ) + typer.echo(f"Grounding {len(intents)} questions for frame {frame_index}") ground_truth = VqaGroundTruthGenerator( detector := MoondreamObjectDetector(model), segmenter := EdgeTamObjectSegmenter(EdgeTAMImageSegmenter()), @@ -78,55 +106,44 @@ def single_frame( min_foreground_points=min_foreground_points, ), ) - results = [ground_truth.answer(frame, intent) for intent in intents] - examples = [result.question for result in results if result.status == "answered"] + results: list[GroundTruthResult] | list[AcceptedOracleResult | RejectedOracleResult] = ( + _answer_agentic( + ground_truth, frame, cast("list[QuestionProposal]", intents), oracle_model + ) + if question_mode == "agentic" + else _answer_intents( + ground_truth, frame, cast("list[QuestionIntent]", intents), f"Frame {frame_index}" + ) + ) + examples = [ + result + for result in results + if isinstance(result, AcceptedOracleResult) + or (isinstance(result, GroundTruthResult) and result.status == "answered") + ] finally: model.stop() - output.mkdir(parents=True) - image_path = output / "image.jpg" - if not cv2.imwrite(str(image_path), frame.image.data): - raise RuntimeError(f"failed to write {image_path}") - original_image_path = output / "original_image.jpg" - if frame.original_image is not None and not cv2.imwrite( - str(original_image_path), frame.original_image.data - ): - raise RuntimeError(f"failed to write {original_image_path}") - overlay_path = output / "grounding_overlay.jpg" - ground_truth.write_overlay(frame, str(overlay_path)) - (output / "frame.json").write_text( - json.dumps( - { - "schema_version": "1.0", - "frame_id": frame.id, - "recording": recording, - "frame_index": frame_index, - "image": image_path.name, - "original_image": original_image_path.name - if frame.original_image is not None - else None, - "grounding_overlay": overlay_path.name, - "question_count": len(intents), - "accepted_question_count": len(examples), - "rejected_question_count": len(results) - len(examples), - "question_source": "image_agent" if propose_questions else "explicit_queries", - "question_model": question_model if propose_questions else None, - "grounding": { - "min_mask_area_px": min_mask_area_px, - "min_foreground_points": min_foreground_points, - }, + write_frame_record( + output, + frame, + recording, + frame_index, + cast("list[QuestionIntent | QuestionProposal]", intents), + cast("list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult]", results), + ground_truth, + { + "question_source": "agentic_image_author" + if question_mode == "agentic" + else "openai_image_agent" + if uses_image_author + else "explicit_queries", + "question_model": question_model if uses_image_author else None, + "oracle_model": oracle_model if question_mode == "agentic" else None, + "grounding": { + "min_mask_area_px": min_mask_area_px, + "min_foreground_points": min_foreground_points, }, - indent=2, - ) - + "\n" - ) - (output / "intents.json").write_text( - json.dumps([asdict(item) for item in intents], indent=2) + "\n" - ) - (output / "examples.json").write_text( - json.dumps([asdict(item) for item in examples], indent=2) + "\n" - ) - (output / "ground_truth.json").write_text( - json.dumps([asdict(item) for item in results], indent=2) + "\n" + }, ) typer.echo(f"Wrote {len(examples)} examples to {output}") @@ -139,40 +156,54 @@ def generate( stride: int = typer.Option(1, "--stride"), query: list[str] = typer.Option([], "--query"), propose_questions: bool = typer.Option(False, "--propose-questions"), + question_mode: str = typer.Option("constrained", "--question-mode"), question_model: str = typer.Option("gpt-4o-mini", "--question-model"), + oracle_model: str = typer.Option("gpt-4o-mini", "--oracle-model"), min_mask_area_px: int = typer.Option(128, "--min-mask-area-px"), min_foreground_points: int = typer.Option(3, "--min-foreground-points"), output: Path | None = typer.Option(None, "--output"), ) -> None: """Generate a resumable VQA dataset from sampled Go2 recording frames.""" - if ( - start_index < 0 - or stop_index <= start_index - or stride < 1 - or (not query and not propose_questions) - ): - raise typer.BadParameter( - "provide valid frame bounds and either --query or --propose-questions" - ) + if start_index < 0 or stop_index <= start_index or stride < 1: + raise typer.BadParameter("provide valid frame bounds") output = output or (STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frames") + _validate_question_mode(question_mode) + uses_image_author = _uses_image_question_author(question_mode, query, propose_questions) + _require_openai_for_question_author(uses_image_author) output.mkdir(parents=True, exist_ok=True) + _require_edgetam_cuda() + frame_indices = range(start_index, stop_index, stride) + typer.echo(f"Generating {len(frame_indices)} sampled frames from {recording} into {output}") model = MoondreamVlModel() + typer.echo("Loading private MoonDream model") model.start() try: detector = MoondreamObjectDetector(model) segmenter = EdgeTamObjectSegmenter(EdgeTAMImageSegmenter()) question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=question_model)) - for frame_index in range(start_index, stop_index, stride): + for frame_number, frame_index in enumerate(frame_indices, start=1): frame_output = output / f"frame-{frame_index:06d}" if (frame_output / "frame.json").is_file(): + typer.echo( + f"Skipping completed frame {frame_number}/{len(frame_indices)}: {frame_index}" + ) continue + typer.echo( + f"Frame {frame_number}/{len(frame_indices)}: loading recording index {frame_index}" + ) frame = load_go2_frame(str(resolve_named_path(recording, ".db")), frame_index) - intents = ( - question_agent.propose(frame.image) - if propose_questions + if uses_image_author: + typer.echo(f"Frame {frame_index}: proposing questions with {question_model}") + intents: list[QuestionIntent] | list[QuestionProposal] = ( + OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=question_model)).propose( + frame.image + ) + if question_mode == "agentic" + else question_agent.propose(frame.image) + if uses_image_author else [ QuestionIntent( - kind=kind, + kind=cast("Any", kind), object_query=item, threshold_m=3.0 if kind == "within_distance" else None, ) @@ -185,6 +216,7 @@ def generate( ) ] ) + typer.echo(f"Frame {frame_index}: grounding {len(intents)} questions") ground_truth = VqaGroundTruthGenerator( detector, segmenter, @@ -194,20 +226,36 @@ def generate( min_mask_area_px=min_mask_area_px, min_foreground_points=min_foreground_points ), ) - results = [ground_truth.answer(frame, intent) for intent in intents] + results: list[GroundTruthResult] | list[AcceptedOracleResult | RejectedOracleResult] = ( + _answer_agentic( + ground_truth, frame, cast("list[QuestionProposal]", intents), oracle_model + ) + if question_mode == "agentic" + else _answer_intents( + ground_truth, + frame, + cast("list[QuestionIntent]", intents), + f"Frame {frame_index}", + ) + ) write_frame_record( frame_output, frame, recording, frame_index, - intents, - results, + cast("list[QuestionIntent | QuestionProposal]", intents), + cast( + "list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult]", results + ), ground_truth, { - "question_source": "openai_image_agent" - if propose_questions + "question_source": "agentic_image_author" + if question_mode == "agentic" + else "openai_image_agent" + if uses_image_author else "explicit_queries", - "question_model": question_model if propose_questions else None, + "question_model": question_model if uses_image_author else None, + "oracle_model": oracle_model if question_mode == "agentic" else None, "grounding": { "min_mask_area_px": min_mask_area_px, "min_foreground_points": min_foreground_points, @@ -219,3 +267,64 @@ def generate( model.stop() summary = write_dataset_manifest(output) typer.echo(f"Dataset manifest: {summary}") + + +def _answer_intents( + ground_truth: VqaGroundTruthGenerator, + frame: CalibratedFrame, + intents: list[QuestionIntent], + label: str, +) -> list[GroundTruthResult]: + results: list[GroundTruthResult] = [] + for number, intent in enumerate(intents, start=1): + typer.echo( + f"{label}: grounding question {number}/{len(intents)}: " + f"{intent.kind} ({intent.object_query})" + ) + result = ground_truth.answer(frame, intent) + results.append(result) + typer.echo(f"{label}: question {number}/{len(intents)} {result.status}") + return results + + +def _answer_agentic( + ground_truth: VqaGroundTruthGenerator, + frame: CalibratedFrame, + proposals: list[QuestionProposal], + oracle_model: str, +) -> list[AcceptedOracleResult | RejectedOracleResult]: + oracle = create_openai_oracle(oracle_model) + results: list[AcceptedOracleResult | RejectedOracleResult] = [] + for number, proposal in enumerate(proposals, start=1): + typer.echo(f"Agentic question {number}/{len(proposals)}: {proposal.question}") + result = oracle.answer(proposal, LocalOracleToolRegistry(frame, ground_truth)) + results.append(result) + if isinstance(result, AcceptedOracleResult): + typer.echo(f"Agentic question {number}/{len(proposals)} accepted") + else: + typer.echo(f"Agentic question {number}/{len(proposals)} rejected: {result.reason}") + return results + + +def _validate_question_mode(question_mode: str) -> None: + if question_mode not in ("constrained", "agentic"): + raise typer.BadParameter("question mode must be constrained or agentic") + + +def _require_openai_for_question_author(uses_image_author: bool) -> None: + if uses_image_author and not os.environ.get("OPENAI_API_KEY"): + raise typer.BadParameter("OPENAI_API_KEY must be set for image-authored question modes") + + +def _uses_image_question_author( + question_mode: str, query: list[str], propose_questions: bool +) -> bool: + """Choose authored questions unless constrained mode has explicit queries.""" + return question_mode == "agentic" or propose_questions or not query + + +def _require_edgetam_cuda() -> None: + if default_local_model_device() != "cuda": + raise typer.BadParameter( + "VQA generation requires an installed PyTorch CUDA build that supports this GPU for EdgeTAM" + ) diff --git a/dimos/models/base.py b/dimos/models/base.py index 65393405c5..0fd8291366 100644 --- a/dimos/models/base.py +++ b/dimos/models/base.py @@ -28,8 +28,20 @@ DeviceType = Annotated[str, "Device identifier (e.g., 'cuda', 'cpu', 'cuda:0')"] +def default_local_model_device() -> str: + """Select CUDA only when this Torch build supports the detected GPU.""" + if not torch.cuda.is_available(): + return "cpu" + try: + major, minor = torch.cuda.get_device_capability() + capability = f"sm_{major}{minor}" + return "cuda" if capability in torch.cuda.get_arch_list() else "cpu" + except RuntimeError: + return "cpu" + + class LocalModelConfig(BaseConfig): - device: DeviceType = "cuda" if torch.cuda.is_available() else "cpu" + device: DeviceType = default_local_model_device() dtype: torch.dtype = torch.float32 warmup: bool = False autostart: bool = False diff --git a/docs/docs.json b/docs/docs.json index 9b4246f2be..6c1374be4a 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -124,7 +124,8 @@ { "group": "Perception", "pages": [ - "capabilities/perception/index" + "capabilities/perception/index", + "vqa-benchmark" ] }, { diff --git a/docs/vqa-benchmark.md b/docs/vqa-benchmark.md new file mode 100644 index 0000000000..8960f54d06 --- /dev/null +++ b/docs/vqa-benchmark.md @@ -0,0 +1,79 @@ +--- +title: "VQA Benchmark" +--- + +# Point-Cloud-Grounded VQA Benchmark + +DimOS generates image-question-multiple-choice VQA cases from frozen robot recordings. Private point-cloud tools establish and validate answers during generation; evaluation sees only public images, questions, choices, and private answer labels. + +## Pipeline Flow + +```text +Frozen recording -> rectified RGB image + calibrated visible point cloud + | + +-- constrained: image object author -> deterministic question families + | -> private grounding -> quality-gate validation + | + +-- agentic: image question author -> frozen choice question + -> private local oracle tools -> validation + | + v +accepted cases.jsonl + private labels.jsonl + | + v +point-cloud-vqa Evaluation -> image-only vision model -> exact choice scoring +``` + +Rejected questions and private evidence remain in each frame's generation record. The evaluator does not load point clouds, calibration, tool traces, overlays, or rejection records. + +## Generation Modes + +Constrained generation expands visible objects into fixed choice-question families: presence (`yes`/`no`), horizontal direction, distance threshold, and nearest left-versus-right comparison. Additional deterministic question families will be added as their evidence programs mature. + +Agentic generation freezes a free-form image-authored question, then a private oracle uses read-only local tools to establish its answer. Height questions use the fixed choices `under 0.5 m`, `0.5-1.0 m`, `1.0-1.5 m`, and `over 1.5 m`; the private tool measures height and maps it deterministically to one choice. Additional oracle tools will be added as new evidence capabilities mature. + +Geometry quality gates are the private validation step. They reject insufficient point support, ambiguous masks, unreliable ground planes, and incomplete height evidence before a case becomes public. + +## Dataset Format + +The root evaluation export follows the common image-question-choice benchmark pattern: + +```json +{"id":"go2-40-chair-height","image":"frame-000040/image.jpg","question":"How tall is the chair?","choices":["under 0.5 m","0.5-1.0 m","1.0-1.5 m","over 1.5 m"]} +``` + +`cases.jsonl` contains public rows only. `labels.jsonl` contains the matching private `id` and `answer`. Frame directories retain `ground_truth.json`, which includes the full private generation audit data. + +## Generate + +```bash +OPENAI_API_KEY="$OPENAI_API_KEY" dimos vqa generate \ + --recording go2_short \ + --start-index 0 --stop-index 100 --stride 20 \ + --question-mode constrained \ + --output ~/.local/state/dimos/datasets/vqa/go2-short +``` + +## Evaluate + +Create an Evaluation Run Specification next to the generated dataset: + +```json +{ + "evaluation": { + "name": "point-cloud-vqa", + "config": { + "dataset": "./go2-short", + "model": "gpt-4o-mini" + } + } +} +``` + +Then run the shared evaluator: + +```bash +OPENAI_API_KEY="$OPENAI_API_KEY" dimos eval run vqa-run.json --output /tmp/vqa-evaluation +``` + +The evaluation writes the shared immutable `run.json` and a VQA-native `vqa-results.json` artifact containing each model response, normalized answer, expected answer, and pass/fail result. From e53e4d1019334771ec6fca2737ea4e9c969dc5e8 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Mon, 10 Aug 2026 10:07:14 -0700 Subject: [PATCH 05/12] refactor(cli): simplify VQA generation options --- dimos/cli/test_vqa.py | 9 ++++++++ dimos/cli/vqa.py | 49 +++++++++++++++++-------------------------- docs/vqa-benchmark.md | 2 ++ 3 files changed, 30 insertions(+), 30 deletions(-) create mode 100644 dimos/cli/test_vqa.py diff --git a/dimos/cli/test_vqa.py b/dimos/cli/test_vqa.py new file mode 100644 index 0000000000..cc94510983 --- /dev/null +++ b/dimos/cli/test_vqa.py @@ -0,0 +1,9 @@ +# Copyright 2026 Dimensional Inc. + +from dimos.cli import vqa + + +def test_question_mode_selects_the_image_author_without_an_explicit_query() -> None: + assert vqa._uses_image_question_author("constrained", []) is True + assert vqa._uses_image_question_author("constrained", ["chair"]) is False + assert vqa._uses_image_question_author("agentic", ["chair"]) is True diff --git a/dimos/cli/vqa.py b/dimos/cli/vqa.py index 7c652067a1..e2a00ed31f 100644 --- a/dimos/cli/vqa.py +++ b/dimos/cli/vqa.py @@ -39,6 +39,8 @@ from dimos.utils.data import resolve_named_path app = typer.Typer(help="Generate point-cloud-grounded VQA benchmark examples") +QUESTION_MODEL = "gpt-4o-mini" +ORACLE_MODEL = "gpt-4o-mini" @app.command("single-frame") @@ -46,10 +48,7 @@ def single_frame( recording: str = typer.Option(..., "--recording"), frame_index: int = typer.Option(0, "--frame-index"), query: list[str] = typer.Option([], "--query"), - propose_questions: bool = typer.Option(False, "--propose-questions"), question_mode: str = typer.Option("constrained", "--question-mode"), - question_model: str = typer.Option("gpt-4o-mini", "--question-model"), - oracle_model: str = typer.Option("gpt-4o-mini", "--oracle-model"), min_mask_area_px: int = typer.Option(128, "--min-mask-area-px"), min_foreground_points: int = typer.Option(3, "--min-foreground-points"), output: Path | None = typer.Option(None, "--output"), @@ -61,7 +60,7 @@ def single_frame( if output.exists(): raise typer.BadParameter("output must not already exist") _validate_question_mode(question_mode) - uses_image_author = _uses_image_question_author(question_mode, query, propose_questions) + uses_image_author = _uses_image_question_author(question_mode, query) _require_openai_for_question_author(uses_image_author) _require_edgetam_cuda() typer.echo(f"Loading frame {frame_index} from {recording}") @@ -69,12 +68,12 @@ def single_frame( model = MoondreamVlModel() typer.echo("Loading private MoonDream model") model.start() - question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=question_model)) + question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=QUESTION_MODEL)) try: if uses_image_author: - typer.echo(f"Proposing questions with {question_model}") + typer.echo(f"Proposing questions with {QUESTION_MODEL}") intents: list[QuestionIntent] | list[QuestionProposal] = ( - OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=question_model)).propose( + OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=QUESTION_MODEL)).propose( frame.image ) if question_mode == "agentic" @@ -107,9 +106,7 @@ def single_frame( ), ) results: list[GroundTruthResult] | list[AcceptedOracleResult | RejectedOracleResult] = ( - _answer_agentic( - ground_truth, frame, cast("list[QuestionProposal]", intents), oracle_model - ) + _answer_agentic(ground_truth, frame, cast("list[QuestionProposal]", intents)) if question_mode == "agentic" else _answer_intents( ground_truth, frame, cast("list[QuestionIntent]", intents), f"Frame {frame_index}" @@ -137,8 +134,8 @@ def single_frame( else "openai_image_agent" if uses_image_author else "explicit_queries", - "question_model": question_model if uses_image_author else None, - "oracle_model": oracle_model if question_mode == "agentic" else None, + "question_model": QUESTION_MODEL if uses_image_author else None, + "oracle_model": ORACLE_MODEL if question_mode == "agentic" else None, "grounding": { "min_mask_area_px": min_mask_area_px, "min_foreground_points": min_foreground_points, @@ -155,10 +152,7 @@ def generate( stop_index: int = typer.Option(..., "--stop-index"), stride: int = typer.Option(1, "--stride"), query: list[str] = typer.Option([], "--query"), - propose_questions: bool = typer.Option(False, "--propose-questions"), question_mode: str = typer.Option("constrained", "--question-mode"), - question_model: str = typer.Option("gpt-4o-mini", "--question-model"), - oracle_model: str = typer.Option("gpt-4o-mini", "--oracle-model"), min_mask_area_px: int = typer.Option(128, "--min-mask-area-px"), min_foreground_points: int = typer.Option(3, "--min-foreground-points"), output: Path | None = typer.Option(None, "--output"), @@ -168,7 +162,7 @@ def generate( raise typer.BadParameter("provide valid frame bounds") output = output or (STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frames") _validate_question_mode(question_mode) - uses_image_author = _uses_image_question_author(question_mode, query, propose_questions) + uses_image_author = _uses_image_question_author(question_mode, query) _require_openai_for_question_author(uses_image_author) output.mkdir(parents=True, exist_ok=True) _require_edgetam_cuda() @@ -180,7 +174,7 @@ def generate( try: detector = MoondreamObjectDetector(model) segmenter = EdgeTamObjectSegmenter(EdgeTAMImageSegmenter()) - question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=question_model)) + question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=QUESTION_MODEL)) for frame_number, frame_index in enumerate(frame_indices, start=1): frame_output = output / f"frame-{frame_index:06d}" if (frame_output / "frame.json").is_file(): @@ -193,9 +187,9 @@ def generate( ) frame = load_go2_frame(str(resolve_named_path(recording, ".db")), frame_index) if uses_image_author: - typer.echo(f"Frame {frame_index}: proposing questions with {question_model}") + typer.echo(f"Frame {frame_index}: proposing questions with {QUESTION_MODEL}") intents: list[QuestionIntent] | list[QuestionProposal] = ( - OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=question_model)).propose( + OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=QUESTION_MODEL)).propose( frame.image ) if question_mode == "agentic" @@ -227,9 +221,7 @@ def generate( ), ) results: list[GroundTruthResult] | list[AcceptedOracleResult | RejectedOracleResult] = ( - _answer_agentic( - ground_truth, frame, cast("list[QuestionProposal]", intents), oracle_model - ) + _answer_agentic(ground_truth, frame, cast("list[QuestionProposal]", intents)) if question_mode == "agentic" else _answer_intents( ground_truth, @@ -254,8 +246,8 @@ def generate( else "openai_image_agent" if uses_image_author else "explicit_queries", - "question_model": question_model if uses_image_author else None, - "oracle_model": oracle_model if question_mode == "agentic" else None, + "question_model": QUESTION_MODEL if uses_image_author else None, + "oracle_model": ORACLE_MODEL if question_mode == "agentic" else None, "grounding": { "min_mask_area_px": min_mask_area_px, "min_foreground_points": min_foreground_points, @@ -291,9 +283,8 @@ def _answer_agentic( ground_truth: VqaGroundTruthGenerator, frame: CalibratedFrame, proposals: list[QuestionProposal], - oracle_model: str, ) -> list[AcceptedOracleResult | RejectedOracleResult]: - oracle = create_openai_oracle(oracle_model) + oracle = create_openai_oracle(ORACLE_MODEL) results: list[AcceptedOracleResult | RejectedOracleResult] = [] for number, proposal in enumerate(proposals, start=1): typer.echo(f"Agentic question {number}/{len(proposals)}: {proposal.question}") @@ -316,11 +307,9 @@ def _require_openai_for_question_author(uses_image_author: bool) -> None: raise typer.BadParameter("OPENAI_API_KEY must be set for image-authored question modes") -def _uses_image_question_author( - question_mode: str, query: list[str], propose_questions: bool -) -> bool: +def _uses_image_question_author(question_mode: str, query: list[str]) -> bool: """Choose authored questions unless constrained mode has explicit queries.""" - return question_mode == "agentic" or propose_questions or not query + return question_mode == "agentic" or not query def _require_edgetam_cuda() -> None: diff --git a/docs/vqa-benchmark.md b/docs/vqa-benchmark.md index 8960f54d06..a6ba7558e4 100644 --- a/docs/vqa-benchmark.md +++ b/docs/vqa-benchmark.md @@ -32,6 +32,8 @@ Constrained generation expands visible objects into fixed choice-question famili Agentic generation freezes a free-form image-authored question, then a private oracle uses read-only local tools to establish its answer. Height questions use the fixed choices `under 0.5 m`, `0.5-1.0 m`, `1.0-1.5 m`, and `over 1.5 m`; the private tool measures height and maps it deterministically to one choice. Additional oracle tools will be added as new evidence capabilities mature. +The generation models are code-level defaults, not CLI settings. Constrained mode uses the image author when no `--query` is provided; supplying one or more `--query` values selects the deterministic families for those objects. + Geometry quality gates are the private validation step. They reject insufficient point support, ambiguous masks, unreliable ground planes, and incomplete height evidence before a case becomes public. ## Dataset Format From 4c5e3476738c856a643a2c0d2150631cd8648b41 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Mon, 10 Aug 2026 14:26:49 -0700 Subject: [PATCH 06/12] refactor(benchmark): consolidate VQA generation pipeline --- dimos/benchmark/vqa/generation/dataset.py | 42 +-- .../vqa/generation/ground_truth_generator.py | 89 +++-- dimos/benchmark/vqa/generation/oracle.py | 55 ++- .../benchmark/vqa/generation/oracle_tools.py | 342 +++++++++++------- .../vqa/generation/question_agent.py | 14 +- dimos/benchmark/vqa/generation/questions.py | 6 +- dimos/benchmark/vqa/generation/selection.py | 14 + dimos/benchmark/vqa/generation/test_agents.py | 22 -- .../benchmark/vqa/generation/test_dataset.py | 30 +- dimos/benchmark/vqa/generation/test_oracle.py | 144 ++++++-- .../vqa/generation/test_selection.py | 15 + dimos/benchmark/vqa/models.py | 13 +- dimos/cli/test_vqa.py | 13 +- dimos/cli/vqa.py | 69 +--- .../vqa-generation/README.md} | 16 +- docs/benchmarking/vqa-generation/pipeline.md | 209 +++++++++++ docs/docs.json | 3 +- 17 files changed, 755 insertions(+), 341 deletions(-) create mode 100644 dimos/benchmark/vqa/generation/selection.py create mode 100644 dimos/benchmark/vqa/generation/test_selection.py rename docs/{vqa-benchmark.md => benchmarking/vqa-generation/README.md} (72%) create mode 100644 docs/benchmarking/vqa-generation/pipeline.md diff --git a/dimos/benchmark/vqa/generation/dataset.py b/dimos/benchmark/vqa/generation/dataset.py index ed2bf905a5..987e1ced7d 100644 --- a/dimos/benchmark/vqa/generation/dataset.py +++ b/dimos/benchmark/vqa/generation/dataset.py @@ -10,7 +10,6 @@ import cv2 -from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator from dimos.benchmark.vqa.models import ( AcceptedOracleResult, BooleanAnswerContract, @@ -29,7 +28,6 @@ def write_frame_record( frame_index: int, intents: list[QuestionIntent | QuestionProposal], results: list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult], - ground_truth: VqaGroundTruthGenerator, metadata: dict[str, Any], ) -> None: """Write one frame's public cases alongside its private generation audit record.""" @@ -37,13 +35,6 @@ def write_frame_record( image_path = output / "image.jpg" if not cv2.imwrite(str(image_path), frame.image.data): raise RuntimeError(f"failed to write {image_path}") - original_image_path = output / "original_image.jpg" - if frame.original_image is not None and not cv2.imwrite( - str(original_image_path), frame.original_image.data - ): - raise RuntimeError(f"failed to write {original_image_path}") - overlay_path = output / "grounding_overlay.jpg" - ground_truth.write_overlay(frame, str(overlay_path)) accepted = [result for result in results if _is_accepted(result)] cases, labels = _evaluation_rows(frame.id, accepted) _write_json( @@ -54,18 +45,12 @@ def write_frame_record( "recording": recording, "frame_index": frame_index, "image": image_path.name, - "original_image": original_image_path.name - if frame.original_image is not None - else None, - "grounding_overlay": overlay_path.name, "question_count": len(intents), "accepted_question_count": len(accepted), "rejected_question_count": len(results) - len(accepted), **metadata, }, ) - _write_json(output / "intents.json", [asdict(item) for item in intents]) - _write_json(output / "examples.json", [_public_example(item, frame.id) for item in accepted]) _write_json(output / "ground_truth.json", [_private_result(item) for item in results]) _write_json(output / "cases.json", cases) _write_json(output / "labels.json", labels) @@ -74,14 +59,12 @@ def write_frame_record( def write_dataset_manifest(output: Path) -> dict[str, int]: """Build aggregate public cases and private labels from completed frame records.""" frames = sorted(path for path in output.glob("frame-*") if (path / "frame.json").is_file()) - frame_rows: list[dict[str, Any]] = [] case_rows: list[dict[str, Any]] = [] label_rows: list[dict[str, Any]] = [] accepted = 0 rejected = 0 for path in frames: frame = json.loads((path / "frame.json").read_text()) - frame_rows.append(frame) case_rows.extend( {**case, "image": f"{path.name}/{case['image']}"} for case in json.loads((path / "cases.json").read_text()) @@ -89,17 +72,8 @@ def write_dataset_manifest(output: Path) -> dict[str, int]: label_rows.extend(json.loads((path / "labels.json").read_text())) accepted += frame["accepted_question_count"] rejected += frame["rejected_question_count"] - _write_jsonl(output / "frames.jsonl", frame_rows) _write_jsonl(output / "cases.jsonl", case_rows) _write_jsonl(output / "labels.jsonl", label_rows) - _write_json( - output / "manifest.json", - { - "frame_count": len(frames), - "accepted_question_count": accepted, - "rejected_question_count": rejected, - }, - ) return { "frame_count": len(frames), "accepted_question_count": accepted, @@ -122,7 +96,7 @@ def _evaluation_rows( if isinstance(result, RejectedOracleResult): continue if isinstance(result, AcceptedOracleResult): - contract = result.proposal.answer_contract + contract = result.answer_contract choices = ( ("yes", "no") if isinstance(contract, BooleanAnswerContract) else contract.choices ) @@ -143,19 +117,6 @@ def _evaluation_rows( return cases, labels -def _public_example( - result: GroundTruthResult | AcceptedOracleResult, frame_id: str -) -> dict[str, Any]: - if isinstance(result, AcceptedOracleResult): - return { - "case_id": f"{frame_id}-{result.proposal.id}", - "question": result.proposal.question, - "answer_contract": asdict(result.proposal.answer_contract), - "object_queries": result.proposal.object_queries, - } - return asdict(result.question) - - def _private_result( result: GroundTruthResult | AcceptedOracleResult | RejectedOracleResult, ) -> dict[str, Any]: @@ -164,6 +125,7 @@ def _private_result( "status": "answered", "answer": result.answer, "proposal": asdict(result.proposal), + "answer_contract": asdict(result.answer_contract), "evidence_ids": result.evidence_ids, "tool_results": [asdict(item) for item in result.tool_results], "trace": [asdict(item) for item in result.trace], diff --git a/dimos/benchmark/vqa/generation/ground_truth_generator.py b/dimos/benchmark/vqa/generation/ground_truth_generator.py index 1ea0dbe0b4..fad215fb85 100644 --- a/dimos/benchmark/vqa/generation/ground_truth_generator.py +++ b/dimos/benchmark/vqa/generation/ground_truth_generator.py @@ -5,6 +5,7 @@ from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects from dimos.benchmark.vqa.generation.questions import generate_questions +from dimos.benchmark.vqa.generation.selection import select_nearest_object from dimos.benchmark.vqa.models import ( CalibratedFrame, GroundedObject, @@ -19,6 +20,7 @@ VqaExample, ) from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D from dimos.perception.detection.type.detection2d.point import Detection2DPoint from dimos.perception.detection.type.detection2d.seg import Detection2DSeg @@ -41,45 +43,62 @@ def __init__( if config.min_mask_area_px < 1 or config.min_foreground_points < 1: raise ValueError("grounding thresholds must be positive") self._config = config - self._groundings: dict[str, tuple[list[GroundedObject], tuple[ToolTrace, ...]]] = {} + self._groundings: dict[str, list[GroundedObject]] = {} + self._detected: dict[str, ImageDetections2D] = {} + self._segmented_queries: set[str] = set() self._masks: dict[str, list[Detection2DSeg]] = {} self._detections: dict[str, list[Detection2DBBox]] = {} self._points: dict[str, list[Detection2DPoint]] = {} - self._overlay_results: list[GroundTruthResult] = [] def answer(self, frame: CalibratedFrame, intent: QuestionIntent) -> GroundTruthResult: objects, trace = self.ground(frame, intent.object_query) - result = self._answer_from_objects(frame, intent, objects, trace) - self._overlay_results.append(result) - return result + return self._answer_from_objects(frame, intent, objects, trace) def ground( self, frame: CalibratedFrame, object_query: str ) -> tuple[list[GroundedObject], tuple[ToolTrace, ...]]: - """Ground one semantic query for direct local oracle tools. + """Run the fixed constrained grounding recipe over shared primitives.""" + if object_query in self._groundings: + return self._groundings[object_query], (ToolTrace("reuse_grounding", object_query),) + trace: list[ToolTrace] = [ToolTrace("detect_objects", object_query)] + detections = self.detect_objects(frame, object_query) + if len(detections): + trace.append(ToolTrace("segment_objects", f"count={len(detections)}")) + elif self._localizer is not None and self._point_segmenter is not None: + trace.append(ToolTrace("locate_object_point", object_query)) + masks = self.segment_detections(frame, object_query) + if not len(detections) and object_query in self._points: + trace.append( + ToolTrace("segment_object_point", f"count={len(self._points[object_query])}") + ) + trace.append(ToolTrace("get_foreground_geometry", f"masks={len(masks)}")) + objects = self.ground_masks(frame, object_query) + return objects, tuple(trace) - The method operates solely on the supplied calibrated frame and exposes no - model or recording state to the caller. - """ - cached = self._groundings.get(object_query) + def detect_objects(self, frame: CalibratedFrame, object_query: str) -> ImageDetections2D: + """Run private MoonDream detection once for an opaque object query.""" + cached = self._detected.get(object_query) if cached is not None: - objects, prior_trace = cached - return objects, (ToolTrace("reuse_grounding", object_query), *prior_trace) - trace: list[ToolTrace] = [ToolTrace("detect_objects", object_query)] + return cached detections = self._detector.detect(frame.image, object_query) + self._detected[object_query] = detections self._detections[object_query] = [ item for item in detections if isinstance(item, Detection2DBBox) ] + return detections + + def segment_detections(self, frame: CalibratedFrame, object_query: str) -> list[Detection2DSeg]: + """Run private EdgeTAM segmentation once for a detected object query.""" + if object_query in self._segmented_queries: + return self._masks.get(object_query, []) + detections = self.detect_objects(frame, object_query) if len(detections): - trace.append(ToolTrace("segment_objects", f"count={len(detections)}")) segmented = self._segmenter.segment(detections) elif self._localizer is not None and self._point_segmenter is not None: - trace.append(ToolTrace("locate_object_point", object_query)) points = self._localizer.locate(frame.image, object_query) self._points[object_query] = [ item for item in points if isinstance(item, Detection2DPoint) ] - trace.append(ToolTrace("segment_object_point", f"count={len(points)}")) segmented = self._point_segmenter.segment_points(points) else: segmented = detections @@ -89,14 +108,21 @@ def ground( if isinstance(item, Detection2DSeg) and int((item.mask > 0).sum()) >= self._config.min_mask_area_px ] - trace.append(ToolTrace("get_foreground_geometry", f"masks={len(masks)}")) self._masks[object_query] = masks + self._segmented_queries.add(object_query) + return masks + + def ground_masks(self, frame: CalibratedFrame, object_query: str) -> list[GroundedObject]: + """Project visible point-cloud support through an accepted mask set.""" + cached = self._groundings.get(object_query) + if cached is not None: + return cached + masks = self.segment_detections(frame, object_query) objects = ground_segmented_objects( frame, masks, min_foreground_points=self._config.min_foreground_points ) - stored_trace = tuple(trace) - self._groundings[object_query] = (objects, stored_trace) - return objects, stored_trace + self._groundings[object_query] = objects + return objects def masks_for_query(self, object_query: str) -> tuple[Detection2DSeg, ...]: """Return masks produced by the most recent local grounding for a query.""" @@ -148,21 +174,6 @@ def _answer_from_objects( intent, rejected, "rejected", None, "no_grounded_object", tuple(objects), trace ) - def write_overlay(self, frame: CalibratedFrame, path: str) -> None: - """Save private masks and detector prompts over the rectified image.""" - import cv2 - - overlay = frame.image.data.copy() - colors = ((255, 128, 0), (0, 200, 255), (180, 0, 255), (0, 200, 80)) - for index, query in enumerate(self._masks): - color = colors[index % len(colors)] - for mask in self._masks.get(query, []): - overlay[mask.mask > 0] = color - rendered = cv2.addWeighted(frame.image.data, 0.55, overlay, 0.45, 0) - if not cv2.imwrite(path, rendered): - raise RuntimeError(f"failed to write {path}") - - def _render_question(intent: QuestionIntent) -> str: if intent.kind == "presence": return f"Is there a {intent.object_query} in the image? Answer yes or no." @@ -179,12 +190,10 @@ def _compare_nearest_by_side( objects: list[GroundedObject], trace: tuple[ToolTrace, ...], ) -> GroundTruthResult: - left_matches = [item for item in objects if item.horizontal_direction == "left"] - right_matches = [item for item in objects if item.horizontal_direction == "right"] - if not left_matches or not right_matches: + left = select_nearest_object(objects, "left") + right = select_nearest_object(objects, "right") + if left is None or right is None: return _rejected_result(frame, intent, objects, trace, "missing_grounded_side") - left = min(left_matches, key=lambda item: item.range_m) - right = min(right_matches, key=lambda item: item.range_m) if left.range_m == right.range_m: return _rejected_result(frame, intent, objects, trace, "ambiguous_nearest_by_side") answer = "left" if left.range_m < right.range_m else "right" diff --git a/dimos/benchmark/vqa/generation/oracle.py b/dimos/benchmark/vqa/generation/oracle.py index 0a174a4cb2..53b76192a2 100644 --- a/dimos/benchmark/vqa/generation/oracle.py +++ b/dimos/benchmark/vqa/generation/oracle.py @@ -14,10 +14,12 @@ AnswerContract, BooleanAnswerContract, ChoiceAnswerContract, + DeferredHeightChoiceContract, OracleToolResult, OracleTrace, QuestionProposal, RejectedOracleResult, + ResolvedAnswerContract, ) if TYPE_CHECKING: @@ -97,7 +99,7 @@ class PrivateToolCallingOracle: def __init__( self, model: BaseChatModel, - max_tool_calls: int = 4, + max_tool_calls: int = 8, semantic_validator: SemanticEvidenceValidator | None = None, ) -> None: if max_tool_calls < 1: @@ -151,7 +153,7 @@ def answer( return _rejected(proposal, "tool_call_limit", registry.results, trace) -def create_openai_oracle(model: str, max_tool_calls: int = 4) -> PrivateToolCallingOracle: +def create_openai_oracle(model: str, max_tool_calls: int = 8) -> PrivateToolCallingOracle: """Construct the private-only OpenAI tool-calling oracle.""" from langchain_openai import ChatOpenAI @@ -169,6 +171,16 @@ def validate_oracle_answer( results: tuple[OracleToolResult, ...], ) -> str: """Deterministically validate an answer format and citations.""" + return _resolve_oracle_answer(proposal, answer, evidence_ids, results)[0] + + +def _resolve_oracle_answer( + proposal: QuestionProposal, + answer: Any, + evidence_ids: Any, + results: tuple[OracleToolResult, ...], +) -> tuple[str, ResolvedAnswerContract]: + """Return a validated answer and the public contract resolved from private evidence.""" if ( not isinstance(evidence_ids, list) or not evidence_ids @@ -182,14 +194,35 @@ def validate_oracle_answer( if isinstance(contract, BooleanAnswerContract): if answer not in ("yes", "no"): raise ValueError("boolean answer must be yes or no") - return str(answer) + return str(answer), contract if isinstance(contract, ChoiceAnswerContract): if not isinstance(answer, str) or answer not in contract.choices: raise ValueError("choice answer is not allowed") - measured_choices = {result.choice for result in results if result.choice is not None} + cited = set(evidence_ids) + measured_choices = { + result.choice + for result in results + if result.choice is not None and any(item.id in cited for item in result.evidence) + } if measured_choices and answer not in measured_choices: raise ValueError("choice answer does not match cited measurement bucket") - return answer + return answer, contract + if isinstance(contract, DeferredHeightChoiceContract): + bucket_results = [ + result + for result in results + if result.tool == "bucket_measurement" and result.choice is not None and result.choices + ] + if len(bucket_results) != 1: + raise ValueError("deferred height answer requires exactly one measurement bucket") + bucket = bucket_results[0] + if not any(item.id in evidence_ids for item in bucket.evidence): + raise ValueError("deferred height answer must cite its measurement") + if answer != bucket.choice: + raise ValueError("deferred height answer does not match measurement bucket") + if bucket.choice not in bucket.choices: + raise ValueError("measurement bucket choice is not public") + return bucket.choice, ChoiceAnswerContract(bucket.choices) raise ValueError("unsupported answer contract") @@ -202,7 +235,7 @@ def _validated_result( ) -> AcceptedOracleResult | RejectedOracleResult: try: payload = _parse_json_object(response) - answer = validate_oracle_answer( + answer, answer_contract = _resolve_oracle_answer( proposal, payload.get("answer"), payload.get("evidence_ids"), results ) evidence_ids = tuple(payload["evidence_ids"]) @@ -211,7 +244,8 @@ def _validated_result( cited_results = _cited_results(evidence_ids, results) if semantic_validator is None: return _rejected(proposal, "semantic_validator_not_configured", results, trace) - verdict = semantic_validator.validate(proposal, answer, cited_results) + resolved_proposal = replace(proposal, answer_contract=answer_contract) + verdict = semantic_validator.validate(resolved_proposal, answer, cited_results) trace.append( OracleTrace( "semantic_validation", @@ -220,7 +254,7 @@ def _validated_result( ) if not verdict.accepted: return _rejected(proposal, f"unsupported_evidence:{verdict.reason}", results, trace) - return AcceptedOracleResult(proposal, answer, evidence_ids, results, tuple(trace)) + return AcceptedOracleResult(proposal, answer, answer_contract, evidence_ids, results, tuple(trace)) def _cited_results( @@ -258,6 +292,11 @@ def _contract_prompt(contract: AnswerContract) -> str: return "boolean: yes or no" if isinstance(contract, ChoiceAnswerContract): return f"choice: {', '.join(contract.choices)}" + if isinstance(contract, DeferredHeightChoiceContract): + return ( + "deferred height choice: call measure_height, then bucket_measurement, and return " + "the exact choice from that result" + ) raise ValueError("unsupported answer contract") diff --git a/dimos/benchmark/vqa/generation/oracle_tools.py b/dimos/benchmark/vqa/generation/oracle_tools.py index ff74e785f9..a13e61fb10 100644 --- a/dimos/benchmark/vqa/generation/oracle_tools.py +++ b/dimos/benchmark/vqa/generation/oracle_tools.py @@ -1,8 +1,9 @@ # Copyright 2026 Dimensional Inc. -"""Direct local LangChain tools over one frozen VQA frame.""" +"""Typed private perception primitives over one frozen VQA frame.""" from __future__ import annotations +from bisect import bisect_right import json from typing import Any @@ -11,21 +12,31 @@ from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator from dimos.benchmark.vqa.generation.measurements import estimate_ground_plane, points_in_mask +from dimos.benchmark.vqa.generation.selection import select_nearest_object from dimos.benchmark.vqa.models import ( CalibratedFrame, + GroundedObject, + GroundPlaneEstimate, OracleEvidence, OracleMeasurement, OracleToolResult, ) +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg class LocalOracleToolRegistry: - """Expose only named, side-effect-free perception operations to an oracle.""" + """Expose the same private perception primitives used by constrained recipes.""" def __init__(self, frame: CalibratedFrame, grounding: VqaGroundTruthGenerator) -> None: self._frame = frame self._grounding = grounding self._results: list[OracleToolResult] = [] + self._detections: dict[str, str] = {} + self._masks: dict[str, str] = {} + self._objects: dict[str, tuple[GroundedObject, Detection2DSeg]] = {} + self._planes: dict[str, GroundPlaneEstimate] = {} + self._measurements: dict[str, OracleToolResult] = {} + self._next_id = 0 @property def results(self) -> tuple[OracleToolResult, ...]: @@ -34,177 +45,221 @@ def results(self) -> tuple[OracleToolResult, ...]: def tools(self) -> list[StructuredTool]: return [ StructuredTool.from_function( - self.ground_semantic_object, - name="ground_semantic_object", + self.detect_objects, + name="detect_objects", + description="Run private MoonDream detection for one visible semantic query.", + ), + StructuredTool.from_function( + self.segment_detections, + name="segment_detections", + description="Run private EdgeTAM segmentation for one opaque detection ID.", + ), + StructuredTool.from_function( + self.ground_masks, + name="ground_masks", description=( - "Ground a visible semantic object query with private MoonDream, EdgeTAM, " - "and calibrated point-cloud geometry. Returns object geometry and evidence IDs." + "Project visible calibrated point-cloud support through one opaque mask ID. " + "Returns grounded object IDs and citable evidence." ), ), StructuredTool.from_function( - self.estimate_ground_plane, - name="estimate_ground_plane", + self.select_nearest_object, + name="select_nearest_object", description=( - "Estimate a visible ground plane from the frozen calibrated point cloud. " - "Returns a quality-gated local geometric result." + "Select the nearest opaque grounded object ID, optionally restricted to left, center, " + "or right." ), ), StructuredTool.from_function( - self.measure_object_height, - name="measure_object_height", + self.fit_ground_plane, + name="fit_ground_plane", + description="Fit a quality-gated Open3D ground plane to the frozen visible point cloud.", + ), + StructuredTool.from_function( + self.measure_height, + name="measure_height", description=( - "Ground one visible semantic object with local MoonDream, EdgeTAM, and LiDAR " - "then measure its visible point-cloud height above the estimated ground plane." + "Measure one opaque grounded object above one opaque accepted ground-plane ID." ), ), StructuredTool.from_function( - self.measure_object_height_bucket, - name="measure_object_height_bucket", + self.bucket_measurement, + name="bucket_measurement", description=( - "Measure one visible object's height, then return its public choice: under 0.5 m, " - "0.5-1.0 m, 1.0-1.5 m, or over 1.5 m." + "Map one opaque height measurement ID to four public, answer-conditioned " + "height choices and the one matching choice." ), ), ] - def ground_semantic_object(self, query: str) -> str: - """Return grounded objects for a visible object query in the frozen frame.""" - objects, _ = self._grounding.ground(self._frame, query) - evidence = tuple( - OracleEvidence( - id=f"grounding:v1:{item.id}", - version="v1", - object_id=item.id, - label=item.label, - range_m=item.range_m, - side=item.horizontal_direction, - point_count=item.point_count, - ) - for item in objects - ) - result = OracleToolResult("ground_semantic_object", query, evidence) + def detect_objects(self, query: str) -> str: + """Detect objects and return an opaque ID for a later segmentation call.""" + detections = self._grounding.detect_objects(self._frame, query) + detection_id = self._id("detection") + self._detections[detection_id] = query + result = OracleToolResult("detect_objects", query, ()) self._results.append(result) - return json.dumps(_tool_payload(result)) + boxes = [list(item.bbox) for item in detections] + return json.dumps(_tool_payload(result, detection_id=detection_id, boxes=boxes)) - def estimate_ground_plane(self) -> str: - """Estimate the lower-image visible ground plane without fabricating a result.""" - fit = estimate_ground_plane(self._frame) - if fit.estimate is None: - result = OracleToolResult( - "estimate_ground_plane", - "", - (), - quality_flags=fit.quality_flags, - rejection_reason=fit.rejection_reason, - ) - else: - measurement = OracleMeasurement( - fit.estimate.offset_m, - "m", - max(fit.estimate.residual_m, 0.01), - fit.quality_flags, - (f"frame:{self._frame.id}",), - ) - evidence = OracleEvidence( - f"ground-plane:v1:{self._frame.id}", - "v1", - "ground-plane", - "ground", - 0.0, - "n/a", - fit.estimate.inlier_count, - measurement, - ) - result = OracleToolResult( - "estimate_ground_plane", - "", - (evidence,), - measurement=measurement, - plane=fit.estimate, - quality_flags=fit.quality_flags, - ) + def segment_detections(self, detection_id: str) -> str: + """Segment one earlier opaque detection result and return an opaque mask ID.""" + query = self._detections.get(detection_id) + if query is None: + return self._record_rejection("segment_detections", "", [], "unknown_detection_id") + masks = self._grounding.segment_detections(self._frame, query) + mask_id = self._id("mask") + self._masks[mask_id] = query + result = OracleToolResult("segment_detections", query, ()) self._results.append(result) - return json.dumps(_tool_payload(result)) + return json.dumps(_tool_payload(result, mask_id=mask_id, mask_count=len(masks))) + + def ground_masks(self, mask_id: str) -> str: + """Ground one earlier opaque mask result against the visible point cloud.""" + query = self._masks.get(mask_id) + if query is None: + return self._record_rejection("ground_masks", "", [], "unknown_mask_id") + objects = self._grounding.ground_masks(self._frame, query) + masks = self._grounding.masks_for_query(query) + evidence = tuple(_grounding_evidence(item) for item in objects) + for item in objects: + index = _object_mask_index(item) + if index < len(masks): + self._objects[item.id] = (item, masks[index]) + result = OracleToolResult("ground_masks", query, evidence) + self._results.append(result) + return json.dumps(_tool_payload(result, object_ids=[item.id for item in objects])) - def measure_object_height(self, query: str) -> str: - """Measure one unambiguous grounded object's visible height above local ground.""" - objects, _ = self._grounding.ground(self._frame, query) - masks_for_query = getattr(self._grounding, "masks_for_query", None) - masks = tuple(masks_for_query(query)) if callable(masks_for_query) else () + def fit_ground_plane(self) -> str: + """Fit the private ground plane and return an opaque ID for measurements.""" fit = estimate_ground_plane(self._frame) - flags = list(fit.quality_flags) if fit.estimate is None: return self._record_rejection( - "measure_object_height", query, flags, fit.rejection_reason - ) - if len(objects) != 1 or len(masks) != 1: - flags.append("ambiguous_or_missing_object_mask") - return self._record_rejection( - "measure_object_height", query, flags, "ambiguous_object_evidence" + "fit_ground_plane", "", list(fit.quality_flags), fit.rejection_reason ) - selected = points_in_mask(self._frame, masks[0].mask) + plane_id = self._id("plane") + self._planes[plane_id] = fit.estimate + measurement = OracleMeasurement( + fit.estimate.offset_m, + "m", + max(fit.estimate.residual_m, 0.01), + fit.quality_flags, + (f"frame:{self._frame.id}",), + ) + evidence = OracleEvidence( + f"ground-plane:v1:{self._frame.id}", + "v1", + "ground-plane", + "ground", + 0.0, + "n/a", + fit.estimate.inlier_count, + measurement, + ) + result = OracleToolResult( + "fit_ground_plane", + "", + (evidence,), + measurement=measurement, + plane=fit.estimate, + quality_flags=fit.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result, plane_id=plane_id)) + + def select_nearest_object(self, object_ids: list[str], side: str | None = None) -> str: + """Select one opaque grounded object by private point-cloud range.""" + selected: list[GroundedObject] = [] + for object_id in object_ids: + item = self._objects.get(object_id) + if item is None: + return self._record_rejection( + "select_nearest_object", object_id, [], "unknown_object_id" + ) + selected.append(item[0]) + nearest = select_nearest_object(selected, side) + if nearest is None: + return self._record_rejection("select_nearest_object", "", [], "no_object_matches_side") + result = OracleToolResult( + "select_nearest_object", nearest.label, (_grounding_evidence(nearest),) + ) + self._results.append(result) + return json.dumps(_tool_payload(result, object_id=nearest.id)) + + def measure_height(self, object_id: str, plane_id: str) -> str: + """Measure one grounded object against one previously accepted plane.""" + object_and_mask = self._objects.get(object_id) + plane = self._planes.get(plane_id) + if object_and_mask is None: + return self._record_rejection("measure_height", object_id, [], "unknown_object_id") + if plane is None: + return self._record_rejection("measure_height", object_id, [], "unknown_plane_id") + item, mask = object_and_mask + selected = points_in_mask(self._frame, mask.mask) + flags = ["visible_point_cloud_height"] if len(selected) < 6: flags.append("sparse_object_point_support") return self._record_rejection( - "measure_object_height", query, flags, "insufficient_object_support" + "measure_height", item.label, flags, "insufficient_object_support" ) - normal = np.asarray(fit.estimate.normal) - distances = selected @ normal + fit.estimate.offset_m + normal = np.asarray(plane.normal) + distances = selected @ normal + plane.offset_m positive = distances[distances > 0.02] if len(positive) < 4 or len(positive) / len(selected) < 0.6: flags.append("partial_or_non_elevated_object_support") return self._record_rejection( - "measure_object_height", query, flags, "ambiguous_object_extent" + "measure_height", item.label, flags, "ambiguous_object_extent" ) - height = float(np.percentile(positive, 85)) - tolerance = float(max(0.05, fit.estimate.residual_m + np.std(positive) * 0.25)) - flags.extend(("visible_point_cloud_height", "conservative_upper_percentile")) + flags.append("conservative_upper_percentile") measurement = OracleMeasurement( - height, + float(np.percentile(positive, 85)), "m", - tolerance, + float(max(0.05, plane.residual_m + np.std(positive) * 0.25)), tuple(flags), ( f"frame:{self._frame.id}", f"ground-plane:v1:{self._frame.id}", - f"grounding:v1:{objects[0].id}", + f"grounding:v1:{item.id}", ), ) evidence = OracleEvidence( - f"height:v1:{objects[0].id}", + f"height:v1:{item.id}", "v1", - objects[0].id, - objects[0].label, - objects[0].range_m, - objects[0].horizontal_direction, + item.id, + item.label, + item.range_m, + item.horizontal_direction, len(selected), measurement, ) result = OracleToolResult( - "measure_object_height", - query, + "measure_height", + item.label, (evidence,), measurement=measurement, - plane=fit.estimate, + plane=plane, quality_flags=tuple(flags), ) + measurement_id = self._id("measurement") + self._measurements[measurement_id] = result self._results.append(result) - return json.dumps(_tool_payload(result)) + return json.dumps(_tool_payload(result, measurement_id=measurement_id)) - def measure_object_height_bucket(self, query: str) -> str: - """Measure an object, then map its private height to a fixed public choice.""" - self.measure_object_height(query) - height_result = self._results[-1] - if height_result.measurement is None: - return json.dumps(_tool_payload(height_result)) + def bucket_measurement(self, measurement_id: str) -> str: + """Map one accepted private height measurement to its public choice.""" + source = self._measurements.get(measurement_id) + if source is None or source.measurement is None: + return self._record_rejection("bucket_measurement", "", [], "unknown_measurement_id") + choices, choice = _height_choice_window(source.measurement.value) result = OracleToolResult( - "measure_object_height_bucket", - query, - height_result.evidence, - measurement=height_result.measurement, - choice=_height_bucket(height_result.measurement.value), - plane=height_result.plane, - quality_flags=height_result.quality_flags, + "bucket_measurement", + source.query, + source.evidence, + measurement=source.measurement, + choice=choice, + choices=choices, + plane=source.plane, + quality_flags=source.quality_flags, ) self._results.append(result) return json.dumps(_tool_payload(result)) @@ -216,8 +271,31 @@ def _record_rejection(self, tool: str, query: str, flags: list[str], reason: str self._results.append(result) return json.dumps(_tool_payload(result)) + def _id(self, kind: str) -> str: + self._next_id += 1 + return f"{kind}:v1:{self._next_id:04d}" + -def _tool_payload(result: OracleToolResult) -> dict[str, Any]: +def _grounding_evidence(item: GroundedObject) -> OracleEvidence: + return OracleEvidence( + f"grounding:v1:{item.id}", + "v1", + item.id, + item.label, + item.range_m, + item.horizontal_direction, + item.point_count, + ) + + +def _object_mask_index(item: GroundedObject) -> int: + try: + return int(item.id.rsplit("-", 1)[1]) + except (IndexError, ValueError) as exc: + raise ValueError(f"grounded object ID lacks mask index: {item.id}") from exc + + +def _tool_payload(result: OracleToolResult, **identifiers: Any) -> dict[str, Any]: return { "tool": result.tool, "query": result.query, @@ -234,6 +312,7 @@ def _tool_payload(result: OracleToolResult) -> dict[str, Any]: else None ), "choice": result.choice, + "choices": result.choices, "quality_flags": result.quality_flags, "rejection_reason": result.rejection_reason, "plane": ( @@ -248,6 +327,7 @@ def _tool_payload(result: OracleToolResult) -> dict[str, Any]: else None ), "objects": [_evidence_payload(item) for item in result.evidence], + **identifiers, } @@ -271,11 +351,19 @@ def _evidence_payload(item: OracleEvidence) -> dict[str, Any]: return payload -def _height_bucket(height_m: float) -> str: - if height_m < 0.5: - return "under 0.5 m" - if height_m < 1.0: - return "0.5-1.0 m" - if height_m < 1.5: - return "1.0-1.5 m" - return "over 1.5 m" +def _height_choice_window(height_m: float) -> tuple[tuple[str, ...], str]: + """Generate a local, exhaustive four-choice window around a private height.""" + breakpoints = (0.1, 0.2, 0.6, 1.0, 2.0) + start = min(max(bisect_right(breakpoints, height_m) - 1, 0), len(breakpoints) - 3) + lower, middle, upper = breakpoints[start : start + 3] + choices = ( + f"under {_format_height(lower)} m", + f"{_format_height(lower)}-{_format_height(middle)} m", + f"{_format_height(middle)}-{_format_height(upper)} m", + f"over {_format_height(upper)} m", + ) + return choices, choices[bisect_right((lower, middle, upper), height_m)] + + +def _format_height(value: float) -> str: + return f"{value:.1f}" diff --git a/dimos/benchmark/vqa/generation/question_agent.py b/dimos/benchmark/vqa/generation/question_agent.py index cf1b21daee..fb62f89b97 100644 --- a/dimos/benchmark/vqa/generation/question_agent.py +++ b/dimos/benchmark/vqa/generation/question_agent.py @@ -11,6 +11,7 @@ AnswerContract, BooleanAnswerContract, ChoiceAnswerContract, + DeferredHeightChoiceContract, QuestionIntent, QuestionProposal, ) @@ -26,14 +27,15 @@ AGENTIC_QUESTION_PROMPT = """Author up to 5 challenging, visually answerable single-frame VQA questions. Inspect only this image. Do not use or infer depth, point clouds, calibration, metadata, or answers. Return JSON only: an array of objects with question, answer_contract, optional object_queries, -and optional tool_hints. answer_contract is {"kind":"boolean"} or {"kind":"choice","choices":[...]}. +and optional tool_hints. answer_contract is {"kind":"boolean"}, {"kind":"choice","choices":[...]}, +or {"kind":"deferred_height_choice","strategy":"height-window-v1"}. Prioritize questions that a private point-cloud oracle can validate. For the height of one upright -object resting on visible ground, use exactly these choices: ["under 0.5 m", "0.5-1.0 m", -"1.0-1.5 m", "over 1.5 m"] and tool_hints ["measure_object_height_bucket"]. Also prefer which of +object resting on visible ground, use deferred_height_choice. Its choices are generated privately +from a successful height measurement. Also prefer which of two named objects is closer (choice, with those object names as choices), object count, left/right spatial relation, and distance-threshold questions. Use object_queries for every referenced object -and tool_hints from "measure_object_height_bucket", "measure_object_height", "estimate_ground_plane", -or "ground_semantic_object" when applicable. +and tool_hints from "detect_objects", "segment_detections", "ground_masks", "fit_ground_plane", +"select_nearest_object", "measure_height", or "bucket_measurement" when applicable. Use visibility/presence questions only when no stronger geometric question is available. Do not ask about color, material, text, intent, full physical size, hidden parts, or terrain. Use only visible objects. Do not include answers, explanations, Markdown, or background surfaces.""" @@ -132,6 +134,8 @@ def _proposal_from_json(item: Any, index: int) -> QuestionProposal: if len(choices) < 2: raise ValueError("choice contract requires at least two choices") answer_contract = ChoiceAnswerContract(choices) + elif kind == "deferred_height_choice" and contract.get("strategy") == "height-window-v1": + answer_contract = DeferredHeightChoiceContract() else: raise ValueError("unsupported answer contract") return QuestionProposal(identifier, question, answer_contract, queries, hints) diff --git a/dimos/benchmark/vqa/generation/questions.py b/dimos/benchmark/vqa/generation/questions.py index 56b425e261..35c6866326 100644 --- a/dimos/benchmark/vqa/generation/questions.py +++ b/dimos/benchmark/vqa/generation/questions.py @@ -16,6 +16,7 @@ from __future__ import annotations +from dimos.benchmark.vqa.generation.selection import select_nearest_object from dimos.benchmark.vqa.models import GroundedObject, VqaExample @@ -28,10 +29,7 @@ def generate_questions( examples: list[VqaExample] = [] for query in queries: - matches = sorted( - (item for item in objects if item.label == query), key=lambda item: item.range_m - ) - nearest = matches[0] if matches else None + nearest = select_nearest_object([item for item in objects if item.label == query]) examples.append( VqaExample( id=f"{frame_id}-{query}-presence", diff --git a/dimos/benchmark/vqa/generation/selection.py b/dimos/benchmark/vqa/generation/selection.py new file mode 100644 index 0000000000..3c32b1d70b --- /dev/null +++ b/dimos/benchmark/vqa/generation/selection.py @@ -0,0 +1,14 @@ +# Copyright 2026 Dimensional Inc. +"""Shared deterministic selection over grounded object evidence.""" + +from __future__ import annotations + +from dimos.benchmark.vqa.models import GroundedObject + + +def select_nearest_object( + objects: list[GroundedObject], side: str | None = None +) -> GroundedObject | None: + """Return the nearest grounded object, optionally restricted to one image side.""" + candidates = [item for item in objects if side is None or item.horizontal_direction == side] + return min(candidates, key=lambda item: item.range_m) if candidates else None diff --git a/dimos/benchmark/vqa/generation/test_agents.py b/dimos/benchmark/vqa/generation/test_agents.py index f1fbe106e6..bda560a164 100644 --- a/dimos/benchmark/vqa/generation/test_agents.py +++ b/dimos/benchmark/vqa/generation/test_agents.py @@ -2,10 +2,8 @@ from __future__ import annotations -from pathlib import Path from typing import cast -import cv2 import numpy as np from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator @@ -219,23 +217,3 @@ def test_ground_truth_agent_rejects_side_comparison_without_both_sides() -> None assert result.status == "rejected" assert result.reason == "missing_grounded_side" - - -def test_ground_truth_agent_overlay_contains_only_rectified_mask_view( - tmp_path: Path, -) -> None: - frame, detection = _frame_and_detection() - agent = VqaGroundTruthGenerator( - _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) - ) - agent.answer(frame, QuestionIntent(kind="presence", object_query="chair")) - agent.answer( - frame, QuestionIntent(kind="within_distance", object_query="chair", threshold_m=3.0) - ) - path = tmp_path / "overlay.jpg" - - agent.write_overlay(frame, str(path)) - - rendered = cv2.imread(str(path)) - assert rendered is not None - assert rendered.shape[:2] == frame.image.data.shape[:2] diff --git a/dimos/benchmark/vqa/generation/test_dataset.py b/dimos/benchmark/vqa/generation/test_dataset.py index cda4830c33..6790044cd4 100644 --- a/dimos/benchmark/vqa/generation/test_dataset.py +++ b/dimos/benchmark/vqa/generation/test_dataset.py @@ -4,7 +4,16 @@ from pathlib import Path from dimos.benchmark.vqa.generation.dataset import _evaluation_rows, write_dataset_manifest -from dimos.benchmark.vqa.models import GroundTruthResult, QuestionIntent, ToolTrace, VqaExample +from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + ChoiceAnswerContract, + DeferredHeightChoiceContract, + GroundTruthResult, + QuestionIntent, + QuestionProposal, + ToolTrace, + VqaExample, +) def test_constrained_results_export_simple_multiple_choice_rows() -> None: @@ -38,6 +47,23 @@ def test_constrained_results_export_simple_multiple_choice_rows() -> None: assert labels == [{"id": "frame-chair-presence", "answer": "yes"}] +def test_deferred_height_result_exports_resolved_public_choices() -> None: + choices = ("under 0.2 m", "0.2-0.6 m", "0.6-1.0 m", "over 1.0 m") + result = AcceptedOracleResult( + QuestionProposal("chair-height", "How tall is the chair?", DeferredHeightChoiceContract()), + "0.2-0.6 m", + ChoiceAnswerContract(choices), + ("height-1",), + (), + (), + ) + + cases, labels = _evaluation_rows("frame", [result]) + + assert cases[0]["choices"] == choices + assert labels == [{"id": "frame-chair-height", "answer": "0.2-0.6 m"}] + + def test_dataset_manifest_exports_public_cases_and_private_labels(tmp_path: Path) -> None: frame = tmp_path / "frame-000040" frame.mkdir() @@ -74,3 +100,5 @@ def test_dataset_manifest_exports_public_cases_and_private_labels(tmp_path: Path "choices": ["yes", "no"], } assert json.loads((tmp_path / "labels.jsonl").read_text()) == {"id": "case-1", "answer": "yes"} + assert not (tmp_path / "frames.jsonl").exists() + assert not (tmp_path / "manifest.json").exists() diff --git a/dimos/benchmark/vqa/generation/test_oracle.py b/dimos/benchmark/vqa/generation/test_oracle.py index 143f35a554..1405ded30a 100644 --- a/dimos/benchmark/vqa/generation/test_oracle.py +++ b/dimos/benchmark/vqa/generation/test_oracle.py @@ -15,7 +15,10 @@ SemanticEvidenceValidation, validate_oracle_answer, ) -from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.generation.oracle_tools import ( + LocalOracleToolRegistry, + _height_choice_window, +) from dimos.benchmark.vqa.generation.question_agent import ( AGENTIC_QUESTION_PROMPT, OpenAIFreeformQuestionAuthor, @@ -24,6 +27,7 @@ BooleanAnswerContract, CalibratedFrame, ChoiceAnswerContract, + DeferredHeightChoiceContract, OracleEvidence, OracleToolResult, QuestionProposal, @@ -42,20 +46,29 @@ def query(self, image: Image, prompt: str) -> str: class _Grounding: - def ground(self, frame: Any, query: str) -> tuple[list[Any], tuple[Any, ...]]: + def detect_objects(self, frame: Any, query: str) -> list[Any]: + return [] + + def segment_detections(self, frame: Any, query: str) -> list[Any]: + return [] + + def ground_masks(self, frame: Any, query: str) -> list[Any]: return [ type( "Object", (), { - "id": "chair-1", + "id": "synthetic-chair-0", "label": query, "range_m": 1.0, "horizontal_direction": "left", "point_count": 4, }, )() - ], () + ] + + def masks_for_query(self, query: str) -> tuple[Any, ...]: + return () class _GroundingWithMask(_Grounding): @@ -65,6 +78,9 @@ def __init__(self, mask: np.ndarray) -> None: def masks_for_query(self, query: str) -> tuple[Any, ...]: return (type("Mask", (), {"mask": self._mask})(),) + def segment_detections(self, frame: Any, query: str) -> list[Any]: + return list(self.masks_for_query(query)) + def _measurement_frame(points: np.ndarray | None = None) -> CalibratedFrame: ground = [[x, 1.0, z] for z in (3.0, 4.0, 5.0) for x in (-1.0, -0.5, 0.0, 0.5, 1.0)] @@ -94,13 +110,31 @@ def bind_tools(self, tools: Any) -> _BoundModel: def invoke(self, messages: Any) -> AIMessage: self._calls += 1 if self._calls == 1: + return AIMessage( + content="", + tool_calls=[{"name": "detect_objects", "args": {"query": "chair"}, "id": "call-1"}], + ) + if self._calls == 2: return AIMessage( content="", tool_calls=[ - {"name": "ground_semantic_object", "args": {"query": "chair"}, "id": "call-1"} + { + "name": "segment_detections", + "args": {"detection_id": "detection:v1:0001"}, + "id": "call-2", + } ], ) - return AIMessage(content='{"answer":"yes","evidence_ids":["grounding:v1:chair-1"]}') + if self._calls == 3: + return AIMessage( + content="", + tool_calls=[ + {"name": "ground_masks", "args": {"mask_id": "mask:v1:0002"}, "id": "call-3"} + ], + ) + return AIMessage( + content='{"answer":"yes","evidence_ids":["grounding:v1:synthetic-chair-0"]}' + ) class _SemanticValidator: @@ -131,7 +165,7 @@ def test_freeform_question_author_parses_public_contract() -> None: def test_freeform_author_prompt_prioritizes_geometric_questions() -> None: - assert "measure_object_height" in AGENTIC_QUESTION_PROMPT + assert "measure_height" in AGENTIC_QUESTION_PROMPT assert "two named objects is closer" in AGENTIC_QUESTION_PROMPT assert "closer (choice" in AGENTIC_QUESTION_PROMPT assert "Use visibility/presence questions only" in AGENTIC_QUESTION_PROMPT @@ -169,6 +203,25 @@ def query(self, image: Image, prompt: str) -> str: raise AssertionError("numeric answer contract was accepted") +def test_freeform_question_author_parses_deferred_height_contract() -> None: + class _HeightContractModel: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"deferred_height_choice","strategy":"height-window-v1"},' + '"object_queries":["chair"]}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposal = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _HeightContractModel())).propose( + image + )[0] + + assert proposal.answer_contract == DeferredHeightChoiceContract() + assert proposal.object_queries == ("chair",) + + def test_freeform_question_author_normalizes_optional_query_hints() -> None: class _ModelWithStringHints: def query(self, image: Image, prompt: str) -> str: @@ -209,17 +262,19 @@ def query(self, image: Image, prompt: str) -> str: def test_local_tool_returns_geometry_and_evidence_ids() -> None: registry = LocalOracleToolRegistry(cast("Any", object()), cast("Any", _Grounding())) - payload = json.loads(registry.ground_semantic_object("chair")) + detection = json.loads(registry.detect_objects("chair")) + masks = json.loads(registry.segment_detections(detection["detection_id"])) + payload = json.loads(registry.ground_masks(masks["mask_id"])) assert payload["objects"][0] == { - "evidence_id": "grounding:v1:chair-1", - "id": "chair-1", + "evidence_id": "grounding:v1:synthetic-chair-0", + "id": "synthetic-chair-0", "label": "chair", "range_m": 1.0, "side": "left", "point_count": 4, } - assert registry.results[0].version == "v1" + assert registry.results[-1].version == "v1" def test_ground_plane_estimator_fits_visible_lower_band() -> None: @@ -237,7 +292,7 @@ def test_ground_plane_tool_returns_quality_gated_rejection() -> None: frame = _measurement_frame(np.asarray([[0.0, 1.0, 3.0]], dtype=np.float32)) registry = LocalOracleToolRegistry(frame, cast("Any", _Grounding())) - payload = json.loads(registry.estimate_ground_plane()) + payload = json.loads(registry.fit_ground_plane()) assert payload["measurement"] is None assert payload["rejection_reason"] == "insufficient_support" @@ -253,12 +308,16 @@ def test_height_tool_measures_visible_object_points_above_plane() -> None: mask[y, x] = 255 registry = LocalOracleToolRegistry(frame, cast("Any", _GroundingWithMask(mask))) - payload = json.loads(registry.measure_object_height("chair")) + detection = json.loads(registry.detect_objects("chair")) + masks = json.loads(registry.segment_detections(detection["detection_id"])) + grounded = json.loads(registry.ground_masks(masks["mask_id"])) + plane = json.loads(registry.fit_ground_plane()) + payload = json.loads(registry.measure_height(grounded["object_ids"][0], plane["plane_id"])) assert payload["measurement"]["unit"] == "m" assert 0.8 < payload["measurement"]["value"] < 1.0 assert payload["measurement"]["tolerance"] >= 0.05 - assert payload["objects"][0]["evidence_id"] == "height:v1:chair-1" + assert payload["objects"][0]["evidence_id"] == "height:v1:synthetic-chair-0" assert "visible_point_cloud_height" in payload["quality_flags"] @@ -266,13 +325,29 @@ def test_local_registry_exposes_geometry_tools() -> None: registry = LocalOracleToolRegistry(cast("Any", object()), cast("Any", _Grounding())) assert {tool.name for tool in registry.tools()} == { - "ground_semantic_object", - "estimate_ground_plane", - "measure_object_height", - "measure_object_height_bucket", + "detect_objects", + "segment_detections", + "ground_masks", + "select_nearest_object", + "fit_ground_plane", + "measure_height", + "bucket_measurement", } +def test_height_choice_window_is_local_and_deterministic() -> None: + choices, answer = _height_choice_window(0.42) + + assert choices == ( + "under 0.2 m", + "0.2-0.6 m", + "0.6-1.0 m", + "over 1.0 m", + ) + assert answer == "0.2-0.6 m" + assert _height_choice_window(3.0)[1] == "over 2.0 m" + + def test_oracle_validates_evidence_and_answer_contract() -> None: proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract()) result = OracleToolResult( @@ -294,6 +369,28 @@ def test_oracle_validates_evidence_and_answer_contract() -> None: raise AssertionError("unknown evidence was accepted") +def test_oracle_derives_deferred_height_answer_from_measurement_bucket() -> None: + proposal = QuestionProposal( + "q", "How tall is the chair?", DeferredHeightChoiceContract(), ("chair",) + ) + evidence = OracleEvidence("height-1", "v1", "chair-1", "chair", 1.0, "left", 8) + result = OracleToolResult( + "bucket_measurement", + "chair", + (evidence,), + choice="0.2-0.6 m", + choices=("under 0.2 m", "0.2-0.6 m", "0.6-1.0 m", "over 1.0 m"), + ) + + assert validate_oracle_answer(proposal, "0.2-0.6 m", ["height-1"], (result,)) == "0.2-0.6 m" + try: + validate_oracle_answer(proposal, "under 0.2 m", ["height-1"], (result,)) + except ValueError as exc: + assert "does not match measurement bucket" in str(exc) + else: + raise AssertionError("non-derived deferred height answer was accepted") + + def test_private_oracle_runs_direct_structured_tool() -> None: proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) registry = LocalOracleToolRegistry(cast("Any", object()), cast("Any", _Grounding())) @@ -304,8 +401,8 @@ def test_private_oracle_runs_direct_structured_tool() -> None: ).answer(proposal, registry) assert result.answer == "yes" - assert result.evidence_ids == ("grounding:v1:chair-1",) - assert validator.calls[0][2][0].evidence[0].id == "grounding:v1:chair-1" + assert result.evidence_ids == ("grounding:v1:synthetic-chair-0",) + assert validator.calls[0][2][-1].evidence[0].id == "grounding:v1:synthetic-chair-0" assert result.trace[-1].detail == "accepted:chair grounding supports yes" @@ -318,14 +415,14 @@ def invoke(self, messages: Any) -> AIMessage: content="", tool_calls=[ { - "name": "ground_semantic_object", + "name": "detect_objects", "args": {"query": "chair"}, "id": "call-1", } ], ) return AIMessage( - content='{"answer":"0.5-1.0 m","evidence_ids":["grounding:v1:chair-1"]}' + content='{"answer":"0.5-1.0 m","evidence_ids":["grounding:v1:synthetic-chair-0"]}' ) proposal = QuestionProposal( @@ -340,8 +437,7 @@ def invoke(self, messages: Any) -> AIMessage: cast("Any", _ChoiceModel()), semantic_validator=validator ).answer(proposal, registry) - assert result.reason == "unsupported_evidence:range and side do not measure height" - assert result.trace[-1].detail == "rejected:range and side do not measure height" + assert result.reason == "invalid_final_answer:answer cites unknown evidence" def test_agentic_oracle_never_uses_legacy_answer_program() -> None: diff --git a/dimos/benchmark/vqa/generation/test_selection.py b/dimos/benchmark/vqa/generation/test_selection.py new file mode 100644 index 0000000000..fa6d7df077 --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_selection.py @@ -0,0 +1,15 @@ +# Copyright 2026 Dimensional Inc. + +from dimos.benchmark.vqa.generation.selection import select_nearest_object +from dimos.benchmark.vqa.models import GroundedObject + + +def test_select_nearest_object_optionally_restricts_to_image_side() -> None: + objects = [ + GroundedObject("left", "chair", 3, 2.0, "left"), + GroundedObject("right", "chair", 3, 1.0, "right"), + ] + + assert select_nearest_object(objects).id == "right" + assert select_nearest_object(objects, "left").id == "left" + assert select_nearest_object(objects, "center") is None diff --git a/dimos/benchmark/vqa/models.py b/dimos/benchmark/vqa/models.py index f66150d8c8..a5221f6b29 100644 --- a/dimos/benchmark/vqa/models.py +++ b/dimos/benchmark/vqa/models.py @@ -136,7 +136,16 @@ class ChoiceAnswerContract: kind: Literal["choice"] = "choice" -AnswerContract = BooleanAnswerContract | ChoiceAnswerContract +@dataclass(frozen=True) +class DeferredHeightChoiceContract: + """A height question whose public choices follow a private measurement.""" + + strategy: Literal["height-window-v1"] = "height-window-v1" + kind: Literal["deferred_height_choice"] = "deferred_height_choice" + + +AnswerContract = BooleanAnswerContract | ChoiceAnswerContract | DeferredHeightChoiceContract +ResolvedAnswerContract = BooleanAnswerContract | ChoiceAnswerContract @dataclass(frozen=True) @@ -196,6 +205,7 @@ class OracleToolResult: version: str = "v1" measurement: OracleMeasurement | None = None choice: str | None = None + choices: tuple[str, ...] = () plane: GroundPlaneEstimate | None = None quality_flags: tuple[str, ...] = () rejection_reason: str | None = None @@ -215,6 +225,7 @@ class AcceptedOracleResult: proposal: QuestionProposal answer: str + answer_contract: ResolvedAnswerContract evidence_ids: tuple[str, ...] tool_results: tuple[OracleToolResult, ...] trace: tuple[OracleTrace, ...] diff --git a/dimos/cli/test_vqa.py b/dimos/cli/test_vqa.py index cc94510983..a0d8d4e85d 100644 --- a/dimos/cli/test_vqa.py +++ b/dimos/cli/test_vqa.py @@ -1,9 +1,14 @@ # Copyright 2026 Dimensional Inc. +from typer.testing import CliRunner + from dimos.cli import vqa -def test_question_mode_selects_the_image_author_without_an_explicit_query() -> None: - assert vqa._uses_image_question_author("constrained", []) is True - assert vqa._uses_image_question_author("constrained", ["chair"]) is False - assert vqa._uses_image_question_author("agentic", ["chair"]) is True +def test_vqa_generation_cli_has_no_explicit_query_or_model_options() -> None: + output = CliRunner().invoke(vqa.app, ["generate", "--help"]).output + + assert "--query" not in output + assert "--propose-questions" not in output + assert "--question-model" not in output + assert "--oracle-model" not in output diff --git a/dimos/cli/vqa.py b/dimos/cli/vqa.py index e2a00ed31f..1416307e25 100644 --- a/dimos/cli/vqa.py +++ b/dimos/cli/vqa.py @@ -5,7 +5,7 @@ import os from pathlib import Path -from typing import Any, cast +from typing import cast import typer @@ -47,7 +47,6 @@ def single_frame( recording: str = typer.Option(..., "--recording"), frame_index: int = typer.Option(0, "--frame-index"), - query: list[str] = typer.Option([], "--query"), question_mode: str = typer.Option("constrained", "--question-mode"), min_mask_area_px: int = typer.Option(128, "--min-mask-area-px"), min_foreground_points: int = typer.Option(3, "--min-foreground-points"), @@ -60,8 +59,7 @@ def single_frame( if output.exists(): raise typer.BadParameter("output must not already exist") _validate_question_mode(question_mode) - uses_image_author = _uses_image_question_author(question_mode, query) - _require_openai_for_question_author(uses_image_author) + _require_openai_for_question_author() _require_edgetam_cuda() typer.echo(f"Loading frame {frame_index} from {recording}") frame = load_go2_frame(str(resolve_named_path(recording, ".db")), frame_index) @@ -70,29 +68,13 @@ def single_frame( model.start() question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=QUESTION_MODEL)) try: - if uses_image_author: - typer.echo(f"Proposing questions with {QUESTION_MODEL}") + typer.echo(f"Proposing questions with {QUESTION_MODEL}") intents: list[QuestionIntent] | list[QuestionProposal] = ( OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=QUESTION_MODEL)).propose( frame.image ) if question_mode == "agentic" else question_agent.propose(frame.image) - if uses_image_author - else [ - QuestionIntent( - kind=cast("Any", kind), - object_query=item, - threshold_m=3.0 if kind == "within_distance" else None, - ) - for item in query - for kind in ( - "presence", - "horizontal_direction", - "within_distance", - "compare_nearest_by_side", - ) - ] ) typer.echo(f"Grounding {len(intents)} questions for frame {frame_index}") ground_truth = VqaGroundTruthGenerator( @@ -127,14 +109,11 @@ def single_frame( frame_index, cast("list[QuestionIntent | QuestionProposal]", intents), cast("list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult]", results), - ground_truth, { "question_source": "agentic_image_author" if question_mode == "agentic" - else "openai_image_agent" - if uses_image_author - else "explicit_queries", - "question_model": QUESTION_MODEL if uses_image_author else None, + else "openai_image_agent", + "question_model": QUESTION_MODEL, "oracle_model": ORACLE_MODEL if question_mode == "agentic" else None, "grounding": { "min_mask_area_px": min_mask_area_px, @@ -151,7 +130,6 @@ def generate( start_index: int = typer.Option(0, "--start-index"), stop_index: int = typer.Option(..., "--stop-index"), stride: int = typer.Option(1, "--stride"), - query: list[str] = typer.Option([], "--query"), question_mode: str = typer.Option("constrained", "--question-mode"), min_mask_area_px: int = typer.Option(128, "--min-mask-area-px"), min_foreground_points: int = typer.Option(3, "--min-foreground-points"), @@ -162,8 +140,7 @@ def generate( raise typer.BadParameter("provide valid frame bounds") output = output or (STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frames") _validate_question_mode(question_mode) - uses_image_author = _uses_image_question_author(question_mode, query) - _require_openai_for_question_author(uses_image_author) + _require_openai_for_question_author() output.mkdir(parents=True, exist_ok=True) _require_edgetam_cuda() frame_indices = range(start_index, stop_index, stride) @@ -186,29 +163,13 @@ def generate( f"Frame {frame_number}/{len(frame_indices)}: loading recording index {frame_index}" ) frame = load_go2_frame(str(resolve_named_path(recording, ".db")), frame_index) - if uses_image_author: - typer.echo(f"Frame {frame_index}: proposing questions with {QUESTION_MODEL}") + typer.echo(f"Frame {frame_index}: proposing questions with {QUESTION_MODEL}") intents: list[QuestionIntent] | list[QuestionProposal] = ( OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=QUESTION_MODEL)).propose( frame.image ) if question_mode == "agentic" else question_agent.propose(frame.image) - if uses_image_author - else [ - QuestionIntent( - kind=cast("Any", kind), - object_query=item, - threshold_m=3.0 if kind == "within_distance" else None, - ) - for item in query - for kind in ( - "presence", - "horizontal_direction", - "within_distance", - "compare_nearest_by_side", - ) - ] ) typer.echo(f"Frame {frame_index}: grounding {len(intents)} questions") ground_truth = VqaGroundTruthGenerator( @@ -239,14 +200,11 @@ def generate( cast( "list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult]", results ), - ground_truth, { "question_source": "agentic_image_author" if question_mode == "agentic" - else "openai_image_agent" - if uses_image_author - else "explicit_queries", - "question_model": QUESTION_MODEL if uses_image_author else None, + else "openai_image_agent", + "question_model": QUESTION_MODEL, "oracle_model": ORACLE_MODEL if question_mode == "agentic" else None, "grounding": { "min_mask_area_px": min_mask_area_px, @@ -302,16 +260,11 @@ def _validate_question_mode(question_mode: str) -> None: raise typer.BadParameter("question mode must be constrained or agentic") -def _require_openai_for_question_author(uses_image_author: bool) -> None: - if uses_image_author and not os.environ.get("OPENAI_API_KEY"): +def _require_openai_for_question_author() -> None: + if not os.environ.get("OPENAI_API_KEY"): raise typer.BadParameter("OPENAI_API_KEY must be set for image-authored question modes") -def _uses_image_question_author(question_mode: str, query: list[str]) -> bool: - """Choose authored questions unless constrained mode has explicit queries.""" - return question_mode == "agentic" or not query - - def _require_edgetam_cuda() -> None: if default_local_model_device() != "cuda": raise typer.BadParameter( diff --git a/docs/vqa-benchmark.md b/docs/benchmarking/vqa-generation/README.md similarity index 72% rename from docs/vqa-benchmark.md rename to docs/benchmarking/vqa-generation/README.md index a6ba7558e4..5ab1a40350 100644 --- a/docs/vqa-benchmark.md +++ b/docs/benchmarking/vqa-generation/README.md @@ -6,6 +6,8 @@ title: "VQA Benchmark" DimOS generates image-question-multiple-choice VQA cases from frozen robot recordings. Private point-cloud tools establish and validate answers during generation; evaluation sees only public images, questions, choices, and private answer labels. +For the exact generation stages, checks, tool contracts, and output files, see [Pipeline](/docs/benchmarking/vqa-generation/pipeline.md). + ## Pipeline Flow ```text @@ -14,8 +16,8 @@ Frozen recording -> rectified RGB image + calibrated visible point cloud +-- constrained: image object author -> deterministic question families | -> private grounding -> quality-gate validation | - +-- agentic: image question author -> frozen choice question - -> private local oracle tools -> validation + +-- agentic: image question author -> frozen question + -> private local oracle tools -> validation | v accepted cases.jsonl + private labels.jsonl @@ -24,24 +26,26 @@ accepted cases.jsonl + private labels.jsonl point-cloud-vqa Evaluation -> image-only vision model -> exact choice scoring ``` -Rejected questions and private evidence remain in each frame's generation record. The evaluator does not load point clouds, calibration, tool traces, overlays, or rejection records. +Rejected questions and private evidence remain in each frame's generation record. The evaluator does not load point clouds, calibration, tool traces, or rejection records. ## Generation Modes Constrained generation expands visible objects into fixed choice-question families: presence (`yes`/`no`), horizontal direction, distance threshold, and nearest left-versus-right comparison. Additional deterministic question families will be added as their evidence programs mature. -Agentic generation freezes a free-form image-authored question, then a private oracle uses read-only local tools to establish its answer. Height questions use the fixed choices `under 0.5 m`, `0.5-1.0 m`, `1.0-1.5 m`, and `over 1.5 m`; the private tool measures height and maps it deterministically to one choice. Additional oracle tools will be added as new evidence capabilities mature. +Agentic generation freezes a free-form image-authored question, then a private oracle uses read-only local tools to establish its answer. Height questions generate four public choices deterministically after private measurement. For example, `0.42 m` produces `under 0.2 m`, `0.2-0.6 m`, `0.6-1.0 m`, and `over 1.0 m`, with `0.2-0.6 m` as the answer. Additional oracle tools will be added as new evidence capabilities mature. -The generation models are code-level defaults, not CLI settings. Constrained mode uses the image author when no `--query` is provided; supplying one or more `--query` values selects the deterministic families for those objects. +The generation models are code-level defaults, not CLI settings. Constrained mode always uses the image author, then expands its object queries into the deterministic families. Geometry quality gates are the private validation step. They reject insufficient point support, ambiguous masks, unreliable ground planes, and incomplete height evidence before a case becomes public. +Constrained families run predefined sequences over the private perception primitives. Agentic generation exposes those same primitives to the oracle, which chooses its own sequence after the public question is frozen. + ## Dataset Format The root evaluation export follows the common image-question-choice benchmark pattern: ```json -{"id":"go2-40-chair-height","image":"frame-000040/image.jpg","question":"How tall is the chair?","choices":["under 0.5 m","0.5-1.0 m","1.0-1.5 m","over 1.5 m"]} +{"id":"go2-40-chair-height","image":"frame-000040/image.jpg","question":"How tall is the chair?","choices":["under 0.2 m","0.2-0.6 m","0.6-1.0 m","over 1.0 m"]} ``` `cases.jsonl` contains public rows only. `labels.jsonl` contains the matching private `id` and `answer`. Frame directories retain `ground_truth.json`, which includes the full private generation audit data. diff --git a/docs/benchmarking/vqa-generation/pipeline.md b/docs/benchmarking/vqa-generation/pipeline.md new file mode 100644 index 0000000000..22b0bc4b20 --- /dev/null +++ b/docs/benchmarking/vqa-generation/pipeline.md @@ -0,0 +1,209 @@ +--- +title: "VQA Generation Pipeline" +--- + +# VQA Generation Pipeline + +## 1. Run Generation + +```bash +dimos vqa generate \ + --recording \ + --start-index \ + --stop-index \ + --stride \ + --question-mode constrained|agentic \ + --output +``` + +Flags: + +- `--recording`: Memory2 Go2 recording path or named recording. +- `--start-index`, `--stop-index`, `--stride`: sampled recording frames. +- `--question-mode`: `constrained` or `agentic`; default is `constrained`. +- `--min-mask-area-px`: minimum accepted segmentation-mask area; default `128`. +- `--min-foreground-points`: minimum point-cloud support inside a mask; default `3`. +- `--output`: dataset root; completed frame directories are skipped on rerun. + +`dimos vqa single-frame` accepts the same generation settings and uses `--frame-index` instead of frame bounds. + +## 2. Create Questions + +### Constrained + +The image author proposes visible object queries, then the generator creates these deterministic families for every query: + +| Family | Sample question | Choices | +|---|---|---| +| Presence | `Is there a chair in the image?` | `yes`, `no` | +| Horizontal direction | `Where is the nearest chair?` | `left`, `center`, `right` | +| Distance threshold | `Is the nearest chair within 3 meters?` | `yes`, `no` | +| Nearest by side | `Which chair is closer, the left or right one?` | `left`, `right` | + +### Agentic + +The image-only author returns a frozen question with one contract: + +- Boolean: `{"kind":"boolean"}`; public choices are `yes`, `no`. +- Choice: `{"kind":"choice","choices":[...]}`; at least two choices. +- Height: `{"kind":"deferred_height_choice","strategy":"height-window-v1"}`; the question is + frozen before private geometry, while four public choices are generated from a successful private + height measurement. + +Numeric contracts are rejected. `height-window-v1` selects a local four-choice window from private +measurement against the fixed internal breakpoints `0.1`, `0.2`, `0.6`, `1.0`, and `2.0` meters. For +example, a private height of `0.42 m` produces: + +```text +under 0.2 m | 0.2-0.6 m | 0.6-1.0 m | over 1.0 m +``` + +## 3. Pre-Answer Grounding Checks + +For each referenced object: + +1. MoonDream detects or point-localizes the object. +2. EdgeTAM produces a mask. +3. Visible calibrated point-cloud samples are projected into the mask. +4. Mask area must meet `--min-mask-area-px`. +5. Point support must meet `--min-foreground-points`. + +Height questions also require: + +1. Accepted Open3D RANSAC ground plane. +2. Exactly one grounded object and one mask. +3. At least six points inside the object mask. +4. At least four elevated points, with at least 60% of selected points elevated more than `0.02 m` above the plane. + +## 4. Create Answers + +### Constrained + +Each deterministic family runs its own fixed sequence. + +```text +presence(A) +-> detect_objects(A) -> detection_id +-> segment_detections(detection_id) -> mask_id +-> ground_masks(mask_id) -> grounded A instances +-> no grounded A: reject +-> one or more grounded A instances: yes +``` + +```text +horizontal_direction(A) +-> detect_objects(A) -> detection_id +-> segment_detections(detection_id) -> mask_id +-> ground_masks(mask_id) -> grounded A instances +-> select_nearest_object(grounded A instances) -> nearest A +-> nearest A horizontal_direction: left/center/right +``` + +```text +within_distance(A, T) +-> detect_objects(A) -> detection_id +-> segment_detections(detection_id) -> mask_id +-> ground_masks(mask_id) -> grounded A instances +-> select_nearest_object(grounded A instances) -> nearest A +-> nearest A range_m <= T: yes/no +``` + +```text +compare_nearest_by_side(A) +-> detect_objects(A) -> detection_id +-> segment_detections(detection_id) -> mask_id +-> ground_masks(mask_id) -> grounded A instances +-> select_nearest_object(grounded A instances, left) -> nearest left A +-> select_nearest_object(grounded A instances, right) -> nearest right A +-> either side missing or tied: reject +-> compare the two range_m values: left/right +``` + +### Agentic + +The private oracle chooses a sequence from the same read-only primitives used by constrained recipes: + +| Tool | Input | Output | +|---|---|---| +| `detect_objects` | semantic query | Detection ID and private boxes. | +| `segment_detections` | detection ID | Mask ID and accepted mask count. | +| `ground_masks` | mask ID | Grounded object IDs, range, side, point support, evidence IDs. | +| `select_nearest_object` | object IDs, optional side | Nearest grounded object ID. | +| `fit_ground_plane` | none | Plane ID, plane estimate, residual, inlier support, quality flags. | +| `measure_height` | object ID, plane ID | Measurement ID, private height, uncertainty, provenance, quality flags. | +| `bucket_measurement` | measurement ID | Public answer-conditioned height choices and matching choice. | + +Opaque IDs chain tool results; raw masks and point-cloud arrays are not exposed to the oracle. The +oracle returns a candidate answer and cited evidence IDs. Deferred height choices are the sole +exception to frozen public choices: the question remains frozen, but the public options and answer +are deterministically derived from the private measurement. + +## 5. Post-Answer Validation + +The candidate is rejected unless: + +1. Its answer exactly matches the fixed public choices, or the public choices deterministically + generated from a cited private height measurement. +2. It cites one or more known evidence IDs. +3. A cited height bucket matches the deterministic measurement bucket. +4. The private semantic validator confirms the cited tool output supports the question and answer. + +Tool failures, quality-gate failures, invalid citations, invalid answer format, and unsupported claims are retained as rejected generation records. + +## 6. Write Dataset Artifacts + +Each `frame-*` directory contains: + +```text +image.jpg public rectified image +frame.json frame metadata and accepted/rejected counts +ground_truth.json private tool evidence, checks, answers, and rejections +cases.json public per-frame image/question/choice rows +labels.json private per-frame correct-choice rows +``` + +The dataset root aggregates: + +```text +cases.jsonl public id, image path, question, choices +labels.jsonl private id and expected choice +``` + +Each line in `cases.jsonl` is one public evaluation case. Formatted for readability, one record is: + +```json +{ + "id": "go2-40-chair-height", + "image": "frame-000040/image.jpg", + "question": "How tall is the chair?", + "choices": [ + "under 0.2 m", + "0.2-0.6 m", + "0.6-1.0 m", + "over 1.0 m" + ] +} +``` + +- `id`: unique case identifier. +- `image`: dataset-relative path to the public image. +- `question`: question shown to the evaluated vision model. +- `choices`: at least two allowed answer strings. + +Each line in `labels.jsonl` is the corresponding private correct answer. Formatted for readability, +one record is: + +```json +{ + "id": "go2-40-chair-height", + "answer": "0.2-0.6 m" +} +``` + +- `id`: case identifier matching exactly one `cases.jsonl` row. +- `answer`: one of that case's `choices`. + +The files store each record as one JSON object per physical line; the examples above are expanded +only for documentation readability. + +The shared `point-cloud-vqa` evaluator reads public `cases.jsonl`, public images, and private `labels.jsonl`. It does not read generation evidence or point-cloud data. diff --git a/docs/docs.json b/docs/docs.json index 6c1374be4a..eda15b0eee 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -125,7 +125,8 @@ "group": "Perception", "pages": [ "capabilities/perception/index", - "vqa-benchmark" + "benchmarking/vqa-generation/README", + "benchmarking/vqa-generation/pipeline" ] }, { From e85c48f40b8d5f61438dc0b8f1f464fc098282b3 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Mon, 10 Aug 2026 16:32:44 -0700 Subject: [PATCH 07/12] refactor(benchmark): share VQA perception primitives --- .../vqa/generation/ground_truth_generator.py | 110 ++---------- dimos/benchmark/vqa/generation/oracle.py | 4 +- .../benchmark/vqa/generation/oracle_tools.py | 116 ++++-------- .../vqa/generation/primitives/__init__.py | 5 + .../vqa/generation/primitives/choices.py | 23 +++ .../vqa/generation/primitives/contracts.py | 18 ++ .../vqa/generation/primitives/frame.py | 166 ++++++++++++++++++ .../geometry.py} | 6 +- .../generation/{ => primitives}/selection.py | 3 +- dimos/benchmark/vqa/generation/questions.py | 2 +- dimos/benchmark/vqa/generation/test_agents.py | 46 ++++- dimos/benchmark/vqa/generation/test_oracle.py | 85 +++++---- .../vqa/generation/test_selection.py | 2 +- dimos/cli/vqa.py | 11 +- .../vqa-generation/infrastructure.md | 157 +++++++++++++++++ docs/docs.json | 3 +- 16 files changed, 524 insertions(+), 233 deletions(-) create mode 100644 dimos/benchmark/vqa/generation/primitives/__init__.py create mode 100644 dimos/benchmark/vqa/generation/primitives/choices.py create mode 100644 dimos/benchmark/vqa/generation/primitives/contracts.py create mode 100644 dimos/benchmark/vqa/generation/primitives/frame.py rename dimos/benchmark/vqa/generation/{measurements.py => primitives/geometry.py} (90%) rename dimos/benchmark/vqa/generation/{ => primitives}/selection.py (82%) create mode 100644 docs/benchmarking/vqa-generation/infrastructure.md diff --git a/dimos/benchmark/vqa/generation/ground_truth_generator.py b/dimos/benchmark/vqa/generation/ground_truth_generator.py index fad215fb85..d37f7e9fda 100644 --- a/dimos/benchmark/vqa/generation/ground_truth_generator.py +++ b/dimos/benchmark/vqa/generation/ground_truth_generator.py @@ -3,52 +3,24 @@ from __future__ import annotations -from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object from dimos.benchmark.vqa.generation.questions import generate_questions -from dimos.benchmark.vqa.generation.selection import select_nearest_object from dimos.benchmark.vqa.models import ( CalibratedFrame, GroundedObject, - GroundingConfig, GroundTruthResult, - ObjectDetector, - ObjectPointLocalizer, - ObjectSegmenter, - PointObjectSegmenter, QuestionIntent, ToolTrace, VqaExample, ) -from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox -from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D -from dimos.perception.detection.type.detection2d.point import Detection2DPoint -from dimos.perception.detection.type.detection2d.seg import Detection2DSeg class VqaGroundTruthGenerator: """Answer constrained questions by calling detection, segmentation, and geometry tools.""" - def __init__( - self, - detector: ObjectDetector, - segmenter: ObjectSegmenter, - localizer: ObjectPointLocalizer | None = None, - point_segmenter: PointObjectSegmenter | None = None, - config: GroundingConfig = GroundingConfig(), - ) -> None: - self._detector = detector - self._segmenter = segmenter - self._localizer = localizer - self._point_segmenter = point_segmenter - if config.min_mask_area_px < 1 or config.min_foreground_points < 1: - raise ValueError("grounding thresholds must be positive") - self._config = config - self._groundings: dict[str, list[GroundedObject]] = {} - self._detected: dict[str, ImageDetections2D] = {} - self._segmented_queries: set[str] = set() - self._masks: dict[str, list[Detection2DSeg]] = {} - self._detections: dict[str, list[Detection2DBBox]] = {} - self._points: dict[str, list[Detection2DPoint]] = {} + def __init__(self, primitives: FramePerceptionPrimitives) -> None: + self.primitives = primitives def answer(self, frame: CalibratedFrame, intent: QuestionIntent) -> GroundTruthResult: objects, trace = self.ground(frame, intent.object_query) @@ -58,76 +30,23 @@ def ground( self, frame: CalibratedFrame, object_query: str ) -> tuple[list[GroundedObject], tuple[ToolTrace, ...]]: """Run the fixed constrained grounding recipe over shared primitives.""" - if object_query in self._groundings: - return self._groundings[object_query], (ToolTrace("reuse_grounding", object_query),) + if self.primitives.has_grounding(object_query): + return self.primitives.ground_masks(object_query), ( + ToolTrace("reuse_grounding", object_query), + ) trace: list[ToolTrace] = [ToolTrace("detect_objects", object_query)] - detections = self.detect_objects(frame, object_query) + detections = self.primitives.detect_objects(object_query) if len(detections): trace.append(ToolTrace("segment_objects", f"count={len(detections)}")) - elif self._localizer is not None and self._point_segmenter is not None: + elif self.primitives.can_localize_points: trace.append(ToolTrace("locate_object_point", object_query)) - masks = self.segment_detections(frame, object_query) - if not len(detections) and object_query in self._points: - trace.append( - ToolTrace("segment_object_point", f"count={len(self._points[object_query])}") - ) + masks = self.primitives.segment_detections(object_query) + if not len(detections) and self.primitives.used_point_localization(object_query): + trace.append(ToolTrace("segment_object_point", object_query)) trace.append(ToolTrace("get_foreground_geometry", f"masks={len(masks)}")) - objects = self.ground_masks(frame, object_query) + objects = self.primitives.ground_masks(object_query) return objects, tuple(trace) - def detect_objects(self, frame: CalibratedFrame, object_query: str) -> ImageDetections2D: - """Run private MoonDream detection once for an opaque object query.""" - cached = self._detected.get(object_query) - if cached is not None: - return cached - detections = self._detector.detect(frame.image, object_query) - self._detected[object_query] = detections - self._detections[object_query] = [ - item for item in detections if isinstance(item, Detection2DBBox) - ] - return detections - - def segment_detections(self, frame: CalibratedFrame, object_query: str) -> list[Detection2DSeg]: - """Run private EdgeTAM segmentation once for a detected object query.""" - if object_query in self._segmented_queries: - return self._masks.get(object_query, []) - detections = self.detect_objects(frame, object_query) - if len(detections): - segmented = self._segmenter.segment(detections) - elif self._localizer is not None and self._point_segmenter is not None: - points = self._localizer.locate(frame.image, object_query) - self._points[object_query] = [ - item for item in points if isinstance(item, Detection2DPoint) - ] - segmented = self._point_segmenter.segment_points(points) - else: - segmented = detections - masks = [ - item - for item in segmented - if isinstance(item, Detection2DSeg) - and int((item.mask > 0).sum()) >= self._config.min_mask_area_px - ] - self._masks[object_query] = masks - self._segmented_queries.add(object_query) - return masks - - def ground_masks(self, frame: CalibratedFrame, object_query: str) -> list[GroundedObject]: - """Project visible point-cloud support through an accepted mask set.""" - cached = self._groundings.get(object_query) - if cached is not None: - return cached - masks = self.segment_detections(frame, object_query) - objects = ground_segmented_objects( - frame, masks, min_foreground_points=self._config.min_foreground_points - ) - self._groundings[object_query] = objects - return objects - - def masks_for_query(self, object_query: str) -> tuple[Detection2DSeg, ...]: - """Return masks produced by the most recent local grounding for a query.""" - return tuple(self._masks.get(object_query, [])) - def _answer_from_objects( self, frame: CalibratedFrame, @@ -174,6 +93,7 @@ def _answer_from_objects( intent, rejected, "rejected", None, "no_grounded_object", tuple(objects), trace ) + def _render_question(intent: QuestionIntent) -> str: if intent.kind == "presence": return f"Is there a {intent.object_query} in the image? Answer yes or no." diff --git a/dimos/benchmark/vqa/generation/oracle.py b/dimos/benchmark/vqa/generation/oracle.py index 53b76192a2..2007082afc 100644 --- a/dimos/benchmark/vqa/generation/oracle.py +++ b/dimos/benchmark/vqa/generation/oracle.py @@ -254,7 +254,9 @@ def _validated_result( ) if not verdict.accepted: return _rejected(proposal, f"unsupported_evidence:{verdict.reason}", results, trace) - return AcceptedOracleResult(proposal, answer, answer_contract, evidence_ids, results, tuple(trace)) + return AcceptedOracleResult( + proposal, answer, answer_contract, evidence_ids, results, tuple(trace) + ) def _cited_results( diff --git a/dimos/benchmark/vqa/generation/oracle_tools.py b/dimos/benchmark/vqa/generation/oracle_tools.py index a13e61fb10..623249eb80 100644 --- a/dimos/benchmark/vqa/generation/oracle_tools.py +++ b/dimos/benchmark/vqa/generation/oracle_tools.py @@ -3,37 +3,32 @@ from __future__ import annotations -from bisect import bisect_right import json from typing import Any from langchain_core.tools import StructuredTool -import numpy as np -from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator -from dimos.benchmark.vqa.generation.measurements import estimate_ground_plane, points_in_mask -from dimos.benchmark.vqa.generation.selection import select_nearest_object +from dimos.benchmark.vqa.generation.primitives.choices import height_choice_window +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object from dimos.benchmark.vqa.models import ( - CalibratedFrame, GroundedObject, GroundPlaneEstimate, OracleEvidence, OracleMeasurement, OracleToolResult, ) -from dimos.perception.detection.type.detection2d.seg import Detection2DSeg class LocalOracleToolRegistry: """Expose the same private perception primitives used by constrained recipes.""" - def __init__(self, frame: CalibratedFrame, grounding: VqaGroundTruthGenerator) -> None: - self._frame = frame - self._grounding = grounding + def __init__(self, primitives: FramePerceptionPrimitives) -> None: + self._primitives = primitives self._results: list[OracleToolResult] = [] self._detections: dict[str, str] = {} self._masks: dict[str, str] = {} - self._objects: dict[str, tuple[GroundedObject, Detection2DSeg]] = {} + self._objects: dict[str, GroundedObject] = {} self._planes: dict[str, GroundPlaneEstimate] = {} self._measurements: dict[str, OracleToolResult] = {} self._next_id = 0 @@ -94,7 +89,7 @@ def tools(self) -> list[StructuredTool]: def detect_objects(self, query: str) -> str: """Detect objects and return an opaque ID for a later segmentation call.""" - detections = self._grounding.detect_objects(self._frame, query) + detections = self._primitives.detect_objects(query) detection_id = self._id("detection") self._detections[detection_id] = query result = OracleToolResult("detect_objects", query, ()) @@ -107,7 +102,7 @@ def segment_detections(self, detection_id: str) -> str: query = self._detections.get(detection_id) if query is None: return self._record_rejection("segment_detections", "", [], "unknown_detection_id") - masks = self._grounding.segment_detections(self._frame, query) + masks = self._primitives.segment_detections(query) mask_id = self._id("mask") self._masks[mask_id] = query result = OracleToolResult("segment_detections", query, ()) @@ -119,20 +114,17 @@ def ground_masks(self, mask_id: str) -> str: query = self._masks.get(mask_id) if query is None: return self._record_rejection("ground_masks", "", [], "unknown_mask_id") - objects = self._grounding.ground_masks(self._frame, query) - masks = self._grounding.masks_for_query(query) + objects = self._primitives.ground_masks(query) evidence = tuple(_grounding_evidence(item) for item in objects) for item in objects: - index = _object_mask_index(item) - if index < len(masks): - self._objects[item.id] = (item, masks[index]) + self._objects[item.id] = item result = OracleToolResult("ground_masks", query, evidence) self._results.append(result) return json.dumps(_tool_payload(result, object_ids=[item.id for item in objects])) def fit_ground_plane(self) -> str: """Fit the private ground plane and return an opaque ID for measurements.""" - fit = estimate_ground_plane(self._frame) + fit = self._primitives.fit_ground_plane() if fit.estimate is None: return self._record_rejection( "fit_ground_plane", "", list(fit.quality_flags), fit.rejection_reason @@ -144,10 +136,10 @@ def fit_ground_plane(self) -> str: "m", max(fit.estimate.residual_m, 0.01), fit.quality_flags, - (f"frame:{self._frame.id}",), + (f"frame:{self._primitives.frame.id}",), ) evidence = OracleEvidence( - f"ground-plane:v1:{self._frame.id}", + f"ground-plane:v1:{self._primitives.frame.id}", "v1", "ground-plane", "ground", @@ -176,7 +168,7 @@ def select_nearest_object(self, object_ids: list[str], side: str | None = None) return self._record_rejection( "select_nearest_object", object_id, [], "unknown_object_id" ) - selected.append(item[0]) + selected.append(item) nearest = select_nearest_object(selected, side) if nearest is None: return self._record_rejection("select_nearest_object", "", [], "no_object_matches_side") @@ -188,57 +180,38 @@ def select_nearest_object(self, object_ids: list[str], side: str | None = None) def measure_height(self, object_id: str, plane_id: str) -> str: """Measure one grounded object against one previously accepted plane.""" - object_and_mask = self._objects.get(object_id) + object = self._objects.get(object_id) plane = self._planes.get(plane_id) - if object_and_mask is None: + if object is None: return self._record_rejection("measure_height", object_id, [], "unknown_object_id") if plane is None: return self._record_rejection("measure_height", object_id, [], "unknown_plane_id") - item, mask = object_and_mask - selected = points_in_mask(self._frame, mask.mask) - flags = ["visible_point_cloud_height"] - if len(selected) < 6: - flags.append("sparse_object_point_support") + measured = self._primitives.measure_height(object, plane) + if measured.measurement is None: return self._record_rejection( - "measure_height", item.label, flags, "insufficient_object_support" + "measure_height", + object.label, + list(measured.quality_flags), + measured.rejection_reason, ) - normal = np.asarray(plane.normal) - distances = selected @ normal + plane.offset_m - positive = distances[distances > 0.02] - if len(positive) < 4 or len(positive) / len(selected) < 0.6: - flags.append("partial_or_non_elevated_object_support") - return self._record_rejection( - "measure_height", item.label, flags, "ambiguous_object_extent" - ) - flags.append("conservative_upper_percentile") - measurement = OracleMeasurement( - float(np.percentile(positive, 85)), - "m", - float(max(0.05, plane.residual_m + np.std(positive) * 0.25)), - tuple(flags), - ( - f"frame:{self._frame.id}", - f"ground-plane:v1:{self._frame.id}", - f"grounding:v1:{item.id}", - ), - ) + measurement = measured.measurement evidence = OracleEvidence( - f"height:v1:{item.id}", + f"height:v1:{object.id}", "v1", - item.id, - item.label, - item.range_m, - item.horizontal_direction, - len(selected), + object.id, + object.label, + object.range_m, + object.horizontal_direction, + object.point_count, measurement, ) result = OracleToolResult( "measure_height", - item.label, + object.label, (evidence,), measurement=measurement, plane=plane, - quality_flags=tuple(flags), + quality_flags=measured.quality_flags, ) measurement_id = self._id("measurement") self._measurements[measurement_id] = result @@ -250,7 +223,7 @@ def bucket_measurement(self, measurement_id: str) -> str: source = self._measurements.get(measurement_id) if source is None or source.measurement is None: return self._record_rejection("bucket_measurement", "", [], "unknown_measurement_id") - choices, choice = _height_choice_window(source.measurement.value) + choices, choice = height_choice_window(source.measurement.value) result = OracleToolResult( "bucket_measurement", source.query, @@ -288,13 +261,6 @@ def _grounding_evidence(item: GroundedObject) -> OracleEvidence: ) -def _object_mask_index(item: GroundedObject) -> int: - try: - return int(item.id.rsplit("-", 1)[1]) - except (IndexError, ValueError) as exc: - raise ValueError(f"grounded object ID lacks mask index: {item.id}") from exc - - def _tool_payload(result: OracleToolResult, **identifiers: Any) -> dict[str, Any]: return { "tool": result.tool, @@ -349,21 +315,3 @@ def _evidence_payload(item: OracleEvidence) -> dict[str, Any]: "provenance_ids": item.measurement.provenance_ids, } return payload - - -def _height_choice_window(height_m: float) -> tuple[tuple[str, ...], str]: - """Generate a local, exhaustive four-choice window around a private height.""" - breakpoints = (0.1, 0.2, 0.6, 1.0, 2.0) - start = min(max(bisect_right(breakpoints, height_m) - 1, 0), len(breakpoints) - 3) - lower, middle, upper = breakpoints[start : start + 3] - choices = ( - f"under {_format_height(lower)} m", - f"{_format_height(lower)}-{_format_height(middle)} m", - f"{_format_height(middle)}-{_format_height(upper)} m", - f"over {_format_height(upper)} m", - ) - return choices, choices[bisect_right((lower, middle, upper), height_m)] - - -def _format_height(value: float) -> str: - return f"{value:.1f}" diff --git a/dimos/benchmark/vqa/generation/primitives/__init__.py b/dimos/benchmark/vqa/generation/primitives/__init__.py new file mode 100644 index 0000000000..6b00b2e258 --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/__init__.py @@ -0,0 +1,5 @@ +"""Private reusable perception primitives for one frozen VQA frame.""" + +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives + +__all__ = ["FramePerceptionPrimitives"] diff --git a/dimos/benchmark/vqa/generation/primitives/choices.py b/dimos/benchmark/vqa/generation/primitives/choices.py new file mode 100644 index 0000000000..595a8a0b72 --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/choices.py @@ -0,0 +1,23 @@ +"""Deterministic public-choice resolution from private measurements.""" + +from __future__ import annotations + +from bisect import bisect_right + + +def height_choice_window(height_m: float) -> tuple[tuple[str, ...], str]: + """Generate a local, exhaustive four-choice window around a private height.""" + breakpoints = (0.1, 0.2, 0.6, 1.0, 2.0) + start = min(max(bisect_right(breakpoints, height_m) - 1, 0), len(breakpoints) - 3) + lower, middle, upper = breakpoints[start : start + 3] + choices = ( + f"under {_format_height(lower)} m", + f"{_format_height(lower)}-{_format_height(middle)} m", + f"{_format_height(middle)}-{_format_height(upper)} m", + f"over {_format_height(upper)} m", + ) + return choices, choices[bisect_right((lower, middle, upper), height_m)] + + +def _format_height(value: float) -> str: + return f"{value:.1f}" diff --git a/dimos/benchmark/vqa/generation/primitives/contracts.py b/dimos/benchmark/vqa/generation/primitives/contracts.py new file mode 100644 index 0000000000..e5dc46c819 --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/contracts.py @@ -0,0 +1,18 @@ +"""Typed results returned by frame-scoped private perception primitives.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from dimos.benchmark.vqa.models import GroundedObject, GroundPlaneEstimate, OracleMeasurement + + +@dataclass(frozen=True) +class HeightMeasurementResult: + """A private object-height measurement or its explicit rejection.""" + + object: GroundedObject + plane: GroundPlaneEstimate + measurement: OracleMeasurement | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None diff --git a/dimos/benchmark/vqa/generation/primitives/frame.py b/dimos/benchmark/vqa/generation/primitives/frame.py new file mode 100644 index 0000000000..6b9ca07b2b --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/frame.py @@ -0,0 +1,166 @@ +"""Frame-scoped private perception primitives shared by VQA generation modes.""" + +from __future__ import annotations + +import numpy as np + +from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects +from dimos.benchmark.vqa.generation.primitives.contracts import HeightMeasurementResult +from dimos.benchmark.vqa.generation.primitives.geometry import ( + PlaneFitResult, + estimate_ground_plane, + points_in_mask, +) +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + GroundedObject, + GroundingConfig, + GroundPlaneEstimate, + ObjectDetector, + ObjectPointLocalizer, + ObjectSegmenter, + OracleMeasurement, + PointObjectSegmenter, +) +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.point import Detection2DPoint +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +class FramePerceptionPrimitives: + """Cached private perception and geometry operations over one frozen frame.""" + + def __init__( + self, + frame: CalibratedFrame, + detector: ObjectDetector, + segmenter: ObjectSegmenter, + localizer: ObjectPointLocalizer | None = None, + point_segmenter: PointObjectSegmenter | None = None, + config: GroundingConfig = GroundingConfig(), + ) -> None: + if config.min_mask_area_px < 1 or config.min_foreground_points < 1: + raise ValueError("grounding thresholds must be positive") + self.frame = frame + self._detector = detector + self._segmenter = segmenter + self._localizer = localizer + self._point_segmenter = point_segmenter + self._config = config + self._detected: dict[str, ImageDetections2D] = {} + self._segmented_queries: set[str] = set() + self._masks: dict[str, list[Detection2DSeg]] = {} + self._points: dict[str, list[Detection2DPoint]] = {} + self._groundings: dict[str, list[GroundedObject]] = {} + self._object_masks: dict[str, Detection2DSeg] = {} + self._plane_fit: PlaneFitResult | None = None + + def detect_objects(self, query: str) -> ImageDetections2D: + """Run private object detection once for a semantic query.""" + cached = self._detected.get(query) + if cached is not None: + return cached + detections = self._detector.detect(self.frame.image, query) + self._detected[query] = detections + return detections + + def segment_detections(self, query: str) -> list[Detection2DSeg]: + """Segment accepted detections, falling back to point localization when available.""" + if query in self._segmented_queries: + return self._masks.get(query, []) + detections = self.detect_objects(query) + if len(detections): + segmented = self._segmenter.segment(detections) + elif self._localizer is not None and self._point_segmenter is not None: + points = self._localizer.locate(self.frame.image, query) + self._points[query] = [item for item in points if isinstance(item, Detection2DPoint)] + segmented = self._point_segmenter.segment_points(points) + else: + segmented = detections + masks = [ + item + for item in segmented + if isinstance(item, Detection2DSeg) + and int((item.mask > 0).sum()) >= self._config.min_mask_area_px + ] + self._masks[query] = masks + self._segmented_queries.add(query) + return masks + + def ground_masks(self, query: str) -> list[GroundedObject]: + """Project visible point-cloud support through accepted masks.""" + cached = self._groundings.get(query) + if cached is not None: + return cached + masks = self.segment_detections(query) + objects = ground_segmented_objects( + self.frame, masks, min_foreground_points=self._config.min_foreground_points + ) + for item in objects: + index = _object_mask_index(item) + if index < len(masks): + self._object_masks[item.id] = masks[index] + self._groundings[query] = objects + return objects + + def used_point_localization(self, query: str) -> bool: + """Return whether segmentation for a query used positive-point localization.""" + return query in self._points + + def has_grounding(self, query: str) -> bool: + """Return whether a query has already been grounded for this frame.""" + return query in self._groundings + + @property + def can_localize_points(self) -> bool: + """Return whether point localization can supplement empty detections.""" + return self._localizer is not None and self._point_segmenter is not None + + def fit_ground_plane(self) -> PlaneFitResult: + """Fit and cache one quality-gated ground plane for the frozen frame.""" + if self._plane_fit is None: + self._plane_fit = estimate_ground_plane(self.frame) + return self._plane_fit + + def measure_height( + self, object: GroundedObject, plane: GroundPlaneEstimate + ) -> HeightMeasurementResult: + """Measure one grounded object's visible height above an accepted plane.""" + mask = self._object_masks.get(object.id) + if mask is None: + raise ValueError(f"unknown grounded object: {object.id}") + selected = points_in_mask(self.frame, mask.mask) + flags = ["visible_point_cloud_height"] + if len(selected) < 6: + flags.append("sparse_object_point_support") + return HeightMeasurementResult( + object, plane, None, tuple(flags), "insufficient_object_support" + ) + normal = np.asarray(plane.normal) + distances = selected @ normal + plane.offset_m + positive = distances[distances > 0.02] + if len(positive) < 4 or len(positive) / len(selected) < 0.6: + flags.append("partial_or_non_elevated_object_support") + return HeightMeasurementResult( + object, plane, None, tuple(flags), "ambiguous_object_extent" + ) + flags.append("conservative_upper_percentile") + measurement = OracleMeasurement( + float(np.percentile(positive, 85)), + "m", + float(max(0.05, plane.residual_m + np.std(positive) * 0.25)), + tuple(flags), + ( + f"frame:{self.frame.id}", + f"ground-plane:v1:{self.frame.id}", + f"grounding:v1:{object.id}", + ), + ) + return HeightMeasurementResult(object, plane, measurement, tuple(flags)) + + +def _object_mask_index(item: GroundedObject) -> int: + try: + return int(item.id.rsplit("-", 1)[1]) + except (IndexError, ValueError) as exc: + raise ValueError(f"grounded object ID lacks mask index: {item.id}") from exc diff --git a/dimos/benchmark/vqa/generation/measurements.py b/dimos/benchmark/vqa/generation/primitives/geometry.py similarity index 90% rename from dimos/benchmark/vqa/generation/measurements.py rename to dimos/benchmark/vqa/generation/primitives/geometry.py index b85872e7a3..5b2215f0c7 100644 --- a/dimos/benchmark/vqa/generation/measurements.py +++ b/dimos/benchmark/vqa/generation/primitives/geometry.py @@ -1,5 +1,4 @@ -# Copyright 2026 Dimensional Inc. -"""Deterministic point-cloud measurements for private VQA oracle tools.""" +"""Deterministic point-cloud geometry helpers for private VQA primitives.""" from __future__ import annotations @@ -35,8 +34,6 @@ def estimate_ground_plane(frame: CalibratedFrame) -> PlaneFitResult: if len(candidates) < min_points: return PlaneFitResult(None, ("insufficient_ground_band_points",), "insufficient_support") - # Open3D owns the randomized consensus search; seed its process-global RNG - # so repeated generation runs are stable for a fixed Open3D release. import open3d as o3d o3d.utility.random.seed(0) @@ -54,7 +51,6 @@ def estimate_ground_plane(frame: CalibratedFrame) -> PlaneFitResult: _, _, right = np.linalg.svd(inlier_points - center, full_matrices=False) normal = right[-1] offset = -float(normal @ center) - # The camera is normally above the support plane; this fixes signed heights. if offset < 0: normal, offset = -normal, -offset residuals = np.abs(candidates @ normal + offset) diff --git a/dimos/benchmark/vqa/generation/selection.py b/dimos/benchmark/vqa/generation/primitives/selection.py similarity index 82% rename from dimos/benchmark/vqa/generation/selection.py rename to dimos/benchmark/vqa/generation/primitives/selection.py index 3c32b1d70b..a9263ab1fc 100644 --- a/dimos/benchmark/vqa/generation/selection.py +++ b/dimos/benchmark/vqa/generation/primitives/selection.py @@ -1,5 +1,4 @@ -# Copyright 2026 Dimensional Inc. -"""Shared deterministic selection over grounded object evidence.""" +"""Deterministic selection over grounded object evidence.""" from __future__ import annotations diff --git a/dimos/benchmark/vqa/generation/questions.py b/dimos/benchmark/vqa/generation/questions.py index 35c6866326..d442232972 100644 --- a/dimos/benchmark/vqa/generation/questions.py +++ b/dimos/benchmark/vqa/generation/questions.py @@ -16,7 +16,7 @@ from __future__ import annotations -from dimos.benchmark.vqa.generation.selection import select_nearest_object +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object from dimos.benchmark.vqa.models import GroundedObject, VqaExample diff --git a/dimos/benchmark/vqa/generation/test_agents.py b/dimos/benchmark/vqa/generation/test_agents.py index bda560a164..468461cf00 100644 --- a/dimos/benchmark/vqa/generation/test_agents.py +++ b/dimos/benchmark/vqa/generation/test_agents.py @@ -7,6 +7,7 @@ import numpy as np from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives from dimos.benchmark.vqa.generation.question_agent import OpenAIQuestionAgent from dimos.benchmark.vqa.models import CalibratedFrame, GroundingConfig, QuestionIntent from dimos.models.vl.openai import OpenAIVlModel @@ -44,6 +45,20 @@ def detect(self, image: Image, query: str) -> ImageDetections2D: return ImageDetections2D(self._image, self._detections if query == "chair" else []) +def _agent( + frame: CalibratedFrame, + detector: _Detector | _MultiDetector, + segmenter: _Segmenter, + *, + localizer: _PointLocalizer | None = None, + point_segmenter: _PointSegmenter | None = None, + config: GroundingConfig = GroundingConfig(), +) -> VqaGroundTruthGenerator: + return VqaGroundTruthGenerator( + FramePerceptionPrimitives(frame, detector, segmenter, localizer, point_segmenter, config) + ) + + class _Segmenter: def segment(self, detections: ImageDetections2D) -> ImageDetections2D: return detections @@ -95,8 +110,11 @@ def test_question_agent_returns_constrained_intents() -> None: def test_ground_truth_agent_records_tools_and_rejects_unsupported_question() -> None: frame, detection = _frame_and_detection() - agent = VqaGroundTruthGenerator( - _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), ) answered = agent.answer( @@ -122,8 +140,11 @@ def test_ground_truth_agent_records_tools_and_rejects_unsupported_question() -> def test_ground_truth_agent_rejects_small_masks() -> None: frame, detection = _frame_and_detection() - agent = VqaGroundTruthGenerator( - _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=37) + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=37), ) result = agent.answer(frame, QuestionIntent(kind="presence", object_query="chair")) @@ -143,7 +164,8 @@ def test_ground_truth_agent_falls_back_to_point_prompt() -> None: detection.image, detection.mask, ) - agent = VqaGroundTruthGenerator( + agent = _agent( + frame, _Detector(frame.image, detection), _Segmenter(), localizer=_PointLocalizer(), @@ -192,8 +214,11 @@ def test_ground_truth_agent_compares_nearest_objects_by_side() -> None: Detection2DSeg((0.0, 0.0, 2.0, 5.0), 0, -1, 1.0, "chair", 0.0, image, left_mask), Detection2DSeg((3.0, 0.0, 5.0, 5.0), 1, -1, 1.0, "chair", 0.0, image, right_mask), ] - agent = VqaGroundTruthGenerator( - _MultiDetector(image, detections), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) + agent = _agent( + frame, + _MultiDetector(image, detections), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), ) result = agent.answer( @@ -207,8 +232,11 @@ def test_ground_truth_agent_compares_nearest_objects_by_side() -> None: def test_ground_truth_agent_rejects_side_comparison_without_both_sides() -> None: frame, detection = _frame_and_detection() - agent = VqaGroundTruthGenerator( - _Detector(frame.image, detection), _Segmenter(), config=GroundingConfig(min_mask_area_px=1) + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), ) result = agent.answer( diff --git a/dimos/benchmark/vqa/generation/test_oracle.py b/dimos/benchmark/vqa/generation/test_oracle.py index 1405ded30a..4b1a8b3917 100644 --- a/dimos/benchmark/vqa/generation/test_oracle.py +++ b/dimos/benchmark/vqa/generation/test_oracle.py @@ -9,16 +9,15 @@ import numpy as np from dimos.benchmark.vqa.generation.geometry import project_visible_points -from dimos.benchmark.vqa.generation.measurements import estimate_ground_plane from dimos.benchmark.vqa.generation.oracle import ( PrivateToolCallingOracle, SemanticEvidenceValidation, validate_oracle_answer, ) -from dimos.benchmark.vqa.generation.oracle_tools import ( - LocalOracleToolRegistry, - _height_choice_window, -) +from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.generation.primitives.choices import height_choice_window +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.geometry import estimate_ground_plane from dimos.benchmark.vqa.generation.question_agent import ( AGENTIC_QUESTION_PROMPT, OpenAIFreeformQuestionAuthor, @@ -28,6 +27,7 @@ CalibratedFrame, ChoiceAnswerContract, DeferredHeightChoiceContract, + GroundingConfig, OracleEvidence, OracleToolResult, QuestionProposal, @@ -37,6 +37,8 @@ from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo from dimos.msgs.sensor_msgs.Image import Image from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg class _QuestionModel: @@ -46,13 +48,15 @@ def query(self, image: Image, prompt: str) -> str: class _Grounding: - def detect_objects(self, frame: Any, query: str) -> list[Any]: + frame = cast("Any", object()) + + def detect_objects(self, query: str) -> list[Any]: return [] - def segment_detections(self, frame: Any, query: str) -> list[Any]: + def segment_detections(self, query: str) -> list[Any]: return [] - def ground_masks(self, frame: Any, query: str) -> list[Any]: + def ground_masks(self, query: str) -> list[Any]: return [ type( "Object", @@ -67,20 +71,6 @@ def ground_masks(self, frame: Any, query: str) -> list[Any]: )() ] - def masks_for_query(self, query: str) -> tuple[Any, ...]: - return () - - -class _GroundingWithMask(_Grounding): - def __init__(self, mask: np.ndarray) -> None: - self._mask = mask - - def masks_for_query(self, query: str) -> tuple[Any, ...]: - return (type("Mask", (), {"mask": self._mask})(),) - - def segment_detections(self, frame: Any, query: str) -> list[Any]: - return list(self.masks_for_query(query)) - def _measurement_frame(points: np.ndarray | None = None) -> CalibratedFrame: ground = [[x, 1.0, z] for z in (3.0, 4.0, 5.0) for x in (-1.0, -0.5, 0.0, 0.5, 1.0)] @@ -100,6 +90,39 @@ def _measurement_frame(points: np.ndarray | None = None) -> CalibratedFrame: ) +class _MaskDetector: + def __init__(self, mask: np.ndarray) -> None: + self._mask = mask + + def detect(self, image: Image, query: str) -> ImageDetections2D: + detection = Detection2DSeg( + (0.0, 0.0, float(image.width - 1), float(image.height - 1)), + 0, + -1, + 1.0, + query, + 0.0, + image, + self._mask, + ) + return ImageDetections2D(image, [detection]) + + +class _IdentitySegmenter: + def segment(self, detections: Any) -> Any: + return detections + + +def _frame_primitives( + frame: CalibratedFrame, mask: np.ndarray | None = None +) -> FramePerceptionPrimitives: + if mask is None: + mask = np.full((frame.image.height, frame.image.width), 255, dtype=np.uint8) + return FramePerceptionPrimitives( + frame, _MaskDetector(mask), _IdentitySegmenter(), config=GroundingConfig(min_mask_area_px=1) + ) + + class _BoundModel: def __init__(self) -> None: self._calls = 0 @@ -260,7 +283,7 @@ def query(self, image: Image, prompt: str) -> str: def test_local_tool_returns_geometry_and_evidence_ids() -> None: - registry = LocalOracleToolRegistry(cast("Any", object()), cast("Any", _Grounding())) + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) detection = json.loads(registry.detect_objects("chair")) masks = json.loads(registry.segment_detections(detection["detection_id"])) @@ -290,7 +313,7 @@ def test_ground_plane_estimator_fits_visible_lower_band() -> None: def test_ground_plane_tool_returns_quality_gated_rejection() -> None: frame = _measurement_frame(np.asarray([[0.0, 1.0, 3.0]], dtype=np.float32)) - registry = LocalOracleToolRegistry(frame, cast("Any", _Grounding())) + registry = LocalOracleToolRegistry(_frame_primitives(frame)) payload = json.loads(registry.fit_ground_plane()) @@ -306,7 +329,7 @@ def test_height_tool_measures_visible_object_points_above_plane() -> None: for (x, y), point in zip(projected.pixels, projected.camera_points, strict=True): if point[0] > 1.5: mask[y, x] = 255 - registry = LocalOracleToolRegistry(frame, cast("Any", _GroundingWithMask(mask))) + registry = LocalOracleToolRegistry(_frame_primitives(frame, mask)) detection = json.loads(registry.detect_objects("chair")) masks = json.loads(registry.segment_detections(detection["detection_id"])) @@ -322,7 +345,7 @@ def test_height_tool_measures_visible_object_points_above_plane() -> None: def test_local_registry_exposes_geometry_tools() -> None: - registry = LocalOracleToolRegistry(cast("Any", object()), cast("Any", _Grounding())) + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) assert {tool.name for tool in registry.tools()} == { "detect_objects", @@ -336,7 +359,7 @@ def test_local_registry_exposes_geometry_tools() -> None: def test_height_choice_window_is_local_and_deterministic() -> None: - choices, answer = _height_choice_window(0.42) + choices, answer = height_choice_window(0.42) assert choices == ( "under 0.2 m", @@ -345,7 +368,7 @@ def test_height_choice_window_is_local_and_deterministic() -> None: "over 1.0 m", ) assert answer == "0.2-0.6 m" - assert _height_choice_window(3.0)[1] == "over 2.0 m" + assert height_choice_window(3.0)[1] == "over 2.0 m" def test_oracle_validates_evidence_and_answer_contract() -> None: @@ -393,7 +416,7 @@ def test_oracle_derives_deferred_height_answer_from_measurement_bucket() -> None def test_private_oracle_runs_direct_structured_tool() -> None: proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) - registry = LocalOracleToolRegistry(cast("Any", object()), cast("Any", _Grounding())) + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) validator = _SemanticValidator(SemanticEvidenceValidation(True, "chair grounding supports yes")) result = PrivateToolCallingOracle( @@ -428,7 +451,7 @@ def invoke(self, messages: Any) -> AIMessage: proposal = QuestionProposal( "q", "How tall is the chair?", ChoiceAnswerContract(("under 0.5 m", "0.5-1.0 m")) ) - registry = LocalOracleToolRegistry(cast("Any", object()), cast("Any", _Grounding())) + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) validator = _SemanticValidator( SemanticEvidenceValidation(False, "range and side do not measure height") ) @@ -446,7 +469,7 @@ def answer(self, frame: Any, intent: Any) -> None: raise AssertionError("agentic oracle must not call legacy answer") proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) - registry = LocalOracleToolRegistry(cast("Any", object()), _GroundingWithoutLegacyAnswer()) + registry = LocalOracleToolRegistry(cast("Any", _GroundingWithoutLegacyAnswer())) validator = _SemanticValidator(SemanticEvidenceValidation(True, "chair grounding supports yes")) result = PrivateToolCallingOracle( diff --git a/dimos/benchmark/vqa/generation/test_selection.py b/dimos/benchmark/vqa/generation/test_selection.py index fa6d7df077..2218a72df8 100644 --- a/dimos/benchmark/vqa/generation/test_selection.py +++ b/dimos/benchmark/vqa/generation/test_selection.py @@ -1,6 +1,6 @@ # Copyright 2026 Dimensional Inc. -from dimos.benchmark.vqa.generation.selection import select_nearest_object +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object from dimos.benchmark.vqa.models import GroundedObject diff --git a/dimos/cli/vqa.py b/dimos/cli/vqa.py index 1416307e25..9c447d170b 100644 --- a/dimos/cli/vqa.py +++ b/dimos/cli/vqa.py @@ -17,6 +17,7 @@ from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator from dimos.benchmark.vqa.generation.oracle import create_openai_oracle from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives from dimos.benchmark.vqa.generation.question_agent import ( OpenAIFreeformQuestionAuthor, OpenAIQuestionAgent, @@ -77,7 +78,8 @@ def single_frame( else question_agent.propose(frame.image) ) typer.echo(f"Grounding {len(intents)} questions for frame {frame_index}") - ground_truth = VqaGroundTruthGenerator( + primitives = FramePerceptionPrimitives( + frame, detector := MoondreamObjectDetector(model), segmenter := EdgeTamObjectSegmenter(EdgeTAMImageSegmenter()), localizer=detector, @@ -87,6 +89,7 @@ def single_frame( min_foreground_points=min_foreground_points, ), ) + ground_truth = VqaGroundTruthGenerator(primitives) results: list[GroundTruthResult] | list[AcceptedOracleResult | RejectedOracleResult] = ( _answer_agentic(ground_truth, frame, cast("list[QuestionProposal]", intents)) if question_mode == "agentic" @@ -172,7 +175,8 @@ def generate( else question_agent.propose(frame.image) ) typer.echo(f"Frame {frame_index}: grounding {len(intents)} questions") - ground_truth = VqaGroundTruthGenerator( + primitives = FramePerceptionPrimitives( + frame, detector, segmenter, localizer=detector, @@ -181,6 +185,7 @@ def generate( min_mask_area_px=min_mask_area_px, min_foreground_points=min_foreground_points ), ) + ground_truth = VqaGroundTruthGenerator(primitives) results: list[GroundTruthResult] | list[AcceptedOracleResult | RejectedOracleResult] = ( _answer_agentic(ground_truth, frame, cast("list[QuestionProposal]", intents)) if question_mode == "agentic" @@ -246,7 +251,7 @@ def _answer_agentic( results: list[AcceptedOracleResult | RejectedOracleResult] = [] for number, proposal in enumerate(proposals, start=1): typer.echo(f"Agentic question {number}/{len(proposals)}: {proposal.question}") - result = oracle.answer(proposal, LocalOracleToolRegistry(frame, ground_truth)) + result = oracle.answer(proposal, LocalOracleToolRegistry(ground_truth.primitives)) results.append(result) if isinstance(result, AcceptedOracleResult): typer.echo(f"Agentic question {number}/{len(proposals)} accepted") diff --git a/docs/benchmarking/vqa-generation/infrastructure.md b/docs/benchmarking/vqa-generation/infrastructure.md new file mode 100644 index 0000000000..aca2cf9fb2 --- /dev/null +++ b/docs/benchmarking/vqa-generation/infrastructure.md @@ -0,0 +1,157 @@ +--- +title: "VQA Generation Infrastructure" +--- + +# VQA Generation Infrastructure + +This document describes the software stack behind the point-cloud-grounded VQA benchmark. For the +step-by-step generation procedure and dataset schema, see [Pipeline](/docs/benchmarking/vqa-generation/pipeline.md). + +## Design Boundary + +The benchmark has two intentionally separate phases: + +1. Generation privately uses calibrated point clouds and perception tools to establish labels. +2. Evaluation receives only public images, questions, choices, and private expected-choice labels. + +The evaluated vision model never receives point clouds, camera calibration, masks, measurements, +tool traces, or rejected attempts. + +```text +recording + calibration + point cloud + | + v + private generation runtime + | + +-- public: image.jpg, cases.jsonl + +-- private: labels.jsonl, ground_truth.json + | + v + image-only evaluation runtime + | + v + run.json + vqa-results.json +``` + +## Package Layout + +```text +dimos/benchmark/vqa/ + models.py shared immutable VQA contracts + generation/ + primitives/ + frame.py cached frame-scoped perception runtime + contracts.py typed private primitive results + geometry.py plane fitting and masked-point helpers + selection.py nearest-object selection + choices.py deterministic answer-choice resolution + ground_truth_generator.py deterministic constrained recipe runner + oracle_tools.py opaque-ID and LangChain adapter for primitives + oracle.py bounded tool-calling and evidence validation + question_agent.py image-only question author + dataset.py frame records and evaluation export + evaluation.py shared point-cloud-vqa Evaluation plugin + +dimos/cli/vqa.py generation CLI commands +``` + +## Generation Runtime + +`dimos vqa generate` and `dimos vqa single-frame` load a frozen Go2 recording frame containing: + +- A rectified RGB image. +- A calibrated visible point cloud. +- Camera intrinsics and the point-cloud-to-camera transform. + +The generator constructs one `FramePerceptionPrimitives` instance per frame. It owns MoonDream and +EdgeTAM calls, intermediate-result caches, grounded masks, and the accepted ground-plane fit. +Projected visible point-cloud samples establish whether a mask has enough foreground support to +become a grounded object. The generation runtime writes complete frame directories, so multi-frame +generation can skip completed frames after an interrupted run. + +### Shared Perception Primitives + +Constrained and agentic generation use the same private primitives: + +```text +detect_objects(query) +-> segment_detections(detection_id) +-> ground_masks(mask_id) +-> select_nearest_object(object_ids, side) +-> fit_ground_plane() +-> measure_height(object_id, plane_id) +-> bucket_measurement(measurement_id) +``` + +Constrained generation selects a fixed sequence for each question family. Agentic generation lets +the oracle select a bounded sequence of these tools, passing opaque IDs rather than masks or point +arrays between calls. + +## Question And Answer Contracts + +The image author sees only RGB and proposes a question plus one of these contracts: + +- Boolean: the final public choices are `yes` and `no`. +- Fixed choice: the author supplies at least two public choices. +- Deferred height choice: the author freezes the question, then private geometry deterministically + creates the public height choices and matching answer after a valid measurement. + +The deferred height contract prevents the image author from guessing a numeric range. The private +`height-window-v1` policy derives four local, exhaustive choices around a measured height. The raw +measurement, uncertainty, plane fit, and support remain private. + +## Private Validation + +Every accepted agentic answer must have valid cited evidence. The deterministic validator checks: + +- Cited IDs were emitted by private tools for the current frame. +- The answer is allowed by the resolved public contract. +- A deferred height answer exactly matches the cited measurement bucket. + +A private semantic validator then determines whether the cited structured evidence supports the +question and answer. Invalid calls, inadequate geometry, bad citations, and unsupported answers +are retained as private rejected records rather than exported as evaluation cases. + +## Dataset Assembly + +Each completed frame retains the public `image.jpg`, resumability metadata, private generation +audit, and temporary per-frame case/label rows. `write_dataset_manifest()` aggregates those rows +into root `cases.jsonl` and `labels.jsonl` files. + +The root dataset is deliberately minimal: + +```text +cases.jsonl public case ID, image path, question, and choices +labels.jsonl private case ID and expected choice +frame-*/image.jpg +``` + +`ground_truth.json` is not an evaluator dependency. It exists solely to make generated labels +auditable. + +## Evaluation Runtime + +`point-cloud-vqa` is registered in the shared evaluation framework and is invoked through: + +```bash +dimos eval run --output +``` + +For each case, the evaluator loads the referenced public image, sends the question and choices to +the configured vision model, normalizes an `ANSWER: ` response, and compares it with the +private label. It writes `vqa-results.json` with the expected answer, normalized model answer, raw +model response, and pass/fail result. The shared evaluation runtime also writes its immutable +`run.json` record. + +## Operational Dependencies + +Generation requires local CUDA support for EdgeTAM and credentials for the image author and, in +agentic mode, the private oracle and semantic validator. Evaluation requires only the selected +vision-model credentials and the exported public images/cases plus private labels; it has no +perception-model, CUDA, recording, or point-cloud dependency. + +## Future Work + +Evaluation visualization is intentionally separate from generation. The planned report should be +derived from evaluator inputs and `vqa-results.json` only, so it can show public images, questions, +choices, predictions, labels, and pass/fail results without exposing private generation evidence. diff --git a/docs/docs.json b/docs/docs.json index eda15b0eee..d934ef899b 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -126,7 +126,8 @@ "pages": [ "capabilities/perception/index", "benchmarking/vqa-generation/README", - "benchmarking/vqa-generation/pipeline" + "benchmarking/vqa-generation/pipeline", + "benchmarking/vqa-generation/infrastructure" ] }, { From ca1e701040103be01c4e1ab2920a9cfb22e067e5 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Mon, 10 Aug 2026 17:41:54 -0700 Subject: [PATCH 08/12] feat(benchmark): extend VQA generation capabilities --- .../vqa/generation/ground_truth_generator.py | 35 ++++++ .../benchmark/vqa/generation/oracle_tools.py | 44 +++++++ .../vqa/generation/primitives/__init__.py | 5 - .../vqa/generation/primitives/contracts.py | 12 ++ .../vqa/generation/primitives/frame.py | 44 ++++++- .../vqa/generation/primitives/geometry.py | 74 +++++++++++ .../vqa/generation/question_agent.py | 39 ++++-- .../benchmark/vqa/generation/specification.py | 30 +++++ dimos/benchmark/vqa/generation/test_agents.py | 29 +++++ dimos/benchmark/vqa/generation/test_oracle.py | 23 +++- dimos/benchmark/vqa/models.py | 2 +- dimos/cli/test_vqa.py | 56 +++++++++ dimos/cli/vqa.py | 116 ++++++++++++++++-- docs/benchmarking/vqa-generation/README.md | 85 ------------- .../vqa-generation/infrastructure.md | 6 +- docs/benchmarking/vqa-generation/pipeline.md | 35 +++++- docs/docs.json | 1 - 17 files changed, 515 insertions(+), 121 deletions(-) delete mode 100644 dimos/benchmark/vqa/generation/primitives/__init__.py create mode 100644 dimos/benchmark/vqa/generation/specification.py delete mode 100644 docs/benchmarking/vqa-generation/README.md diff --git a/dimos/benchmark/vqa/generation/ground_truth_generator.py b/dimos/benchmark/vqa/generation/ground_truth_generator.py index d37f7e9fda..5db08c84ea 100644 --- a/dimos/benchmark/vqa/generation/ground_truth_generator.py +++ b/dimos/benchmark/vqa/generation/ground_truth_generator.py @@ -67,6 +67,8 @@ def _answer_from_objects( ) if intent.kind == "compare_nearest_by_side": return _compare_nearest_by_side(frame, intent, objects, trace) + if intent.kind == "door_state": + return _classify_door_state(frame, intent, objects, trace, self.primitives) examples = generate_questions( frame.id, objects, [intent.object_query], distance_m=intent.threshold_m or 3.0 ) @@ -101,6 +103,8 @@ def _render_question(intent: QuestionIntent) -> str: return f"Where is the nearest {intent.object_query}: left, center, or right?" if intent.kind == "compare_nearest_by_side": return f"Which {intent.object_query} is closer: the left one or the right one?" + if intent.kind == "door_state": + return f"Is the {intent.object_query} open or closed?" return f"Is the nearest {intent.object_query} within {intent.threshold_m or 3:g} meters? Answer yes or no." @@ -128,6 +132,37 @@ def _compare_nearest_by_side( return GroundTruthResult(intent, example, "answered", answer, None, (left, right), trace) +def _classify_door_state( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, +) -> GroundTruthResult: + if "door" not in intent.object_query.lower(): + return _rejected_result(frame, intent, objects, trace, "door_state_requires_door_query") + if len(objects) != 1: + return _rejected_result(frame, intent, objects, trace, "ambiguous_door_instances") + result = primitives.classify_door_state(objects[0]) + trace = ( + *trace, + ToolTrace("classify_door_state", result.state or result.rejection_reason or "rejected"), + ) + if result.state is None: + return _rejected_result( + frame, intent, objects, trace, result.rejection_reason or "door_state_rejected" + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-state", + _render_question(intent), + result.state, + "choice", + (objects[0].id,), + ("open", "closed"), + ) + return GroundTruthResult(intent, example, "answered", result.state, None, tuple(objects), trace) + + def _rejected_result( frame: CalibratedFrame, intent: QuestionIntent, diff --git a/dimos/benchmark/vqa/generation/oracle_tools.py b/dimos/benchmark/vqa/generation/oracle_tools.py index 623249eb80..d77c5f4e03 100644 --- a/dimos/benchmark/vqa/generation/oracle_tools.py +++ b/dimos/benchmark/vqa/generation/oracle_tools.py @@ -77,6 +77,14 @@ def tools(self) -> list[StructuredTool]: "Measure one opaque grounded object above one opaque accepted ground-plane ID." ), ), + StructuredTool.from_function( + self.classify_door_state, + name="classify_door_state", + description=( + "Classify one opaque grounded door as open or closed from point-cloud planes. " + "Rejects insufficient or ambiguous geometry." + ), + ), StructuredTool.from_function( self.bucket_measurement, name="bucket_measurement", @@ -218,6 +226,42 @@ def measure_height(self, object_id: str, plane_id: str) -> str: self._results.append(result) return json.dumps(_tool_payload(result, measurement_id=measurement_id)) + def classify_door_state(self, object_id: str) -> str: + """Classify one grounded door against the surrounding point-cloud plane.""" + object = self._objects.get(object_id) + if object is None: + return self._record_rejection("classify_door_state", object_id, [], "unknown_object_id") + if "door" not in object.label.lower(): + return self._record_rejection( + "classify_door_state", object.label, [], "door_state_requires_door_query" + ) + result = self._primitives.classify_door_state(object) + if result.state is None: + return self._record_rejection( + "classify_door_state", + object.label, + list(result.quality_flags), + result.rejection_reason, + ) + evidence = OracleEvidence( + f"door-state:v1:{object.id}", + "v1", + object.id, + object.label, + object.range_m, + object.horizontal_direction, + object.point_count, + ) + tool_result = OracleToolResult( + "classify_door_state", + object.label, + (evidence,), + choice=result.state, + quality_flags=result.quality_flags, + ) + self._results.append(tool_result) + return json.dumps(_tool_payload(tool_result)) + def bucket_measurement(self, measurement_id: str) -> str: """Map one accepted private height measurement to its public choice.""" source = self._measurements.get(measurement_id) diff --git a/dimos/benchmark/vqa/generation/primitives/__init__.py b/dimos/benchmark/vqa/generation/primitives/__init__.py deleted file mode 100644 index 6b00b2e258..0000000000 --- a/dimos/benchmark/vqa/generation/primitives/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Private reusable perception primitives for one frozen VQA frame.""" - -from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives - -__all__ = ["FramePerceptionPrimitives"] diff --git a/dimos/benchmark/vqa/generation/primitives/contracts.py b/dimos/benchmark/vqa/generation/primitives/contracts.py index e5dc46c819..2630271910 100644 --- a/dimos/benchmark/vqa/generation/primitives/contracts.py +++ b/dimos/benchmark/vqa/generation/primitives/contracts.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Literal from dimos.benchmark.vqa.models import GroundedObject, GroundPlaneEstimate, OracleMeasurement @@ -16,3 +17,14 @@ class HeightMeasurementResult: measurement: OracleMeasurement | None quality_flags: tuple[str, ...] rejection_reason: str | None = None + + +@dataclass(frozen=True) +class DoorStateResult: + """A conservative point-cloud classification of one door's state.""" + + object: GroundedObject + state: Literal["open", "closed"] | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + angle_deg: float | None = None diff --git a/dimos/benchmark/vqa/generation/primitives/frame.py b/dimos/benchmark/vqa/generation/primitives/frame.py index 6b9ca07b2b..aad6c420ab 100644 --- a/dimos/benchmark/vqa/generation/primitives/frame.py +++ b/dimos/benchmark/vqa/generation/primitives/frame.py @@ -5,10 +5,16 @@ import numpy as np from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects -from dimos.benchmark.vqa.generation.primitives.contracts import HeightMeasurementResult +from dimos.benchmark.vqa.generation.primitives.contracts import ( + DoorStateResult, + HeightMeasurementResult, +) from dimos.benchmark.vqa.generation.primitives.geometry import ( PlaneFitResult, + classify_door_plane_angle, estimate_ground_plane, + fit_surface_plane, + points_around_mask, points_in_mask, ) from dimos.benchmark.vqa.models import ( @@ -158,6 +164,42 @@ def measure_height( ) return HeightMeasurementResult(object, plane, measurement, tuple(flags)) + def classify_door_state(self, object: GroundedObject) -> DoorStateResult: + """Classify a door as open or closed from its plane relative to nearby structure.""" + mask = self._object_masks.get(object.id) + if mask is None: + raise ValueError(f"unknown grounded object: {object.id}") + door_fit = fit_surface_plane(points_in_mask(self.frame, mask.mask)) + if door_fit.estimate is None: + return DoorStateResult( + object, + None, + ("door_plane_rejected", *door_fit.quality_flags), + door_fit.rejection_reason, + ) + surrounding_fit = fit_surface_plane(points_around_mask(self.frame, mask.mask)) + if surrounding_fit.estimate is None: + return DoorStateResult( + object, + None, + ("surrounding_plane_rejected", *surrounding_fit.quality_flags), + surrounding_fit.rejection_reason, + ) + state, reason, angle_deg = classify_door_plane_angle( + door_fit.estimate, surrounding_fit.estimate + ) + return DoorStateResult( + object, + state, + ( + "door_and_surrounding_planes_accepted", + *door_fit.quality_flags, + *surrounding_fit.quality_flags, + ), + reason, + angle_deg, + ) + def _object_mask_index(item: GroundedObject) -> int: try: diff --git a/dimos/benchmark/vqa/generation/primitives/geometry.py b/dimos/benchmark/vqa/generation/primitives/geometry.py index 5b2215f0c7..75da5dc9eb 100644 --- a/dimos/benchmark/vqa/generation/primitives/geometry.py +++ b/dimos/benchmark/vqa/generation/primitives/geometry.py @@ -4,6 +4,7 @@ from dataclasses import dataclass +import cv2 import numpy as np from dimos.benchmark.vqa.generation.geometry import project_visible_points @@ -87,3 +88,76 @@ def points_in_mask(frame: CalibratedFrame, mask: np.ndarray) -> np.ndarray: ], dtype=np.float64, ) + + +def points_around_mask(frame: CalibratedFrame, mask: np.ndarray, radius_px: int = 12) -> np.ndarray: + """Return visible points in an annulus surrounding one foreground mask.""" + if radius_px < 1: + raise ValueError("mask ring radius must be positive") + if mask.shape != (frame.image.height, frame.image.width): + raise ValueError("segmentation mask dimensions must match the image") + foreground = mask > 0 + expanded = cv2.dilate( + foreground.astype(np.uint8), + np.ones((radius_px * 2 + 1, radius_px * 2 + 1), dtype=np.uint8), + ).astype(bool) + return points_in_mask(frame, expanded & ~foreground) + + +def fit_surface_plane(points: np.ndarray) -> PlaneFitResult: + """Fit a robust plane to private surface points without assuming ground orientation.""" + min_points, min_inliers, threshold = 12, 10, 0.06 + if len(points) < min_points: + return PlaneFitResult(None, ("insufficient_surface_points",), "insufficient_support") + + import open3d as o3d + + o3d.utility.random.seed(0) + point_cloud = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(points)) + _, inlier_indices = point_cloud.segment_plane( + distance_threshold=threshold, + ransac_n=3, + num_iterations=816, + ) + if len(inlier_indices) < min_inliers: + return PlaneFitResult(None, ("insufficient_surface_inliers",), "insufficient_inliers") + + inlier_points = points[np.asarray(inlier_indices, dtype=int)] + center = np.mean(inlier_points, axis=0) + _, _, right = np.linalg.svd(inlier_points - center, full_matrices=False) + normal = right[-1] + offset = -float(normal @ center) + residuals = np.abs(points @ normal + offset) + inliers = residuals <= threshold + residual_m = ( + float(np.sqrt(np.mean(np.square(residuals[inliers])))) if inliers.any() else float("inf") + ) + if int(inliers.sum()) < min_inliers: + return PlaneFitResult( + None, ("insufficient_refined_surface_inliers",), "insufficient_inliers" + ) + if residual_m > 0.035: + return PlaneFitResult(None, ("high_surface_residual",), "residual_too_high") + return PlaneFitResult( + GroundPlaneEstimate( + tuple(float(value) for value in normal), + float(offset), + len(points), + int(inliers.sum()), + residual_m, + ), + ("surface_plane_accepted",), + ) + + +def classify_door_plane_angle( + door: GroundPlaneEstimate, surrounding: GroundPlaneEstimate +) -> tuple[str | None, str | None, float]: + """Classify only clearly coplanar or rotated door and surrounding planes.""" + alignment = abs(float(np.dot(door.normal, surrounding.normal))) + angle_deg = float(np.degrees(np.arccos(np.clip(alignment, -1.0, 1.0)))) + if angle_deg <= 12.0: + return "closed", None, angle_deg + if angle_deg >= 25.0: + return "open", None, angle_deg + return None, "ambiguous_door_angle", angle_deg diff --git a/dimos/benchmark/vqa/generation/question_agent.py b/dimos/benchmark/vqa/generation/question_agent.py index fb62f89b97..389c85e419 100644 --- a/dimos/benchmark/vqa/generation/question_agent.py +++ b/dimos/benchmark/vqa/generation/question_agent.py @@ -18,11 +18,18 @@ from dimos.models.vl.openai import OpenAIVlModel from dimos.msgs.sensor_msgs.Image import Image -QUESTION_PROMPT = """You generate challenging but visually answerable single-frame VQA questions. -Inspect only this image. Do not assume depth, point clouds, metadata, or temporal context. -Return JSON only: an array of at most 5 visible, salient object names as strings. -Do not return floors, walls, ceilings, background surfaces, questions, explanations, Markdown, -or information not visible in the image.""" +QUESTION_PROMPT = """Select up to 5 challenging but visually well-supported single-frame VQA intents. +Inspect only this image. Do not assume depth, point clouds, calibration, metadata, or temporal context. +Return JSON only: an array of objects with kind, object_query, and threshold_m only for +within_distance. kind must be one of presence, horizontal_direction, within_distance, or +compare_nearest_by_side, or door_state. Use threshold_m: 3.0 for within_distance. +Use image context to select only intents likely to produce a useful geometric case: emit +compare_nearest_by_side only when at least two visible instances of the same object appear on +opposite image sides; emit horizontal_direction only for a visible object; and prefer relational +or directional intents over presence. Emit door_state only for a clearly visible door with nearby +visible structure. Diversify object classes and question families when the scene supports them. +Do not return bare object names, floors, walls, ceilings, background surfaces, +questions, answers, explanations, Markdown, or information not visible in the image.""" AGENTIC_QUESTION_PROMPT = """Author up to 5 challenging, visually answerable single-frame VQA questions. Inspect only this image. Do not use or infer depth, point clouds, calibration, metadata, or answers. @@ -31,14 +38,19 @@ or {"kind":"deferred_height_choice","strategy":"height-window-v1"}. Prioritize questions that a private point-cloud oracle can validate. For the height of one upright object resting on visible ground, use deferred_height_choice. Its choices are generated privately -from a successful height measurement. Also prefer which of -two named objects is closer (choice, with those object names as choices), object count, left/right -spatial relation, and distance-threshold questions. Use object_queries for every referenced object -and tool_hints from "detect_objects", "segment_detections", "ground_masks", "fit_ground_plane", -"select_nearest_object", "measure_height", or "bucket_measurement" when applicable. -Use visibility/presence questions only when no stronger geometric question is available. Do not ask -about color, material, text, intent, full physical size, hidden parts, or terrain. Use only visible -objects. Do not include answers, explanations, Markdown, or background surfaces.""" +from a successful height measurement. Aim for a diverse set of questions that use the visible scene +composition: relative left/center/right position, which of two named object types is closer, count +choices for a visibly repeated object, and height for an upright grounded object. Use concise, +mutually exclusive fixed choices with two to four options. For a clearly visible door with nearby +visible structure, you may ask whether it is open or closed with exactly ["open", "closed"]. Use +object_queries for every referenced +object and tool_hints from "detect_objects", "segment_detections", "ground_masks", +"fit_ground_plane", "select_nearest_object", "measure_height", "classify_door_state", or +"bucket_measurement" when applicable. Use visibility/presence questions only when no stronger +geometric question is available. +Do not ask about color, material, text, intent, full physical size, hidden parts, or exact +metric distances without a supplied choice contract. Use only visible objects. Do not include +answers, explanations, Markdown, or background surfaces.""" _UNSUPPORTED_QUERIES = {"background", "ceiling", "floor", "ground", "room", "wall"} @@ -77,6 +89,7 @@ def propose(self, image: Image) -> list[QuestionIntent]: "horizontal_direction", "within_distance", "compare_nearest_by_side", + "door_state", ): raise ValueError(f"unsupported question kind: {kind!r}") if not isinstance(query, str) or not query: diff --git a/dimos/benchmark/vqa/generation/specification.py b/dimos/benchmark/vqa/generation/specification.py new file mode 100644 index 0000000000..f0c9ce20af --- /dev/null +++ b/dimos/benchmark/vqa/generation/specification.py @@ -0,0 +1,30 @@ +"""Validated input configuration for resumable VQA dataset generation.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class VqaGroundingSpecification(BaseModel): + """Private grounding quality thresholds for generated VQA labels.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + min_mask_area_px: int = Field(default=128, ge=1) + min_foreground_points: int = Field(default=3, ge=1) + + +class VqaGenerationSpecification(BaseModel): + """One reproducible multi-frame VQA generation request.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + recording: str = Field(min_length=1) + start_index: int = Field(default=0, ge=0) + stop_index: int = Field(gt=0) + stride: int = Field(default=1, ge=1) + question_mode: Literal["constrained", "agentic"] = "constrained" + grounding: VqaGroundingSpecification = VqaGroundingSpecification() + output: str | None = None diff --git a/dimos/benchmark/vqa/generation/test_agents.py b/dimos/benchmark/vqa/generation/test_agents.py index 468461cf00..16b3a86019 100644 --- a/dimos/benchmark/vqa/generation/test_agents.py +++ b/dimos/benchmark/vqa/generation/test_agents.py @@ -108,6 +108,35 @@ def test_question_agent_returns_constrained_intents() -> None: ] +def test_question_agent_uses_image_selected_structured_intents() -> None: + class _StructuredQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "Do not return bare object names" in prompt + assert "opposite image sides" in prompt + return '[{"kind":"compare_nearest_by_side","object_query":"chair"}]' + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _StructuredQuestionModel())).propose( + frame.image + ) + + assert intents == [QuestionIntent(kind="compare_nearest_by_side", object_query="chair")] + + +def test_question_agent_accepts_door_state_intent() -> None: + class _DoorQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "door_state" in prompt + return '[{"kind":"door_state","object_query":"door"}]' + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _DoorQuestionModel())).propose(frame.image) + + assert intents == [QuestionIntent(kind="door_state", object_query="door")] + + def test_ground_truth_agent_records_tools_and_rejects_unsupported_question() -> None: frame, detection = _frame_and_detection() agent = _agent( diff --git a/dimos/benchmark/vqa/generation/test_oracle.py b/dimos/benchmark/vqa/generation/test_oracle.py index 4b1a8b3917..fd026e3aa0 100644 --- a/dimos/benchmark/vqa/generation/test_oracle.py +++ b/dimos/benchmark/vqa/generation/test_oracle.py @@ -17,7 +17,10 @@ from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry from dimos.benchmark.vqa.generation.primitives.choices import height_choice_window from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives -from dimos.benchmark.vqa.generation.primitives.geometry import estimate_ground_plane +from dimos.benchmark.vqa.generation.primitives.geometry import ( + classify_door_plane_angle, + estimate_ground_plane, +) from dimos.benchmark.vqa.generation.question_agent import ( AGENTIC_QUESTION_PROMPT, OpenAIFreeformQuestionAuthor, @@ -28,6 +31,7 @@ ChoiceAnswerContract, DeferredHeightChoiceContract, GroundingConfig, + GroundPlaneEstimate, OracleEvidence, OracleToolResult, QuestionProposal, @@ -189,8 +193,9 @@ def test_freeform_question_author_parses_public_contract() -> None: def test_freeform_author_prompt_prioritizes_geometric_questions() -> None: assert "measure_height" in AGENTIC_QUESTION_PROMPT - assert "two named objects is closer" in AGENTIC_QUESTION_PROMPT - assert "closer (choice" in AGENTIC_QUESTION_PROMPT + assert "two named object types is closer" in AGENTIC_QUESTION_PROMPT + assert "visibly repeated object" in AGENTIC_QUESTION_PROMPT + assert "diverse set of questions" in AGENTIC_QUESTION_PROMPT assert "Use visibility/presence questions only" in AGENTIC_QUESTION_PROMPT @@ -354,6 +359,7 @@ def test_local_registry_exposes_geometry_tools() -> None: "select_nearest_object", "fit_ground_plane", "measure_height", + "classify_door_state", "bucket_measurement", } @@ -371,6 +377,17 @@ def test_height_choice_window_is_local_and_deterministic() -> None: assert height_choice_window(3.0)[1] == "over 2.0 m" +def test_door_state_accepts_clear_plane_angles_and_rejects_ajar() -> None: + door = GroundPlaneEstimate((1.0, 0.0, 0.0), 0.0, 20, 20, 0.01) + closed = GroundPlaneEstimate((1.0, 0.0, 0.0), 0.0, 20, 20, 0.01) + open_door = GroundPlaneEstimate((0.0, 0.0, 1.0), 0.0, 20, 20, 0.01) + ajar_door = GroundPlaneEstimate((0.95, 0.0, 0.31), 0.0, 20, 20, 0.01) + + assert classify_door_plane_angle(door, closed)[0] == "closed" + assert classify_door_plane_angle(door, open_door)[0] == "open" + assert classify_door_plane_angle(door, ajar_door)[1] == "ambiguous_door_angle" + + def test_oracle_validates_evidence_and_answer_contract() -> None: proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract()) result = OracleToolResult( diff --git a/dimos/benchmark/vqa/models.py b/dimos/benchmark/vqa/models.py index a5221f6b29..904c6b9883 100644 --- a/dimos/benchmark/vqa/models.py +++ b/dimos/benchmark/vqa/models.py @@ -87,7 +87,7 @@ class VqaExample: QuestionKind = Literal[ - "presence", "horizontal_direction", "within_distance", "compare_nearest_by_side" + "presence", "horizontal_direction", "within_distance", "compare_nearest_by_side", "door_state" ] diff --git a/dimos/cli/test_vqa.py b/dimos/cli/test_vqa.py index a0d8d4e85d..abf7e5bf49 100644 --- a/dimos/cli/test_vqa.py +++ b/dimos/cli/test_vqa.py @@ -1,7 +1,11 @@ # Copyright 2026 Dimensional Inc. +import json +from pathlib import Path + from typer.testing import CliRunner +from dimos.benchmark.vqa.generation.specification import VqaGenerationSpecification from dimos.cli import vqa @@ -12,3 +16,55 @@ def test_vqa_generation_cli_has_no_explicit_query_or_model_options() -> None: assert "--propose-questions" not in output assert "--question-model" not in output assert "--oracle-model" not in output + assert "--spec" in output + + +def test_generation_spec_resolves_the_same_options_as_the_cli(tmp_path: Path) -> None: + spec = tmp_path / "generation.json" + spec.write_text( + json.dumps( + { + "recording": "go2_bigoffice.db", + "start_index": 10, + "stop_index": 40, + "stride": 5, + "question_mode": "agentic", + "grounding": {"min_mask_area_px": 256, "min_foreground_points": 4}, + "output": "/tmp/vqa", + } + ) + ) + + generation = vqa._resolve_generation_spec(spec, None, None, None, None, None, None, None, None) + + assert generation.recording == "go2_bigoffice.db" + assert generation.question_mode == "agentic" + assert generation.grounding.min_mask_area_px == 256 + assert generation.output == "/tmp/vqa" + + +def test_generation_spec_rejects_mixed_cli_options(tmp_path: Path) -> None: + spec = tmp_path / "generation.json" + spec.write_text('{"recording":"go2.db","stop_index":10}') + + result = CliRunner().invoke( + vqa.app, + ["generate", "--spec", str(spec), "--recording", "other.db"], + ) + + assert result.exit_code != 0 + assert "cannot be combined" in result.output + + +def test_generation_run_records_resolved_request(tmp_path: Path) -> None: + vqa._write_generation_run( + tmp_path, + VqaGenerationSpecification(recording="go2.db", stop_index=10), + {"frame_count": 2, "accepted_question_count": 3, "rejected_question_count": 1}, + ) + + payload = json.loads((tmp_path / "run.json").read_text()) + + assert payload["generation"]["recording"] == "go2.db" + assert payload["generation"]["output"] == str(tmp_path) + assert payload["summary"]["accepted_question_count"] == 3 diff --git a/dimos/cli/vqa.py b/dimos/cli/vqa.py index 9c447d170b..3d0260dc12 100644 --- a/dimos/cli/vqa.py +++ b/dimos/cli/vqa.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import os from pathlib import Path from typing import cast @@ -23,6 +24,7 @@ OpenAIQuestionAgent, ) from dimos.benchmark.vqa.generation.recording import load_go2_frame +from dimos.benchmark.vqa.generation.specification import VqaGenerationSpecification from dimos.benchmark.vqa.models import ( AcceptedOracleResult, CalibratedFrame, @@ -129,20 +131,42 @@ def single_frame( @app.command("generate") def generate( - recording: str = typer.Option(..., "--recording"), - start_index: int = typer.Option(0, "--start-index"), - stop_index: int = typer.Option(..., "--stop-index"), - stride: int = typer.Option(1, "--stride"), - question_mode: str = typer.Option("constrained", "--question-mode"), - min_mask_area_px: int = typer.Option(128, "--min-mask-area-px"), - min_foreground_points: int = typer.Option(3, "--min-foreground-points"), + recording: str | None = typer.Option(None, "--recording"), + start_index: int | None = typer.Option(None, "--start-index"), + stop_index: int | None = typer.Option(None, "--stop-index"), + stride: int | None = typer.Option(None, "--stride"), + question_mode: str | None = typer.Option(None, "--question-mode"), + min_mask_area_px: int | None = typer.Option(None, "--min-mask-area-px"), + min_foreground_points: int | None = typer.Option(None, "--min-foreground-points"), output: Path | None = typer.Option(None, "--output"), + spec: Path | None = typer.Option(None, "--spec", exists=True, dir_okay=False, readable=True), ) -> None: """Generate a resumable VQA dataset from sampled Go2 recording frames.""" - if start_index < 0 or stop_index <= start_index or stride < 1: + generation = _resolve_generation_spec( + spec, + recording, + start_index, + stop_index, + stride, + question_mode, + min_mask_area_px, + min_foreground_points, + output, + ) + if generation.stop_index <= generation.start_index: raise typer.BadParameter("provide valid frame bounds") - output = output or (STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frames") - _validate_question_mode(question_mode) + recording = generation.recording + start_index = generation.start_index + stop_index = generation.stop_index + stride = generation.stride + question_mode = generation.question_mode + min_mask_area_px = generation.grounding.min_mask_area_px + min_foreground_points = generation.grounding.min_foreground_points + output = ( + Path(generation.output).expanduser() + if generation.output is not None + else STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frames" + ) _require_openai_for_question_author() output.mkdir(parents=True, exist_ok=True) _require_edgetam_cuda() @@ -221,6 +245,7 @@ def generate( finally: model.stop() summary = write_dataset_manifest(output) + _write_generation_run(output, generation, summary) typer.echo(f"Dataset manifest: {summary}") @@ -265,6 +290,77 @@ def _validate_question_mode(question_mode: str) -> None: raise typer.BadParameter("question mode must be constrained or agentic") +def _resolve_generation_spec( + spec: Path | None, + recording: str | None, + start_index: int | None, + stop_index: int | None, + stride: int | None, + question_mode: str | None, + min_mask_area_px: int | None, + min_foreground_points: int | None, + output: Path | None, +) -> VqaGenerationSpecification: + """Load a JSON generation specification or resolve the explicit CLI alternatives.""" + values = ( + recording, + start_index, + stop_index, + stride, + question_mode, + min_mask_area_px, + min_foreground_points, + output, + ) + if spec is not None: + if any(value is not None for value in values): + raise typer.BadParameter("--spec cannot be combined with generation options") + try: + return VqaGenerationSpecification.model_validate_json(spec.read_bytes()) + except ValueError as exc: + raise typer.BadParameter(f"invalid generation specification: {exc}") from exc + if recording is None or stop_index is None: + raise typer.BadParameter("--recording and --stop-index are required without --spec") + try: + return VqaGenerationSpecification( + recording=recording, + start_index=0 if start_index is None else start_index, + stop_index=stop_index, + stride=1 if stride is None else stride, + question_mode="constrained" if question_mode is None else question_mode, + grounding={ + "min_mask_area_px": 128 if min_mask_area_px is None else min_mask_area_px, + "min_foreground_points": 3 + if min_foreground_points is None + else min_foreground_points, + }, + output=str(output) if output is not None else None, + ) + except ValueError as exc: + raise typer.BadParameter(f"invalid generation options: {exc}") from exc + + +def _write_generation_run( + output: Path, + generation: VqaGenerationSpecification, + summary: dict[str, int], +) -> None: + """Record the resolved request that produced one generated dataset.""" + payload = { + "schema_version": "1.0", + "generation": { + **generation.model_dump(mode="json"), + "output": str(output), + }, + "models": { + "question_author": QUESTION_MODEL, + "oracle": ORACLE_MODEL if generation.question_mode == "agentic" else None, + }, + "summary": summary, + } + (output / "run.json").write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + def _require_openai_for_question_author() -> None: if not os.environ.get("OPENAI_API_KEY"): raise typer.BadParameter("OPENAI_API_KEY must be set for image-authored question modes") diff --git a/docs/benchmarking/vqa-generation/README.md b/docs/benchmarking/vqa-generation/README.md deleted file mode 100644 index 5ab1a40350..0000000000 --- a/docs/benchmarking/vqa-generation/README.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: "VQA Benchmark" ---- - -# Point-Cloud-Grounded VQA Benchmark - -DimOS generates image-question-multiple-choice VQA cases from frozen robot recordings. Private point-cloud tools establish and validate answers during generation; evaluation sees only public images, questions, choices, and private answer labels. - -For the exact generation stages, checks, tool contracts, and output files, see [Pipeline](/docs/benchmarking/vqa-generation/pipeline.md). - -## Pipeline Flow - -```text -Frozen recording -> rectified RGB image + calibrated visible point cloud - | - +-- constrained: image object author -> deterministic question families - | -> private grounding -> quality-gate validation - | - +-- agentic: image question author -> frozen question - -> private local oracle tools -> validation - | - v -accepted cases.jsonl + private labels.jsonl - | - v -point-cloud-vqa Evaluation -> image-only vision model -> exact choice scoring -``` - -Rejected questions and private evidence remain in each frame's generation record. The evaluator does not load point clouds, calibration, tool traces, or rejection records. - -## Generation Modes - -Constrained generation expands visible objects into fixed choice-question families: presence (`yes`/`no`), horizontal direction, distance threshold, and nearest left-versus-right comparison. Additional deterministic question families will be added as their evidence programs mature. - -Agentic generation freezes a free-form image-authored question, then a private oracle uses read-only local tools to establish its answer. Height questions generate four public choices deterministically after private measurement. For example, `0.42 m` produces `under 0.2 m`, `0.2-0.6 m`, `0.6-1.0 m`, and `over 1.0 m`, with `0.2-0.6 m` as the answer. Additional oracle tools will be added as new evidence capabilities mature. - -The generation models are code-level defaults, not CLI settings. Constrained mode always uses the image author, then expands its object queries into the deterministic families. - -Geometry quality gates are the private validation step. They reject insufficient point support, ambiguous masks, unreliable ground planes, and incomplete height evidence before a case becomes public. - -Constrained families run predefined sequences over the private perception primitives. Agentic generation exposes those same primitives to the oracle, which chooses its own sequence after the public question is frozen. - -## Dataset Format - -The root evaluation export follows the common image-question-choice benchmark pattern: - -```json -{"id":"go2-40-chair-height","image":"frame-000040/image.jpg","question":"How tall is the chair?","choices":["under 0.2 m","0.2-0.6 m","0.6-1.0 m","over 1.0 m"]} -``` - -`cases.jsonl` contains public rows only. `labels.jsonl` contains the matching private `id` and `answer`. Frame directories retain `ground_truth.json`, which includes the full private generation audit data. - -## Generate - -```bash -OPENAI_API_KEY="$OPENAI_API_KEY" dimos vqa generate \ - --recording go2_short \ - --start-index 0 --stop-index 100 --stride 20 \ - --question-mode constrained \ - --output ~/.local/state/dimos/datasets/vqa/go2-short -``` - -## Evaluate - -Create an Evaluation Run Specification next to the generated dataset: - -```json -{ - "evaluation": { - "name": "point-cloud-vqa", - "config": { - "dataset": "./go2-short", - "model": "gpt-4o-mini" - } - } -} -``` - -Then run the shared evaluator: - -```bash -OPENAI_API_KEY="$OPENAI_API_KEY" dimos eval run vqa-run.json --output /tmp/vqa-evaluation -``` - -The evaluation writes the shared immutable `run.json` and a VQA-native `vqa-results.json` artifact containing each model response, normalized answer, expected answer, and pass/fail result. diff --git a/docs/benchmarking/vqa-generation/infrastructure.md b/docs/benchmarking/vqa-generation/infrastructure.md index aca2cf9fb2..bfd03e4a9f 100644 --- a/docs/benchmarking/vqa-generation/infrastructure.md +++ b/docs/benchmarking/vqa-generation/infrastructure.md @@ -49,6 +49,7 @@ dimos/benchmark/vqa/ oracle_tools.py opaque-ID and LangChain adapter for primitives oracle.py bounded tool-calling and evidence validation question_agent.py image-only question author + specification.py validated generation-specification schema dataset.py frame records and evaluation export evaluation.py shared point-cloud-vqa Evaluation plugin @@ -63,7 +64,10 @@ dimos/cli/vqa.py generation CLI commands - A calibrated visible point cloud. - Camera intrinsics and the point-cloud-to-camera transform. -The generator constructs one `FramePerceptionPrimitives` instance per frame. It owns MoonDream and +The generator accepts either explicit CLI flags or a reproducible JSON specification through +`dimos vqa generate --spec `. It writes the resolved generation request and +aggregate counts to `run.json` at the dataset root. The generator constructs one +`FramePerceptionPrimitives` instance per frame. It owns MoonDream and EdgeTAM calls, intermediate-result caches, grounded masks, and the accepted ground-plane fit. Projected visible point-cloud samples establish whether a mask has enough foreground support to become a grounded object. The generation runtime writes complete frame directories, so multi-frame diff --git a/docs/benchmarking/vqa-generation/pipeline.md b/docs/benchmarking/vqa-generation/pipeline.md index 22b0bc4b20..95411847b2 100644 --- a/docs/benchmarking/vqa-generation/pipeline.md +++ b/docs/benchmarking/vqa-generation/pipeline.md @@ -27,11 +27,37 @@ Flags: `dimos vqa single-frame` accepts the same generation settings and uses `--frame-index` instead of frame bounds. +### Generation Specification + +`dimos vqa generate --spec ` is an alternative to the explicit generation flags. +Do not combine `--spec` with `--recording`, frame bounds, question mode, grounding thresholds, or +output flags. A specification is a reproducible generation request: + +```json +{ + "recording": "go2_bigoffice.db", + "start_index": 0, + "stop_index": 100, + "stride": 20, + "question_mode": "agentic", + "grounding": { + "min_mask_area_px": 128, + "min_foreground_points": 3 + }, + "output": "~/.local/state/dimos/datasets/vqa/go2-bigoffice" +} +``` + +Generation writes the resolved request, model IDs, and aggregate counts to the dataset root's +private `run.json`. This is an output record, not the input specification. + ## 2. Create Questions ### Constrained -The image author proposes visible object queries, then the generator creates these deterministic families for every query: +The image author inspects the scene and returns up to five structured intents. It selects only +families likely to be useful for the visible arrangement, rather than expanding every object into +every family. Private grounding still rejects unsupported or ambiguous candidates. | Family | Sample question | Choices | |---|---|---| @@ -39,6 +65,7 @@ The image author proposes visible object queries, then the generator creates the | Horizontal direction | `Where is the nearest chair?` | `left`, `center`, `right` | | Distance threshold | `Is the nearest chair within 3 meters?` | `yes`, `no` | | Nearest by side | `Which chair is closer, the left or right one?` | `left`, `right` | +| Door state | `Is the door open or closed?` | `open`, `closed` | ### Agentic @@ -75,6 +102,10 @@ Height questions also require: 3. At least six points inside the object mask. 4. At least four elevated points, with at least 60% of selected points elevated more than `0.02 m` above the plane. +Door-state questions also require one grounded door, a robust plane fit for the door mask, and a +robust plane fit in a narrow ring around that mask. The planes must be either nearly aligned +(`closed`) or clearly rotated (`open`); slightly ajar or otherwise ambiguous doors are rejected. + ## 4. Create Answers ### Constrained @@ -131,6 +162,7 @@ The private oracle chooses a sequence from the same read-only primitives used by | `select_nearest_object` | object IDs, optional side | Nearest grounded object ID. | | `fit_ground_plane` | none | Plane ID, plane estimate, residual, inlier support, quality flags. | | `measure_height` | object ID, plane ID | Measurement ID, private height, uncertainty, provenance, quality flags. | +| `classify_door_state` | object ID | Public `open` or `closed` choice, or a private geometry rejection. | | `bucket_measurement` | measurement ID | Public answer-conditioned height choices and matching choice. | Opaque IDs chain tool results; raw masks and point-cloud arrays are not exposed to the oracle. The @@ -167,6 +199,7 @@ The dataset root aggregates: ```text cases.jsonl public id, image path, question, choices labels.jsonl private id and expected choice +run.json private resolved generation request and aggregate counts ``` Each line in `cases.jsonl` is one public evaluation case. Formatted for readability, one record is: diff --git a/docs/docs.json b/docs/docs.json index d934ef899b..94659c93a6 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -125,7 +125,6 @@ "group": "Perception", "pages": [ "capabilities/perception/index", - "benchmarking/vqa-generation/README", "benchmarking/vqa-generation/pipeline", "benchmarking/vqa-generation/infrastructure" ] From b0aee60f8f6fdf7c6494bb569b40d3cf33aa205e Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Mon, 10 Aug 2026 17:49:21 -0700 Subject: [PATCH 09/12] feat(benchmark): add closest-object VQA questions --- .../vqa/generation/ground_truth_generator.py | 61 +++++++++++++++++++ .../benchmark/vqa/generation/oracle_tools.py | 41 +++++++++++++ .../vqa/generation/primitives/contracts.py | 10 +++ .../vqa/generation/primitives/frame.py | 32 ++++++++++ .../vqa/generation/question_agent.py | 20 ++++-- dimos/benchmark/vqa/generation/test_agents.py | 18 ++++++ dimos/benchmark/vqa/generation/test_oracle.py | 31 +++++++++- dimos/benchmark/vqa/models.py | 8 ++- docs/benchmarking/vqa-generation/pipeline.md | 6 ++ 9 files changed, 219 insertions(+), 8 deletions(-) diff --git a/dimos/benchmark/vqa/generation/ground_truth_generator.py b/dimos/benchmark/vqa/generation/ground_truth_generator.py index 5db08c84ea..383e81ec36 100644 --- a/dimos/benchmark/vqa/generation/ground_truth_generator.py +++ b/dimos/benchmark/vqa/generation/ground_truth_generator.py @@ -69,6 +69,8 @@ def _answer_from_objects( return _compare_nearest_by_side(frame, intent, objects, trace) if intent.kind == "door_state": return _classify_door_state(frame, intent, objects, trace, self.primitives) + if intent.kind == "closest_object": + return _select_closest_object(frame, intent, objects, trace, self) examples = generate_questions( frame.id, objects, [intent.object_query], distance_m=intent.threshold_m or 3.0 ) @@ -105,6 +107,8 @@ def _render_question(intent: QuestionIntent) -> str: return f"Which {intent.object_query} is closer: the left one or the right one?" if intent.kind == "door_state": return f"Is the {intent.object_query} open or closed?" + if intent.kind == "closest_object": + return f"Which object is closest to the {intent.object_query}: {', '.join(intent.candidate_queries)}?" return f"Is the nearest {intent.object_query} within {intent.threshold_m or 3:g} meters? Answer yes or no." @@ -163,6 +167,63 @@ def _classify_door_state( return GroundTruthResult(intent, example, "answered", result.state, None, tuple(objects), trace) +def _select_closest_object( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + generator: VqaGroundTruthGenerator, +) -> GroundTruthResult: + if len(objects) != 1: + return _rejected_result(frame, intent, objects, trace, "ambiguous_target_object") + candidates: list[GroundedObject] = [] + for query in intent.candidate_queries: + matches, candidate_trace = generator.ground(frame, query) + trace = (*trace, *candidate_trace) + if len(matches) != 1: + return _rejected_result( + frame, + intent, + [*objects, *candidates, *matches], + trace, + "ambiguous_candidate_object", + ) + candidates.append(matches[0]) + selected = generator.primitives.select_closest_object(objects[0], candidates) + trace = ( + *trace, + ToolTrace( + "select_closest_object", + selected.object.id if selected.object else selected.rejection_reason or "rejected", + ), + ) + if selected.object is None: + return _rejected_result( + frame, + intent, + [*objects, *candidates], + trace, + selected.rejection_reason or "closest_object_rejected", + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-closest-object", + _render_question(intent), + selected.object.label, + "choice", + (objects[0].id, *(item.id for item in candidates)), + intent.candidate_queries, + ) + return GroundTruthResult( + intent, + example, + "answered", + selected.object.label, + None, + tuple([*objects, *candidates]), + trace, + ) + + def _rejected_result( frame: CalibratedFrame, intent: QuestionIntent, diff --git a/dimos/benchmark/vqa/generation/oracle_tools.py b/dimos/benchmark/vqa/generation/oracle_tools.py index d77c5f4e03..5b8eeffad1 100644 --- a/dimos/benchmark/vqa/generation/oracle_tools.py +++ b/dimos/benchmark/vqa/generation/oracle_tools.py @@ -65,6 +65,14 @@ def tools(self) -> list[StructuredTool]: "or right." ), ), + StructuredTool.from_function( + self.select_closest_object, + name="select_closest_object", + description=( + "Select which opaque candidate object is closest to one opaque target object " + "by private point-cloud support. Rejects ambiguous proximity." + ), + ), StructuredTool.from_function( self.fit_ground_plane, name="fit_ground_plane", @@ -186,6 +194,39 @@ def select_nearest_object(self, object_ids: list[str], side: str | None = None) self._results.append(result) return json.dumps(_tool_payload(result, object_id=nearest.id)) + def select_closest_object(self, target_id: str, candidate_ids: list[str]) -> str: + """Select one candidate closest to a target by private 3D support-point proximity.""" + target = self._objects.get(target_id) + if target is None: + return self._record_rejection( + "select_closest_object", target_id, [], "unknown_target_id" + ) + candidates: list[GroundedObject] = [] + for candidate_id in candidate_ids: + candidate = self._objects.get(candidate_id) + if candidate is None: + return self._record_rejection( + "select_closest_object", candidate_id, [], "unknown_candidate_id" + ) + candidates.append(candidate) + selected = self._primitives.select_closest_object(target, candidates) + if selected.object is None: + return self._record_rejection( + "select_closest_object", + target.label, + list(selected.quality_flags), + selected.rejection_reason, + ) + result = OracleToolResult( + "select_closest_object", + target.label, + (_grounding_evidence(target), _grounding_evidence(selected.object)), + choice=selected.object.label, + quality_flags=selected.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result, object_id=selected.object.id)) + def measure_height(self, object_id: str, plane_id: str) -> str: """Measure one grounded object against one previously accepted plane.""" object = self._objects.get(object_id) diff --git a/dimos/benchmark/vqa/generation/primitives/contracts.py b/dimos/benchmark/vqa/generation/primitives/contracts.py index 2630271910..ddebec942a 100644 --- a/dimos/benchmark/vqa/generation/primitives/contracts.py +++ b/dimos/benchmark/vqa/generation/primitives/contracts.py @@ -28,3 +28,13 @@ class DoorStateResult: quality_flags: tuple[str, ...] rejection_reason: str | None = None angle_deg: float | None = None + + +@dataclass(frozen=True) +class ClosestObjectResult: + """A point-cloud selected candidate nearest to one grounded target object.""" + + object: GroundedObject | None + distance_m: float | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None diff --git a/dimos/benchmark/vqa/generation/primitives/frame.py b/dimos/benchmark/vqa/generation/primitives/frame.py index aad6c420ab..3e66efb15a 100644 --- a/dimos/benchmark/vqa/generation/primitives/frame.py +++ b/dimos/benchmark/vqa/generation/primitives/frame.py @@ -6,6 +6,7 @@ from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects from dimos.benchmark.vqa.generation.primitives.contracts import ( + ClosestObjectResult, DoorStateResult, HeightMeasurementResult, ) @@ -200,6 +201,37 @@ def classify_door_state(self, object: GroundedObject) -> DoorStateResult: angle_deg, ) + def select_closest_object( + self, target: GroundedObject, candidates: list[GroundedObject] + ) -> ClosestObjectResult: + """Select the unambiguously closest candidate by private support-point centroids.""" + if not candidates: + return ClosestObjectResult(None, None, (), "no_candidate_objects") + if any(item.id == target.id for item in candidates): + return ClosestObjectResult(None, None, (), "target_cannot_be_candidate") + target_points = self._object_points(target) + if target_points is None: + return ClosestObjectResult(None, None, (), "insufficient_target_support") + target_center = np.median(target_points, axis=0) + distances: list[tuple[float, GroundedObject]] = [] + for candidate in candidates: + candidate_points = self._object_points(candidate) + if candidate_points is None: + return ClosestObjectResult(None, None, (), "insufficient_candidate_support") + distance = float(np.linalg.norm(np.median(candidate_points, axis=0) - target_center)) + distances.append((distance, candidate)) + distances.sort(key=lambda item: item[0]) + if len(distances) > 1 and distances[1][0] - distances[0][0] < 0.15: + return ClosestObjectResult(None, None, (), "ambiguous_object_proximity") + return ClosestObjectResult(distances[0][1], distances[0][0], ("object_centroid_proximity",)) + + def _object_points(self, object: GroundedObject) -> np.ndarray | None: + mask = self._object_masks.get(object.id) + if mask is None: + raise ValueError(f"unknown grounded object: {object.id}") + points = points_in_mask(self.frame, mask.mask) + return points if len(points) >= 6 else None + def _object_mask_index(item: GroundedObject) -> int: try: diff --git a/dimos/benchmark/vqa/generation/question_agent.py b/dimos/benchmark/vqa/generation/question_agent.py index 389c85e419..ef1183e1e6 100644 --- a/dimos/benchmark/vqa/generation/question_agent.py +++ b/dimos/benchmark/vqa/generation/question_agent.py @@ -21,13 +21,16 @@ QUESTION_PROMPT = """Select up to 5 challenging but visually well-supported single-frame VQA intents. Inspect only this image. Do not assume depth, point clouds, calibration, metadata, or temporal context. Return JSON only: an array of objects with kind, object_query, and threshold_m only for -within_distance. kind must be one of presence, horizontal_direction, within_distance, or -compare_nearest_by_side, or door_state. Use threshold_m: 3.0 for within_distance. +within_distance, plus candidate_queries only for closest_object. kind must be one of presence, +horizontal_direction, within_distance, compare_nearest_by_side, door_state, or closest_object. +Use threshold_m: 3.0 for within_distance. Use image context to select only intents likely to produce a useful geometric case: emit compare_nearest_by_side only when at least two visible instances of the same object appear on opposite image sides; emit horizontal_direction only for a visible object; and prefer relational or directional intents over presence. Emit door_state only for a clearly visible door with nearby visible structure. Diversify object classes and question families when the scene supports them. +Emit closest_object only when one target and at least two distinct candidate object types are +visible; candidate_queries must name the visible candidate types. Do not return bare object names, floors, walls, ceilings, background surfaces, questions, answers, explanations, Markdown, or information not visible in the image.""" @@ -39,14 +42,15 @@ Prioritize questions that a private point-cloud oracle can validate. For the height of one upright object resting on visible ground, use deferred_height_choice. Its choices are generated privately from a successful height measurement. Aim for a diverse set of questions that use the visible scene -composition: relative left/center/right position, which of two named object types is closer, count -choices for a visibly repeated object, and height for an upright grounded object. Use concise, +composition: relative left/center/right position, which listed object is closest to a named target, +count choices for a visibly repeated object, and height for an upright grounded object. For +closest-object questions, use distinct visible candidate object types as the fixed choices. Use concise, mutually exclusive fixed choices with two to four options. For a clearly visible door with nearby visible structure, you may ask whether it is open or closed with exactly ["open", "closed"]. Use object_queries for every referenced object and tool_hints from "detect_objects", "segment_detections", "ground_masks", "fit_ground_plane", "select_nearest_object", "measure_height", "classify_door_state", or -"bucket_measurement" when applicable. Use visibility/presence questions only when no stronger +"select_closest_object", "bucket_measurement" when applicable. Use visibility/presence questions only when no stronger geometric question is available. Do not ask about color, material, text, intent, full physical size, hidden parts, or exact metric distances without a supplied choice contract. Use only visible objects. Do not include @@ -90,6 +94,7 @@ def propose(self, image: Image) -> list[QuestionIntent]: "within_distance", "compare_nearest_by_side", "door_state", + "closest_object", ): raise ValueError(f"unsupported question kind: {kind!r}") if not isinstance(query, str) or not query: @@ -100,7 +105,10 @@ def propose(self, image: Image) -> list[QuestionIntent]: raise ValueError("within_distance requires a positive threshold_m") if kind != "within_distance": threshold = None - intents.append(QuestionIntent(kind=kind, object_query=query, threshold_m=threshold)) + candidates = _string_tuple(item.get("candidate_queries"), "candidate_queries") + if kind == "closest_object" and (len(candidates) < 2 or query in candidates): + raise ValueError("closest_object requires two distinct candidate_queries") + intents.append(QuestionIntent(kind, query, threshold, candidates)) return intents diff --git a/dimos/benchmark/vqa/generation/test_agents.py b/dimos/benchmark/vqa/generation/test_agents.py index 16b3a86019..586d758c02 100644 --- a/dimos/benchmark/vqa/generation/test_agents.py +++ b/dimos/benchmark/vqa/generation/test_agents.py @@ -137,6 +137,24 @@ def query(self, image: Image, prompt: str) -> str: assert intents == [QuestionIntent(kind="door_state", object_query="door")] +def test_question_agent_accepts_closest_object_intent() -> None: + class _ClosestQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "candidate_queries" in prompt + return ( + '[{"kind":"closest_object","object_query":"chair",' + '"candidate_queries":["table","lamp"]}]' + ) + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _ClosestQuestionModel())).propose( + frame.image + ) + + assert intents == [QuestionIntent("closest_object", "chair", None, ("table", "lamp"))] + + def test_ground_truth_agent_records_tools_and_rejects_unsupported_question() -> None: frame, detection = _frame_and_detection() agent = _agent( diff --git a/dimos/benchmark/vqa/generation/test_oracle.py b/dimos/benchmark/vqa/generation/test_oracle.py index fd026e3aa0..409ab80ad2 100644 --- a/dimos/benchmark/vqa/generation/test_oracle.py +++ b/dimos/benchmark/vqa/generation/test_oracle.py @@ -30,6 +30,7 @@ CalibratedFrame, ChoiceAnswerContract, DeferredHeightChoiceContract, + GroundedObject, GroundingConfig, GroundPlaneEstimate, OracleEvidence, @@ -193,7 +194,7 @@ def test_freeform_question_author_parses_public_contract() -> None: def test_freeform_author_prompt_prioritizes_geometric_questions() -> None: assert "measure_height" in AGENTIC_QUESTION_PROMPT - assert "two named object types is closer" in AGENTIC_QUESTION_PROMPT + assert "closest to a named target" in AGENTIC_QUESTION_PROMPT assert "visibly repeated object" in AGENTIC_QUESTION_PROMPT assert "diverse set of questions" in AGENTIC_QUESTION_PROMPT assert "Use visibility/presence questions only" in AGENTIC_QUESTION_PROMPT @@ -357,6 +358,7 @@ def test_local_registry_exposes_geometry_tools() -> None: "segment_detections", "ground_masks", "select_nearest_object", + "select_closest_object", "fit_ground_plane", "measure_height", "classify_door_state", @@ -388,6 +390,33 @@ def test_door_state_accepts_clear_plane_angles_and_rejects_ajar() -> None: assert classify_door_plane_angle(door, ajar_door)[1] == "ambiguous_door_angle" +def test_closest_object_uses_point_cloud_centroids_and_rejects_ties(monkeypatch: Any) -> None: + target = GroundedObject("target", "chair", 8, 1.0, "left") + close = GroundedObject("close", "table", 8, 2.0, "center") + far = GroundedObject("far", "lamp", 8, 3.0, "right") + primitives = _frame_primitives(_measurement_frame()) + centers = { + "target": np.zeros((6, 3)), + "close": np.tile((1.0, 0.0, 0.0), (6, 1)), + "far": np.tile((2.0, 0.0, 0.0), (6, 1)), + } + monkeypatch.setattr(primitives, "_object_points", lambda item: centers[item.id]) + + selected = primitives.select_closest_object(target, [close, far]) + + assert selected.object == close + assert selected.distance_m == 1.0 + monkeypatch.setattr( + primitives, + "_object_points", + lambda item: np.tile((1.0, 0.0, 0.0), (6, 1)) if item.id != "target" else centers["target"], + ) + assert ( + primitives.select_closest_object(target, [close, far]).rejection_reason + == "ambiguous_object_proximity" + ) + + def test_oracle_validates_evidence_and_answer_contract() -> None: proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract()) result = OracleToolResult( diff --git a/dimos/benchmark/vqa/models.py b/dimos/benchmark/vqa/models.py index 904c6b9883..c562350fbd 100644 --- a/dimos/benchmark/vqa/models.py +++ b/dimos/benchmark/vqa/models.py @@ -87,7 +87,12 @@ class VqaExample: QuestionKind = Literal[ - "presence", "horizontal_direction", "within_distance", "compare_nearest_by_side", "door_state" + "presence", + "horizontal_direction", + "within_distance", + "compare_nearest_by_side", + "door_state", + "closest_object", ] @@ -98,6 +103,7 @@ class QuestionIntent: kind: QuestionKind object_query: str threshold_m: float | None = None + candidate_queries: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/docs/benchmarking/vqa-generation/pipeline.md b/docs/benchmarking/vqa-generation/pipeline.md index 95411847b2..ae1283419c 100644 --- a/docs/benchmarking/vqa-generation/pipeline.md +++ b/docs/benchmarking/vqa-generation/pipeline.md @@ -65,6 +65,7 @@ every family. Private grounding still rejects unsupported or ambiguous candidate | Horizontal direction | `Where is the nearest chair?` | `left`, `center`, `right` | | Distance threshold | `Is the nearest chair within 3 meters?` | `yes`, `no` | | Nearest by side | `Which chair is closer, the left or right one?` | `left`, `right` | +| Closest object | `Which object is closest to the chair: table or lamp?` | `table`, `lamp` | | Door state | `Is the door open or closed?` | `open`, `closed` | ### Agentic @@ -106,6 +107,10 @@ Door-state questions also require one grounded door, a robust plane fit for the robust plane fit in a narrow ring around that mask. The planes must be either nearly aligned (`closed`) or clearly rotated (`open`); slightly ajar or otherwise ambiguous doors are rejected. +Closest-object questions require exactly one grounded target and one grounded instance for every +candidate choice. They compare private support-point centroids and reject candidates whose nearest +two distances are within `0.15 m`. + ## 4. Create Answers ### Constrained @@ -160,6 +165,7 @@ The private oracle chooses a sequence from the same read-only primitives used by | `segment_detections` | detection ID | Mask ID and accepted mask count. | | `ground_masks` | mask ID | Grounded object IDs, range, side, point support, evidence IDs. | | `select_nearest_object` | object IDs, optional side | Nearest grounded object ID. | +| `select_closest_object` | target ID, candidate IDs | Candidate nearest to target by private support-point centroids. | | `fit_ground_plane` | none | Plane ID, plane estimate, residual, inlier support, quality flags. | | `measure_height` | object ID, plane ID | Measurement ID, private height, uncertainty, provenance, quality flags. | | `classify_door_state` | object ID | Public `open` or `closed` choice, or a private geometry rejection. | From ad1699b38da716d3663c1bb82395e373ca240a66 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Mon, 10 Aug 2026 17:57:47 -0700 Subject: [PATCH 10/12] feat(benchmark): add forward-path VQA questions --- .../vqa/generation/ground_truth_generator.py | 26 +++++++++++++ .../benchmark/vqa/generation/oracle_tools.py | 37 +++++++++++++++++++ .../vqa/generation/primitives/contracts.py | 10 +++++ .../vqa/generation/primitives/frame.py | 18 +++++++++ .../vqa/generation/primitives/geometry.py | 24 ++++++++++++ .../vqa/generation/question_agent.py | 13 +++++-- dimos/benchmark/vqa/generation/test_agents.py | 15 ++++++++ dimos/benchmark/vqa/generation/test_oracle.py | 16 ++++++++ dimos/benchmark/vqa/models.py | 1 + docs/benchmarking/vqa-generation/pipeline.md | 6 +++ 10 files changed, 162 insertions(+), 4 deletions(-) diff --git a/dimos/benchmark/vqa/generation/ground_truth_generator.py b/dimos/benchmark/vqa/generation/ground_truth_generator.py index 383e81ec36..2c322ff124 100644 --- a/dimos/benchmark/vqa/generation/ground_truth_generator.py +++ b/dimos/benchmark/vqa/generation/ground_truth_generator.py @@ -23,6 +23,8 @@ def __init__(self, primitives: FramePerceptionPrimitives) -> None: self.primitives = primitives def answer(self, frame: CalibratedFrame, intent: QuestionIntent) -> GroundTruthResult: + if intent.kind == "forward_path": + return _classify_forward_path(frame, intent, self.primitives) objects, trace = self.ground(frame, intent.object_query) return self._answer_from_objects(frame, intent, objects, trace) @@ -109,6 +111,8 @@ def _render_question(intent: QuestionIntent) -> str: return f"Is the {intent.object_query} open or closed?" if intent.kind == "closest_object": return f"Which object is closest to the {intent.object_query}: {', '.join(intent.candidate_queries)}?" + if intent.kind == "forward_path": + return "Is the path directly ahead clear or blocked?" return f"Is the nearest {intent.object_query} within {intent.threshold_m or 3:g} meters? Answer yes or no." @@ -224,6 +228,28 @@ def _select_closest_object( ) +def _classify_forward_path( + frame: CalibratedFrame, intent: QuestionIntent, primitives: FramePerceptionPrimitives +) -> GroundTruthResult: + result = primitives.classify_forward_path() + trace = ( + ToolTrace("classify_forward_path", result.state or result.rejection_reason or "rejected"), + ) + if result.state is None: + return _rejected_result( + frame, intent, [], trace, result.rejection_reason or "forward_path_rejected" + ) + example = VqaExample( + f"{frame.id}-forward-path", + _render_question(intent), + result.state, + "choice", + (), + ("clear", "blocked"), + ) + return GroundTruthResult(intent, example, "answered", result.state, None, (), trace) + + def _rejected_result( frame: CalibratedFrame, intent: QuestionIntent, diff --git a/dimos/benchmark/vqa/generation/oracle_tools.py b/dimos/benchmark/vqa/generation/oracle_tools.py index 5b8eeffad1..e3c3be974a 100644 --- a/dimos/benchmark/vqa/generation/oracle_tools.py +++ b/dimos/benchmark/vqa/generation/oracle_tools.py @@ -93,6 +93,14 @@ def tools(self) -> list[StructuredTool]: "Rejects insufficient or ambiguous geometry." ), ), + StructuredTool.from_function( + self.classify_forward_path, + name="classify_forward_path", + description=( + "Classify the visible camera-forward corridor as clear or blocked from point-cloud " + "ground and obstacle support. Rejects incomplete or ambiguous visibility." + ), + ), StructuredTool.from_function( self.bucket_measurement, name="bucket_measurement", @@ -303,6 +311,35 @@ def classify_door_state(self, object_id: str) -> str: self._results.append(tool_result) return json.dumps(_tool_payload(tool_result)) + def classify_forward_path(self) -> str: + """Classify the observed local corridor directly ahead of the camera.""" + result = self._primitives.classify_forward_path() + if result.state is None: + return self._record_rejection( + "classify_forward_path", + "", + list(result.quality_flags), + result.rejection_reason, + ) + evidence = OracleEvidence( + f"forward-path:v1:{self._primitives.frame.id}", + "v1", + "forward-path", + "forward path", + 0.0, + "center", + result.point_count, + ) + tool_result = OracleToolResult( + "classify_forward_path", + "forward path", + (evidence,), + choice=result.state, + quality_flags=result.quality_flags, + ) + self._results.append(tool_result) + return json.dumps(_tool_payload(tool_result)) + def bucket_measurement(self, measurement_id: str) -> str: """Map one accepted private height measurement to its public choice.""" source = self._measurements.get(measurement_id) diff --git a/dimos/benchmark/vqa/generation/primitives/contracts.py b/dimos/benchmark/vqa/generation/primitives/contracts.py index ddebec942a..7aa9d59774 100644 --- a/dimos/benchmark/vqa/generation/primitives/contracts.py +++ b/dimos/benchmark/vqa/generation/primitives/contracts.py @@ -38,3 +38,13 @@ class ClosestObjectResult: distance_m: float | None quality_flags: tuple[str, ...] rejection_reason: str | None = None + + +@dataclass(frozen=True) +class ForwardPathResult: + """A conservative visible-corridor classification from point-cloud evidence.""" + + state: Literal["clear", "blocked"] | None + point_count: int + quality_flags: tuple[str, ...] + rejection_reason: str | None = None diff --git a/dimos/benchmark/vqa/generation/primitives/frame.py b/dimos/benchmark/vqa/generation/primitives/frame.py index 3e66efb15a..028d7da9fa 100644 --- a/dimos/benchmark/vqa/generation/primitives/frame.py +++ b/dimos/benchmark/vqa/generation/primitives/frame.py @@ -4,15 +4,18 @@ import numpy as np +from dimos.benchmark.vqa.generation.geometry import project_visible_points from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects from dimos.benchmark.vqa.generation.primitives.contracts import ( ClosestObjectResult, DoorStateResult, + ForwardPathResult, HeightMeasurementResult, ) from dimos.benchmark.vqa.generation.primitives.geometry import ( PlaneFitResult, classify_door_plane_angle, + classify_forward_corridor, estimate_ground_plane, fit_surface_plane, points_around_mask, @@ -225,6 +228,21 @@ def select_closest_object( return ClosestObjectResult(None, None, (), "ambiguous_object_proximity") return ClosestObjectResult(distances[0][1], distances[0][0], ("object_centroid_proximity",)) + def classify_forward_path(self) -> ForwardPathResult: + """Classify the observed camera-forward corridor as clear or blocked.""" + ground_fit = self.fit_ground_plane() + if ground_fit.estimate is None: + return ForwardPathResult( + None, + 0, + ("ground_plane_rejected", *ground_fit.quality_flags), + ground_fit.rejection_reason, + ) + projected = project_visible_points(self.frame) + points = np.asarray(projected.camera_points, dtype=np.float64) + state, flags, reason = classify_forward_corridor(points, ground_fit.estimate) + return ForwardPathResult(state, len(points), flags, reason) + def _object_points(self, object: GroundedObject) -> np.ndarray | None: mask = self._object_masks.get(object.id) if mask is None: diff --git a/dimos/benchmark/vqa/generation/primitives/geometry.py b/dimos/benchmark/vqa/generation/primitives/geometry.py index 75da5dc9eb..a1314506a9 100644 --- a/dimos/benchmark/vqa/generation/primitives/geometry.py +++ b/dimos/benchmark/vqa/generation/primitives/geometry.py @@ -161,3 +161,27 @@ def classify_door_plane_angle( if angle_deg >= 25.0: return "open", None, angle_deg return None, "ambiguous_door_angle", angle_deg + + +def classify_forward_corridor( + points: np.ndarray, ground: GroundPlaneEstimate +) -> tuple[str | None, tuple[str, ...], str | None]: + """Classify a visible camera-forward corridor as clear or blocked.""" + if len(points) == 0: + return None, (), "insufficient_forward_support" + depth = points[:, 2] + lateral_limit = depth * np.tan(np.radians(20.0)) + corridor = points[(depth >= 0.5) & (depth <= 3.0) & (np.abs(points[:, 0]) <= lateral_limit)] + if len(corridor) < 12: + return None, (), "insufficient_forward_support" + elevation = corridor @ np.asarray(ground.normal) + ground.offset_m + ground_points = corridor[np.abs(elevation) <= 0.08] + for start, stop in ((0.5, 1.33), (1.33, 2.16), (2.16, 3.0)): + if int(((ground_points[:, 2] >= start) & (ground_points[:, 2] < stop)).sum()) < 3: + return None, (), "incomplete_forward_ground_support" + obstacle_count = int((elevation > 0.15).sum()) + if obstacle_count >= 4: + return "blocked", ("visible_forward_obstacle", "forward_ground_supported"), None + if obstacle_count: + return None, (), "ambiguous_forward_obstacle" + return "clear", ("forward_ground_supported", "no_supported_forward_obstacle"), None diff --git a/dimos/benchmark/vqa/generation/question_agent.py b/dimos/benchmark/vqa/generation/question_agent.py index ef1183e1e6..5fe9d65b8d 100644 --- a/dimos/benchmark/vqa/generation/question_agent.py +++ b/dimos/benchmark/vqa/generation/question_agent.py @@ -22,7 +22,7 @@ Inspect only this image. Do not assume depth, point clouds, calibration, metadata, or temporal context. Return JSON only: an array of objects with kind, object_query, and threshold_m only for within_distance, plus candidate_queries only for closest_object. kind must be one of presence, -horizontal_direction, within_distance, compare_nearest_by_side, door_state, or closest_object. +horizontal_direction, within_distance, compare_nearest_by_side, door_state, closest_object, or forward_path. Use threshold_m: 3.0 for within_distance. Use image context to select only intents likely to produce a useful geometric case: emit compare_nearest_by_side only when at least two visible instances of the same object appear on @@ -31,6 +31,8 @@ visible structure. Diversify object classes and question families when the scene supports them. Emit closest_object only when one target and at least two distinct candidate object types are visible; candidate_queries must name the visible candidate types. +Emit forward_path only when the center foreground and visible floor provide enough context to judge +the local path directly ahead. Use object_query: "forward path" for forward_path. Do not return bare object names, floors, walls, ceilings, background surfaces, questions, answers, explanations, Markdown, or information not visible in the image.""" @@ -47,10 +49,10 @@ closest-object questions, use distinct visible candidate object types as the fixed choices. Use concise, mutually exclusive fixed choices with two to four options. For a clearly visible door with nearby visible structure, you may ask whether it is open or closed with exactly ["open", "closed"]. Use -object_queries for every referenced -object and tool_hints from "detect_objects", "segment_detections", "ground_masks", +the fixed choices ["clear", "blocked"] only for a visibly supported local path directly ahead. +Use object_queries for every referenced object and tool_hints from "detect_objects", "segment_detections", "ground_masks", "fit_ground_plane", "select_nearest_object", "measure_height", "classify_door_state", or -"select_closest_object", "bucket_measurement" when applicable. Use visibility/presence questions only when no stronger +"select_closest_object", "classify_forward_path", "bucket_measurement" when applicable. Use visibility/presence questions only when no stronger geometric question is available. Do not ask about color, material, text, intent, full physical size, hidden parts, or exact metric distances without a supplied choice contract. Use only visible objects. Do not include @@ -95,6 +97,7 @@ def propose(self, image: Image) -> list[QuestionIntent]: "compare_nearest_by_side", "door_state", "closest_object", + "forward_path", ): raise ValueError(f"unsupported question kind: {kind!r}") if not isinstance(query, str) or not query: @@ -108,6 +111,8 @@ def propose(self, image: Image) -> list[QuestionIntent]: candidates = _string_tuple(item.get("candidate_queries"), "candidate_queries") if kind == "closest_object" and (len(candidates) < 2 or query in candidates): raise ValueError("closest_object requires two distinct candidate_queries") + if kind == "forward_path" and query != "forward path": + raise ValueError('forward_path requires object_query "forward path"') intents.append(QuestionIntent(kind, query, threshold, candidates)) return intents diff --git a/dimos/benchmark/vqa/generation/test_agents.py b/dimos/benchmark/vqa/generation/test_agents.py index 586d758c02..68205344b0 100644 --- a/dimos/benchmark/vqa/generation/test_agents.py +++ b/dimos/benchmark/vqa/generation/test_agents.py @@ -155,6 +155,21 @@ def query(self, image: Image, prompt: str) -> str: assert intents == [QuestionIntent("closest_object", "chair", None, ("table", "lamp"))] +def test_question_agent_accepts_forward_path_intent() -> None: + class _ForwardPathQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "forward_path" in prompt + return '[{"kind":"forward_path","object_query":"forward path"}]' + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _ForwardPathQuestionModel())).propose( + frame.image + ) + + assert intents == [QuestionIntent(kind="forward_path", object_query="forward path")] + + def test_ground_truth_agent_records_tools_and_rejects_unsupported_question() -> None: frame, detection = _frame_and_detection() agent = _agent( diff --git a/dimos/benchmark/vqa/generation/test_oracle.py b/dimos/benchmark/vqa/generation/test_oracle.py index 409ab80ad2..aa039e98a3 100644 --- a/dimos/benchmark/vqa/generation/test_oracle.py +++ b/dimos/benchmark/vqa/generation/test_oracle.py @@ -19,6 +19,7 @@ from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives from dimos.benchmark.vqa.generation.primitives.geometry import ( classify_door_plane_angle, + classify_forward_corridor, estimate_ground_plane, ) from dimos.benchmark.vqa.generation.question_agent import ( @@ -362,6 +363,7 @@ def test_local_registry_exposes_geometry_tools() -> None: "fit_ground_plane", "measure_height", "classify_door_state", + "classify_forward_path", "bucket_measurement", } @@ -417,6 +419,20 @@ def test_closest_object_uses_point_cloud_centroids_and_rejects_ties(monkeypatch: ) +def test_forward_corridor_requires_ground_support_and_detects_obstacles() -> None: + ground = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + floor = np.asarray( + [ + [0.0, 1.0, depth] + for depth in (0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8) + ] + ) + + assert classify_forward_corridor(floor, ground)[0] == "clear" + obstacle = np.repeat(np.asarray([[0.0, 0.5, 1.0]]), 4, axis=0) + assert classify_forward_corridor(np.vstack((floor, obstacle)), ground)[0] == "blocked" + + def test_oracle_validates_evidence_and_answer_contract() -> None: proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract()) result = OracleToolResult( diff --git a/dimos/benchmark/vqa/models.py b/dimos/benchmark/vqa/models.py index c562350fbd..db79115c2e 100644 --- a/dimos/benchmark/vqa/models.py +++ b/dimos/benchmark/vqa/models.py @@ -93,6 +93,7 @@ class VqaExample: "compare_nearest_by_side", "door_state", "closest_object", + "forward_path", ] diff --git a/docs/benchmarking/vqa-generation/pipeline.md b/docs/benchmarking/vqa-generation/pipeline.md index ae1283419c..18199768b0 100644 --- a/docs/benchmarking/vqa-generation/pipeline.md +++ b/docs/benchmarking/vqa-generation/pipeline.md @@ -67,6 +67,7 @@ every family. Private grounding still rejects unsupported or ambiguous candidate | Nearest by side | `Which chair is closer, the left or right one?` | `left`, `right` | | Closest object | `Which object is closest to the chair: table or lamp?` | `table`, `lamp` | | Door state | `Is the door open or closed?` | `open`, `closed` | +| Forward path | `Is the path directly ahead clear or blocked?` | `clear`, `blocked` | ### Agentic @@ -111,6 +112,10 @@ Closest-object questions require exactly one grounded target and one grounded in candidate choice. They compare private support-point centroids and reject candidates whose nearest two distances are within `0.15 m`. +Forward-path questions require a fitted visible ground plane and enough ground support in each third +of the center camera-forward corridor from `0.5-3.0 m`. Supported non-ground points block the +corridor; incomplete ground support or sparse obstacle evidence is rejected. + ## 4. Create Answers ### Constrained @@ -169,6 +174,7 @@ The private oracle chooses a sequence from the same read-only primitives used by | `fit_ground_plane` | none | Plane ID, plane estimate, residual, inlier support, quality flags. | | `measure_height` | object ID, plane ID | Measurement ID, private height, uncertainty, provenance, quality flags. | | `classify_door_state` | object ID | Public `open` or `closed` choice, or a private geometry rejection. | +| `classify_forward_path` | none | Public `clear` or `blocked` choice, or a private visibility rejection. | | `bucket_measurement` | measurement ID | Public answer-conditioned height choices and matching choice. | Opaque IDs chain tool results; raw masks and point-cloud arrays are not exposed to the oracle. The From a4f7f9f67debe9878ecdb2378e0529ee59ae0218 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Mon, 10 Aug 2026 20:02:08 -0700 Subject: [PATCH 11/12] feat(benchmark): add VQA geometry comparisons --- .../vqa/generation/ground_truth_generator.py | 175 ++++++++++++++ .../benchmark/vqa/generation/oracle_tools.py | 215 +++++++++++++++++- .../vqa/generation/primitives/choices.py | 29 +++ .../vqa/generation/primitives/contracts.py | 9 + .../vqa/generation/primitives/frame.py | 18 ++ .../vqa/generation/question_agent.py | 54 +++-- dimos/benchmark/vqa/generation/test_agents.py | 137 ++++++++++- dimos/benchmark/vqa/generation/test_oracle.py | 98 +++++++- dimos/benchmark/vqa/models.py | 5 + docs/benchmarking/vqa-generation/pipeline.md | 42 ++++ 10 files changed, 755 insertions(+), 27 deletions(-) diff --git a/dimos/benchmark/vqa/generation/ground_truth_generator.py b/dimos/benchmark/vqa/generation/ground_truth_generator.py index 2c322ff124..3987d5ee2d 100644 --- a/dimos/benchmark/vqa/generation/ground_truth_generator.py +++ b/dimos/benchmark/vqa/generation/ground_truth_generator.py @@ -3,6 +3,12 @@ from __future__ import annotations +from dimos.benchmark.vqa.generation.primitives.choices import ( + CAMERA_RANGE_CHOICES, + COUNT_CHOICES, + camera_range_choice, + count_choice, +) from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object from dimos.benchmark.vqa.generation.questions import generate_questions @@ -69,6 +75,14 @@ def _answer_from_objects( ) if intent.kind == "compare_nearest_by_side": return _compare_nearest_by_side(frame, intent, objects, trace) + if intent.kind == "visible_count": + return _count_visible_objects(frame, intent, objects, trace) + if intent.kind == "camera_range": + return _bucket_camera_range(frame, intent, objects, trace) + if intent.kind == "compare_left_right": + return _compare_left_right(frame, intent, objects, trace, self) + if intent.kind == "compare_height": + return _compare_heights(frame, intent, objects, trace, self) if intent.kind == "door_state": return _classify_door_state(frame, intent, objects, trace, self.primitives) if intent.kind == "closest_object": @@ -105,8 +119,18 @@ def _render_question(intent: QuestionIntent) -> str: return f"Is there a {intent.object_query} in the image? Answer yes or no." if intent.kind == "horizontal_direction": return f"Where is the nearest {intent.object_query}: left, center, or right?" + if intent.kind == "visible_count": + return f"How many {intent.object_query}s are visible?" + if intent.kind == "camera_range": + return f"How far is the nearest {intent.object_query} from the camera?" if intent.kind == "compare_nearest_by_side": return f"Which {intent.object_query} is closer: the left one or the right one?" + if intent.kind == "compare_left_right": + return ( + f"Is the {intent.object_query} to the left or right of the {intent.comparison_query}?" + ) + if intent.kind == "compare_height": + return f"Which is taller: the {intent.object_query} or the {intent.comparison_query}?" if intent.kind == "door_state": return f"Is the {intent.object_query} open or closed?" if intent.kind == "closest_object": @@ -116,6 +140,157 @@ def _render_question(intent: QuestionIntent) -> str: return f"Is the nearest {intent.object_query} within {intent.threshold_m or 3:g} meters? Answer yes or no." +def _count_visible_objects( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + answer = count_choice(len(objects)) + example = VqaExample( + f"{frame.id}-{intent.object_query}-visible-count", + _render_question(intent), + answer, + "choice", + tuple(item.id for item in objects), + COUNT_CHOICES, + ) + return GroundTruthResult(intent, example, "answered", answer, None, tuple(objects), trace) + + +def _bucket_camera_range( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + selected = select_nearest_object(objects) + if selected is None: + return _rejected_result(frame, intent, objects, trace, "no_grounded_object") + answer = camera_range_choice(selected.range_m) + example = VqaExample( + f"{frame.id}-{intent.object_query}-camera-range", + _render_question(intent), + answer, + "choice", + (selected.id,), + CAMERA_RANGE_CHOICES, + ) + return GroundTruthResult(intent, example, "answered", answer, None, (selected,), trace) + + +def _compare_heights( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + generator: VqaGroundTruthGenerator, +) -> GroundTruthResult: + if len(objects) != 1 or intent.comparison_query is None: + return _rejected_result(frame, intent, objects, trace, "ambiguous_first_height_object") + other_objects, other_trace = generator.ground(frame, intent.comparison_query) + trace = (*trace, *other_trace) + if len(other_objects) != 1: + return _rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_second_height_object" + ) + plane_fit = generator.primitives.fit_ground_plane() + trace = (*trace, ToolTrace("fit_ground_plane", plane_fit.rejection_reason or "accepted")) + if plane_fit.estimate is None: + return _rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + plane_fit.rejection_reason or "ground_plane_rejected", + ) + first = generator.primitives.measure_height(objects[0], plane_fit.estimate) + second = generator.primitives.measure_height(other_objects[0], plane_fit.estimate) + trace = ( + *trace, + ToolTrace("measure_height", first.rejection_reason or objects[0].id), + ToolTrace("measure_height", second.rejection_reason or other_objects[0].id), + ) + if first.measurement is None or second.measurement is None: + return _rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + first.rejection_reason or second.rejection_reason or "height_measurement_rejected", + ) + first_lower = first.measurement.value - first.measurement.tolerance + second_lower = second.measurement.value - second.measurement.tolerance + first_upper = first.measurement.value + first.measurement.tolerance + second_upper = second.measurement.value + second.measurement.tolerance + if first_lower <= second_upper and second_lower <= first_upper: + return _rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_height_comparison" + ) + answer = intent.object_query if first_lower > second_upper else intent.comparison_query + example = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.comparison_query}-height-comparison", + _render_question(intent), + answer, + "choice", + (objects[0].id, other_objects[0].id), + (intent.object_query, intent.comparison_query), + ) + return GroundTruthResult( + intent, example, "answered", answer, None, (objects[0], other_objects[0]), trace + ) + + +def _compare_left_right( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + generator: VqaGroundTruthGenerator, +) -> GroundTruthResult: + if len(objects) != 1 or intent.comparison_query is None: + return _rejected_result(frame, intent, objects, trace, "ambiguous_first_relation_object") + other_objects, other_trace = generator.ground(frame, intent.comparison_query) + trace = (*trace, *other_trace) + if len(other_objects) != 1: + return _rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_second_relation_object" + ) + relation = generator.primitives.classify_horizontal_relation(objects[0], other_objects[0]) + trace = ( + *trace, + ToolTrace( + "classify_horizontal_relation", + relation.relation or relation.rejection_reason or "rejected", + ), + ) + if relation.relation is None: + return _rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + relation.rejection_reason or "horizontal_relation_rejected", + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.comparison_query}-left-right", + _render_question(intent), + relation.relation, + "choice", + (objects[0].id, other_objects[0].id), + ("left", "right"), + ) + return GroundTruthResult( + intent, + example, + "answered", + relation.relation, + None, + (objects[0], other_objects[0]), + trace, + ) + + def _compare_nearest_by_side( frame: CalibratedFrame, intent: QuestionIntent, diff --git a/dimos/benchmark/vqa/generation/oracle_tools.py b/dimos/benchmark/vqa/generation/oracle_tools.py index e3c3be974a..73cca5958d 100644 --- a/dimos/benchmark/vqa/generation/oracle_tools.py +++ b/dimos/benchmark/vqa/generation/oracle_tools.py @@ -8,7 +8,13 @@ from langchain_core.tools import StructuredTool -from dimos.benchmark.vqa.generation.primitives.choices import height_choice_window +from dimos.benchmark.vqa.generation.primitives.choices import ( + CAMERA_RANGE_CHOICES, + COUNT_CHOICES, + camera_range_choice, + count_choice, + height_choice_window, +) from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object from dimos.benchmark.vqa.models import ( @@ -65,6 +71,29 @@ def tools(self) -> list[StructuredTool]: "or right." ), ), + StructuredTool.from_function( + self.count_grounded_objects, + name="count_grounded_objects", + description="Count opaque grounded object IDs into fixed public count buckets.", + ), + StructuredTool.from_function( + self.bucket_camera_range, + name="bucket_camera_range", + description="Bucket one opaque object's private camera-origin range into fixed public choices.", + ), + StructuredTool.from_function( + self.compare_nearest_by_side, + name="compare_nearest_by_side", + description="Choose whether the nearest opaque left or right object is closer to the camera.", + ), + StructuredTool.from_function( + self.compare_left_right, + name="compare_left_right", + description=( + "Choose whether one opaque object is left or right of another from private " + "camera-frame support centroids. Rejects ambiguous separation." + ), + ), StructuredTool.from_function( self.select_closest_object, name="select_closest_object", @@ -85,6 +114,14 @@ def tools(self) -> list[StructuredTool]: "Measure one opaque grounded object above one opaque accepted ground-plane ID." ), ), + StructuredTool.from_function( + self.compare_heights, + name="compare_heights", + description=( + "Compare two opaque grounded objects against one opaque accepted ground-plane ID. " + "Rejects overlapping physical-height uncertainty." + ), + ), StructuredTool.from_function( self.classify_door_state, name="classify_door_state", @@ -202,6 +239,101 @@ def select_nearest_object(self, object_ids: list[str], side: str | None = None) self._results.append(result) return json.dumps(_tool_payload(result, object_id=nearest.id)) + def count_grounded_objects(self, object_ids: list[str]) -> str: + """Count unique grounded object IDs into fixed public count choices.""" + objects = self._lookup_objects("count_grounded_objects", object_ids) + if objects is None: + return self._record_rejection( + "count_grounded_objects", "", [], "unknown_or_duplicate_object_id" + ) + if not objects: + return self._record_rejection("count_grounded_objects", "", [], "no_grounded_object") + result = OracleToolResult( + "count_grounded_objects", + objects[0].label, + tuple(_grounding_evidence(item) for item in objects), + choice=count_choice(len(objects)), + choices=COUNT_CHOICES, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def bucket_camera_range(self, object_id: str) -> str: + """Map one grounded object's camera-origin range into fixed public choices.""" + object = self._objects.get(object_id) + if object is None: + return self._record_rejection("bucket_camera_range", object_id, [], "unknown_object_id") + measurement = OracleMeasurement( + object.range_m, + "m", + 0.0, + ("camera_origin_euclidean_range",), + (f"grounding:v1:{object.id}",), + ) + result = OracleToolResult( + "bucket_camera_range", + object.label, + (_grounding_evidence(object),), + measurement=measurement, + choice=camera_range_choice(object.range_m), + choices=CAMERA_RANGE_CHOICES, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def compare_nearest_by_side(self, object_ids: list[str]) -> str: + """Compare the nearest left and right grounded objects by camera range.""" + objects = self._lookup_objects("compare_nearest_by_side", object_ids) + if objects is None: + return self._record_rejection( + "compare_nearest_by_side", "", [], "unknown_or_duplicate_object_id" + ) + left = select_nearest_object(objects, "left") + right = select_nearest_object(objects, "right") + if left is None or right is None: + return self._record_rejection( + "compare_nearest_by_side", "", [], "missing_grounded_side" + ) + if left.range_m == right.range_m: + return self._record_rejection( + "compare_nearest_by_side", "", [], "ambiguous_nearest_by_side" + ) + choice = "left" if left.range_m < right.range_m else "right" + result = OracleToolResult( + "compare_nearest_by_side", + left.label, + (_grounding_evidence(left), _grounding_evidence(right)), + choice=choice, + choices=("left", "right"), + ) + self._results.append(result) + return json.dumps(_tool_payload(result, left_object_id=left.id, right_object_id=right.id)) + + def compare_left_right(self, first_object_id: str, second_object_id: str) -> str: + """Classify one grounded object's left/right relation to another grounded object.""" + first = self._objects.get(first_object_id) + second = self._objects.get(second_object_id) + if first is None or second is None: + return self._record_rejection("compare_left_right", "", [], "unknown_object_id") + relation = self._primitives.classify_horizontal_relation(first, second) + if relation.relation is None: + return self._record_rejection( + "compare_left_right", + f"{first.label},{second.label}", + list(relation.quality_flags), + relation.rejection_reason, + ) + result = OracleToolResult( + "compare_left_right", + f"{first.label},{second.label}", + (_grounding_evidence(first), _grounding_evidence(second)), + choice=relation.relation, + choices=("left", "right"), + quality_flags=relation.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + def select_closest_object(self, target_id: str, candidate_ids: list[str]) -> str: """Select one candidate closest to a target by private 3D support-point proximity.""" target = self._objects.get(target_id) @@ -252,16 +384,7 @@ def measure_height(self, object_id: str, plane_id: str) -> str: measured.rejection_reason, ) measurement = measured.measurement - evidence = OracleEvidence( - f"height:v1:{object.id}", - "v1", - object.id, - object.label, - object.range_m, - object.horizontal_direction, - object.point_count, - measurement, - ) + evidence = _height_evidence(object, measurement) result = OracleToolResult( "measure_height", object.label, @@ -275,6 +398,52 @@ def measure_height(self, object_id: str, plane_id: str) -> str: self._results.append(result) return json.dumps(_tool_payload(result, measurement_id=measurement_id)) + def compare_heights(self, first_object_id: str, second_object_id: str, plane_id: str) -> str: + """Choose the taller object only when one shared-plane measurement is unambiguous.""" + first = self._objects.get(first_object_id) + second = self._objects.get(second_object_id) + plane = self._planes.get(plane_id) + if first is None or second is None: + return self._record_rejection("compare_heights", "", [], "unknown_object_id") + if first.id == second.id: + return self._record_rejection("compare_heights", first.label, [], "duplicate_object_id") + if plane is None: + return self._record_rejection("compare_heights", "", [], "unknown_plane_id") + first_height = self._primitives.measure_height(first, plane) + second_height = self._primitives.measure_height(second, plane) + if first_height.measurement is None or second_height.measurement is None: + return self._record_rejection( + "compare_heights", + "", + [*first_height.quality_flags, *second_height.quality_flags], + first_height.rejection_reason + or second_height.rejection_reason + or "height_measurement_rejected", + ) + first_measurement = first_height.measurement + second_measurement = second_height.measurement + first_lower = first_measurement.value - first_measurement.tolerance + second_lower = second_measurement.value - second_measurement.tolerance + first_upper = first_measurement.value + first_measurement.tolerance + second_upper = second_measurement.value + second_measurement.tolerance + if first_lower <= second_upper and second_lower <= first_upper: + return self._record_rejection("compare_heights", "", [], "ambiguous_height_comparison") + choice = first.label if first_lower > second_upper else second.label + result = OracleToolResult( + "compare_heights", + f"{first.label},{second.label}", + ( + _height_evidence(first, first_measurement), + _height_evidence(second, second_measurement), + ), + choice=choice, + choices=(first.label, second.label), + plane=plane, + quality_flags=(*first_height.quality_flags, *second_height.quality_flags), + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + def classify_door_state(self, object_id: str) -> str: """Classify one grounded door against the surrounding point-cloud plane.""" object = self._objects.get(object_id) @@ -366,6 +535,17 @@ def _record_rejection(self, tool: str, query: str, flags: list[str], reason: str self._results.append(result) return json.dumps(_tool_payload(result)) + def _lookup_objects(self, tool: str, object_ids: list[str]) -> list[GroundedObject] | None: + if len(set(object_ids)) != len(object_ids): + return None + objects: list[GroundedObject] = [] + for object_id in object_ids: + object = self._objects.get(object_id) + if object is None: + return None + objects.append(object) + return objects + def _id(self, kind: str) -> str: self._next_id += 1 return f"{kind}:v1:{self._next_id:04d}" @@ -383,6 +563,19 @@ def _grounding_evidence(item: GroundedObject) -> OracleEvidence: ) +def _height_evidence(item: GroundedObject, measurement: OracleMeasurement) -> OracleEvidence: + return OracleEvidence( + f"height:v1:{item.id}", + "v1", + item.id, + item.label, + item.range_m, + item.horizontal_direction, + item.point_count, + measurement, + ) + + def _tool_payload(result: OracleToolResult, **identifiers: Any) -> dict[str, Any]: return { "tool": result.tool, diff --git a/dimos/benchmark/vqa/generation/primitives/choices.py b/dimos/benchmark/vqa/generation/primitives/choices.py index 595a8a0b72..af45ca20c8 100644 --- a/dimos/benchmark/vqa/generation/primitives/choices.py +++ b/dimos/benchmark/vqa/generation/primitives/choices.py @@ -4,6 +4,35 @@ from bisect import bisect_right +COUNT_CHOICES = ("1-2", "3-4", "5-7", "8+") +CAMERA_RANGE_CHOICES = ("under 1 m", "1 to under 2 m", "2 to under 4 m", "4 m or more") + + +def count_choice(count: int) -> str: + """Return the public count bucket for one or more grounded instances.""" + if count < 1: + raise ValueError("count must be positive") + if count <= 2: + return COUNT_CHOICES[0] + if count <= 4: + return COUNT_CHOICES[1] + if count <= 7: + return COUNT_CHOICES[2] + return COUNT_CHOICES[3] + + +def camera_range_choice(range_m: float) -> str: + """Return the public camera-origin range bucket for a grounded object.""" + if range_m < 0: + raise ValueError("range must be non-negative") + if range_m < 1.0: + return CAMERA_RANGE_CHOICES[0] + if range_m < 2.0: + return CAMERA_RANGE_CHOICES[1] + if range_m < 4.0: + return CAMERA_RANGE_CHOICES[2] + return CAMERA_RANGE_CHOICES[3] + def height_choice_window(height_m: float) -> tuple[tuple[str, ...], str]: """Generate a local, exhaustive four-choice window around a private height.""" diff --git a/dimos/benchmark/vqa/generation/primitives/contracts.py b/dimos/benchmark/vqa/generation/primitives/contracts.py index 7aa9d59774..e27e6b027f 100644 --- a/dimos/benchmark/vqa/generation/primitives/contracts.py +++ b/dimos/benchmark/vqa/generation/primitives/contracts.py @@ -40,6 +40,15 @@ class ClosestObjectResult: rejection_reason: str | None = None +@dataclass(frozen=True) +class HorizontalRelationResult: + """A pairwise camera-frame horizontal relation or its explicit rejection.""" + + relation: Literal["left", "right"] | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + @dataclass(frozen=True) class ForwardPathResult: """A conservative visible-corridor classification from point-cloud evidence.""" diff --git a/dimos/benchmark/vqa/generation/primitives/frame.py b/dimos/benchmark/vqa/generation/primitives/frame.py index 028d7da9fa..80009f7281 100644 --- a/dimos/benchmark/vqa/generation/primitives/frame.py +++ b/dimos/benchmark/vqa/generation/primitives/frame.py @@ -11,6 +11,7 @@ DoorStateResult, ForwardPathResult, HeightMeasurementResult, + HorizontalRelationResult, ) from dimos.benchmark.vqa.generation.primitives.geometry import ( PlaneFitResult, @@ -228,6 +229,23 @@ def select_closest_object( return ClosestObjectResult(None, None, (), "ambiguous_object_proximity") return ClosestObjectResult(distances[0][1], distances[0][0], ("object_centroid_proximity",)) + def classify_horizontal_relation( + self, first: GroundedObject, second: GroundedObject + ) -> HorizontalRelationResult: + """Classify whether the first object's support centroid is left or right of the second's.""" + if first.id == second.id: + return HorizontalRelationResult(None, (), "duplicate_object_id") + first_points = self._object_points(first) + second_points = self._object_points(second) + if first_points is None or second_points is None: + return HorizontalRelationResult(None, (), "insufficient_object_support") + horizontal_offset_m = float(np.median(first_points[:, 0]) - np.median(second_points[:, 0])) + if abs(horizontal_offset_m) < 0.1: + return HorizontalRelationResult(None, (), "ambiguous_horizontal_relation") + return HorizontalRelationResult( + "left" if horizontal_offset_m < 0 else "right", ("camera_frame_support_centroids",) + ) + def classify_forward_path(self) -> ForwardPathResult: """Classify the observed camera-forward corridor as clear or blocked.""" ground_fit = self.fit_ground_plane() diff --git a/dimos/benchmark/vqa/generation/question_agent.py b/dimos/benchmark/vqa/generation/question_agent.py index 5fe9d65b8d..1f2cdb5fba 100644 --- a/dimos/benchmark/vqa/generation/question_agent.py +++ b/dimos/benchmark/vqa/generation/question_agent.py @@ -21,16 +21,22 @@ QUESTION_PROMPT = """Select up to 5 challenging but visually well-supported single-frame VQA intents. Inspect only this image. Do not assume depth, point clouds, calibration, metadata, or temporal context. Return JSON only: an array of objects with kind, object_query, and threshold_m only for -within_distance, plus candidate_queries only for closest_object. kind must be one of presence, -horizontal_direction, within_distance, compare_nearest_by_side, door_state, closest_object, or forward_path. +within_distance, candidate_queries only for closest_object, and comparison_query only for +compare_height. kind must be one of presence, horizontal_direction, within_distance, visible_count, +camera_range, compare_nearest_by_side, compare_left_right, compare_height, door_state, closest_object, +or forward_path. Use threshold_m: 3.0 for within_distance. Use image context to select only intents likely to produce a useful geometric case: emit compare_nearest_by_side only when at least two visible instances of the same object appear on opposite image sides; emit horizontal_direction only for a visible object; and prefer relational or directional intents over presence. Emit door_state only for a clearly visible door with nearby visible structure. Diversify object classes and question families when the scene supports them. -Emit closest_object only when one target and at least two distinct candidate object types are -visible; candidate_queries must name the visible candidate types. +Emit visible_count only when at least one repeated visible object is present. Emit camera_range only +for a visible object. Emit compare_height only for two distinct visible upright object types resting +on the visible ground; comparison_query must name the other object type. Emit compare_left_right only +for two distinct visible object types; comparison_query must name the other object type. Emit closest_object only when +one target and at least two distinct candidate object types are visible; candidate_queries must name +the visible candidate types. Emit forward_path only when the center foreground and visible floor provide enough context to judge the local path directly ahead. Use object_query: "forward path" for forward_path. Do not return bare object names, floors, walls, ceilings, background surfaces, @@ -45,15 +51,20 @@ object resting on visible ground, use deferred_height_choice. Its choices are generated privately from a successful height measurement. Aim for a diverse set of questions that use the visible scene composition: relative left/center/right position, which listed object is closest to a named target, -count choices for a visibly repeated object, and height for an upright grounded object. For -closest-object questions, use distinct visible candidate object types as the fixed choices. Use concise, -mutually exclusive fixed choices with two to four options. For a clearly visible door with nearby -visible structure, you may ask whether it is open or closed with exactly ["open", "closed"]. Use -the fixed choices ["clear", "blocked"] only for a visibly supported local path directly ahead. -Use object_queries for every referenced object and tool_hints from "detect_objects", "segment_detections", "ground_masks", -"fit_ground_plane", "select_nearest_object", "measure_height", "classify_door_state", or -"select_closest_object", "classify_forward_path", "bucket_measurement" when applicable. Use visibility/presence questions only when no stronger -geometric question is available. +and height for an upright grounded object. For a visibly repeated object, count questions must use exactly +["1-2", "3-4", "5-7", "8+"]; camera-distance questions must use exactly ["under 1 m", +"1 to under 2 m", "2 to under 4 m", "4 m or more"]. For a pairwise left/right range question, +use exactly ["left", "right"]. For an A/B left-right relation question, use exactly ["left", "right"] +and name both objects. For a pairwise height question, use the two distinct object types as the choices. +For closest-object questions, use distinct visible candidate object types as the fixed choices. Use concise, +mutually exclusive fixed choices with two to four options. For a clearly visible door with nearby visible +structure, you may ask whether it is open or closed with exactly ["open", "closed"]. Use the fixed choices +["clear", "blocked"] only for a visibly supported local path directly ahead. +Use object_queries for every referenced object and tool_hints from "detect_objects", "segment_detections", +"ground_masks", "count_grounded_objects", "bucket_camera_range", "compare_nearest_by_side", +"compare_left_right", "fit_ground_plane", "measure_height", "compare_heights", "classify_door_state", "select_closest_object", +"classify_forward_path", or "bucket_measurement" when applicable. Use visibility/presence questions only +when no stronger geometric question is available. Do not ask about color, material, text, intent, full physical size, hidden parts, or exact metric distances without a supplied choice contract. Use only visible objects. Do not include answers, explanations, Markdown, or background surfaces.""" @@ -94,7 +105,11 @@ def propose(self, image: Image) -> list[QuestionIntent]: "presence", "horizontal_direction", "within_distance", + "visible_count", + "camera_range", "compare_nearest_by_side", + "compare_left_right", + "compare_height", "door_state", "closest_object", "forward_path", @@ -109,11 +124,20 @@ def propose(self, image: Image) -> list[QuestionIntent]: if kind != "within_distance": threshold = None candidates = _string_tuple(item.get("candidate_queries"), "candidate_queries") + comparison_query = item.get("comparison_query") if kind == "closest_object" and (len(candidates) < 2 or query in candidates): raise ValueError("closest_object requires two distinct candidate_queries") if kind == "forward_path" and query != "forward path": raise ValueError('forward_path requires object_query "forward path"') - intents.append(QuestionIntent(kind, query, threshold, candidates)) + if kind in ("compare_left_right", "compare_height") and ( + not isinstance(comparison_query, str) + or not comparison_query + or comparison_query == query + ): + raise ValueError(f"{kind} requires a distinct comparison_query") + if kind not in ("compare_left_right", "compare_height"): + comparison_query = None + intents.append(QuestionIntent(kind, query, threshold, candidates, comparison_query)) return intents @@ -182,6 +206,8 @@ def _intents_for_query(query: str) -> list[QuestionIntent]: QuestionIntent(kind="presence", object_query=query), QuestionIntent(kind="horizontal_direction", object_query=query), QuestionIntent(kind="within_distance", object_query=query, threshold_m=3.0), + QuestionIntent(kind="visible_count", object_query=query), + QuestionIntent(kind="camera_range", object_query=query), QuestionIntent(kind="compare_nearest_by_side", object_query=query), ] diff --git a/dimos/benchmark/vqa/generation/test_agents.py b/dimos/benchmark/vqa/generation/test_agents.py index 68205344b0..6b001341e9 100644 --- a/dimos/benchmark/vqa/generation/test_agents.py +++ b/dimos/benchmark/vqa/generation/test_agents.py @@ -7,9 +7,20 @@ import numpy as np from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator +from dimos.benchmark.vqa.generation.primitives.contracts import ( + HeightMeasurementResult, + HorizontalRelationResult, +) from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives from dimos.benchmark.vqa.generation.question_agent import OpenAIQuestionAgent -from dimos.benchmark.vqa.models import CalibratedFrame, GroundingConfig, QuestionIntent +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + GroundedObject, + GroundingConfig, + GroundPlaneEstimate, + OracleMeasurement, + QuestionIntent, +) from dimos.models.vl.openai import OpenAIVlModel from dimos.msgs.geometry_msgs.Transform import Transform from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo @@ -104,6 +115,8 @@ def test_question_agent_returns_constrained_intents() -> None: QuestionIntent(kind="presence", object_query="chair"), QuestionIntent(kind="horizontal_direction", object_query="chair"), QuestionIntent(kind="within_distance", object_query="chair", threshold_m=3.0), + QuestionIntent(kind="visible_count", object_query="chair"), + QuestionIntent(kind="camera_range", object_query="chair"), QuestionIntent(kind="compare_nearest_by_side", object_query="chair"), ] @@ -170,6 +183,32 @@ def query(self, image: Image, prompt: str) -> str: assert intents == [QuestionIntent(kind="forward_path", object_query="forward path")] +def test_question_agent_accepts_count_range_and_height_comparison_intents() -> None: + class _GeometryQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "visible_count" in prompt + assert "comparison_query" in prompt + return """[ + {"kind":"visible_count","object_query":"chair"}, + {"kind":"camera_range","object_query":"lamp"}, + {"kind":"compare_left_right","object_query":"chair","comparison_query":"table"}, + {"kind":"compare_height","object_query":"chair","comparison_query":"table"} + ]""" + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _GeometryQuestionModel())).propose( + frame.image + ) + + assert intents == [ + QuestionIntent(kind="visible_count", object_query="chair"), + QuestionIntent(kind="camera_range", object_query="lamp"), + QuestionIntent(kind="compare_left_right", object_query="chair", comparison_query="table"), + QuestionIntent(kind="compare_height", object_query="chair", comparison_query="table"), + ] + + def test_ground_truth_agent_records_tools_and_rejects_unsupported_question() -> None: frame, detection = _frame_and_detection() agent = _agent( @@ -307,3 +346,99 @@ def test_ground_truth_agent_rejects_side_comparison_without_both_sides() -> None assert result.status == "rejected" assert result.reason == "missing_grounded_side" + + +def test_ground_truth_agent_buckets_visible_count_and_camera_range() -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + + count = agent.answer(frame, QuestionIntent(kind="visible_count", object_query="chair")) + camera_range = agent.answer(frame, QuestionIntent(kind="camera_range", object_query="chair")) + + assert count.answer == "1-2" + assert count.question.allowed_answers == ("1-2", "3-4", "5-7", "8+") + assert camera_range.answer == "1 to under 2 m" + assert camera_range.question.allowed_answers == ( + "under 1 m", + "1 to under 2 m", + "2 to under 4 m", + "4 m or more", + ) + + +def test_ground_truth_agent_compares_ground_plane_relative_heights(monkeypatch: object) -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + chair = GroundedObject("chair-0", "chair", 8, 1.0, "left") + table = GroundedObject("table-0", "table", 8, 2.0, "right") + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + monkeypatch.setattr( + agent, + "ground", + lambda _frame, query: ([chair] if query == "chair" else [table], ()), + ) + monkeypatch.setattr( + agent.primitives, + "fit_ground_plane", + lambda: type("Fit", (), {"estimate": plane, "rejection_reason": None})(), + ) + monkeypatch.setattr( + agent.primitives, + "measure_height", + lambda item, accepted_plane: HeightMeasurementResult( + item, + accepted_plane, + OracleMeasurement(0.8 if item == chair else 0.5, "m", 0.05, (), ()), + (), + ), + ) + + result = agent.answer( + frame, + QuestionIntent(kind="compare_height", object_query="chair", comparison_query="table"), + ) + + assert result.status == "answered" + assert result.answer == "chair" + assert result.question.allowed_answers == ("chair", "table") + + +def test_ground_truth_agent_compares_pairwise_left_right(monkeypatch: object) -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + chair = GroundedObject("chair-0", "chair", 8, 1.0, "left") + table = GroundedObject("table-0", "table", 8, 2.0, "right") + monkeypatch.setattr( + agent, + "ground", + lambda _frame, query: ([chair] if query == "chair" else [table], ()), + ) + monkeypatch.setattr( + agent.primitives, + "classify_horizontal_relation", + lambda first, second: HorizontalRelationResult("left", ("camera_frame_support_centroids",)), + ) + + result = agent.answer( + frame, + QuestionIntent(kind="compare_left_right", object_query="chair", comparison_query="table"), + ) + + assert result.status == "answered" + assert result.answer == "left" + assert result.question.allowed_answers == ("left", "right") diff --git a/dimos/benchmark/vqa/generation/test_oracle.py b/dimos/benchmark/vqa/generation/test_oracle.py index aa039e98a3..53fcec067e 100644 --- a/dimos/benchmark/vqa/generation/test_oracle.py +++ b/dimos/benchmark/vqa/generation/test_oracle.py @@ -15,7 +15,15 @@ validate_oracle_answer, ) from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry -from dimos.benchmark.vqa.generation.primitives.choices import height_choice_window +from dimos.benchmark.vqa.generation.primitives.choices import ( + camera_range_choice, + count_choice, + height_choice_window, +) +from dimos.benchmark.vqa.generation.primitives.contracts import ( + HeightMeasurementResult, + HorizontalRelationResult, +) from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives from dimos.benchmark.vqa.generation.primitives.geometry import ( classify_door_plane_angle, @@ -35,6 +43,7 @@ GroundingConfig, GroundPlaneEstimate, OracleEvidence, + OracleMeasurement, OracleToolResult, QuestionProposal, ) @@ -359,9 +368,14 @@ def test_local_registry_exposes_geometry_tools() -> None: "segment_detections", "ground_masks", "select_nearest_object", + "count_grounded_objects", + "bucket_camera_range", + "compare_nearest_by_side", + "compare_left_right", "select_closest_object", "fit_ground_plane", "measure_height", + "compare_heights", "classify_door_state", "classify_forward_path", "bucket_measurement", @@ -381,6 +395,70 @@ def test_height_choice_window_is_local_and_deterministic() -> None: assert height_choice_window(3.0)[1] == "over 2.0 m" +def test_count_and_camera_range_choices_are_fixed_and_non_overlapping() -> None: + assert [count_choice(value) for value in (1, 2, 3, 4, 5, 7, 8)] == [ + "1-2", + "1-2", + "3-4", + "3-4", + "5-7", + "5-7", + "8+", + ] + assert [camera_range_choice(value) for value in (0.99, 1.0, 2.0, 4.0)] == [ + "under 1 m", + "1 to under 2 m", + "2 to under 4 m", + "4 m or more", + ] + + +def test_local_registry_buckets_count_range_and_pairwise_side() -> None: + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + left = GroundedObject("left", "chair", 8, 1.0, "left") + right = GroundedObject("right", "chair", 8, 2.0, "right") + registry._objects = {left.id: left, right.id: right} + + count = json.loads(registry.count_grounded_objects([left.id, right.id])) + camera_range = json.loads(registry.bucket_camera_range(right.id)) + side = json.loads(registry.compare_nearest_by_side([left.id, right.id])) + + assert count["choice"] == "1-2" + assert camera_range["choice"] == "2 to under 4 m" + assert side["choice"] == "left" + + +def test_local_registry_compares_pairwise_relation_and_height(monkeypatch: Any) -> None: + primitives = _frame_primitives(_measurement_frame()) + registry = LocalOracleToolRegistry(primitives) + chair = GroundedObject("chair", "chair", 8, 1.0, "left") + table = GroundedObject("table", "table", 8, 2.0, "right") + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + registry._objects = {chair.id: chair, table.id: table} + registry._planes = {"plane": plane} + monkeypatch.setattr( + primitives, + "classify_horizontal_relation", + lambda first, second: HorizontalRelationResult("left", ("camera_frame_support_centroids",)), + ) + monkeypatch.setattr( + primitives, + "measure_height", + lambda item, accepted_plane: HeightMeasurementResult( + item, + accepted_plane, + OracleMeasurement(0.8 if item == chair else 0.5, "m", 0.05, (), ()), + (), + ), + ) + + relation = json.loads(registry.compare_left_right(chair.id, table.id)) + height = json.loads(registry.compare_heights(chair.id, table.id, "plane")) + + assert relation["choice"] == "left" + assert height["choice"] == "chair" + + def test_door_state_accepts_clear_plane_angles_and_rejects_ajar() -> None: door = GroundPlaneEstimate((1.0, 0.0, 0.0), 0.0, 20, 20, 0.01) closed = GroundPlaneEstimate((1.0, 0.0, 0.0), 0.0, 20, 20, 0.01) @@ -419,6 +497,24 @@ def test_closest_object_uses_point_cloud_centroids_and_rejects_ties(monkeypatch: ) +def test_horizontal_relation_uses_camera_frame_support_centroids(monkeypatch: Any) -> None: + left = GroundedObject("left", "chair", 8, 1.0, "left") + right = GroundedObject("right", "table", 8, 2.0, "right") + primitives = _frame_primitives(_measurement_frame()) + centers = { + "left": np.tile((-0.3, 0.0, 1.0), (6, 1)), + "right": np.tile((0.3, 0.0, 1.0), (6, 1)), + } + monkeypatch.setattr(primitives, "_object_points", lambda item: centers[item.id]) + + assert primitives.classify_horizontal_relation(left, right).relation == "left" + monkeypatch.setattr(primitives, "_object_points", lambda item: np.zeros((6, 3))) + assert ( + primitives.classify_horizontal_relation(left, right).rejection_reason + == "ambiguous_horizontal_relation" + ) + + def test_forward_corridor_requires_ground_support_and_detects_obstacles() -> None: ground = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) floor = np.asarray( diff --git a/dimos/benchmark/vqa/models.py b/dimos/benchmark/vqa/models.py index db79115c2e..33c2f81061 100644 --- a/dimos/benchmark/vqa/models.py +++ b/dimos/benchmark/vqa/models.py @@ -90,7 +90,11 @@ class VqaExample: "presence", "horizontal_direction", "within_distance", + "visible_count", + "camera_range", "compare_nearest_by_side", + "compare_left_right", + "compare_height", "door_state", "closest_object", "forward_path", @@ -105,6 +109,7 @@ class QuestionIntent: object_query: str threshold_m: float | None = None candidate_queries: tuple[str, ...] = () + comparison_query: str | None = None @dataclass(frozen=True) diff --git a/docs/benchmarking/vqa-generation/pipeline.md b/docs/benchmarking/vqa-generation/pipeline.md index 18199768b0..487c0bbfbe 100644 --- a/docs/benchmarking/vqa-generation/pipeline.md +++ b/docs/benchmarking/vqa-generation/pipeline.md @@ -64,7 +64,11 @@ every family. Private grounding still rejects unsupported or ambiguous candidate | Presence | `Is there a chair in the image?` | `yes`, `no` | | Horizontal direction | `Where is the nearest chair?` | `left`, `center`, `right` | | Distance threshold | `Is the nearest chair within 3 meters?` | `yes`, `no` | +| Visible count | `How many chairs are visible?` | `1-2`, `3-4`, `5-7`, `8+` | +| Camera range | `How far is the nearest chair from the camera?` | `under 1 m`, `1 to under 2 m`, `2 to under 4 m`, `4 m or more` | | Nearest by side | `Which chair is closer, the left or right one?` | `left`, `right` | +| Pairwise left/right | `Is the chair to the left or right of the table?` | `left`, `right` | +| Height comparison | `Which is taller: the chair or the table?` | `chair`, `table` | | Closest object | `Which object is closest to the chair: table or lamp?` | `table`, `lamp` | | Door state | `Is the door open or closed?` | `open`, `closed` | | Forward path | `Is the path directly ahead clear or blocked?` | `clear`, `blocked` | @@ -112,6 +116,14 @@ Closest-object questions require exactly one grounded target and one grounded in candidate choice. They compare private support-point centroids and reject candidates whose nearest two distances are within `0.15 m`. +Visible-count questions count only accepted grounded instances, rather than raw detections. Camera-range +questions use the nearest grounded instance's camera-origin Euclidean range. Height comparisons require +one accepted shared ground plane and one successful physical-height measurement per distinct object; +overlapping measurement uncertainty intervals are rejected. + +Pairwise left/right questions require exactly one grounded instance for each named object. They compare +their visible support-point centroids in the camera horizontal axis and reject separation under `0.1 m`. + Forward-path questions require a fitted visible ground plane and enough ground support in each third of the center camera-forward corridor from `0.5-3.0 m`. Supported non-ground points block the corridor; incomplete ground support or sparse obstacle evidence is rejected. @@ -160,6 +172,31 @@ compare_nearest_by_side(A) -> compare the two range_m values: left/right ``` +```text +visible_count(A) +-> detect_objects(A) -> segment_detections(...) -> ground_masks(...) -> grounded A instances +-> bucket accepted instance count: 1-2 / 3-4 / 5-7 / 8+ +``` + +```text +camera_range(A) +-> detect_objects(A) -> segment_detections(...) -> ground_masks(...) -> grounded A instances +-> select_nearest_object(...) -> bucket camera-origin range +``` + +```text +compare_left_right(A, B) +-> ground exactly one A and one B -> compare camera-frame support centroids +-> separation under 0.1 m: reject -> otherwise choose left/right +``` + +```text +compare_height(A, B) +-> ground exactly one A and one B -> fit_ground_plane() +-> measure_height(A, plane) and measure_height(B, plane) +-> reject overlapping uncertainty intervals -> choose taller A/B +``` + ### Agentic The private oracle chooses a sequence from the same read-only primitives used by constrained recipes: @@ -170,9 +207,14 @@ The private oracle chooses a sequence from the same read-only primitives used by | `segment_detections` | detection ID | Mask ID and accepted mask count. | | `ground_masks` | mask ID | Grounded object IDs, range, side, point support, evidence IDs. | | `select_nearest_object` | object IDs, optional side | Nearest grounded object ID. | +| `count_grounded_objects` | object IDs | Fixed count bucket and cited grounded instances. | +| `bucket_camera_range` | object ID | Fixed camera-origin range bucket and cited object. | +| `compare_nearest_by_side` | object IDs | Public `left` or `right` choice from nearest grounded objects. | +| `compare_left_right` | two object IDs | Public pairwise `left` or `right` relation, or an ambiguity rejection. | | `select_closest_object` | target ID, candidate IDs | Candidate nearest to target by private support-point centroids. | | `fit_ground_plane` | none | Plane ID, plane estimate, residual, inlier support, quality flags. | | `measure_height` | object ID, plane ID | Measurement ID, private height, uncertainty, provenance, quality flags. | +| `compare_heights` | two object IDs, plane ID | Taller object choice from shared-plane measurements, or rejection. | | `classify_door_state` | object ID | Public `open` or `closed` choice, or a private geometry rejection. | | `classify_forward_path` | none | Public `clear` or `blocked` choice, or a private visibility rejection. | | `bucket_measurement` | measurement ID | Public answer-conditioned height choices and matching choice. | From 337cc5852aec7d8d4a25b3c81c159adc73fad352 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Tue, 11 Aug 2026 15:11:34 -0700 Subject: [PATCH 12/12] refactor(benchmark): unify VQA oracle primitives --- dimos/benchmark/vqa/generation/families.py | 485 ++++++++++++++++++ .../vqa/generation/ground_truth_generator.py | 390 +------------- dimos/benchmark/vqa/generation/oracle.py | 52 +- .../benchmark/vqa/generation/oracle_tools.py | 477 ++++++++++++----- .../vqa/generation/primitives/choices.py | 14 + .../vqa/generation/primitives/contracts.py | 45 +- .../vqa/generation/primitives/frame.py | 287 +++++++++-- .../vqa/generation/primitives/geometry.py | 178 ++++++- .../vqa/generation/question_agent.py | 56 +- dimos/benchmark/vqa/generation/test_agents.py | 105 +++- dimos/benchmark/vqa/generation/test_oracle.py | 273 +++++++--- dimos/benchmark/vqa/models.py | 3 + .../vqa-generation/infrastructure.md | 21 +- docs/benchmarking/vqa-generation/pipeline.md | 157 +++--- 14 files changed, 1801 insertions(+), 742 deletions(-) create mode 100644 dimos/benchmark/vqa/generation/families.py diff --git a/dimos/benchmark/vqa/generation/families.py b/dimos/benchmark/vqa/generation/families.py new file mode 100644 index 0000000000..5119f31a9c --- /dev/null +++ b/dimos/benchmark/vqa/generation/families.py @@ -0,0 +1,485 @@ +# Copyright 2026 Dimensional Inc. +"""Recipes and result construction for constrained single-frame VQA families.""" + +from __future__ import annotations + +from collections.abc import Callable + +import numpy as np + +from dimos.benchmark.vqa.generation.primitives.choices import ( + CAMERA_RANGE_CHOICES, + COUNT_CHOICES, + OPENING_WIDTH_CHOICES, + camera_range_choice, + count_choice, + opening_width_choice, +) +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + GroundedObject, + GroundTruthResult, + QuestionIntent, + ToolTrace, + VqaExample, +) + +Ground = Callable[[CalibratedFrame, str], tuple[list[GroundedObject], tuple[ToolTrace, ...]]] + + +def render_question(intent: QuestionIntent) -> str: + if intent.kind == "presence": + return f"Is there a {intent.object_query} in the image? Answer yes or no." + if intent.kind == "horizontal_direction": + return f"Where is the nearest {intent.object_query}: left, center, or right?" + if intent.kind == "visible_count": + return f"How many {intent.object_query}s are visible?" + if intent.kind == "camera_range": + return f"How far is the nearest {intent.object_query} from the camera?" + if intent.kind == "compare_nearest_by_side": + return f"Which {intent.object_query} is closer: the left one or the right one?" + if intent.kind == "compare_left_right": + return ( + f"Is the {intent.object_query} to the left or right of the {intent.comparison_query}?" + ) + if intent.kind == "compare_height": + return f"Which is taller: the {intent.object_query} or the {intent.comparison_query}?" + if intent.kind == "object_on_support": + return f"Is the {intent.object_query} on the {intent.comparison_query}? Answer yes or no." + if intent.kind == "opening_width": + return f"How wide is the {intent.object_query}?" + if intent.kind == "door_state": + return f"Is the {intent.object_query} open or closed?" + if intent.kind == "closest_object": + return f"Which object is closest to the {intent.object_query}: {', '.join(intent.candidate_queries)}?" + if intent.kind == "forward_path": + return "Is the path directly ahead clear or blocked?" + return f"Is the nearest {intent.object_query} within {intent.threshold_m or 3:g} meters? Answer yes or no." + + +def rejected_result( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + reason: str, +) -> GroundTruthResult: + rejected = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.kind}", render_question(intent), "", "", () + ) + return GroundTruthResult(intent, rejected, "rejected", None, reason, tuple(objects), trace) + + +def count_visible_objects( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + answer = count_choice(len(objects)) + example = VqaExample( + f"{frame.id}-{intent.object_query}-visible-count", + render_question(intent), + answer, + "choice", + tuple(item.id for item in objects), + COUNT_CHOICES, + ) + return GroundTruthResult(intent, example, "answered", answer, None, tuple(objects), trace) + + +def bucket_camera_range( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + selected = select_nearest_object(objects) + if selected is None: + return rejected_result(frame, intent, objects, trace, "no_grounded_object") + answer = camera_range_choice(selected.range_m) + example = VqaExample( + f"{frame.id}-{intent.object_query}-camera-range", + render_question(intent), + answer, + "choice", + (selected.id,), + CAMERA_RANGE_CHOICES, + ) + return GroundTruthResult(intent, example, "answered", answer, None, (selected,), trace) + + +def compare_heights( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, + ground: Ground, +) -> GroundTruthResult: + if len(objects) != 1 or intent.comparison_query is None: + return rejected_result(frame, intent, objects, trace, "ambiguous_first_height_object") + other_objects, other_trace = ground(frame, intent.comparison_query) + trace = (*trace, *other_trace) + if len(other_objects) != 1: + return rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_second_height_object" + ) + plane_fit = primitives.fit_ground_plane() + trace = (*trace, ToolTrace("fit_ground_plane", plane_fit.rejection_reason or "accepted")) + if plane_fit.estimate is None: + return rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + plane_fit.rejection_reason or "ground_plane_rejected", + ) + first = primitives.measure_height(objects[0], plane_fit.estimate) + second = primitives.measure_height(other_objects[0], plane_fit.estimate) + trace = ( + *trace, + ToolTrace("measure_height", first.rejection_reason or objects[0].id), + ToolTrace("measure_height", second.rejection_reason or other_objects[0].id), + ) + if first.measurement is None or second.measurement is None: + return rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + first.rejection_reason or second.rejection_reason or "height_measurement_rejected", + ) + first_lower = first.measurement.value - first.measurement.tolerance + second_lower = second.measurement.value - second.measurement.tolerance + first_upper = first.measurement.value + first.measurement.tolerance + second_upper = second.measurement.value + second.measurement.tolerance + if first_lower <= second_upper and second_lower <= first_upper: + return rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_height_comparison" + ) + answer = intent.object_query if first_lower > second_upper else intent.comparison_query + example = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.comparison_query}-height-comparison", + render_question(intent), + answer, + "choice", + (objects[0].id, other_objects[0].id), + (intent.object_query, intent.comparison_query), + ) + return GroundTruthResult( + intent, example, "answered", answer, None, (objects[0], other_objects[0]), trace + ) + + +def compare_left_right( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, + ground: Ground, +) -> GroundTruthResult: + if len(objects) != 1 or intent.comparison_query is None: + return rejected_result(frame, intent, objects, trace, "ambiguous_first_relation_object") + other_objects, other_trace = ground(frame, intent.comparison_query) + trace = (*trace, *other_trace) + if len(other_objects) != 1: + return rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_second_relation_object" + ) + relation = primitives.classify_horizontal_relation(objects[0], other_objects[0]) + trace = ( + *trace, + ToolTrace( + "classify_horizontal_relation", + relation.relation or relation.rejection_reason or "rejected", + ), + ) + if relation.relation is None: + return rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + relation.rejection_reason or "horizontal_relation_rejected", + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.comparison_query}-left-right", + render_question(intent), + relation.relation, + "choice", + (objects[0].id, other_objects[0].id), + ("left", "right"), + ) + return GroundTruthResult( + intent, example, "answered", relation.relation, None, (objects[0], other_objects[0]), trace + ) + + +def classify_object_on_support( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, + ground: Ground, +) -> GroundTruthResult: + if len(objects) != 1 or intent.comparison_query is None: + return rejected_result(frame, intent, objects, trace, "ambiguous_supported_object") + supports, support_trace = ground(frame, intent.comparison_query) + trace = (*trace, *support_trace) + if len(supports) != 1: + return rejected_result( + frame, intent, [*objects, *supports], trace, "ambiguous_support_object" + ) + ground_fit = primitives.fit_ground_plane() + support_fit = primitives.fit_object_surface_plane(supports[0]) + trace = ( + *trace, + ToolTrace("fit_ground_plane", ground_fit.rejection_reason or "accepted"), + ToolTrace("fit_object_surface_plane", support_fit.rejection_reason or "accepted"), + ) + if ground_fit.estimate is None or support_fit.estimate is None: + return rejected_result( + frame, + intent, + [*objects, *supports], + trace, + ground_fit.rejection_reason + or support_fit.rejection_reason + or "support_relation_rejected", + ) + if abs(float(np.dot(ground_fit.estimate.normal, support_fit.estimate.normal))) < np.cos( + np.radians(12.0) + ): + return rejected_result( + frame, intent, [*objects, *supports], trace, "support_not_horizontal" + ) + relation = primitives.measure_object_plane_relation( + objects[0], supports[0], support_fit.estimate, ground_fit.estimate.normal + ) + trace = ( + *trace, + ToolTrace("measure_object_plane_relation", relation.rejection_reason or "accepted"), + ) + if relation.rejection_reason is not None: + return rejected_result( + frame, intent, [*objects, *supports], trace, relation.rejection_reason + ) + if relation.planar_separation_m is not None and relation.planar_separation_m > 0.2: + answer = "no" + elif relation.lower_clearance_m is not None and relation.lower_clearance_m > 0.2: + answer = "no" + elif ( + relation.lower_clearance_m is not None + and relation.upper_clearance_m is not None + and relation.elevated_fraction is not None + and relation.contact_point_count >= 4 + and relation.lower_clearance_m <= 0.08 + and relation.upper_clearance_m >= 0.15 + and relation.elevated_fraction >= 0.7 + and relation.contact_overlap_count >= 3 + ): + answer = "yes" + else: + return rejected_result( + frame, intent, [*objects, *supports], trace, "insufficient_contact_evidence" + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.comparison_query}-on-support", + render_question(intent), + answer, + "boolean", + (objects[0].id, supports[0].id), + ("yes", "no"), + ) + return GroundTruthResult( + intent, example, "answered", answer, None, (objects[0], supports[0]), trace + ) + + +def measure_opening_width( + frame: CalibratedFrame, intent: QuestionIntent, primitives: FramePerceptionPrimitives +) -> GroundTruthResult: + primitives.detect_objects(intent.object_query) + masks = primitives.segment_detections(intent.object_query) + trace = ( + ToolTrace("detect_objects", intent.object_query), + ToolTrace("segment_detections", f"count={len(masks)}"), + ) + if len(masks) != 1: + return rejected_result(frame, intent, [], trace, "ambiguous_opening_instances") + ground_fit = primitives.fit_ground_plane() + trace = (*trace, ToolTrace("fit_ground_plane", ground_fit.rejection_reason or "accepted")) + if ground_fit.estimate is None: + return rejected_result( + frame, intent, [], trace, ground_fit.rejection_reason or "ground_plane_rejected" + ) + result = primitives.measure_opening_width_from_mask(masks[0], ground_fit.estimate) + trace = (*trace, ToolTrace("measure_opening_width", result.rejection_reason or "accepted")) + if result.measurement is None: + return rejected_result( + frame, intent, [], trace, result.rejection_reason or "opening_width_rejected" + ) + answer = opening_width_choice(result.measurement.value) + example = VqaExample( + f"{frame.id}-{intent.object_query}-opening-width", + render_question(intent), + answer, + "choice", + (), + OPENING_WIDTH_CHOICES, + ) + return GroundTruthResult(intent, example, "answered", answer, None, (), trace) + + +def compare_nearest_by_side( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + left = select_nearest_object(objects, "left") + right = select_nearest_object(objects, "right") + if left is None or right is None: + return rejected_result(frame, intent, objects, trace, "missing_grounded_side") + if left.range_m == right.range_m: + return rejected_result(frame, intent, objects, trace, "ambiguous_nearest_by_side") + answer = "left" if left.range_m < right.range_m else "right" + example = VqaExample( + f"{frame.id}-{intent.object_query}-nearest-by-side", + render_question(intent), + answer, + "choice", + (left.id, right.id), + ("left", "right"), + ) + return GroundTruthResult(intent, example, "answered", answer, None, (left, right), trace) + + +def classify_door_state( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, +) -> GroundTruthResult: + if "door" not in intent.object_query.lower(): + return rejected_result(frame, intent, objects, trace, "door_state_requires_door_query") + if len(objects) != 1: + return rejected_result(frame, intent, objects, trace, "ambiguous_door_instances") + door_fit = primitives.fit_object_surface_plane(objects[0]) + surrounding_fit = primitives.fit_object_surrounding_plane(objects[0]) + trace = ( + *trace, + ToolTrace("fit_object_surface_plane", door_fit.rejection_reason or "accepted"), + ToolTrace("fit_mask_surrounding_plane", surrounding_fit.rejection_reason or "accepted"), + ) + if door_fit.estimate is None or surrounding_fit.estimate is None: + return rejected_result( + frame, + intent, + objects, + trace, + door_fit.rejection_reason or surrounding_fit.rejection_reason or "door_state_rejected", + ) + angle = primitives.measure_relative_plane_angle(door_fit.estimate, surrounding_fit.estimate) + trace = (*trace, ToolTrace("measure_relative_plane_angle", f"{angle.value:.1f} deg")) + if angle.value <= 12.0: + state = "closed" + elif angle.value >= 25.0: + state = "open" + else: + return rejected_result(frame, intent, objects, trace, "ambiguous_door_angle") + example = VqaExample( + f"{frame.id}-{intent.object_query}-state", + render_question(intent), + state, + "choice", + (objects[0].id,), + ("open", "closed"), + ) + return GroundTruthResult(intent, example, "answered", state, None, tuple(objects), trace) + + +def select_closest_object( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, + ground: Ground, +) -> GroundTruthResult: + if len(objects) != 1: + return rejected_result(frame, intent, objects, trace, "ambiguous_target_object") + candidates: list[GroundedObject] = [] + for query in intent.candidate_queries: + matches, candidate_trace = ground(frame, query) + trace = (*trace, *candidate_trace) + if len(matches) != 1: + return rejected_result( + frame, + intent, + [*objects, *candidates, *matches], + trace, + "ambiguous_candidate_object", + ) + candidates.append(matches[0]) + selected = primitives.select_closest_object(objects[0], candidates) + trace = ( + *trace, + ToolTrace( + "select_closest_object", + selected.object.id if selected.object else selected.rejection_reason or "rejected", + ), + ) + if selected.object is None: + return rejected_result( + frame, + intent, + [*objects, *candidates], + trace, + selected.rejection_reason or "closest_object_rejected", + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-closest-object", + render_question(intent), + selected.object.label, + "choice", + (objects[0].id, *(item.id for item in candidates)), + intent.candidate_queries, + ) + return GroundTruthResult( + intent, + example, + "answered", + selected.object.label, + None, + tuple([*objects, *candidates]), + trace, + ) + + +def classify_forward_path( + frame: CalibratedFrame, intent: QuestionIntent, primitives: FramePerceptionPrimitives +) -> GroundTruthResult: + result = primitives.classify_forward_path() + trace = ( + ToolTrace("classify_forward_path", result.state or result.rejection_reason or "rejected"), + ) + if result.state is None: + return rejected_result( + frame, intent, [], trace, result.rejection_reason or "forward_path_rejected" + ) + example = VqaExample( + f"{frame.id}-forward-path", + render_question(intent), + result.state, + "choice", + (), + ("clear", "blocked"), + ) + return GroundTruthResult(intent, example, "answered", result.state, None, (), trace) diff --git a/dimos/benchmark/vqa/generation/ground_truth_generator.py b/dimos/benchmark/vqa/generation/ground_truth_generator.py index 3987d5ee2d..2dd4eb71d5 100644 --- a/dimos/benchmark/vqa/generation/ground_truth_generator.py +++ b/dimos/benchmark/vqa/generation/ground_truth_generator.py @@ -3,14 +3,8 @@ from __future__ import annotations -from dimos.benchmark.vqa.generation.primitives.choices import ( - CAMERA_RANGE_CHOICES, - COUNT_CHOICES, - camera_range_choice, - count_choice, -) +from dimos.benchmark.vqa.generation import families from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives -from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object from dimos.benchmark.vqa.generation.questions import generate_questions from dimos.benchmark.vqa.models import ( CalibratedFrame, @@ -18,7 +12,6 @@ GroundTruthResult, QuestionIntent, ToolTrace, - VqaExample, ) @@ -30,7 +23,9 @@ def __init__(self, primitives: FramePerceptionPrimitives) -> None: def answer(self, frame: CalibratedFrame, intent: QuestionIntent) -> GroundTruthResult: if intent.kind == "forward_path": - return _classify_forward_path(frame, intent, self.primitives) + return families.classify_forward_path(frame, intent, self.primitives) + if intent.kind == "opening_width": + return families.measure_opening_width(frame, intent, self.primitives) objects, trace = self.ground(frame, intent.object_query) return self._answer_from_objects(frame, intent, objects, trace) @@ -52,8 +47,7 @@ def ground( if not len(detections) and self.primitives.used_point_localization(object_query): trace.append(ToolTrace("segment_object_point", object_query)) trace.append(ToolTrace("get_foreground_geometry", f"masks={len(masks)}")) - objects = self.primitives.ground_masks(object_query) - return objects, tuple(trace) + return self.primitives.ground_masks(object_query), tuple(trace) def _answer_from_objects( self, @@ -63,30 +57,31 @@ def _answer_from_objects( trace: tuple[ToolTrace, ...], ) -> GroundTruthResult: if not objects: - rejected = VqaExample( - f"{frame.id}-{intent.object_query}-{intent.kind}", - _render_question(intent), - "", - "", - (), - ) - return GroundTruthResult( - intent, rejected, "rejected", None, "no_grounded_object", (), trace - ) + return families.rejected_result(frame, intent, objects, trace, "no_grounded_object") if intent.kind == "compare_nearest_by_side": - return _compare_nearest_by_side(frame, intent, objects, trace) + return families.compare_nearest_by_side(frame, intent, objects, trace) if intent.kind == "visible_count": - return _count_visible_objects(frame, intent, objects, trace) + return families.count_visible_objects(frame, intent, objects, trace) if intent.kind == "camera_range": - return _bucket_camera_range(frame, intent, objects, trace) + return families.bucket_camera_range(frame, intent, objects, trace) if intent.kind == "compare_left_right": - return _compare_left_right(frame, intent, objects, trace, self) + return families.compare_left_right( + frame, intent, objects, trace, self.primitives, self.ground + ) if intent.kind == "compare_height": - return _compare_heights(frame, intent, objects, trace, self) + return families.compare_heights( + frame, intent, objects, trace, self.primitives, self.ground + ) + if intent.kind == "object_on_support": + return families.classify_object_on_support( + frame, intent, objects, trace, self.primitives, self.ground + ) if intent.kind == "door_state": - return _classify_door_state(frame, intent, objects, trace, self.primitives) + return families.classify_door_state(frame, intent, objects, trace, self.primitives) if intent.kind == "closest_object": - return _select_closest_object(frame, intent, objects, trace, self) + return families.select_closest_object( + frame, intent, objects, trace, self.primitives, self.ground + ) examples = generate_questions( frame.id, objects, [intent.object_query], distance_m=intent.threshold_m or 3.0 ) @@ -98,341 +93,6 @@ def _answer_from_objects( example = next((item for item in examples if item.id.endswith(f"-{suffix}")), None) if example is not None: return GroundTruthResult( - intent, - example, - "answered", - example.expected_answer, - None, - tuple(objects), - trace, + intent, example, "answered", example.expected_answer, None, tuple(objects), trace ) - rejected = VqaExample( - f"{frame.id}-{intent.object_query}-{intent.kind}", _render_question(intent), "", "", () - ) - return GroundTruthResult( - intent, rejected, "rejected", None, "no_grounded_object", tuple(objects), trace - ) - - -def _render_question(intent: QuestionIntent) -> str: - if intent.kind == "presence": - return f"Is there a {intent.object_query} in the image? Answer yes or no." - if intent.kind == "horizontal_direction": - return f"Where is the nearest {intent.object_query}: left, center, or right?" - if intent.kind == "visible_count": - return f"How many {intent.object_query}s are visible?" - if intent.kind == "camera_range": - return f"How far is the nearest {intent.object_query} from the camera?" - if intent.kind == "compare_nearest_by_side": - return f"Which {intent.object_query} is closer: the left one or the right one?" - if intent.kind == "compare_left_right": - return ( - f"Is the {intent.object_query} to the left or right of the {intent.comparison_query}?" - ) - if intent.kind == "compare_height": - return f"Which is taller: the {intent.object_query} or the {intent.comparison_query}?" - if intent.kind == "door_state": - return f"Is the {intent.object_query} open or closed?" - if intent.kind == "closest_object": - return f"Which object is closest to the {intent.object_query}: {', '.join(intent.candidate_queries)}?" - if intent.kind == "forward_path": - return "Is the path directly ahead clear or blocked?" - return f"Is the nearest {intent.object_query} within {intent.threshold_m or 3:g} meters? Answer yes or no." - - -def _count_visible_objects( - frame: CalibratedFrame, - intent: QuestionIntent, - objects: list[GroundedObject], - trace: tuple[ToolTrace, ...], -) -> GroundTruthResult: - answer = count_choice(len(objects)) - example = VqaExample( - f"{frame.id}-{intent.object_query}-visible-count", - _render_question(intent), - answer, - "choice", - tuple(item.id for item in objects), - COUNT_CHOICES, - ) - return GroundTruthResult(intent, example, "answered", answer, None, tuple(objects), trace) - - -def _bucket_camera_range( - frame: CalibratedFrame, - intent: QuestionIntent, - objects: list[GroundedObject], - trace: tuple[ToolTrace, ...], -) -> GroundTruthResult: - selected = select_nearest_object(objects) - if selected is None: - return _rejected_result(frame, intent, objects, trace, "no_grounded_object") - answer = camera_range_choice(selected.range_m) - example = VqaExample( - f"{frame.id}-{intent.object_query}-camera-range", - _render_question(intent), - answer, - "choice", - (selected.id,), - CAMERA_RANGE_CHOICES, - ) - return GroundTruthResult(intent, example, "answered", answer, None, (selected,), trace) - - -def _compare_heights( - frame: CalibratedFrame, - intent: QuestionIntent, - objects: list[GroundedObject], - trace: tuple[ToolTrace, ...], - generator: VqaGroundTruthGenerator, -) -> GroundTruthResult: - if len(objects) != 1 or intent.comparison_query is None: - return _rejected_result(frame, intent, objects, trace, "ambiguous_first_height_object") - other_objects, other_trace = generator.ground(frame, intent.comparison_query) - trace = (*trace, *other_trace) - if len(other_objects) != 1: - return _rejected_result( - frame, intent, [*objects, *other_objects], trace, "ambiguous_second_height_object" - ) - plane_fit = generator.primitives.fit_ground_plane() - trace = (*trace, ToolTrace("fit_ground_plane", plane_fit.rejection_reason or "accepted")) - if plane_fit.estimate is None: - return _rejected_result( - frame, - intent, - [*objects, *other_objects], - trace, - plane_fit.rejection_reason or "ground_plane_rejected", - ) - first = generator.primitives.measure_height(objects[0], plane_fit.estimate) - second = generator.primitives.measure_height(other_objects[0], plane_fit.estimate) - trace = ( - *trace, - ToolTrace("measure_height", first.rejection_reason or objects[0].id), - ToolTrace("measure_height", second.rejection_reason or other_objects[0].id), - ) - if first.measurement is None or second.measurement is None: - return _rejected_result( - frame, - intent, - [*objects, *other_objects], - trace, - first.rejection_reason or second.rejection_reason or "height_measurement_rejected", - ) - first_lower = first.measurement.value - first.measurement.tolerance - second_lower = second.measurement.value - second.measurement.tolerance - first_upper = first.measurement.value + first.measurement.tolerance - second_upper = second.measurement.value + second.measurement.tolerance - if first_lower <= second_upper and second_lower <= first_upper: - return _rejected_result( - frame, intent, [*objects, *other_objects], trace, "ambiguous_height_comparison" - ) - answer = intent.object_query if first_lower > second_upper else intent.comparison_query - example = VqaExample( - f"{frame.id}-{intent.object_query}-{intent.comparison_query}-height-comparison", - _render_question(intent), - answer, - "choice", - (objects[0].id, other_objects[0].id), - (intent.object_query, intent.comparison_query), - ) - return GroundTruthResult( - intent, example, "answered", answer, None, (objects[0], other_objects[0]), trace - ) - - -def _compare_left_right( - frame: CalibratedFrame, - intent: QuestionIntent, - objects: list[GroundedObject], - trace: tuple[ToolTrace, ...], - generator: VqaGroundTruthGenerator, -) -> GroundTruthResult: - if len(objects) != 1 or intent.comparison_query is None: - return _rejected_result(frame, intent, objects, trace, "ambiguous_first_relation_object") - other_objects, other_trace = generator.ground(frame, intent.comparison_query) - trace = (*trace, *other_trace) - if len(other_objects) != 1: - return _rejected_result( - frame, intent, [*objects, *other_objects], trace, "ambiguous_second_relation_object" - ) - relation = generator.primitives.classify_horizontal_relation(objects[0], other_objects[0]) - trace = ( - *trace, - ToolTrace( - "classify_horizontal_relation", - relation.relation or relation.rejection_reason or "rejected", - ), - ) - if relation.relation is None: - return _rejected_result( - frame, - intent, - [*objects, *other_objects], - trace, - relation.rejection_reason or "horizontal_relation_rejected", - ) - example = VqaExample( - f"{frame.id}-{intent.object_query}-{intent.comparison_query}-left-right", - _render_question(intent), - relation.relation, - "choice", - (objects[0].id, other_objects[0].id), - ("left", "right"), - ) - return GroundTruthResult( - intent, - example, - "answered", - relation.relation, - None, - (objects[0], other_objects[0]), - trace, - ) - - -def _compare_nearest_by_side( - frame: CalibratedFrame, - intent: QuestionIntent, - objects: list[GroundedObject], - trace: tuple[ToolTrace, ...], -) -> GroundTruthResult: - left = select_nearest_object(objects, "left") - right = select_nearest_object(objects, "right") - if left is None or right is None: - return _rejected_result(frame, intent, objects, trace, "missing_grounded_side") - if left.range_m == right.range_m: - return _rejected_result(frame, intent, objects, trace, "ambiguous_nearest_by_side") - answer = "left" if left.range_m < right.range_m else "right" - example = VqaExample( - f"{frame.id}-{intent.object_query}-nearest-by-side", - _render_question(intent), - answer, - "choice", - (left.id, right.id), - ("left", "right"), - ) - return GroundTruthResult(intent, example, "answered", answer, None, (left, right), trace) - - -def _classify_door_state( - frame: CalibratedFrame, - intent: QuestionIntent, - objects: list[GroundedObject], - trace: tuple[ToolTrace, ...], - primitives: FramePerceptionPrimitives, -) -> GroundTruthResult: - if "door" not in intent.object_query.lower(): - return _rejected_result(frame, intent, objects, trace, "door_state_requires_door_query") - if len(objects) != 1: - return _rejected_result(frame, intent, objects, trace, "ambiguous_door_instances") - result = primitives.classify_door_state(objects[0]) - trace = ( - *trace, - ToolTrace("classify_door_state", result.state or result.rejection_reason or "rejected"), - ) - if result.state is None: - return _rejected_result( - frame, intent, objects, trace, result.rejection_reason or "door_state_rejected" - ) - example = VqaExample( - f"{frame.id}-{intent.object_query}-state", - _render_question(intent), - result.state, - "choice", - (objects[0].id,), - ("open", "closed"), - ) - return GroundTruthResult(intent, example, "answered", result.state, None, tuple(objects), trace) - - -def _select_closest_object( - frame: CalibratedFrame, - intent: QuestionIntent, - objects: list[GroundedObject], - trace: tuple[ToolTrace, ...], - generator: VqaGroundTruthGenerator, -) -> GroundTruthResult: - if len(objects) != 1: - return _rejected_result(frame, intent, objects, trace, "ambiguous_target_object") - candidates: list[GroundedObject] = [] - for query in intent.candidate_queries: - matches, candidate_trace = generator.ground(frame, query) - trace = (*trace, *candidate_trace) - if len(matches) != 1: - return _rejected_result( - frame, - intent, - [*objects, *candidates, *matches], - trace, - "ambiguous_candidate_object", - ) - candidates.append(matches[0]) - selected = generator.primitives.select_closest_object(objects[0], candidates) - trace = ( - *trace, - ToolTrace( - "select_closest_object", - selected.object.id if selected.object else selected.rejection_reason or "rejected", - ), - ) - if selected.object is None: - return _rejected_result( - frame, - intent, - [*objects, *candidates], - trace, - selected.rejection_reason or "closest_object_rejected", - ) - example = VqaExample( - f"{frame.id}-{intent.object_query}-closest-object", - _render_question(intent), - selected.object.label, - "choice", - (objects[0].id, *(item.id for item in candidates)), - intent.candidate_queries, - ) - return GroundTruthResult( - intent, - example, - "answered", - selected.object.label, - None, - tuple([*objects, *candidates]), - trace, - ) - - -def _classify_forward_path( - frame: CalibratedFrame, intent: QuestionIntent, primitives: FramePerceptionPrimitives -) -> GroundTruthResult: - result = primitives.classify_forward_path() - trace = ( - ToolTrace("classify_forward_path", result.state or result.rejection_reason or "rejected"), - ) - if result.state is None: - return _rejected_result( - frame, intent, [], trace, result.rejection_reason or "forward_path_rejected" - ) - example = VqaExample( - f"{frame.id}-forward-path", - _render_question(intent), - result.state, - "choice", - (), - ("clear", "blocked"), - ) - return GroundTruthResult(intent, example, "answered", result.state, None, (), trace) - - -def _rejected_result( - frame: CalibratedFrame, - intent: QuestionIntent, - objects: list[GroundedObject], - trace: tuple[ToolTrace, ...], - reason: str, -) -> GroundTruthResult: - rejected = VqaExample( - f"{frame.id}-{intent.object_query}-{intent.kind}", _render_question(intent), "", "", () - ) - return GroundTruthResult(intent, rejected, "rejected", None, reason, tuple(objects), trace) + return families.rejected_result(frame, intent, objects, trace, "no_grounded_object") diff --git a/dimos/benchmark/vqa/generation/oracle.py b/dimos/benchmark/vqa/generation/oracle.py index 2007082afc..e8b8e51b6d 100644 --- a/dimos/benchmark/vqa/generation/oracle.py +++ b/dimos/benchmark/vqa/generation/oracle.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Protocol from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.generation.primitives.choices import height_choice_window from dimos.benchmark.vqa.models import ( AcceptedOracleResult, AnswerContract, @@ -99,7 +100,7 @@ class PrivateToolCallingOracle: def __init__( self, model: BaseChatModel, - max_tool_calls: int = 8, + max_tool_calls: int = 25, semantic_validator: SemanticEvidenceValidator | None = None, ) -> None: if max_tool_calls < 1: @@ -118,7 +119,13 @@ def answer( messages: list[Any] = [ SystemMessage( "You are a private VQA oracle. Use only supplied local tools. Do not invent " - 'evidence. Finish with JSON only: {"answer": value, "evidence_ids": [..]}.' + "evidence. Detection IDs can only be passed to segment_detection. Mask IDs can only " + "be passed to ground_mask. Only grounded object IDs can be passed to object and " + "geometry measurements. If grounding fails, reject rather than trying unrelated " + "tools. Finish with JSON only: either " + '{"answer": value, "evidence_ids": [..]} or ' + '{"status": "rejected", "reason": "non-empty reason", "evidence_ids": [..]}. ' + "Rejected results must not include an answer." ), HumanMessage(_proposal_prompt(proposal)), ] @@ -153,7 +160,7 @@ def answer( return _rejected(proposal, "tool_call_limit", registry.results, trace) -def create_openai_oracle(model: str, max_tool_calls: int = 8) -> PrivateToolCallingOracle: +def create_openai_oracle(model: str, max_tool_calls: int = 25) -> PrivateToolCallingOracle: """Construct the private-only OpenAI tool-calling oracle.""" from langchain_openai import ChatOpenAI @@ -208,21 +215,19 @@ def _resolve_oracle_answer( raise ValueError("choice answer does not match cited measurement bucket") return answer, contract if isinstance(contract, DeferredHeightChoiceContract): - bucket_results = [ + height_results = [ result for result in results - if result.tool == "bucket_measurement" and result.choice is not None and result.choices + if ( + result.tool == "measure_height" + and result.measurement is not None + and any(item.id in evidence_ids for item in result.evidence) + ) ] - if len(bucket_results) != 1: - raise ValueError("deferred height answer requires exactly one measurement bucket") - bucket = bucket_results[0] - if not any(item.id in evidence_ids for item in bucket.evidence): - raise ValueError("deferred height answer must cite its measurement") - if answer != bucket.choice: - raise ValueError("deferred height answer does not match measurement bucket") - if bucket.choice not in bucket.choices: - raise ValueError("measurement bucket choice is not public") - return bucket.choice, ChoiceAnswerContract(bucket.choices) + if len(height_results) != 1: + raise ValueError("deferred height answer requires exactly one cited height measurement") + choices, choice = height_choice_window(height_results[0].measurement.value) + return choice, ChoiceAnswerContract(choices) raise ValueError("unsupported answer contract") @@ -235,6 +240,19 @@ def _validated_result( ) -> AcceptedOracleResult | RejectedOracleResult: try: payload = _parse_json_object(response) + if payload.get("status") == "rejected": + if set(payload) - {"status", "reason", "evidence_ids"}: + raise ValueError("rejected final response contains unsupported fields") + reason = payload.get("reason") + evidence_ids = payload.get("evidence_ids") + if not isinstance(reason, str) or not reason: + raise ValueError("rejected final response requires non-empty reason") + if evidence_ids is not None and ( + not isinstance(evidence_ids, list) + or not all(isinstance(item, str) for item in evidence_ids) + ): + raise ValueError("rejected evidence_ids must be a list of strings") + return _rejected(proposal, reason, results, trace) answer, answer_contract = _resolve_oracle_answer( proposal, payload.get("answer"), payload.get("evidence_ids"), results ) @@ -296,8 +314,8 @@ def _contract_prompt(contract: AnswerContract) -> str: return f"choice: {', '.join(contract.choices)}" if isinstance(contract, DeferredHeightChoiceContract): return ( - "deferred height choice: call measure_height, then bucket_measurement, and return " - "the exact choice from that result" + "deferred height choice: call measure_height, cite exactly one accepted height evidence ID, " + "and return answer: null. The private pipeline derives the public choice from the measurement" ) raise ValueError("unsupported answer contract") diff --git a/dimos/benchmark/vqa/generation/oracle_tools.py b/dimos/benchmark/vqa/generation/oracle_tools.py index 73cca5958d..0dd7b32f99 100644 --- a/dimos/benchmark/vqa/generation/oracle_tools.py +++ b/dimos/benchmark/vqa/generation/oracle_tools.py @@ -13,9 +13,9 @@ COUNT_CHOICES, camera_range_choice, count_choice, - height_choice_window, ) from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.geometry import ForwardCorridorMeasurement from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object from dimos.benchmark.vqa.models import ( GroundedObject, @@ -24,6 +24,7 @@ OracleMeasurement, OracleToolResult, ) +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg class LocalOracleToolRegistry: @@ -32,11 +33,11 @@ class LocalOracleToolRegistry: def __init__(self, primitives: FramePerceptionPrimitives) -> None: self._primitives = primitives self._results: list[OracleToolResult] = [] - self._detections: dict[str, str] = {} - self._masks: dict[str, str] = {} + self._detections: dict[str, tuple[str, int]] = {} + self._masks: dict[str, tuple[str, Detection2DSeg]] = {} self._objects: dict[str, GroundedObject] = {} self._planes: dict[str, GroundPlaneEstimate] = {} - self._measurements: dict[str, OracleToolResult] = {} + self._ground_planes: set[str] = set() self._next_id = 0 @property @@ -51,99 +52,76 @@ def tools(self) -> list[StructuredTool]: description="Run private MoonDream detection for one visible semantic query.", ), StructuredTool.from_function( - self.segment_detections, - name="segment_detections", + self.segment_detection, + name="segment_detection", description="Run private EdgeTAM segmentation for one opaque detection ID.", ), StructuredTool.from_function( - self.ground_masks, - name="ground_masks", + self.ground_mask, + name="ground_mask", description=( - "Project visible calibrated point-cloud support through one opaque mask ID. " - "Returns grounded object IDs and citable evidence." + "Project visible calibrated point-cloud support through one opaque mask ID." ), ), StructuredTool.from_function( - self.select_nearest_object, - name="select_nearest_object", - description=( - "Select the nearest opaque grounded object ID, optionally restricted to left, center, " - "or right." - ), - ), - StructuredTool.from_function( - self.count_grounded_objects, - name="count_grounded_objects", - description="Count opaque grounded object IDs into fixed public count buckets.", - ), - StructuredTool.from_function( - self.bucket_camera_range, - name="bucket_camera_range", - description="Bucket one opaque object's private camera-origin range into fixed public choices.", + self.fit_ground_plane, + name="fit_ground_plane", + description="Fit a quality-gated Open3D ground plane to the frozen visible point cloud.", ), StructuredTool.from_function( - self.compare_nearest_by_side, - name="compare_nearest_by_side", - description="Choose whether the nearest opaque left or right object is closer to the camera.", + self.get_object_pose, + name="get_object_pose", + description="Return robust private camera-frame position evidence for one grounded object.", ), StructuredTool.from_function( - self.compare_left_right, - name="compare_left_right", - description=( - "Choose whether one opaque object is left or right of another from private " - "camera-frame support centroids. Rejects ambiguous separation." - ), + self.fit_object_surface_plane, + name="fit_object_surface_plane", + description="Fit a private surface plane to one grounded object's visible support.", ), StructuredTool.from_function( - self.select_closest_object, - name="select_closest_object", - description=( - "Select which opaque candidate object is closest to one opaque target object " - "by private point-cloud support. Rejects ambiguous proximity." - ), + self.fit_mask_surrounding_plane, + name="fit_mask_surrounding_plane", + description="Fit a private structural plane from the visible ring around one mask.", ), StructuredTool.from_function( - self.fit_ground_plane, - name="fit_ground_plane", - description="Fit a quality-gated Open3D ground plane to the frozen visible point cloud.", + self.measure_object_pair_distance, + name="measure_object_pair_distance", + description="Measure private 3D support-centroid distance between two grounded objects.", ), StructuredTool.from_function( - self.measure_height, - name="measure_height", - description=( - "Measure one opaque grounded object above one opaque accepted ground-plane ID." - ), + self.measure_relative_plane_angle, + name="measure_relative_plane_angle", + description="Measure the unsigned angle between two accepted opaque plane IDs.", ), StructuredTool.from_function( - self.compare_heights, - name="compare_heights", + self.measure_object_plane_relation, + name="measure_object_plane_relation", description=( - "Compare two opaque grounded objects against one opaque accepted ground-plane ID. " - "Rejects overlapping physical-height uncertainty." + "Measure private clearance, contact support, and projected separation of one object " + "relative to a support object plane." ), ), StructuredTool.from_function( - self.classify_door_state, - name="classify_door_state", + self.measure_aperture_geometry, + name="measure_aperture_geometry", description=( - "Classify one opaque grounded door as open or closed from point-cloud planes. " - "Rejects insufficient or ambiguous geometry." + "Measure a selected mask's ground-connected aperture geometry against an accepted " + "ground plane." ), ), StructuredTool.from_function( - self.classify_forward_path, - name="classify_forward_path", + self.measure_forward_corridor, + name="measure_forward_corridor", description=( - "Classify the visible camera-forward corridor as clear or blocked from point-cloud " - "ground and obstacle support. Rejects incomplete or ambiguous visibility." + "Measure private ground and elevated-obstacle support in the camera-forward corridor " + "against an accepted ground plane." ), ), StructuredTool.from_function( - self.bucket_measurement, - name="bucket_measurement", + self.measure_height, + name="measure_height", description=( - "Map one opaque height measurement ID to four public, answer-conditioned " - "height choices and the one matching choice." + "Measure one opaque grounded object above one opaque accepted ground-plane ID." ), ), ] @@ -151,37 +129,51 @@ def tools(self) -> list[StructuredTool]: def detect_objects(self, query: str) -> str: """Detect objects and return an opaque ID for a later segmentation call.""" detections = self._primitives.detect_objects(query) - detection_id = self._id("detection") - self._detections[detection_id] = query result = OracleToolResult("detect_objects", query, ()) self._results.append(result) - boxes = [list(item.bbox) for item in detections] - return json.dumps(_tool_payload(result, detection_id=detection_id, boxes=boxes)) - - def segment_detections(self, detection_id: str) -> str: - """Segment one earlier opaque detection result and return an opaque mask ID.""" - query = self._detections.get(detection_id) - if query is None: - return self._record_rejection("segment_detections", "", [], "unknown_detection_id") - masks = self._primitives.segment_detections(query) - mask_id = self._id("mask") - self._masks[mask_id] = query - result = OracleToolResult("segment_detections", query, ()) + handles = [] + for index, detection in enumerate(detections): + detection_id = self._id("detection") + self._detections[detection_id] = (query, index) + handles.append({"detection_id": detection_id, "confidence": detection.confidence}) + return json.dumps(_tool_payload(result, detections=handles)) + + def segment_detection(self, detection_id: str) -> str: + """Segment one earlier opaque detection and return opaque per-mask handles.""" + handle = self._detections.get(detection_id) + if handle is None: + return self._record_rejection("segment_detection", "", [], "unknown_detection_id") + query, index = handle + masks = self._primitives.segment_detection(query, index) + mask_ids = [] + for mask in masks: + mask_id = self._id("mask") + self._masks[mask_id] = (query, mask) + mask_ids.append(mask_id) + result = OracleToolResult("segment_detection", query, ()) self._results.append(result) - return json.dumps(_tool_payload(result, mask_id=mask_id, mask_count=len(masks))) - - def ground_masks(self, mask_id: str) -> str: - """Ground one earlier opaque mask result against the visible point cloud.""" - query = self._masks.get(mask_id) - if query is None: - return self._record_rejection("ground_masks", "", [], "unknown_mask_id") - objects = self._primitives.ground_masks(query) - evidence = tuple(_grounding_evidence(item) for item in objects) - for item in objects: - self._objects[item.id] = item - result = OracleToolResult("ground_masks", query, evidence) + return json.dumps(_tool_payload(result, mask_ids=mask_ids)) + + def ground_mask(self, mask_id: str) -> str: + """Ground one earlier opaque mask against the visible point cloud.""" + handle = self._masks.get(mask_id) + if handle is None: + return self._record_rejection("ground_mask", "", [], "unknown_mask_id") + query, mask = handle + object_id = self._id("object") + if hasattr(self._primitives, "ground_mask"): + object = self._primitives.ground_mask(mask, object_id) + else: + objects = self._primitives.ground_masks(query) + object = objects[0] if objects else None + if object is None: + return self._record_rejection( + "ground_mask", query, [], "insufficient_foreground_support" + ) + self._objects[object.id] = object + result = OracleToolResult("ground_mask", query, (_grounding_evidence(object),)) self._results.append(result) - return json.dumps(_tool_payload(result, object_ids=[item.id for item in objects])) + return json.dumps(_tool_payload(result, object_id=object.id)) def fit_ground_plane(self) -> str: """Fit the private ground plane and return an opaque ID for measurements.""" @@ -192,6 +184,7 @@ def fit_ground_plane(self) -> str: ) plane_id = self._id("plane") self._planes[plane_id] = fit.estimate + self._ground_planes.add(plane_id) measurement = OracleMeasurement( fit.estimate.offset_m, "m", @@ -220,6 +213,220 @@ def fit_ground_plane(self) -> str: self._results.append(result) return json.dumps(_tool_payload(result, plane_id=plane_id)) + def get_object_pose(self, object_id: str) -> str: + """Return existing grounding evidence for one object without selecting an answer.""" + object = self._objects.get(object_id) + if object is None: + return self._record_rejection("get_object_pose", object_id, [], "unknown_object_id") + result = OracleToolResult("get_object_pose", object.label, (_grounding_evidence(object),)) + self._results.append(result) + return json.dumps(_tool_payload(result, object_id=object.id)) + + def fit_object_surface_plane(self, object_id: str) -> str: + """Fit a plane to one grounded object's visible point support.""" + object = self._objects.get(object_id) + if object is None: + return self._record_rejection( + "fit_object_surface_plane", object_id, [], "unknown_object_id" + ) + fit = self._primitives.fit_object_surface_plane(object) + if fit.estimate is None: + return self._record_rejection( + "fit_object_surface_plane", + object.label, + list(fit.quality_flags), + fit.rejection_reason, + ) + plane_id = self._id("plane") + self._planes[plane_id] = fit.estimate + result = OracleToolResult( + "fit_object_surface_plane", + object.label, + (_grounding_evidence(object),), + plane=fit.estimate, + quality_flags=fit.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result, plane_id=plane_id)) + + def fit_mask_surrounding_plane(self, mask_id: str) -> str: + """Fit a plane around one selected segmentation mask.""" + handle = self._masks.get(mask_id) + if handle is None or not isinstance(handle, tuple) or handle[1] is None: + return self._record_rejection("fit_mask_surrounding_plane", "", [], "unknown_mask_id") + query, mask = handle + fit = self._primitives.fit_mask_surrounding_plane(mask) + if fit.estimate is None: + return self._record_rejection( + "fit_mask_surrounding_plane", query, list(fit.quality_flags), fit.rejection_reason + ) + plane_id = self._id("plane") + self._planes[plane_id] = fit.estimate + result = OracleToolResult( + "fit_mask_surrounding_plane", + query, + (), + plane=fit.estimate, + quality_flags=fit.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result, plane_id=plane_id)) + + def measure_object_pair_distance(self, first_object_id: str, second_object_id: str) -> str: + """Measure support-centroid distance between two grounded objects.""" + first, second = self._objects.get(first_object_id), self._objects.get(second_object_id) + if first is None or second is None: + return self._record_rejection( + "measure_object_pair_distance", "", [], "unknown_object_id" + ) + measurement = self._primitives.measure_object_pair_distance(first, second) + if measurement is None: + return self._record_rejection( + "measure_object_pair_distance", "", [], "insufficient_object_support" + ) + result = OracleToolResult( + "measure_object_pair_distance", + f"{first.label},{second.label}", + (_grounding_evidence(first), _grounding_evidence(second)), + measurement=measurement, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def measure_relative_plane_angle(self, first_plane_id: str, second_plane_id: str) -> str: + """Measure the unsigned angle between two accepted planes.""" + first, second = self._planes.get(first_plane_id), self._planes.get(second_plane_id) + if first is None or second is None: + return self._record_rejection( + "measure_relative_plane_angle", "", [], "unknown_plane_id" + ) + measurement = self._primitives.measure_relative_plane_angle(first, second) + result = OracleToolResult("measure_relative_plane_angle", "", (), measurement=measurement) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def measure_object_plane_relation( + self, object_id: str, support_id: str, plane_id: str, ground_plane_id: str + ) -> str: + """Measure one object's clearance/contact relation to a selected support plane.""" + object, support = self._objects.get(object_id), self._objects.get(support_id) + plane, ground = self._planes.get(plane_id), self._planes.get(ground_plane_id) + if object is None or support is None: + return self._record_rejection( + "measure_object_plane_relation", "", [], "unknown_object_id" + ) + if plane is None or ground is None or ground_plane_id not in self._ground_planes: + return self._record_rejection( + "measure_object_plane_relation", "", [], "unknown_plane_id" + ) + relation = self._primitives.measure_object_plane_relation( + object, support, plane, ground.normal + ) + if relation.rejection_reason is not None: + return self._record_rejection( + "measure_object_plane_relation", + f"{object.label},{support.label}", + list(relation.quality_flags), + relation.rejection_reason, + ) + metrics = tuple( + (name, value) + for name, value in ( + ("lower_clearance_m", relation.lower_clearance_m), + ("upper_clearance_m", relation.upper_clearance_m), + ("elevated_fraction", relation.elevated_fraction), + ("contact_point_count", float(relation.contact_point_count)), + ("planar_separation_m", relation.planar_separation_m), + ("contact_overlap_count", float(relation.contact_overlap_count)), + ) + if value is not None + ) + result = OracleToolResult( + "measure_object_plane_relation", + f"{object.label},{support.label}", + (_grounding_evidence(object), _grounding_evidence(support)), + quality_flags=relation.quality_flags, + metrics=metrics, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def measure_aperture_geometry(self, mask_id: str, ground_plane_id: str) -> str: + """Measure one selected aperture mask using an accepted ground plane.""" + handle = self._masks.get(mask_id) + ground = self._planes.get(ground_plane_id) + if handle is None or not isinstance(handle, tuple) or handle[1] is None: + return self._record_rejection("measure_aperture_geometry", "", [], "unknown_mask_id") + if ground is None or ground_plane_id not in self._ground_planes: + return self._record_rejection( + "measure_aperture_geometry", "", [], "unknown_ground_plane_id" + ) + query, mask = handle + result = self._primitives.measure_opening_width_from_mask(mask, ground) + if result.measurement is None: + return self._record_rejection( + "measure_aperture_geometry", + query, + list(result.quality_flags), + result.rejection_reason, + ) + evidence = OracleEvidence( + f"aperture:v1:{self._primitives.frame.id}:{mask_id}", + "v1", + mask_id, + query, + 0.0, + "n/a", + 0, + result.measurement, + ) + tool_result = OracleToolResult( + "measure_aperture_geometry", + query, + (evidence,), + measurement=result.measurement, + quality_flags=result.quality_flags, + ) + self._results.append(tool_result) + return json.dumps(_tool_payload(tool_result)) + + def measure_forward_corridor(self, ground_plane_id: str) -> str: + """Measure visible forward ground and obstacle support using one accepted ground plane.""" + ground = self._planes.get(ground_plane_id) + if ground is None or ground_plane_id not in self._ground_planes: + return self._record_rejection( + "measure_forward_corridor", "", [], "unknown_ground_plane_id" + ) + measured: ForwardCorridorMeasurement = self._primitives.measure_forward_corridor(ground) + if measured.rejection_reason is not None: + return self._record_rejection( + "measure_forward_corridor", "", [], measured.rejection_reason + ) + evidence = OracleEvidence( + f"forward-corridor:v1:{self._primitives.frame.id}", + "v1", + "forward-corridor", + "forward corridor", + 0.0, + "n/a", + measured.point_count, + ) + result = OracleToolResult( + "measure_forward_corridor", + "", + (evidence,), + quality_flags=("forward_ground_supported",), + metrics=( + ("ground_band_1_count", float(measured.ground_band_counts[0])), + ("ground_band_2_count", float(measured.ground_band_counts[1])), + ("ground_band_3_count", float(measured.ground_band_counts[2])), + ("elevated_obstacle_count", float(measured.obstacle_count)), + ("corridor_point_count", float(measured.point_count)), + ), + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + def select_nearest_object(self, object_ids: list[str], side: str | None = None) -> str: """Select one opaque grounded object by private point-cloud range.""" selected: list[GroundedObject] = [] @@ -393,10 +600,8 @@ def measure_height(self, object_id: str, plane_id: str) -> str: plane=plane, quality_flags=measured.quality_flags, ) - measurement_id = self._id("measurement") - self._measurements[measurement_id] = result self._results.append(result) - return json.dumps(_tool_payload(result, measurement_id=measurement_id)) + return json.dumps(_tool_payload(result)) def compare_heights(self, first_object_id: str, second_object_id: str, plane_id: str) -> str: """Choose the taller object only when one shared-plane measurement is unambiguous.""" @@ -444,37 +649,57 @@ def compare_heights(self, first_object_id: str, second_object_id: str, plane_id: self._results.append(result) return json.dumps(_tool_payload(result)) - def classify_door_state(self, object_id: str) -> str: - """Classify one grounded door against the surrounding point-cloud plane.""" + def classify_object_on_support(self, object_id: str, support_id: str) -> str: + """Verify that one grounded object directly rests on another grounded object.""" object = self._objects.get(object_id) - if object is None: - return self._record_rejection("classify_door_state", object_id, [], "unknown_object_id") - if "door" not in object.label.lower(): + support = self._objects.get(support_id) + if object is None or support is None: + return self._record_rejection("classify_object_on_support", "", [], "unknown_object_id") + result = self._primitives.classify_object_on_support(object, support) + if result.answer is None: return self._record_rejection( - "classify_door_state", object.label, [], "door_state_requires_door_query" - ) - result = self._primitives.classify_door_state(object) - if result.state is None: - return self._record_rejection( - "classify_door_state", - object.label, + "classify_object_on_support", + f"{object.label},{support.label}", list(result.quality_flags), result.rejection_reason, ) + tool_result = OracleToolResult( + "classify_object_on_support", + f"{object.label},{support.label}", + (_grounding_evidence(object), _grounding_evidence(support)), + choice=result.answer, + choices=("yes", "no"), + quality_flags=result.quality_flags, + ) + self._results.append(tool_result) + return json.dumps(_tool_payload(tool_result)) + + def measure_opening_width(self, mask_id: str) -> str: + """Measure one earlier segmented doorway/opening mask in metres.""" + handle = self._masks.get(mask_id) + if handle is None: + return self._record_rejection("measure_opening_width", "", [], "unknown_mask_id") + query = handle[0] if isinstance(handle, tuple) else handle + result = self._primitives.measure_opening_width(query) + if result.measurement is None: + return self._record_rejection( + "measure_opening_width", query, list(result.quality_flags), result.rejection_reason + ) evidence = OracleEvidence( - f"door-state:v1:{object.id}", + f"opening-width:v1:{self._primitives.frame.id}:{mask_id}", "v1", - object.id, - object.label, - object.range_m, - object.horizontal_direction, - object.point_count, + mask_id, + query, + 0.0, + "n/a", + 0, + result.measurement, ) tool_result = OracleToolResult( - "classify_door_state", - object.label, + "measure_opening_width", + query, (evidence,), - choice=result.state, + measurement=result.measurement, quality_flags=result.quality_flags, ) self._results.append(tool_result) @@ -509,25 +734,6 @@ def classify_forward_path(self) -> str: self._results.append(tool_result) return json.dumps(_tool_payload(tool_result)) - def bucket_measurement(self, measurement_id: str) -> str: - """Map one accepted private height measurement to its public choice.""" - source = self._measurements.get(measurement_id) - if source is None or source.measurement is None: - return self._record_rejection("bucket_measurement", "", [], "unknown_measurement_id") - choices, choice = height_choice_window(source.measurement.value) - result = OracleToolResult( - "bucket_measurement", - source.query, - source.evidence, - measurement=source.measurement, - choice=choice, - choices=choices, - plane=source.plane, - quality_flags=source.quality_flags, - ) - self._results.append(result) - return json.dumps(_tool_payload(result)) - def _record_rejection(self, tool: str, query: str, flags: list[str], reason: str | None) -> str: result = OracleToolResult( tool, query, (), quality_flags=tuple(flags), rejection_reason=reason @@ -595,6 +801,7 @@ def _tool_payload(result: OracleToolResult, **identifiers: Any) -> dict[str, Any "choice": result.choice, "choices": result.choices, "quality_flags": result.quality_flags, + "metrics": dict(result.metrics), "rejection_reason": result.rejection_reason, "plane": ( { diff --git a/dimos/benchmark/vqa/generation/primitives/choices.py b/dimos/benchmark/vqa/generation/primitives/choices.py index af45ca20c8..cdd7028a45 100644 --- a/dimos/benchmark/vqa/generation/primitives/choices.py +++ b/dimos/benchmark/vqa/generation/primitives/choices.py @@ -6,6 +6,7 @@ COUNT_CHOICES = ("1-2", "3-4", "5-7", "8+") CAMERA_RANGE_CHOICES = ("under 1 m", "1 to under 2 m", "2 to under 4 m", "4 m or more") +OPENING_WIDTH_CHOICES = ("under 0.2 m", "0.2 to under 0.5 m", "0.5 to under 0.8 m", "0.8 m or more") def count_choice(count: int) -> str: @@ -34,6 +35,19 @@ def camera_range_choice(range_m: float) -> str: return CAMERA_RANGE_CHOICES[3] +def opening_width_choice(width_m: float) -> str: + """Return the public doorway-width bucket for a positive metric measurement.""" + if width_m < 0: + raise ValueError("width must be non-negative") + if width_m < 0.2: + return OPENING_WIDTH_CHOICES[0] + if width_m < 0.5: + return OPENING_WIDTH_CHOICES[1] + if width_m < 0.8: + return OPENING_WIDTH_CHOICES[2] + return OPENING_WIDTH_CHOICES[3] + + def height_choice_window(height_m: float) -> tuple[tuple[str, ...], str]: """Generate a local, exhaustive four-choice window around a private height.""" breakpoints = (0.1, 0.2, 0.6, 1.0, 2.0) diff --git a/dimos/benchmark/vqa/generation/primitives/contracts.py b/dimos/benchmark/vqa/generation/primitives/contracts.py index e27e6b027f..0e8b171539 100644 --- a/dimos/benchmark/vqa/generation/primitives/contracts.py +++ b/dimos/benchmark/vqa/generation/primitives/contracts.py @@ -19,17 +19,6 @@ class HeightMeasurementResult: rejection_reason: str | None = None -@dataclass(frozen=True) -class DoorStateResult: - """A conservative point-cloud classification of one door's state.""" - - object: GroundedObject - state: Literal["open", "closed"] | None - quality_flags: tuple[str, ...] - rejection_reason: str | None = None - angle_deg: float | None = None - - @dataclass(frozen=True) class ClosestObjectResult: """A point-cloud selected candidate nearest to one grounded target object.""" @@ -49,6 +38,40 @@ class HorizontalRelationResult: rejection_reason: str | None = None +@dataclass(frozen=True) +class ObjectOnSupportResult: + """Verified direct support/contact separation, or an explicit conservative rejection.""" + + object: GroundedObject + support: GroundedObject + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + answer: Literal["yes", "no"] | None = None + + +@dataclass(frozen=True) +class ObjectPlaneRelationResult: + """Private geometric relation of one visible object to a selected support plane.""" + + lower_clearance_m: float | None + upper_clearance_m: float | None + elevated_fraction: float | None + contact_point_count: int + planar_separation_m: float | None + contact_overlap_count: int + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class OpeningWidthResult: + """A metric doorway opening width, or an explicit conservative rejection.""" + + measurement: OracleMeasurement | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + @dataclass(frozen=True) class ForwardPathResult: """A conservative visible-corridor classification from point-cloud evidence.""" diff --git a/dimos/benchmark/vqa/generation/primitives/frame.py b/dimos/benchmark/vqa/generation/primitives/frame.py index 80009f7281..e1031ee11e 100644 --- a/dimos/benchmark/vqa/generation/primitives/frame.py +++ b/dimos/benchmark/vqa/generation/primitives/frame.py @@ -2,23 +2,29 @@ from __future__ import annotations +from dataclasses import replace + import numpy as np from dimos.benchmark.vqa.generation.geometry import project_visible_points from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects from dimos.benchmark.vqa.generation.primitives.contracts import ( ClosestObjectResult, - DoorStateResult, ForwardPathResult, HeightMeasurementResult, HorizontalRelationResult, + ObjectOnSupportResult, + ObjectPlaneRelationResult, + OpeningWidthResult, ) from dimos.benchmark.vqa.generation.primitives.geometry import ( PlaneFitResult, - classify_door_plane_angle, classify_forward_corridor, estimate_ground_plane, fit_surface_plane, + measure_forward_corridor, + measure_opening_width, + measure_relative_plane_angle, points_around_mask, points_in_mask, ) @@ -98,6 +104,30 @@ def segment_detections(self, query: str) -> list[Detection2DSeg]: self._segmented_queries.add(query) return masks + def segment_detection(self, query: str, index: int) -> list[Detection2DSeg]: + """Segment one frozen detection selected by its query-local index.""" + detections = self.detect_objects(query) + if index < 0 or index >= len(detections): + raise ValueError("unknown_detection_index") + selected = ImageDetections2D(self.frame.image, [detections[index]]) + return [ + item + for item in self._segmenter.segment(selected) + if isinstance(item, Detection2DSeg) + and int((item.mask > 0).sum()) >= self._config.min_mask_area_px + ] + + def ground_mask(self, mask: Detection2DSeg, object_id: str) -> GroundedObject | None: + """Ground one selected segmentation mask under a caller-owned opaque object ID.""" + objects = ground_segmented_objects( + self.frame, [mask], min_foreground_points=self._config.min_foreground_points + ) + if len(objects) != 1: + return None + object = replace(objects[0], id=object_id) + self._object_masks[object.id] = mask + return object + def ground_masks(self, query: str) -> list[GroundedObject]: """Project visible point-cloud support through accepted masks.""" cached = self._groundings.get(query) @@ -133,6 +163,85 @@ def fit_ground_plane(self) -> PlaneFitResult: self._plane_fit = estimate_ground_plane(self.frame) return self._plane_fit + def fit_object_surface_plane(self, object: GroundedObject) -> PlaneFitResult: + """Fit one object-supported surface plane without assigning it a semantic role.""" + points = self._object_points(object) + if points is None: + return PlaneFitResult(None, ("insufficient_object_support",), "insufficient_support") + return fit_surface_plane(points) + + def fit_mask_surrounding_plane(self, mask: Detection2DSeg) -> PlaneFitResult: + """Fit a surface plane from visible support around one selected mask.""" + return fit_surface_plane(points_around_mask(self.frame, mask.mask)) + + def fit_object_surrounding_plane(self, object: GroundedObject) -> PlaneFitResult: + """Fit a surface plane around one grounded object's selected mask.""" + mask = self._object_masks.get(object.id) + if mask is None: + return PlaneFitResult(None, ("unknown_grounded_object",), "unknown_object_id") + return self.fit_mask_surrounding_plane(mask) + + def measure_object_pair_distance( + self, first: GroundedObject, second: GroundedObject + ) -> OracleMeasurement | None: + """Measure support-centroid distance between two grounded objects.""" + first_points = self._object_points(first) + second_points = self._object_points(second) + if first_points is None or second_points is None: + return None + value = float( + np.linalg.norm(np.median(first_points, axis=0) - np.median(second_points, axis=0)) + ) + return OracleMeasurement( + value, + "m", + 0.05, + ("visible_support_centroids",), + (f"grounding:v1:{first.id}", f"grounding:v1:{second.id}"), + ) + + def measure_object_plane_relation( + self, + object: GroundedObject, + support: GroundedObject, + plane: GroundPlaneEstimate, + reference_normal: tuple[float, float, float], + ) -> ObjectPlaneRelationResult: + """Measure clearance and projected support proximity without assigning a relation label.""" + object_points = self._object_points(object) + support_points = self._object_points(support) + if object_points is None or support_points is None: + return ObjectPlaneRelationResult( + None, None, None, 0, None, 0, (), "insufficient_object_support" + ) + normal = np.asarray(plane.normal) + offset = plane.offset_m + if float(normal @ np.asarray(reference_normal)) < 0: + normal, offset = -normal, -offset + elevation = object_points @ normal + offset + projected_support = support_points - np.outer(support_points @ normal + offset, normal) + projected_object = object_points - np.outer(elevation, normal) + separation = np.linalg.norm( + projected_object[:, np.newaxis, :] - projected_support[np.newaxis, :, :], axis=2 + ).min(axis=1) + contact_points = object_points[np.abs(elevation) <= 0.08] + overlap_count = 0 + if len(contact_points): + projected_contact = contact_points - np.outer(contact_points @ normal + offset, normal) + contact_separation = np.linalg.norm( + projected_contact[:, np.newaxis, :] - projected_support[np.newaxis, :, :], axis=2 + ).min(axis=1) + overlap_count = int((contact_separation <= 0.1).sum()) + return ObjectPlaneRelationResult( + float(np.percentile(elevation, 15)), + float(np.percentile(elevation, 85)), + float((elevation >= 0.02).mean()), + len(contact_points), + float(separation.min()), + overlap_count, + ("visible_object_plane_relation",), + ) + def measure_height( self, object: GroundedObject, plane: GroundPlaneEstimate ) -> HeightMeasurementResult: @@ -169,41 +278,11 @@ def measure_height( ) return HeightMeasurementResult(object, plane, measurement, tuple(flags)) - def classify_door_state(self, object: GroundedObject) -> DoorStateResult: - """Classify a door as open or closed from its plane relative to nearby structure.""" - mask = self._object_masks.get(object.id) - if mask is None: - raise ValueError(f"unknown grounded object: {object.id}") - door_fit = fit_surface_plane(points_in_mask(self.frame, mask.mask)) - if door_fit.estimate is None: - return DoorStateResult( - object, - None, - ("door_plane_rejected", *door_fit.quality_flags), - door_fit.rejection_reason, - ) - surrounding_fit = fit_surface_plane(points_around_mask(self.frame, mask.mask)) - if surrounding_fit.estimate is None: - return DoorStateResult( - object, - None, - ("surrounding_plane_rejected", *surrounding_fit.quality_flags), - surrounding_fit.rejection_reason, - ) - state, reason, angle_deg = classify_door_plane_angle( - door_fit.estimate, surrounding_fit.estimate - ) - return DoorStateResult( - object, - state, - ( - "door_and_surrounding_planes_accepted", - *door_fit.quality_flags, - *surrounding_fit.quality_flags, - ), - reason, - angle_deg, - ) + def measure_relative_plane_angle( + self, first: GroundPlaneEstimate, second: GroundPlaneEstimate + ) -> OracleMeasurement: + """Measure the unsigned angle between two accepted planes.""" + return measure_relative_plane_angle(first, second) def select_closest_object( self, target: GroundedObject, candidates: list[GroundedObject] @@ -246,6 +325,134 @@ def classify_horizontal_relation( "left" if horizontal_offset_m < 0 else "right", ("camera_frame_support_centroids",) ) + def classify_object_on_support( + self, object: GroundedObject, support: GroundedObject + ) -> ObjectOnSupportResult: + """Verify direct contact between an object and a horizontal support surface.""" + if object.id == support.id: + return ObjectOnSupportResult(object, support, (), "duplicate_object_id") + object_points = self._object_points(object) + support_points = self._object_points(support) + if object_points is None or support_points is None: + return ObjectOnSupportResult(object, support, (), "insufficient_object_support") + ground_fit = self.fit_ground_plane() + if ground_fit.estimate is None: + return ObjectOnSupportResult( + object, + support, + ground_fit.quality_flags, + ground_fit.rejection_reason or "ground_plane_rejected", + ) + support_fit = fit_surface_plane(support_points) + if support_fit.estimate is None: + return ObjectOnSupportResult( + object, + support, + support_fit.quality_flags, + support_fit.rejection_reason or "support_plane_rejected", + ) + normal = np.asarray(support_fit.estimate.normal) + ground_normal = np.asarray(ground_fit.estimate.normal) + if abs(float(normal @ ground_normal)) < np.cos(np.radians(12.0)): + return ObjectOnSupportResult( + object, support, support_fit.quality_flags, "support_not_horizontal" + ) + if float(normal @ ground_normal) < 0: + normal = -normal + elevation = object_points @ normal + support_fit.estimate.offset_m + projected_support = support_points - np.outer( + support_points @ normal + support_fit.estimate.offset_m, normal + ) + projected_object = object_points - np.outer( + object_points @ normal + support_fit.estimate.offset_m, normal + ) + nearest_support_distance = np.linalg.norm( + projected_object[:, np.newaxis, :] - projected_support[np.newaxis, :, :], axis=2 + ).min(axis=1) + if float(nearest_support_distance.min()) > 0.2: + return ObjectOnSupportResult( + object, + support, + (*support_fit.quality_flags, "objects_separated_in_plane"), + answer="no", + ) + if float(np.percentile(elevation, 15)) > 0.2: + return ObjectOnSupportResult( + object, + support, + (*support_fit.quality_flags, "object_clearly_above_support"), + answer="no", + ) + contact_points = object_points[np.abs(elevation) <= 0.08] + if ( + len(contact_points) < 4 + or float(np.percentile(elevation, 15)) > 0.08 + or float(np.percentile(elevation, 85)) < 0.15 + or float((elevation >= 0.02).mean()) < 0.7 + ): + return ObjectOnSupportResult( + object, support, support_fit.quality_flags, "insufficient_contact_evidence" + ) + projected_contact = contact_points - np.outer( + contact_points @ normal + support_fit.estimate.offset_m, normal + ) + distances = np.linalg.norm( + projected_contact[:, np.newaxis, :] - projected_support[np.newaxis, :, :], axis=2 + ) + if int((distances.min(axis=1) <= 0.1).sum()) < 3: + return ObjectOnSupportResult( + object, support, support_fit.quality_flags, "insufficient_in_plane_support_overlap" + ) + return ObjectOnSupportResult( + object, + support, + ( + *ground_fit.quality_flags, + *support_fit.quality_flags, + "support_horizontal", + "contact_band_supported", + "in_plane_support_overlap", + ), + answer="yes", + ) + + def measure_opening_width(self, query: str) -> OpeningWidthResult: + """Measure one segmented doorway aperture using its adjacent structural plane.""" + masks = self.segment_detections(query) + if len(masks) != 1: + return OpeningWidthResult(None, (), "ambiguous_opening_instances") + ground_fit = self.fit_ground_plane() + if ground_fit.estimate is None: + return OpeningWidthResult( + None, + ground_fit.quality_flags, + ground_fit.rejection_reason or "ground_plane_rejected", + ) + return self.measure_opening_width_from_mask(masks[0], ground_fit.estimate) + + def measure_opening_width_from_mask( + self, mask: Detection2DSeg, ground: GroundPlaneEstimate + ) -> OpeningWidthResult: + """Measure one selected aperture mask against an already accepted ground plane.""" + result = measure_opening_width(self.frame, mask.mask, ground) + if result.width_m is None or result.tolerance_m is None: + return OpeningWidthResult( + None, + result.quality_flags, + result.rejection_reason or "opening_width_rejected", + ) + measurement = OracleMeasurement( + result.width_m, + "m", + result.tolerance_m, + result.quality_flags, + (f"frame:{self.frame.id}", f"opening-mask:v1:{query}"), + ) + return OpeningWidthResult( + measurement, + result.quality_flags, + ) + def classify_forward_path(self) -> ForwardPathResult: """Classify the observed camera-forward corridor as clear or blocked.""" ground_fit = self.fit_ground_plane() @@ -261,6 +468,12 @@ def classify_forward_path(self) -> ForwardPathResult: state, flags, reason = classify_forward_corridor(points, ground_fit.estimate) return ForwardPathResult(state, len(points), flags, reason) + def measure_forward_corridor(self, plane: GroundPlaneEstimate): + """Measure visible forward corridor support against an accepted ground plane.""" + projected = project_visible_points(self.frame) + points = np.asarray(projected.camera_points, dtype=np.float64) + return measure_forward_corridor(points, plane) + def _object_points(self, object: GroundedObject) -> np.ndarray | None: mask = self._object_masks.get(object.id) if mask is None: diff --git a/dimos/benchmark/vqa/generation/primitives/geometry.py b/dimos/benchmark/vqa/generation/primitives/geometry.py index a1314506a9..0e149b3be9 100644 --- a/dimos/benchmark/vqa/generation/primitives/geometry.py +++ b/dimos/benchmark/vqa/generation/primitives/geometry.py @@ -8,7 +8,7 @@ import numpy as np from dimos.benchmark.vqa.generation.geometry import project_visible_points -from dimos.benchmark.vqa.models import CalibratedFrame, GroundPlaneEstimate +from dimos.benchmark.vqa.models import CalibratedFrame, GroundPlaneEstimate, OracleMeasurement @dataclass(frozen=True) @@ -20,6 +20,26 @@ class PlaneFitResult: rejection_reason: str | None = None +@dataclass(frozen=True) +class OpeningGeometryResult: + """Opening width from a mask and surrounding structural plane.""" + + width_m: float | None + tolerance_m: float | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class ForwardCorridorMeasurement: + """Private point-support metrics for the camera-forward corridor.""" + + ground_band_counts: tuple[int, int, int] + obstacle_count: int + point_count: int + rejection_reason: str | None = None + + def estimate_ground_plane(frame: CalibratedFrame) -> PlaneFitResult: """Fit a robust plane to visible points in the lower image ground band.""" projected = project_visible_points(frame) @@ -150,38 +170,152 @@ def fit_surface_plane(points: np.ndarray) -> PlaneFitResult: ) -def classify_door_plane_angle( - door: GroundPlaneEstimate, surrounding: GroundPlaneEstimate -) -> tuple[str | None, str | None, float]: - """Classify only clearly coplanar or rotated door and surrounding planes.""" - alignment = abs(float(np.dot(door.normal, surrounding.normal))) - angle_deg = float(np.degrees(np.arccos(np.clip(alignment, -1.0, 1.0)))) - if angle_deg <= 12.0: - return "closed", None, angle_deg - if angle_deg >= 25.0: - return "open", None, angle_deg - return None, "ambiguous_door_angle", angle_deg +def measure_relative_plane_angle( + first: GroundPlaneEstimate, second: GroundPlaneEstimate +) -> OracleMeasurement: + """Measure the unsigned angle between two accepted plane normals.""" + alignment = abs(float(np.dot(first.normal, second.normal))) + return OracleMeasurement( + float(np.degrees(np.arccos(np.clip(alignment, -1.0, 1.0)))), + "deg", + 2.0, + ("accepted_plane_normals",), + (), + ) def classify_forward_corridor( points: np.ndarray, ground: GroundPlaneEstimate ) -> tuple[str | None, tuple[str, ...], str | None]: """Classify a visible camera-forward corridor as clear or blocked.""" + measured = measure_forward_corridor(points, ground) + if measured.rejection_reason is not None: + return None, (), measured.rejection_reason + if measured.obstacle_count >= 4: + return "blocked", ("visible_forward_obstacle", "forward_ground_supported"), None + if measured.obstacle_count: + return None, (), "ambiguous_forward_obstacle" + return "clear", ("forward_ground_supported", "no_supported_forward_obstacle"), None + + +def measure_forward_corridor( + points: np.ndarray, ground: GroundPlaneEstimate +) -> ForwardCorridorMeasurement: + """Measure private floor and elevated-obstacle support in the camera-forward corridor.""" if len(points) == 0: - return None, (), "insufficient_forward_support" + return ForwardCorridorMeasurement((0, 0, 0), 0, 0, "insufficient_forward_support") depth = points[:, 2] lateral_limit = depth * np.tan(np.radians(20.0)) corridor = points[(depth >= 0.5) & (depth <= 3.0) & (np.abs(points[:, 0]) <= lateral_limit)] if len(corridor) < 12: - return None, (), "insufficient_forward_support" + return ForwardCorridorMeasurement( + (0, 0, 0), 0, len(corridor), "insufficient_forward_support" + ) elevation = corridor @ np.asarray(ground.normal) + ground.offset_m ground_points = corridor[np.abs(elevation) <= 0.08] - for start, stop in ((0.5, 1.33), (1.33, 2.16), (2.16, 3.0)): - if int(((ground_points[:, 2] >= start) & (ground_points[:, 2] < stop)).sum()) < 3: - return None, (), "incomplete_forward_ground_support" + bands = tuple( + int(((ground_points[:, 2] >= start) & (ground_points[:, 2] < stop)).sum()) + for start, stop in ((0.5, 1.33), (1.33, 2.16), (2.16, 3.0)) + ) + if any(count < 3 for count in bands): + return ForwardCorridorMeasurement( + bands, 0, len(corridor), "incomplete_forward_ground_support" + ) obstacle_count = int((elevation > 0.15).sum()) - if obstacle_count >= 4: - return "blocked", ("visible_forward_obstacle", "forward_ground_supported"), None - if obstacle_count: - return None, (), "ambiguous_forward_obstacle" - return "clear", ("forward_ground_supported", "no_supported_forward_obstacle"), None + return ForwardCorridorMeasurement(bands, obstacle_count, len(corridor)) + + +def measure_opening_width( + frame: CalibratedFrame, mask: np.ndarray, ground: GroundPlaneEstimate +) -> OpeningGeometryResult: + """Measure a ground-connected vertical aperture from its silhouette and surrounding wall plane.""" + if mask.shape != (frame.image.height, frame.image.width): + raise ValueError("segmentation mask dimensions must match the image") + component_count, labels, stats, _ = cv2.connectedComponentsWithStats( + (mask > 0).astype(np.uint8) + ) + components = [ + index for index in range(1, component_count) if stats[index, cv2.CC_STAT_AREA] >= 128 + ] + if len(components) != 1: + return OpeningGeometryResult(None, None, (), "ambiguous_opening_component") + component = components[0] + x, y, width_px, height_px, _ = stats[component] + if x == 0 or y == 0 or x + width_px >= frame.image.width or y + height_px >= frame.image.height: + return OpeningGeometryResult(None, None, (), "opening_touches_image_edge") + structure = fit_surface_plane(points_around_mask(frame, (labels == component).astype(np.uint8))) + if structure.estimate is None: + return OpeningGeometryResult( + None, + None, + structure.quality_flags, + structure.rejection_reason or "structure_plane_rejected", + ) + normal = np.asarray(structure.estimate.normal) + ground_normal = np.asarray(ground.normal) + if abs(float(normal @ ground_normal)) > np.sin(np.radians(20.0)): + return OpeningGeometryResult( + None, None, structure.quality_flags, "opening_structure_not_vertical" + ) + direction = np.cross(ground_normal, normal) + direction_norm = float(np.linalg.norm(direction)) + if direction_norm == 0: + return OpeningGeometryResult( + None, None, structure.quality_flags, "opening_width_axis_rejected" + ) + direction /= direction_norm + component_mask = labels == component + rows = range(y + int(height_px * 0.3), y + max(int(height_px * 0.7), 1)) + widths: list[float] = [] + for row in rows: + columns = np.flatnonzero(component_mask[row]) + if len(columns) < 2: + continue + left = _intersect_pixel_with_plane(frame, int(columns[0]), row, structure.estimate) + right = _intersect_pixel_with_plane(frame, int(columns[-1]), row, structure.estimate) + if left is not None and right is not None: + widths.append(abs(float((right - left) @ direction))) + if len(widths) < 5: + return OpeningGeometryResult( + None, None, structure.quality_flags, "insufficient_opening_scanlines" + ) + median_width = float(np.median(widths)) + mad = float(np.median(np.abs(np.asarray(widths) - median_width))) + if median_width < 0.4 or max(widths) - min(widths) > max(0.1, median_width * 0.1): + return OpeningGeometryResult(None, None, structure.quality_flags, "ambiguous_opening_width") + bottom_columns = np.flatnonzero(component_mask[y + height_px - 1]) + if not len(bottom_columns): + return OpeningGeometryResult( + None, None, structure.quality_flags, "opening_not_ground_connected" + ) + bottom = _intersect_pixel_with_plane( + frame, int(np.median(bottom_columns)), y + height_px - 1, structure.estimate + ) + if bottom is None or abs(float(bottom @ ground_normal + ground.offset_m)) > 0.08: + return OpeningGeometryResult( + None, None, structure.quality_flags, "opening_not_ground_connected" + ) + return OpeningGeometryResult( + median_width, + max(0.05, structure.estimate.residual_m + 1.4826 * mad), + (*structure.quality_flags, "ground_connected_vertical_opening", "stable_opening_scanlines"), + ) + + +def _intersect_pixel_with_plane( + frame: CalibratedFrame, x: int, y: int, plane: GroundPlaneEstimate +) -> np.ndarray | None: + fx, fy, cx, cy = ( + frame.camera_info.K[0], + frame.camera_info.K[4], + frame.camera_info.K[2], + frame.camera_info.K[5], + ) + if fx <= 0 or fy <= 0: + return None + ray = np.asarray(((x - cx) / fx, (y - cy) / fy, 1.0)) + denominator = float(np.asarray(plane.normal) @ ray) + if abs(denominator) < 1e-6: + return None + distance = -plane.offset_m / denominator + return ray * distance if distance > 0 else None diff --git a/dimos/benchmark/vqa/generation/question_agent.py b/dimos/benchmark/vqa/generation/question_agent.py index 1f2cdb5fba..573c54c2e2 100644 --- a/dimos/benchmark/vqa/generation/question_agent.py +++ b/dimos/benchmark/vqa/generation/question_agent.py @@ -18,13 +18,13 @@ from dimos.models.vl.openai import OpenAIVlModel from dimos.msgs.sensor_msgs.Image import Image -QUESTION_PROMPT = """Select up to 5 challenging but visually well-supported single-frame VQA intents. +QUESTION_PROMPT = """Select up to 15 challenging but visually well-supported single-frame VQA intents. Inspect only this image. Do not assume depth, point clouds, calibration, metadata, or temporal context. Return JSON only: an array of objects with kind, object_query, and threshold_m only for within_distance, candidate_queries only for closest_object, and comparison_query only for compare_height. kind must be one of presence, horizontal_direction, within_distance, visible_count, camera_range, compare_nearest_by_side, compare_left_right, compare_height, door_state, closest_object, -or forward_path. +object_on_support, opening_width, or forward_path. Use threshold_m: 3.0 for within_distance. Use image context to select only intents likely to produce a useful geometric case: emit compare_nearest_by_side only when at least two visible instances of the same object appear on @@ -37,12 +37,15 @@ for two distinct visible object types; comparison_query must name the other object type. Emit closest_object only when one target and at least two distinct candidate object types are visible; candidate_queries must name the visible candidate types. +Emit object_on_support only when one visible object is clearly resting on a distinct visible table, +bench, or other horizontal support; comparison_query must name that support. Emit opening_width only +for one clearly visible doorway aperture with nearby wall structure. Emit forward_path only when the center foreground and visible floor provide enough context to judge the local path directly ahead. Use object_query: "forward path" for forward_path. Do not return bare object names, floors, walls, ceilings, background surfaces, questions, answers, explanations, Markdown, or information not visible in the image.""" -AGENTIC_QUESTION_PROMPT = """Author up to 5 challenging, visually answerable single-frame VQA questions. +AGENTIC_QUESTION_PROMPT = """Author up to 15 challenging, visually answerable single-frame VQA questions. Inspect only this image. Do not use or infer depth, point clouds, calibration, metadata, or answers. Return JSON only: an array of objects with question, answer_contract, optional object_queries, and optional tool_hints. answer_contract is {"kind":"boolean"}, {"kind":"choice","choices":[...]}, @@ -60,11 +63,9 @@ mutually exclusive fixed choices with two to four options. For a clearly visible door with nearby visible structure, you may ask whether it is open or closed with exactly ["open", "closed"]. Use the fixed choices ["clear", "blocked"] only for a visibly supported local path directly ahead. -Use object_queries for every referenced object and tool_hints from "detect_objects", "segment_detections", -"ground_masks", "count_grounded_objects", "bucket_camera_range", "compare_nearest_by_side", -"compare_left_right", "fit_ground_plane", "measure_height", "compare_heights", "classify_door_state", "select_closest_object", -"classify_forward_path", or "bucket_measurement" when applicable. Use visibility/presence questions only -when no stronger geometric question is available. +Use object_queries for every referenced object. The private oracle chooses its own sequence of reusable +perception and geometry tools; do not prescribe an answer-level tool sequence. Use visibility/presence questions +only when no stronger geometric question is available. Do not ask about color, material, text, intent, full physical size, hidden parts, or exact metric distances without a supplied choice contract. Use only visible objects. Do not include answers, explanations, Markdown, or background surfaces.""" @@ -83,8 +84,8 @@ def propose(self, image: Image) -> list[QuestionIntent]: payload: Any = _parse_json_array(self._model.query(image, QUESTION_PROMPT)) except json.JSONDecodeError as exc: raise ValueError("question agent did not return JSON") from exc - if not isinstance(payload, list) or len(payload) > 5: - raise ValueError("question agent must return an array of at most five intents") + if not isinstance(payload, list) or len(payload) > 15: + raise ValueError("question agent must return an array of at most 15 intents") intents: list[QuestionIntent] = [] if all(isinstance(item, str) for item in payload): return [ @@ -95,7 +96,7 @@ def propose(self, image: Image) -> list[QuestionIntent]: ] for item in payload: if not isinstance(item, dict): - raise ValueError("question intent must be an object") + continue kind, query, threshold = ( item.get("kind"), item.get("object_query"), @@ -110,32 +111,34 @@ def propose(self, image: Image) -> list[QuestionIntent]: "compare_nearest_by_side", "compare_left_right", "compare_height", + "object_on_support", + "opening_width", "door_state", "closest_object", "forward_path", ): - raise ValueError(f"unsupported question kind: {kind!r}") + continue if not isinstance(query, str) or not query: - raise ValueError("question intent requires object_query") + continue if kind == "within_distance" and ( not isinstance(threshold, (int, float)) or threshold <= 0 ): - raise ValueError("within_distance requires a positive threshold_m") + continue if kind != "within_distance": threshold = None candidates = _string_tuple(item.get("candidate_queries"), "candidate_queries") comparison_query = item.get("comparison_query") if kind == "closest_object" and (len(candidates) < 2 or query in candidates): - raise ValueError("closest_object requires two distinct candidate_queries") + continue if kind == "forward_path" and query != "forward path": - raise ValueError('forward_path requires object_query "forward path"') - if kind in ("compare_left_right", "compare_height") and ( + continue + if kind in ("compare_left_right", "compare_height", "object_on_support") and ( not isinstance(comparison_query, str) or not comparison_query or comparison_query == query ): - raise ValueError(f"{kind} requires a distinct comparison_query") - if kind not in ("compare_left_right", "compare_height"): + continue + if kind not in ("compare_left_right", "compare_height", "object_on_support"): comparison_query = None intents.append(QuestionIntent(kind, query, threshold, candidates, comparison_query)) return intents @@ -144,7 +147,7 @@ def propose(self, image: Image) -> list[QuestionIntent]: class OpenAIFreeformQuestionAuthor: """Image-only author for generic public questions and answer contracts.""" - def __init__(self, model: OpenAIVlModel, max_questions: int = 5) -> None: + def __init__(self, model: OpenAIVlModel, max_questions: int = 15) -> None: self._model = model self._max_questions = max_questions @@ -155,9 +158,16 @@ def propose(self, image: Image) -> list[QuestionProposal]: raise ValueError("question author did not return JSON") from exc if not isinstance(payload, list) or len(payload) > self._max_questions: raise ValueError("question author returned too many questions") - proposals = [ - _proposal_from_json(item, index) for index, item in enumerate(payload, start=1) - ] + proposals = [] + errors: list[ValueError] = [] + for index, item in enumerate(payload, start=1): + try: + proposals.append(_proposal_from_json(item, index)) + except ValueError as exc: + errors.append(exc) + continue + if not proposals and errors: + raise errors[0] if len({item.id for item in proposals}) != len(proposals): raise ValueError("question ids must be unique") return proposals diff --git a/dimos/benchmark/vqa/generation/test_agents.py b/dimos/benchmark/vqa/generation/test_agents.py index 6b001341e9..a99cdf8448 100644 --- a/dimos/benchmark/vqa/generation/test_agents.py +++ b/dimos/benchmark/vqa/generation/test_agents.py @@ -10,8 +10,11 @@ from dimos.benchmark.vqa.generation.primitives.contracts import ( HeightMeasurementResult, HorizontalRelationResult, + ObjectPlaneRelationResult, + OpeningWidthResult, ) from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.geometry import PlaneFitResult from dimos.benchmark.vqa.generation.question_agent import OpenAIQuestionAgent from dimos.benchmark.vqa.models import ( CalibratedFrame, @@ -190,9 +193,10 @@ def query(self, image: Image, prompt: str) -> str: assert "comparison_query" in prompt return """[ {"kind":"visible_count","object_query":"chair"}, - {"kind":"camera_range","object_query":"lamp"}, {"kind":"compare_left_right","object_query":"chair","comparison_query":"table"}, {"kind":"compare_height","object_query":"chair","comparison_query":"table"} + ,{"kind":"object_on_support","object_query":"box","comparison_query":"table"} + ,{"kind":"opening_width","object_query":"doorway"} ]""" frame, _ = _frame_and_detection() @@ -203,9 +207,10 @@ def query(self, image: Image, prompt: str) -> str: assert intents == [ QuestionIntent(kind="visible_count", object_query="chair"), - QuestionIntent(kind="camera_range", object_query="lamp"), QuestionIntent(kind="compare_left_right", object_query="chair", comparison_query="table"), QuestionIntent(kind="compare_height", object_query="chair", comparison_query="table"), + QuestionIntent(kind="object_on_support", object_query="box", comparison_query="table"), + QuestionIntent(kind="opening_width", object_query="doorway"), ] @@ -442,3 +447,99 @@ def test_ground_truth_agent_compares_pairwise_left_right(monkeypatch: object) -> assert result.status == "answered" assert result.answer == "left" assert result.question.allowed_answers == ("left", "right") + + +def test_ground_truth_agent_generates_verified_support_and_opening_width( + monkeypatch: object, +) -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + box = GroundedObject("box-0", "box", 8, 1.0, "left") + table = GroundedObject("table-0", "table", 8, 2.0, "right") + monkeypatch.setattr( + agent, + "ground", + lambda _frame, query: ([box] if query == "box" else [table], ()), + ) + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + monkeypatch.setattr(agent.primitives, "fit_ground_plane", lambda: PlaneFitResult(plane, ())) + monkeypatch.setattr( + agent.primitives, "fit_object_surface_plane", lambda item: PlaneFitResult(plane, ()) + ) + monkeypatch.setattr( + agent.primitives, + "measure_object_plane_relation", + lambda item, support, support_plane, normal: ObjectPlaneRelationResult( + 0.0, + 0.8, + 0.8, + 4, + 0.0, + 3, + (), + ), + ) + monkeypatch.setattr( + agent.primitives, + "measure_opening_width_from_mask", + lambda mask, ground: OpeningWidthResult(OracleMeasurement(0.6, "m", 0.05, (), ()), ()), + ) + monkeypatch.setattr(agent.primitives, "segment_detections", lambda query: [detection]) + + support = agent.answer( + frame, + QuestionIntent(kind="object_on_support", object_query="box", comparison_query="table"), + ) + opening = agent.answer(frame, QuestionIntent(kind="opening_width", object_query="doorway")) + + assert support.answer == "yes" + assert support.question.allowed_answers == ("yes", "no") + assert opening.answer == "0.5 to under 0.8 m" + + +def test_ground_truth_agent_keeps_verified_non_support_cases(monkeypatch: object) -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + box = GroundedObject("box-0", "box", 8, 1.0, "left") + table = GroundedObject("table-0", "table", 8, 2.0, "right") + monkeypatch.setattr( + agent, + "ground", + lambda _frame, query: ([box] if query == "box" else [table], ()), + ) + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + monkeypatch.setattr(agent.primitives, "fit_ground_plane", lambda: PlaneFitResult(plane, ())) + monkeypatch.setattr( + agent.primitives, "fit_object_surface_plane", lambda item: PlaneFitResult(plane, ()) + ) + monkeypatch.setattr( + agent.primitives, + "measure_object_plane_relation", + lambda item, support, support_plane, normal: ObjectPlaneRelationResult( + 0.0, + 0.8, + 0.8, + 0, + 0.3, + 0, + (), + ), + ) + + result = agent.answer( + frame, + QuestionIntent(kind="object_on_support", object_query="box", comparison_query="table"), + ) + + assert result.status == "answered" + assert result.answer == "no" diff --git a/dimos/benchmark/vqa/generation/test_oracle.py b/dimos/benchmark/vqa/generation/test_oracle.py index 53fcec067e..6642870e1b 100644 --- a/dimos/benchmark/vqa/generation/test_oracle.py +++ b/dimos/benchmark/vqa/generation/test_oracle.py @@ -23,12 +23,17 @@ from dimos.benchmark.vqa.generation.primitives.contracts import ( HeightMeasurementResult, HorizontalRelationResult, + ObjectOnSupportResult, + OpeningWidthResult, ) from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives from dimos.benchmark.vqa.generation.primitives.geometry import ( - classify_door_plane_angle, + ForwardCorridorMeasurement, + PlaneFitResult, classify_forward_corridor, estimate_ground_plane, + measure_opening_width, + measure_relative_plane_angle, ) from dimos.benchmark.vqa.generation.question_agent import ( AGENTIC_QUESTION_PROMPT, @@ -46,6 +51,7 @@ OracleMeasurement, OracleToolResult, QuestionProposal, + RejectedOracleResult, ) from dimos.models.vl.openai import OpenAIVlModel from dimos.msgs.geometry_msgs.Transform import Transform @@ -148,33 +154,32 @@ def bind_tools(self, tools: Any) -> _BoundModel: def invoke(self, messages: Any) -> AIMessage: self._calls += 1 if self._calls == 1: - return AIMessage( - content="", - tool_calls=[{"name": "detect_objects", "args": {"query": "chair"}, "id": "call-1"}], - ) - if self._calls == 2: return AIMessage( content="", tool_calls=[ { - "name": "segment_detections", - "args": {"detection_id": "detection:v1:0001"}, + "name": "get_object_pose", + "args": {"object_id": "synthetic-chair-0"}, "id": "call-2", } ], ) - if self._calls == 3: - return AIMessage( - content="", - tool_calls=[ - {"name": "ground_masks", "args": {"mask_id": "mask:v1:0002"}, "id": "call-3"} - ], - ) return AIMessage( content='{"answer":"yes","evidence_ids":["grounding:v1:synthetic-chair-0"]}' ) +class _ScriptedModel: + def __init__(self, responses: list[AIMessage]) -> None: + self._responses = responses + + def bind_tools(self, tools: Any) -> _ScriptedModel: + return self + + def invoke(self, messages: Any) -> AIMessage: + return self._responses.pop(0) + + class _SemanticValidator: def __init__(self, verdict: SemanticEvidenceValidation) -> None: self.verdict = verdict @@ -203,11 +208,11 @@ def test_freeform_question_author_parses_public_contract() -> None: def test_freeform_author_prompt_prioritizes_geometric_questions() -> None: - assert "measure_height" in AGENTIC_QUESTION_PROMPT + assert "chooses its own sequence" in AGENTIC_QUESTION_PROMPT assert "closest to a named target" in AGENTIC_QUESTION_PROMPT assert "visibly repeated object" in AGENTIC_QUESTION_PROMPT assert "diverse set of questions" in AGENTIC_QUESTION_PROMPT - assert "Use visibility/presence questions only" in AGENTIC_QUESTION_PROMPT + assert "visibility/presence" in AGENTIC_QUESTION_PROMPT def test_freeform_question_author_assigns_missing_ids() -> None: @@ -299,20 +304,15 @@ def query(self, image: Image, prompt: str) -> str: def test_local_tool_returns_geometry_and_evidence_ids() -> None: - registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + registry = LocalOracleToolRegistry(_frame_primitives(_measurement_frame())) detection = json.loads(registry.detect_objects("chair")) - masks = json.loads(registry.segment_detections(detection["detection_id"])) - payload = json.loads(registry.ground_masks(masks["mask_id"])) - - assert payload["objects"][0] == { - "evidence_id": "grounding:v1:synthetic-chair-0", - "id": "synthetic-chair-0", - "label": "chair", - "range_m": 1.0, - "side": "left", - "point_count": 4, - } + masks = json.loads(registry.segment_detection(detection["detections"][0]["detection_id"])) + payload = json.loads(registry.ground_mask(masks["mask_ids"][0])) + + assert "detection_id" not in detection + assert payload["object_id"].startswith("object:v1:") + assert payload["objects"][0]["evidence_id"] == f"grounding:v1:{payload['object_id']}" assert registry.results[-1].version == "v1" @@ -348,15 +348,15 @@ def test_height_tool_measures_visible_object_points_above_plane() -> None: registry = LocalOracleToolRegistry(_frame_primitives(frame, mask)) detection = json.loads(registry.detect_objects("chair")) - masks = json.loads(registry.segment_detections(detection["detection_id"])) - grounded = json.loads(registry.ground_masks(masks["mask_id"])) + masks = json.loads(registry.segment_detection(detection["detections"][0]["detection_id"])) + grounded = json.loads(registry.ground_mask(masks["mask_ids"][0])) plane = json.loads(registry.fit_ground_plane()) - payload = json.loads(registry.measure_height(grounded["object_ids"][0], plane["plane_id"])) + payload = json.loads(registry.measure_height(grounded["object_id"], plane["plane_id"])) assert payload["measurement"]["unit"] == "m" assert 0.8 < payload["measurement"]["value"] < 1.0 assert payload["measurement"]["tolerance"] >= 0.05 - assert payload["objects"][0]["evidence_id"] == "height:v1:synthetic-chair-0" + assert payload["objects"][0]["evidence_id"] == f"height:v1:{grounded['object_id']}" assert "visible_point_cloud_height" in payload["quality_flags"] @@ -365,23 +365,47 @@ def test_local_registry_exposes_geometry_tools() -> None: assert {tool.name for tool in registry.tools()} == { "detect_objects", - "segment_detections", - "ground_masks", - "select_nearest_object", - "count_grounded_objects", - "bucket_camera_range", - "compare_nearest_by_side", - "compare_left_right", - "select_closest_object", + "segment_detection", + "ground_mask", "fit_ground_plane", + "get_object_pose", + "fit_object_surface_plane", + "fit_mask_surrounding_plane", + "measure_object_pair_distance", + "measure_relative_plane_angle", + "measure_object_plane_relation", + "measure_aperture_geometry", + "measure_forward_corridor", "measure_height", - "compare_heights", - "classify_door_state", - "classify_forward_path", - "bucket_measurement", } +def test_local_registry_exposes_forward_corridor_metrics(monkeypatch: Any) -> None: + primitives = _frame_primitives(_measurement_frame()) + registry = LocalOracleToolRegistry(primitives) + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + registry._planes["ground"] = plane + registry._ground_planes.add("ground") + monkeypatch.setattr( + primitives, + "measure_forward_corridor", + lambda accepted_plane: ForwardCorridorMeasurement((3, 4, 5), 6, 24), + ) + + payload = json.loads(registry.measure_forward_corridor("ground")) + rejected = json.loads(registry.measure_forward_corridor("unknown")) + + assert payload["metrics"] == { + "ground_band_1_count": 3.0, + "ground_band_2_count": 4.0, + "ground_band_3_count": 5.0, + "elevated_obstacle_count": 6.0, + "corridor_point_count": 24.0, + } + assert payload["objects"][0]["evidence_id"] == "forward-corridor:v1:synthetic" + assert rejected["rejection_reason"] == "unknown_ground_plane_id" + + def test_height_choice_window_is_local_and_deterministic() -> None: choices, answer = height_choice_window(0.42) @@ -415,6 +439,9 @@ def test_count_and_camera_range_choices_are_fixed_and_non_overlapping() -> None: def test_local_registry_buckets_count_range_and_pairwise_side() -> None: registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + registry._objects["synthetic-chair-0"] = GroundedObject( + "synthetic-chair-0", "chair", 4, 1.0, "left" + ) left = GroundedObject("left", "chair", 8, 1.0, "left") right = GroundedObject("right", "chair", 8, 2.0, "right") registry._objects = {left.id: left, right.id: right} @@ -459,15 +486,75 @@ def test_local_registry_compares_pairwise_relation_and_height(monkeypatch: Any) assert height["choice"] == "chair" -def test_door_state_accepts_clear_plane_angles_and_rejects_ajar() -> None: +def test_local_registry_verifies_support_and_measures_opening(monkeypatch: Any) -> None: + primitives = _frame_primitives(_measurement_frame()) + registry = LocalOracleToolRegistry(primitives) + box = GroundedObject("box", "box", 8, 1.0, "left") + table = GroundedObject("table", "table", 8, 2.0, "right") + registry._objects = {box.id: box, table.id: table} + registry._masks = {"mask": "doorway"} + measurement = OracleMeasurement(0.9, "m", 0.05, (), ()) + monkeypatch.setattr( + primitives, + "classify_object_on_support", + lambda item, support: ObjectOnSupportResult( + item, support, ("contact_band_supported",), answer="yes" + ), + ) + monkeypatch.setattr( + primitives, + "measure_opening_width", + lambda query: OpeningWidthResult(measurement, ("stable_opening_scanlines",)), + ) + + support = json.loads(registry.classify_object_on_support(box.id, table.id)) + opening = json.loads(registry.measure_opening_width("mask")) + + assert support["choice"] == "yes" + assert opening["measurement"]["value"] == 0.9 + + +def test_opening_width_uses_wall_geometry_outside_the_aperture() -> None: + image = Image.from_numpy(np.zeros((100, 100, 3), dtype=np.uint8)) + camera_info = CameraInfo.from_intrinsics(50.0, 50.0, 50.0, 50.0, 100, 100) + opening = np.zeros((100, 100), dtype=np.uint8) + opening[35:61, 40:61] = 255 + points: list[list[float]] = [] + for y in range(20, 61, 4): + for x in range(28, 73, 4): + if opening[y, x]: + continue + points.append([(x - 50) / 50 * 5.0, (y - 50) / 50 * 5.0, 5.0]) + for y in (65, 75, 85, 95): + for x in range(0, 100, 10): + depth = 1.0 / ((y - 50) / 50) + points.append([(x - 50) / 50 * depth, 1.0, depth]) + frame = CalibratedFrame( + "opening", + image, + PointCloud2.from_numpy(np.asarray(points, dtype=np.float32)), + camera_info, + Transform.identity(), + True, + ) + ground = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + + result = measure_opening_width(frame, opening, ground) + + assert result.rejection_reason is None + assert result.width_m is not None + assert 1.9 < result.width_m < 2.1 + + +def test_relative_plane_angle_measurement_is_unsigned() -> None: door = GroundPlaneEstimate((1.0, 0.0, 0.0), 0.0, 20, 20, 0.01) closed = GroundPlaneEstimate((1.0, 0.0, 0.0), 0.0, 20, 20, 0.01) open_door = GroundPlaneEstimate((0.0, 0.0, 1.0), 0.0, 20, 20, 0.01) ajar_door = GroundPlaneEstimate((0.95, 0.0, 0.31), 0.0, 20, 20, 0.01) - assert classify_door_plane_angle(door, closed)[0] == "closed" - assert classify_door_plane_angle(door, open_door)[0] == "open" - assert classify_door_plane_angle(door, ajar_door)[1] == "ambiguous_door_angle" + assert measure_relative_plane_angle(door, closed).value == 0.0 + assert measure_relative_plane_angle(door, open_door).value == 90.0 + assert 17.0 < measure_relative_plane_angle(door, ajar_door).value < 19.0 def test_closest_object_uses_point_cloud_centroids_and_rejects_ties(monkeypatch: Any) -> None: @@ -515,6 +602,28 @@ def test_horizontal_relation_uses_camera_frame_support_centroids(monkeypatch: An ) +def test_object_on_support_keeps_clearly_separated_objects_as_no(monkeypatch: Any) -> None: + box = GroundedObject("box", "box", 8, 1.0, "left") + support = GroundedObject("table", "table", 8, 2.0, "right") + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + primitives = _frame_primitives(_measurement_frame()) + points = { + box.id: np.tile((2.0, 0.5, 4.0), (6, 1)), + support.id: np.tile((0.0, 1.0, 4.0), (6, 1)), + } + monkeypatch.setattr(primitives, "_object_points", lambda item: points[item.id]) + monkeypatch.setattr(primitives, "fit_ground_plane", lambda: PlaneFitResult(plane, ())) + monkeypatch.setattr( + "dimos.benchmark.vqa.generation.primitives.frame.fit_surface_plane", + lambda support_points: PlaneFitResult(plane, ("surface_plane_accepted",)), + ) + + result = primitives.classify_object_on_support(box, support) + + assert result.answer == "no" + assert "objects_separated_in_plane" in result.quality_flags + + def test_forward_corridor_requires_ground_support_and_detects_obstacles() -> None: ground = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) floor = np.asarray( @@ -550,31 +659,34 @@ def test_oracle_validates_evidence_and_answer_contract() -> None: raise AssertionError("unknown evidence was accepted") -def test_oracle_derives_deferred_height_answer_from_measurement_bucket() -> None: +def test_oracle_derives_deferred_height_answer_from_cited_measurement() -> None: proposal = QuestionProposal( "q", "How tall is the chair?", DeferredHeightChoiceContract(), ("chair",) ) - evidence = OracleEvidence("height-1", "v1", "chair-1", "chair", 1.0, "left", 8) + measurement = OracleMeasurement(0.42, "m", 0.05, (), ()) + evidence = OracleEvidence("height-1", "v1", "chair-1", "chair", 1.0, "left", 8, measurement) result = OracleToolResult( - "bucket_measurement", + "measure_height", "chair", (evidence,), - choice="0.2-0.6 m", - choices=("under 0.2 m", "0.2-0.6 m", "0.6-1.0 m", "over 1.0 m"), + measurement=measurement, ) - assert validate_oracle_answer(proposal, "0.2-0.6 m", ["height-1"], (result,)) == "0.2-0.6 m" + assert validate_oracle_answer(proposal, None, ["height-1"], (result,)) == "0.2-0.6 m" try: - validate_oracle_answer(proposal, "under 0.2 m", ["height-1"], (result,)) + validate_oracle_answer(proposal, None, [], (result,)) except ValueError as exc: - assert "does not match measurement bucket" in str(exc) + assert "evidence_ids" in str(exc) else: - raise AssertionError("non-derived deferred height answer was accepted") + raise AssertionError("deferred height answer without evidence was accepted") def test_private_oracle_runs_direct_structured_tool() -> None: proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + registry._objects["synthetic-chair-0"] = GroundedObject( + "synthetic-chair-0", "chair", 4, 1.0, "left" + ) validator = _SemanticValidator(SemanticEvidenceValidation(True, "chair grounding supports yes")) result = PrivateToolCallingOracle( @@ -621,6 +733,48 @@ def invoke(self, messages: Any) -> AIMessage: assert result.reason == "invalid_final_answer:answer cites unknown evidence" +def test_private_oracle_allows_explicit_rejection_without_answer_resolution() -> None: + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + validator = _SemanticValidator(SemanticEvidenceValidation(True, "should not be called")) + model = _ScriptedModel( + [ + AIMessage( + content="", + tool_calls=[{"name": "detect_objects", "args": {"query": "chair"}, "id": "call-1"}], + ), + AIMessage( + content='{"status":"rejected","reason":"chair cannot be verified","evidence_ids":[]}' + ), + ] + ) + + result = PrivateToolCallingOracle(cast("Any", model), semantic_validator=validator).answer( + proposal, registry + ) + + assert isinstance(result, RejectedOracleResult) + assert result.reason == "chair cannot be verified" + assert result.tool_results == registry.results + assert len(result.trace) == 1 + assert result.trace[0].operation == "tool" + assert validator.calls == [] + + +def test_private_oracle_rejects_malformed_explicit_rejection() -> None: + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract()) + validator = _SemanticValidator(SemanticEvidenceValidation(True, "should not be called")) + model = _ScriptedModel([AIMessage(content='{"status":"rejected","reason":""}')]) + + result = PrivateToolCallingOracle(cast("Any", model), semantic_validator=validator).answer( + proposal, LocalOracleToolRegistry(cast("Any", _Grounding())) + ) + + assert isinstance(result, RejectedOracleResult) + assert result.reason.startswith("invalid_final_answer:") + assert validator.calls == [] + + def test_agentic_oracle_never_uses_legacy_answer_program() -> None: class _GroundingWithoutLegacyAnswer(_Grounding): def answer(self, frame: Any, intent: Any) -> None: @@ -628,6 +782,9 @@ def answer(self, frame: Any, intent: Any) -> None: proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) registry = LocalOracleToolRegistry(cast("Any", _GroundingWithoutLegacyAnswer())) + registry._objects["synthetic-chair-0"] = GroundedObject( + "synthetic-chair-0", "chair", 4, 1.0, "left" + ) validator = _SemanticValidator(SemanticEvidenceValidation(True, "chair grounding supports yes")) result = PrivateToolCallingOracle( diff --git a/dimos/benchmark/vqa/models.py b/dimos/benchmark/vqa/models.py index 33c2f81061..6ae4d4f231 100644 --- a/dimos/benchmark/vqa/models.py +++ b/dimos/benchmark/vqa/models.py @@ -95,6 +95,8 @@ class VqaExample: "compare_nearest_by_side", "compare_left_right", "compare_height", + "object_on_support", + "opening_width", "door_state", "closest_object", "forward_path", @@ -221,6 +223,7 @@ class OracleToolResult: plane: GroundPlaneEstimate | None = None quality_flags: tuple[str, ...] = () rejection_reason: str | None = None + metrics: tuple[tuple[str, float], ...] = () @dataclass(frozen=True) diff --git a/docs/benchmarking/vqa-generation/infrastructure.md b/docs/benchmarking/vqa-generation/infrastructure.md index bfd03e4a9f..509102f29e 100644 --- a/docs/benchmarking/vqa-generation/infrastructure.md +++ b/docs/benchmarking/vqa-generation/infrastructure.md @@ -45,7 +45,8 @@ dimos/benchmark/vqa/ geometry.py plane fitting and masked-point helpers selection.py nearest-object selection choices.py deterministic answer-choice resolution - ground_truth_generator.py deterministic constrained recipe runner + ground_truth_generator.py primitive-owning answer dispatch and grounding coordinator + families.py deterministic constrained family recipes and result helpers oracle_tools.py opaque-ID and LangChain adapter for primitives oracle.py bounded tool-calling and evidence validation question_agent.py image-only question author @@ -75,21 +76,21 @@ generation can skip completed frames after an interrupted run. ### Shared Perception Primitives -Constrained and agentic generation use the same private primitives: +Constrained and agentic generation share the same private geometry runtime. Constrained families select a +fixed recipe; the agentic oracle may inspect intermediate results and choose its own next operation. Handles +are immutable and scoped to one frozen frame. ```text detect_objects(query) --> segment_detections(detection_id) --> ground_masks(mask_id) --> select_nearest_object(object_ids, side) +-> segment_detection(detection_id) +-> ground_mask(mask_id) +-> object_id / pose / supported point set -> fit_ground_plane() --> measure_height(object_id, plane_id) --> bucket_measurement(measurement_id) +-> fitted planes and reusable geometric measurements ``` -Constrained generation selects a fixed sequence for each question family. Agentic generation lets -the oracle select a bounded sequence of these tools, passing opaque IDs rather than masks or point -arrays between calls. +Agentic answers may explicitly reject a question when the available primitive results cannot establish it; +the rejection and missing-evidence reason remain private generation audit data. ## Question And Answer Contracts diff --git a/docs/benchmarking/vqa-generation/pipeline.md b/docs/benchmarking/vqa-generation/pipeline.md index 487c0bbfbe..4dcd334925 100644 --- a/docs/benchmarking/vqa-generation/pipeline.md +++ b/docs/benchmarking/vqa-generation/pipeline.md @@ -69,6 +69,8 @@ every family. Private grounding still rejects unsupported or ambiguous candidate | Nearest by side | `Which chair is closer, the left or right one?` | `left`, `right` | | Pairwise left/right | `Is the chair to the left or right of the table?` | `left`, `right` | | Height comparison | `Which is taller: the chair or the table?` | `chair`, `table` | +| Direct support | `Is the box on the table?` | `yes`, `no` | +| Opening width | `How wide is the doorway?` | `under 0.2 m`, `0.2 to under 0.5 m`, `0.5 to under 0.8 m`, `0.8 m or more` | | Closest object | `Which object is closest to the chair: table or lamp?` | `table`, `lamp` | | Door state | `Is the door open or closed?` | `open`, `closed` | | Forward path | `Is the path directly ahead clear or blocked?` | `clear`, `blocked` | @@ -91,81 +93,64 @@ example, a private height of `0.42 m` produces: under 0.2 m | 0.2-0.6 m | 0.6-1.0 m | over 1.0 m ``` -## 3. Pre-Answer Grounding Checks - -For each referenced object: - -1. MoonDream detects or point-localizes the object. -2. EdgeTAM produces a mask. -3. Visible calibrated point-cloud samples are projected into the mask. -4. Mask area must meet `--min-mask-area-px`. -5. Point support must meet `--min-foreground-points`. - -Height questions also require: +The agentic oracle is not bound to a constrained family. It can inspect individual detection, mask, +object, and plane handles, choose its own tool sequence, and either return a cited answer or privately +reject the proposal with the missing evidence. -1. Accepted Open3D RANSAC ground plane. -2. Exactly one grounded object and one mask. -3. At least six points inside the object mask. -4. At least four elevated points, with at least 60% of selected points elevated more than `0.02 m` above the plane. - -Door-state questions also require one grounded door, a robust plane fit for the door mask, and a -robust plane fit in a narrow ring around that mask. The planes must be either nearly aligned -(`closed`) or clearly rotated (`open`); slightly ajar or otherwise ambiguous doors are rejected. - -Closest-object questions require exactly one grounded target and one grounded instance for every -candidate choice. They compare private support-point centroids and reject candidates whose nearest -two distances are within `0.15 m`. +## 3. Pre-Answer Grounding Checks -Visible-count questions count only accepted grounded instances, rather than raw detections. Camera-range -questions use the nearest grounded instance's camera-origin Euclidean range. Height comparisons require -one accepted shared ground plane and one successful physical-height measurement per distinct object; -overlapping measurement uncertainty intervals are rejected. +All referenced objects need a detected mask with enough visible 3D point support. -Pairwise left/right questions require exactly one grounded instance for each named object. They compare -their visible support-point centroids in the camera horizontal axis and reject separation under `0.1 m`. +- Height: one grounded object and an accepted ground plane. +- Door state: a door plane and nearby surrounding plane with a clear relative angle. +- Closest object: one grounded target and one grounded instance per candidate. +- Count and range: accepted grounded instances only. +- Left/right and height comparison: one grounded instance per named object with clear separation. +- Object on support: contact or clear separation relative to the support plane. +- Opening width: one ground-connected aperture and stable surrounding-wall geometry. +- Forward path: visible ground support across the forward corridor and supported obstacle evidence. -Forward-path questions require a fitted visible ground plane and enough ground support in each third -of the center camera-forward corridor from `0.5-3.0 m`. Supported non-ground points block the -corridor; incomplete ground support or sparse obstacle evidence is rejected. +Missing, sparse, or ambiguous evidence rejects the question. ## 4. Create Answers ### Constrained -Each deterministic family runs its own fixed sequence. +Each deterministic family runs its own fixed sequence. `ground(A)` below is always the same private +primitive chain: + +```text +ground(A) +-> detect_objects(A) +-> segment_detections(A) +-> ground_masks(A) +-> accepted grounded A instances, or reject if the family requires an unavailable instance +``` ```text presence(A) --> detect_objects(A) -> detection_id --> segment_detections(detection_id) -> mask_id --> ground_masks(mask_id) -> grounded A instances +-> ground(A) -> no grounded A: reject -> one or more grounded A instances: yes ``` ```text horizontal_direction(A) --> detect_objects(A) -> detection_id --> segment_detections(detection_id) -> mask_id --> ground_masks(mask_id) -> grounded A instances +-> ground(A) -> select_nearest_object(grounded A instances) -> nearest A -> nearest A horizontal_direction: left/center/right ``` ```text within_distance(A, T) --> detect_objects(A) -> detection_id --> segment_detections(detection_id) -> mask_id --> ground_masks(mask_id) -> grounded A instances +-> ground(A) -> select_nearest_object(grounded A instances) -> nearest A -> nearest A range_m <= T: yes/no ``` ```text compare_nearest_by_side(A) --> detect_objects(A) -> detection_id --> segment_detections(detection_id) -> mask_id --> ground_masks(mask_id) -> grounded A instances +-> ground(A) -> select_nearest_object(grounded A instances, left) -> nearest left A -> select_nearest_object(grounded A instances, right) -> nearest right A -> either side missing or tied: reject @@ -174,50 +159,98 @@ compare_nearest_by_side(A) ```text visible_count(A) --> detect_objects(A) -> segment_detections(...) -> ground_masks(...) -> grounded A instances +-> ground(A) -> bucket accepted instance count: 1-2 / 3-4 / 5-7 / 8+ ``` ```text camera_range(A) --> detect_objects(A) -> segment_detections(...) -> ground_masks(...) -> grounded A instances +-> ground(A) -> select_nearest_object(...) -> bucket camera-origin range ``` ```text compare_left_right(A, B) --> ground exactly one A and one B -> compare camera-frame support centroids +-> ground(A), ground(B) +-> require exactly one accepted A and B +-> compare camera-frame support centroids -> separation under 0.1 m: reject -> otherwise choose left/right ``` ```text compare_height(A, B) --> ground exactly one A and one B -> fit_ground_plane() +-> ground(A), ground(B) +-> require exactly one accepted A and B +-> fit_ground_plane() -> measure_height(A, plane) and measure_height(B, plane) -> reject overlapping uncertainty intervals -> choose taller A/B ``` +```text +object_on_support(A, B) +-> ground(A), ground(B) +-> require exactly one accepted A and B +-> fit_ground_plane() -> ground plane +-> fit_object_surface_plane(B) -> support plane +-> reject non-horizontal support plane +-> measure_object_plane_relation(A, B, support plane, ground normal) +-> clear separation: no -> unsupported or ambiguous geometry: reject -> otherwise yes +``` + +```text +opening_width(A) +-> detect_objects(A) -> segment_detections(A) +-> require exactly one accepted aperture mask +-> fit_ground_plane() -> ground plane +-> measure_opening_width_from_mask(mask, ground plane) +-> reject non-ground-connected, nonvertical, edge-clipped, or unstable aperture geometry +-> bucket: under 0.2 / 0.2-0.5 / 0.5-0.8 / 0.8 m or more +``` + +```text +closest_object(A, B, C, ...) +-> ground(A), ground(B), ground(C), ... +-> require exactly one target and one instance for every candidate type +-> select_closest_object(target, candidates) from private support-point centroids +-> reject ties within 0.15 m -> otherwise choose the closest candidate +``` + +```text +door_state(A) +-> ground(A) -> require exactly one door +-> fit_object_surface_plane(door) -> door plane +-> fit_object_surrounding_plane(door) -> surrounding plane +-> measure_relative_plane_angle(door plane, surrounding plane) +-> nearly aligned: closed; clearly rotated: open; intermediate/failed geometry: reject +``` + +```text +forward_path() +-> fit_ground_plane() +-> classify_forward_corridor(visible camera points, ground plane) +-> require ground support across each forward distance band +-> supported elevated points: blocked; no supported obstacle: clear; otherwise reject +``` + ### Agentic The private oracle chooses a sequence from the same read-only primitives used by constrained recipes: | Tool | Input | Output | |---|---|---| -| `detect_objects` | semantic query | Detection ID and private boxes. | -| `segment_detections` | detection ID | Mask ID and accepted mask count. | -| `ground_masks` | mask ID | Grounded object IDs, range, side, point support, evidence IDs. | -| `select_nearest_object` | object IDs, optional side | Nearest grounded object ID. | -| `count_grounded_objects` | object IDs | Fixed count bucket and cited grounded instances. | -| `bucket_camera_range` | object ID | Fixed camera-origin range bucket and cited object. | -| `compare_nearest_by_side` | object IDs | Public `left` or `right` choice from nearest grounded objects. | -| `compare_left_right` | two object IDs | Public pairwise `left` or `right` relation, or an ambiguity rejection. | -| `select_closest_object` | target ID, candidate IDs | Candidate nearest to target by private support-point centroids. | +| `detect_objects` | semantic query | Individual opaque detection IDs and confidence. | +| `segment_detection` | detection ID | Individual opaque mask IDs. | +| `ground_mask` | mask ID | One grounded object ID and citable support evidence. | | `fit_ground_plane` | none | Plane ID, plane estimate, residual, inlier support, quality flags. | +| `get_object_pose` | object ID | Camera-frame grounding evidence: range, side, support count. | +| `fit_object_surface_plane` | object ID | Plane ID from one object's visible support. | +| `fit_mask_surrounding_plane` | mask ID | Plane ID from visible support around a mask. | +| `measure_object_pair_distance` | two object IDs | Private 3D support-centroid distance. | +| `measure_relative_plane_angle` | two plane IDs | Private unsigned angle between accepted planes. | +| `measure_object_plane_relation` | object ID, support ID, support plane, ground plane | Clearance, contact, and projected-separation metrics. | +| `measure_aperture_geometry` | mask ID, ground plane | Private ground-connected aperture span and uncertainty. | +| `measure_forward_corridor` | ground plane | Private floor-support and elevated-obstacle metrics. | | `measure_height` | object ID, plane ID | Measurement ID, private height, uncertainty, provenance, quality flags. | -| `compare_heights` | two object IDs, plane ID | Taller object choice from shared-plane measurements, or rejection. | -| `classify_door_state` | object ID | Public `open` or `closed` choice, or a private geometry rejection. | -| `classify_forward_path` | none | Public `clear` or `blocked` choice, or a private visibility rejection. | -| `bucket_measurement` | measurement ID | Public answer-conditioned height choices and matching choice. | Opaque IDs chain tool results; raw masks and point-cloud arrays are not exposed to the oracle. The oracle returns a candidate answer and cited evidence IDs. Deferred height choices are the sole