diff --git a/dimos/benchmark/evaluation/registry.py b/dimos/benchmark/evaluation/registry.py index a6608daf67..14e929d911 100644 --- a/dimos/benchmark/evaluation/registry.py +++ b/dimos/benchmark/evaluation/registry.py @@ -31,6 +31,7 @@ LOCAL_NAME_PATTERN = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") BUILTIN_EVALUATIONS = { "frozen-integer-qa": ("dimos.benchmark.short_horizon_qa.evaluation:frozen_integer_qa"), + "point-cloud-vqa": ("dimos.benchmark.vqa.evaluation:point_cloud_vqa"), } diff --git a/dimos/benchmark/vqa/evaluation.py b/dimos/benchmark/vqa/evaluation.py new file mode 100644 index 0000000000..c7ae259156 --- /dev/null +++ b/dimos/benchmark/vqa/evaluation.py @@ -0,0 +1,144 @@ +# Copyright 2026 Dimensional Inc. +"""Multiple-choice VQA evaluation plugin using public image artifacts only.""" + +from __future__ import annotations + +from collections.abc import Callable +import json +from pathlib import Path +import re + +import cv2 +from pydantic import BaseModel, ConfigDict, Field + +from dimos.benchmark.evaluation.models import ( + ArtifactNativeResult, + ArtifactReference, + EvaluationReport, + SummaryItem, +) +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.sensor_msgs.Image import Image + + +class VqaEvaluationConfig(BaseModel): + """Location and vision model for a generated VQA evaluation dataset.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + dataset: str = Field(min_length=1) + model: str = Field(default="gpt-4o-mini", min_length=1) + + +class MultipleChoiceVqaEvaluation: + """Score a vision model against generated image-question-choice VQA cases.""" + + name = "point-cloud-vqa" + config_model: type[BaseModel] = VqaEvaluationConfig + + def __init__(self, vision_factory: Callable[[str], OpenAIVlModel] | None = None) -> None: + self._vision_factory = vision_factory or (lambda model: OpenAIVlModel(model_name=model)) + + def run(self, config: BaseModel, context: EvaluationContext) -> EvaluationReport: + if not isinstance(config, VqaEvaluationConfig): + raise TypeError("point-cloud-vqa received the wrong configuration type") + dataset = Path(config.dataset).expanduser() + if not dataset.is_absolute(): + dataset = context.spec_dir / dataset + dataset = dataset.resolve() + cases = _load_jsonl(dataset / "cases.jsonl") + labels = {item["id"]: item["answer"] for item in _load_jsonl(dataset / "labels.jsonl")} + model = self._vision_factory(config.model) + results = [_evaluate_case(dataset, model, case, labels) for case in cases] + artifact = context.workspace / "vqa-results.json" + artifact.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + passed = sum(item["passed"] for item in results) + total = len(results) + return EvaluationReport( + summary=( + SummaryItem(key="cases", label="Cases", value=total), + SummaryItem(key="passed", label="Passed", value=passed), + SummaryItem( + key="accuracy", label="Accuracy", value=passed / total if total else 0.0 + ), + ), + native_result=ArtifactNativeResult( + artifact=ArtifactReference( + path=artifact.relative_to(context.workspace).as_posix(), + label="VQA case results", + media_type="application/json", + ) + ), + artifacts=( + ArtifactReference( + path=artifact.relative_to(context.workspace).as_posix(), + label="VQA case results", + media_type="application/json", + ), + ), + ) + + +def _evaluate_case( + dataset: Path, model: OpenAIVlModel, case: dict[str, object], labels: dict[str, str] +) -> dict[str, object]: + case_id = _required_string(case, "id") + choices = _choices(case) + expected = labels.get(case_id) + if expected is None: + raise ValueError(f"missing private label for case {case_id}") + if expected not in choices: + raise ValueError(f"private label for {case_id} is not an allowed choice") + image = cv2.imread(str(dataset / _required_string(case, "image"))) + if image is None: + raise ValueError(f"unable to load public image for case {case_id}") + prompt = ( + f"{_required_string(case, 'question')}\n\n" + f"Choices: {', '.join(choices)}.\n" + "Use only the supplied image. End with exactly `ANSWER: `." + ) + response = model.query(Image.from_numpy(image), prompt) + answer = _parse_choice(response, choices) + return { + "id": case_id, + "expected": expected, + "answer": answer, + "passed": answer == expected, + "raw_response": response, + } + + +def _load_jsonl(path: Path) -> list[dict[str, object]]: + if not path.is_file(): + raise ValueError(f"missing VQA dataset file: {path}") + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] + + +def _required_string(item: dict[str, object], key: str) -> str: + value = item.get(key) + if not isinstance(value, str) or not value: + raise ValueError(f"VQA case requires non-empty {key}") + return value + + +def _choices(case: dict[str, object]) -> tuple[str, ...]: + value = case.get("choices") + if ( + not isinstance(value, list) + or len(value) < 2 + or not all(isinstance(item, str) for item in value) + ): + raise ValueError("VQA case requires at least two string choices") + return tuple(value) + + +def _parse_choice(response: str, choices: tuple[str, ...]) -> str | None: + match = re.search(r"^ANSWER:\s*(.+?)\s*$", response, re.MULTILINE) + if match is None: + return None + answer = match.group(1) + return answer if answer in choices else None + + +point_cloud_vqa = MultipleChoiceVqaEvaluation() diff --git a/dimos/benchmark/vqa/generation/adapters.py b/dimos/benchmark/vqa/generation/adapters.py new file mode 100644 index 0000000000..ebe4323599 --- /dev/null +++ b/dimos/benchmark/vqa/generation/adapters.py @@ -0,0 +1,51 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MoonDream and EdgeTAM adapters for the single-frame VQA pipeline.""" + +from __future__ import annotations + +from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter +from dimos.models.vl.moondream import MoondreamVlModel +from dimos.msgs.sensor_msgs.Image import Image +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D + + +class MoondreamObjectDetector: + """Adapt MoonDream's query detection API to the grounding interface.""" + + def __init__(self, model: MoondreamVlModel) -> None: + self._model = model + + def detect(self, image: Image, query: str) -> ImageDetections2D: + return self._model.query_detections(image, query) + + def locate(self, image: Image, query: str) -> ImageDetections2D: + points = self._model.query_points(image, f"center of the {query}") + for point in points: + point.name = query + return points + + +class EdgeTamObjectSegmenter: + """Adapt EdgeTAM single-image segmentation to the grounding interface.""" + + def __init__(self, segmenter: EdgeTAMImageSegmenter) -> None: + self._segmenter = segmenter + + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: + return self._segmenter.segment(detections) + + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: + return self._segmenter.segment_points(points) diff --git a/dimos/benchmark/vqa/generation/dataset.py b/dimos/benchmark/vqa/generation/dataset.py new file mode 100644 index 0000000000..987e1ced7d --- /dev/null +++ b/dimos/benchmark/vqa/generation/dataset.py @@ -0,0 +1,149 @@ +# Copyright 2026 Dimensional Inc. +"""Persist VQA generation evidence and a simple multiple-choice evaluation export.""" + +from __future__ import annotations + +from dataclasses import asdict +import json +from pathlib import Path +from typing import Any + +import cv2 + +from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + BooleanAnswerContract, + CalibratedFrame, + GroundTruthResult, + QuestionIntent, + QuestionProposal, + RejectedOracleResult, +) + + +def write_frame_record( + output: Path, + frame: CalibratedFrame, + recording: str, + frame_index: int, + intents: list[QuestionIntent | QuestionProposal], + results: list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult], + metadata: dict[str, Any], +) -> None: + """Write one frame's public cases alongside its private generation audit record.""" + output.mkdir(parents=True, exist_ok=False) + image_path = output / "image.jpg" + if not cv2.imwrite(str(image_path), frame.image.data): + raise RuntimeError(f"failed to write {image_path}") + accepted = [result for result in results if _is_accepted(result)] + cases, labels = _evaluation_rows(frame.id, accepted) + _write_json( + output / "frame.json", + { + "schema_version": "1.0", + "frame_id": frame.id, + "recording": recording, + "frame_index": frame_index, + "image": image_path.name, + "question_count": len(intents), + "accepted_question_count": len(accepted), + "rejected_question_count": len(results) - len(accepted), + **metadata, + }, + ) + _write_json(output / "ground_truth.json", [_private_result(item) for item in results]) + _write_json(output / "cases.json", cases) + _write_json(output / "labels.json", labels) + + +def write_dataset_manifest(output: Path) -> dict[str, int]: + """Build aggregate public cases and private labels from completed frame records.""" + frames = sorted(path for path in output.glob("frame-*") if (path / "frame.json").is_file()) + case_rows: list[dict[str, Any]] = [] + label_rows: list[dict[str, Any]] = [] + accepted = 0 + rejected = 0 + for path in frames: + frame = json.loads((path / "frame.json").read_text()) + case_rows.extend( + {**case, "image": f"{path.name}/{case['image']}"} + for case in json.loads((path / "cases.json").read_text()) + ) + label_rows.extend(json.loads((path / "labels.json").read_text())) + accepted += frame["accepted_question_count"] + rejected += frame["rejected_question_count"] + _write_jsonl(output / "cases.jsonl", case_rows) + _write_jsonl(output / "labels.jsonl", label_rows) + return { + "frame_count": len(frames), + "accepted_question_count": accepted, + "rejected_question_count": rejected, + } + + +def _is_accepted(result: GroundTruthResult | AcceptedOracleResult | RejectedOracleResult) -> bool: + return isinstance(result, AcceptedOracleResult) or ( + isinstance(result, GroundTruthResult) and result.status == "answered" + ) + + +def _evaluation_rows( + frame_id: str, results: list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult] +) -> tuple[list[dict[str, Any]], list[dict[str, str]]]: + cases: list[dict[str, Any]] = [] + labels: list[dict[str, str]] = [] + for result in results: + if isinstance(result, RejectedOracleResult): + continue + if isinstance(result, AcceptedOracleResult): + contract = result.answer_contract + choices = ( + ("yes", "no") if isinstance(contract, BooleanAnswerContract) else contract.choices + ) + case_id = f"{frame_id}-{result.proposal.id}" + question = result.proposal.question + answer = result.answer + else: + case_id = result.question.id + question = result.question.question + choices = result.question.allowed_answers + answer = result.answer + if answer is None or answer not in choices: + raise ValueError(f"accepted VQA case {case_id} must have a choice answer") + cases.append( + {"id": case_id, "image": "image.jpg", "question": question, "choices": choices} + ) + labels.append({"id": case_id, "answer": answer}) + return cases, labels + + +def _private_result( + result: GroundTruthResult | AcceptedOracleResult | RejectedOracleResult, +) -> dict[str, Any]: + if isinstance(result, AcceptedOracleResult): + return { + "status": "answered", + "answer": result.answer, + "proposal": asdict(result.proposal), + "answer_contract": asdict(result.answer_contract), + "evidence_ids": result.evidence_ids, + "tool_results": [asdict(item) for item in result.tool_results], + "trace": [asdict(item) for item in result.trace], + } + if isinstance(result, RejectedOracleResult): + return { + "status": "rejected", + "reason": result.reason, + "proposal": asdict(result.proposal), + "tool_results": [asdict(item) for item in result.tool_results], + "trace": [asdict(item) for item in result.trace], + } + return asdict(result) + + +def _write_json(path: Path, payload: Any) -> None: + path.write_text(json.dumps(payload, indent=2) + "\n") + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.write_text("".join(f"{json.dumps(row, sort_keys=True)}\n" for row in rows)) diff --git a/dimos/benchmark/vqa/generation/families.py b/dimos/benchmark/vqa/generation/families.py new file mode 100644 index 0000000000..5119f31a9c --- /dev/null +++ b/dimos/benchmark/vqa/generation/families.py @@ -0,0 +1,485 @@ +# Copyright 2026 Dimensional Inc. +"""Recipes and result construction for constrained single-frame VQA families.""" + +from __future__ import annotations + +from collections.abc import Callable + +import numpy as np + +from dimos.benchmark.vqa.generation.primitives.choices import ( + CAMERA_RANGE_CHOICES, + COUNT_CHOICES, + OPENING_WIDTH_CHOICES, + camera_range_choice, + count_choice, + opening_width_choice, +) +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + GroundedObject, + GroundTruthResult, + QuestionIntent, + ToolTrace, + VqaExample, +) + +Ground = Callable[[CalibratedFrame, str], tuple[list[GroundedObject], tuple[ToolTrace, ...]]] + + +def render_question(intent: QuestionIntent) -> str: + if intent.kind == "presence": + return f"Is there a {intent.object_query} in the image? Answer yes or no." + if intent.kind == "horizontal_direction": + return f"Where is the nearest {intent.object_query}: left, center, or right?" + if intent.kind == "visible_count": + return f"How many {intent.object_query}s are visible?" + if intent.kind == "camera_range": + return f"How far is the nearest {intent.object_query} from the camera?" + if intent.kind == "compare_nearest_by_side": + return f"Which {intent.object_query} is closer: the left one or the right one?" + if intent.kind == "compare_left_right": + return ( + f"Is the {intent.object_query} to the left or right of the {intent.comparison_query}?" + ) + if intent.kind == "compare_height": + return f"Which is taller: the {intent.object_query} or the {intent.comparison_query}?" + if intent.kind == "object_on_support": + return f"Is the {intent.object_query} on the {intent.comparison_query}? Answer yes or no." + if intent.kind == "opening_width": + return f"How wide is the {intent.object_query}?" + if intent.kind == "door_state": + return f"Is the {intent.object_query} open or closed?" + if intent.kind == "closest_object": + return f"Which object is closest to the {intent.object_query}: {', '.join(intent.candidate_queries)}?" + if intent.kind == "forward_path": + return "Is the path directly ahead clear or blocked?" + return f"Is the nearest {intent.object_query} within {intent.threshold_m or 3:g} meters? Answer yes or no." + + +def rejected_result( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + reason: str, +) -> GroundTruthResult: + rejected = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.kind}", render_question(intent), "", "", () + ) + return GroundTruthResult(intent, rejected, "rejected", None, reason, tuple(objects), trace) + + +def count_visible_objects( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + answer = count_choice(len(objects)) + example = VqaExample( + f"{frame.id}-{intent.object_query}-visible-count", + render_question(intent), + answer, + "choice", + tuple(item.id for item in objects), + COUNT_CHOICES, + ) + return GroundTruthResult(intent, example, "answered", answer, None, tuple(objects), trace) + + +def bucket_camera_range( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + selected = select_nearest_object(objects) + if selected is None: + return rejected_result(frame, intent, objects, trace, "no_grounded_object") + answer = camera_range_choice(selected.range_m) + example = VqaExample( + f"{frame.id}-{intent.object_query}-camera-range", + render_question(intent), + answer, + "choice", + (selected.id,), + CAMERA_RANGE_CHOICES, + ) + return GroundTruthResult(intent, example, "answered", answer, None, (selected,), trace) + + +def compare_heights( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, + ground: Ground, +) -> GroundTruthResult: + if len(objects) != 1 or intent.comparison_query is None: + return rejected_result(frame, intent, objects, trace, "ambiguous_first_height_object") + other_objects, other_trace = ground(frame, intent.comparison_query) + trace = (*trace, *other_trace) + if len(other_objects) != 1: + return rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_second_height_object" + ) + plane_fit = primitives.fit_ground_plane() + trace = (*trace, ToolTrace("fit_ground_plane", plane_fit.rejection_reason or "accepted")) + if plane_fit.estimate is None: + return rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + plane_fit.rejection_reason or "ground_plane_rejected", + ) + first = primitives.measure_height(objects[0], plane_fit.estimate) + second = primitives.measure_height(other_objects[0], plane_fit.estimate) + trace = ( + *trace, + ToolTrace("measure_height", first.rejection_reason or objects[0].id), + ToolTrace("measure_height", second.rejection_reason or other_objects[0].id), + ) + if first.measurement is None or second.measurement is None: + return rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + first.rejection_reason or second.rejection_reason or "height_measurement_rejected", + ) + first_lower = first.measurement.value - first.measurement.tolerance + second_lower = second.measurement.value - second.measurement.tolerance + first_upper = first.measurement.value + first.measurement.tolerance + second_upper = second.measurement.value + second.measurement.tolerance + if first_lower <= second_upper and second_lower <= first_upper: + return rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_height_comparison" + ) + answer = intent.object_query if first_lower > second_upper else intent.comparison_query + example = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.comparison_query}-height-comparison", + render_question(intent), + answer, + "choice", + (objects[0].id, other_objects[0].id), + (intent.object_query, intent.comparison_query), + ) + return GroundTruthResult( + intent, example, "answered", answer, None, (objects[0], other_objects[0]), trace + ) + + +def compare_left_right( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, + ground: Ground, +) -> GroundTruthResult: + if len(objects) != 1 or intent.comparison_query is None: + return rejected_result(frame, intent, objects, trace, "ambiguous_first_relation_object") + other_objects, other_trace = ground(frame, intent.comparison_query) + trace = (*trace, *other_trace) + if len(other_objects) != 1: + return rejected_result( + frame, intent, [*objects, *other_objects], trace, "ambiguous_second_relation_object" + ) + relation = primitives.classify_horizontal_relation(objects[0], other_objects[0]) + trace = ( + *trace, + ToolTrace( + "classify_horizontal_relation", + relation.relation or relation.rejection_reason or "rejected", + ), + ) + if relation.relation is None: + return rejected_result( + frame, + intent, + [*objects, *other_objects], + trace, + relation.rejection_reason or "horizontal_relation_rejected", + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.comparison_query}-left-right", + render_question(intent), + relation.relation, + "choice", + (objects[0].id, other_objects[0].id), + ("left", "right"), + ) + return GroundTruthResult( + intent, example, "answered", relation.relation, None, (objects[0], other_objects[0]), trace + ) + + +def classify_object_on_support( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, + ground: Ground, +) -> GroundTruthResult: + if len(objects) != 1 or intent.comparison_query is None: + return rejected_result(frame, intent, objects, trace, "ambiguous_supported_object") + supports, support_trace = ground(frame, intent.comparison_query) + trace = (*trace, *support_trace) + if len(supports) != 1: + return rejected_result( + frame, intent, [*objects, *supports], trace, "ambiguous_support_object" + ) + ground_fit = primitives.fit_ground_plane() + support_fit = primitives.fit_object_surface_plane(supports[0]) + trace = ( + *trace, + ToolTrace("fit_ground_plane", ground_fit.rejection_reason or "accepted"), + ToolTrace("fit_object_surface_plane", support_fit.rejection_reason or "accepted"), + ) + if ground_fit.estimate is None or support_fit.estimate is None: + return rejected_result( + frame, + intent, + [*objects, *supports], + trace, + ground_fit.rejection_reason + or support_fit.rejection_reason + or "support_relation_rejected", + ) + if abs(float(np.dot(ground_fit.estimate.normal, support_fit.estimate.normal))) < np.cos( + np.radians(12.0) + ): + return rejected_result( + frame, intent, [*objects, *supports], trace, "support_not_horizontal" + ) + relation = primitives.measure_object_plane_relation( + objects[0], supports[0], support_fit.estimate, ground_fit.estimate.normal + ) + trace = ( + *trace, + ToolTrace("measure_object_plane_relation", relation.rejection_reason or "accepted"), + ) + if relation.rejection_reason is not None: + return rejected_result( + frame, intent, [*objects, *supports], trace, relation.rejection_reason + ) + if relation.planar_separation_m is not None and relation.planar_separation_m > 0.2: + answer = "no" + elif relation.lower_clearance_m is not None and relation.lower_clearance_m > 0.2: + answer = "no" + elif ( + relation.lower_clearance_m is not None + and relation.upper_clearance_m is not None + and relation.elevated_fraction is not None + and relation.contact_point_count >= 4 + and relation.lower_clearance_m <= 0.08 + and relation.upper_clearance_m >= 0.15 + and relation.elevated_fraction >= 0.7 + and relation.contact_overlap_count >= 3 + ): + answer = "yes" + else: + return rejected_result( + frame, intent, [*objects, *supports], trace, "insufficient_contact_evidence" + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-{intent.comparison_query}-on-support", + render_question(intent), + answer, + "boolean", + (objects[0].id, supports[0].id), + ("yes", "no"), + ) + return GroundTruthResult( + intent, example, "answered", answer, None, (objects[0], supports[0]), trace + ) + + +def measure_opening_width( + frame: CalibratedFrame, intent: QuestionIntent, primitives: FramePerceptionPrimitives +) -> GroundTruthResult: + primitives.detect_objects(intent.object_query) + masks = primitives.segment_detections(intent.object_query) + trace = ( + ToolTrace("detect_objects", intent.object_query), + ToolTrace("segment_detections", f"count={len(masks)}"), + ) + if len(masks) != 1: + return rejected_result(frame, intent, [], trace, "ambiguous_opening_instances") + ground_fit = primitives.fit_ground_plane() + trace = (*trace, ToolTrace("fit_ground_plane", ground_fit.rejection_reason or "accepted")) + if ground_fit.estimate is None: + return rejected_result( + frame, intent, [], trace, ground_fit.rejection_reason or "ground_plane_rejected" + ) + result = primitives.measure_opening_width_from_mask(masks[0], ground_fit.estimate) + trace = (*trace, ToolTrace("measure_opening_width", result.rejection_reason or "accepted")) + if result.measurement is None: + return rejected_result( + frame, intent, [], trace, result.rejection_reason or "opening_width_rejected" + ) + answer = opening_width_choice(result.measurement.value) + example = VqaExample( + f"{frame.id}-{intent.object_query}-opening-width", + render_question(intent), + answer, + "choice", + (), + OPENING_WIDTH_CHOICES, + ) + return GroundTruthResult(intent, example, "answered", answer, None, (), trace) + + +def compare_nearest_by_side( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], +) -> GroundTruthResult: + left = select_nearest_object(objects, "left") + right = select_nearest_object(objects, "right") + if left is None or right is None: + return rejected_result(frame, intent, objects, trace, "missing_grounded_side") + if left.range_m == right.range_m: + return rejected_result(frame, intent, objects, trace, "ambiguous_nearest_by_side") + answer = "left" if left.range_m < right.range_m else "right" + example = VqaExample( + f"{frame.id}-{intent.object_query}-nearest-by-side", + render_question(intent), + answer, + "choice", + (left.id, right.id), + ("left", "right"), + ) + return GroundTruthResult(intent, example, "answered", answer, None, (left, right), trace) + + +def classify_door_state( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, +) -> GroundTruthResult: + if "door" not in intent.object_query.lower(): + return rejected_result(frame, intent, objects, trace, "door_state_requires_door_query") + if len(objects) != 1: + return rejected_result(frame, intent, objects, trace, "ambiguous_door_instances") + door_fit = primitives.fit_object_surface_plane(objects[0]) + surrounding_fit = primitives.fit_object_surrounding_plane(objects[0]) + trace = ( + *trace, + ToolTrace("fit_object_surface_plane", door_fit.rejection_reason or "accepted"), + ToolTrace("fit_mask_surrounding_plane", surrounding_fit.rejection_reason or "accepted"), + ) + if door_fit.estimate is None or surrounding_fit.estimate is None: + return rejected_result( + frame, + intent, + objects, + trace, + door_fit.rejection_reason or surrounding_fit.rejection_reason or "door_state_rejected", + ) + angle = primitives.measure_relative_plane_angle(door_fit.estimate, surrounding_fit.estimate) + trace = (*trace, ToolTrace("measure_relative_plane_angle", f"{angle.value:.1f} deg")) + if angle.value <= 12.0: + state = "closed" + elif angle.value >= 25.0: + state = "open" + else: + return rejected_result(frame, intent, objects, trace, "ambiguous_door_angle") + example = VqaExample( + f"{frame.id}-{intent.object_query}-state", + render_question(intent), + state, + "choice", + (objects[0].id,), + ("open", "closed"), + ) + return GroundTruthResult(intent, example, "answered", state, None, tuple(objects), trace) + + +def select_closest_object( + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + primitives: FramePerceptionPrimitives, + ground: Ground, +) -> GroundTruthResult: + if len(objects) != 1: + return rejected_result(frame, intent, objects, trace, "ambiguous_target_object") + candidates: list[GroundedObject] = [] + for query in intent.candidate_queries: + matches, candidate_trace = ground(frame, query) + trace = (*trace, *candidate_trace) + if len(matches) != 1: + return rejected_result( + frame, + intent, + [*objects, *candidates, *matches], + trace, + "ambiguous_candidate_object", + ) + candidates.append(matches[0]) + selected = primitives.select_closest_object(objects[0], candidates) + trace = ( + *trace, + ToolTrace( + "select_closest_object", + selected.object.id if selected.object else selected.rejection_reason or "rejected", + ), + ) + if selected.object is None: + return rejected_result( + frame, + intent, + [*objects, *candidates], + trace, + selected.rejection_reason or "closest_object_rejected", + ) + example = VqaExample( + f"{frame.id}-{intent.object_query}-closest-object", + render_question(intent), + selected.object.label, + "choice", + (objects[0].id, *(item.id for item in candidates)), + intent.candidate_queries, + ) + return GroundTruthResult( + intent, + example, + "answered", + selected.object.label, + None, + tuple([*objects, *candidates]), + trace, + ) + + +def classify_forward_path( + frame: CalibratedFrame, intent: QuestionIntent, primitives: FramePerceptionPrimitives +) -> GroundTruthResult: + result = primitives.classify_forward_path() + trace = ( + ToolTrace("classify_forward_path", result.state or result.rejection_reason or "rejected"), + ) + if result.state is None: + return rejected_result( + frame, intent, [], trace, result.rejection_reason or "forward_path_rejected" + ) + example = VqaExample( + f"{frame.id}-forward-path", + render_question(intent), + result.state, + "choice", + (), + ("clear", "blocked"), + ) + return GroundTruthResult(intent, example, "answered", result.state, None, (), trace) diff --git a/dimos/benchmark/vqa/generation/geometry.py b/dimos/benchmark/vqa/generation/geometry.py new file mode 100644 index 0000000000..db094027b6 --- /dev/null +++ b/dimos/benchmark/vqa/generation/geometry.py @@ -0,0 +1,85 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Static calibrated point-cloud projection for single-frame VQA.""" + +from __future__ import annotations + +import numpy as np + +from dimos.benchmark.vqa.models import CalibratedFrame, ProjectedPoints, ProjectionConfig + + +def project_visible_points( + frame: CalibratedFrame, config: ProjectionConfig = ProjectionConfig() +) -> ProjectedPoints: + """Project the nearest point at each image pixel using static calibration. + + The supplied transform maps point-cloud coordinates to the camera optical + frame. This function intentionally has no time or TF dependency. + """ + if not frame.image_is_rectified: + raise ValueError("VQA projection requires a rectified pinhole image") + if config.min_depth_m <= 0: + raise ValueError("min_depth_m must be positive") + + camera_info = frame.camera_info + if camera_info.width != frame.image.width or camera_info.height != frame.image.height: + raise ValueError("camera intrinsics dimensions must match the image") + if len(camera_info.K) != 9: + raise ValueError("camera intrinsics must contain a 3x3 matrix") + + points, _ = frame.pointcloud.as_numpy() + if len(points) == 0: + return ProjectedPoints([], [], []) + + homogeneous = np.column_stack((points, np.ones(len(points), dtype=points.dtype))) + camera_points = (frame.pointcloud_to_camera.to_matrix() @ homogeneous.T).T[:, :3] + source_indices = np.arange(len(points)) + + depth_mask = camera_points[:, 2] >= config.min_depth_m + camera_points = camera_points[depth_mask] + source_indices = source_indices[depth_mask] + if len(camera_points) == 0: + return ProjectedPoints([], [], []) + + fx, fy = camera_info.K[0], camera_info.K[4] + cx, cy = camera_info.K[2], camera_info.K[5] + if fx <= 0 or fy <= 0: + raise ValueError("camera focal lengths must be positive") + + u = np.floor(fx * camera_points[:, 0] / camera_points[:, 2] + cx).astype(np.int64) + v = np.floor(fy * camera_points[:, 1] / camera_points[:, 2] + cy).astype(np.int64) + in_image = (u >= 0) & (u < frame.image.width) & (v >= 0) & (v < frame.image.height) + camera_points = camera_points[in_image] + source_indices = source_indices[in_image] + u = u[in_image] + v = v[in_image] + if len(camera_points) == 0: + return ProjectedPoints([], [], []) + + pixel_ids = v * frame.image.width + u + nearest_first = np.lexsort((camera_points[:, 2], pixel_ids)) + first_per_pixel = np.concatenate( + ([True], pixel_ids[nearest_first][1:] != pixel_ids[nearest_first][:-1]) + ) + visible = nearest_first[first_per_pixel] + + return ProjectedPoints( + camera_points=[ + (float(point[0]), float(point[1]), float(point[2])) for point in camera_points[visible] + ], + pixels=[(int(x), int(y)) for x, y in zip(u[visible], v[visible], strict=True)], + source_indices=[int(index) for index in source_indices[visible]], + ) diff --git a/dimos/benchmark/vqa/generation/ground_truth_generator.py b/dimos/benchmark/vqa/generation/ground_truth_generator.py new file mode 100644 index 0000000000..2dd4eb71d5 --- /dev/null +++ b/dimos/benchmark/vqa/generation/ground_truth_generator.py @@ -0,0 +1,98 @@ +# Copyright 2026 Dimensional Inc. +"""Tool-driven private answer generation for single-frame VQA.""" + +from __future__ import annotations + +from dimos.benchmark.vqa.generation import families +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.questions import generate_questions +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + GroundedObject, + GroundTruthResult, + QuestionIntent, + ToolTrace, +) + + +class VqaGroundTruthGenerator: + """Answer constrained questions by calling detection, segmentation, and geometry tools.""" + + def __init__(self, primitives: FramePerceptionPrimitives) -> None: + self.primitives = primitives + + def answer(self, frame: CalibratedFrame, intent: QuestionIntent) -> GroundTruthResult: + if intent.kind == "forward_path": + return families.classify_forward_path(frame, intent, self.primitives) + if intent.kind == "opening_width": + return families.measure_opening_width(frame, intent, self.primitives) + objects, trace = self.ground(frame, intent.object_query) + return self._answer_from_objects(frame, intent, objects, trace) + + def ground( + self, frame: CalibratedFrame, object_query: str + ) -> tuple[list[GroundedObject], tuple[ToolTrace, ...]]: + """Run the fixed constrained grounding recipe over shared primitives.""" + if self.primitives.has_grounding(object_query): + return self.primitives.ground_masks(object_query), ( + ToolTrace("reuse_grounding", object_query), + ) + trace: list[ToolTrace] = [ToolTrace("detect_objects", object_query)] + detections = self.primitives.detect_objects(object_query) + if len(detections): + trace.append(ToolTrace("segment_objects", f"count={len(detections)}")) + elif self.primitives.can_localize_points: + trace.append(ToolTrace("locate_object_point", object_query)) + masks = self.primitives.segment_detections(object_query) + if not len(detections) and self.primitives.used_point_localization(object_query): + trace.append(ToolTrace("segment_object_point", object_query)) + trace.append(ToolTrace("get_foreground_geometry", f"masks={len(masks)}")) + return self.primitives.ground_masks(object_query), tuple(trace) + + def _answer_from_objects( + self, + frame: CalibratedFrame, + intent: QuestionIntent, + objects: list[GroundedObject], + trace: tuple[ToolTrace, ...], + ) -> GroundTruthResult: + if not objects: + return families.rejected_result(frame, intent, objects, trace, "no_grounded_object") + if intent.kind == "compare_nearest_by_side": + return families.compare_nearest_by_side(frame, intent, objects, trace) + if intent.kind == "visible_count": + return families.count_visible_objects(frame, intent, objects, trace) + if intent.kind == "camera_range": + return families.bucket_camera_range(frame, intent, objects, trace) + if intent.kind == "compare_left_right": + return families.compare_left_right( + frame, intent, objects, trace, self.primitives, self.ground + ) + if intent.kind == "compare_height": + return families.compare_heights( + frame, intent, objects, trace, self.primitives, self.ground + ) + if intent.kind == "object_on_support": + return families.classify_object_on_support( + frame, intent, objects, trace, self.primitives, self.ground + ) + if intent.kind == "door_state": + return families.classify_door_state(frame, intent, objects, trace, self.primitives) + if intent.kind == "closest_object": + return families.select_closest_object( + frame, intent, objects, trace, self.primitives, self.ground + ) + examples = generate_questions( + frame.id, objects, [intent.object_query], distance_m=intent.threshold_m or 3.0 + ) + suffix = { + "presence": "presence", + "horizontal_direction": "direction", + "within_distance": "range", + }[intent.kind] + example = next((item for item in examples if item.id.endswith(f"-{suffix}")), None) + if example is not None: + return GroundTruthResult( + intent, example, "answered", example.expected_answer, None, tuple(objects), trace + ) + return families.rejected_result(frame, intent, objects, trace, "no_grounded_object") diff --git a/dimos/benchmark/vqa/generation/grounding.py b/dimos/benchmark/vqa/generation/grounding.py new file mode 100644 index 0000000000..b6b6aa6c61 --- /dev/null +++ b/dimos/benchmark/vqa/generation/grounding.py @@ -0,0 +1,72 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Foreground-mask point-cloud grounding for one calibrated frame.""" + +from __future__ import annotations + +import numpy as np + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.models import CalibratedFrame, GroundedObject, ProjectionConfig +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +def ground_segmented_objects( + frame: CalibratedFrame, + detections: list[Detection2DSeg], + *, + min_foreground_points: int = 3, + projection: ProjectionConfig = ProjectionConfig(), +) -> list[GroundedObject]: + """Create object geometry from visible points covered by each foreground mask.""" + if min_foreground_points < 1: + raise ValueError("min_foreground_points must be positive") + + projected = project_visible_points(frame, projection) + grounded: list[GroundedObject] = [] + for index, detection in enumerate(detections): + mask = detection.mask + if mask.shape != (frame.image.height, frame.image.width): + raise ValueError("segmentation mask dimensions must match the image") + selected = [ + point + for point, (x, y) in zip(projected.camera_points, projected.pixels, strict=True) + if mask[y, x] > 0 + ] + if len(selected) < min_foreground_points: + continue + + ranges = np.linalg.norm(np.asarray(selected), axis=1) + image_x = [x for x, y in projected.pixels if mask[y, x] > 0] + median_x = float(np.median(image_x)) + direction = _horizontal_direction(median_x, frame.image.width) + grounded.append( + GroundedObject( + id=f"{frame.id}-{detection.name}-{index}", + label=detection.name, + point_count=len(selected), + range_m=float(np.median(ranges)), + horizontal_direction=direction, + ) + ) + return grounded + + +def _horizontal_direction(x: float, width: int) -> str: + if x < width / 3: + return "left" + if x >= 2 * width / 3: + return "right" + return "center" diff --git a/dimos/benchmark/vqa/generation/oracle.py b/dimos/benchmark/vqa/generation/oracle.py new file mode 100644 index 0000000000..e8b8e51b6d --- /dev/null +++ b/dimos/benchmark/vqa/generation/oracle.py @@ -0,0 +1,346 @@ +# Copyright 2026 Dimensional Inc. +"""Private bounded LangChain oracle for generic VQA question proposals.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, replace +import json +import re +from typing import TYPE_CHECKING, Any, Protocol + +from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.generation.primitives.choices import height_choice_window +from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + AnswerContract, + BooleanAnswerContract, + ChoiceAnswerContract, + DeferredHeightChoiceContract, + OracleToolResult, + OracleTrace, + QuestionProposal, + RejectedOracleResult, + ResolvedAnswerContract, +) + +if TYPE_CHECKING: + from langchain_core.language_models.chat_models import BaseChatModel + + +@dataclass(frozen=True) +class SemanticEvidenceValidation: + """Private verdict on whether cited tool evidence supports an oracle answer.""" + + accepted: bool + reason: str + + +class SemanticEvidenceValidator(Protocol): + """Validate an answer only against the frozen question and cited local evidence.""" + + def validate( + self, + proposal: QuestionProposal, + answer: str, + cited_results: tuple[OracleToolResult, ...], + ) -> SemanticEvidenceValidation: ... + + +class OpenAISemanticEvidenceValidator: + """Private no-tools model judge for semantic grounding of a proposed answer.""" + + def __init__(self, model: BaseChatModel) -> None: + self._model = model + + def validate( + self, + proposal: QuestionProposal, + answer: str, + cited_results: tuple[OracleToolResult, ...], + ) -> SemanticEvidenceValidation: + from langchain_core.messages import HumanMessage, SystemMessage + + response = self._model.invoke( + [ + SystemMessage( + "You validate a private VQA oracle answer. Decide only whether the cited " + "structured local-tool evidence supports the answer to the frozen question. " + "Reject claims requiring measurements not present in the evidence; for example, " + "height is not supported by range or side alone. A cited measurement bucket must " + "exactly match the selected choice. Return strict JSON only: " + '{"accepted": true|false, "reason": "concise reason"}. Do not call tools.' + ), + HumanMessage( + json.dumps( + { + "question": proposal.question, + "answer": answer, + "answer_contract": asdict(proposal.answer_contract), + "cited_evidence": [asdict(result) for result in cited_results], + } + ) + ), + ] + ) + try: + payload = _parse_strict_json_object(_response_text(response.content)) + accepted, reason = payload.get("accepted"), payload.get("reason") + if not isinstance(accepted, bool) or not isinstance(reason, str) or not reason: + raise ValueError( + "validator response requires boolean accepted and non-empty reason" + ) + except (ValueError, json.JSONDecodeError, AttributeError) as exc: + return SemanticEvidenceValidation(False, f"invalid_validator_response:{exc}") + return SemanticEvidenceValidation(accepted, reason) + + +class PrivateToolCallingOracle: + """Use direct local tools only, then validate a model's final JSON response.""" + + def __init__( + self, + model: BaseChatModel, + max_tool_calls: int = 25, + semantic_validator: SemanticEvidenceValidator | None = None, + ) -> None: + if max_tool_calls < 1: + raise ValueError("max_tool_calls must be positive") + self._model = model + self._max_tool_calls = max_tool_calls + self._semantic_validator = semantic_validator + + def answer( + self, proposal: QuestionProposal, registry: LocalOracleToolRegistry + ) -> AcceptedOracleResult | RejectedOracleResult: + from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage + + tools = registry.tools() + model = self._model.bind_tools(tools) + messages: list[Any] = [ + SystemMessage( + "You are a private VQA oracle. Use only supplied local tools. Do not invent " + "evidence. Detection IDs can only be passed to segment_detection. Mask IDs can only " + "be passed to ground_mask. Only grounded object IDs can be passed to object and " + "geometry measurements. If grounding fails, reject rather than trying unrelated " + "tools. Finish with JSON only: either " + '{"answer": value, "evidence_ids": [..]} or ' + '{"status": "rejected", "reason": "non-empty reason", "evidence_ids": [..]}. ' + "Rejected results must not include an answer." + ), + HumanMessage(_proposal_prompt(proposal)), + ] + trace: list[OracleTrace] = [] + calls = 0 + while calls < self._max_tool_calls: + response = model.invoke(messages) + messages.append(response) + tool_calls = getattr(response, "tool_calls", []) + if not tool_calls: + return _validated_result( + proposal, + _response_text(response.content), + registry.results, + trace, + self._semantic_validator, + ) + for call in tool_calls: + if calls >= self._max_tool_calls: + break + name = call.get("name") + tool = next((item for item in tools if item.name == name), None) + if tool is None: + return _rejected(proposal, "unsupported_tool", registry.results, trace) + try: + output = tool.invoke(call.get("args", {})) + except (TypeError, ValueError) as exc: + return _rejected(proposal, f"tool_error:{exc}", registry.results, trace) + calls += 1 + trace.append(OracleTrace("tool", str(name))) + messages.append(ToolMessage(content=str(output), tool_call_id=call["id"])) + return _rejected(proposal, "tool_call_limit", registry.results, trace) + + +def create_openai_oracle(model: str, max_tool_calls: int = 25) -> PrivateToolCallingOracle: + """Construct the private-only OpenAI tool-calling oracle.""" + from langchain_openai import ChatOpenAI + + return PrivateToolCallingOracle( + ChatOpenAI(model=model), + max_tool_calls=max_tool_calls, + semantic_validator=OpenAISemanticEvidenceValidator(ChatOpenAI(model=model)), + ) + + +def validate_oracle_answer( + proposal: QuestionProposal, + answer: Any, + evidence_ids: Any, + results: tuple[OracleToolResult, ...], +) -> str: + """Deterministically validate an answer format and citations.""" + return _resolve_oracle_answer(proposal, answer, evidence_ids, results)[0] + + +def _resolve_oracle_answer( + proposal: QuestionProposal, + answer: Any, + evidence_ids: Any, + results: tuple[OracleToolResult, ...], +) -> tuple[str, ResolvedAnswerContract]: + """Return a validated answer and the public contract resolved from private evidence.""" + if ( + not isinstance(evidence_ids, list) + or not evidence_ids + or not all(isinstance(item, str) for item in evidence_ids) + ): + raise ValueError("answer requires non-empty evidence_ids") + known_ids = {item.id for result in results for item in result.evidence} + if not set(evidence_ids).issubset(known_ids): + raise ValueError("answer cites unknown evidence") + contract = proposal.answer_contract + if isinstance(contract, BooleanAnswerContract): + if answer not in ("yes", "no"): + raise ValueError("boolean answer must be yes or no") + return str(answer), contract + if isinstance(contract, ChoiceAnswerContract): + if not isinstance(answer, str) or answer not in contract.choices: + raise ValueError("choice answer is not allowed") + cited = set(evidence_ids) + measured_choices = { + result.choice + for result in results + if result.choice is not None and any(item.id in cited for item in result.evidence) + } + if measured_choices and answer not in measured_choices: + raise ValueError("choice answer does not match cited measurement bucket") + return answer, contract + if isinstance(contract, DeferredHeightChoiceContract): + height_results = [ + result + for result in results + if ( + result.tool == "measure_height" + and result.measurement is not None + and any(item.id in evidence_ids for item in result.evidence) + ) + ] + if len(height_results) != 1: + raise ValueError("deferred height answer requires exactly one cited height measurement") + choices, choice = height_choice_window(height_results[0].measurement.value) + return choice, ChoiceAnswerContract(choices) + raise ValueError("unsupported answer contract") + + +def _validated_result( + proposal: QuestionProposal, + response: str, + results: tuple[OracleToolResult, ...], + trace: list[OracleTrace], + semantic_validator: SemanticEvidenceValidator | None, +) -> AcceptedOracleResult | RejectedOracleResult: + try: + payload = _parse_json_object(response) + if payload.get("status") == "rejected": + if set(payload) - {"status", "reason", "evidence_ids"}: + raise ValueError("rejected final response contains unsupported fields") + reason = payload.get("reason") + evidence_ids = payload.get("evidence_ids") + if not isinstance(reason, str) or not reason: + raise ValueError("rejected final response requires non-empty reason") + if evidence_ids is not None and ( + not isinstance(evidence_ids, list) + or not all(isinstance(item, str) for item in evidence_ids) + ): + raise ValueError("rejected evidence_ids must be a list of strings") + return _rejected(proposal, reason, results, trace) + answer, answer_contract = _resolve_oracle_answer( + proposal, payload.get("answer"), payload.get("evidence_ids"), results + ) + evidence_ids = tuple(payload["evidence_ids"]) + except (ValueError, json.JSONDecodeError, AttributeError) as exc: + return _rejected(proposal, f"invalid_final_answer:{exc}", results, trace) + cited_results = _cited_results(evidence_ids, results) + if semantic_validator is None: + return _rejected(proposal, "semantic_validator_not_configured", results, trace) + resolved_proposal = replace(proposal, answer_contract=answer_contract) + verdict = semantic_validator.validate(resolved_proposal, answer, cited_results) + trace.append( + OracleTrace( + "semantic_validation", + f"{'accepted' if verdict.accepted else 'rejected'}:{verdict.reason}", + ) + ) + if not verdict.accepted: + return _rejected(proposal, f"unsupported_evidence:{verdict.reason}", results, trace) + return AcceptedOracleResult( + proposal, answer, answer_contract, evidence_ids, results, tuple(trace) + ) + + +def _cited_results( + evidence_ids: tuple[str, ...], results: tuple[OracleToolResult, ...] +) -> tuple[OracleToolResult, ...]: + cited = set(evidence_ids) + return tuple( + replace( + result, + evidence=tuple(evidence for evidence in result.evidence if evidence.id in cited), + ) + for result in results + if any(evidence.id in cited for evidence in result.evidence) + ) + + +def _rejected( + proposal: QuestionProposal, + reason: str, + results: tuple[OracleToolResult, ...], + trace: list[OracleTrace], +) -> RejectedOracleResult: + return RejectedOracleResult(proposal, reason, results, tuple(trace)) + + +def _proposal_prompt(proposal: QuestionProposal) -> str: + return ( + f"Question: {proposal.question}\nAnswer contract: {_contract_prompt(proposal.answer_contract)}\n" + f"Suggested object queries: {', '.join(proposal.object_queries) or 'none'}" + ) + + +def _contract_prompt(contract: AnswerContract) -> str: + if isinstance(contract, BooleanAnswerContract): + return "boolean: yes or no" + if isinstance(contract, ChoiceAnswerContract): + return f"choice: {', '.join(contract.choices)}" + if isinstance(contract, DeferredHeightChoiceContract): + return ( + "deferred height choice: call measure_height, cite exactly one accepted height evidence ID, " + "and return answer: null. The private pipeline derives the public choice from the measurement" + ) + raise ValueError("unsupported answer contract") + + +def _parse_json_object(response: str) -> dict[str, Any]: + stripped = re.sub(r"^```(?:json)?\s*|\s*```$", "", response.strip(), flags=re.IGNORECASE) + start, end = stripped.find("{"), stripped.rfind("}") + if start < 0 or end < start: + raise json.JSONDecodeError("expected JSON object", stripped, 0) + payload: Any = json.loads(stripped[start : end + 1]) + if not isinstance(payload, dict): + raise ValueError("final response must be an object") + return payload + + +def _parse_strict_json_object(response: str) -> dict[str, Any]: + payload: Any = json.loads(response) + if not isinstance(payload, dict): + raise ValueError("validator response must be an object") + return payload + + +def _response_text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return "".join(str(item.get("text", "")) for item in content if isinstance(item, dict)) + return str(content) diff --git a/dimos/benchmark/vqa/generation/oracle_tools.py b/dimos/benchmark/vqa/generation/oracle_tools.py new file mode 100644 index 0000000000..0dd7b32f99 --- /dev/null +++ b/dimos/benchmark/vqa/generation/oracle_tools.py @@ -0,0 +1,839 @@ +# Copyright 2026 Dimensional Inc. +"""Typed private perception primitives over one frozen VQA frame.""" + +from __future__ import annotations + +import json +from typing import Any + +from langchain_core.tools import StructuredTool + +from dimos.benchmark.vqa.generation.primitives.choices import ( + CAMERA_RANGE_CHOICES, + COUNT_CHOICES, + camera_range_choice, + count_choice, +) +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.geometry import ForwardCorridorMeasurement +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object +from dimos.benchmark.vqa.models import ( + GroundedObject, + GroundPlaneEstimate, + OracleEvidence, + OracleMeasurement, + OracleToolResult, +) +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +class LocalOracleToolRegistry: + """Expose the same private perception primitives used by constrained recipes.""" + + def __init__(self, primitives: FramePerceptionPrimitives) -> None: + self._primitives = primitives + self._results: list[OracleToolResult] = [] + self._detections: dict[str, tuple[str, int]] = {} + self._masks: dict[str, tuple[str, Detection2DSeg]] = {} + self._objects: dict[str, GroundedObject] = {} + self._planes: dict[str, GroundPlaneEstimate] = {} + self._ground_planes: set[str] = set() + self._next_id = 0 + + @property + def results(self) -> tuple[OracleToolResult, ...]: + return tuple(self._results) + + def tools(self) -> list[StructuredTool]: + return [ + StructuredTool.from_function( + self.detect_objects, + name="detect_objects", + description="Run private MoonDream detection for one visible semantic query.", + ), + StructuredTool.from_function( + self.segment_detection, + name="segment_detection", + description="Run private EdgeTAM segmentation for one opaque detection ID.", + ), + StructuredTool.from_function( + self.ground_mask, + name="ground_mask", + description=( + "Project visible calibrated point-cloud support through one opaque mask ID." + ), + ), + StructuredTool.from_function( + self.fit_ground_plane, + name="fit_ground_plane", + description="Fit a quality-gated Open3D ground plane to the frozen visible point cloud.", + ), + StructuredTool.from_function( + self.get_object_pose, + name="get_object_pose", + description="Return robust private camera-frame position evidence for one grounded object.", + ), + StructuredTool.from_function( + self.fit_object_surface_plane, + name="fit_object_surface_plane", + description="Fit a private surface plane to one grounded object's visible support.", + ), + StructuredTool.from_function( + self.fit_mask_surrounding_plane, + name="fit_mask_surrounding_plane", + description="Fit a private structural plane from the visible ring around one mask.", + ), + StructuredTool.from_function( + self.measure_object_pair_distance, + name="measure_object_pair_distance", + description="Measure private 3D support-centroid distance between two grounded objects.", + ), + StructuredTool.from_function( + self.measure_relative_plane_angle, + name="measure_relative_plane_angle", + description="Measure the unsigned angle between two accepted opaque plane IDs.", + ), + StructuredTool.from_function( + self.measure_object_plane_relation, + name="measure_object_plane_relation", + description=( + "Measure private clearance, contact support, and projected separation of one object " + "relative to a support object plane." + ), + ), + StructuredTool.from_function( + self.measure_aperture_geometry, + name="measure_aperture_geometry", + description=( + "Measure a selected mask's ground-connected aperture geometry against an accepted " + "ground plane." + ), + ), + StructuredTool.from_function( + self.measure_forward_corridor, + name="measure_forward_corridor", + description=( + "Measure private ground and elevated-obstacle support in the camera-forward corridor " + "against an accepted ground plane." + ), + ), + StructuredTool.from_function( + self.measure_height, + name="measure_height", + description=( + "Measure one opaque grounded object above one opaque accepted ground-plane ID." + ), + ), + ] + + def detect_objects(self, query: str) -> str: + """Detect objects and return an opaque ID for a later segmentation call.""" + detections = self._primitives.detect_objects(query) + result = OracleToolResult("detect_objects", query, ()) + self._results.append(result) + handles = [] + for index, detection in enumerate(detections): + detection_id = self._id("detection") + self._detections[detection_id] = (query, index) + handles.append({"detection_id": detection_id, "confidence": detection.confidence}) + return json.dumps(_tool_payload(result, detections=handles)) + + def segment_detection(self, detection_id: str) -> str: + """Segment one earlier opaque detection and return opaque per-mask handles.""" + handle = self._detections.get(detection_id) + if handle is None: + return self._record_rejection("segment_detection", "", [], "unknown_detection_id") + query, index = handle + masks = self._primitives.segment_detection(query, index) + mask_ids = [] + for mask in masks: + mask_id = self._id("mask") + self._masks[mask_id] = (query, mask) + mask_ids.append(mask_id) + result = OracleToolResult("segment_detection", query, ()) + self._results.append(result) + return json.dumps(_tool_payload(result, mask_ids=mask_ids)) + + def ground_mask(self, mask_id: str) -> str: + """Ground one earlier opaque mask against the visible point cloud.""" + handle = self._masks.get(mask_id) + if handle is None: + return self._record_rejection("ground_mask", "", [], "unknown_mask_id") + query, mask = handle + object_id = self._id("object") + if hasattr(self._primitives, "ground_mask"): + object = self._primitives.ground_mask(mask, object_id) + else: + objects = self._primitives.ground_masks(query) + object = objects[0] if objects else None + if object is None: + return self._record_rejection( + "ground_mask", query, [], "insufficient_foreground_support" + ) + self._objects[object.id] = object + result = OracleToolResult("ground_mask", query, (_grounding_evidence(object),)) + self._results.append(result) + return json.dumps(_tool_payload(result, object_id=object.id)) + + def fit_ground_plane(self) -> str: + """Fit the private ground plane and return an opaque ID for measurements.""" + fit = self._primitives.fit_ground_plane() + if fit.estimate is None: + return self._record_rejection( + "fit_ground_plane", "", list(fit.quality_flags), fit.rejection_reason + ) + plane_id = self._id("plane") + self._planes[plane_id] = fit.estimate + self._ground_planes.add(plane_id) + measurement = OracleMeasurement( + fit.estimate.offset_m, + "m", + max(fit.estimate.residual_m, 0.01), + fit.quality_flags, + (f"frame:{self._primitives.frame.id}",), + ) + evidence = OracleEvidence( + f"ground-plane:v1:{self._primitives.frame.id}", + "v1", + "ground-plane", + "ground", + 0.0, + "n/a", + fit.estimate.inlier_count, + measurement, + ) + result = OracleToolResult( + "fit_ground_plane", + "", + (evidence,), + measurement=measurement, + plane=fit.estimate, + quality_flags=fit.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result, plane_id=plane_id)) + + def get_object_pose(self, object_id: str) -> str: + """Return existing grounding evidence for one object without selecting an answer.""" + object = self._objects.get(object_id) + if object is None: + return self._record_rejection("get_object_pose", object_id, [], "unknown_object_id") + result = OracleToolResult("get_object_pose", object.label, (_grounding_evidence(object),)) + self._results.append(result) + return json.dumps(_tool_payload(result, object_id=object.id)) + + def fit_object_surface_plane(self, object_id: str) -> str: + """Fit a plane to one grounded object's visible point support.""" + object = self._objects.get(object_id) + if object is None: + return self._record_rejection( + "fit_object_surface_plane", object_id, [], "unknown_object_id" + ) + fit = self._primitives.fit_object_surface_plane(object) + if fit.estimate is None: + return self._record_rejection( + "fit_object_surface_plane", + object.label, + list(fit.quality_flags), + fit.rejection_reason, + ) + plane_id = self._id("plane") + self._planes[plane_id] = fit.estimate + result = OracleToolResult( + "fit_object_surface_plane", + object.label, + (_grounding_evidence(object),), + plane=fit.estimate, + quality_flags=fit.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result, plane_id=plane_id)) + + def fit_mask_surrounding_plane(self, mask_id: str) -> str: + """Fit a plane around one selected segmentation mask.""" + handle = self._masks.get(mask_id) + if handle is None or not isinstance(handle, tuple) or handle[1] is None: + return self._record_rejection("fit_mask_surrounding_plane", "", [], "unknown_mask_id") + query, mask = handle + fit = self._primitives.fit_mask_surrounding_plane(mask) + if fit.estimate is None: + return self._record_rejection( + "fit_mask_surrounding_plane", query, list(fit.quality_flags), fit.rejection_reason + ) + plane_id = self._id("plane") + self._planes[plane_id] = fit.estimate + result = OracleToolResult( + "fit_mask_surrounding_plane", + query, + (), + plane=fit.estimate, + quality_flags=fit.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result, plane_id=plane_id)) + + def measure_object_pair_distance(self, first_object_id: str, second_object_id: str) -> str: + """Measure support-centroid distance between two grounded objects.""" + first, second = self._objects.get(first_object_id), self._objects.get(second_object_id) + if first is None or second is None: + return self._record_rejection( + "measure_object_pair_distance", "", [], "unknown_object_id" + ) + measurement = self._primitives.measure_object_pair_distance(first, second) + if measurement is None: + return self._record_rejection( + "measure_object_pair_distance", "", [], "insufficient_object_support" + ) + result = OracleToolResult( + "measure_object_pair_distance", + f"{first.label},{second.label}", + (_grounding_evidence(first), _grounding_evidence(second)), + measurement=measurement, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def measure_relative_plane_angle(self, first_plane_id: str, second_plane_id: str) -> str: + """Measure the unsigned angle between two accepted planes.""" + first, second = self._planes.get(first_plane_id), self._planes.get(second_plane_id) + if first is None or second is None: + return self._record_rejection( + "measure_relative_plane_angle", "", [], "unknown_plane_id" + ) + measurement = self._primitives.measure_relative_plane_angle(first, second) + result = OracleToolResult("measure_relative_plane_angle", "", (), measurement=measurement) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def measure_object_plane_relation( + self, object_id: str, support_id: str, plane_id: str, ground_plane_id: str + ) -> str: + """Measure one object's clearance/contact relation to a selected support plane.""" + object, support = self._objects.get(object_id), self._objects.get(support_id) + plane, ground = self._planes.get(plane_id), self._planes.get(ground_plane_id) + if object is None or support is None: + return self._record_rejection( + "measure_object_plane_relation", "", [], "unknown_object_id" + ) + if plane is None or ground is None or ground_plane_id not in self._ground_planes: + return self._record_rejection( + "measure_object_plane_relation", "", [], "unknown_plane_id" + ) + relation = self._primitives.measure_object_plane_relation( + object, support, plane, ground.normal + ) + if relation.rejection_reason is not None: + return self._record_rejection( + "measure_object_plane_relation", + f"{object.label},{support.label}", + list(relation.quality_flags), + relation.rejection_reason, + ) + metrics = tuple( + (name, value) + for name, value in ( + ("lower_clearance_m", relation.lower_clearance_m), + ("upper_clearance_m", relation.upper_clearance_m), + ("elevated_fraction", relation.elevated_fraction), + ("contact_point_count", float(relation.contact_point_count)), + ("planar_separation_m", relation.planar_separation_m), + ("contact_overlap_count", float(relation.contact_overlap_count)), + ) + if value is not None + ) + result = OracleToolResult( + "measure_object_plane_relation", + f"{object.label},{support.label}", + (_grounding_evidence(object), _grounding_evidence(support)), + quality_flags=relation.quality_flags, + metrics=metrics, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def measure_aperture_geometry(self, mask_id: str, ground_plane_id: str) -> str: + """Measure one selected aperture mask using an accepted ground plane.""" + handle = self._masks.get(mask_id) + ground = self._planes.get(ground_plane_id) + if handle is None or not isinstance(handle, tuple) or handle[1] is None: + return self._record_rejection("measure_aperture_geometry", "", [], "unknown_mask_id") + if ground is None or ground_plane_id not in self._ground_planes: + return self._record_rejection( + "measure_aperture_geometry", "", [], "unknown_ground_plane_id" + ) + query, mask = handle + result = self._primitives.measure_opening_width_from_mask(mask, ground) + if result.measurement is None: + return self._record_rejection( + "measure_aperture_geometry", + query, + list(result.quality_flags), + result.rejection_reason, + ) + evidence = OracleEvidence( + f"aperture:v1:{self._primitives.frame.id}:{mask_id}", + "v1", + mask_id, + query, + 0.0, + "n/a", + 0, + result.measurement, + ) + tool_result = OracleToolResult( + "measure_aperture_geometry", + query, + (evidence,), + measurement=result.measurement, + quality_flags=result.quality_flags, + ) + self._results.append(tool_result) + return json.dumps(_tool_payload(tool_result)) + + def measure_forward_corridor(self, ground_plane_id: str) -> str: + """Measure visible forward ground and obstacle support using one accepted ground plane.""" + ground = self._planes.get(ground_plane_id) + if ground is None or ground_plane_id not in self._ground_planes: + return self._record_rejection( + "measure_forward_corridor", "", [], "unknown_ground_plane_id" + ) + measured: ForwardCorridorMeasurement = self._primitives.measure_forward_corridor(ground) + if measured.rejection_reason is not None: + return self._record_rejection( + "measure_forward_corridor", "", [], measured.rejection_reason + ) + evidence = OracleEvidence( + f"forward-corridor:v1:{self._primitives.frame.id}", + "v1", + "forward-corridor", + "forward corridor", + 0.0, + "n/a", + measured.point_count, + ) + result = OracleToolResult( + "measure_forward_corridor", + "", + (evidence,), + quality_flags=("forward_ground_supported",), + metrics=( + ("ground_band_1_count", float(measured.ground_band_counts[0])), + ("ground_band_2_count", float(measured.ground_band_counts[1])), + ("ground_band_3_count", float(measured.ground_band_counts[2])), + ("elevated_obstacle_count", float(measured.obstacle_count)), + ("corridor_point_count", float(measured.point_count)), + ), + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def select_nearest_object(self, object_ids: list[str], side: str | None = None) -> str: + """Select one opaque grounded object by private point-cloud range.""" + selected: list[GroundedObject] = [] + for object_id in object_ids: + item = self._objects.get(object_id) + if item is None: + return self._record_rejection( + "select_nearest_object", object_id, [], "unknown_object_id" + ) + selected.append(item) + nearest = select_nearest_object(selected, side) + if nearest is None: + return self._record_rejection("select_nearest_object", "", [], "no_object_matches_side") + result = OracleToolResult( + "select_nearest_object", nearest.label, (_grounding_evidence(nearest),) + ) + self._results.append(result) + return json.dumps(_tool_payload(result, object_id=nearest.id)) + + def count_grounded_objects(self, object_ids: list[str]) -> str: + """Count unique grounded object IDs into fixed public count choices.""" + objects = self._lookup_objects("count_grounded_objects", object_ids) + if objects is None: + return self._record_rejection( + "count_grounded_objects", "", [], "unknown_or_duplicate_object_id" + ) + if not objects: + return self._record_rejection("count_grounded_objects", "", [], "no_grounded_object") + result = OracleToolResult( + "count_grounded_objects", + objects[0].label, + tuple(_grounding_evidence(item) for item in objects), + choice=count_choice(len(objects)), + choices=COUNT_CHOICES, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def bucket_camera_range(self, object_id: str) -> str: + """Map one grounded object's camera-origin range into fixed public choices.""" + object = self._objects.get(object_id) + if object is None: + return self._record_rejection("bucket_camera_range", object_id, [], "unknown_object_id") + measurement = OracleMeasurement( + object.range_m, + "m", + 0.0, + ("camera_origin_euclidean_range",), + (f"grounding:v1:{object.id}",), + ) + result = OracleToolResult( + "bucket_camera_range", + object.label, + (_grounding_evidence(object),), + measurement=measurement, + choice=camera_range_choice(object.range_m), + choices=CAMERA_RANGE_CHOICES, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def compare_nearest_by_side(self, object_ids: list[str]) -> str: + """Compare the nearest left and right grounded objects by camera range.""" + objects = self._lookup_objects("compare_nearest_by_side", object_ids) + if objects is None: + return self._record_rejection( + "compare_nearest_by_side", "", [], "unknown_or_duplicate_object_id" + ) + left = select_nearest_object(objects, "left") + right = select_nearest_object(objects, "right") + if left is None or right is None: + return self._record_rejection( + "compare_nearest_by_side", "", [], "missing_grounded_side" + ) + if left.range_m == right.range_m: + return self._record_rejection( + "compare_nearest_by_side", "", [], "ambiguous_nearest_by_side" + ) + choice = "left" if left.range_m < right.range_m else "right" + result = OracleToolResult( + "compare_nearest_by_side", + left.label, + (_grounding_evidence(left), _grounding_evidence(right)), + choice=choice, + choices=("left", "right"), + ) + self._results.append(result) + return json.dumps(_tool_payload(result, left_object_id=left.id, right_object_id=right.id)) + + def compare_left_right(self, first_object_id: str, second_object_id: str) -> str: + """Classify one grounded object's left/right relation to another grounded object.""" + first = self._objects.get(first_object_id) + second = self._objects.get(second_object_id) + if first is None or second is None: + return self._record_rejection("compare_left_right", "", [], "unknown_object_id") + relation = self._primitives.classify_horizontal_relation(first, second) + if relation.relation is None: + return self._record_rejection( + "compare_left_right", + f"{first.label},{second.label}", + list(relation.quality_flags), + relation.rejection_reason, + ) + result = OracleToolResult( + "compare_left_right", + f"{first.label},{second.label}", + (_grounding_evidence(first), _grounding_evidence(second)), + choice=relation.relation, + choices=("left", "right"), + quality_flags=relation.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def select_closest_object(self, target_id: str, candidate_ids: list[str]) -> str: + """Select one candidate closest to a target by private 3D support-point proximity.""" + target = self._objects.get(target_id) + if target is None: + return self._record_rejection( + "select_closest_object", target_id, [], "unknown_target_id" + ) + candidates: list[GroundedObject] = [] + for candidate_id in candidate_ids: + candidate = self._objects.get(candidate_id) + if candidate is None: + return self._record_rejection( + "select_closest_object", candidate_id, [], "unknown_candidate_id" + ) + candidates.append(candidate) + selected = self._primitives.select_closest_object(target, candidates) + if selected.object is None: + return self._record_rejection( + "select_closest_object", + target.label, + list(selected.quality_flags), + selected.rejection_reason, + ) + result = OracleToolResult( + "select_closest_object", + target.label, + (_grounding_evidence(target), _grounding_evidence(selected.object)), + choice=selected.object.label, + quality_flags=selected.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result, object_id=selected.object.id)) + + def measure_height(self, object_id: str, plane_id: str) -> str: + """Measure one grounded object against one previously accepted plane.""" + object = self._objects.get(object_id) + plane = self._planes.get(plane_id) + if object is None: + return self._record_rejection("measure_height", object_id, [], "unknown_object_id") + if plane is None: + return self._record_rejection("measure_height", object_id, [], "unknown_plane_id") + measured = self._primitives.measure_height(object, plane) + if measured.measurement is None: + return self._record_rejection( + "measure_height", + object.label, + list(measured.quality_flags), + measured.rejection_reason, + ) + measurement = measured.measurement + evidence = _height_evidence(object, measurement) + result = OracleToolResult( + "measure_height", + object.label, + (evidence,), + measurement=measurement, + plane=plane, + quality_flags=measured.quality_flags, + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def compare_heights(self, first_object_id: str, second_object_id: str, plane_id: str) -> str: + """Choose the taller object only when one shared-plane measurement is unambiguous.""" + first = self._objects.get(first_object_id) + second = self._objects.get(second_object_id) + plane = self._planes.get(plane_id) + if first is None or second is None: + return self._record_rejection("compare_heights", "", [], "unknown_object_id") + if first.id == second.id: + return self._record_rejection("compare_heights", first.label, [], "duplicate_object_id") + if plane is None: + return self._record_rejection("compare_heights", "", [], "unknown_plane_id") + first_height = self._primitives.measure_height(first, plane) + second_height = self._primitives.measure_height(second, plane) + if first_height.measurement is None or second_height.measurement is None: + return self._record_rejection( + "compare_heights", + "", + [*first_height.quality_flags, *second_height.quality_flags], + first_height.rejection_reason + or second_height.rejection_reason + or "height_measurement_rejected", + ) + first_measurement = first_height.measurement + second_measurement = second_height.measurement + first_lower = first_measurement.value - first_measurement.tolerance + second_lower = second_measurement.value - second_measurement.tolerance + first_upper = first_measurement.value + first_measurement.tolerance + second_upper = second_measurement.value + second_measurement.tolerance + if first_lower <= second_upper and second_lower <= first_upper: + return self._record_rejection("compare_heights", "", [], "ambiguous_height_comparison") + choice = first.label if first_lower > second_upper else second.label + result = OracleToolResult( + "compare_heights", + f"{first.label},{second.label}", + ( + _height_evidence(first, first_measurement), + _height_evidence(second, second_measurement), + ), + choice=choice, + choices=(first.label, second.label), + plane=plane, + quality_flags=(*first_height.quality_flags, *second_height.quality_flags), + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def classify_object_on_support(self, object_id: str, support_id: str) -> str: + """Verify that one grounded object directly rests on another grounded object.""" + object = self._objects.get(object_id) + support = self._objects.get(support_id) + if object is None or support is None: + return self._record_rejection("classify_object_on_support", "", [], "unknown_object_id") + result = self._primitives.classify_object_on_support(object, support) + if result.answer is None: + return self._record_rejection( + "classify_object_on_support", + f"{object.label},{support.label}", + list(result.quality_flags), + result.rejection_reason, + ) + tool_result = OracleToolResult( + "classify_object_on_support", + f"{object.label},{support.label}", + (_grounding_evidence(object), _grounding_evidence(support)), + choice=result.answer, + choices=("yes", "no"), + quality_flags=result.quality_flags, + ) + self._results.append(tool_result) + return json.dumps(_tool_payload(tool_result)) + + def measure_opening_width(self, mask_id: str) -> str: + """Measure one earlier segmented doorway/opening mask in metres.""" + handle = self._masks.get(mask_id) + if handle is None: + return self._record_rejection("measure_opening_width", "", [], "unknown_mask_id") + query = handle[0] if isinstance(handle, tuple) else handle + result = self._primitives.measure_opening_width(query) + if result.measurement is None: + return self._record_rejection( + "measure_opening_width", query, list(result.quality_flags), result.rejection_reason + ) + evidence = OracleEvidence( + f"opening-width:v1:{self._primitives.frame.id}:{mask_id}", + "v1", + mask_id, + query, + 0.0, + "n/a", + 0, + result.measurement, + ) + tool_result = OracleToolResult( + "measure_opening_width", + query, + (evidence,), + measurement=result.measurement, + quality_flags=result.quality_flags, + ) + self._results.append(tool_result) + return json.dumps(_tool_payload(tool_result)) + + def classify_forward_path(self) -> str: + """Classify the observed local corridor directly ahead of the camera.""" + result = self._primitives.classify_forward_path() + if result.state is None: + return self._record_rejection( + "classify_forward_path", + "", + list(result.quality_flags), + result.rejection_reason, + ) + evidence = OracleEvidence( + f"forward-path:v1:{self._primitives.frame.id}", + "v1", + "forward-path", + "forward path", + 0.0, + "center", + result.point_count, + ) + tool_result = OracleToolResult( + "classify_forward_path", + "forward path", + (evidence,), + choice=result.state, + quality_flags=result.quality_flags, + ) + self._results.append(tool_result) + return json.dumps(_tool_payload(tool_result)) + + def _record_rejection(self, tool: str, query: str, flags: list[str], reason: str | None) -> str: + result = OracleToolResult( + tool, query, (), quality_flags=tuple(flags), rejection_reason=reason + ) + self._results.append(result) + return json.dumps(_tool_payload(result)) + + def _lookup_objects(self, tool: str, object_ids: list[str]) -> list[GroundedObject] | None: + if len(set(object_ids)) != len(object_ids): + return None + objects: list[GroundedObject] = [] + for object_id in object_ids: + object = self._objects.get(object_id) + if object is None: + return None + objects.append(object) + return objects + + def _id(self, kind: str) -> str: + self._next_id += 1 + return f"{kind}:v1:{self._next_id:04d}" + + +def _grounding_evidence(item: GroundedObject) -> OracleEvidence: + return OracleEvidence( + f"grounding:v1:{item.id}", + "v1", + item.id, + item.label, + item.range_m, + item.horizontal_direction, + item.point_count, + ) + + +def _height_evidence(item: GroundedObject, measurement: OracleMeasurement) -> OracleEvidence: + return OracleEvidence( + f"height:v1:{item.id}", + "v1", + item.id, + item.label, + item.range_m, + item.horizontal_direction, + item.point_count, + measurement, + ) + + +def _tool_payload(result: OracleToolResult, **identifiers: Any) -> dict[str, Any]: + return { + "tool": result.tool, + "query": result.query, + "version": result.version, + "measurement": ( + { + "value": result.measurement.value, + "unit": result.measurement.unit, + "tolerance": result.measurement.tolerance, + "quality_flags": result.measurement.quality_flags, + "provenance_ids": result.measurement.provenance_ids, + } + if result.measurement is not None + else None + ), + "choice": result.choice, + "choices": result.choices, + "quality_flags": result.quality_flags, + "metrics": dict(result.metrics), + "rejection_reason": result.rejection_reason, + "plane": ( + { + "normal": result.plane.normal, + "offset_m": result.plane.offset_m, + "sample_count": result.plane.sample_count, + "inlier_count": result.plane.inlier_count, + "residual_m": result.plane.residual_m, + } + if result.plane is not None + else None + ), + "objects": [_evidence_payload(item) for item in result.evidence], + **identifiers, + } + + +def _evidence_payload(item: OracleEvidence) -> dict[str, Any]: + payload: dict[str, Any] = { + "evidence_id": item.id, + "id": item.object_id, + "label": item.label, + "range_m": item.range_m, + "side": item.side, + "point_count": item.point_count, + } + if item.measurement is not None: + payload["measurement"] = { + "value": item.measurement.value, + "unit": item.measurement.unit, + "tolerance": item.measurement.tolerance, + "quality_flags": item.measurement.quality_flags, + "provenance_ids": item.measurement.provenance_ids, + } + return payload diff --git a/dimos/benchmark/vqa/generation/pipeline.py b/dimos/benchmark/vqa/generation/pipeline.py new file mode 100644 index 0000000000..19476557f6 --- /dev/null +++ b/dimos/benchmark/vqa/generation/pipeline.py @@ -0,0 +1,43 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Single-frame VQA private ground-truth generation.""" + +from __future__ import annotations + +from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects +from dimos.benchmark.vqa.generation.questions import generate_questions +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + ObjectDetector, + ObjectSegmenter, + VqaExample, +) +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +def generate_ground_truth( + frame: CalibratedFrame, + queries: list[str], + detector: ObjectDetector, + segmenter: ObjectSegmenter, +) -> list[VqaExample]: + """Generate VQA examples from MoonDream/EdgeTAM-style image models and LiDAR.""" + segmented: list[Detection2DSeg] = [] + for query in queries: + detections = detector.detect(frame.image, query) + result = segmenter.segment(detections) + segmented.extend(detection for detection in result if isinstance(detection, Detection2DSeg)) + objects = ground_segmented_objects(frame, segmented) + return generate_questions(frame.id, objects, queries) diff --git a/dimos/benchmark/vqa/generation/primitives/choices.py b/dimos/benchmark/vqa/generation/primitives/choices.py new file mode 100644 index 0000000000..cdd7028a45 --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/choices.py @@ -0,0 +1,66 @@ +"""Deterministic public-choice resolution from private measurements.""" + +from __future__ import annotations + +from bisect import bisect_right + +COUNT_CHOICES = ("1-2", "3-4", "5-7", "8+") +CAMERA_RANGE_CHOICES = ("under 1 m", "1 to under 2 m", "2 to under 4 m", "4 m or more") +OPENING_WIDTH_CHOICES = ("under 0.2 m", "0.2 to under 0.5 m", "0.5 to under 0.8 m", "0.8 m or more") + + +def count_choice(count: int) -> str: + """Return the public count bucket for one or more grounded instances.""" + if count < 1: + raise ValueError("count must be positive") + if count <= 2: + return COUNT_CHOICES[0] + if count <= 4: + return COUNT_CHOICES[1] + if count <= 7: + return COUNT_CHOICES[2] + return COUNT_CHOICES[3] + + +def camera_range_choice(range_m: float) -> str: + """Return the public camera-origin range bucket for a grounded object.""" + if range_m < 0: + raise ValueError("range must be non-negative") + if range_m < 1.0: + return CAMERA_RANGE_CHOICES[0] + if range_m < 2.0: + return CAMERA_RANGE_CHOICES[1] + if range_m < 4.0: + return CAMERA_RANGE_CHOICES[2] + return CAMERA_RANGE_CHOICES[3] + + +def opening_width_choice(width_m: float) -> str: + """Return the public doorway-width bucket for a positive metric measurement.""" + if width_m < 0: + raise ValueError("width must be non-negative") + if width_m < 0.2: + return OPENING_WIDTH_CHOICES[0] + if width_m < 0.5: + return OPENING_WIDTH_CHOICES[1] + if width_m < 0.8: + return OPENING_WIDTH_CHOICES[2] + return OPENING_WIDTH_CHOICES[3] + + +def height_choice_window(height_m: float) -> tuple[tuple[str, ...], str]: + """Generate a local, exhaustive four-choice window around a private height.""" + breakpoints = (0.1, 0.2, 0.6, 1.0, 2.0) + start = min(max(bisect_right(breakpoints, height_m) - 1, 0), len(breakpoints) - 3) + lower, middle, upper = breakpoints[start : start + 3] + choices = ( + f"under {_format_height(lower)} m", + f"{_format_height(lower)}-{_format_height(middle)} m", + f"{_format_height(middle)}-{_format_height(upper)} m", + f"over {_format_height(upper)} m", + ) + return choices, choices[bisect_right((lower, middle, upper), height_m)] + + +def _format_height(value: float) -> str: + return f"{value:.1f}" diff --git a/dimos/benchmark/vqa/generation/primitives/contracts.py b/dimos/benchmark/vqa/generation/primitives/contracts.py new file mode 100644 index 0000000000..0e8b171539 --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/contracts.py @@ -0,0 +1,82 @@ +"""Typed results returned by frame-scoped private perception primitives.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from dimos.benchmark.vqa.models import GroundedObject, GroundPlaneEstimate, OracleMeasurement + + +@dataclass(frozen=True) +class HeightMeasurementResult: + """A private object-height measurement or its explicit rejection.""" + + object: GroundedObject + plane: GroundPlaneEstimate + measurement: OracleMeasurement | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class ClosestObjectResult: + """A point-cloud selected candidate nearest to one grounded target object.""" + + object: GroundedObject | None + distance_m: float | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class HorizontalRelationResult: + """A pairwise camera-frame horizontal relation or its explicit rejection.""" + + relation: Literal["left", "right"] | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class ObjectOnSupportResult: + """Verified direct support/contact separation, or an explicit conservative rejection.""" + + object: GroundedObject + support: GroundedObject + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + answer: Literal["yes", "no"] | None = None + + +@dataclass(frozen=True) +class ObjectPlaneRelationResult: + """Private geometric relation of one visible object to a selected support plane.""" + + lower_clearance_m: float | None + upper_clearance_m: float | None + elevated_fraction: float | None + contact_point_count: int + planar_separation_m: float | None + contact_overlap_count: int + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class OpeningWidthResult: + """A metric doorway opening width, or an explicit conservative rejection.""" + + measurement: OracleMeasurement | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class ForwardPathResult: + """A conservative visible-corridor classification from point-cloud evidence.""" + + state: Literal["clear", "blocked"] | None + point_count: int + quality_flags: tuple[str, ...] + rejection_reason: str | None = None diff --git a/dimos/benchmark/vqa/generation/primitives/frame.py b/dimos/benchmark/vqa/generation/primitives/frame.py new file mode 100644 index 0000000000..e1031ee11e --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/frame.py @@ -0,0 +1,489 @@ +"""Frame-scoped private perception primitives shared by VQA generation modes.""" + +from __future__ import annotations + +from dataclasses import replace + +import numpy as np + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.generation.grounding import ground_segmented_objects +from dimos.benchmark.vqa.generation.primitives.contracts import ( + ClosestObjectResult, + ForwardPathResult, + HeightMeasurementResult, + HorizontalRelationResult, + ObjectOnSupportResult, + ObjectPlaneRelationResult, + OpeningWidthResult, +) +from dimos.benchmark.vqa.generation.primitives.geometry import ( + PlaneFitResult, + classify_forward_corridor, + estimate_ground_plane, + fit_surface_plane, + measure_forward_corridor, + measure_opening_width, + measure_relative_plane_angle, + points_around_mask, + points_in_mask, +) +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + GroundedObject, + GroundingConfig, + GroundPlaneEstimate, + ObjectDetector, + ObjectPointLocalizer, + ObjectSegmenter, + OracleMeasurement, + PointObjectSegmenter, +) +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.point import Detection2DPoint +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +class FramePerceptionPrimitives: + """Cached private perception and geometry operations over one frozen frame.""" + + def __init__( + self, + frame: CalibratedFrame, + detector: ObjectDetector, + segmenter: ObjectSegmenter, + localizer: ObjectPointLocalizer | None = None, + point_segmenter: PointObjectSegmenter | None = None, + config: GroundingConfig = GroundingConfig(), + ) -> None: + if config.min_mask_area_px < 1 or config.min_foreground_points < 1: + raise ValueError("grounding thresholds must be positive") + self.frame = frame + self._detector = detector + self._segmenter = segmenter + self._localizer = localizer + self._point_segmenter = point_segmenter + self._config = config + self._detected: dict[str, ImageDetections2D] = {} + self._segmented_queries: set[str] = set() + self._masks: dict[str, list[Detection2DSeg]] = {} + self._points: dict[str, list[Detection2DPoint]] = {} + self._groundings: dict[str, list[GroundedObject]] = {} + self._object_masks: dict[str, Detection2DSeg] = {} + self._plane_fit: PlaneFitResult | None = None + + def detect_objects(self, query: str) -> ImageDetections2D: + """Run private object detection once for a semantic query.""" + cached = self._detected.get(query) + if cached is not None: + return cached + detections = self._detector.detect(self.frame.image, query) + self._detected[query] = detections + return detections + + def segment_detections(self, query: str) -> list[Detection2DSeg]: + """Segment accepted detections, falling back to point localization when available.""" + if query in self._segmented_queries: + return self._masks.get(query, []) + detections = self.detect_objects(query) + if len(detections): + segmented = self._segmenter.segment(detections) + elif self._localizer is not None and self._point_segmenter is not None: + points = self._localizer.locate(self.frame.image, query) + self._points[query] = [item for item in points if isinstance(item, Detection2DPoint)] + segmented = self._point_segmenter.segment_points(points) + else: + segmented = detections + masks = [ + item + for item in segmented + if isinstance(item, Detection2DSeg) + and int((item.mask > 0).sum()) >= self._config.min_mask_area_px + ] + self._masks[query] = masks + self._segmented_queries.add(query) + return masks + + def segment_detection(self, query: str, index: int) -> list[Detection2DSeg]: + """Segment one frozen detection selected by its query-local index.""" + detections = self.detect_objects(query) + if index < 0 or index >= len(detections): + raise ValueError("unknown_detection_index") + selected = ImageDetections2D(self.frame.image, [detections[index]]) + return [ + item + for item in self._segmenter.segment(selected) + if isinstance(item, Detection2DSeg) + and int((item.mask > 0).sum()) >= self._config.min_mask_area_px + ] + + def ground_mask(self, mask: Detection2DSeg, object_id: str) -> GroundedObject | None: + """Ground one selected segmentation mask under a caller-owned opaque object ID.""" + objects = ground_segmented_objects( + self.frame, [mask], min_foreground_points=self._config.min_foreground_points + ) + if len(objects) != 1: + return None + object = replace(objects[0], id=object_id) + self._object_masks[object.id] = mask + return object + + def ground_masks(self, query: str) -> list[GroundedObject]: + """Project visible point-cloud support through accepted masks.""" + cached = self._groundings.get(query) + if cached is not None: + return cached + masks = self.segment_detections(query) + objects = ground_segmented_objects( + self.frame, masks, min_foreground_points=self._config.min_foreground_points + ) + for item in objects: + index = _object_mask_index(item) + if index < len(masks): + self._object_masks[item.id] = masks[index] + self._groundings[query] = objects + return objects + + def used_point_localization(self, query: str) -> bool: + """Return whether segmentation for a query used positive-point localization.""" + return query in self._points + + def has_grounding(self, query: str) -> bool: + """Return whether a query has already been grounded for this frame.""" + return query in self._groundings + + @property + def can_localize_points(self) -> bool: + """Return whether point localization can supplement empty detections.""" + return self._localizer is not None and self._point_segmenter is not None + + def fit_ground_plane(self) -> PlaneFitResult: + """Fit and cache one quality-gated ground plane for the frozen frame.""" + if self._plane_fit is None: + self._plane_fit = estimate_ground_plane(self.frame) + return self._plane_fit + + def fit_object_surface_plane(self, object: GroundedObject) -> PlaneFitResult: + """Fit one object-supported surface plane without assigning it a semantic role.""" + points = self._object_points(object) + if points is None: + return PlaneFitResult(None, ("insufficient_object_support",), "insufficient_support") + return fit_surface_plane(points) + + def fit_mask_surrounding_plane(self, mask: Detection2DSeg) -> PlaneFitResult: + """Fit a surface plane from visible support around one selected mask.""" + return fit_surface_plane(points_around_mask(self.frame, mask.mask)) + + def fit_object_surrounding_plane(self, object: GroundedObject) -> PlaneFitResult: + """Fit a surface plane around one grounded object's selected mask.""" + mask = self._object_masks.get(object.id) + if mask is None: + return PlaneFitResult(None, ("unknown_grounded_object",), "unknown_object_id") + return self.fit_mask_surrounding_plane(mask) + + def measure_object_pair_distance( + self, first: GroundedObject, second: GroundedObject + ) -> OracleMeasurement | None: + """Measure support-centroid distance between two grounded objects.""" + first_points = self._object_points(first) + second_points = self._object_points(second) + if first_points is None or second_points is None: + return None + value = float( + np.linalg.norm(np.median(first_points, axis=0) - np.median(second_points, axis=0)) + ) + return OracleMeasurement( + value, + "m", + 0.05, + ("visible_support_centroids",), + (f"grounding:v1:{first.id}", f"grounding:v1:{second.id}"), + ) + + def measure_object_plane_relation( + self, + object: GroundedObject, + support: GroundedObject, + plane: GroundPlaneEstimate, + reference_normal: tuple[float, float, float], + ) -> ObjectPlaneRelationResult: + """Measure clearance and projected support proximity without assigning a relation label.""" + object_points = self._object_points(object) + support_points = self._object_points(support) + if object_points is None or support_points is None: + return ObjectPlaneRelationResult( + None, None, None, 0, None, 0, (), "insufficient_object_support" + ) + normal = np.asarray(plane.normal) + offset = plane.offset_m + if float(normal @ np.asarray(reference_normal)) < 0: + normal, offset = -normal, -offset + elevation = object_points @ normal + offset + projected_support = support_points - np.outer(support_points @ normal + offset, normal) + projected_object = object_points - np.outer(elevation, normal) + separation = np.linalg.norm( + projected_object[:, np.newaxis, :] - projected_support[np.newaxis, :, :], axis=2 + ).min(axis=1) + contact_points = object_points[np.abs(elevation) <= 0.08] + overlap_count = 0 + if len(contact_points): + projected_contact = contact_points - np.outer(contact_points @ normal + offset, normal) + contact_separation = np.linalg.norm( + projected_contact[:, np.newaxis, :] - projected_support[np.newaxis, :, :], axis=2 + ).min(axis=1) + overlap_count = int((contact_separation <= 0.1).sum()) + return ObjectPlaneRelationResult( + float(np.percentile(elevation, 15)), + float(np.percentile(elevation, 85)), + float((elevation >= 0.02).mean()), + len(contact_points), + float(separation.min()), + overlap_count, + ("visible_object_plane_relation",), + ) + + def measure_height( + self, object: GroundedObject, plane: GroundPlaneEstimate + ) -> HeightMeasurementResult: + """Measure one grounded object's visible height above an accepted plane.""" + mask = self._object_masks.get(object.id) + if mask is None: + raise ValueError(f"unknown grounded object: {object.id}") + selected = points_in_mask(self.frame, mask.mask) + flags = ["visible_point_cloud_height"] + if len(selected) < 6: + flags.append("sparse_object_point_support") + return HeightMeasurementResult( + object, plane, None, tuple(flags), "insufficient_object_support" + ) + normal = np.asarray(plane.normal) + distances = selected @ normal + plane.offset_m + positive = distances[distances > 0.02] + if len(positive) < 4 or len(positive) / len(selected) < 0.6: + flags.append("partial_or_non_elevated_object_support") + return HeightMeasurementResult( + object, plane, None, tuple(flags), "ambiguous_object_extent" + ) + flags.append("conservative_upper_percentile") + measurement = OracleMeasurement( + float(np.percentile(positive, 85)), + "m", + float(max(0.05, plane.residual_m + np.std(positive) * 0.25)), + tuple(flags), + ( + f"frame:{self.frame.id}", + f"ground-plane:v1:{self.frame.id}", + f"grounding:v1:{object.id}", + ), + ) + return HeightMeasurementResult(object, plane, measurement, tuple(flags)) + + def measure_relative_plane_angle( + self, first: GroundPlaneEstimate, second: GroundPlaneEstimate + ) -> OracleMeasurement: + """Measure the unsigned angle between two accepted planes.""" + return measure_relative_plane_angle(first, second) + + def select_closest_object( + self, target: GroundedObject, candidates: list[GroundedObject] + ) -> ClosestObjectResult: + """Select the unambiguously closest candidate by private support-point centroids.""" + if not candidates: + return ClosestObjectResult(None, None, (), "no_candidate_objects") + if any(item.id == target.id for item in candidates): + return ClosestObjectResult(None, None, (), "target_cannot_be_candidate") + target_points = self._object_points(target) + if target_points is None: + return ClosestObjectResult(None, None, (), "insufficient_target_support") + target_center = np.median(target_points, axis=0) + distances: list[tuple[float, GroundedObject]] = [] + for candidate in candidates: + candidate_points = self._object_points(candidate) + if candidate_points is None: + return ClosestObjectResult(None, None, (), "insufficient_candidate_support") + distance = float(np.linalg.norm(np.median(candidate_points, axis=0) - target_center)) + distances.append((distance, candidate)) + distances.sort(key=lambda item: item[0]) + if len(distances) > 1 and distances[1][0] - distances[0][0] < 0.15: + return ClosestObjectResult(None, None, (), "ambiguous_object_proximity") + return ClosestObjectResult(distances[0][1], distances[0][0], ("object_centroid_proximity",)) + + def classify_horizontal_relation( + self, first: GroundedObject, second: GroundedObject + ) -> HorizontalRelationResult: + """Classify whether the first object's support centroid is left or right of the second's.""" + if first.id == second.id: + return HorizontalRelationResult(None, (), "duplicate_object_id") + first_points = self._object_points(first) + second_points = self._object_points(second) + if first_points is None or second_points is None: + return HorizontalRelationResult(None, (), "insufficient_object_support") + horizontal_offset_m = float(np.median(first_points[:, 0]) - np.median(second_points[:, 0])) + if abs(horizontal_offset_m) < 0.1: + return HorizontalRelationResult(None, (), "ambiguous_horizontal_relation") + return HorizontalRelationResult( + "left" if horizontal_offset_m < 0 else "right", ("camera_frame_support_centroids",) + ) + + def classify_object_on_support( + self, object: GroundedObject, support: GroundedObject + ) -> ObjectOnSupportResult: + """Verify direct contact between an object and a horizontal support surface.""" + if object.id == support.id: + return ObjectOnSupportResult(object, support, (), "duplicate_object_id") + object_points = self._object_points(object) + support_points = self._object_points(support) + if object_points is None or support_points is None: + return ObjectOnSupportResult(object, support, (), "insufficient_object_support") + ground_fit = self.fit_ground_plane() + if ground_fit.estimate is None: + return ObjectOnSupportResult( + object, + support, + ground_fit.quality_flags, + ground_fit.rejection_reason or "ground_plane_rejected", + ) + support_fit = fit_surface_plane(support_points) + if support_fit.estimate is None: + return ObjectOnSupportResult( + object, + support, + support_fit.quality_flags, + support_fit.rejection_reason or "support_plane_rejected", + ) + normal = np.asarray(support_fit.estimate.normal) + ground_normal = np.asarray(ground_fit.estimate.normal) + if abs(float(normal @ ground_normal)) < np.cos(np.radians(12.0)): + return ObjectOnSupportResult( + object, support, support_fit.quality_flags, "support_not_horizontal" + ) + if float(normal @ ground_normal) < 0: + normal = -normal + elevation = object_points @ normal + support_fit.estimate.offset_m + projected_support = support_points - np.outer( + support_points @ normal + support_fit.estimate.offset_m, normal + ) + projected_object = object_points - np.outer( + object_points @ normal + support_fit.estimate.offset_m, normal + ) + nearest_support_distance = np.linalg.norm( + projected_object[:, np.newaxis, :] - projected_support[np.newaxis, :, :], axis=2 + ).min(axis=1) + if float(nearest_support_distance.min()) > 0.2: + return ObjectOnSupportResult( + object, + support, + (*support_fit.quality_flags, "objects_separated_in_plane"), + answer="no", + ) + if float(np.percentile(elevation, 15)) > 0.2: + return ObjectOnSupportResult( + object, + support, + (*support_fit.quality_flags, "object_clearly_above_support"), + answer="no", + ) + contact_points = object_points[np.abs(elevation) <= 0.08] + if ( + len(contact_points) < 4 + or float(np.percentile(elevation, 15)) > 0.08 + or float(np.percentile(elevation, 85)) < 0.15 + or float((elevation >= 0.02).mean()) < 0.7 + ): + return ObjectOnSupportResult( + object, support, support_fit.quality_flags, "insufficient_contact_evidence" + ) + projected_contact = contact_points - np.outer( + contact_points @ normal + support_fit.estimate.offset_m, normal + ) + distances = np.linalg.norm( + projected_contact[:, np.newaxis, :] - projected_support[np.newaxis, :, :], axis=2 + ) + if int((distances.min(axis=1) <= 0.1).sum()) < 3: + return ObjectOnSupportResult( + object, support, support_fit.quality_flags, "insufficient_in_plane_support_overlap" + ) + return ObjectOnSupportResult( + object, + support, + ( + *ground_fit.quality_flags, + *support_fit.quality_flags, + "support_horizontal", + "contact_band_supported", + "in_plane_support_overlap", + ), + answer="yes", + ) + + def measure_opening_width(self, query: str) -> OpeningWidthResult: + """Measure one segmented doorway aperture using its adjacent structural plane.""" + masks = self.segment_detections(query) + if len(masks) != 1: + return OpeningWidthResult(None, (), "ambiguous_opening_instances") + ground_fit = self.fit_ground_plane() + if ground_fit.estimate is None: + return OpeningWidthResult( + None, + ground_fit.quality_flags, + ground_fit.rejection_reason or "ground_plane_rejected", + ) + return self.measure_opening_width_from_mask(masks[0], ground_fit.estimate) + + def measure_opening_width_from_mask( + self, mask: Detection2DSeg, ground: GroundPlaneEstimate + ) -> OpeningWidthResult: + """Measure one selected aperture mask against an already accepted ground plane.""" + result = measure_opening_width(self.frame, mask.mask, ground) + if result.width_m is None or result.tolerance_m is None: + return OpeningWidthResult( + None, + result.quality_flags, + result.rejection_reason or "opening_width_rejected", + ) + measurement = OracleMeasurement( + result.width_m, + "m", + result.tolerance_m, + result.quality_flags, + (f"frame:{self.frame.id}", f"opening-mask:v1:{query}"), + ) + return OpeningWidthResult( + measurement, + result.quality_flags, + ) + + def classify_forward_path(self) -> ForwardPathResult: + """Classify the observed camera-forward corridor as clear or blocked.""" + ground_fit = self.fit_ground_plane() + if ground_fit.estimate is None: + return ForwardPathResult( + None, + 0, + ("ground_plane_rejected", *ground_fit.quality_flags), + ground_fit.rejection_reason, + ) + projected = project_visible_points(self.frame) + points = np.asarray(projected.camera_points, dtype=np.float64) + state, flags, reason = classify_forward_corridor(points, ground_fit.estimate) + return ForwardPathResult(state, len(points), flags, reason) + + def measure_forward_corridor(self, plane: GroundPlaneEstimate): + """Measure visible forward corridor support against an accepted ground plane.""" + projected = project_visible_points(self.frame) + points = np.asarray(projected.camera_points, dtype=np.float64) + return measure_forward_corridor(points, plane) + + def _object_points(self, object: GroundedObject) -> np.ndarray | None: + mask = self._object_masks.get(object.id) + if mask is None: + raise ValueError(f"unknown grounded object: {object.id}") + points = points_in_mask(self.frame, mask.mask) + return points if len(points) >= 6 else None + + +def _object_mask_index(item: GroundedObject) -> int: + try: + return int(item.id.rsplit("-", 1)[1]) + except (IndexError, ValueError) as exc: + raise ValueError(f"grounded object ID lacks mask index: {item.id}") from exc diff --git a/dimos/benchmark/vqa/generation/primitives/geometry.py b/dimos/benchmark/vqa/generation/primitives/geometry.py new file mode 100644 index 0000000000..0e149b3be9 --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/geometry.py @@ -0,0 +1,321 @@ +"""Deterministic point-cloud geometry helpers for private VQA primitives.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import cv2 +import numpy as np + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.models import CalibratedFrame, GroundPlaneEstimate, OracleMeasurement + + +@dataclass(frozen=True) +class PlaneFitResult: + """Accepted plane or explicit quality-gated rejection.""" + + estimate: GroundPlaneEstimate | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class OpeningGeometryResult: + """Opening width from a mask and surrounding structural plane.""" + + width_m: float | None + tolerance_m: float | None + quality_flags: tuple[str, ...] + rejection_reason: str | None = None + + +@dataclass(frozen=True) +class ForwardCorridorMeasurement: + """Private point-support metrics for the camera-forward corridor.""" + + ground_band_counts: tuple[int, int, int] + obstacle_count: int + point_count: int + rejection_reason: str | None = None + + +def estimate_ground_plane(frame: CalibratedFrame) -> PlaneFitResult: + """Fit a robust plane to visible points in the lower image ground band.""" + projected = project_visible_points(frame) + candidates = np.asarray( + [ + point + for point, (_, y) in zip(projected.camera_points, projected.pixels, strict=True) + if y >= int(frame.image.height * 0.6) + ], + dtype=np.float64, + ) + min_points, min_inliers, threshold = 12, 10, 0.06 + if len(candidates) < min_points: + return PlaneFitResult(None, ("insufficient_ground_band_points",), "insufficient_support") + + import open3d as o3d + + o3d.utility.random.seed(0) + point_cloud = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(candidates)) + _, inlier_indices = point_cloud.segment_plane( + distance_threshold=threshold, + ransac_n=3, + num_iterations=816, + ) + if len(inlier_indices) < min_inliers: + return PlaneFitResult(None, ("insufficient_plane_inliers",), "insufficient_inliers") + + inlier_points = candidates[np.asarray(inlier_indices, dtype=int)] + center = np.mean(inlier_points, axis=0) + _, _, right = np.linalg.svd(inlier_points - center, full_matrices=False) + normal = right[-1] + offset = -float(normal @ center) + if offset < 0: + normal, offset = -normal, -offset + residuals = np.abs(candidates @ normal + offset) + inliers = residuals <= threshold + residual_m = ( + float(np.sqrt(np.mean(np.square(residuals[inliers])))) if inliers.any() else float("inf") + ) + if int(inliers.sum()) < min_inliers: + return PlaneFitResult(None, ("insufficient_refined_inliers",), "insufficient_inliers") + if residual_m > 0.035: + return PlaneFitResult(None, ("high_plane_residual",), "residual_too_high") + return PlaneFitResult( + GroundPlaneEstimate( + tuple(float(value) for value in normal), + float(offset), + len(candidates), + int(inliers.sum()), + residual_m, + ), + ("ground_band_visible", "ransac_inliers_accepted"), + ) + + +def points_in_mask(frame: CalibratedFrame, mask: np.ndarray) -> np.ndarray: + """Return nearest visible camera points covered by one foreground mask.""" + if mask.shape != (frame.image.height, frame.image.width): + raise ValueError("segmentation mask dimensions must match the image") + projected = project_visible_points(frame) + return np.asarray( + [ + point + for point, (x, y) in zip(projected.camera_points, projected.pixels, strict=True) + if mask[y, x] > 0 + ], + dtype=np.float64, + ) + + +def points_around_mask(frame: CalibratedFrame, mask: np.ndarray, radius_px: int = 12) -> np.ndarray: + """Return visible points in an annulus surrounding one foreground mask.""" + if radius_px < 1: + raise ValueError("mask ring radius must be positive") + if mask.shape != (frame.image.height, frame.image.width): + raise ValueError("segmentation mask dimensions must match the image") + foreground = mask > 0 + expanded = cv2.dilate( + foreground.astype(np.uint8), + np.ones((radius_px * 2 + 1, radius_px * 2 + 1), dtype=np.uint8), + ).astype(bool) + return points_in_mask(frame, expanded & ~foreground) + + +def fit_surface_plane(points: np.ndarray) -> PlaneFitResult: + """Fit a robust plane to private surface points without assuming ground orientation.""" + min_points, min_inliers, threshold = 12, 10, 0.06 + if len(points) < min_points: + return PlaneFitResult(None, ("insufficient_surface_points",), "insufficient_support") + + import open3d as o3d + + o3d.utility.random.seed(0) + point_cloud = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(points)) + _, inlier_indices = point_cloud.segment_plane( + distance_threshold=threshold, + ransac_n=3, + num_iterations=816, + ) + if len(inlier_indices) < min_inliers: + return PlaneFitResult(None, ("insufficient_surface_inliers",), "insufficient_inliers") + + inlier_points = points[np.asarray(inlier_indices, dtype=int)] + center = np.mean(inlier_points, axis=0) + _, _, right = np.linalg.svd(inlier_points - center, full_matrices=False) + normal = right[-1] + offset = -float(normal @ center) + residuals = np.abs(points @ normal + offset) + inliers = residuals <= threshold + residual_m = ( + float(np.sqrt(np.mean(np.square(residuals[inliers])))) if inliers.any() else float("inf") + ) + if int(inliers.sum()) < min_inliers: + return PlaneFitResult( + None, ("insufficient_refined_surface_inliers",), "insufficient_inliers" + ) + if residual_m > 0.035: + return PlaneFitResult(None, ("high_surface_residual",), "residual_too_high") + return PlaneFitResult( + GroundPlaneEstimate( + tuple(float(value) for value in normal), + float(offset), + len(points), + int(inliers.sum()), + residual_m, + ), + ("surface_plane_accepted",), + ) + + +def measure_relative_plane_angle( + first: GroundPlaneEstimate, second: GroundPlaneEstimate +) -> OracleMeasurement: + """Measure the unsigned angle between two accepted plane normals.""" + alignment = abs(float(np.dot(first.normal, second.normal))) + return OracleMeasurement( + float(np.degrees(np.arccos(np.clip(alignment, -1.0, 1.0)))), + "deg", + 2.0, + ("accepted_plane_normals",), + (), + ) + + +def classify_forward_corridor( + points: np.ndarray, ground: GroundPlaneEstimate +) -> tuple[str | None, tuple[str, ...], str | None]: + """Classify a visible camera-forward corridor as clear or blocked.""" + measured = measure_forward_corridor(points, ground) + if measured.rejection_reason is not None: + return None, (), measured.rejection_reason + if measured.obstacle_count >= 4: + return "blocked", ("visible_forward_obstacle", "forward_ground_supported"), None + if measured.obstacle_count: + return None, (), "ambiguous_forward_obstacle" + return "clear", ("forward_ground_supported", "no_supported_forward_obstacle"), None + + +def measure_forward_corridor( + points: np.ndarray, ground: GroundPlaneEstimate +) -> ForwardCorridorMeasurement: + """Measure private floor and elevated-obstacle support in the camera-forward corridor.""" + if len(points) == 0: + return ForwardCorridorMeasurement((0, 0, 0), 0, 0, "insufficient_forward_support") + depth = points[:, 2] + lateral_limit = depth * np.tan(np.radians(20.0)) + corridor = points[(depth >= 0.5) & (depth <= 3.0) & (np.abs(points[:, 0]) <= lateral_limit)] + if len(corridor) < 12: + return ForwardCorridorMeasurement( + (0, 0, 0), 0, len(corridor), "insufficient_forward_support" + ) + elevation = corridor @ np.asarray(ground.normal) + ground.offset_m + ground_points = corridor[np.abs(elevation) <= 0.08] + bands = tuple( + int(((ground_points[:, 2] >= start) & (ground_points[:, 2] < stop)).sum()) + for start, stop in ((0.5, 1.33), (1.33, 2.16), (2.16, 3.0)) + ) + if any(count < 3 for count in bands): + return ForwardCorridorMeasurement( + bands, 0, len(corridor), "incomplete_forward_ground_support" + ) + obstacle_count = int((elevation > 0.15).sum()) + return ForwardCorridorMeasurement(bands, obstacle_count, len(corridor)) + + +def measure_opening_width( + frame: CalibratedFrame, mask: np.ndarray, ground: GroundPlaneEstimate +) -> OpeningGeometryResult: + """Measure a ground-connected vertical aperture from its silhouette and surrounding wall plane.""" + if mask.shape != (frame.image.height, frame.image.width): + raise ValueError("segmentation mask dimensions must match the image") + component_count, labels, stats, _ = cv2.connectedComponentsWithStats( + (mask > 0).astype(np.uint8) + ) + components = [ + index for index in range(1, component_count) if stats[index, cv2.CC_STAT_AREA] >= 128 + ] + if len(components) != 1: + return OpeningGeometryResult(None, None, (), "ambiguous_opening_component") + component = components[0] + x, y, width_px, height_px, _ = stats[component] + if x == 0 or y == 0 or x + width_px >= frame.image.width or y + height_px >= frame.image.height: + return OpeningGeometryResult(None, None, (), "opening_touches_image_edge") + structure = fit_surface_plane(points_around_mask(frame, (labels == component).astype(np.uint8))) + if structure.estimate is None: + return OpeningGeometryResult( + None, + None, + structure.quality_flags, + structure.rejection_reason or "structure_plane_rejected", + ) + normal = np.asarray(structure.estimate.normal) + ground_normal = np.asarray(ground.normal) + if abs(float(normal @ ground_normal)) > np.sin(np.radians(20.0)): + return OpeningGeometryResult( + None, None, structure.quality_flags, "opening_structure_not_vertical" + ) + direction = np.cross(ground_normal, normal) + direction_norm = float(np.linalg.norm(direction)) + if direction_norm == 0: + return OpeningGeometryResult( + None, None, structure.quality_flags, "opening_width_axis_rejected" + ) + direction /= direction_norm + component_mask = labels == component + rows = range(y + int(height_px * 0.3), y + max(int(height_px * 0.7), 1)) + widths: list[float] = [] + for row in rows: + columns = np.flatnonzero(component_mask[row]) + if len(columns) < 2: + continue + left = _intersect_pixel_with_plane(frame, int(columns[0]), row, structure.estimate) + right = _intersect_pixel_with_plane(frame, int(columns[-1]), row, structure.estimate) + if left is not None and right is not None: + widths.append(abs(float((right - left) @ direction))) + if len(widths) < 5: + return OpeningGeometryResult( + None, None, structure.quality_flags, "insufficient_opening_scanlines" + ) + median_width = float(np.median(widths)) + mad = float(np.median(np.abs(np.asarray(widths) - median_width))) + if median_width < 0.4 or max(widths) - min(widths) > max(0.1, median_width * 0.1): + return OpeningGeometryResult(None, None, structure.quality_flags, "ambiguous_opening_width") + bottom_columns = np.flatnonzero(component_mask[y + height_px - 1]) + if not len(bottom_columns): + return OpeningGeometryResult( + None, None, structure.quality_flags, "opening_not_ground_connected" + ) + bottom = _intersect_pixel_with_plane( + frame, int(np.median(bottom_columns)), y + height_px - 1, structure.estimate + ) + if bottom is None or abs(float(bottom @ ground_normal + ground.offset_m)) > 0.08: + return OpeningGeometryResult( + None, None, structure.quality_flags, "opening_not_ground_connected" + ) + return OpeningGeometryResult( + median_width, + max(0.05, structure.estimate.residual_m + 1.4826 * mad), + (*structure.quality_flags, "ground_connected_vertical_opening", "stable_opening_scanlines"), + ) + + +def _intersect_pixel_with_plane( + frame: CalibratedFrame, x: int, y: int, plane: GroundPlaneEstimate +) -> np.ndarray | None: + fx, fy, cx, cy = ( + frame.camera_info.K[0], + frame.camera_info.K[4], + frame.camera_info.K[2], + frame.camera_info.K[5], + ) + if fx <= 0 or fy <= 0: + return None + ray = np.asarray(((x - cx) / fx, (y - cy) / fy, 1.0)) + denominator = float(np.asarray(plane.normal) @ ray) + if abs(denominator) < 1e-6: + return None + distance = -plane.offset_m / denominator + return ray * distance if distance > 0 else None diff --git a/dimos/benchmark/vqa/generation/primitives/selection.py b/dimos/benchmark/vqa/generation/primitives/selection.py new file mode 100644 index 0000000000..a9263ab1fc --- /dev/null +++ b/dimos/benchmark/vqa/generation/primitives/selection.py @@ -0,0 +1,13 @@ +"""Deterministic selection over grounded object evidence.""" + +from __future__ import annotations + +from dimos.benchmark.vqa.models import GroundedObject + + +def select_nearest_object( + objects: list[GroundedObject], side: str | None = None +) -> GroundedObject | None: + """Return the nearest grounded object, optionally restricted to one image side.""" + candidates = [item for item in objects if side is None or item.horizontal_direction == side] + return min(candidates, key=lambda item: item.range_m) if candidates else None diff --git a/dimos/benchmark/vqa/generation/question_agent.py b/dimos/benchmark/vqa/generation/question_agent.py new file mode 100644 index 0000000000..573c54c2e2 --- /dev/null +++ b/dimos/benchmark/vqa/generation/question_agent.py @@ -0,0 +1,232 @@ +# Copyright 2026 Dimensional Inc. +"""Image-only constrained VQA question proposal.""" + +from __future__ import annotations + +import json +import re +from typing import Any + +from dimos.benchmark.vqa.models import ( + AnswerContract, + BooleanAnswerContract, + ChoiceAnswerContract, + DeferredHeightChoiceContract, + QuestionIntent, + QuestionProposal, +) +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.sensor_msgs.Image import Image + +QUESTION_PROMPT = """Select up to 15 challenging but visually well-supported single-frame VQA intents. +Inspect only this image. Do not assume depth, point clouds, calibration, metadata, or temporal context. +Return JSON only: an array of objects with kind, object_query, and threshold_m only for +within_distance, candidate_queries only for closest_object, and comparison_query only for +compare_height. kind must be one of presence, horizontal_direction, within_distance, visible_count, +camera_range, compare_nearest_by_side, compare_left_right, compare_height, door_state, closest_object, +object_on_support, opening_width, or forward_path. +Use threshold_m: 3.0 for within_distance. +Use image context to select only intents likely to produce a useful geometric case: emit +compare_nearest_by_side only when at least two visible instances of the same object appear on +opposite image sides; emit horizontal_direction only for a visible object; and prefer relational +or directional intents over presence. Emit door_state only for a clearly visible door with nearby +visible structure. Diversify object classes and question families when the scene supports them. +Emit visible_count only when at least one repeated visible object is present. Emit camera_range only +for a visible object. Emit compare_height only for two distinct visible upright object types resting +on the visible ground; comparison_query must name the other object type. Emit compare_left_right only +for two distinct visible object types; comparison_query must name the other object type. Emit closest_object only when +one target and at least two distinct candidate object types are visible; candidate_queries must name +the visible candidate types. +Emit object_on_support only when one visible object is clearly resting on a distinct visible table, +bench, or other horizontal support; comparison_query must name that support. Emit opening_width only +for one clearly visible doorway aperture with nearby wall structure. +Emit forward_path only when the center foreground and visible floor provide enough context to judge +the local path directly ahead. Use object_query: "forward path" for forward_path. +Do not return bare object names, floors, walls, ceilings, background surfaces, +questions, answers, explanations, Markdown, or information not visible in the image.""" + +AGENTIC_QUESTION_PROMPT = """Author up to 15 challenging, visually answerable single-frame VQA questions. +Inspect only this image. Do not use or infer depth, point clouds, calibration, metadata, or answers. +Return JSON only: an array of objects with question, answer_contract, optional object_queries, +and optional tool_hints. answer_contract is {"kind":"boolean"}, {"kind":"choice","choices":[...]}, +or {"kind":"deferred_height_choice","strategy":"height-window-v1"}. +Prioritize questions that a private point-cloud oracle can validate. For the height of one upright +object resting on visible ground, use deferred_height_choice. Its choices are generated privately +from a successful height measurement. Aim for a diverse set of questions that use the visible scene +composition: relative left/center/right position, which listed object is closest to a named target, +and height for an upright grounded object. For a visibly repeated object, count questions must use exactly +["1-2", "3-4", "5-7", "8+"]; camera-distance questions must use exactly ["under 1 m", +"1 to under 2 m", "2 to under 4 m", "4 m or more"]. For a pairwise left/right range question, +use exactly ["left", "right"]. For an A/B left-right relation question, use exactly ["left", "right"] +and name both objects. For a pairwise height question, use the two distinct object types as the choices. +For closest-object questions, use distinct visible candidate object types as the fixed choices. Use concise, +mutually exclusive fixed choices with two to four options. For a clearly visible door with nearby visible +structure, you may ask whether it is open or closed with exactly ["open", "closed"]. Use the fixed choices +["clear", "blocked"] only for a visibly supported local path directly ahead. +Use object_queries for every referenced object. The private oracle chooses its own sequence of reusable +perception and geometry tools; do not prescribe an answer-level tool sequence. Use visibility/presence questions +only when no stronger geometric question is available. +Do not ask about color, material, text, intent, full physical size, hidden parts, or exact +metric distances without a supplied choice contract. Use only visible objects. Do not include +answers, explanations, Markdown, or background surfaces.""" + +_UNSUPPORTED_QUERIES = {"background", "ceiling", "floor", "ground", "room", "wall"} + + +class OpenAIQuestionAgent: + """Propose constrained VQA intents from an image without geometry access.""" + + def __init__(self, model: OpenAIVlModel) -> None: + self._model = model + + def propose(self, image: Image) -> list[QuestionIntent]: + try: + payload: Any = _parse_json_array(self._model.query(image, QUESTION_PROMPT)) + except json.JSONDecodeError as exc: + raise ValueError("question agent did not return JSON") from exc + if not isinstance(payload, list) or len(payload) > 15: + raise ValueError("question agent must return an array of at most 15 intents") + intents: list[QuestionIntent] = [] + if all(isinstance(item, str) for item in payload): + return [ + intent + for query in dict.fromkeys(item.strip() for item in payload if item.strip()) + if query.lower() not in _UNSUPPORTED_QUERIES + for intent in _intents_for_query(query) + ] + for item in payload: + if not isinstance(item, dict): + continue + kind, query, threshold = ( + item.get("kind"), + item.get("object_query"), + item.get("threshold_m"), + ) + if kind not in ( + "presence", + "horizontal_direction", + "within_distance", + "visible_count", + "camera_range", + "compare_nearest_by_side", + "compare_left_right", + "compare_height", + "object_on_support", + "opening_width", + "door_state", + "closest_object", + "forward_path", + ): + continue + if not isinstance(query, str) or not query: + continue + if kind == "within_distance" and ( + not isinstance(threshold, (int, float)) or threshold <= 0 + ): + continue + if kind != "within_distance": + threshold = None + candidates = _string_tuple(item.get("candidate_queries"), "candidate_queries") + comparison_query = item.get("comparison_query") + if kind == "closest_object" and (len(candidates) < 2 or query in candidates): + continue + if kind == "forward_path" and query != "forward path": + continue + if kind in ("compare_left_right", "compare_height", "object_on_support") and ( + not isinstance(comparison_query, str) + or not comparison_query + or comparison_query == query + ): + continue + if kind not in ("compare_left_right", "compare_height", "object_on_support"): + comparison_query = None + intents.append(QuestionIntent(kind, query, threshold, candidates, comparison_query)) + return intents + + +class OpenAIFreeformQuestionAuthor: + """Image-only author for generic public questions and answer contracts.""" + + def __init__(self, model: OpenAIVlModel, max_questions: int = 15) -> None: + self._model = model + self._max_questions = max_questions + + def propose(self, image: Image) -> list[QuestionProposal]: + try: + payload: Any = _parse_json_array(self._model.query(image, AGENTIC_QUESTION_PROMPT)) + except json.JSONDecodeError as exc: + raise ValueError("question author did not return JSON") from exc + if not isinstance(payload, list) or len(payload) > self._max_questions: + raise ValueError("question author returned too many questions") + proposals = [] + errors: list[ValueError] = [] + for index, item in enumerate(payload, start=1): + try: + proposals.append(_proposal_from_json(item, index)) + except ValueError as exc: + errors.append(exc) + continue + if not proposals and errors: + raise errors[0] + if len({item.id for item in proposals}) != len(proposals): + raise ValueError("question ids must be unique") + return proposals + + +def _proposal_from_json(item: Any, index: int) -> QuestionProposal: + if not isinstance(item, dict): + raise ValueError("question proposal must be an object") + identifier, question = item.get("id"), item.get("question") + if not isinstance(question, str) or not question: + raise ValueError("question proposal requires question") + if not isinstance(identifier, str) or not identifier: + identifier = f"proposal-{index:02d}" + queries = _string_tuple(item.get("object_queries", []), "object_queries") + hints = _string_tuple(item.get("tool_hints", []), "tool_hints") + contract = item.get("answer_contract") + if not isinstance(contract, dict): + raise ValueError("question proposal requires answer_contract") + kind = contract.get("kind") + if kind == "boolean": + answer_contract: AnswerContract = BooleanAnswerContract() + elif kind == "choice": + choices = _string_tuple(contract.get("choices"), "choices") + if len(choices) < 2: + raise ValueError("choice contract requires at least two choices") + answer_contract = ChoiceAnswerContract(choices) + elif kind == "deferred_height_choice" and contract.get("strategy") == "height-window-v1": + answer_contract = DeferredHeightChoiceContract() + else: + raise ValueError("unsupported answer contract") + return QuestionProposal(identifier, question, answer_contract, queries, hints) + + +def _string_tuple(value: Any, name: str) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str): + value = [value] + if not isinstance(value, list): + return () + return tuple(item.strip() for item in value if isinstance(item, str) and item.strip()) + + +def _intents_for_query(query: str) -> list[QuestionIntent]: + return [ + QuestionIntent(kind="presence", object_query=query), + QuestionIntent(kind="horizontal_direction", object_query=query), + QuestionIntent(kind="within_distance", object_query=query, threshold_m=3.0), + QuestionIntent(kind="visible_count", object_query=query), + QuestionIntent(kind="camera_range", object_query=query), + QuestionIntent(kind="compare_nearest_by_side", object_query=query), + ] + + +def _parse_json_array(response: str) -> Any: + stripped = response.strip() + if stripped.startswith("```"): + stripped = re.sub(r"^```(?:json)?\s*|\s*```$", "", stripped, flags=re.IGNORECASE) + start, end = stripped.find("["), stripped.rfind("]") + if start < 0 or end < start: + raise json.JSONDecodeError("expected JSON array", stripped, 0) + return json.loads(stripped[start : end + 1]) diff --git a/dimos/benchmark/vqa/generation/questions.py b/dimos/benchmark/vqa/generation/questions.py new file mode 100644 index 0000000000..d442232972 --- /dev/null +++ b/dimos/benchmark/vqa/generation/questions.py @@ -0,0 +1,65 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Deterministic closed-answer questions from grounded objects.""" + +from __future__ import annotations + +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object +from dimos.benchmark.vqa.models import GroundedObject, VqaExample + + +def generate_questions( + frame_id: str, objects: list[GroundedObject], queries: list[str], *, distance_m: float = 3.0 +) -> list[VqaExample]: + """Generate presence, range, and direction questions for one frame.""" + if distance_m <= 0: + raise ValueError("distance_m must be positive") + + examples: list[VqaExample] = [] + for query in queries: + nearest = select_nearest_object([item for item in objects if item.label == query]) + examples.append( + VqaExample( + id=f"{frame_id}-{query}-presence", + question=f"Is there a {query} in the image? Answer yes or no.", + expected_answer="yes" if nearest is not None else "no", + answer_type="boolean", + object_ids=(nearest.id,) if nearest is not None else (), + allowed_answers=("yes", "no"), + ) + ) + if nearest is None: + continue + examples.extend( + [ + VqaExample( + id=f"{frame_id}-{query}-direction", + question=f"Where is the nearest {query}: left, center, or right?", + expected_answer=nearest.horizontal_direction, + answer_type="choice", + object_ids=(nearest.id,), + allowed_answers=("left", "center", "right"), + ), + VqaExample( + id=f"{frame_id}-{query}-range", + question=f"Is the nearest {query} within {distance_m:g} meters? Answer yes or no.", + expected_answer="yes" if nearest.range_m <= distance_m else "no", + answer_type="boolean", + object_ids=(nearest.id,), + allowed_answers=("yes", "no"), + ), + ] + ) + return examples diff --git a/dimos/benchmark/vqa/generation/recording.py b/dimos/benchmark/vqa/generation/recording.py new file mode 100644 index 0000000000..ff3cd6d49e --- /dev/null +++ b/dimos/benchmark/vqa/generation/recording.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +"""Build self-contained VQA frames from Go2 Memory2 recordings.""" + +from __future__ import annotations + +import numpy as np + +from dimos.benchmark.vqa.models import CalibratedFrame +from dimos.memory2.store.sqlite import SqliteStore +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.robot.unitree.go2.connection import BASE_TO_OPTICAL, GO2Connection + + +def load_go2_frame(recording: str, frame_index: int, tolerance_s: float = 0.25) -> CalibratedFrame: + """Load one image with its nearest LiDAR and odometry observations.""" + if frame_index < 0 or tolerance_s <= 0: + raise ValueError("frame_index must be non-negative and tolerance_s must be positive") + store = SqliteStore(path=recording, must_exist=True) + store.start() + try: + image_obs = store.streams.color_image.offset(frame_index).first() + lidar_obs = store.streams.lidar.at(image_obs.ts, tolerance_s).first() + odom_obs = store.streams.odom.at(image_obs.ts, tolerance_s).first() + image_data = image_obs.data + lidar_data = lidar_obs.data + odom_data = odom_obs.data + finally: + store.stop() + image, camera_info = _rectify_go2_image(image_data) + world_to_camera = -(Transform.from_pose("base_link", odom_data) + BASE_TO_OPTICAL) + return CalibratedFrame( + id=f"go2-{frame_index}", + image=image, + pointcloud=lidar_data, + camera_info=camera_info, + pointcloud_to_camera=world_to_camera, + image_is_rectified=True, + original_image=image_data, + ) + + +def _rectify_go2_image(image: Image) -> tuple[Image, CameraInfo]: + import cv2 + + source = GO2Connection.camera_info_static + matrix = np.asarray(source.K, dtype=np.float64).reshape(3, 3) + distortion = np.asarray(source.D, dtype=np.float64) + size = (image.width, image.height) + map_x, map_y = cv2.fisheye.initUndistortRectifyMap( + matrix, distortion, np.eye(3), matrix, size, cv2.CV_32FC1 + ) + data = cv2.remap(image.data, map_x, map_y, interpolation=cv2.INTER_LINEAR) + camera_info = CameraInfo.from_intrinsics( + matrix[0, 0], + matrix[1, 1], + matrix[0, 2], + matrix[1, 2], + image.width, + image.height, + frame_id="camera_optical", + ) + return Image(data=data, format=image.format, frame_id=image.frame_id, ts=image.ts), camera_info diff --git a/dimos/benchmark/vqa/generation/specification.py b/dimos/benchmark/vqa/generation/specification.py new file mode 100644 index 0000000000..f0c9ce20af --- /dev/null +++ b/dimos/benchmark/vqa/generation/specification.py @@ -0,0 +1,30 @@ +"""Validated input configuration for resumable VQA dataset generation.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class VqaGroundingSpecification(BaseModel): + """Private grounding quality thresholds for generated VQA labels.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + min_mask_area_px: int = Field(default=128, ge=1) + min_foreground_points: int = Field(default=3, ge=1) + + +class VqaGenerationSpecification(BaseModel): + """One reproducible multi-frame VQA generation request.""" + + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + recording: str = Field(min_length=1) + start_index: int = Field(default=0, ge=0) + stop_index: int = Field(gt=0) + stride: int = Field(default=1, ge=1) + question_mode: Literal["constrained", "agentic"] = "constrained" + grounding: VqaGroundingSpecification = VqaGroundingSpecification() + output: str | None = None diff --git a/dimos/benchmark/vqa/generation/test_agents.py b/dimos/benchmark/vqa/generation/test_agents.py new file mode 100644 index 0000000000..a99cdf8448 --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_agents.py @@ -0,0 +1,545 @@ +# Copyright 2026 Dimensional Inc. + +from __future__ import annotations + +from typing import cast + +import numpy as np + +from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator +from dimos.benchmark.vqa.generation.primitives.contracts import ( + HeightMeasurementResult, + HorizontalRelationResult, + ObjectPlaneRelationResult, + OpeningWidthResult, +) +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.geometry import PlaneFitResult +from dimos.benchmark.vqa.generation.question_agent import OpenAIQuestionAgent +from dimos.benchmark.vqa.models import ( + CalibratedFrame, + GroundedObject, + GroundingConfig, + GroundPlaneEstimate, + OracleMeasurement, + QuestionIntent, +) +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.point import Detection2DPoint +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +class _QuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert image.width == 6 + assert "point clouds" in prompt + return '```json\n["chair"]\n```' + + +class _Detector: + def __init__(self, image: Image, detection: Detection2DSeg) -> None: + self._image = image + self._detection = detection + + def detect(self, image: Image, query: str) -> ImageDetections2D: + return ImageDetections2D(image, [self._detection] if query == "chair" else []) + + +class _MultiDetector: + def __init__(self, image: Image, detections: list[Detection2DSeg]) -> None: + self._image = image + self._detections = detections + + def detect(self, image: Image, query: str) -> ImageDetections2D: + return ImageDetections2D(self._image, self._detections if query == "chair" else []) + + +def _agent( + frame: CalibratedFrame, + detector: _Detector | _MultiDetector, + segmenter: _Segmenter, + *, + localizer: _PointLocalizer | None = None, + point_segmenter: _PointSegmenter | None = None, + config: GroundingConfig = GroundingConfig(), +) -> VqaGroundTruthGenerator: + return VqaGroundTruthGenerator( + FramePerceptionPrimitives(frame, detector, segmenter, localizer, point_segmenter, config) + ) + + +class _Segmenter: + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: + return detections + + +class _PointLocalizer: + def locate(self, image: Image, query: str) -> ImageDetections2D: + return ImageDetections2D(image, [Detection2DPoint(3.0, 3.0, query, 0.0, image)]) + + +class _PointSegmenter: + def __init__(self, detection: Detection2DSeg) -> None: + self._detection = detection + + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: + return ImageDetections2D(points.image, [self._detection]) + + +def _frame_and_detection() -> tuple[CalibratedFrame, Detection2DSeg]: + image = Image.from_numpy(np.zeros((6, 6, 3), dtype=np.uint8)) + frame = CalibratedFrame( + id="frame-1", + image=image, + pointcloud=PointCloud2.from_numpy( + np.array([[0.0, 0.0, 1.0], [0.4, 0.0, 1.0], [-0.4, 0.0, 1.0]], dtype=np.float32) + ), + camera_info=CameraInfo.from_intrinsics(3.0, 3.0, 3.0, 3.0, 6, 6), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + detection = Detection2DSeg( + (0.0, 0.0, 5.0, 5.0), 0, -1, 1.0, "chair", 0.0, image, np.full((6, 6), 255, dtype=np.uint8) + ) + return frame, detection + + +def test_question_agent_returns_constrained_intents() -> None: + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _QuestionModel())).propose(frame.image) + + assert intents == [ + QuestionIntent(kind="presence", object_query="chair"), + QuestionIntent(kind="horizontal_direction", object_query="chair"), + QuestionIntent(kind="within_distance", object_query="chair", threshold_m=3.0), + QuestionIntent(kind="visible_count", object_query="chair"), + QuestionIntent(kind="camera_range", object_query="chair"), + QuestionIntent(kind="compare_nearest_by_side", object_query="chair"), + ] + + +def test_question_agent_uses_image_selected_structured_intents() -> None: + class _StructuredQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "Do not return bare object names" in prompt + assert "opposite image sides" in prompt + return '[{"kind":"compare_nearest_by_side","object_query":"chair"}]' + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _StructuredQuestionModel())).propose( + frame.image + ) + + assert intents == [QuestionIntent(kind="compare_nearest_by_side", object_query="chair")] + + +def test_question_agent_accepts_door_state_intent() -> None: + class _DoorQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "door_state" in prompt + return '[{"kind":"door_state","object_query":"door"}]' + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _DoorQuestionModel())).propose(frame.image) + + assert intents == [QuestionIntent(kind="door_state", object_query="door")] + + +def test_question_agent_accepts_closest_object_intent() -> None: + class _ClosestQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "candidate_queries" in prompt + return ( + '[{"kind":"closest_object","object_query":"chair",' + '"candidate_queries":["table","lamp"]}]' + ) + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _ClosestQuestionModel())).propose( + frame.image + ) + + assert intents == [QuestionIntent("closest_object", "chair", None, ("table", "lamp"))] + + +def test_question_agent_accepts_forward_path_intent() -> None: + class _ForwardPathQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "forward_path" in prompt + return '[{"kind":"forward_path","object_query":"forward path"}]' + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _ForwardPathQuestionModel())).propose( + frame.image + ) + + assert intents == [QuestionIntent(kind="forward_path", object_query="forward path")] + + +def test_question_agent_accepts_count_range_and_height_comparison_intents() -> None: + class _GeometryQuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "visible_count" in prompt + assert "comparison_query" in prompt + return """[ + {"kind":"visible_count","object_query":"chair"}, + {"kind":"compare_left_right","object_query":"chair","comparison_query":"table"}, + {"kind":"compare_height","object_query":"chair","comparison_query":"table"} + ,{"kind":"object_on_support","object_query":"box","comparison_query":"table"} + ,{"kind":"opening_width","object_query":"doorway"} + ]""" + + frame, _ = _frame_and_detection() + + intents = OpenAIQuestionAgent(cast("OpenAIVlModel", _GeometryQuestionModel())).propose( + frame.image + ) + + assert intents == [ + QuestionIntent(kind="visible_count", object_query="chair"), + QuestionIntent(kind="compare_left_right", object_query="chair", comparison_query="table"), + QuestionIntent(kind="compare_height", object_query="chair", comparison_query="table"), + QuestionIntent(kind="object_on_support", object_query="box", comparison_query="table"), + QuestionIntent(kind="opening_width", object_query="doorway"), + ] + + +def test_ground_truth_agent_records_tools_and_rejects_unsupported_question() -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + + answered = agent.answer( + frame, QuestionIntent(kind="within_distance", object_query="chair", threshold_m=3.0) + ) + rejected = agent.answer( + frame, QuestionIntent(kind="horizontal_direction", object_query="table") + ) + absent = agent.answer(frame, QuestionIntent(kind="presence", object_query="table")) + + assert answered.status == "answered" + assert answered.answer == "yes" + assert answered.evidence[0].point_count == 3 + assert [item.tool for item in answered.trace] == [ + "detect_objects", + "segment_objects", + "get_foreground_geometry", + ] + assert rejected.status == "rejected" + assert rejected.reason == "no_grounded_object" + assert absent.status == "rejected" + + +def test_ground_truth_agent_rejects_small_masks() -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=37), + ) + + result = agent.answer(frame, QuestionIntent(kind="presence", object_query="chair")) + + assert result.status == "rejected" + + +def test_ground_truth_agent_falls_back_to_point_prompt() -> None: + frame, detection = _frame_and_detection() + point_detection = Detection2DSeg( + detection.bbox, + detection.track_id, + detection.class_id, + detection.confidence, + "plant", + detection.ts, + detection.image, + detection.mask, + ) + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + localizer=_PointLocalizer(), + point_segmenter=_PointSegmenter(point_detection), + config=GroundingConfig(min_mask_area_px=1), + ) + + result = agent.answer(frame, QuestionIntent(kind="presence", object_query="plant")) + + assert result.answer == "yes" + assert [item.tool for item in result.trace] == [ + "detect_objects", + "locate_object_point", + "segment_object_point", + "get_foreground_geometry", + ] + + +def test_ground_truth_agent_compares_nearest_objects_by_side() -> None: + image = Image.from_numpy(np.zeros((6, 6, 3), dtype=np.uint8)) + frame = CalibratedFrame( + id="frame-1", + image=image, + pointcloud=PointCloud2.from_numpy( + np.array( + [ + [-0.5, -0.6, 1.0], + [-0.5, 0.0, 1.0], + [-0.5, 0.6, 1.0], + [1.0, -1.0, 2.0], + [1.0, 0.0, 2.0], + [1.0, 1.0, 2.0], + ], + dtype=np.float32, + ) + ), + camera_info=CameraInfo.from_intrinsics(3.0, 3.0, 3.0, 3.0, 6, 6), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + left_mask = np.zeros((6, 6), dtype=np.uint8) + left_mask[:, :3] = 255 + right_mask = np.zeros((6, 6), dtype=np.uint8) + right_mask[:, 3:] = 255 + detections = [ + Detection2DSeg((0.0, 0.0, 2.0, 5.0), 0, -1, 1.0, "chair", 0.0, image, left_mask), + Detection2DSeg((3.0, 0.0, 5.0, 5.0), 1, -1, 1.0, "chair", 0.0, image, right_mask), + ] + agent = _agent( + frame, + _MultiDetector(image, detections), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + + result = agent.answer( + frame, QuestionIntent(kind="compare_nearest_by_side", object_query="chair") + ) + + assert result.status == "answered" + assert result.answer == "left" + assert result.question.object_ids == ("frame-1-chair-0", "frame-1-chair-1") + + +def test_ground_truth_agent_rejects_side_comparison_without_both_sides() -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + + result = agent.answer( + frame, QuestionIntent(kind="compare_nearest_by_side", object_query="chair") + ) + + assert result.status == "rejected" + assert result.reason == "missing_grounded_side" + + +def test_ground_truth_agent_buckets_visible_count_and_camera_range() -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + + count = agent.answer(frame, QuestionIntent(kind="visible_count", object_query="chair")) + camera_range = agent.answer(frame, QuestionIntent(kind="camera_range", object_query="chair")) + + assert count.answer == "1-2" + assert count.question.allowed_answers == ("1-2", "3-4", "5-7", "8+") + assert camera_range.answer == "1 to under 2 m" + assert camera_range.question.allowed_answers == ( + "under 1 m", + "1 to under 2 m", + "2 to under 4 m", + "4 m or more", + ) + + +def test_ground_truth_agent_compares_ground_plane_relative_heights(monkeypatch: object) -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + chair = GroundedObject("chair-0", "chair", 8, 1.0, "left") + table = GroundedObject("table-0", "table", 8, 2.0, "right") + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + monkeypatch.setattr( + agent, + "ground", + lambda _frame, query: ([chair] if query == "chair" else [table], ()), + ) + monkeypatch.setattr( + agent.primitives, + "fit_ground_plane", + lambda: type("Fit", (), {"estimate": plane, "rejection_reason": None})(), + ) + monkeypatch.setattr( + agent.primitives, + "measure_height", + lambda item, accepted_plane: HeightMeasurementResult( + item, + accepted_plane, + OracleMeasurement(0.8 if item == chair else 0.5, "m", 0.05, (), ()), + (), + ), + ) + + result = agent.answer( + frame, + QuestionIntent(kind="compare_height", object_query="chair", comparison_query="table"), + ) + + assert result.status == "answered" + assert result.answer == "chair" + assert result.question.allowed_answers == ("chair", "table") + + +def test_ground_truth_agent_compares_pairwise_left_right(monkeypatch: object) -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + chair = GroundedObject("chair-0", "chair", 8, 1.0, "left") + table = GroundedObject("table-0", "table", 8, 2.0, "right") + monkeypatch.setattr( + agent, + "ground", + lambda _frame, query: ([chair] if query == "chair" else [table], ()), + ) + monkeypatch.setattr( + agent.primitives, + "classify_horizontal_relation", + lambda first, second: HorizontalRelationResult("left", ("camera_frame_support_centroids",)), + ) + + result = agent.answer( + frame, + QuestionIntent(kind="compare_left_right", object_query="chair", comparison_query="table"), + ) + + assert result.status == "answered" + assert result.answer == "left" + assert result.question.allowed_answers == ("left", "right") + + +def test_ground_truth_agent_generates_verified_support_and_opening_width( + monkeypatch: object, +) -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + box = GroundedObject("box-0", "box", 8, 1.0, "left") + table = GroundedObject("table-0", "table", 8, 2.0, "right") + monkeypatch.setattr( + agent, + "ground", + lambda _frame, query: ([box] if query == "box" else [table], ()), + ) + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + monkeypatch.setattr(agent.primitives, "fit_ground_plane", lambda: PlaneFitResult(plane, ())) + monkeypatch.setattr( + agent.primitives, "fit_object_surface_plane", lambda item: PlaneFitResult(plane, ()) + ) + monkeypatch.setattr( + agent.primitives, + "measure_object_plane_relation", + lambda item, support, support_plane, normal: ObjectPlaneRelationResult( + 0.0, + 0.8, + 0.8, + 4, + 0.0, + 3, + (), + ), + ) + monkeypatch.setattr( + agent.primitives, + "measure_opening_width_from_mask", + lambda mask, ground: OpeningWidthResult(OracleMeasurement(0.6, "m", 0.05, (), ()), ()), + ) + monkeypatch.setattr(agent.primitives, "segment_detections", lambda query: [detection]) + + support = agent.answer( + frame, + QuestionIntent(kind="object_on_support", object_query="box", comparison_query="table"), + ) + opening = agent.answer(frame, QuestionIntent(kind="opening_width", object_query="doorway")) + + assert support.answer == "yes" + assert support.question.allowed_answers == ("yes", "no") + assert opening.answer == "0.5 to under 0.8 m" + + +def test_ground_truth_agent_keeps_verified_non_support_cases(monkeypatch: object) -> None: + frame, detection = _frame_and_detection() + agent = _agent( + frame, + _Detector(frame.image, detection), + _Segmenter(), + config=GroundingConfig(min_mask_area_px=1), + ) + box = GroundedObject("box-0", "box", 8, 1.0, "left") + table = GroundedObject("table-0", "table", 8, 2.0, "right") + monkeypatch.setattr( + agent, + "ground", + lambda _frame, query: ([box] if query == "box" else [table], ()), + ) + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + monkeypatch.setattr(agent.primitives, "fit_ground_plane", lambda: PlaneFitResult(plane, ())) + monkeypatch.setattr( + agent.primitives, "fit_object_surface_plane", lambda item: PlaneFitResult(plane, ()) + ) + monkeypatch.setattr( + agent.primitives, + "measure_object_plane_relation", + lambda item, support, support_plane, normal: ObjectPlaneRelationResult( + 0.0, + 0.8, + 0.8, + 0, + 0.3, + 0, + (), + ), + ) + + result = agent.answer( + frame, + QuestionIntent(kind="object_on_support", object_query="box", comparison_query="table"), + ) + + assert result.status == "answered" + assert result.answer == "no" diff --git a/dimos/benchmark/vqa/generation/test_dataset.py b/dimos/benchmark/vqa/generation/test_dataset.py new file mode 100644 index 0000000000..6790044cd4 --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_dataset.py @@ -0,0 +1,104 @@ +# Copyright 2026 Dimensional Inc. + +import json +from pathlib import Path + +from dimos.benchmark.vqa.generation.dataset import _evaluation_rows, write_dataset_manifest +from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + ChoiceAnswerContract, + DeferredHeightChoiceContract, + GroundTruthResult, + QuestionIntent, + QuestionProposal, + ToolTrace, + VqaExample, +) + + +def test_constrained_results_export_simple_multiple_choice_rows() -> None: + result = GroundTruthResult( + intent=QuestionIntent("presence", "chair"), + question=VqaExample( + "frame-chair-presence", + "Is there a chair?", + "yes", + "boolean", + (), + ("yes", "no"), + ), + status="answered", + answer="yes", + reason=None, + evidence=(), + trace=(ToolTrace("ground", "chair"),), + ) + + cases, labels = _evaluation_rows("frame", [result]) + + assert cases == [ + { + "id": "frame-chair-presence", + "image": "image.jpg", + "question": "Is there a chair?", + "choices": ("yes", "no"), + } + ] + assert labels == [{"id": "frame-chair-presence", "answer": "yes"}] + + +def test_deferred_height_result_exports_resolved_public_choices() -> None: + choices = ("under 0.2 m", "0.2-0.6 m", "0.6-1.0 m", "over 1.0 m") + result = AcceptedOracleResult( + QuestionProposal("chair-height", "How tall is the chair?", DeferredHeightChoiceContract()), + "0.2-0.6 m", + ChoiceAnswerContract(choices), + ("height-1",), + (), + (), + ) + + cases, labels = _evaluation_rows("frame", [result]) + + assert cases[0]["choices"] == choices + assert labels == [{"id": "frame-chair-height", "answer": "0.2-0.6 m"}] + + +def test_dataset_manifest_exports_public_cases_and_private_labels(tmp_path: Path) -> None: + frame = tmp_path / "frame-000040" + frame.mkdir() + (frame / "frame.json").write_text( + json.dumps( + { + "frame_id": "frame-40", + "accepted_question_count": 1, + "rejected_question_count": 2, + } + ) + ) + (frame / "cases.json").write_text( + json.dumps( + [ + { + "id": "case-1", + "image": "image.jpg", + "question": "Is it visible?", + "choices": ["yes", "no"], + } + ] + ) + ) + (frame / "labels.json").write_text(json.dumps([{"id": "case-1", "answer": "yes"}])) + + summary = write_dataset_manifest(tmp_path) + + assert summary == {"frame_count": 1, "accepted_question_count": 1, "rejected_question_count": 2} + assert json.loads((tmp_path / "cases.jsonl").read_text()) == { + "id": "case-1", + "image": "frame-000040/image.jpg", + "question": "Is it visible?", + "choices": ["yes", "no"], + } + assert json.loads((tmp_path / "labels.jsonl").read_text()) == {"id": "case-1", "answer": "yes"} + assert not (tmp_path / "frames.jsonl").exists() + assert not (tmp_path / "manifest.json").exists() diff --git a/dimos/benchmark/vqa/generation/test_geometry.py b/dimos/benchmark/vqa/generation/test_geometry.py new file mode 100644 index 0000000000..d4977c0adb --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_geometry.py @@ -0,0 +1,61 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import numpy as np +import pytest + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.models import CalibratedFrame +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 + + +def _frame(points: np.ndarray, *, rectified: bool = True) -> CalibratedFrame: + return CalibratedFrame( + id="frame-1", + image=Image.from_numpy(np.zeros((4, 4, 3), dtype=np.uint8)), + pointcloud=PointCloud2.from_numpy(points), + camera_info=CameraInfo.from_intrinsics(2.0, 2.0, 2.0, 2.0, 4, 4), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=rectified, + ) + + +def test_project_visible_points_keeps_nearest_point_per_pixel() -> None: + frame = _frame( + np.array( + [ + [0.0, 0.0, 2.0], + [0.0, 0.0, 1.0], + [0.5, 0.0, 1.0], + [0.0, 0.0, -1.0], + ], + dtype=np.float32, + ) + ) + + projected = project_visible_points(frame) + + assert projected.pixels == [(2, 2), (3, 2)] + assert projected.source_indices == [1, 2] + assert projected.camera_points == [(0.0, 0.0, 1.0), (0.5, 0.0, 1.0)] + + +def test_project_visible_points_rejects_unrectified_images() -> None: + with pytest.raises(ValueError, match="rectified"): + project_visible_points(_frame(np.zeros((1, 3), dtype=np.float32), rectified=False)) diff --git a/dimos/benchmark/vqa/generation/test_oracle.py b/dimos/benchmark/vqa/generation/test_oracle.py new file mode 100644 index 0000000000..6642870e1b --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_oracle.py @@ -0,0 +1,794 @@ +# Copyright 2026 Dimensional Inc. + +from __future__ import annotations + +import json +from typing import Any, cast + +from langchain_core.messages import AIMessage +import numpy as np + +from dimos.benchmark.vqa.generation.geometry import project_visible_points +from dimos.benchmark.vqa.generation.oracle import ( + PrivateToolCallingOracle, + SemanticEvidenceValidation, + validate_oracle_answer, +) +from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.generation.primitives.choices import ( + camera_range_choice, + count_choice, + height_choice_window, +) +from dimos.benchmark.vqa.generation.primitives.contracts import ( + HeightMeasurementResult, + HorizontalRelationResult, + ObjectOnSupportResult, + OpeningWidthResult, +) +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.primitives.geometry import ( + ForwardCorridorMeasurement, + PlaneFitResult, + classify_forward_corridor, + estimate_ground_plane, + measure_opening_width, + measure_relative_plane_angle, +) +from dimos.benchmark.vqa.generation.question_agent import ( + AGENTIC_QUESTION_PROMPT, + OpenAIFreeformQuestionAuthor, +) +from dimos.benchmark.vqa.models import ( + BooleanAnswerContract, + CalibratedFrame, + ChoiceAnswerContract, + DeferredHeightChoiceContract, + GroundedObject, + GroundingConfig, + GroundPlaneEstimate, + OracleEvidence, + OracleMeasurement, + OracleToolResult, + QuestionProposal, + RejectedOracleResult, +) +from dimos.models.vl.openai import OpenAIVlModel +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +class _QuestionModel: + def query(self, image: Image, prompt: str) -> str: + assert "answer_contract" in prompt + return '[{"id":"chair-presence","question":"Is there a chair?","answer_contract":{"kind":"boolean"},"object_queries":["chair"]}]' + + +class _Grounding: + frame = cast("Any", object()) + + def detect_objects(self, query: str) -> list[Any]: + return [] + + def segment_detections(self, query: str) -> list[Any]: + return [] + + def ground_masks(self, query: str) -> list[Any]: + return [ + type( + "Object", + (), + { + "id": "synthetic-chair-0", + "label": query, + "range_m": 1.0, + "horizontal_direction": "left", + "point_count": 4, + }, + )() + ] + + +def _measurement_frame(points: np.ndarray | None = None) -> CalibratedFrame: + ground = [[x, 1.0, z] for z in (3.0, 4.0, 5.0) for x in (-1.0, -0.5, 0.0, 0.5, 1.0)] + object_points = [[2.0, 1.0 - height, 4.0] for height in np.linspace(0.2, 1.0, 9)] + image = Image.from_numpy(np.zeros((100, 100, 3), dtype=np.uint8)) + return CalibratedFrame( + id="synthetic", + image=image, + pointcloud=PointCloud2.from_numpy( + points + if points is not None + else np.asarray([*ground, *object_points], dtype=np.float32) + ), + camera_info=CameraInfo.from_intrinsics(50.0, 50.0, 50.0, 50.0, 100, 100), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + + +class _MaskDetector: + def __init__(self, mask: np.ndarray) -> None: + self._mask = mask + + def detect(self, image: Image, query: str) -> ImageDetections2D: + detection = Detection2DSeg( + (0.0, 0.0, float(image.width - 1), float(image.height - 1)), + 0, + -1, + 1.0, + query, + 0.0, + image, + self._mask, + ) + return ImageDetections2D(image, [detection]) + + +class _IdentitySegmenter: + def segment(self, detections: Any) -> Any: + return detections + + +def _frame_primitives( + frame: CalibratedFrame, mask: np.ndarray | None = None +) -> FramePerceptionPrimitives: + if mask is None: + mask = np.full((frame.image.height, frame.image.width), 255, dtype=np.uint8) + return FramePerceptionPrimitives( + frame, _MaskDetector(mask), _IdentitySegmenter(), config=GroundingConfig(min_mask_area_px=1) + ) + + +class _BoundModel: + def __init__(self) -> None: + self._calls = 0 + + def bind_tools(self, tools: Any) -> _BoundModel: + return self + + def invoke(self, messages: Any) -> AIMessage: + self._calls += 1 + if self._calls == 1: + return AIMessage( + content="", + tool_calls=[ + { + "name": "get_object_pose", + "args": {"object_id": "synthetic-chair-0"}, + "id": "call-2", + } + ], + ) + return AIMessage( + content='{"answer":"yes","evidence_ids":["grounding:v1:synthetic-chair-0"]}' + ) + + +class _ScriptedModel: + def __init__(self, responses: list[AIMessage]) -> None: + self._responses = responses + + def bind_tools(self, tools: Any) -> _ScriptedModel: + return self + + def invoke(self, messages: Any) -> AIMessage: + return self._responses.pop(0) + + +class _SemanticValidator: + def __init__(self, verdict: SemanticEvidenceValidation) -> None: + self.verdict = verdict + self.calls: list[tuple[QuestionProposal, str | float, tuple[OracleToolResult, ...]]] = [] + + def validate( + self, + proposal: QuestionProposal, + answer: str | float, + cited_results: tuple[OracleToolResult, ...], + ) -> SemanticEvidenceValidation: + self.calls.append((proposal, answer, cited_results)) + return self.verdict + + +def test_freeform_question_author_parses_public_contract() -> None: + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposals = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _QuestionModel())).propose(image) + + assert proposals == [ + QuestionProposal( + "chair-presence", "Is there a chair?", BooleanAnswerContract(), ("chair",), () + ) + ] + + +def test_freeform_author_prompt_prioritizes_geometric_questions() -> None: + assert "chooses its own sequence" in AGENTIC_QUESTION_PROMPT + assert "closest to a named target" in AGENTIC_QUESTION_PROMPT + assert "visibly repeated object" in AGENTIC_QUESTION_PROMPT + assert "diverse set of questions" in AGENTIC_QUESTION_PROMPT + assert "visibility/presence" in AGENTIC_QUESTION_PROMPT + + +def test_freeform_question_author_assigns_missing_ids() -> None: + class _ModelWithoutId: + def query(self, image: Image, prompt: str) -> str: + return '[{"question":"Is there a chair?","answer_contract":{"kind":"boolean"}}]' + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposals = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _ModelWithoutId())).propose( + image + ) + + assert proposals[0].id == "proposal-01" + + +def test_freeform_question_author_rejects_numeric_contracts() -> None: + class _NumericContractModel: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"numeric","unit":"m","tolerance":0.1}}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + try: + OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _NumericContractModel())).propose(image) + except ValueError as exc: + assert "unsupported answer contract" in str(exc) + else: + raise AssertionError("numeric answer contract was accepted") + + +def test_freeform_question_author_parses_deferred_height_contract() -> None: + class _HeightContractModel: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"deferred_height_choice","strategy":"height-window-v1"},' + '"object_queries":["chair"]}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposal = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _HeightContractModel())).propose( + image + )[0] + + assert proposal.answer_contract == DeferredHeightChoiceContract() + assert proposal.object_queries == ("chair",) + + +def test_freeform_question_author_normalizes_optional_query_hints() -> None: + class _ModelWithStringHints: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"choice","choices":["under 0.5 m","0.5-1.0 m"]},' + '"object_queries":"chair","tool_hints":null}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposal = OpenAIFreeformQuestionAuthor(cast("OpenAIVlModel", _ModelWithStringHints())).propose( + image + )[0] + + assert proposal.object_queries == ("chair",) + assert proposal.tool_hints == () + + +def test_freeform_question_author_ignores_malformed_optional_query_hints() -> None: + class _ModelWithMalformedHints: + def query(self, image: Image, prompt: str) -> str: + return ( + '[{"question":"How tall is the chair?","answer_contract":' + '{"kind":"choice","choices":["under 0.5 m","0.5-1.0 m"]},' + '"object_queries":{"query":"chair"}}]' + ) + + image = Image.from_numpy(np.zeros((1, 1, 3), dtype=np.uint8)) + + proposal = OpenAIFreeformQuestionAuthor( + cast("OpenAIVlModel", _ModelWithMalformedHints()) + ).propose(image)[0] + + assert proposal.object_queries == () + + +def test_local_tool_returns_geometry_and_evidence_ids() -> None: + registry = LocalOracleToolRegistry(_frame_primitives(_measurement_frame())) + + detection = json.loads(registry.detect_objects("chair")) + masks = json.loads(registry.segment_detection(detection["detections"][0]["detection_id"])) + payload = json.loads(registry.ground_mask(masks["mask_ids"][0])) + + assert "detection_id" not in detection + assert payload["object_id"].startswith("object:v1:") + assert payload["objects"][0]["evidence_id"] == f"grounding:v1:{payload['object_id']}" + assert registry.results[-1].version == "v1" + + +def test_ground_plane_estimator_fits_visible_lower_band() -> None: + fit = estimate_ground_plane(_measurement_frame()) + + assert fit.rejection_reason is None + assert fit.estimate is not None + assert fit.estimate.inlier_count >= 12 + assert fit.estimate.residual_m < 0.001 + assert np.allclose(fit.estimate.normal, (0.0, -1.0, 0.0), atol=0.001) + assert fit.estimate.offset_m == 1.0 + + +def test_ground_plane_tool_returns_quality_gated_rejection() -> None: + frame = _measurement_frame(np.asarray([[0.0, 1.0, 3.0]], dtype=np.float32)) + registry = LocalOracleToolRegistry(_frame_primitives(frame)) + + payload = json.loads(registry.fit_ground_plane()) + + assert payload["measurement"] is None + assert payload["rejection_reason"] == "insufficient_support" + assert "insufficient_ground_band_points" in payload["quality_flags"] + + +def test_height_tool_measures_visible_object_points_above_plane() -> None: + frame = _measurement_frame() + mask = np.zeros((100, 100), dtype=np.uint8) + projected = project_visible_points(frame) + for (x, y), point in zip(projected.pixels, projected.camera_points, strict=True): + if point[0] > 1.5: + mask[y, x] = 255 + registry = LocalOracleToolRegistry(_frame_primitives(frame, mask)) + + detection = json.loads(registry.detect_objects("chair")) + masks = json.loads(registry.segment_detection(detection["detections"][0]["detection_id"])) + grounded = json.loads(registry.ground_mask(masks["mask_ids"][0])) + plane = json.loads(registry.fit_ground_plane()) + payload = json.loads(registry.measure_height(grounded["object_id"], plane["plane_id"])) + + assert payload["measurement"]["unit"] == "m" + assert 0.8 < payload["measurement"]["value"] < 1.0 + assert payload["measurement"]["tolerance"] >= 0.05 + assert payload["objects"][0]["evidence_id"] == f"height:v1:{grounded['object_id']}" + assert "visible_point_cloud_height" in payload["quality_flags"] + + +def test_local_registry_exposes_geometry_tools() -> None: + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + + assert {tool.name for tool in registry.tools()} == { + "detect_objects", + "segment_detection", + "ground_mask", + "fit_ground_plane", + "get_object_pose", + "fit_object_surface_plane", + "fit_mask_surrounding_plane", + "measure_object_pair_distance", + "measure_relative_plane_angle", + "measure_object_plane_relation", + "measure_aperture_geometry", + "measure_forward_corridor", + "measure_height", + } + + +def test_local_registry_exposes_forward_corridor_metrics(monkeypatch: Any) -> None: + primitives = _frame_primitives(_measurement_frame()) + registry = LocalOracleToolRegistry(primitives) + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + registry._planes["ground"] = plane + registry._ground_planes.add("ground") + monkeypatch.setattr( + primitives, + "measure_forward_corridor", + lambda accepted_plane: ForwardCorridorMeasurement((3, 4, 5), 6, 24), + ) + + payload = json.loads(registry.measure_forward_corridor("ground")) + rejected = json.loads(registry.measure_forward_corridor("unknown")) + + assert payload["metrics"] == { + "ground_band_1_count": 3.0, + "ground_band_2_count": 4.0, + "ground_band_3_count": 5.0, + "elevated_obstacle_count": 6.0, + "corridor_point_count": 24.0, + } + assert payload["objects"][0]["evidence_id"] == "forward-corridor:v1:synthetic" + assert rejected["rejection_reason"] == "unknown_ground_plane_id" + + +def test_height_choice_window_is_local_and_deterministic() -> None: + choices, answer = height_choice_window(0.42) + + assert choices == ( + "under 0.2 m", + "0.2-0.6 m", + "0.6-1.0 m", + "over 1.0 m", + ) + assert answer == "0.2-0.6 m" + assert height_choice_window(3.0)[1] == "over 2.0 m" + + +def test_count_and_camera_range_choices_are_fixed_and_non_overlapping() -> None: + assert [count_choice(value) for value in (1, 2, 3, 4, 5, 7, 8)] == [ + "1-2", + "1-2", + "3-4", + "3-4", + "5-7", + "5-7", + "8+", + ] + assert [camera_range_choice(value) for value in (0.99, 1.0, 2.0, 4.0)] == [ + "under 1 m", + "1 to under 2 m", + "2 to under 4 m", + "4 m or more", + ] + + +def test_local_registry_buckets_count_range_and_pairwise_side() -> None: + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + registry._objects["synthetic-chair-0"] = GroundedObject( + "synthetic-chair-0", "chair", 4, 1.0, "left" + ) + left = GroundedObject("left", "chair", 8, 1.0, "left") + right = GroundedObject("right", "chair", 8, 2.0, "right") + registry._objects = {left.id: left, right.id: right} + + count = json.loads(registry.count_grounded_objects([left.id, right.id])) + camera_range = json.loads(registry.bucket_camera_range(right.id)) + side = json.loads(registry.compare_nearest_by_side([left.id, right.id])) + + assert count["choice"] == "1-2" + assert camera_range["choice"] == "2 to under 4 m" + assert side["choice"] == "left" + + +def test_local_registry_compares_pairwise_relation_and_height(monkeypatch: Any) -> None: + primitives = _frame_primitives(_measurement_frame()) + registry = LocalOracleToolRegistry(primitives) + chair = GroundedObject("chair", "chair", 8, 1.0, "left") + table = GroundedObject("table", "table", 8, 2.0, "right") + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + registry._objects = {chair.id: chair, table.id: table} + registry._planes = {"plane": plane} + monkeypatch.setattr( + primitives, + "classify_horizontal_relation", + lambda first, second: HorizontalRelationResult("left", ("camera_frame_support_centroids",)), + ) + monkeypatch.setattr( + primitives, + "measure_height", + lambda item, accepted_plane: HeightMeasurementResult( + item, + accepted_plane, + OracleMeasurement(0.8 if item == chair else 0.5, "m", 0.05, (), ()), + (), + ), + ) + + relation = json.loads(registry.compare_left_right(chair.id, table.id)) + height = json.loads(registry.compare_heights(chair.id, table.id, "plane")) + + assert relation["choice"] == "left" + assert height["choice"] == "chair" + + +def test_local_registry_verifies_support_and_measures_opening(monkeypatch: Any) -> None: + primitives = _frame_primitives(_measurement_frame()) + registry = LocalOracleToolRegistry(primitives) + box = GroundedObject("box", "box", 8, 1.0, "left") + table = GroundedObject("table", "table", 8, 2.0, "right") + registry._objects = {box.id: box, table.id: table} + registry._masks = {"mask": "doorway"} + measurement = OracleMeasurement(0.9, "m", 0.05, (), ()) + monkeypatch.setattr( + primitives, + "classify_object_on_support", + lambda item, support: ObjectOnSupportResult( + item, support, ("contact_band_supported",), answer="yes" + ), + ) + monkeypatch.setattr( + primitives, + "measure_opening_width", + lambda query: OpeningWidthResult(measurement, ("stable_opening_scanlines",)), + ) + + support = json.loads(registry.classify_object_on_support(box.id, table.id)) + opening = json.loads(registry.measure_opening_width("mask")) + + assert support["choice"] == "yes" + assert opening["measurement"]["value"] == 0.9 + + +def test_opening_width_uses_wall_geometry_outside_the_aperture() -> None: + image = Image.from_numpy(np.zeros((100, 100, 3), dtype=np.uint8)) + camera_info = CameraInfo.from_intrinsics(50.0, 50.0, 50.0, 50.0, 100, 100) + opening = np.zeros((100, 100), dtype=np.uint8) + opening[35:61, 40:61] = 255 + points: list[list[float]] = [] + for y in range(20, 61, 4): + for x in range(28, 73, 4): + if opening[y, x]: + continue + points.append([(x - 50) / 50 * 5.0, (y - 50) / 50 * 5.0, 5.0]) + for y in (65, 75, 85, 95): + for x in range(0, 100, 10): + depth = 1.0 / ((y - 50) / 50) + points.append([(x - 50) / 50 * depth, 1.0, depth]) + frame = CalibratedFrame( + "opening", + image, + PointCloud2.from_numpy(np.asarray(points, dtype=np.float32)), + camera_info, + Transform.identity(), + True, + ) + ground = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + + result = measure_opening_width(frame, opening, ground) + + assert result.rejection_reason is None + assert result.width_m is not None + assert 1.9 < result.width_m < 2.1 + + +def test_relative_plane_angle_measurement_is_unsigned() -> None: + door = GroundPlaneEstimate((1.0, 0.0, 0.0), 0.0, 20, 20, 0.01) + closed = GroundPlaneEstimate((1.0, 0.0, 0.0), 0.0, 20, 20, 0.01) + open_door = GroundPlaneEstimate((0.0, 0.0, 1.0), 0.0, 20, 20, 0.01) + ajar_door = GroundPlaneEstimate((0.95, 0.0, 0.31), 0.0, 20, 20, 0.01) + + assert measure_relative_plane_angle(door, closed).value == 0.0 + assert measure_relative_plane_angle(door, open_door).value == 90.0 + assert 17.0 < measure_relative_plane_angle(door, ajar_door).value < 19.0 + + +def test_closest_object_uses_point_cloud_centroids_and_rejects_ties(monkeypatch: Any) -> None: + target = GroundedObject("target", "chair", 8, 1.0, "left") + close = GroundedObject("close", "table", 8, 2.0, "center") + far = GroundedObject("far", "lamp", 8, 3.0, "right") + primitives = _frame_primitives(_measurement_frame()) + centers = { + "target": np.zeros((6, 3)), + "close": np.tile((1.0, 0.0, 0.0), (6, 1)), + "far": np.tile((2.0, 0.0, 0.0), (6, 1)), + } + monkeypatch.setattr(primitives, "_object_points", lambda item: centers[item.id]) + + selected = primitives.select_closest_object(target, [close, far]) + + assert selected.object == close + assert selected.distance_m == 1.0 + monkeypatch.setattr( + primitives, + "_object_points", + lambda item: np.tile((1.0, 0.0, 0.0), (6, 1)) if item.id != "target" else centers["target"], + ) + assert ( + primitives.select_closest_object(target, [close, far]).rejection_reason + == "ambiguous_object_proximity" + ) + + +def test_horizontal_relation_uses_camera_frame_support_centroids(monkeypatch: Any) -> None: + left = GroundedObject("left", "chair", 8, 1.0, "left") + right = GroundedObject("right", "table", 8, 2.0, "right") + primitives = _frame_primitives(_measurement_frame()) + centers = { + "left": np.tile((-0.3, 0.0, 1.0), (6, 1)), + "right": np.tile((0.3, 0.0, 1.0), (6, 1)), + } + monkeypatch.setattr(primitives, "_object_points", lambda item: centers[item.id]) + + assert primitives.classify_horizontal_relation(left, right).relation == "left" + monkeypatch.setattr(primitives, "_object_points", lambda item: np.zeros((6, 3))) + assert ( + primitives.classify_horizontal_relation(left, right).rejection_reason + == "ambiguous_horizontal_relation" + ) + + +def test_object_on_support_keeps_clearly_separated_objects_as_no(monkeypatch: Any) -> None: + box = GroundedObject("box", "box", 8, 1.0, "left") + support = GroundedObject("table", "table", 8, 2.0, "right") + plane = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + primitives = _frame_primitives(_measurement_frame()) + points = { + box.id: np.tile((2.0, 0.5, 4.0), (6, 1)), + support.id: np.tile((0.0, 1.0, 4.0), (6, 1)), + } + monkeypatch.setattr(primitives, "_object_points", lambda item: points[item.id]) + monkeypatch.setattr(primitives, "fit_ground_plane", lambda: PlaneFitResult(plane, ())) + monkeypatch.setattr( + "dimos.benchmark.vqa.generation.primitives.frame.fit_surface_plane", + lambda support_points: PlaneFitResult(plane, ("surface_plane_accepted",)), + ) + + result = primitives.classify_object_on_support(box, support) + + assert result.answer == "no" + assert "objects_separated_in_plane" in result.quality_flags + + +def test_forward_corridor_requires_ground_support_and_detects_obstacles() -> None: + ground = GroundPlaneEstimate((0.0, -1.0, 0.0), 1.0, 20, 20, 0.01) + floor = np.asarray( + [ + [0.0, 1.0, depth] + for depth in (0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2, 2.4, 2.6, 2.8) + ] + ) + + assert classify_forward_corridor(floor, ground)[0] == "clear" + obstacle = np.repeat(np.asarray([[0.0, 0.5, 1.0]]), 4, axis=0) + assert classify_forward_corridor(np.vstack((floor, obstacle)), ground)[0] == "blocked" + + +def test_oracle_validates_evidence_and_answer_contract() -> None: + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract()) + result = OracleToolResult( + "ground", "chair", (OracleEvidence("e1", "v1", "o1", "chair", 1.0, "left", 3),) + ) + + assert validate_oracle_answer(proposal, "yes", ["e1"], (result,)) == "yes" + try: + validate_oracle_answer(proposal, "maybe", ["e1"], (result,)) + except ValueError as exc: + assert "boolean" in str(exc) + else: + raise AssertionError("invalid boolean answer was accepted") + try: + validate_oracle_answer(proposal, "yes", ["unknown"], (result,)) + except ValueError as exc: + assert "unknown evidence" in str(exc) + else: + raise AssertionError("unknown evidence was accepted") + + +def test_oracle_derives_deferred_height_answer_from_cited_measurement() -> None: + proposal = QuestionProposal( + "q", "How tall is the chair?", DeferredHeightChoiceContract(), ("chair",) + ) + measurement = OracleMeasurement(0.42, "m", 0.05, (), ()) + evidence = OracleEvidence("height-1", "v1", "chair-1", "chair", 1.0, "left", 8, measurement) + result = OracleToolResult( + "measure_height", + "chair", + (evidence,), + measurement=measurement, + ) + + assert validate_oracle_answer(proposal, None, ["height-1"], (result,)) == "0.2-0.6 m" + try: + validate_oracle_answer(proposal, None, [], (result,)) + except ValueError as exc: + assert "evidence_ids" in str(exc) + else: + raise AssertionError("deferred height answer without evidence was accepted") + + +def test_private_oracle_runs_direct_structured_tool() -> None: + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + registry._objects["synthetic-chair-0"] = GroundedObject( + "synthetic-chair-0", "chair", 4, 1.0, "left" + ) + validator = _SemanticValidator(SemanticEvidenceValidation(True, "chair grounding supports yes")) + + result = PrivateToolCallingOracle( + cast("Any", _BoundModel()), semantic_validator=validator + ).answer(proposal, registry) + + assert result.answer == "yes" + assert result.evidence_ids == ("grounding:v1:synthetic-chair-0",) + assert validator.calls[0][2][-1].evidence[0].id == "grounding:v1:synthetic-chair-0" + assert result.trace[-1].detail == "accepted:chair grounding supports yes" + + +def test_private_oracle_rejects_unsupported_measurement_claim() -> None: + class _ChoiceModel(_BoundModel): + def invoke(self, messages: Any) -> AIMessage: + self._calls += 1 + if self._calls == 1: + return AIMessage( + content="", + tool_calls=[ + { + "name": "detect_objects", + "args": {"query": "chair"}, + "id": "call-1", + } + ], + ) + return AIMessage( + content='{"answer":"0.5-1.0 m","evidence_ids":["grounding:v1:synthetic-chair-0"]}' + ) + + proposal = QuestionProposal( + "q", "How tall is the chair?", ChoiceAnswerContract(("under 0.5 m", "0.5-1.0 m")) + ) + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + validator = _SemanticValidator( + SemanticEvidenceValidation(False, "range and side do not measure height") + ) + + result = PrivateToolCallingOracle( + cast("Any", _ChoiceModel()), semantic_validator=validator + ).answer(proposal, registry) + + assert result.reason == "invalid_final_answer:answer cites unknown evidence" + + +def test_private_oracle_allows_explicit_rejection_without_answer_resolution() -> None: + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) + registry = LocalOracleToolRegistry(cast("Any", _Grounding())) + validator = _SemanticValidator(SemanticEvidenceValidation(True, "should not be called")) + model = _ScriptedModel( + [ + AIMessage( + content="", + tool_calls=[{"name": "detect_objects", "args": {"query": "chair"}, "id": "call-1"}], + ), + AIMessage( + content='{"status":"rejected","reason":"chair cannot be verified","evidence_ids":[]}' + ), + ] + ) + + result = PrivateToolCallingOracle(cast("Any", model), semantic_validator=validator).answer( + proposal, registry + ) + + assert isinstance(result, RejectedOracleResult) + assert result.reason == "chair cannot be verified" + assert result.tool_results == registry.results + assert len(result.trace) == 1 + assert result.trace[0].operation == "tool" + assert validator.calls == [] + + +def test_private_oracle_rejects_malformed_explicit_rejection() -> None: + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract()) + validator = _SemanticValidator(SemanticEvidenceValidation(True, "should not be called")) + model = _ScriptedModel([AIMessage(content='{"status":"rejected","reason":""}')]) + + result = PrivateToolCallingOracle(cast("Any", model), semantic_validator=validator).answer( + proposal, LocalOracleToolRegistry(cast("Any", _Grounding())) + ) + + assert isinstance(result, RejectedOracleResult) + assert result.reason.startswith("invalid_final_answer:") + assert validator.calls == [] + + +def test_agentic_oracle_never_uses_legacy_answer_program() -> None: + class _GroundingWithoutLegacyAnswer(_Grounding): + def answer(self, frame: Any, intent: Any) -> None: + raise AssertionError("agentic oracle must not call legacy answer") + + proposal = QuestionProposal("q", "Is there a chair?", BooleanAnswerContract(), ("chair",)) + registry = LocalOracleToolRegistry(cast("Any", _GroundingWithoutLegacyAnswer())) + registry._objects["synthetic-chair-0"] = GroundedObject( + "synthetic-chair-0", "chair", 4, 1.0, "left" + ) + validator = _SemanticValidator(SemanticEvidenceValidation(True, "chair grounding supports yes")) + + result = PrivateToolCallingOracle( + cast("Any", _BoundModel()), semantic_validator=validator + ).answer(proposal, registry) + + assert result.answer == "yes" diff --git a/dimos/benchmark/vqa/generation/test_selection.py b/dimos/benchmark/vqa/generation/test_selection.py new file mode 100644 index 0000000000..2218a72df8 --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_selection.py @@ -0,0 +1,15 @@ +# Copyright 2026 Dimensional Inc. + +from dimos.benchmark.vqa.generation.primitives.selection import select_nearest_object +from dimos.benchmark.vqa.models import GroundedObject + + +def test_select_nearest_object_optionally_restricts_to_image_side() -> None: + objects = [ + GroundedObject("left", "chair", 3, 2.0, "left"), + GroundedObject("right", "chair", 3, 1.0, "right"), + ] + + assert select_nearest_object(objects).id == "right" + assert select_nearest_object(objects, "left").id == "left" + assert select_nearest_object(objects, "center") is None diff --git a/dimos/benchmark/vqa/generation/test_single_frame.py b/dimos/benchmark/vqa/generation/test_single_frame.py new file mode 100644 index 0000000000..be6ef78c29 --- /dev/null +++ b/dimos/benchmark/vqa/generation/test_single_frame.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import numpy as np + +from dimos.benchmark.vqa.generation.pipeline import generate_ground_truth +from dimos.benchmark.vqa.models import CalibratedFrame +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + + +class _Detector: + def __init__(self, image: Image, detection: Detection2DSeg) -> None: + self._image = image + self._detection = detection + + def detect(self, image: Image, query: str) -> ImageDetections2D: + assert image is self._image + return ImageDetections2D(image, [self._detection] if query == "chair" else []) + + +class _Segmenter: + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: + return detections + + +def test_single_frame_ground_truth_generates_multiple_choice_cases() -> None: + image = Image.from_numpy(np.zeros((6, 6, 3), dtype=np.uint8)) + frame = CalibratedFrame( + id="frame-1", + image=image, + pointcloud=PointCloud2.from_numpy( + np.array([[0.0, 0.0, 1.0], [0.4, 0.0, 1.0], [-0.4, 0.0, 1.0]], dtype=np.float32) + ), + camera_info=CameraInfo.from_intrinsics(3.0, 3.0, 3.0, 3.0, 6, 6), + pointcloud_to_camera=Transform.identity(), + image_is_rectified=True, + ) + mask = np.full((6, 6), 255, dtype=np.uint8) + detection = Detection2DSeg((0.0, 0.0, 5.0, 5.0), 0, -1, 1.0, "chair", 0.0, image, mask) + + examples = generate_ground_truth( + frame, ["chair", "table"], _Detector(image, detection), _Segmenter() + ) + + assert {example.expected_answer for example in examples} == {"yes", "no", "center"} + assert all(example.expected_answer in example.allowed_answers for example in examples) diff --git a/dimos/benchmark/vqa/models.py b/dimos/benchmark/vqa/models.py new file mode 100644 index 0000000000..6ae4d4f231 --- /dev/null +++ b/dimos/benchmark/vqa/models.py @@ -0,0 +1,280 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contracts shared by the single-frame perception VQA pipeline.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal, Protocol + +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D + + +@dataclass(frozen=True) +class CalibratedFrame: + """One self-contained image and point-cloud pair for VQA generation.""" + + id: str + image: Image + pointcloud: PointCloud2 + camera_info: CameraInfo + pointcloud_to_camera: Transform + image_is_rectified: bool + original_image: Image | None = None + + +@dataclass(frozen=True) +class ProjectionConfig: + """Controls static pinhole projection of a frame's point cloud.""" + + min_depth_m: float = 0.01 + + +@dataclass(frozen=True) +class GroundingConfig: + """Quality thresholds for accepting an image mask as a grounded object.""" + + min_mask_area_px: int = 128 + min_foreground_points: int = 3 + + +@dataclass(frozen=True) +class ProjectedPoints: + """Visible point-cloud samples represented in the camera image.""" + + camera_points: list[tuple[float, float, float]] + pixels: list[tuple[int, int]] + source_indices: list[int] + + +@dataclass(frozen=True) +class GroundedObject: + """One semantic object with foreground point-cloud support.""" + + id: str + label: str + point_count: int + range_m: float + horizontal_direction: str + + +@dataclass(frozen=True) +class VqaExample: + """A closed-answer question generated from grounded objects.""" + + id: str + question: str + expected_answer: str + answer_type: str + object_ids: tuple[str, ...] + allowed_answers: tuple[str, ...] = () + + +QuestionKind = Literal[ + "presence", + "horizontal_direction", + "within_distance", + "visible_count", + "camera_range", + "compare_nearest_by_side", + "compare_left_right", + "compare_height", + "object_on_support", + "opening_width", + "door_state", + "closest_object", + "forward_path", +] + + +@dataclass(frozen=True) +class QuestionIntent: + """A constrained question proposed from an image.""" + + kind: QuestionKind + object_query: str + threshold_m: float | None = None + candidate_queries: tuple[str, ...] = () + comparison_query: str | None = None + + +@dataclass(frozen=True) +class ToolTrace: + """One perception operation used to establish a ground-truth answer.""" + + tool: str + detail: str + + +@dataclass(frozen=True) +class GroundTruthResult: + """An answered or rejected question with private perception evidence.""" + + intent: QuestionIntent + question: VqaExample + status: Literal["answered", "rejected"] + answer: str | None + reason: str | None + evidence: tuple[GroundedObject, ...] + trace: tuple[ToolTrace, ...] + + +@dataclass(frozen=True) +class BooleanAnswerContract: + """A yes/no answer required from a private oracle.""" + + kind: Literal["boolean"] = "boolean" + + +@dataclass(frozen=True) +class ChoiceAnswerContract: + """An answer selected exactly from the supplied choices.""" + + choices: tuple[str, ...] + kind: Literal["choice"] = "choice" + + +@dataclass(frozen=True) +class DeferredHeightChoiceContract: + """A height question whose public choices follow a private measurement.""" + + strategy: Literal["height-window-v1"] = "height-window-v1" + kind: Literal["deferred_height_choice"] = "deferred_height_choice" + + +AnswerContract = BooleanAnswerContract | ChoiceAnswerContract | DeferredHeightChoiceContract +ResolvedAnswerContract = BooleanAnswerContract | ChoiceAnswerContract + + +@dataclass(frozen=True) +class QuestionProposal: + """Public image-only question frozen before private oracle execution.""" + + id: str + question: str + answer_contract: AnswerContract + object_queries: tuple[str, ...] = () + tool_hints: tuple[str, ...] = () + + +@dataclass(frozen=True) +class OracleEvidence: + """A private evidence item emitted by a registered local tool.""" + + id: str + version: str + object_id: str + label: str + range_m: float + side: str + point_count: int + measurement: OracleMeasurement | None = None + + +@dataclass(frozen=True) +class OracleMeasurement: + """A private scalar measurement with its uncertainty and provenance.""" + + value: float + unit: str + tolerance: float + quality_flags: tuple[str, ...] + provenance_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class GroundPlaneEstimate: + """A robust ground-plane fit in frozen camera coordinates.""" + + normal: tuple[float, float, float] + offset_m: float + sample_count: int + inlier_count: int + residual_m: float + + +@dataclass(frozen=True) +class OracleToolResult: + """Structured result and evidence IDs from one local tool invocation.""" + + tool: str + query: str + evidence: tuple[OracleEvidence, ...] + version: str = "v1" + measurement: OracleMeasurement | None = None + choice: str | None = None + choices: tuple[str, ...] = () + plane: GroundPlaneEstimate | None = None + quality_flags: tuple[str, ...] = () + rejection_reason: str | None = None + metrics: tuple[tuple[str, float], ...] = () + + +@dataclass(frozen=True) +class OracleTrace: + """Audit record for a private oracle model or tool operation.""" + + operation: str + detail: str + + +@dataclass(frozen=True) +class AcceptedOracleResult: + """Validated private answer for a frozen public proposal.""" + + proposal: QuestionProposal + answer: str + answer_contract: ResolvedAnswerContract + evidence_ids: tuple[str, ...] + tool_results: tuple[OracleToolResult, ...] + trace: tuple[OracleTrace, ...] + + +@dataclass(frozen=True) +class RejectedOracleResult: + """Private oracle attempt that cannot be safely exported as a case.""" + + proposal: QuestionProposal + reason: str + tool_results: tuple[OracleToolResult, ...] + trace: tuple[OracleTrace, ...] + + +class ObjectDetector(Protocol): + """Produces object-query detections from one image.""" + + def detect(self, image: Image, query: str) -> ImageDetections2D: ... + + +class ObjectSegmenter(Protocol): + """Refines object detections into foreground masks in one image.""" + + def segment(self, detections: ImageDetections2D) -> ImageDetections2D: ... + + +class ObjectPointLocalizer(Protocol): + """Locates a queried object with positive image points.""" + + def locate(self, image: Image, query: str) -> ImageDetections2D: ... + + +class PointObjectSegmenter(Protocol): + """Creates foreground masks from positive image-point prompts.""" + + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: ... diff --git a/dimos/benchmark/vqa/test_evaluation.py b/dimos/benchmark/vqa/test_evaluation.py new file mode 100644 index 0000000000..2fab9b85c8 --- /dev/null +++ b/dimos/benchmark/vqa/test_evaluation.py @@ -0,0 +1,47 @@ +# Copyright 2026 Dimensional Inc. + +import json +from pathlib import Path +from typing import Any, cast + +import cv2 +import numpy as np + +from dimos.benchmark.evaluation.models import ArtifactNativeResult +from dimos.benchmark.evaluation.protocol import EvaluationContext +from dimos.benchmark.vqa.evaluation import MultipleChoiceVqaEvaluation, VqaEvaluationConfig + + +class _VisionModel: + def query(self, image: Any, prompt: str) -> str: + assert "Choices: left, right." in prompt + return "ANSWER: left" + + +def test_vqa_evaluation_uses_only_public_case_image_and_private_label(tmp_path: Path) -> None: + dataset = tmp_path / "dataset" + dataset.mkdir() + assert cv2.imwrite(str(dataset / "image.jpg"), np.zeros((1, 1, 3), dtype=np.uint8)) + (dataset / "cases.jsonl").write_text( + json.dumps( + { + "id": "case-1", + "image": "image.jpg", + "question": "Which side?", + "choices": ["left", "right"], + } + ) + + "\n" + ) + (dataset / "labels.jsonl").write_text(json.dumps({"id": "case-1", "answer": "left"}) + "\n") + workspace = tmp_path / "workspace" + workspace.mkdir() + + report = MultipleChoiceVqaEvaluation(lambda _: cast("Any", _VisionModel())).run( + VqaEvaluationConfig(dataset=str(dataset)), + EvaluationContext("run", tmp_path, workspace, cast("Any", None), None), + ) + + assert report.summary[1].value == 1 + assert isinstance(report.native_result, ArtifactNativeResult) + assert json.loads((workspace / "vqa-results.json").read_text())[0]["passed"] is True diff --git a/dimos/cli/dimos.py b/dimos/cli/dimos.py index 0f50caaaea..9137621c5f 100644 --- a/dimos/cli/dimos.py +++ b/dimos/cli/dimos.py @@ -53,6 +53,7 @@ from dimos.cli.eval import app as eval_app from dimos.cli.hardware_cli import app as hardware_app from dimos.cli.shell import shell +from dimos.cli.vqa import app as vqa_app from dimos.constants import CONFIG_DIR, LOG_DIR from dimos.core.daemon import daemonize, install_signal_handlers from dimos.core.global_config import GlobalConfig, global_config @@ -77,6 +78,7 @@ help="Dimensional CLI", no_args_is_help=True, ) +main.add_typer(vqa_app, name="vqa") load_dotenv() diff --git a/dimos/cli/test_vqa.py b/dimos/cli/test_vqa.py new file mode 100644 index 0000000000..abf7e5bf49 --- /dev/null +++ b/dimos/cli/test_vqa.py @@ -0,0 +1,70 @@ +# Copyright 2026 Dimensional Inc. + +import json +from pathlib import Path + +from typer.testing import CliRunner + +from dimos.benchmark.vqa.generation.specification import VqaGenerationSpecification +from dimos.cli import vqa + + +def test_vqa_generation_cli_has_no_explicit_query_or_model_options() -> None: + output = CliRunner().invoke(vqa.app, ["generate", "--help"]).output + + assert "--query" not in output + assert "--propose-questions" not in output + assert "--question-model" not in output + assert "--oracle-model" not in output + assert "--spec" in output + + +def test_generation_spec_resolves_the_same_options_as_the_cli(tmp_path: Path) -> None: + spec = tmp_path / "generation.json" + spec.write_text( + json.dumps( + { + "recording": "go2_bigoffice.db", + "start_index": 10, + "stop_index": 40, + "stride": 5, + "question_mode": "agentic", + "grounding": {"min_mask_area_px": 256, "min_foreground_points": 4}, + "output": "/tmp/vqa", + } + ) + ) + + generation = vqa._resolve_generation_spec(spec, None, None, None, None, None, None, None, None) + + assert generation.recording == "go2_bigoffice.db" + assert generation.question_mode == "agentic" + assert generation.grounding.min_mask_area_px == 256 + assert generation.output == "/tmp/vqa" + + +def test_generation_spec_rejects_mixed_cli_options(tmp_path: Path) -> None: + spec = tmp_path / "generation.json" + spec.write_text('{"recording":"go2.db","stop_index":10}') + + result = CliRunner().invoke( + vqa.app, + ["generate", "--spec", str(spec), "--recording", "other.db"], + ) + + assert result.exit_code != 0 + assert "cannot be combined" in result.output + + +def test_generation_run_records_resolved_request(tmp_path: Path) -> None: + vqa._write_generation_run( + tmp_path, + VqaGenerationSpecification(recording="go2.db", stop_index=10), + {"frame_count": 2, "accepted_question_count": 3, "rejected_question_count": 1}, + ) + + payload = json.loads((tmp_path / "run.json").read_text()) + + assert payload["generation"]["recording"] == "go2.db" + assert payload["generation"]["output"] == str(tmp_path) + assert payload["summary"]["accepted_question_count"] == 3 diff --git a/dimos/cli/vqa.py b/dimos/cli/vqa.py new file mode 100644 index 0000000000..3d0260dc12 --- /dev/null +++ b/dimos/cli/vqa.py @@ -0,0 +1,373 @@ +# Copyright 2026 Dimensional Inc. +"""Single-frame point-cloud-grounded VQA commands.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import cast + +import typer + +from dimos.benchmark.vqa.generation.adapters import ( + EdgeTamObjectSegmenter, + MoondreamObjectDetector, +) +from dimos.benchmark.vqa.generation.dataset import write_dataset_manifest, write_frame_record +from dimos.benchmark.vqa.generation.ground_truth_generator import VqaGroundTruthGenerator +from dimos.benchmark.vqa.generation.oracle import create_openai_oracle +from dimos.benchmark.vqa.generation.oracle_tools import LocalOracleToolRegistry +from dimos.benchmark.vqa.generation.primitives.frame import FramePerceptionPrimitives +from dimos.benchmark.vqa.generation.question_agent import ( + OpenAIFreeformQuestionAuthor, + OpenAIQuestionAgent, +) +from dimos.benchmark.vqa.generation.recording import load_go2_frame +from dimos.benchmark.vqa.generation.specification import VqaGenerationSpecification +from dimos.benchmark.vqa.models import ( + AcceptedOracleResult, + CalibratedFrame, + GroundingConfig, + GroundTruthResult, + QuestionIntent, + QuestionProposal, + RejectedOracleResult, +) +from dimos.constants import STATE_DIR +from dimos.models.base import default_local_model_device +from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter +from dimos.models.vl.moondream import MoondreamVlModel +from dimos.models.vl.openai import OpenAIVlModel +from dimos.utils.data import resolve_named_path + +app = typer.Typer(help="Generate point-cloud-grounded VQA benchmark examples") +QUESTION_MODEL = "gpt-4o-mini" +ORACLE_MODEL = "gpt-4o-mini" + + +@app.command("single-frame") +def single_frame( + recording: str = typer.Option(..., "--recording"), + frame_index: int = typer.Option(0, "--frame-index"), + question_mode: str = typer.Option("constrained", "--question-mode"), + min_mask_area_px: int = typer.Option(128, "--min-mask-area-px"), + min_foreground_points: int = typer.Option(3, "--min-foreground-points"), + output: Path | None = typer.Option(None, "--output"), +) -> None: + """Generate private-grounded questions for one Go2 recording frame.""" + output = output or ( + STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frame-{frame_index:06d}" + ) + if output.exists(): + raise typer.BadParameter("output must not already exist") + _validate_question_mode(question_mode) + _require_openai_for_question_author() + _require_edgetam_cuda() + typer.echo(f"Loading frame {frame_index} from {recording}") + frame = load_go2_frame(str(resolve_named_path(recording, ".db")), frame_index) + model = MoondreamVlModel() + typer.echo("Loading private MoonDream model") + model.start() + question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=QUESTION_MODEL)) + try: + typer.echo(f"Proposing questions with {QUESTION_MODEL}") + intents: list[QuestionIntent] | list[QuestionProposal] = ( + OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=QUESTION_MODEL)).propose( + frame.image + ) + if question_mode == "agentic" + else question_agent.propose(frame.image) + ) + typer.echo(f"Grounding {len(intents)} questions for frame {frame_index}") + primitives = FramePerceptionPrimitives( + frame, + detector := MoondreamObjectDetector(model), + segmenter := EdgeTamObjectSegmenter(EdgeTAMImageSegmenter()), + localizer=detector, + point_segmenter=segmenter, + config=GroundingConfig( + min_mask_area_px=min_mask_area_px, + min_foreground_points=min_foreground_points, + ), + ) + ground_truth = VqaGroundTruthGenerator(primitives) + results: list[GroundTruthResult] | list[AcceptedOracleResult | RejectedOracleResult] = ( + _answer_agentic(ground_truth, frame, cast("list[QuestionProposal]", intents)) + if question_mode == "agentic" + else _answer_intents( + ground_truth, frame, cast("list[QuestionIntent]", intents), f"Frame {frame_index}" + ) + ) + examples = [ + result + for result in results + if isinstance(result, AcceptedOracleResult) + or (isinstance(result, GroundTruthResult) and result.status == "answered") + ] + finally: + model.stop() + write_frame_record( + output, + frame, + recording, + frame_index, + cast("list[QuestionIntent | QuestionProposal]", intents), + cast("list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult]", results), + { + "question_source": "agentic_image_author" + if question_mode == "agentic" + else "openai_image_agent", + "question_model": QUESTION_MODEL, + "oracle_model": ORACLE_MODEL if question_mode == "agentic" else None, + "grounding": { + "min_mask_area_px": min_mask_area_px, + "min_foreground_points": min_foreground_points, + }, + }, + ) + typer.echo(f"Wrote {len(examples)} examples to {output}") + + +@app.command("generate") +def generate( + recording: str | None = typer.Option(None, "--recording"), + start_index: int | None = typer.Option(None, "--start-index"), + stop_index: int | None = typer.Option(None, "--stop-index"), + stride: int | None = typer.Option(None, "--stride"), + question_mode: str | None = typer.Option(None, "--question-mode"), + min_mask_area_px: int | None = typer.Option(None, "--min-mask-area-px"), + min_foreground_points: int | None = typer.Option(None, "--min-foreground-points"), + output: Path | None = typer.Option(None, "--output"), + spec: Path | None = typer.Option(None, "--spec", exists=True, dir_okay=False, readable=True), +) -> None: + """Generate a resumable VQA dataset from sampled Go2 recording frames.""" + generation = _resolve_generation_spec( + spec, + recording, + start_index, + stop_index, + stride, + question_mode, + min_mask_area_px, + min_foreground_points, + output, + ) + if generation.stop_index <= generation.start_index: + raise typer.BadParameter("provide valid frame bounds") + recording = generation.recording + start_index = generation.start_index + stop_index = generation.stop_index + stride = generation.stride + question_mode = generation.question_mode + min_mask_area_px = generation.grounding.min_mask_area_px + min_foreground_points = generation.grounding.min_foreground_points + output = ( + Path(generation.output).expanduser() + if generation.output is not None + else STATE_DIR / "datasets" / "vqa" / f"{Path(recording).stem}-frames" + ) + _require_openai_for_question_author() + output.mkdir(parents=True, exist_ok=True) + _require_edgetam_cuda() + frame_indices = range(start_index, stop_index, stride) + typer.echo(f"Generating {len(frame_indices)} sampled frames from {recording} into {output}") + model = MoondreamVlModel() + typer.echo("Loading private MoonDream model") + model.start() + try: + detector = MoondreamObjectDetector(model) + segmenter = EdgeTamObjectSegmenter(EdgeTAMImageSegmenter()) + question_agent = OpenAIQuestionAgent(OpenAIVlModel(model_name=QUESTION_MODEL)) + for frame_number, frame_index in enumerate(frame_indices, start=1): + frame_output = output / f"frame-{frame_index:06d}" + if (frame_output / "frame.json").is_file(): + typer.echo( + f"Skipping completed frame {frame_number}/{len(frame_indices)}: {frame_index}" + ) + continue + typer.echo( + f"Frame {frame_number}/{len(frame_indices)}: loading recording index {frame_index}" + ) + frame = load_go2_frame(str(resolve_named_path(recording, ".db")), frame_index) + typer.echo(f"Frame {frame_index}: proposing questions with {QUESTION_MODEL}") + intents: list[QuestionIntent] | list[QuestionProposal] = ( + OpenAIFreeformQuestionAuthor(OpenAIVlModel(model_name=QUESTION_MODEL)).propose( + frame.image + ) + if question_mode == "agentic" + else question_agent.propose(frame.image) + ) + typer.echo(f"Frame {frame_index}: grounding {len(intents)} questions") + primitives = FramePerceptionPrimitives( + frame, + detector, + segmenter, + localizer=detector, + point_segmenter=segmenter, + config=GroundingConfig( + min_mask_area_px=min_mask_area_px, min_foreground_points=min_foreground_points + ), + ) + ground_truth = VqaGroundTruthGenerator(primitives) + results: list[GroundTruthResult] | list[AcceptedOracleResult | RejectedOracleResult] = ( + _answer_agentic(ground_truth, frame, cast("list[QuestionProposal]", intents)) + if question_mode == "agentic" + else _answer_intents( + ground_truth, + frame, + cast("list[QuestionIntent]", intents), + f"Frame {frame_index}", + ) + ) + write_frame_record( + frame_output, + frame, + recording, + frame_index, + cast("list[QuestionIntent | QuestionProposal]", intents), + cast( + "list[GroundTruthResult | AcceptedOracleResult | RejectedOracleResult]", results + ), + { + "question_source": "agentic_image_author" + if question_mode == "agentic" + else "openai_image_agent", + "question_model": QUESTION_MODEL, + "oracle_model": ORACLE_MODEL if question_mode == "agentic" else None, + "grounding": { + "min_mask_area_px": min_mask_area_px, + "min_foreground_points": min_foreground_points, + }, + }, + ) + typer.echo(f"Generated {frame_output}") + finally: + model.stop() + summary = write_dataset_manifest(output) + _write_generation_run(output, generation, summary) + typer.echo(f"Dataset manifest: {summary}") + + +def _answer_intents( + ground_truth: VqaGroundTruthGenerator, + frame: CalibratedFrame, + intents: list[QuestionIntent], + label: str, +) -> list[GroundTruthResult]: + results: list[GroundTruthResult] = [] + for number, intent in enumerate(intents, start=1): + typer.echo( + f"{label}: grounding question {number}/{len(intents)}: " + f"{intent.kind} ({intent.object_query})" + ) + result = ground_truth.answer(frame, intent) + results.append(result) + typer.echo(f"{label}: question {number}/{len(intents)} {result.status}") + return results + + +def _answer_agentic( + ground_truth: VqaGroundTruthGenerator, + frame: CalibratedFrame, + proposals: list[QuestionProposal], +) -> list[AcceptedOracleResult | RejectedOracleResult]: + oracle = create_openai_oracle(ORACLE_MODEL) + results: list[AcceptedOracleResult | RejectedOracleResult] = [] + for number, proposal in enumerate(proposals, start=1): + typer.echo(f"Agentic question {number}/{len(proposals)}: {proposal.question}") + result = oracle.answer(proposal, LocalOracleToolRegistry(ground_truth.primitives)) + results.append(result) + if isinstance(result, AcceptedOracleResult): + typer.echo(f"Agentic question {number}/{len(proposals)} accepted") + else: + typer.echo(f"Agentic question {number}/{len(proposals)} rejected: {result.reason}") + return results + + +def _validate_question_mode(question_mode: str) -> None: + if question_mode not in ("constrained", "agentic"): + raise typer.BadParameter("question mode must be constrained or agentic") + + +def _resolve_generation_spec( + spec: Path | None, + recording: str | None, + start_index: int | None, + stop_index: int | None, + stride: int | None, + question_mode: str | None, + min_mask_area_px: int | None, + min_foreground_points: int | None, + output: Path | None, +) -> VqaGenerationSpecification: + """Load a JSON generation specification or resolve the explicit CLI alternatives.""" + values = ( + recording, + start_index, + stop_index, + stride, + question_mode, + min_mask_area_px, + min_foreground_points, + output, + ) + if spec is not None: + if any(value is not None for value in values): + raise typer.BadParameter("--spec cannot be combined with generation options") + try: + return VqaGenerationSpecification.model_validate_json(spec.read_bytes()) + except ValueError as exc: + raise typer.BadParameter(f"invalid generation specification: {exc}") from exc + if recording is None or stop_index is None: + raise typer.BadParameter("--recording and --stop-index are required without --spec") + try: + return VqaGenerationSpecification( + recording=recording, + start_index=0 if start_index is None else start_index, + stop_index=stop_index, + stride=1 if stride is None else stride, + question_mode="constrained" if question_mode is None else question_mode, + grounding={ + "min_mask_area_px": 128 if min_mask_area_px is None else min_mask_area_px, + "min_foreground_points": 3 + if min_foreground_points is None + else min_foreground_points, + }, + output=str(output) if output is not None else None, + ) + except ValueError as exc: + raise typer.BadParameter(f"invalid generation options: {exc}") from exc + + +def _write_generation_run( + output: Path, + generation: VqaGenerationSpecification, + summary: dict[str, int], +) -> None: + """Record the resolved request that produced one generated dataset.""" + payload = { + "schema_version": "1.0", + "generation": { + **generation.model_dump(mode="json"), + "output": str(output), + }, + "models": { + "question_author": QUESTION_MODEL, + "oracle": ORACLE_MODEL if generation.question_mode == "agentic" else None, + }, + "summary": summary, + } + (output / "run.json").write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def _require_openai_for_question_author() -> None: + if not os.environ.get("OPENAI_API_KEY"): + raise typer.BadParameter("OPENAI_API_KEY must be set for image-authored question modes") + + +def _require_edgetam_cuda() -> None: + if default_local_model_device() != "cuda": + raise typer.BadParameter( + "VQA generation requires an installed PyTorch CUDA build that supports this GPU for EdgeTAM" + ) diff --git a/dimos/models/base.py b/dimos/models/base.py index 65393405c5..0fd8291366 100644 --- a/dimos/models/base.py +++ b/dimos/models/base.py @@ -28,8 +28,20 @@ DeviceType = Annotated[str, "Device identifier (e.g., 'cuda', 'cpu', 'cuda:0')"] +def default_local_model_device() -> str: + """Select CUDA only when this Torch build supports the detected GPU.""" + if not torch.cuda.is_available(): + return "cpu" + try: + major, minor = torch.cuda.get_device_capability() + capability = f"sm_{major}{minor}" + return "cuda" if capability in torch.cuda.get_arch_list() else "cpu" + except RuntimeError: + return "cpu" + + class LocalModelConfig(BaseConfig): - device: DeviceType = "cuda" if torch.cuda.is_available() else "cpu" + device: DeviceType = default_local_model_device() dtype: torch.dtype = torch.float32 warmup: bool = False autostart: bool = False diff --git a/dimos/models/segmentation/edge_tam.py b/dimos/models/segmentation/edge_tam.py index 72f1484af8..3f644bd0a0 100644 --- a/dimos/models/segmentation/edge_tam.py +++ b/dimos/models/segmentation/edge_tam.py @@ -30,6 +30,7 @@ from dimos.perception.detection.detectors.base import Detector from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection2d.point import Detection2DPoint from dimos.perception.detection.type.detection2d.seg import Detection2DSeg from dimos.utils.data import get_data from dimos.utils.logging_config import setup_logger @@ -132,6 +133,37 @@ def segment(self, detections: ImageDetections2D) -> ImageDetections2D: ] return ImageDetections2D(image, segmented) + def segment_points(self, points: ImageDetections2D) -> ImageDetections2D: + """Create one foreground mask for every positive point prompt.""" + import cv2 + + if not len(points): + return points + image = points.image + rgb = cv2.cvtColor(image.to_opencv(), cv2.COLOR_BGR2RGB) + segmented: list[Detection2DBBox] = [] + with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16): + self._predictor.set_image(rgb) + for point in points: + if not isinstance(point, Detection2DPoint): + continue + mask, _, _ = self._predictor.predict( + point_coords=np.array([[point.x, point.y]], dtype=np.float32), + point_labels=np.array([1], dtype=np.int32), + multimask_output=False, + ) + segmented.append( + Detection2DSeg.from_sam2_result( + mask.squeeze(), + point.track_id, + image, + class_id=point.class_id, + name=point.name, + confidence=point.confidence, + ) + ) + return ImageDetections2D(image, segmented) + class EdgeTAMProcessor(Detector): _predictor: "SAM2VideoPredictor" diff --git a/docs/benchmarking/vqa-generation/infrastructure.md b/docs/benchmarking/vqa-generation/infrastructure.md new file mode 100644 index 0000000000..509102f29e --- /dev/null +++ b/docs/benchmarking/vqa-generation/infrastructure.md @@ -0,0 +1,162 @@ +--- +title: "VQA Generation Infrastructure" +--- + +# VQA Generation Infrastructure + +This document describes the software stack behind the point-cloud-grounded VQA benchmark. For the +step-by-step generation procedure and dataset schema, see [Pipeline](/docs/benchmarking/vqa-generation/pipeline.md). + +## Design Boundary + +The benchmark has two intentionally separate phases: + +1. Generation privately uses calibrated point clouds and perception tools to establish labels. +2. Evaluation receives only public images, questions, choices, and private expected-choice labels. + +The evaluated vision model never receives point clouds, camera calibration, masks, measurements, +tool traces, or rejected attempts. + +```text +recording + calibration + point cloud + | + v + private generation runtime + | + +-- public: image.jpg, cases.jsonl + +-- private: labels.jsonl, ground_truth.json + | + v + image-only evaluation runtime + | + v + run.json + vqa-results.json +``` + +## Package Layout + +```text +dimos/benchmark/vqa/ + models.py shared immutable VQA contracts + generation/ + primitives/ + frame.py cached frame-scoped perception runtime + contracts.py typed private primitive results + geometry.py plane fitting and masked-point helpers + selection.py nearest-object selection + choices.py deterministic answer-choice resolution + ground_truth_generator.py primitive-owning answer dispatch and grounding coordinator + families.py deterministic constrained family recipes and result helpers + oracle_tools.py opaque-ID and LangChain adapter for primitives + oracle.py bounded tool-calling and evidence validation + question_agent.py image-only question author + specification.py validated generation-specification schema + dataset.py frame records and evaluation export + evaluation.py shared point-cloud-vqa Evaluation plugin + +dimos/cli/vqa.py generation CLI commands +``` + +## Generation Runtime + +`dimos vqa generate` and `dimos vqa single-frame` load a frozen Go2 recording frame containing: + +- A rectified RGB image. +- A calibrated visible point cloud. +- Camera intrinsics and the point-cloud-to-camera transform. + +The generator accepts either explicit CLI flags or a reproducible JSON specification through +`dimos vqa generate --spec `. It writes the resolved generation request and +aggregate counts to `run.json` at the dataset root. The generator constructs one +`FramePerceptionPrimitives` instance per frame. It owns MoonDream and +EdgeTAM calls, intermediate-result caches, grounded masks, and the accepted ground-plane fit. +Projected visible point-cloud samples establish whether a mask has enough foreground support to +become a grounded object. The generation runtime writes complete frame directories, so multi-frame +generation can skip completed frames after an interrupted run. + +### Shared Perception Primitives + +Constrained and agentic generation share the same private geometry runtime. Constrained families select a +fixed recipe; the agentic oracle may inspect intermediate results and choose its own next operation. Handles +are immutable and scoped to one frozen frame. + +```text +detect_objects(query) +-> segment_detection(detection_id) +-> ground_mask(mask_id) +-> object_id / pose / supported point set +-> fit_ground_plane() +-> fitted planes and reusable geometric measurements +``` + +Agentic answers may explicitly reject a question when the available primitive results cannot establish it; +the rejection and missing-evidence reason remain private generation audit data. + +## Question And Answer Contracts + +The image author sees only RGB and proposes a question plus one of these contracts: + +- Boolean: the final public choices are `yes` and `no`. +- Fixed choice: the author supplies at least two public choices. +- Deferred height choice: the author freezes the question, then private geometry deterministically + creates the public height choices and matching answer after a valid measurement. + +The deferred height contract prevents the image author from guessing a numeric range. The private +`height-window-v1` policy derives four local, exhaustive choices around a measured height. The raw +measurement, uncertainty, plane fit, and support remain private. + +## Private Validation + +Every accepted agentic answer must have valid cited evidence. The deterministic validator checks: + +- Cited IDs were emitted by private tools for the current frame. +- The answer is allowed by the resolved public contract. +- A deferred height answer exactly matches the cited measurement bucket. + +A private semantic validator then determines whether the cited structured evidence supports the +question and answer. Invalid calls, inadequate geometry, bad citations, and unsupported answers +are retained as private rejected records rather than exported as evaluation cases. + +## Dataset Assembly + +Each completed frame retains the public `image.jpg`, resumability metadata, private generation +audit, and temporary per-frame case/label rows. `write_dataset_manifest()` aggregates those rows +into root `cases.jsonl` and `labels.jsonl` files. + +The root dataset is deliberately minimal: + +```text +cases.jsonl public case ID, image path, question, and choices +labels.jsonl private case ID and expected choice +frame-*/image.jpg +``` + +`ground_truth.json` is not an evaluator dependency. It exists solely to make generated labels +auditable. + +## Evaluation Runtime + +`point-cloud-vqa` is registered in the shared evaluation framework and is invoked through: + +```bash +dimos eval run --output +``` + +For each case, the evaluator loads the referenced public image, sends the question and choices to +the configured vision model, normalizes an `ANSWER: ` response, and compares it with the +private label. It writes `vqa-results.json` with the expected answer, normalized model answer, raw +model response, and pass/fail result. The shared evaluation runtime also writes its immutable +`run.json` record. + +## Operational Dependencies + +Generation requires local CUDA support for EdgeTAM and credentials for the image author and, in +agentic mode, the private oracle and semantic validator. Evaluation requires only the selected +vision-model credentials and the exported public images/cases plus private labels; it has no +perception-model, CUDA, recording, or point-cloud dependency. + +## Future Work + +Evaluation visualization is intentionally separate from generation. The planned report should be +derived from evaluator inputs and `vqa-results.json` only, so it can show public images, questions, +choices, predictions, labels, and pass/fail results without exposing private generation evidence. diff --git a/docs/benchmarking/vqa-generation/pipeline.md b/docs/benchmarking/vqa-generation/pipeline.md new file mode 100644 index 0000000000..4dcd334925 --- /dev/null +++ b/docs/benchmarking/vqa-generation/pipeline.md @@ -0,0 +1,329 @@ +--- +title: "VQA Generation Pipeline" +--- + +# VQA Generation Pipeline + +## 1. Run Generation + +```bash +dimos vqa generate \ + --recording \ + --start-index \ + --stop-index \ + --stride \ + --question-mode constrained|agentic \ + --output +``` + +Flags: + +- `--recording`: Memory2 Go2 recording path or named recording. +- `--start-index`, `--stop-index`, `--stride`: sampled recording frames. +- `--question-mode`: `constrained` or `agentic`; default is `constrained`. +- `--min-mask-area-px`: minimum accepted segmentation-mask area; default `128`. +- `--min-foreground-points`: minimum point-cloud support inside a mask; default `3`. +- `--output`: dataset root; completed frame directories are skipped on rerun. + +`dimos vqa single-frame` accepts the same generation settings and uses `--frame-index` instead of frame bounds. + +### Generation Specification + +`dimos vqa generate --spec ` is an alternative to the explicit generation flags. +Do not combine `--spec` with `--recording`, frame bounds, question mode, grounding thresholds, or +output flags. A specification is a reproducible generation request: + +```json +{ + "recording": "go2_bigoffice.db", + "start_index": 0, + "stop_index": 100, + "stride": 20, + "question_mode": "agentic", + "grounding": { + "min_mask_area_px": 128, + "min_foreground_points": 3 + }, + "output": "~/.local/state/dimos/datasets/vqa/go2-bigoffice" +} +``` + +Generation writes the resolved request, model IDs, and aggregate counts to the dataset root's +private `run.json`. This is an output record, not the input specification. + +## 2. Create Questions + +### Constrained + +The image author inspects the scene and returns up to five structured intents. It selects only +families likely to be useful for the visible arrangement, rather than expanding every object into +every family. Private grounding still rejects unsupported or ambiguous candidates. + +| Family | Sample question | Choices | +|---|---|---| +| Presence | `Is there a chair in the image?` | `yes`, `no` | +| Horizontal direction | `Where is the nearest chair?` | `left`, `center`, `right` | +| Distance threshold | `Is the nearest chair within 3 meters?` | `yes`, `no` | +| Visible count | `How many chairs are visible?` | `1-2`, `3-4`, `5-7`, `8+` | +| Camera range | `How far is the nearest chair from the camera?` | `under 1 m`, `1 to under 2 m`, `2 to under 4 m`, `4 m or more` | +| Nearest by side | `Which chair is closer, the left or right one?` | `left`, `right` | +| Pairwise left/right | `Is the chair to the left or right of the table?` | `left`, `right` | +| Height comparison | `Which is taller: the chair or the table?` | `chair`, `table` | +| Direct support | `Is the box on the table?` | `yes`, `no` | +| Opening width | `How wide is the doorway?` | `under 0.2 m`, `0.2 to under 0.5 m`, `0.5 to under 0.8 m`, `0.8 m or more` | +| Closest object | `Which object is closest to the chair: table or lamp?` | `table`, `lamp` | +| Door state | `Is the door open or closed?` | `open`, `closed` | +| Forward path | `Is the path directly ahead clear or blocked?` | `clear`, `blocked` | + +### Agentic + +The image-only author returns a frozen question with one contract: + +- Boolean: `{"kind":"boolean"}`; public choices are `yes`, `no`. +- Choice: `{"kind":"choice","choices":[...]}`; at least two choices. +- Height: `{"kind":"deferred_height_choice","strategy":"height-window-v1"}`; the question is + frozen before private geometry, while four public choices are generated from a successful private + height measurement. + +Numeric contracts are rejected. `height-window-v1` selects a local four-choice window from private +measurement against the fixed internal breakpoints `0.1`, `0.2`, `0.6`, `1.0`, and `2.0` meters. For +example, a private height of `0.42 m` produces: + +```text +under 0.2 m | 0.2-0.6 m | 0.6-1.0 m | over 1.0 m +``` + +The agentic oracle is not bound to a constrained family. It can inspect individual detection, mask, +object, and plane handles, choose its own tool sequence, and either return a cited answer or privately +reject the proposal with the missing evidence. + +## 3. Pre-Answer Grounding Checks + +All referenced objects need a detected mask with enough visible 3D point support. + +- Height: one grounded object and an accepted ground plane. +- Door state: a door plane and nearby surrounding plane with a clear relative angle. +- Closest object: one grounded target and one grounded instance per candidate. +- Count and range: accepted grounded instances only. +- Left/right and height comparison: one grounded instance per named object with clear separation. +- Object on support: contact or clear separation relative to the support plane. +- Opening width: one ground-connected aperture and stable surrounding-wall geometry. +- Forward path: visible ground support across the forward corridor and supported obstacle evidence. + +Missing, sparse, or ambiguous evidence rejects the question. + +## 4. Create Answers + +### Constrained + +Each deterministic family runs its own fixed sequence. `ground(A)` below is always the same private +primitive chain: + +```text +ground(A) +-> detect_objects(A) +-> segment_detections(A) +-> ground_masks(A) +-> accepted grounded A instances, or reject if the family requires an unavailable instance +``` + +```text +presence(A) +-> ground(A) +-> no grounded A: reject +-> one or more grounded A instances: yes +``` + +```text +horizontal_direction(A) +-> ground(A) +-> select_nearest_object(grounded A instances) -> nearest A +-> nearest A horizontal_direction: left/center/right +``` + +```text +within_distance(A, T) +-> ground(A) +-> select_nearest_object(grounded A instances) -> nearest A +-> nearest A range_m <= T: yes/no +``` + +```text +compare_nearest_by_side(A) +-> ground(A) +-> select_nearest_object(grounded A instances, left) -> nearest left A +-> select_nearest_object(grounded A instances, right) -> nearest right A +-> either side missing or tied: reject +-> compare the two range_m values: left/right +``` + +```text +visible_count(A) +-> ground(A) +-> bucket accepted instance count: 1-2 / 3-4 / 5-7 / 8+ +``` + +```text +camera_range(A) +-> ground(A) +-> select_nearest_object(...) -> bucket camera-origin range +``` + +```text +compare_left_right(A, B) +-> ground(A), ground(B) +-> require exactly one accepted A and B +-> compare camera-frame support centroids +-> separation under 0.1 m: reject -> otherwise choose left/right +``` + +```text +compare_height(A, B) +-> ground(A), ground(B) +-> require exactly one accepted A and B +-> fit_ground_plane() +-> measure_height(A, plane) and measure_height(B, plane) +-> reject overlapping uncertainty intervals -> choose taller A/B +``` + +```text +object_on_support(A, B) +-> ground(A), ground(B) +-> require exactly one accepted A and B +-> fit_ground_plane() -> ground plane +-> fit_object_surface_plane(B) -> support plane +-> reject non-horizontal support plane +-> measure_object_plane_relation(A, B, support plane, ground normal) +-> clear separation: no -> unsupported or ambiguous geometry: reject -> otherwise yes +``` + +```text +opening_width(A) +-> detect_objects(A) -> segment_detections(A) +-> require exactly one accepted aperture mask +-> fit_ground_plane() -> ground plane +-> measure_opening_width_from_mask(mask, ground plane) +-> reject non-ground-connected, nonvertical, edge-clipped, or unstable aperture geometry +-> bucket: under 0.2 / 0.2-0.5 / 0.5-0.8 / 0.8 m or more +``` + +```text +closest_object(A, B, C, ...) +-> ground(A), ground(B), ground(C), ... +-> require exactly one target and one instance for every candidate type +-> select_closest_object(target, candidates) from private support-point centroids +-> reject ties within 0.15 m -> otherwise choose the closest candidate +``` + +```text +door_state(A) +-> ground(A) -> require exactly one door +-> fit_object_surface_plane(door) -> door plane +-> fit_object_surrounding_plane(door) -> surrounding plane +-> measure_relative_plane_angle(door plane, surrounding plane) +-> nearly aligned: closed; clearly rotated: open; intermediate/failed geometry: reject +``` + +```text +forward_path() +-> fit_ground_plane() +-> classify_forward_corridor(visible camera points, ground plane) +-> require ground support across each forward distance band +-> supported elevated points: blocked; no supported obstacle: clear; otherwise reject +``` + +### Agentic + +The private oracle chooses a sequence from the same read-only primitives used by constrained recipes: + +| Tool | Input | Output | +|---|---|---| +| `detect_objects` | semantic query | Individual opaque detection IDs and confidence. | +| `segment_detection` | detection ID | Individual opaque mask IDs. | +| `ground_mask` | mask ID | One grounded object ID and citable support evidence. | +| `fit_ground_plane` | none | Plane ID, plane estimate, residual, inlier support, quality flags. | +| `get_object_pose` | object ID | Camera-frame grounding evidence: range, side, support count. | +| `fit_object_surface_plane` | object ID | Plane ID from one object's visible support. | +| `fit_mask_surrounding_plane` | mask ID | Plane ID from visible support around a mask. | +| `measure_object_pair_distance` | two object IDs | Private 3D support-centroid distance. | +| `measure_relative_plane_angle` | two plane IDs | Private unsigned angle between accepted planes. | +| `measure_object_plane_relation` | object ID, support ID, support plane, ground plane | Clearance, contact, and projected-separation metrics. | +| `measure_aperture_geometry` | mask ID, ground plane | Private ground-connected aperture span and uncertainty. | +| `measure_forward_corridor` | ground plane | Private floor-support and elevated-obstacle metrics. | +| `measure_height` | object ID, plane ID | Measurement ID, private height, uncertainty, provenance, quality flags. | + +Opaque IDs chain tool results; raw masks and point-cloud arrays are not exposed to the oracle. The +oracle returns a candidate answer and cited evidence IDs. Deferred height choices are the sole +exception to frozen public choices: the question remains frozen, but the public options and answer +are deterministically derived from the private measurement. + +## 5. Post-Answer Validation + +The candidate is rejected unless: + +1. Its answer exactly matches the fixed public choices, or the public choices deterministically + generated from a cited private height measurement. +2. It cites one or more known evidence IDs. +3. A cited height bucket matches the deterministic measurement bucket. +4. The private semantic validator confirms the cited tool output supports the question and answer. + +Tool failures, quality-gate failures, invalid citations, invalid answer format, and unsupported claims are retained as rejected generation records. + +## 6. Write Dataset Artifacts + +Each `frame-*` directory contains: + +```text +image.jpg public rectified image +frame.json frame metadata and accepted/rejected counts +ground_truth.json private tool evidence, checks, answers, and rejections +cases.json public per-frame image/question/choice rows +labels.json private per-frame correct-choice rows +``` + +The dataset root aggregates: + +```text +cases.jsonl public id, image path, question, choices +labels.jsonl private id and expected choice +run.json private resolved generation request and aggregate counts +``` + +Each line in `cases.jsonl` is one public evaluation case. Formatted for readability, one record is: + +```json +{ + "id": "go2-40-chair-height", + "image": "frame-000040/image.jpg", + "question": "How tall is the chair?", + "choices": [ + "under 0.2 m", + "0.2-0.6 m", + "0.6-1.0 m", + "over 1.0 m" + ] +} +``` + +- `id`: unique case identifier. +- `image`: dataset-relative path to the public image. +- `question`: question shown to the evaluated vision model. +- `choices`: at least two allowed answer strings. + +Each line in `labels.jsonl` is the corresponding private correct answer. Formatted for readability, +one record is: + +```json +{ + "id": "go2-40-chair-height", + "answer": "0.2-0.6 m" +} +``` + +- `id`: case identifier matching exactly one `cases.jsonl` row. +- `answer`: one of that case's `choices`. + +The files store each record as one JSON object per physical line; the examples above are expanded +only for documentation readability. + +The shared `point-cloud-vqa` evaluator reads public `cases.jsonl`, public images, and private `labels.jsonl`. It does not read generation evidence or point-cloud data. diff --git a/docs/docs.json b/docs/docs.json index 9b4246f2be..94659c93a6 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -124,7 +124,9 @@ { "group": "Perception", "pages": [ - "capabilities/perception/index" + "capabilities/perception/index", + "benchmarking/vqa-generation/pipeline", + "benchmarking/vqa-generation/infrastructure" ] }, {