feat: bind each action to its exact retained screen frame

openadapt_capture/capture.py

@@ -5,6 +5,7 @@
 
 from __future__ import annotations
 
+import io
 from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Iterator
@@ -112,6 +113,7 @@ def _convert_action_event(db_event) -> PydanticActionEvent:
         "structural_observation": _parse_structural_observation(
             getattr(db_event, "structural_observation", None)
         ),
+        "screenshot_timestamp": getattr(db_event, "screenshot_timestamp", None),
     }
 
     if db_event.name == "move":
@@ -449,11 +451,22 @@ def structural_observation(self) -> StructuralObservation | None:
 
     @property
     def screenshot(self) -> "Image" | None:
-        """Get the screenshot at the time of this action.
+        """Get the exact retained screen frame this action is bound to.
+
+        The action's ``screenshot_timestamp`` names the one frame the recorder
+        retained as this action's evidence; it is decoded exactly or this
+        raises. Actions recorded before frame binding existed fall back to a
+        nearest-frame lookup at the action timestamp.
 
         Returns:
-            PIL Image of the screen at action time, or None if not available.
+            PIL Image of the bound frame.
+
+        Raises:
+            LookupError: If the bound frame cannot be resolved exactly.
         """
+        bound = getattr(self.event, "screenshot_timestamp", None)
+        if bound is not None:
+            return self._capture.get_exact_frame(bound)
         return self._capture.get_frame_at(self.timestamp)
 
 
@@ -739,6 +752,10 @@ def browser_event_count(self) -> int:
     def get_frame_at(self, timestamp: float, tolerance: float = 0.5) -> "Image" | None:
         """Get the screen frame closest to a timestamp.
 
+        This is the lenient legacy lookup. Evidence consumers should prefer
+        :meth:`get_exact_frame`, which resolves one exact retained frame and
+        fails closed when it cannot.
+
         Args:
             timestamp: Unix timestamp.
             tolerance: Maximum time difference in seconds.
@@ -764,6 +781,36 @@ def get_frame_at(self, timestamp: float, tolerance: float = 0.5) -> "Image" | No
         except Exception:
             return None
 
+    def get_exact_frame(self, capture_timestamp: float) -> "Image":
+        """Decode THE retained frame bound to this exact capture timestamp.
+
+        Prefers the video's capture-timeline binding; for image-only captures
+        (no video stream) the retained PNG row with that exact timestamp is
+        the bound frame.
+
+        Raises:
+            LookupError: If no frame was retained at exactly this timestamp,
+                or the binding metadata is unavailable (fail-closed).
+        """
+        from PIL import Image
+
+        video_path = self.video_path
+        if video_path is not None:
+            from openadapt_capture.video import extract_exact_frame
+
+            return extract_exact_frame(video_path, capture_timestamp)
+        for screenshot in self._recording.screenshots:
+            if screenshot.timestamp == capture_timestamp:
+                if not screenshot.png_data:
+                    raise LookupError(
+                        f"the screenshot retained at {capture_timestamp!r} has no image data"
+                    )
+                return Image.open(io.BytesIO(screenshot.png_data)).convert("RGB")
+        raise LookupError(
+            f"no frame was retained at exactly {capture_timestamp!r} "
+            "(fail-closed; refusing a nearest-frame substitute)"
+        )
+
     def close(self) -> None:
         """Close the capture and release resources.
 

openadapt_capture/events.py

@@ -71,6 +71,14 @@ class ActionBaseEvent(BaseEvent):
         default=None,
         description="Versioned accessibility evidence observed at action time",
     )
+    screenshot_timestamp: float | None = Field(
+        default=None,
+        description=(
+            "Exact capture timestamp of the retained screen frame this action "
+            "is bound to. Extract that exact frame instead of guessing a "
+            "nearest one."
+        ),
+    )
 
 
 # =============================================================================

openadapt_capture/processing.py

@@ -90,6 +90,21 @@ def _first_structural_observation(
     return None
 
 
+def _bound_screenshot_timestamp(events: list[ActionEvent]) -> float | None:
+    """Keep the frame bound where the merged action completed.
+
+    The recorder binds an action to the screen frame retained when the action
+    was emitted (a click at button-up, a typed run at its last key), so the
+    merged event carries the LAST child's binding, not the first's.
+    """
+
+    bound: float | None = None
+    for event in events:
+        if event.screenshot_timestamp is not None:
+            bound = event.screenshot_timestamp
+    return bound
+
+
 # =============================================================================
 # Event Processing Functions
 # =============================================================================
@@ -250,6 +265,9 @@ def flush_buffer() -> None:
                         structural_observation=_first_structural_observation(
                             list(keyboard_buffer)
                         ),
+                        screenshot_timestamp=_bound_screenshot_timestamp(
+                            list(keyboard_buffer)
+                        ),
                     )
                 )
         else:
@@ -266,6 +284,7 @@ def flush_buffer() -> None:
                 structural_observation=_first_structural_observation(
                     list(keyboard_buffer)
                 ),
+                screenshot_timestamp=_bound_screenshot_timestamp(list(keyboard_buffer)),
             )
             result.append(type_event)
 
@@ -328,6 +347,7 @@ def flush_buffer() -> None:
                 structural_observation=_first_structural_observation(
                     list(move_buffer)
                 ),
+                screenshot_timestamp=_bound_screenshot_timestamp(list(move_buffer)),
             )
             result.append(merged)
 
@@ -380,6 +400,7 @@ def flush_buffer() -> None:
                 structural_observation=_first_structural_observation(
                     list(scroll_buffer)
                 ),
+                screenshot_timestamp=_bound_screenshot_timestamp(list(scroll_buffer)),
             )
             result.append(merged)
 
@@ -487,6 +508,9 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float:
                             structural_observation=_first_structural_observation(
                                 [down, up, next_down, next_up]
                             ),
+                            screenshot_timestamp=_bound_screenshot_timestamp(
+                                [down, up, next_down, next_up]
+                            ),
                         )
                         result.append(double_click)
                         skip_timestamps.add(up.timestamp)
@@ -504,6 +528,7 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float:
                     structural_observation=_first_structural_observation(
                         [down, up]
                     ),
+                    screenshot_timestamp=_bound_screenshot_timestamp([down, up]),
                 )
                 result.append(single_click)
                 skip_timestamps.add(up.timestamp)
@@ -571,6 +596,9 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float:
                         structural_observation=_first_structural_observation(
                             [down_event] + moves + [event]
                         ),
+                        screenshot_timestamp=_bound_screenshot_timestamp(
+                            [down_event] + moves + [event]
+                        ),
                     )
                     result.append(drag)
                 else: