From afb7daa97e3de372a7a34f564a0fc8f56b4329f3 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 20 Aug 2026 12:11:37 -0400 Subject: [PATCH 1/3] feat: bind native actions to exact window geometry --- README.md | 97 ++--- docs/DESIGN.md | 54 +-- openadapt_capture/capture.py | 69 ++++ openadapt_capture/db/models.py | 3 + openadapt_capture/events.py | 95 ++++- openadapt_capture/processing.py | 89 +++++ openadapt_capture/recorder.py | 87 ++++- openadapt_capture/window_capture.py | 533 +++++++++++++++++++++++++--- pyproject.toml | 1 - scripts/verify_distribution.py | 5 + tests/test_changelog_contract.py | 24 ++ tests/test_window_capture.py | 355 +++++++++++++++++- 12 files changed, 1263 insertions(+), 149 deletions(-) diff --git a/README.md b/README.md index 5072ee0..e920825 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,17 @@ # OpenAdapt Capture > [!IMPORTANT] -> **Status: Experimental.** OpenAdapt Capture records native mouse, keyboard, -> and screen activity into a time-aligned local capture session. Its current -> product role is the optional cross-platform desktop recorder used by +> OpenAdapt Capture is the canonical native recorder for OpenAdapt. It records +> native mouse, keyboard, screen, timing, window-geometry, and optional +> accessibility evidence into a time-aligned local session for > [`openadapt-flow`](https://github.com/OpenAdaptAI/openadapt-flow). > -> The OpenAdapt product is the demonstration compiler, -> [`openadapt-flow`](https://github.com/OpenAdaptAI/openadapt-flow), installed -> via the [`OpenAdapt`](https://github.com/OpenAdaptAI/OpenAdapt) launcher -> (first run: `pip install 'openadapt[browser]'` then `openadapt quickstart`; -> on Windows `cmd.exe` use `pip install "openadapt[browser]"`): it compiles a -> demonstrated GUI workflow into a -> deterministic, locally executable program. Healthy runs make no model calls, -> and it halts instead of guessing when verification fails. Lifecycle labels for -> every repository are in the -> [repository lifecycle registry](https://github.com/OpenAdaptAI/.github/blob/main/REPOSITORY_LIFECYCLE.md). +> Capture does not use a static Experimental, Beta, or Production package +> label. OpenAdapt selects a release only while it has an active, signed entry +> in the canonical +> [production admission ledger](https://github.com/OpenAdaptAI/.github/blob/main/production-lifecycle-admissions.json). +> An inactive release is not actively admitted. It is not reclassified as an +> Experimental or Beta implementation. [![Build Status](https://github.com/OpenAdaptAI/openadapt-capture/actions/workflows/test.yml/badge.svg)](https://github.com/OpenAdaptAI/openadapt-capture/actions/workflows/test.yml) [![PyPI version](https://img.shields.io/pypi/v/openadapt-capture.svg)](https://pypi.org/project/openadapt-capture/) @@ -38,14 +34,15 @@ model calls on the healthy path. When the live screen does not match what was demonstrated it halts instead of guessing, using identity gates and independent effect verification. Every substrate is first-class. -Substrate maturity, stated the same way across the OpenAdapt repositories: +Each recording path has one canonical implementation. A Production run uses +the exact release and claim scope selected by the signed admission ledger. -| Substrate | Maturity | +| Substrate | Canonical implementation | | --- | --- | -| Browser (web) | Beta; available in production today through the managed browser product | -| Native desktop (Windows, macOS, Linux) | Available for customer-controlled execution; qualification evidence is task- and environment-specific | -| Remote display (RDP) | Available for customer-controlled execution; qualification evidence is task- and environment-specific | -| Citrix / VDI | Available for customer-controlled execution; real-environment ICA/HDX qualification is deployment-specific | +| Browser (web) | The `openadapt-flow` Playwright recorder owns browser context, DOM identity, field geometry, navigation, and source-time secret exclusion. | +| Native desktop (Windows, macOS, Linux) | `openadapt-capture` records native input, media, geometry, and local structural evidence. | +| Remote display (RDP) | `openadapt-capture` records the client window; Flow treats the remote application as a visual black box. | +| Citrix / VDI | `openadapt-capture` records the client window; deployment qualification proves the exact ICA/HDX environment. | The packages in the stack: @@ -65,7 +62,7 @@ Documentation for the whole stack lives at | Recording path | Current implementation | | --- | --- | -| Windows, macOS, and Linux demonstrations | `openadapt-capture` records native input and action-gated screen video; Windows can also retain action-time UI Automation evidence. `openadapt-flow` converts the session into compiler input. | +| Windows, macOS, and Linux demonstrations | `openadapt-capture` records native input and action-gated screen video. It also retains action-time Windows UIA, macOS Accessibility, or Linux AT-SPI evidence when the local provider exposes it. `openadapt-flow` converts the session into compiler input. | | RDP and Citrix/VDI demonstrations | `openadapt-capture` records the selected client window in its own pixel space. The remote application remains externally black-box, and `openadapt-flow` converts the session into compiler input. | | Browser demonstrations | `openadapt-flow` uses its Playwright recorder. It can launch Chromium or attach to one existing signed-in local Chromium tab. It does not require this package. | | Chrome extension in this repository | Repository-only prototype. Its bridge and legacy direct replay are excluded from the wheel and source archive. It is not the supported recorder. | @@ -76,15 +73,12 @@ frames, and source-time secret redaction to one recording contract. A Chrome extension cannot guarantee that contract across browser profiles, extension permissions, browser-internal pages, and process or tab disconnects. -The extension remains useful as a research observer and as a possible future -source of optional DOM evidence. It should become a supported auxiliary -observer only after it emits the shared event schema, has a fail-closed -connection and permission contract, redacts secret fields before persistence, -and passes the same compiler qualification as the Playwright path. It should -not replace the Playwright recorder merely to make the package layout uniform, -create a second compiler format, or bypass governed replay. Flow supports an -existing authenticated browser session through its local-loopback CDP attach -mode. +The extension remains a repository-only research prototype. It is not packaged +or promoted as a recorder or replay path. Browser recording stays in Flow +because DOM identity, field geometry, navigation, source-time secret exclusion, +and browser-context ownership are load-bearing parts of one compiler contract. +Flow supports an existing authenticated browser session through its +local-loopback CDP attach mode. ## Use it with OpenAdapt @@ -225,11 +219,12 @@ retain window-scoped pixels and coordinates for Flow's remote visual compiler. ## Window-scoped recording -**Status: implemented, with display-free unit coverage on every -supported operating system.** The production release gate also requires live +Window-scoped recording is implemented with display-free unit coverage on every +supported operating system. The production release gate also requires live window capture, input injection, movement, resize, video verification, and no -skipped tests on interactive macOS and Windows runners. A customer RDP or -Citrix deployment still requires task- and environment-specific qualification. +skipped tests on interactive Linux, macOS, and Windows runners. A customer RDP +or Citrix deployment still requires task- and environment-specific +qualification. By default the recorder captures the full screen. Window-scoped mode records ONE window in that window's own pixel space. This is the mode built for @@ -249,9 +244,9 @@ with Recorder( input("Perform the task, then press Enter...") ``` -`owner` matches the application (macOS: window owner name; Windows: process -executable name) and `title` optionally disambiguates among its windows; both -are case-insensitive substrings, mirroring how `openadapt-flow`'s +`owner` matches the application (macOS: window owner name; Windows and X11: +process executable name) and `title` optionally disambiguates among its +windows; both are case-insensitive substrings, mirroring how `openadapt-flow`'s remote-display backend identifies the same window at replay time. The selectors can also be set via config/environment (`RECORD_WINDOW_OWNER` / `RECORD_WINDOW_TITLE`). @@ -260,7 +255,9 @@ In this mode: - **Frames are the target window's pixels.** macOS captures the window's own buffer (`CGWindowListCreateImage`, the identical call flow's replay uses); - Windows grabs the window's screen region, so keep the window unoccluded. + Windows and native X11 grab the window's screen region, so keep the window + unoccluded. Native Wayland fails closed until a compositor portal supplies + stable window identity and geometry. - **Input coordinates are translated at capture time** into the captured frame's pixel space (`pixel = (global_point - window_origin) * scale`, the exact inverse of the replay mapping). Input outside the window records @@ -269,13 +266,19 @@ In this mode: target, resolved window, initial bounds, fixed output viewport, current source viewport, scale-to-fit mapping, and content rectangle (`CaptureSession.window_capture`), and the window is re-resolved every - frame with bounds changes recorded as window events, a bounds timeline - converters can use to be exact even when the window moves. + frame. Each frame records a process-bound window identity, physical + coordinate source, and monotonic geometry generation. Every action binds + to that exact published generation. Capture refuses input during an + unobserved move or resize instead of translating against stale bounds. - **Window movement and resize are supported.** The first frame fixes the encoded video size. Later source frames scale to fit and letterbox into that viewport. Input uses the exact current bounds and content rectangle. No frame is silently skipped because the window changed size or moved to a display with a different scale. +- **The geometry contract is versioned.** Current window sessions declare + `openadapt.capture.window-scoped/v2`. The frame pixels and WindowEvent share + one timestamp. Every pointer or keyboard action references that timestamp + and the exact geometry generation. - **Fail-loud guarantees:** recording refuses to start if the window cannot be resolved and captured; input arriving before the first frame is discarded with a warning instead of being recorded in the wrong coordinate space; a @@ -382,15 +385,13 @@ session is required. See the - Display hot-plug, rotation, resolution changes, and scale changes require a new recording because one media stream has one fixed virtual-desktop viewport. - Browser-extension installation, bridge code, and direct replay are not part - of the published Capture artifacts or supported browser path. Promotion - requires the shared Flow schema, source-time secret exclusion, authenticated - and sequenced delivery, exact frame binding, compiler integration, and - removal of direct replay. - -See the organization-wide -[repository lifecycle registry](https://github.com/OpenAdaptAI/.github/blob/main/REPOSITORY_LIFECYCLE.md) -and [`openadapt-flow` product status](https://github.com/OpenAdaptAI/openadapt-flow/blob/main/docs/PRODUCT_STATUS.md) -for the evidence behind current maturity labels. + of the published Capture artifacts or supported browser path. The supported + browser recorder remains the Flow Playwright recorder. + +The canonical +[production admission ledger](https://github.com/OpenAdaptAI/.github/blob/main/production-lifecycle-admissions.json) +binds each admitted claim scope to an exact release and evidence record. Package +metadata and repository copy do not select Production maturity. ## Optional extras diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 18b4fdc..bb222f5 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -7,10 +7,12 @@ screen media, native input, timing, window geometry, and optional action-time UI structure into one local session. `openadapt-flow` consumes the session, applies the compiler and qualification contracts, and owns governed replay. -The package lifecycle is **Experimental**. A successful unit test or a runnable package -does not by itself make a recording path production-qualified. A release must -also pass the exact-commit clean-install and interactive native qualification -described below. +Capture has no static Experimental, Beta, or Production package label. It is a +stable component with one canonical native role. A release becomes selectable +for a Production claim scope only through an active, signed entry in the +[production admission ledger](https://github.com/OpenAdaptAI/.github/blob/main/production-lifecycle-admissions.json). +An inactive release is not actively admitted. Exact-commit clean-install and +interactive native qualification provide the evidence for admission. Capture is local-first. It does not upload a recording. A raw session can contain screen text, typed secrets, accessibility text, and optional narration. @@ -89,6 +91,25 @@ window changed size. Input outside the selected window remains out of range. Capture does not clamp it into a valid-looking target coordinate. +Current sessions declare `openadapt.capture.window-scoped/v2`. Each frame has a +monotonic geometry generation. Capture persists the frame and its WindowEvent +at one timestamp. Every pointer or keyboard action references that timestamp +and generation. The compiler must reject a missing or mismatched v2 binding. It +must not select geometry by a nearest-time estimate. + +Each frame binds the native window handle to the owner PID, process start time, +and platform coordinate source. Capture resolves the identity and bounds on +both sides of the frame grab. It publishes a monotonic geometry generation only +after the frame and its WindowEvent enter the ordered event queue. Every action +re-resolves the live target and records the exact published generation. +An action during an unobserved move, resize, process replacement, or handle +reuse fails the session. + +Windows accepts DWM extended-frame bounds only. It does not fall back to the +DPI-virtualized `GetWindowRect` coordinate space. Linux window scope uses +native X11 EWMH identity and root-pixel geometry. A native Wayland session +fails closed until a compositor portal can supply the same stable contract. + ## Native input Capture records these primitive event classes: @@ -160,20 +181,11 @@ The supported browser recorder remains Playwright-native in `openadapt-flow`. It needs one owner for the browser context, DOM identity, field geometry, ordered before/after frames, page state, and source-time secret redaction. -The Chrome extension can supply useful DOM observations, but it does not yet -provide this complete contract. It can become a supported auxiliary observer -after it has: - -1. a shared versioned event schema; -2. source-time secret redaction; -3. explicit permission and browser-profile boundaries; -4. fail-closed reconnect and tab-lifecycle behavior; -5. exact frame and event binding; and -6. end-to-end compiler qualification. - -It should not replace the Playwright recorder only to consolidate packages. -The stronger ownership and redaction boundary is more important than package -uniformity. +The Chrome extension is a repository-only research prototype. It does not own +the complete browser contract, and Capture does not package or promote it. +Browser recording stays in Flow because DOM identity, field geometry, +navigation, source-time secret exclusion, and browser-context ownership are +load-bearing inputs to compilation and replay. The extension and its unauthenticated development bridge are repository-only. Wheel and source archives exclude the bridge and its legacy direct replay. The @@ -198,13 +210,15 @@ commit. It must: The release workflow accepts only a successful, complete job set for its exact commit. Missing, stale, skipped, partial, or failed evidence blocks publication. +Publication does not select a Production default. The signed admission ledger +binds an exact published release to its approved claim scope and evidence. ## Known boundaries - A visible logged-in desktop session and operating-system permissions are required. -- Windows window capture uses a screen-region grab and requires an unobstructed - target window. +- Windows and X11 window capture use a screen-region grab and require an + unobstructed target window. - A display-topology change requires a new recording. - Browser extension capture is a repository-only development prototype. Its bridge and direct replay are not in release artifacts. It is not the diff --git a/openadapt_capture/capture.py b/openadapt_capture/capture.py index 75574b3..2cf5df3 100644 --- a/openadapt_capture/capture.py +++ b/openadapt_capture/capture.py @@ -27,6 +27,7 @@ ActionEvent as PydanticActionEvent, ) from openadapt_capture.events import ( + CapturedWindowEvent, KeyDownEvent, KeyShortcutEvent, KeyTypeEvent, @@ -35,6 +36,7 @@ MouseMoveEvent, MouseScrollEvent, MouseUpEvent, + WindowCaptureStateV2, ) from openadapt_capture.processing import process_events from openadapt_capture.structural import StructuralObservation @@ -109,9 +111,12 @@ def _convert_action_event(db_event) -> PydanticActionEvent: """ common = { "timestamp": db_event.timestamp, + "screenshot_timestamp": getattr(db_event, "screenshot_timestamp", None), + "window_event_timestamp": getattr(db_event, "window_event_timestamp", None), "structural_observation": _parse_structural_observation( getattr(db_event, "structural_observation", None) ), + "window_geometry_generation": getattr(db_event, "window_geometry_generation", None), } if db_event.name == "move": @@ -447,6 +452,21 @@ def structural_observation(self) -> StructuralObservation | None: """Accessibility evidence captured at this action, when available.""" return self.event.structural_observation + @property + def window_geometry_generation(self) -> int | None: + """Exact published window geometry used for this action.""" + return self.event.window_geometry_generation + + @property + def screenshot_timestamp(self) -> float | None: + """Exact ScreenEvent timestamp bound to this action.""" + return self.event.screenshot_timestamp + + @property + def window_event_timestamp(self) -> float | None: + """Exact WindowEvent timestamp bound to this action.""" + return self.event.window_event_timestamp + @property def screenshot(self) -> "Image" | None: """Get the screenshot at the time of this action. @@ -689,6 +709,55 @@ def raw_events(self) -> list[PydanticActionEvent]: events.append(_convert_action_event(db_event)) return events + def window_events(self) -> list[CapturedWindowEvent]: + """Return stored window rows through the public validated event view.""" + events: list[CapturedWindowEvent] = [] + for row in self._recording.window_events: + state = getattr(row, "state", None) + if not isinstance(state, dict): + raise InvalidCaptureEvent( + f"stored WindowEvent at {row.timestamp!r} has invalid state" + ) + events.append( + CapturedWindowEvent( + timestamp=row.timestamp, + title=row.title, + left=row.left, + top=row.top, + width=row.width, + height=row.height, + window_id=row.window_id, + state=state, + ) + ) + return events + + def window_capture_events_v2(self) -> list[CapturedWindowEvent]: + """Return current scoped rows after strict v2 state validation.""" + result: list[CapturedWindowEvent] = [] + for event in self.window_events(): + if event.state.get("schema_version") != "openadapt.capture.window-scoped/v2": + continue + state: WindowCaptureStateV2 | None = event.window_capture_v2 + if state is None: # pragma: no cover - guarded by the version test + continue + if not event.window_id: + raise InvalidCaptureEvent( + f"stored WindowEvent at {event.timestamp!r} has no window identity" + ) + x, y, width, height = state.bounds + if ( + event.left != int(x) + or event.top != int(y) + or event.width != int(width) + or event.height != int(height) + ): + raise InvalidCaptureEvent( + f"stored WindowEvent at {event.timestamp!r} columns do not match v2 bounds" + ) + result.append(event) + return result + def actions(self, include_moves: bool = False) -> Iterator[Action]: """Iterate over processed actions. diff --git a/openadapt_capture/db/models.py b/openadapt_capture/db/models.py index 59a9dc1..0ff6545 100644 --- a/openadapt_capture/db/models.py +++ b/openadapt_capture/db/models.py @@ -127,6 +127,9 @@ class ActionEvent(Base): # Versioned optional accessibility evidence captured at action time. # Nullable + additive migration keep older recording.db files readable. structural_observation = sa.Column(sa.JSON) + # Exact window-frame geometry bound to this action. + # Nullable keeps full-desktop and pre-1.3 captures compatible. + window_geometry_generation = sa.Column(sa.Integer) disabled = sa.Column(sa.Boolean, default=False) children = sa.orm.relationship("ActionEvent") diff --git a/openadapt_capture/events.py b/openadapt_capture/events.py index 57a7d3c..9d586b7 100644 --- a/openadapt_capture/events.py +++ b/openadapt_capture/events.py @@ -7,12 +7,14 @@ from __future__ import annotations +import math from enum import Enum -from typing import Literal +from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from openadapt_capture.structural import StructuralObservation +from openadapt_capture.window_capture import WINDOW_CAPTURE_SCHEMA_VERSION class EventType(str, Enum): @@ -64,13 +66,102 @@ class BaseEvent(BaseModel): model_config = {"use_enum_values": True} +class WindowCaptureStateV2(BaseModel): + """Exact per-frame geometry contract for a current window session.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal["openadapt.capture.window-scoped/v2"] + window_capture: Literal[True] + owner: str = Field(min_length=1) + pid: int = Field(gt=0) + process_start_time: float + coordinate_source: str = Field(min_length=1) + geometry_generation: int = Field(ge=1) + bounds: tuple[float, float, float, float] + scale: float = Field(gt=0) + scale_x: float = Field(gt=0) + scale_y: float = Field(gt=0) + viewport: tuple[int, int] + source_viewport: tuple[int, int] + content_rect: tuple[int, int, int, int] + fit_scale: float = Field(gt=0) + on_screen: bool + + @model_validator(mode="after") + def validate_geometry(self) -> "WindowCaptureStateV2": + """Reject non-finite or internally inconsistent frame geometry.""" + numeric = ( + self.process_start_time, + *self.bounds, + self.scale, + self.scale_x, + self.scale_y, + self.fit_scale, + ) + if not all(math.isfinite(float(value)) for value in numeric): + raise ValueError("window capture geometry must be finite") + if self.process_start_time <= 0 or self.bounds[2] <= 0 or self.bounds[3] <= 0: + raise ValueError("window capture process identity and bounds must be positive") + if any(value <= 0 for value in (*self.viewport, *self.source_viewport)): + raise ValueError("window capture viewports must be positive") + left, top, width, height = self.content_rect + if ( + left < 0 + or top < 0 + or width <= 0 + or height <= 0 + or left + width > self.viewport[0] + or top + height > self.viewport[1] + ): + raise ValueError("window capture content_rect falls outside the viewport") + return self + + +class CapturedWindowEvent(BaseModel): + """Public validated view of one stored WindowEvent row.""" + + model_config = ConfigDict(extra="forbid") + + timestamp: float + title: str | None = None + left: int | None = None + top: int | None = None + width: int | None = None + height: int | None = None + window_id: str | None = None + state: dict[str, Any] + + @property + def window_capture_v2(self) -> WindowCaptureStateV2 | None: + """Return strict v2 state, or None for a non-v2 legacy row.""" + if self.state.get("schema_version") != WINDOW_CAPTURE_SCHEMA_VERSION: + return None + return WindowCaptureStateV2.model_validate(self.state) + + class ActionBaseEvent(BaseEvent): """Base event for native actions with optional structural evidence.""" + screenshot_timestamp: float | None = Field( + default=None, + description="Exact preceding ScreenEvent timestamp retained by Capture", + ) + window_event_timestamp: float | None = Field( + default=None, + description="Exact preceding WindowEvent timestamp retained by Capture", + ) structural_observation: StructuralObservation | None = Field( default=None, description="Versioned accessibility evidence observed at action time", ) + window_geometry_generation: int | None = Field( + default=None, + ge=1, + description=( + "Exact WindowEvent geometry generation bound to a window-scoped action" + ), + ) # ============================================================================= diff --git a/openadapt_capture/processing.py b/openadapt_capture/processing.py index 49daba5..552fcaa 100644 --- a/openadapt_capture/processing.py +++ b/openadapt_capture/processing.py @@ -90,6 +90,28 @@ def _first_structural_observation( return None +def _first_binding_timestamp(events: list[ActionEvent], field: str) -> float | None: + """Retain the first primitive event's exact evidence timestamp.""" + for event in events: + value = getattr(event, field) + if value is not None: + return value + return None + + +def _consistent_window_geometry_generation(events: list[ActionEvent]) -> int | None: + """Return one shared generation or reject a mixed merged action.""" + values = [event.window_geometry_generation for event in events] + present = [value for value in values if value is not None] + if not present: + return None + if len(present) != len(values) or any(value != present[0] for value in present[1:]): + raise ValueError( + "cannot merge action primitives with missing or mixed window geometry generations" + ) + return present[0] + + # ============================================================================= # Event Processing Functions # ============================================================================= @@ -247,9 +269,18 @@ def flush_buffer() -> None: modifiers=modifiers, key=trigger, children=list(keyboard_buffer), + screenshot_timestamp=_first_binding_timestamp( + list(keyboard_buffer), "screenshot_timestamp" + ), + window_event_timestamp=_first_binding_timestamp( + list(keyboard_buffer), "window_event_timestamp" + ), structural_observation=_first_structural_observation( list(keyboard_buffer) ), + window_geometry_generation=_consistent_window_geometry_generation( + list(keyboard_buffer) + ), ) ) else: @@ -263,9 +294,18 @@ def flush_buffer() -> None: timestamp=first_event.timestamp, text=text, children=list(keyboard_buffer), + screenshot_timestamp=_first_binding_timestamp( + list(keyboard_buffer), "screenshot_timestamp" + ), + window_event_timestamp=_first_binding_timestamp( + list(keyboard_buffer), "window_event_timestamp" + ), structural_observation=_first_structural_observation( list(keyboard_buffer) ), + window_geometry_generation=_consistent_window_geometry_generation( + list(keyboard_buffer) + ), ) result.append(type_event) @@ -325,9 +365,18 @@ def flush_buffer() -> None: timestamp=first.timestamp, x=last.x, y=last.y, + screenshot_timestamp=_first_binding_timestamp( + list(move_buffer), "screenshot_timestamp" + ), + window_event_timestamp=_first_binding_timestamp( + list(move_buffer), "window_event_timestamp" + ), structural_observation=_first_structural_observation( list(move_buffer) ), + window_geometry_generation=_consistent_window_geometry_generation( + list(move_buffer) + ), ) result.append(merged) @@ -377,9 +426,18 @@ def flush_buffer() -> None: y=first.y, dx=total_dx, dy=total_dy, + screenshot_timestamp=_first_binding_timestamp( + list(scroll_buffer), "screenshot_timestamp" + ), + window_event_timestamp=_first_binding_timestamp( + list(scroll_buffer), "window_event_timestamp" + ), structural_observation=_first_structural_observation( list(scroll_buffer) ), + window_geometry_generation=_consistent_window_geometry_generation( + list(scroll_buffer) + ), ) result.append(merged) @@ -484,9 +542,20 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: y=down.y, button=down.button, children=[down, up, next_down, next_up], + screenshot_timestamp=_first_binding_timestamp( + [down, up, next_down, next_up], + "screenshot_timestamp", + ), + window_event_timestamp=_first_binding_timestamp( + [down, up, next_down, next_up], + "window_event_timestamp", + ), structural_observation=_first_structural_observation( [down, up, next_down, next_up] ), + window_geometry_generation=_consistent_window_geometry_generation( + [down, up, next_down, next_up] + ), ) result.append(double_click) skip_timestamps.add(up.timestamp) @@ -501,9 +570,18 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: y=down.y, button=down.button, children=[down, up], + screenshot_timestamp=_first_binding_timestamp( + [down, up], "screenshot_timestamp" + ), + window_event_timestamp=_first_binding_timestamp( + [down, up], "window_event_timestamp" + ), structural_observation=_first_structural_observation( [down, up] ), + window_geometry_generation=_consistent_window_geometry_generation( + [down, up] + ), ) result.append(single_click) skip_timestamps.add(up.timestamp) @@ -568,9 +646,20 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: dy=event.y - down_event.y, button=down_event.button, children=[down_event] + moves + [event], + screenshot_timestamp=_first_binding_timestamp( + [down_event] + moves + [event], + "screenshot_timestamp", + ), + window_event_timestamp=_first_binding_timestamp( + [down_event] + moves + [event], + "window_event_timestamp", + ), structural_observation=_first_structural_observation( [down_event] + moves + [event] ), + window_geometry_generation=_consistent_window_geometry_generation( + [down_event] + moves + [event] + ), ) result.append(drag) else: diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index ead42c3..33f9048 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -31,6 +31,7 @@ import tracemalloc import uuid from collections import namedtuple +from dataclasses import dataclass from functools import partial from typing import Any, Callable @@ -63,12 +64,23 @@ observe_structural_action, ) from openadapt_capture.window_capture import ( + WindowCaptureError, WindowCaptureScope, build_window_scope, ) CoordinateScope = WindowCaptureScope | DesktopCaptureScope + +@dataclass(frozen=True) +class WindowScopedFrame: + """One atomic frame plus the exact geometry that produced it.""" + + image: Any + window_event_data: dict[str, Any] + geometry_generation: int + + try: import soundfile except ImportError: @@ -410,6 +422,21 @@ def process_events( # behavior undefined, swallow for now # XXX TODO: mitigate if event.type == "screen": + if isinstance(event.data, WindowScopedFrame): + scoped_frame = event.data + metadata_generation = scoped_frame.window_event_data.get("state", {}).get( + "geometry_generation" + ) + if metadata_generation != scoped_frame.geometry_generation: + raise WindowCaptureError( + "the scoped frame geometry generation does not match its metadata" + ) + prev_window_event = Event( + event.timestamp, + "window", + scoped_frame.window_event_data, + ) + event = event._replace(data=scoped_frame.image) prev_screen_event = event if config.RECORD_FULL_VIDEO: video_event = event._replace(type="screen/video") @@ -447,6 +474,17 @@ def process_events( # Window capture disabled — skip window timestamp requirement else: event.data["window_event_timestamp"] = prev_window_event.timestamp + action_generation = event.data.get("window_geometry_generation") + if action_generation is not None: + window_generation = prev_window_event.data.get("state", {}).get( + "geometry_generation" + ) + if action_generation != window_generation: + raise WindowCaptureError( + "the action geometry generation does not match its " + f"published frame: action={action_generation}, " + f"frame={window_generation}" + ) process_event( event, @@ -859,9 +897,19 @@ def trigger_action_event( # The recorder captures and validates its first scoped frame before # starting input. A translation failure after that boundary means # evidence is incomplete and must terminate the session. - wx, wy = coordinate_scope.translate(x, y) + if isinstance(coordinate_scope, WindowCaptureScope): + wx, wy, generation = coordinate_scope.translate_with_generation(x, y) + action_event_args["window_geometry_generation"] = generation + else: + wx, wy = coordinate_scope.translate(x, y) action_event_args["mouse_x"] = wx action_event_args["mouse_y"] = wy + elif isinstance(coordinate_scope, WindowCaptureScope): + # Keyboard actions have no pointer coordinate, but still need the + # exact target identity and frame geometry that existed at action time. + action_event_args["window_geometry_generation"] = ( + coordinate_scope.generation_for_action() + ) event_q.put( Event( event_timestamp, @@ -1054,23 +1102,15 @@ def read_screen_events( logger.info(f"Starting (fps={fps}, min_interval={min_interval:.3f}s)") started = False - announced_window = False while not terminate_processing.is_set(): t_start = time.perf_counter() if window_scope is not None: # Any failed capture terminates the session. Retrying would omit a # frame while input continues and could produce complete-looking # evidence with a missing interval. - screenshot, window_changed = window_scope.capture_frame() - if window_changed or not announced_window: - event_q.put( - Event( - utils.get_timestamp(), - "window", - window_scope.window_event_data(), - ) - ) - announced_window = True + screenshot, _window_changed = window_scope.capture_frame(publish=False) + window_event_data = window_scope.window_event_data() + generation = int(window_event_data["state"]["geometry_generation"]) elif desktop_scope is not None: # A monitor can move or change scale while the combined frame keeps # the same dimensions. Check both sides of the grab so neither the @@ -1084,10 +1124,29 @@ def read_screen_events( if screenshot is None: logger.warning("Screenshot was None") continue + frame_timestamp = utils.get_timestamp() + if window_scope is not None: + event_q.put( + Event( + frame_timestamp, + "screen", + WindowScopedFrame( + image=screenshot, + window_event_data=window_event_data, + geometry_generation=generation, + ), + ) + ) + # Action observers can use this geometry only after the exact + # frame/metadata pair has entered the shared ordered queue. + window_scope.publish_frame(generation) + else: + event_q.put(Event(frame_timestamp, "screen", screenshot)) if not started: + # Do not report this reader ready before its first exact frame is + # ordered and, for window scope, published for action translation. started_event.set() started = True - event_q.put(Event(utils.get_timestamp(), "screen", screenshot)) # Throttle: sleep for the remainder of the frame interval if min_interval > 0: elapsed = time.perf_counter() - t_start @@ -1681,7 +1740,7 @@ def record( initial_window_frame = None desktop_scope = None if window_scope is not None: - initial_window_frame, _ = window_scope.capture_frame() + initial_window_frame, _ = window_scope.capture_frame(publish=False) logger.info( f"window-scoped capture resolved: {window_scope.snapshot()} " f"initial frame {initial_window_frame.size}" diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index a4c5f1e..ebf3ca6 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -33,16 +33,20 @@ from __future__ import annotations +import math +import os import sys import threading from dataclasses import dataclass -from typing import TYPE_CHECKING, Callable, Optional +from typing import TYPE_CHECKING, Callable, Mapping, Optional from loguru import logger if TYPE_CHECKING: from PIL import Image +WINDOW_CAPTURE_SCHEMA_VERSION = "openadapt.capture.window-scoped/v2" + class WindowCaptureError(RuntimeError): """The target window could not be resolved or captured. @@ -109,6 +113,18 @@ class TargetWindow: pid: int bounds: tuple[float, float, float, float] on_screen: bool = True + process_start_time: float | None = None + coordinate_source: str = "platform-screen" + + @property + def identity(self) -> tuple[int, int, float | None, str]: + """Return the process-bound identity used for the whole session.""" + return ( + self.window_id, + self.pid, + self.process_start_time, + self.owner.casefold(), + ) def translate_point( @@ -167,7 +183,14 @@ def __init__( self._source_viewport: tuple[int, int] | None = None self._content_rect: tuple[int, int, int, int] | None = None self._fit_scale: float | None = None - self._bound_window_id: int | None = None + self._generation = 0 + self._geometry_signature: tuple | None = None + self._published_generation = 0 + self._published_window: TargetWindow | None = None + self._published_scale_x: float | None = None + self._published_scale_y: float | None = None + self._published_content_rect: tuple[int, int, int, int] | None = None + self._bound_identity: tuple[int, int, float | None, str] | None = None # Window of the last CAPTURED frame (not merely resolved): the # bounds-timeline 'changed' flag compares frame to frame, so a bare # resolve() (e.g. a pre-flight existence check) never suppresses the @@ -187,9 +210,41 @@ def resolve(self) -> TargetWindow: f"title {self.target.title!r}; is the target application " "running with a visible window?" ) + if self.target.owner and self.target.owner.casefold() not in win.owner.casefold(): + raise WindowCaptureError( + "the window resolver returned an owner outside the configured selector: " + f"expected {self.target.owner!r}, got {win.owner!r}" + ) + if self.target.title and self.target.title.casefold() not in win.title.casefold(): + raise WindowCaptureError( + "the window resolver returned a title outside the configured selector: " + f"expected {self.target.title!r}, got {win.title!r}" + ) + if win.pid <= 0: + raise WindowCaptureError("the resolved target has no owning process identity") + if ( + win.process_start_time is None + or not math.isfinite(win.process_start_time) + or win.process_start_time <= 0 + ): + raise WindowCaptureError( + "the resolved target has no stable process start identity" + ) + if not win.coordinate_source.strip(): + raise WindowCaptureError("the resolved target has no coordinate source") return win - def capture_frame(self) -> tuple["Image.Image", bool]: + def _assert_bound_identity(self, win: TargetWindow) -> None: + """Reject a recycled window handle or a different owning process.""" + with self._lock: + bound_identity = self._bound_identity + if bound_identity is not None and win.identity != bound_identity: + raise WindowCaptureError( + "the resolved target changed window identity or owning process during recording: " + f"expected {bound_identity!r}, got {win.identity!r}" + ) + + def capture_frame(self, *, publish: bool = True) -> tuple["Image.Image", bool]: """Capture the target window's current pixels. Re-resolves the window first so bounds/scale can never disagree with @@ -210,18 +265,31 @@ def capture_frame(self) -> tuple["Image.Image", bool]: """ with self._lock: prev = self._frame_window - bound_window_id = self._bound_window_id - win = self.resolve() - if bound_window_id is not None and win.window_id != bound_window_id: + output_viewport = self._viewport + + # A capture is valid only when the process identity and geometry are + # unchanged on both sides of the platform grab. A move or resize can + # happen between those calls. Such a frame has no single coordinate + # space and must never enter a complete recording. + pre = self.resolve() + self._assert_bound_identity(pre) + source_image = self._capturer(pre) + post = self.resolve() + self._assert_bound_identity(post) + if pre.identity != post.identity: raise WindowCaptureError( - "the resolved target changed window identity during recording: " - f"expected {bound_window_id}, got {win.window_id}" + "the target process identity changed while a frame was captured" ) - source_image = self._capturer(win) + if pre.bounds != post.bounds: + raise WindowCaptureError( + "the target moved or resized while a frame was captured; " + "no action can bind to mixed frame geometry" + ) + win = post if source_image.width <= 0 or source_image.height <= 0: raise WindowCaptureError("window capture returned an empty frame") source_viewport = (source_image.width, source_image.height) - output_viewport = self._viewport or source_viewport + output_viewport = output_viewport or source_viewport output_width, output_height = output_viewport fit_scale = min( output_width / source_image.width, @@ -243,6 +311,16 @@ def capture_frame(self) -> tuple["Image.Image", bool]: bounds_h = win.bounds[3] or float(source_image.height) scale_x = fitted_width / bounds_w scale_y = fitted_height / bounds_h + geometry_signature = ( + win.identity, + win.bounds, + win.coordinate_source, + output_viewport, + source_viewport, + (offset_x, offset_y, fitted_width, fitted_height), + scale_x, + scale_y, + ) with self._lock: self._window = win # ``scale`` is the historical scalar field. Keep it as the x-axis @@ -255,8 +333,15 @@ def capture_frame(self) -> tuple["Image.Image", bool]: self._source_viewport = source_viewport self._content_rect = (offset_x, offset_y, fitted_width, fitted_height) self._fit_scale = fit_scale - self._bound_window_id = win.window_id + if self._bound_identity is None: + self._bound_identity = win.identity + if geometry_signature != self._geometry_signature: + self._generation += 1 + self._geometry_signature = geometry_signature self._frame_window = win + generation = self._generation + if publish: + self._publish_locked(generation) changed = ( prev is None or prev.window_id != win.window_id @@ -265,6 +350,24 @@ def capture_frame(self) -> tuple["Image.Image", bool]: ) return image, changed + def _publish_locked(self, generation: int) -> None: + """Expose one committed frame geometry to action observers.""" + if generation != self._generation or self._window is None: + raise WindowCaptureError( + f"cannot publish geometry generation {generation}; " + f"the current generation is {self._generation}" + ) + self._published_generation = generation + self._published_window = self._window + self._published_scale_x = self._scale_x + self._published_scale_y = self._scale_y + self._published_content_rect = self._content_rect + + def publish_frame(self, generation: int) -> None: + """Publish geometry after its window metadata and pixels enter the queue.""" + with self._lock: + self._publish_locked(generation) + def translate(self, x: float, y: float) -> tuple[float, float]: """Translate a global screen point into window-relative pixels. @@ -276,19 +379,52 @@ def translate(self, x: float, y: float) -> tuple[float, float]: :meth:`capture_frame` (no bounds are known yet, and guessing a coordinate space would be a silent wrong action). """ + px, py, _generation = self.translate_with_generation(x, y) + return px, py + + def _geometry_for_action( + self, + ) -> tuple[ + TargetWindow, + float, + float, + tuple[int, int, int, int], + int, + ]: + """Return the published geometry after exact live revalidation.""" with self._lock: - window = self._window - scale_x = self._scale_x - scale_y = self._scale_y - content_rect = self._content_rect + window = self._published_window + scale_x = self._published_scale_x + scale_y = self._published_scale_y + content_rect = self._published_content_rect + generation = self._published_generation if window is None or scale_x is None or scale_y is None or content_rect is None: raise WindowCaptureError( - "translate() called before the first captured frame; " + "an action arrived before the first captured frame; " "capture_frame() must succeed before input can be scoped" ) + live = self.resolve() + self._assert_bound_identity(live) + if not live.on_screen: + raise WindowCaptureError("the target window is not on screen at action time") + if live.bounds != window.bounds: + raise WindowCaptureError( + "the target moved or resized after the last published frame; " + "wait for a matching frame before recording input" + ) + return window, scale_x, scale_y, content_rect, generation + + def generation_for_action(self) -> int: + """Bind a non-pointer action to the exact current published frame.""" + return self._geometry_for_action()[4] + + def translate_with_generation(self, x: float, y: float) -> tuple[float, float, int]: + """Translate against the exact published frame after live revalidation.""" + window, scale_x, scale_y, content_rect, generation = self._geometry_for_action() return ( (x - window.bounds[0]) * scale_x + content_rect[0], (y - window.bounds[1]) * scale_y + content_rect[1], + generation, ) def window_event_data(self) -> dict: @@ -308,6 +444,7 @@ def window_event_data(self) -> dict: source_viewport = self._source_viewport content_rect = self._content_rect fit_scale = self._fit_scale + generation = self._generation if window is None: raise WindowCaptureError("no resolved window; call capture_frame() first") x, y, w, h = window.bounds @@ -319,9 +456,13 @@ def window_event_data(self) -> dict: "height": int(h), "window_id": str(window.window_id), "state": { + "schema_version": WINDOW_CAPTURE_SCHEMA_VERSION, "window_capture": True, "owner": window.owner, "pid": window.pid, + "process_start_time": window.process_start_time, + "coordinate_source": window.coordinate_source, + "geometry_generation": generation, "bounds": [x, y, w, h], "scale": scale, "scale_x": scale_x, @@ -350,7 +491,9 @@ def snapshot(self) -> dict: source_viewport = self._source_viewport content_rect = self._content_rect fit_scale = self._fit_scale + generation = self._generation data: dict = { + "schema_version": WINDOW_CAPTURE_SCHEMA_VERSION, "target": {"owner": self.target.owner, "title": self.target.title}, "coordinate_space": "window_pixels", } @@ -361,6 +504,9 @@ def snapshot(self) -> dict: "owner": window.owner, "title": window.title, "pid": window.pid, + "process_start_time": window.process_start_time, + "coordinate_source": window.coordinate_source, + "geometry_generation": generation, "initial_bounds": list(window.bounds), "scale": scale, "scale_x": scale_x, @@ -385,9 +531,9 @@ def resolve_window(target: WindowTarget) -> TargetWindow | None: return _resolve_window_macos(target) if sys.platform == "win32": return _resolve_window_windows(target) - raise WindowCaptureError( - f"window-scoped capture is not supported on {sys.platform} (supported: darwin, win32)" - ) + if sys.platform.startswith("linux"): + return _resolve_window_linux(target) + raise WindowCaptureError(f"window-scoped capture is not supported on {sys.platform}") def capture_window(window: TargetWindow) -> "Image.Image": @@ -396,9 +542,21 @@ def capture_window(window: TargetWindow) -> "Image.Image": return _capture_window_macos(window) if sys.platform == "win32": return _capture_window_windows(window) - raise WindowCaptureError( - f"window-scoped capture is not supported on {sys.platform} (supported: darwin, win32)" - ) + if sys.platform.startswith("linux"): + return _capture_window_linux(window) + raise WindowCaptureError(f"window-scoped capture is not supported on {sys.platform}") + + +def _process_start_time(pid: int) -> float: + """Return a stable process creation identity or fail closed.""" + import psutil + + try: + return float(psutil.Process(pid).create_time()) + except psutil.Error as exc: + raise WindowCaptureError( + f"could not bind window owner PID {pid} to its process start time: {exc}" + ) from exc def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: @@ -414,8 +572,7 @@ def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: owner_l = target.owner.lower() if target.owner else None title_l = target.title.lower() if target.title else None wins = Quartz.CGWindowListCopyWindowInfo(Quartz.kCGWindowListOptionAll, Quartz.kCGNullWindowID) - best: TargetWindow | None = None - best_area = -1.0 + matches: list[TargetWindow] = [] for w in wins or []: owner = str(w.get("kCGWindowOwnerName", "") or "") name = str(w.get("kCGWindowName", "") or "") @@ -432,18 +589,31 @@ def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: float(b.get("Width", 0.0)), float(b.get("Height", 0.0)), ) - area = bounds[2] * bounds[3] - if area > best_area: - best_area = area - best = TargetWindow( - window_id=int(w.get("kCGWindowNumber", 0) or 0), - owner=owner, - title=name, - pid=int(w.get("kCGWindowOwnerPID", 0) or 0), - bounds=bounds, - on_screen=bool(w.get("kCGWindowIsOnscreen", False)), + if bounds[2] > 0 and bounds[3] > 0: + pid = int(w.get("kCGWindowOwnerPID", 0) or 0) + matches.append( + TargetWindow( + window_id=int(w.get("kCGWindowNumber", 0) or 0), + owner=owner, + title=name, + pid=pid, + bounds=bounds, + on_screen=bool(w.get("kCGWindowIsOnscreen", False)), + process_start_time=_process_start_time(pid), + coordinate_source="quartz-screen-points", + ) ) - return best + if not matches: + return None + # Prefer a visible target. Hidden windows remain resolvable for capture, + # but cannot receive a safe global pointer action. + return max( + matches, + key=lambda candidate: ( + candidate.on_screen, + candidate.bounds[2] * candidate.bounds[3], + ), + ) def _capture_window_macos(window: TargetWindow) -> "Image.Image": @@ -519,9 +689,11 @@ def _enum_cb(hwnd: int, _lparam: int) -> bool: pid = wintypes.DWORD() user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid)) try: - proc_name = psutil.Process(pid.value).name() + process = psutil.Process(pid.value) + proc_name = process.name() + process_start_time = float(process.create_time()) except psutil.Error: - proc_name = "" + return True if owner_l is not None and owner_l not in proc_name.lower(): return True rect = _window_rect(hwnd) @@ -541,6 +713,8 @@ def _enum_cb(hwnd: int, _lparam: int) -> bool: float(bottom - top), ), on_screen=True, + process_start_time=process_start_time, + coordinate_source="dwm-physical-pixels", ) ) return True @@ -552,24 +726,26 @@ def _enum_cb(hwnd: int, _lparam: int) -> bool: def _window_rect(hwnd: int) -> tuple[int, int, int, int] | None: - """Window rectangle in screen coordinates (DWM extended frame preferred).""" + """Return DWM physical bounds; never mix DPI-virtualized coordinates.""" import ctypes import ctypes.wintypes as wintypes rect = wintypes.RECT() try: - DWMWA_EXTENDED_FRAME_BOUNDS = 9 - res = ctypes.windll.dwmapi.DwmGetWindowAttribute( - wintypes.HWND(hwnd), - ctypes.wintypes.DWORD(DWMWA_EXTENDED_FRAME_BOUNDS), - ctypes.byref(rect), - ctypes.sizeof(rect), - ) - if res == 0: - return (rect.left, rect.top, rect.right, rect.bottom) - except (AttributeError, OSError): # pragma: no cover - dwmapi always present - pass - if not ctypes.windll.user32.GetWindowRect(wintypes.HWND(hwnd), ctypes.byref(rect)): + dwmapi = ctypes.windll.dwmapi + except (AttributeError, OSError): # pragma: no cover - supported Windows has DWM + return None + DWMWA_EXTENDED_FRAME_BOUNDS = 9 + res = dwmapi.DwmGetWindowAttribute( + wintypes.HWND(hwnd), + ctypes.wintypes.DWORD(DWMWA_EXTENDED_FRAME_BOUNDS), + ctypes.byref(rect), + ctypes.sizeof(rect), + ) + if res != 0: + # GetWindowRect is intentionally not a fallback. It is DPI virtualized + # for an unaware caller while MSS and low-level mouse hooks use physical + # screen coordinates. Mixing them can record a plausible wrong action. return None return (rect.left, rect.top, rect.right, rect.bottom) @@ -598,6 +774,263 @@ def _capture_window_windows(window: TargetWindow) -> "Image.Image": return Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") +def _require_x11_session(environ: Mapping[str, str] | None = None) -> str: + """Return DISPLAY for a native X11 session or fail closed on Wayland.""" + env = os.environ if environ is None else environ + if env.get("XDG_SESSION_TYPE", "").casefold() == "wayland" or env.get("WAYLAND_DISPLAY"): + raise WindowCaptureError( + "native Wayland window capture requires a compositor portal with " + "stable window identity and geometry; the X11 scope refuses XWayland-only capture" + ) + display_name = env.get("DISPLAY") + if not display_name: + raise WindowCaptureError("Linux window capture requires a native X11 DISPLAY") + return display_name + + +def _x11_library(display_name: str): + """Open one configured libX11 connection.""" + import ctypes + import ctypes.util + + from openadapt_capture.x11_threads import ensure_xlib_thread_support + + ensure_xlib_thread_support() + library_name = ctypes.util.find_library("X11") + if not library_name: + raise WindowCaptureError("Linux window capture requires the system libX11 runtime") + try: + x11 = ctypes.CDLL(library_name) + except OSError as exc: + raise WindowCaptureError(f"could not load libX11: {exc}") from exc + x11.XOpenDisplay.argtypes = [ctypes.c_char_p] + x11.XOpenDisplay.restype = ctypes.c_void_p + x11.XCloseDisplay.argtypes = [ctypes.c_void_p] + x11.XCloseDisplay.restype = ctypes.c_int + x11.XDefaultRootWindow.argtypes = [ctypes.c_void_p] + x11.XDefaultRootWindow.restype = ctypes.c_ulong + x11.XInternAtom.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int] + x11.XInternAtom.restype = ctypes.c_ulong + x11.XGetWindowProperty.argtypes = [ + ctypes.c_void_p, + ctypes.c_ulong, + ctypes.c_ulong, + ctypes.c_long, + ctypes.c_long, + ctypes.c_int, + ctypes.c_ulong, + ctypes.POINTER(ctypes.c_ulong), + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_ulong), + ctypes.POINTER(ctypes.c_ulong), + ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)), + ] + x11.XGetWindowProperty.restype = ctypes.c_int + x11.XGetGeometry.argtypes = [ + ctypes.c_void_p, + ctypes.c_ulong, + ctypes.POINTER(ctypes.c_ulong), + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_uint), + ctypes.POINTER(ctypes.c_uint), + ctypes.POINTER(ctypes.c_uint), + ctypes.POINTER(ctypes.c_uint), + ] + x11.XGetGeometry.restype = ctypes.c_int + x11.XTranslateCoordinates.argtypes = [ + ctypes.c_void_p, + ctypes.c_ulong, + ctypes.c_ulong, + ctypes.c_int, + ctypes.c_int, + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_ulong), + ] + x11.XTranslateCoordinates.restype = ctypes.c_int + x11.XFree.argtypes = [ctypes.c_void_p] + x11.XFree.restype = ctypes.c_int + display = x11.XOpenDisplay(display_name.encode()) + if not display: + raise WindowCaptureError( + f"could not open X11 display {display_name!r}; authorize the recording user" + ) + return x11, display + + +def _x11_property( + x11, + display, + window_id: int, + name: str, + *, + property_type: int = 0, +) -> tuple[int, int, bytes | list[int]] | None: + """Read one complete X11 property and release the Xlib allocation.""" + import ctypes + + atom = int(x11.XInternAtom(display, name.encode(), 1)) + if not atom: + return None + actual_type = ctypes.c_ulong() + actual_format = ctypes.c_int() + item_count = ctypes.c_ulong() + bytes_after = ctypes.c_ulong() + value = ctypes.POINTER(ctypes.c_ubyte)() + status = int( + x11.XGetWindowProperty( + display, + ctypes.c_ulong(window_id), + ctypes.c_ulong(atom), + 0, + 1 << 20, + 0, + ctypes.c_ulong(property_type), + ctypes.byref(actual_type), + ctypes.byref(actual_format), + ctypes.byref(item_count), + ctypes.byref(bytes_after), + ctypes.byref(value), + ) + ) + if status != 0 or not value: + return None + try: + if actual_format.value == 8: + data: bytes | list[int] = ctypes.string_at(value, item_count.value) + elif actual_format.value == 32: + longs = ctypes.cast(value, ctypes.POINTER(ctypes.c_ulong)) + data = [int(longs[index]) for index in range(item_count.value)] + else: + return None + return int(actual_type.value), int(actual_format.value), data + finally: + x11.XFree(value) + + +def _x11_text_property(x11, display, window_id: int, *names: str) -> str: + for name in names: + prop = _x11_property(x11, display, window_id, name) + if prop is None or not isinstance(prop[2], bytes): + continue + text = prop[2].decode("utf-8", errors="replace").rstrip("\x00") + if text: + return text + return "" + + +def _x11_geometry(x11, display, root: int, window_id: int): + """Return root-relative client bounds for one X11 window.""" + import ctypes + + returned_root = ctypes.c_ulong() + x = ctypes.c_int() + y = ctypes.c_int() + width = ctypes.c_uint() + height = ctypes.c_uint() + border = ctypes.c_uint() + depth = ctypes.c_uint() + if not x11.XGetGeometry( + display, + ctypes.c_ulong(window_id), + ctypes.byref(returned_root), + ctypes.byref(x), + ctypes.byref(y), + ctypes.byref(width), + ctypes.byref(height), + ctypes.byref(border), + ctypes.byref(depth), + ): + return None + root_x = ctypes.c_int() + root_y = ctypes.c_int() + child = ctypes.c_ulong() + if not x11.XTranslateCoordinates( + display, + ctypes.c_ulong(window_id), + ctypes.c_ulong(root), + 0, + 0, + ctypes.byref(root_x), + ctypes.byref(root_y), + ctypes.byref(child), + ): + return None + return ( + float(root_x.value), + float(root_y.value), + float(width.value), + float(height.value), + ) + + +def _resolve_window_linux(target: WindowTarget) -> TargetWindow | None: + """Resolve an EWMH client window in native X11 root-pixel coordinates.""" + display_name = _require_x11_session() + x11, display = _x11_library(display_name) + try: + root = int(x11.XDefaultRootWindow(display)) + client_list = _x11_property(x11, display, root, "_NET_CLIENT_LIST_STACKING") + if client_list is None or not isinstance(client_list[2], list): + raise WindowCaptureError( + "the X11 window manager does not expose _NET_CLIENT_LIST_STACKING" + ) + owner_l = target.owner.casefold() if target.owner else None + title_l = target.title.casefold() if target.title else None + matches: list[TargetWindow] = [] + for window_id in client_list[2]: + pid_property = _x11_property(x11, display, window_id, "_NET_WM_PID") + if pid_property is None or not isinstance(pid_property[2], list): + continue + pid = int(pid_property[2][0]) if pid_property[2] else 0 + if pid <= 0: + continue + import psutil + + try: + process = psutil.Process(pid) + owner = process.name() + process_start_time = float(process.create_time()) + except psutil.Error: + continue + title = _x11_text_property(x11, display, window_id, "_NET_WM_NAME", "WM_NAME") + if owner_l is not None and owner_l not in owner.casefold(): + continue + if title_l is not None and title_l not in title.casefold(): + continue + bounds = _x11_geometry(x11, display, root, window_id) + if bounds is None or bounds[2] <= 0 or bounds[3] <= 0: + continue + matches.append( + TargetWindow( + window_id=window_id, + owner=owner, + title=title, + pid=pid, + bounds=bounds, + on_screen=True, + process_start_time=process_start_time, + coordinate_source="x11-root-physical-pixels", + ) + ) + if not matches: + return None + # Stacking order is bottom-to-top. Prefer the largest match, then the + # top-most one for deterministic selection among equal surfaces. + return max( + enumerate(matches), + key=lambda item: (item[1].bounds[2] * item[1].bounds[3], item[0]), + )[1] + finally: + x11.XCloseDisplay(display) + + +def _capture_window_linux(window: TargetWindow) -> "Image.Image": + """Capture the exact X11 root rectangle through the shared MSS runtime.""" + return _capture_window_windows(window) + + def build_window_scope(owner: str | None, title: str | None) -> WindowCaptureScope | None: """Build a :class:`WindowCaptureScope` when a target is configured. diff --git a/pyproject.toml b/pyproject.toml index 8c5387a..eeaafd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,6 @@ authors = [ ] keywords = ["gui", "automation", "capture", "events", "rpa"] classifiers = [ - "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", diff --git a/scripts/verify_distribution.py b/scripts/verify_distribution.py index 970112a..3492bb0 100644 --- a/scripts/verify_distribution.py +++ b/scripts/verify_distribution.py @@ -15,6 +15,7 @@ FORBIDDEN_DEPENDENCIES = ("oa-atomacos", "pynput", "websockets") FORBIDDEN_SOURCE_TOKENS = ("oa_atomacos", "pynput", "EXECUTE_ACTION") +STATIC_LIFECYCLE_CLASSIFIER_PREFIX = "classifier: development status ::" FORBIDDEN_ARCHIVE_PATHS = ( ".env.example", ".github/", @@ -91,6 +92,10 @@ def verify_distribution(path: Path) -> None: ] assert metadata_files, f"{path}: package metadata is missing" metadata = "\n".join(metadata_files).lower() + assert STATIC_LIFECYCLE_CLASSIFIER_PREFIX not in metadata, ( + f"{path}: static lifecycle classifier is in package metadata; " + "the signed admission ledger owns release maturity" + ) for dependency in FORBIDDEN_DEPENDENCIES: assert f"requires-dist: {dependency}" not in metadata, ( f"{path}: forbidden dependency {dependency!r} is in package metadata" diff --git a/tests/test_changelog_contract.py b/tests/test_changelog_contract.py index 9bd0276..b41f4f8 100644 --- a/tests/test_changelog_contract.py +++ b/tests/test_changelog_contract.py @@ -2,6 +2,7 @@ import io import tarfile +import zipfile from pathlib import Path from types import SimpleNamespace @@ -62,6 +63,29 @@ def test_changelog_matches_project_version_and_release_config() -> None: assert releases[-1].version == "0.1.0" +def test_project_metadata_has_no_static_lifecycle_classifier() -> None: + _changelog, pyproject = _documents() + + assert "Development Status ::" not in pyproject + + +def test_distribution_refuses_a_static_lifecycle_classifier(tmp_path: Path) -> None: + wheel = tmp_path / "openadapt_capture-1.2.2-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("openadapt_capture/__init__.py", "") + archive.writestr("openadapt_capture-1.2.2.dist-info/licenses/LICENSE", "MIT\n") + archive.writestr( + "openadapt_capture-1.2.2.dist-info/METADATA", + "Metadata-Version: 2.4\n" + "Name: openadapt-capture\n" + "Version: 1.2.2\n" + "Classifier: Development Status :: 5 - Production/Stable\n", + ) + + with pytest.raises(AssertionError, match="static lifecycle classifier"): + verify_distribution(wheel) + + def test_changelog_refuses_a_project_version_without_release_notes() -> None: changelog, pyproject = _documents() pyproject, _ = _project_with_next_patch(changelog, pyproject) diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index 0783851..90be341 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -8,7 +8,7 @@ gated like the input-injection tests in tests/test_performance.py: marked 'slow', skipped on unsupported platforms and when OPENADAPT_CI_NO_INPUT_INJECTION=1 (hosted-runner session limitation). Run it -on an interactive macOS/Windows desktop (e.g. the Parallels rig): +on an interactive Linux, macOS, or Windows desktop: OPENADAPT_WINDOW_SMOKE_OWNER=Parallels pytest tests/test_window_capture.py -m slow """ @@ -26,12 +26,20 @@ from openadapt_capture.capture import CaptureSession from openadapt_capture.db import create_db, crud -from openadapt_capture.recorder import Recorder, read_screen_events +from openadapt_capture.db.models import Recording +from openadapt_capture.recorder import ( + Recorder, + WindowScopedFrame, + read_screen_events, +) from openadapt_capture.window_capture import ( + WINDOW_CAPTURE_SCHEMA_VERSION, TargetWindow, WindowCaptureError, WindowCaptureScope, WindowTarget, + _require_x11_session, + _x11_library, build_window_scope, translate_point, ) @@ -123,6 +131,23 @@ def test_build_window_scope_with_owner(self): assert isinstance(scope, WindowCaptureScope) assert scope.target.owner == "Parallels" + def test_native_wayland_fails_closed_even_when_xwayland_display_exists(self): + with pytest.raises(WindowCaptureError, match="Wayland"): + _require_x11_session( + { + "XDG_SESSION_TYPE": "wayland", + "WAYLAND_DISPLAY": "wayland-0", + "DISPLAY": ":1", + } + ) + + def test_x11_requires_display(self): + with pytest.raises(WindowCaptureError, match="DISPLAY"): + _require_x11_session({"XDG_SESSION_TYPE": "x11"}) + + def test_x11_display_contract(self): + assert _require_x11_session({"XDG_SESSION_TYPE": "x11", "DISPLAY": ":99"}) == ":99" + # --------------------------------------------------------------------------- # WindowCaptureScope with injected fakes (no display) @@ -137,6 +162,7 @@ def __init__(self, bounds=(300.0, 150.0, 800.0, 600.0), scale=2.0): self.scale = scale self.window_id = 42 self.title = "Fake Window" + self.process_start_time = 100.0 self.missing = False def resolver(self, target: WindowTarget): @@ -148,6 +174,8 @@ def resolver(self, target: WindowTarget): title=self.title, pid=1234, bounds=self.bounds, + process_start_time=self.process_start_time, + coordinate_source="fake-physical-pixels", ) def capturer(self, window: TargetWindow) -> Image.Image: @@ -227,7 +255,7 @@ def test_resize_uses_exact_axis_scales_after_integer_rounding(self, fake): ] ) scope = WindowCaptureScope( - WindowTarget(owner="Parallels"), + WindowTarget(owner="FakeApp"), resolver=fake.resolver, capturer=lambda _window: next(images), ) @@ -253,16 +281,17 @@ def test_translate_uses_latest_bounds(self, scope, fake): scope.capture_frame() assert scope.translate(310.0, 170.0) == (420.0, 240.0) - def test_resolve_does_not_mix_new_bounds_with_previous_frame(self, scope, fake): + def test_action_refuses_bounds_without_a_matching_frame(self, scope, fake): scope.capture_frame() assert scope.translate(310.0, 170.0) == (20.0, 40.0) fake.bounds = (100.0, 50.0, 800.0, 600.0) scope.resolve() - # A resolver poll alone cannot commit geometry. Translation changes - # only after the corresponding frame has been captured. - assert scope.translate(310.0, 170.0) == (20.0, 40.0) + # A resolver poll alone cannot commit geometry. An action must halt + # until a frame with the new geometry enters the event stream. + with pytest.raises(WindowCaptureError, match="last published frame"): + scope.translate(310.0, 170.0) scope.capture_frame() assert scope.translate(310.0, 170.0) == (420.0, 240.0) @@ -273,6 +302,59 @@ def test_window_identity_change_terminates_scope(self, scope, fake): with pytest.raises(WindowCaptureError, match="changed window identity"): scope.capture_frame() + def test_process_replacement_with_recycled_window_id_terminates_scope(self, scope, fake): + scope.capture_frame() + fake.process_start_time += 1.0 + + with pytest.raises(WindowCaptureError, match="owning process"): + scope.capture_frame() + + def test_move_during_capture_never_commits_mixed_geometry(self, fake): + before = fake.bounds + + def capture_then_move(window): + image = fake.capturer(window) + fake.bounds = (before[0] + 10.0, before[1], before[2], before[3]) + return image + + scope = WindowCaptureScope( + WindowTarget(owner="FakeApp"), + resolver=fake.resolver, + capturer=capture_then_move, + ) + + with pytest.raises(WindowCaptureError, match="while a frame was captured"): + scope.capture_frame() + assert scope.snapshot().get("geometry_generation") is None + + def test_generation_changes_only_when_geometry_changes(self, scope, fake): + scope.capture_frame() + first = scope.window_event_data()["state"] + scope.capture_frame() + second = scope.window_event_data()["state"] + fake.bounds = (100.0, 50.0, 800.0, 600.0) + scope.capture_frame() + third = scope.window_event_data()["state"] + + assert first["geometry_generation"] == 1 + assert second["geometry_generation"] == 1 + assert third["geometry_generation"] == 2 + assert second["process_start_time"] == 100.0 + assert second["coordinate_source"] == "fake-physical-pixels" + + def test_unpublished_frame_cannot_change_action_geometry(self, scope, fake): + scope.capture_frame() + assert scope.translate_with_generation(310.0, 170.0)[2] == 1 + fake.bounds = (100.0, 50.0, 800.0, 600.0) + scope.capture_frame(publish=False) + + with pytest.raises(WindowCaptureError, match="last published frame"): + scope.translate(310.0, 170.0) + + generation = scope.window_event_data()["state"]["geometry_generation"] + scope.publish_frame(generation) + assert scope.translate_with_generation(310.0, 170.0) == (420.0, 240.0, 2) + def test_missing_window_raises_loudly(self, scope, fake): fake.missing = True with pytest.raises(WindowCaptureError, match="no window matching"): @@ -290,6 +372,35 @@ def test_screen_reader_propagates_capture_failure_without_retry(self, scope, fak window_scope=scope, ) + def test_screen_reader_queues_frame_and_geometry_atomically(self, scope): + terminate = threading.Event() + + class OneFrameQueue(queue.Queue): + def put(self, item, *args, **kwargs): + super().put(item, *args, **kwargs) + terminate.set() + + events = OneFrameQueue() + started = threading.Event() + read_screen_events( + events, + terminate, + SimpleNamespace(timestamp=time.time()), + started, + window_scope=scope, + ) + + frame = events.get_nowait() + assert frame.type == "screen" + assert isinstance(frame.data, WindowScopedFrame) + assert frame.data.geometry_generation == frame.data.window_event_data[ + "state" + ]["geometry_generation"] + assert scope.translate_with_generation(310.0, 170.0)[2] == ( + frame.data.geometry_generation + ) + assert started.is_set() + def test_window_event_data_matches_window_event_columns(self, scope): scope.capture_frame() data = scope.window_event_data() @@ -308,6 +419,7 @@ def test_window_event_data_matches_window_event_columns(self, scope): assert data["window_id"] == "42" state = data["state"] assert state["window_capture"] is True + assert state["schema_version"] == WINDOW_CAPTURE_SCHEMA_VERSION assert state["scale"] == 2.0 assert state["bounds"] == [300.0, 150.0, 800.0, 600.0] assert state["viewport"] == [1600, 1200] @@ -322,6 +434,7 @@ def test_window_event_data_before_frame_raises(self, scope): def test_snapshot_shape(self, scope): scope.capture_frame() snap = scope.snapshot() + assert snap["schema_version"] == WINDOW_CAPTURE_SCHEMA_VERSION assert snap["coordinate_space"] == "window_pixels" assert snap["target"] == {"owner": "FakeApp", "title": None} assert snap["window_id"] == 42 @@ -405,6 +518,7 @@ def test_mouse_action_translated(self, scope): (event,) = self._drain(q) assert event.data["mouse_x"] == 20.0 assert event.data["mouse_y"] == 40.0 + assert event.data["window_geometry_generation"] == 1 def test_mouse_action_before_first_frame_fails_session(self, scope): import queue @@ -422,17 +536,35 @@ def test_mouse_action_before_first_frame_fails_session(self, scope): ) assert self._drain(q) == [] - def test_key_action_unaffected(self, scope): + def test_key_action_binds_to_exact_window_generation(self, scope): import queue from openadapt_capture import utils from openadapt_capture.recorder import trigger_action_event utils.set_start_time() + scope.capture_frame() q = queue.Queue() trigger_action_event(q, {"name": "press", "key_char": "a"}, scope) (event,) = self._drain(q) assert event.data["key_char"] == "a" + assert event.data["window_geometry_generation"] == 1 + + def test_key_action_refuses_stale_window_geometry(self, scope, fake): + import queue + + from openadapt_capture import utils + from openadapt_capture.recorder import trigger_action_event + + utils.set_start_time() + scope.capture_frame() + fake.bounds = (100.0, 50.0, 800.0, 600.0) + with pytest.raises(WindowCaptureError, match="last published frame"): + trigger_action_event( + queue.Queue(), + {"name": "press", "key_char": "a"}, + scope, + ) def test_no_scope_leaves_globals(self): import queue @@ -446,6 +578,34 @@ def test_no_scope_leaves_globals(self): (event,) = self._drain(q) assert event.data["mouse_x"] == 310.0 + def test_processed_click_rejects_mixed_geometry_generations(self): + from openadapt_capture.events import MouseDownEvent, MouseUpEvent + from openadapt_capture.processing import process_events + + events = [ + MouseDownEvent( + timestamp=1.0, + x=10, + y=20, + button="left", + screenshot_timestamp=0.9, + window_event_timestamp=0.9, + window_geometry_generation=1, + ), + MouseUpEvent( + timestamp=1.1, + x=10, + y=20, + button="left", + screenshot_timestamp=1.05, + window_event_timestamp=1.05, + window_geometry_generation=2, + ), + ] + + with pytest.raises(ValueError, match="mixed window geometry"): + process_events(events) + # --------------------------------------------------------------------------- # Persistence: capture_window config JSON round-trips through CaptureSession @@ -495,15 +655,114 @@ def test_fullscreen_recording_has_no_window_capture(self, tmp_path): with CaptureSession.load(capture_path) as capture: assert capture.window_capture is None + def test_action_geometry_generation_round_trips(self, tmp_path): + capture_path = self._insert_recording(str(tmp_path / "cap")) + engine, Session = create_db(os.path.join(capture_path, "recording.db")) + session = Session() + recording = session.query(Recording).one() + evidence_timestamp = recording.timestamp + 0.01 + crud.insert_action_event( + session, + recording, + recording.timestamp + 0.1, + { + "name": "move", + "mouse_x": 10.0, + "mouse_y": 20.0, + "screenshot_timestamp": evidence_timestamp, + "window_event_timestamp": evidence_timestamp, + "window_geometry_generation": 7, + }, + ) + session.close() + engine.dispose() + + with CaptureSession.load(capture_path) as capture: + raw = capture.raw_events() + assert raw[0].window_geometry_generation == 7 + assert raw[0].screenshot_timestamp == evidence_timestamp + assert raw[0].window_event_timestamp == evidence_timestamp + action = next(capture.actions(include_moves=True)) + assert action.screenshot_timestamp == evidence_timestamp + assert action.window_event_timestamp == evidence_timestamp + + def test_public_window_event_view_validates_v2_geometry(self, tmp_path, scope): + scope.capture_frame() + capture_path = self._insert_recording(str(tmp_path / "cap")) + engine, Session = create_db(os.path.join(capture_path, "recording.db")) + session = Session() + recording = session.query(Recording).one() + timestamp = recording.timestamp + 0.01 + crud.insert_window_event( + session, + recording, + timestamp, + scope.window_event_data(), + ) + session.close() + engine.dispose() + + with CaptureSession.load(capture_path) as capture: + (event,) = capture.window_events() + assert event.timestamp == timestamp + assert event.window_capture_v2 is not None + assert event.window_capture_v2.geometry_generation == 1 + assert capture.window_capture_events_v2() == [event] + + def test_public_v2_window_event_rejects_column_mismatch(self, tmp_path, scope): + scope.capture_frame() + capture_path = self._insert_recording(str(tmp_path / "cap")) + engine, Session = create_db(os.path.join(capture_path, "recording.db")) + session = Session() + recording = session.query(Recording).one() + event_data = scope.window_event_data() + event_data["left"] += 1 + crud.insert_window_event( + session, + recording, + recording.timestamp + 0.01, + event_data, + ) + session.close() + engine.dispose() + + with CaptureSession.load(capture_path) as capture: + with pytest.raises(ValueError, match="columns do not match"): + capture.window_capture_events_v2() + + def test_legacy_action_table_adds_nullable_generation(self, tmp_path): + capture_path = self._insert_recording(str(tmp_path / "legacy")) + database_path = os.path.join(capture_path, "recording.db") + import sqlite3 + + connection = sqlite3.connect(database_path) + connection.execute( + "ALTER TABLE action_event DROP COLUMN window_geometry_generation" + ) + connection.commit() + connection.close() + + with CaptureSession.load(capture_path): + pass + + connection = sqlite3.connect(database_path) + columns = { + row[1] + for row in connection.execute("PRAGMA table_info(action_event)") + } + connection.close() + assert "window_geometry_generation" in columns + # --------------------------------------------------------------------------- # Live smoke test: capture a REAL window (interactive desktop only) # --------------------------------------------------------------------------- -_ON_SUPPORTED_PLATFORM = sys.platform in ("darwin", "win32") +_ON_SUPPORTED_PLATFORM = sys.platform in ("darwin", "win32") or sys.platform.startswith( + "linux" +) _PLATFORM_SKIP_REASON = ( - "window-scoped capture supports macOS (CGWindowListCreateImage) and " - "Windows (Win32 + mss region grab) only; no Linux implementation" + "window-scoped capture supports macOS, Windows, and native X11 Linux" ) # Same gate as the input-injection tests in tests/test_performance.py: on # hosted CI runners the job executes in a non-interactive session, so there @@ -514,13 +773,17 @@ def test_fullscreen_recording_has_no_window_capture(self, tmp_path): _SESSION_SKIP_REASON = ( "OPENADAPT_CI_NO_INPUT_INJECTION=1: non-interactive hosted-runner session " "has no guaranteed capturable application window (hosted CI limitation, " - "not a window-capture bug); run on an interactive macOS/Windows desktop" + "not a window-capture bug); run on an interactive Linux, macOS, or Windows desktop" ) # Default smoke target: a window that exists on any logged-in desktop. # Override on the rig: OPENADAPT_WINDOW_SMOKE_OWNER=Parallels _SMOKE_OWNER = os.environ.get( "OPENADAPT_WINDOW_SMOKE_OWNER", - "Finder" if sys.platform == "darwin" else "explorer", + ( + "Finder" + if sys.platform == "darwin" + else "explorer" if sys.platform == "win32" else "gnome-terminal-server" + ), ) _SMOKE_TITLE = os.environ.get("OPENADAPT_WINDOW_SMOKE_TITLE") or None @@ -752,6 +1015,60 @@ def _temporary_macos_geometry(window: TargetWindow): ) +@contextmanager +def _temporary_linux_geometry(window: TargetWindow): + """Move and resize one X11 client, then restore its exact root geometry.""" + import ctypes + + display_name = _require_x11_session() + x11, display = _x11_library(display_name) + x11.XMoveResizeWindow.argtypes = [ + ctypes.c_void_p, + ctypes.c_ulong, + ctypes.c_int, + ctypes.c_int, + ctypes.c_uint, + ctypes.c_uint, + ] + x11.XMoveResizeWindow.restype = ctypes.c_int + x11.XSync.argtypes = [ctypes.c_void_p, ctypes.c_int] + x11.XSync.restype = ctypes.c_int + + original_x, original_y, original_width, original_height = window.bounds + target_width = max(320, round(original_width) - 137) + target_height = max(240, round(original_height) - 83) + if target_width == round(original_width): + target_width += 137 + if target_height == round(original_height): + target_height += 83 + + mutated = False + try: + x11.XMoveResizeWindow( + display, + ctypes.c_ulong(window.window_id), + round(original_x) + 37, + round(original_y) + 29, + target_width, + target_height, + ) + x11.XSync(display, 0) + mutated = True + yield + finally: + if mutated: + x11.XMoveResizeWindow( + display, + ctypes.c_ulong(window.window_id), + round(original_x), + round(original_y), + round(original_width), + round(original_height), + ) + x11.XSync(display, 0) + x11.XCloseDisplay(display) + + @contextmanager def _temporary_window_geometry(window: TargetWindow): """Dispatch a reversible live geometry change to the current platform.""" @@ -763,6 +1080,10 @@ def _temporary_window_geometry(window: TargetWindow): with _temporary_macos_geometry(window): yield return + if sys.platform.startswith("linux"): + with _temporary_linux_geometry(window): + yield + return raise AssertionError(f"no live window geometry controller for {sys.platform}") @@ -775,7 +1096,7 @@ class TestWindowCaptureLive: def _scope(self) -> WindowCaptureScope: scope = WindowCaptureScope(WindowTarget(owner=_SMOKE_OWNER, title=_SMOKE_TITLE)) try: - scope.resolve() + target = scope.resolve() except WindowCaptureError as exc: if _PRODUCTION_QUALIFICATION: raise AssertionError( @@ -787,6 +1108,12 @@ def _scope(self) -> WindowCaptureScope: f"title {_SMOKE_TITLE!r} on this desktop; open one (or set " "OPENADAPT_WINDOW_SMOKE_OWNER) to run the live smoke test" ) + if not target.on_screen: + if _PRODUCTION_QUALIFICATION: + raise AssertionError( + "production qualification requires the target window to be on screen" + ) + pytest.skip("the resolved smoke target is not currently on screen") return scope def test_live_window_frame_and_translation(self): From 30a28ac94d66d4f64d1180d758345389abb7bfe8 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 20 Aug 2026 12:11:31 -0400 Subject: [PATCH 2/3] feat: add macOS and Linux structural observers --- README.md | 39 +- docs/DESIGN.md | 25 +- openadapt_capture/structural.py | 50 +- .../structural_observer/linux.py | 458 ++++++++++++++++++ .../structural_observer/macos.py | 292 +++++++++++ pyproject.toml | 8 +- tests/test_structural_observation.py | 336 ++++++++++++- 7 files changed, 1167 insertions(+), 41 deletions(-) create mode 100644 openadapt_capture/structural_observer/linux.py create mode 100644 openadapt_capture/structural_observer/macos.py diff --git a/README.md b/README.md index e920825..269b374 100644 --- a/README.md +++ b/README.md @@ -191,17 +191,19 @@ Desktop provisions and probes that exact closure. ## Native structural observations -On Windows, the recorder can retain a versioned UI Automation observation -beside the native action that produced it. When UIA exposes the information, -the observation includes the target's AutomationId, role/control type, name, -bounds, supported patterns, process/window identity, ancestry, and exact -candidate cardinality within the top-level window. Unavailable values remain -absent; capture never infers structural fields from coordinates or pixels. +The recorder can retain a versioned native accessibility observation beside +the action that produced it. Capture uses Windows UI Automation, macOS +Accessibility, or Linux AT-SPI. When a provider exposes the information, the +observation includes the target identifier, role/control type, name, bounds, +supported actions, process/window identity, and ancestry. Windows UIA also +records exact candidate cardinality within the top-level window. Unavailable +values remain absent; Capture never infers structural fields from coordinates +or pixels. This evidence is stored in `recording.db`, exposed on raw events and processed -`Action` objects, and remains optional so existing recordings and non-Windows -hosts continue to load unchanged. It is enabled by default on Windows and can -be disabled with +`Action` objects, and remains optional so existing recordings continue to load +unchanged. It is enabled by default when the native provider is available. It +can be disabled with `Recorder(..., capture_structural_observations=False)`. Applications can inject another read-only observer through `Recorder(..., structural_observer=...)` using the public `StructuralObserver` protocol. @@ -209,12 +211,16 @@ using the public `StructuralObserver` protocol. Accessibility text remains inside the local raw-capture boundary and is bounded to 512 characters per field. Longer provider values are omitted rather than truncated, so partial text is never presented to the compiler as exact identity -evidence. The versioned observation contract accepts namespaced provider IDs for -future macOS Accessibility and Linux AT-SPI observers, but this package currently -emits only `windows_uia` observations. +evidence. Capture emits `windows_uia`, `macos_ax`, or `linux_atspi`. macOS +requires Accessibility permission. Linux requires an available desktop AT-SPI +bus, the system AT-SPI typelib/runtime, and the `linux` package extra: -UIA describes the local Windows accessibility tree. It does not cross an -RDP/Citrix pixel boundary into the remote application; those demonstrations +```bash +pip install "openadapt-capture[linux]" +``` + +The native provider describes the local accessibility tree. It does not cross +an RDP/Citrix pixel boundary into the remote application; those demonstrations retain window-scoped pixels and coordinates for Flow's remote visual compiler. ## Window-scoped recording @@ -376,8 +382,9 @@ session is required. See the - Native recording requires a visible user session plus the operating system's screen-recording and input-monitoring permissions. -- Native Windows capture retains UIA evidence when the application exposes it; - opaque remote applications still require Flow's visual and OCR bindings. +- Native Windows, macOS, and Linux capture retain accessibility evidence when + the application and local provider expose it. Opaque remote applications + still require Flow's visual and OCR bindings. - The Flow adapter compiles left and right clicks, left-button drags, typed text, named keys, modifier chords, and scrolling. It rejects unsupported input such as middle clicks, non-left-button drags, malformed shortcuts, and diff --git a/docs/DESIGN.md b/docs/DESIGN.md index bb222f5..00745a2 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -29,9 +29,9 @@ operation exports it. | Browser | `openadapt-flow` owns the supported Playwright recorder. The Chrome extension and bridge in this repository are source-only development prototypes and are excluded from Capture release artifacts. | Capture supports native input observation on macOS, Windows, and X11 Linux. -Windows can also retain UI Automation evidence at action time. The structural -schema permits another injected provider, but the package does not currently -ship macOS Accessibility or Linux AT-SPI structural observers. +It retains action-time structure through Windows UI Automation, macOS +Accessibility, and Linux AT-SPI when the local provider is available. The +public structural protocol also accepts an injected read-only provider. ## Session pipeline @@ -139,14 +139,19 @@ schema can retain: - bounded ancestry - exact candidate cardinality and its matching fields -The package currently creates a Windows UIA observer. A missing optional field -stays missing. Capture does not infer an accessibility value from a screenshot, -coordinate, or neighboring control. Provider text has strict length and depth -bounds. A transient provider failure omits the optional observation without -corrupting the screen and input evidence. +The package creates a Windows UIA, macOS Accessibility, or Linux AT-SPI +observer for the current platform. A missing optional field stays missing. +Capture does not infer an accessibility value from a screenshot, coordinate, +or neighboring control. Provider text has strict length and depth bounds. A +transient provider failure omits the optional observation without corrupting +the screen and input evidence. -UIA describes the local accessibility tree. It does not describe controls -inside an RDP or Citrix pixel stream. +The Linux provider uses the modern GObject AT-SPI binding. The `linux` package +extra installs PyGObject. The host supplies the AT-SPI typelib/runtime and an +interactive desktop accessibility bus. + +The native provider describes the local accessibility tree. It does not +describe controls inside an RDP or Citrix pixel stream. ## Video and frame timing diff --git a/openadapt_capture/structural.py b/openadapt_capture/structural.py index 8301cf1..33260c9 100644 --- a/openadapt_capture/structural.py +++ b/openadapt_capture/structural.py @@ -8,11 +8,12 @@ from __future__ import annotations import logging +import math import sys from dataclasses import dataclass from typing import Annotated, Literal, Protocol, runtime_checkable -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator STRUCTURAL_OBSERVATION_SCHEMA_VERSION = ( "openadapt.capture.structural-observation/v1" @@ -40,6 +41,16 @@ class StructuralBounds(BaseModel): right: float bottom: float + @model_validator(mode="after") + def validate_bounds(self) -> "StructuralBounds": + """Reject a non-finite or inverted provider rectangle.""" + values = (self.left, self.top, self.right, self.bottom) + if not all(math.isfinite(value) for value in values): + raise ValueError("structural bounds must be finite") + if self.right < self.left or self.bottom < self.top: + raise ValueError("structural bounds must not be inverted") + return self + class StructuralElement(BaseModel): """Stable and semantic fields exposed for one accessibility element.""" @@ -117,8 +128,8 @@ class StructuralObservation(BaseModel): ] = STRUCTURAL_OBSERVATION_SCHEMA_VERSION provider: _ProviderIdentifier = Field( description=( - "Accessibility-provider identifier. Capture currently emits " - "windows_uia; other providers remain opt-in observer extensions." + "Accessibility-provider identifier. Native Capture providers are " + "windows_uia, macos_ax, and linux_atspi." ), ) event_timestamp: float @@ -167,17 +178,32 @@ def create_structural_observer( if not enabled: return None resolved_platform = platform_name or sys.platform - if resolved_platform != "win32": - return None - try: - from openadapt_capture.structural_observer.windows import ( - WindowsUIAStructuralObserver, - ) - - return WindowsUIAStructuralObserver() + if resolved_platform == "win32": + from openadapt_capture.structural_observer.windows import ( + WindowsUIAStructuralObserver, + ) + + return WindowsUIAStructuralObserver() + if resolved_platform == "darwin": + from openadapt_capture.structural_observer.macos import ( + MacOSAXStructuralObserver, + ) + + return MacOSAXStructuralObserver() + if resolved_platform.startswith("linux"): + from openadapt_capture.structural_observer.linux import ( + LinuxATSpiStructuralObserver, + ) + + return LinuxATSpiStructuralObserver() + return None except Exception as exc: - _logger.warning("Windows UIA structural observation is unavailable: %s", exc) + _logger.warning( + "%s structural observation is unavailable: %s", + resolved_platform, + exc, + ) return None diff --git a/openadapt_capture/structural_observer/linux.py b/openadapt_capture/structural_observer/linux.py new file mode 100644 index 0000000..14f8397 --- /dev/null +++ b/openadapt_capture/structural_observer/linux.py @@ -0,0 +1,458 @@ +"""Action-time Linux AT-SPI observations. + +The system AT-SPI GI binding is imported only when this observer is +constructed. Native Wayland and X11 applications both expose structure through +AT-SPI when the desktop accessibility bus is available. +""" + +from __future__ import annotations + +import logging +import threading +import time +from collections.abc import Callable +from typing import Any + +from openadapt_capture.structural import ( + MAX_STRUCTURAL_ANCESTRY_DEPTH, + MAX_STRUCTURAL_TEXT_LENGTH, + StructuralAncestor, + StructuralBounds, + StructuralElement, + StructuralObservation, + StructuralObservationRequest, + StructuralProcessIdentity, + StructuralWindowIdentity, +) + +_logger = logging.getLogger(__name__) + + +def _text(value: Any) -> str | None: + if not isinstance(value, str): + return None + value = value.strip() + if not value or len(value) > MAX_STRUCTURAL_TEXT_LENGTH: + return None + return value + + +def _integer(value: Any) -> int | None: + if isinstance(value, bool): + return None + try: + value = int(value) + except (TypeError, ValueError): + return None + return value if value >= 0 else None + + +def _call(element: Any, *names: str) -> Any: + for name in names: + method = getattr(element, name, None) + if not callable(method): + continue + try: + return method() + except Exception: + continue + return None + + +class _GIAtspiRuntime: + """Bounded adapter for the modern GObject AT-SPI binding.""" + + _MAX_NODES = 4096 + + def __init__(self) -> None: + try: + import gi + + gi.require_version("Atspi", "2.0") + from gi.repository import Atspi + except Exception as exc: + raise RuntimeError( + "Linux structural observation requires PyGObject and the " + "system AT-SPI typelib/runtime" + ) from exc + self.atspi = Atspi + try: + initialized = bool(Atspi.is_initialized()) + status = 0 if initialized else int(Atspi.init()) + except Exception as exc: + raise RuntimeError( + "could not initialize the AT-SPI accessibility registry" + ) from exc + if status != 0: + raise RuntimeError( + f"AT-SPI accessibility registry initialization failed ({status})" + ) + try: + self.desktop = Atspi.get_desktop(0) + except Exception as exc: + raise RuntimeError("AT-SPI did not expose the desktop") from exc + if self.desktop is None: + raise RuntimeError("AT-SPI did not expose a desktop accessibility tree") + + def _screen_coordinates(self) -> Any: + return self.atspi.CoordType.SCREEN + + def _state_contains(self, element: Any, name: str) -> bool: + state = getattr(self.atspi.StateType, name, None) + state_set = _call(element, "get_state_set", "getState") + if state is None or state_set is None: + return False + try: + return bool(state_set.contains(state)) + except Exception: + return False + + def children(self, element: Any) -> list[Any]: + count = _integer( + _call(element, "get_child_count", "getChildCount") + ) + if count is None: + return [] + children: list[Any] = [] + for index in range(min(count, self._MAX_NODES)): + try: + child = element.get_child_at_index(index) + except Exception: + try: + child = element.getChildAtIndex(index) + except Exception: + continue + if child is not None: + children.append(child) + return children + + def component(self, element: Any) -> Any: + return _call( + element, + "get_component_iface", + "queryComponent", + "get_component", + ) + + def bounds(self, element: Any) -> StructuralBounds | None: + component = self.component(element) + if component is None: + return None + try: + extents = component.get_extents(self._screen_coordinates()) + except Exception: + try: + extents = component.getExtents(self._screen_coordinates()) + except Exception: + return None + try: + left = float(extents.x) + top = float(extents.y) + width = float(extents.width) + height = float(extents.height) + return StructuralBounds( + left=left, + top=top, + right=left + width, + bottom=top + height, + ) + except (AttributeError, TypeError, ValueError): + return None + + def _child_at_point(self, element: Any, x: int, y: int) -> Any: + component = self.component(element) + if component is None: + return None + for method_name in ("get_accessible_at_point", "getAccessibleAtPoint"): + method = getattr(component, method_name, None) + if not callable(method): + continue + try: + return method(x, y, self._screen_coordinates()) + except Exception: + continue + return None + + def _deepest_at_point(self, root: Any, x: int, y: int) -> Any: + current = root + seen = {id(current)} + for _ in range(MAX_STRUCTURAL_ANCESTRY_DEPTH): + child = self._child_at_point(current, x, y) + if child is None or id(child) in seen: + return current + current = child + seen.add(id(current)) + # A deeper direct child means the bounded query did not establish the + # actual target. Omit the observation instead of returning an ancestor. + child = self._child_at_point(current, x, y) + return None if child is not None and id(child) not in seen else current + + def element_at_point(self, x: float, y: float) -> Any: + point_x, point_y = int(round(x)), int(round(y)) + + # The desktop Component, when available, owns cross-application + # stacking and gives the only unambiguous top-most child. Descend from + # that result because AT-SPI's point method returns one direct child. + direct = self._child_at_point(self.desktop, point_x, point_y) + if direct is not None: + return self._deepest_at_point(direct, point_x, point_y) + + # Some registries do not expose Component on the desktop. In that + # case, accept only one active visible top-level surface at the point. + # Do not guess between overlapping applications. + matches: list[Any] = [] + for application in self.children(self.desktop): + for surface in self.children(application) or [application]: + bounds = self.bounds(surface) + if bounds is None: + continue + if not ( + bounds.left <= point_x < bounds.right + and bounds.top <= point_y < bounds.bottom + ): + continue + if not self._state_contains(surface, "ACTIVE"): + continue + match = self._deepest_at_point(surface, point_x, point_y) + if match is not None: + matches.append(match) + return matches[0] if len(matches) == 1 else None + + def focused_element(self) -> Any: + stack = list(reversed(self.children(self.desktop))) + visited = 0 + found: list[Any] = [] + while stack and visited < self._MAX_NODES: + element = stack.pop() + visited += 1 + if self._state_contains(element, "FOCUSED"): + found.append(element) + stack.extend(reversed(self.children(element))) + if stack: + return None + # Multiple focused nodes make the structural binding ambiguous. + return found[0] if len(found) == 1 else None + + def parent(self, element: Any) -> Any: + value = getattr(element, "parent", None) + return value if value is not None else _call(element, "get_parent", "getParent") + + def attributes(self, element: Any) -> dict[str, str]: + raw = _call(element, "getAttributes", "get_attributes") + if isinstance(raw, dict): + return {str(key): str(value) for key, value in raw.items()} + result: dict[str, str] = {} + if isinstance(raw, (list, tuple)): + for item in raw: + key, separator, value = str(item).partition(":") + if separator: + result[key] = value + return result + + def role_name(self, element: Any) -> str | None: + return _text(_call(element, "get_role_name", "getRoleName")) + + def process_id(self, element: Any) -> int | None: + current = element + for _ in range(MAX_STRUCTURAL_ANCESTRY_DEPTH): + process_id = _integer( + _call(current, "get_process_id", "getProcessId") + ) + if process_id is not None: + return process_id + parent = self.parent(current) + if parent is None: + break + current = parent + return None + + def action_names(self, element: Any) -> list[str] | None: + action = _call(element, "get_action_iface", "queryAction", "get_action") + if action is None: + return None + count = _integer( + _call(action, "get_n_actions", "getNActions") + ) + if count is None: + return None + names = [] + for index in range(min(count, 64)): + try: + name = action.get_action_name(index) + except Exception: + try: + name = action.getName(index) + except Exception: + continue + if (value := _text(name)) is not None: + names.append(value) + return names or None + + +def _process_name(pid: int) -> str | None: + try: + import psutil + + return _text(psutil.Process(pid).name()) + except Exception: + return None + + +def _fields(runtime: Any, element: Any) -> dict[str, Any]: + attributes = runtime.attributes(element) + role = runtime.role_name(element) + name = _text(getattr(element, "name", None)) or _text( + _call(element, "get_name", "getName") + ) + return { + "automation_id": _text( + attributes.get("accessible-id") or attributes.get("id") + ), + "role": role, + "role_source": "linux_atspi_role" if role else None, + "control_type": role, + "name": name, + "class_name": _text( + attributes.get("class") or attributes.get("toolkit") + ), + "bounds": runtime.bounds(element), + "supported_patterns": runtime.action_names(element), + } + + +def _ancestry(runtime: Any, element: Any, maximum_depth: int): + result: list[StructuralAncestor] = [] + current = element + for _ in range(maximum_depth): + current = runtime.parent(current) + if current is None: + break + fields = _fields(runtime, current) + fields.pop("supported_patterns", None) + ancestor = StructuralAncestor(**fields) + if ancestor.model_dump(exclude_none=True): + result.append(ancestor) + return result or None + + +def _window(runtime: Any, element: Any) -> StructuralWindowIdentity | None: + current = element + candidate = None + for _ in range(MAX_STRUCTURAL_ANCESTRY_DEPTH): + if (runtime.role_name(current) or "").casefold() in { + "alert", + "dialog", + "frame", + "window", + }: + candidate = current + break + parent = runtime.parent(current) + if parent is None: + break + current = parent + if candidate is None: + return None + fields = _fields(runtime, candidate) + identity = StructuralWindowIdentity( + title=fields.get("name"), + automation_id=fields.get("automation_id"), + class_name=fields.get("class_name"), + bounds=fields.get("bounds"), + ) + return identity if identity.model_dump(exclude_none=True) else None + + +class LinuxATSpiStructuralObserver: + """Read exact AT-SPI evidence for a pointer or focused action.""" + + def __init__( + self, + *, + runtime: Any | None = None, + runtime_factory: Callable[[], Any] = _GIAtspiRuntime, + process_name_resolver: Callable[[int], str | None] = _process_name, + clock: Callable[[], float] = time.time, + maximum_ancestry_depth: int = 12, + ) -> None: + if not 0 < maximum_ancestry_depth <= MAX_STRUCTURAL_ANCESTRY_DEPTH: + raise ValueError( + "maximum_ancestry_depth must be between 1 and " + f"{MAX_STRUCTURAL_ANCESTRY_DEPTH}" + ) + self._runtime = runtime + self._runtime_factory = runtime_factory + self._thread_state = threading.local() + self.process_name_resolver = process_name_resolver + self.clock = clock + self.maximum_ancestry_depth = maximum_ancestry_depth + + def open_current_thread(self) -> None: + """Create the AT-SPI runtime on the input delivery thread.""" + if self._runtime is not None: + return + if getattr(self._thread_state, "runtime", None) is not None: + return + if getattr(self._thread_state, "unavailable", False): + return + try: + self._thread_state.runtime = self._runtime_factory() + except Exception as exc: + self._thread_state.unavailable = True + _logger.warning("Linux AT-SPI observation is unavailable: %s", exc) + + def close_current_thread(self) -> None: + """Release a thread-owned AT-SPI adapter when it exposes a close hook.""" + if self._runtime is not None: + return + runtime = getattr(self._thread_state, "runtime", None) + self._thread_state.runtime = None + close = getattr(runtime, "close", None) + if callable(close): + close() + + def observe( + self, request: StructuralObservationRequest + ) -> StructuralObservation | None: + runtime = self._runtime + if runtime is None: + self.open_current_thread() + runtime = getattr(self._thread_state, "runtime", None) + if runtime is None: + return None + if request.x is not None and request.y is not None: + element = runtime.element_at_point(request.x, request.y) + query_kind = "point" + else: + element = runtime.focused_element() + query_kind = "focused" + if element is None: + return None + fields = _fields(runtime, element) + observed_element = StructuralElement(**fields) + if not observed_element.model_dump(exclude_none=True): + return None + pid = _integer(runtime.process_id(element)) + process = None + if pid is not None and pid > 0: + process = StructuralProcessIdentity( + process_id=pid, + process_name=self.process_name_resolver(pid), + ) + return StructuralObservation( + provider="linux_atspi", + event_timestamp=request.event_timestamp, + observed_at=self.clock(), + query_kind=query_kind, + element=observed_element, + process=process, + window=_window(runtime, element), + ancestry=_ancestry( + runtime, + element, + self.maximum_ancestry_depth, + ), + ) + + +__all__ = ["LinuxATSpiStructuralObserver"] diff --git a/openadapt_capture/structural_observer/macos.py b/openadapt_capture/structural_observer/macos.py new file mode 100644 index 0000000..9c2d8b1 --- /dev/null +++ b/openadapt_capture/structural_observer/macos.py @@ -0,0 +1,292 @@ +"""Action-time macOS Accessibility observations. + +ApplicationServices is imported only when this observer is constructed. The +module therefore remains safe to import on other platforms and in headless +package checks. +""" + +from __future__ import annotations + +import logging +import threading +import time +from collections.abc import Callable +from typing import Any + +from openadapt_capture.structural import ( + MAX_STRUCTURAL_ANCESTRY_DEPTH, + MAX_STRUCTURAL_TEXT_LENGTH, + StructuralAncestor, + StructuralBounds, + StructuralElement, + StructuralObservation, + StructuralObservationRequest, + StructuralProcessIdentity, + StructuralWindowIdentity, +) + +_logger = logging.getLogger(__name__) + + +def _text(value: Any) -> str | None: + if not isinstance(value, str): + return None + value = value.strip() + if not value or len(value) > MAX_STRUCTURAL_TEXT_LENGTH: + return None + return value + + +def _integer(value: Any) -> int | None: + if isinstance(value, bool): + return None + try: + value = int(value) + except (TypeError, ValueError): + return None + return value if value >= 0 else None + + +class _AXRuntime: + """Small PyObjC adapter for the native AX API.""" + + def __init__(self) -> None: + import ApplicationServices as ax + + if not bool(ax.AXIsProcessTrusted()): + raise PermissionError( + "macOS Accessibility permission is required for structural observation" + ) + self.ax = ax + self.system = ax.AXUIElementCreateSystemWide() + + def attribute(self, element: Any, name: str) -> Any: + error, value = self.ax.AXUIElementCopyAttributeValue(element, name, None) + if error != self.ax.kAXErrorSuccess: + return None + return value + + def element_at_point(self, x: float, y: float) -> Any: + error, element = self.ax.AXUIElementCopyElementAtPosition( + self.system, float(x), float(y), None + ) + if error != self.ax.kAXErrorSuccess: + return None + return element + + def focused_element(self) -> Any: + return self.attribute(self.system, "AXFocusedUIElement") + + def actions(self, element: Any) -> list[Any] | None: + error, values = self.ax.AXUIElementCopyActionNames(element, None) + if error != self.ax.kAXErrorSuccess or not isinstance(values, (list, tuple)): + return None + return list(values) + + def process_id(self, element: Any) -> int | None: + result = self.ax.AXUIElementGetPid(element, None) + if isinstance(result, tuple): + error, pid = result + if error != self.ax.kAXErrorSuccess: + return None + return _integer(pid) + return _integer(result) + + def _geometry(self, value: Any, value_type: int) -> Any: + if value is None: + return None + success, geometry = self.ax.AXValueGetValue(value, value_type, None) + return geometry if success else None + + def bounds(self, element: Any) -> StructuralBounds | None: + position = self._geometry( + self.attribute(element, "AXPosition"), + self.ax.kAXValueCGPointType, + ) + size = self._geometry( + self.attribute(element, "AXSize"), + self.ax.kAXValueCGSizeType, + ) + if position is None or size is None: + return None + try: + left = float(position.x) + top = float(position.y) + width = float(size.width) + height = float(size.height) + return StructuralBounds( + left=left, + top=top, + right=left + width, + bottom=top + height, + ) + except (AttributeError, TypeError, ValueError): + return None + + +def _process_name(pid: int) -> str | None: + try: + import psutil + + return _text(psutil.Process(pid).name()) + except Exception: + return None + + +def _element_fields(runtime: Any, element: Any) -> dict[str, Any]: + role = _text(runtime.attribute(element, "AXRole")) + title = _text(runtime.attribute(element, "AXTitle")) + description = _text(runtime.attribute(element, "AXDescription")) + action_reader = getattr(runtime, "actions", None) + actions = action_reader(element) if callable(action_reader) else None + supported_patterns = None + if isinstance(actions, (list, tuple)): + supported_patterns = [ + action + for item in actions[:64] + if (action := _text(str(item))) is not None + ] or None + return { + "automation_id": _text(runtime.attribute(element, "AXIdentifier")), + "role": role, + "role_source": "macos_ax_role" if role else None, + "control_type": _text(runtime.attribute(element, "AXSubrole")) or role, + "name": title or description, + "class_name": _text(runtime.attribute(element, "AXSubrole")), + "native_window_handle": _integer( + runtime.attribute(element, "AXWindowNumber") + ), + "bounds": runtime.bounds(element), + "supported_patterns": supported_patterns, + } + + +def _ancestor(runtime: Any, element: Any) -> StructuralAncestor | None: + fields = _element_fields(runtime, element) + fields.pop("native_window_handle", None) + fields.pop("supported_patterns", None) + ancestor = StructuralAncestor(**fields) + return ancestor if ancestor.model_dump(exclude_none=True) else None + + +def _ancestry(runtime: Any, element: Any, maximum_depth: int): + result: list[StructuralAncestor] = [] + current = element + for _ in range(maximum_depth): + current = runtime.attribute(current, "AXParent") + if current is None: + break + ancestor = _ancestor(runtime, current) + if ancestor is not None: + result.append(ancestor) + return result or None + + +def _window(runtime: Any, element: Any) -> StructuralWindowIdentity | None: + window = runtime.attribute(element, "AXWindow") + if window is None: + return None + fields = _element_fields(runtime, window) + identity = StructuralWindowIdentity( + title=fields.get("name"), + automation_id=fields.get("automation_id"), + class_name=fields.get("class_name"), + native_window_handle=fields.get("native_window_handle"), + bounds=fields.get("bounds"), + ) + return identity if identity.model_dump(exclude_none=True) else None + + +class MacOSAXStructuralObserver: + """Read exact AX evidence for a pointer or focused action.""" + + def __init__( + self, + *, + runtime: Any | None = None, + runtime_factory: Callable[[], Any] = _AXRuntime, + process_name_resolver: Callable[[int], str | None] = _process_name, + clock: Callable[[], float] = time.time, + maximum_ancestry_depth: int = 12, + ) -> None: + if not 0 < maximum_ancestry_depth <= MAX_STRUCTURAL_ANCESTRY_DEPTH: + raise ValueError( + "maximum_ancestry_depth must be between 1 and " + f"{MAX_STRUCTURAL_ANCESTRY_DEPTH}" + ) + self._runtime = runtime + self._runtime_factory = runtime_factory + self._thread_state = threading.local() + self.process_name_resolver = process_name_resolver + self.clock = clock + self.maximum_ancestry_depth = maximum_ancestry_depth + + def open_current_thread(self) -> None: + """Create the native AX runtime on the input delivery thread.""" + if self._runtime is not None: + return + if getattr(self._thread_state, "runtime", None) is not None: + return + if getattr(self._thread_state, "unavailable", False): + return + try: + self._thread_state.runtime = self._runtime_factory() + except Exception as exc: + self._thread_state.unavailable = True + _logger.warning("macOS AX observation is unavailable: %s", exc) + + def close_current_thread(self) -> None: + """Release a thread-owned AX adapter when it exposes a close hook.""" + if self._runtime is not None: + return + runtime = getattr(self._thread_state, "runtime", None) + self._thread_state.runtime = None + close = getattr(runtime, "close", None) + if callable(close): + close() + + def observe( + self, request: StructuralObservationRequest + ) -> StructuralObservation | None: + runtime = self._runtime + if runtime is None: + self.open_current_thread() + runtime = getattr(self._thread_state, "runtime", None) + if runtime is None: + return None + if request.x is not None and request.y is not None: + element = runtime.element_at_point(request.x, request.y) + query_kind = "point" + else: + element = runtime.focused_element() + query_kind = "focused" + if element is None: + return None + fields = _element_fields(runtime, element) + observed_element = StructuralElement(**fields) + if not observed_element.model_dump(exclude_none=True): + return None + pid = _integer(runtime.process_id(element)) + process = None + if pid is not None and pid > 0: + process = StructuralProcessIdentity( + process_id=pid, + process_name=self.process_name_resolver(pid), + ) + return StructuralObservation( + provider="macos_ax", + event_timestamp=request.event_timestamp, + observed_at=self.clock(), + query_kind=query_kind, + element=observed_element, + process=process, + window=_window(runtime, element), + ancestry=_ancestry( + runtime, + element, + self.maximum_ancestry_depth, + ), + ) + + +__all__ = ["MacOSAXStructuralObserver"] diff --git a/pyproject.toml b/pyproject.toml index eeaafd5..3564fed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,9 +67,15 @@ share = [ "magic-wormhole>=0.17.0", ] +# Native Linux AT-SPI observation. The host must also provide the AT-SPI +# typelib/runtime (for example gir1.2-atspi-2.0 on Debian/Ubuntu). +linux = [ + "PyGObject>=3.46; sys_platform == 'linux'", +] + # Everything all = [ - "openadapt-capture[transcribe-fast,transcribe,privacy,share]", + "openadapt-capture[transcribe-fast,transcribe,privacy,share,linux]", ] dev = [ diff --git a/tests/test_structural_observation.py b/tests/test_structural_observation.py index e050d4f..8f6fba6 100644 --- a/tests/test_structural_observation.py +++ b/tests/test_structural_observation.py @@ -2,8 +2,10 @@ from __future__ import annotations +import os import queue import sqlite3 +import sys import threading from pathlib import Path from types import SimpleNamespace @@ -16,11 +18,18 @@ from openadapt_capture.structural import ( MAX_STRUCTURAL_ANCESTRY_DEPTH, MAX_STRUCTURAL_TEXT_LENGTH, + StructuralBounds, StructuralElement, StructuralObservation, StructuralObservationRequest, create_structural_observer, + observe_structural_action, ) +from openadapt_capture.structural_observer.linux import ( + LinuxATSpiStructuralObserver, + _GIAtspiRuntime, +) +from openadapt_capture.structural_observer.macos import MacOSAXStructuralObserver from openadapt_capture.structural_observer.windows import ( WindowsUIAStructuralObserver, _PywinautoRuntime, @@ -245,6 +254,13 @@ def test_structural_contract_accepts_namespaced_extension_provider() -> None: assert observation.provider == "example_macos_ax" +def test_structural_bounds_reject_non_finite_or_inverted_provider_geometry() -> None: + with pytest.raises(ValueError, match="finite"): + StructuralBounds(left=0, top=0, right=float("nan"), bottom=10) + with pytest.raises(ValueError, match="inverted"): + StructuralBounds(left=10, top=0, right=5, bottom=10) + + def test_windows_uia_ancestry_depth_is_bounded() -> None: with pytest.raises(ValueError, match="maximum_ancestry_depth"): WindowsUIAStructuralObserver( @@ -253,6 +269,271 @@ def test_windows_uia_ancestry_depth_is_bounded() -> None: ) +class _AXFakeRuntime: + def __init__(self) -> None: + self.window = object() + self.parent = object() + self.target = object() + self.attributes = { + (self.target, "AXIdentifier"): "submit-order", + (self.target, "AXRole"): "AXButton", + (self.target, "AXTitle"): "Submit", + (self.target, "AXParent"): self.parent, + (self.target, "AXWindow"): self.window, + (self.parent, "AXRole"): "AXGroup", + (self.parent, "AXTitle"): "Order", + (self.parent, "AXParent"): self.window, + (self.window, "AXRole"): "AXWindow", + (self.window, "AXTitle"): "Orders", + (self.window, "AXWindowNumber"): 44, + } + + def attribute(self, element, name): + return self.attributes.get((element, name)) + + def element_at_point(self, x, y): + assert (x, y) == (25, 35) + return self.target + + def focused_element(self): + return self.target + + def process_id(self, element): + assert element is self.target + return 42 + + def actions(self, element): + return ["AXPress"] if element is self.target else None + + def bounds(self, element): + if element is self.target: + from openadapt_capture.structural import StructuralBounds + + return StructuralBounds(left=10, top=20, right=110, bottom=60) + return None + + +def test_macos_ax_observer_returns_action_time_evidence() -> None: + observer = MacOSAXStructuralObserver( + runtime=_AXFakeRuntime(), + process_name_resolver=lambda pid: "Example" if pid == 42 else None, + clock=lambda: 101.25, + ) + + observed = observer.observe( + StructuralObservationRequest( + event_timestamp=101.0, + action_name="click", + x=25, + y=35, + ) + ) + + assert observed is not None + assert observed.provider == "macos_ax" + assert observed.query_kind == "point" + assert observed.element.automation_id == "submit-order" + assert observed.element.supported_patterns == ["AXPress"] + assert observed.process is not None + assert observed.process.process_name == "Example" + assert observed.window is not None + assert observed.window.native_window_handle == 44 + assert [item.role for item in observed.ancestry or []] == [ + "AXGroup", + "AXWindow", + ] + + +class _ATSpiElement: + def __init__(self, name: str, parent=None) -> None: + self.name = name + self.parent = parent + + +class _ATSpiFakeRuntime: + def __init__(self) -> None: + self.application = _ATSpiElement("Example") + self.window = _ATSpiElement("Orders", self.application) + self.parent_element = _ATSpiElement("Order", self.window) + self.target = _ATSpiElement("Submit", self.parent_element) + + def element_at_point(self, x, y): + assert (x, y) == (25, 35) + return self.target + + def focused_element(self): + return self.target + + def parent(self, element): + return element.parent + + def attributes(self, element): + if element is self.target: + return {"accessible-id": "submit-order", "class": "GtkButton"} + return {} + + def role_name(self, element): + return { + self.target: "push button", + self.parent_element: "panel", + self.window: "frame", + self.application: "application", + }[element] + + def bounds(self, element): + if element is self.target: + from openadapt_capture.structural import StructuralBounds + + return StructuralBounds(left=10, top=20, right=110, bottom=60) + return None + + def action_names(self, element): + return ["click"] if element is self.target else None + + def process_id(self, element): + assert element is self.target + return 42 + + +def test_linux_atspi_observer_returns_action_time_evidence() -> None: + observer = LinuxATSpiStructuralObserver( + runtime=_ATSpiFakeRuntime(), + process_name_resolver=lambda pid: "example" if pid == 42 else None, + clock=lambda: 101.5, + ) + + observed = observer.observe( + StructuralObservationRequest( + event_timestamp=101.0, + action_name="press", + ) + ) + + assert observed is not None + assert observed.provider == "linux_atspi" + assert observed.query_kind == "focused" + assert observed.element.automation_id == "submit-order" + assert observed.element.class_name == "GtkButton" + assert observed.element.supported_patterns == ["click"] + assert observed.process is not None + assert observed.process.process_name == "example" + assert observed.window is not None + assert observed.window.title == "Orders" + assert [item.role for item in observed.ancestry or []] == [ + "panel", + "frame", + "application", + ] + + +def test_gi_atspi_runtime_descends_from_desktop_point_query() -> None: + class StateSet: + def __init__(self, *states) -> None: + self.states = set(states) + + def contains(self, state) -> bool: + return state in self.states + + class Component: + def __init__(self, child=None) -> None: + self.child = child + + def get_accessible_at_point(self, x, y, coordinates): + assert (x, y, coordinates) == (25, 35, "screen") + return self.child + + class Node: + def __init__(self, *, child=None, states=()) -> None: + self.component = Component(child) + self.states = StateSet(*states) + + def get_component_iface(self): + return self.component + + def get_state_set(self): + return self.states + + target = Node(states=("focused",)) + window = Node(child=target, states=("active",)) + application = Node(child=window) + desktop = Node(child=application) + runtime = _GIAtspiRuntime.__new__(_GIAtspiRuntime) + runtime.atspi = SimpleNamespace( + CoordType=SimpleNamespace(SCREEN="screen"), + StateType=SimpleNamespace(ACTIVE="active", FOCUSED="focused"), + ) + runtime.desktop = desktop + + assert runtime.element_at_point(25, 35) is target + + +def test_gi_atspi_runtime_omits_a_point_query_beyond_the_depth_bound() -> None: + class Component: + def __init__(self) -> None: + self.child = None + + def get_accessible_at_point(self, _x, _y, _coordinates): + return self.child + + class Node: + def __init__(self) -> None: + self.component = Component() + + def get_component_iface(self): + return self.component + + nodes = [Node() for _ in range(MAX_STRUCTURAL_ANCESTRY_DEPTH + 3)] + for current, child in zip(nodes, nodes[1:]): + current.component.child = child + runtime = _GIAtspiRuntime.__new__(_GIAtspiRuntime) + runtime.atspi = SimpleNamespace(CoordType=SimpleNamespace(SCREEN="screen")) + runtime.desktop = nodes[0] + + assert runtime.element_at_point(25, 35) is None + + +@pytest.mark.parametrize( + "observer_type", + [MacOSAXStructuralObserver, LinuxATSpiStructuralObserver], +) +def test_cross_platform_runtime_is_owned_by_the_delivery_thread(observer_type) -> None: + lifecycle: list[tuple[str, int]] = [] + fake = _AXFakeRuntime() if observer_type is MacOSAXStructuralObserver else _ATSpiFakeRuntime() + + class Runtime: + def __getattr__(self, name): + return getattr(fake, name) + + def close(self): + lifecycle.append(("close", threading.get_ident())) + + def factory(): + lifecycle.append(("create", threading.get_ident())) + return Runtime() + + observer = observer_type(runtime_factory=factory) + + def deliver() -> None: + observer.open_current_thread() + try: + observation = observer.observe( + StructuralObservationRequest( + event_timestamp=101.0, + action_name="press", + ) + ) + assert observation is not None + finally: + observer.close_current_thread() + + thread = threading.Thread(target=deliver) + thread.start() + thread.join() + + assert [name for name, _thread_id in lifecycle] == ["create", "close"] + assert len({thread_id for _name, thread_id in lifecycle}) == 1 + + def test_pywinauto_runtime_quantizes_points_and_uses_exact_focused_api() -> None: calls: list[tuple[str, object]] = [] desktop = SimpleNamespace( @@ -462,8 +743,29 @@ def test_legacy_action_table_is_migrated_without_fabricated_evidence( assert "structural_observation" in columns -def test_non_windows_factory_is_headless_safe() -> None: - assert create_structural_observer(platform_name="linux") is None +@pytest.mark.parametrize( + ("platform_name", "target"), + [ + ( + "darwin", + "openadapt_capture.structural_observer.macos.MacOSAXStructuralObserver", + ), + ( + "linux", + "openadapt_capture.structural_observer.linux.LinuxATSpiStructuralObserver", + ), + ], +) +def test_factory_selects_each_native_structural_provider( + monkeypatch, platform_name, target +) -> None: + sentinel = object() + monkeypatch.setattr(target, lambda: sentinel) + assert create_structural_observer(platform_name=platform_name) is sentinel + + +def test_unsupported_platform_factory_is_headless_safe() -> None: + assert create_structural_observer(platform_name="freebsd") is None def test_windows_factory_omits_optional_evidence_when_uia_cannot_start( @@ -503,3 +805,33 @@ def fail_runtime(): assert observer.observe(request) is None assert observer.observe(request) is None assert attempts == 1 + + +@pytest.mark.slow +@pytest.mark.skipif( + os.environ.get("OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION") != "1", + reason="requires the explicit interactive production qualification rig", +) +def test_live_native_structural_provider_returns_exact_focused_evidence() -> None: + observer = create_structural_observer() + assert observer is not None, "the native structural provider is required" + expected_provider = { + "win32": "windows_uia", + "darwin": "macos_ax", + "linux": "linux_atspi", + }["linux" if sys.platform.startswith("linux") else sys.platform] + observation = observe_structural_action( + observer, + StructuralObservationRequest( + event_timestamp=101.0, + action_name="press", + ), + ) + assert observation is not None + assert observation.provider == expected_provider + assert observation.query_kind == "focused" + assert observation.event_timestamp == 101.0 + assert observation.element.model_dump(exclude_none=True) + assert observation.process is not None + assert observation.process.process_id is not None + assert observation.window is not None From 0f0896b3b82be18a77c3207808cdfe568ccd7ced Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 20 Aug 2026 12:16:51 -0400 Subject: [PATCH 3/3] test: gate live X11 missing-window assertion --- tests/test_window_capture.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index 90be341..2864e6d 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -1211,6 +1211,15 @@ def test_live_move_resize_preserves_fixed_viewport_and_restores_window(self): assert restored_data["window_id"] == str(target.window_id) def test_live_missing_window_fails_loud(self): + if sys.platform.startswith("linux"): + try: + _require_x11_session() + except WindowCaptureError as exc: + if _PRODUCTION_QUALIFICATION: + raise AssertionError( + "production qualification requires a native X11 display" + ) from exc + pytest.skip(f"live missing-window test requires native X11: {exc}") scope = WindowCaptureScope(WindowTarget(owner="no-such-app-obviously-not-running-xyz")) with pytest.raises(WindowCaptureError, match="no window matching"): scope.capture_frame()