diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9cfefe0..a8211f83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ name: CI # e2e-browser (the GOLDEN TASK first -- the whole free path to VERIFIED, # pinned against tests/golden/tutorial_run.json -- then record -> # compile -> replay in a headless browser; no OS permissions needed, so -# it runs on PRs, NOT just nightly), +# it runs on PRs, NOT just weekly), # linux-atspi-x11 (real GTK/AT-SPI actuation inside Xvfb + session D-Bus), # wheel (clean-venv wheel install + CLI smoke), # windows-mock (non-injecting Win32 ABI + fake behavior contract). @@ -44,7 +44,7 @@ name: CI # - wheel # - gate # `gate` comes from the separate Validate claims workflow. -# (test-matrix remains nightly/explicit release qualification.) +# (test-matrix remains weekly/explicit release qualification.) on: pull_request: push: @@ -65,7 +65,7 @@ concurrency: jobs: # --- Lint + type check (ruff + mypy) ------------------------------------- - # Runs on PRs (and post-merge/nightly). Deliberately a SEPARATE job from the + # Runs on PRs (and post-merge/weekly). Deliberately a SEPARATE job from the # required `test` gate: it must NOT be wired as a dependency of `test` (that # would leave `test` reporting a compound context and break branch # protection, whose required context is exactly `test`). Add `lint` as its @@ -94,7 +94,7 @@ jobs: run: mypy # --- Supported Python lower bound (required on PRs) ---------------------- - # The full cross-platform matrix remains post-merge/nightly, but the oldest + # The full cross-platform matrix remains post-merge/weekly, but the oldest # supported interpreter must install and execute the release-consistency gate # before merge. This specifically prevents Python 3.11-only stdlib imports or # dependency metadata drift from reaching PyPI unnoticed. @@ -351,7 +351,7 @@ jobs: # when a drift/heal test fails in CI. --ignore=tests/e2e drops the slow # browser/OCR end-to-end suite from the fast required gate (it runs in the # required `e2e-browser` job instead). The complete suite is repeated - # across supported interpreters/OSes nightly and for explicit release + # across supported interpreters/OSes weekly and for explicit release # qualification. --cov collects coverage into # .coverage for the enforced safety floor in the next step. - name: Test (fast unit suite) @@ -390,7 +390,7 @@ jobs: # The end-to-end record -> compile -> replay-under-drift suite. It drives a # HEADLESS Playwright browser against the bundled MockMed app, so it needs no # OS-level input permissions and runs deterministically on a Linux runner -- - # which is exactly why it is a REQUIRED PR check here, not a nightly-only one. + # which is exactly why it is a REQUIRED PR check here, not a weekly-only one. # The desktop/Citrix/Parallels legs under tests/e2e self-skip when their # macOS/VM backends are absent (i.e. on this Linux runner), so running the # whole directory exercises every browser scenario and skips the rest. diff --git a/.github/workflows/quickstart-lifecycle.yml b/.github/workflows/quickstart-lifecycle.yml index 50ebdb0f..2bf65aa3 100644 --- a/.github/workflows/quickstart-lifecycle.yml +++ b/.github/workflows/quickstart-lifecycle.yml @@ -49,6 +49,7 @@ jobs: --work-dir "runs/lifecycle" --install-browser --browser-with-deps + --source-revision "${{ github.sha }}" - name: Full lifecycle (macOS / Windows) if: runner.os != 'Linux' @@ -57,6 +58,7 @@ jobs: --wheel "lifecycle-dist/*.whl" --work-dir "runs/lifecycle" --install-browser + --source-revision "${{ github.sha }}" - name: Upload lifecycle evidence if: always() diff --git a/README.md b/README.md index 3edfac97..e7071317 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ governed lifecycle (`openadapt-flow repair`: reviewed diff, replay + fault campaigns, human approval, staged canary, one-command rollback). See [docs/REPAIR_LIFECYCLE.md](docs/REPAIR_LIFECYCLE.md). -The nightly clean-machine test runs this complete install-to-uninstall journey +The weekly clean-machine test runs this complete install-to-uninstall journey on Linux, macOS, and Windows. See the [capability and qualification matrix](docs/PRODUCT_STATUS.md) for the accepted scope of each substrate. @@ -169,25 +169,49 @@ to the browser, and it prints a visible notice when it does. See [docs/SURFACES.md](docs/SURFACES.md) for the per-surface first-workflow paths and the two remote execution modes. +Install the Capture component together with the runtime for the surface that +will replay the workflow: + +| Workflow surface | Exact install | +|---|---| +| Browser | `pip install 'openadapt-flow[browser]'` | +| Native Windows | `pip install 'openadapt-flow[capture,windows]'` | +| Native macOS | `pip install 'openadapt-flow[capture,macos]'` | +| Native Linux | `pip install 'openadapt-flow[capture,linux]'` plus the AT-SPI system packages in [the Linux guide](docs/desktop/LINUX_NATIVE.md) | +| Network RDP | Recorder: `pip install 'openadapt-flow[capture]'` inside the demonstrated session; runner: `pip install 'openadapt-flow[rdp]'` | +| Local RDP/Citrix client window | macOS host: `pip install 'openadapt-flow[capture,macos]'`; Windows host: `pip install 'openadapt-flow[capture,windows]'` | + ```bash # Browser (Playwright / Chromium): the app is a URL. openadapt-flow record --backend web --url https://your.app --out rec openadapt-flow compile rec --out bundle --name my-task openadapt-flow replay bundle --backend web --url https://your.app -# Native Windows (UI Automation via the in-guest WAA agent). -openadapt-flow record --backend windows --agent-url http://localhost:5001 \ +# Native Windows: Capture records the local target window. WAA drives replay. +openadapt-flow record --backend windows --window "Target App" \ --task "add a patient note" --out rec +openadapt-flow compile rec --out bundle --name my-task +openadapt-flow replay bundle --backend windows \ + --agent-url http://localhost:5001 -# Native macOS (accessibility, one app window). -openadapt-flow record --backend macos --macos-app TextEdit --out rec +# Native macOS: --macos-app scopes Capture and selects the replay app. +openadapt-flow record --backend macos --macos-app TextEdit \ + --macos-window-title notes.txt --out rec +openadapt-flow compile rec --out bundle --name my-task +openadapt-flow replay bundle --backend macos --macos-app TextEdit \ + --macos-window-title notes.txt -# Native Linux (AT-SPI, one exact app window). -openadapt-flow record --backend linux --linux-app gedit \ - --linux-window-title "Untitled Document 1" --out rec +# Native Linux: Capture records the local desktop; AT-SPI selects replay target. +openadapt-flow record --backend linux --out rec +openadapt-flow compile rec --out bundle --name my-task +openadapt-flow replay bundle --backend linux --linux-app gedit \ + --linux-window-title "Untitled Document 1" -# RDP remote display (pixel-only vision ladder over the network session). -openadapt-flow record --backend rdp --rdp-host 10.0.0.5 --out rec +# Network RDP: run record inside the demonstrated remote session. The host is +# a replay target, so it is supplied only when the runner connects. +openadapt-flow record --backend rdp --out rec +openadapt-flow compile rec --out bundle --name my-task +openadapt-flow replay bundle --backend rdp --rdp-host 10.0.0.5 # Citrix / VDI (one exact local Citrix Workspace window). openadapt-flow record --backend citrix \ @@ -196,14 +220,23 @@ openadapt-flow record --backend citrix \ --rdp-window-title "Ward A" \ --rdp-readiness-text "Appointments" \ --out rec +openadapt-flow compile rec --out bundle --name my-task +openadapt-flow replay bundle --backend citrix \ + --rdp-window "Citrix Viewer" \ + --rdp-window-title "Ward A" \ + --rdp-readiness-text "Appointments" ``` `--backend web` is browser-first (the app is a `--url`). For -`windows`, `macos`, `linux`, `rdp`, and `citrix` the capture has no field identity, so the -target is the app window or host: `--agent-url` for Windows, `--macos-app`, -`--linux-app` plus `--linux-window-title`, and `--rdp-host` (or a configured -exact `rdp_window` / `rdp_window_title` for Citrix Workspace). Pass the same `--backend` -plus target flags to `replay`, or drive a real deployment with +`windows`, `macos`, `linux`, `rdp`, and `citrix`, the Capture component records +local screen, mouse, keyboard, timing, and available action-time structure. +`--macos-app` / `--macos-window-title` scope the macOS Capture window. +`--window` / `--window-title` scope a Windows-hosted local capture, and +`--rdp-window` / `--rdp-window-title` bind a local RDP or Citrix client window +to both capture and replay. In contrast, `--agent-url`, `--linux-app`, +`--linux-window-title`, and `--rdp-host` name replay targets that the local +Capture session cannot control. `record` refuses those flags instead of +ignoring them; pass them to `replay` or `run`. Drive a real deployment with `openadapt-flow run bundle --config deploy.yaml`, which reads the backend, effects, actuation, durable, and policy sections from one config. Recorded parameter values are the defaults, and `--param` overrides them at replay. diff --git a/claims.yaml b/claims.yaml index 7dfb72cc..653992d5 100644 --- a/claims.yaml +++ b/claims.yaml @@ -70,8 +70,8 @@ claims: registry. Desktop and remote-display workflows use the separately scoped acceptance and code-qualified claims below. - >- - The full record->compile->replay e2e runs in the post-merge/nightly - full suite; the fast required PR gate covers the replayer + demo units. + The full record->compile->replay browser suite runs in the required + e2e-browser PR gate and repeats in the weekly compatibility matrix. # -------------------------------------------------- deterministic $0 replay - id: deterministic-zero-model-replay diff --git a/docs/ECOSYSTEM_INTEGRATION.md b/docs/ECOSYSTEM_INTEGRATION.md index 83ddab80..4f8739ed 100644 --- a/docs/ECOSYSTEM_INTEGRATION.md +++ b/docs/ECOSYSTEM_INTEGRATION.md @@ -1,411 +1,92 @@ -# OpenAdapt Ecosystem Integration Roadmap +# OpenAdapt ecosystem integration -**Status:** decision-grade architecture memo (read-only analysis, no code changes) -**Scope:** how `openadapt-flow` should — and should not — adopt the rest of the -`openadapt-*` ecosystem. -**Author's note:** this memo covers **openadapt-types**, **openadapt-capture**, and -**openadapt-verifier**, plus overall sequencing. Three integrations are already owned by -other workstreams and are only referenced here for sequencing: **openadapt-privacy** (PHI -scrubbing), the **openadapt-grounding** evaluation, and the **`openadapt[flow]` umbrella -extra**. +**Status: current implementation map.** This file replaces the pre-integration +architecture memo that once described `openadapt-capture` as unimplemented and +experimental. That statement is obsolete. ---- +## Product boundary -## 0. TL;DR — recommendation per package +`openadapt-flow` is the canonical demonstration compiler and governed runtime. +The OpenAdapt launcher installs it. Focused packages provide optional input, +interop, privacy, and operator surfaces. They do not replace Flow's internal +evidence-rich workflow model. -| Package | What it is | Overlap with flow | Recommendation | Risk | -|---|---|---|---|---| -| **openadapt-types** | Canonical Pydantic action schema (pydantic-only, zero heavy deps) | `ir.py` `ActionKind` ⊂ `ActionType`; flow's `Anchor`/`Postcondition`/`Resolution`/`IdentityCheck` are net-new | **Shim, don't swap.** Add an optional `to_openadapt_types()` / `from_openadapt_types()` interop layer at the boundary; keep `ir.py` as the internal source of truth | **Low** (additive, no blast radius) — but a full schema *swap* is **High** risk (44 files import `ir`) | -| **openadapt-capture** | Cross-platform desktop recorder (pynput + mss + PyAV), own SQLAlchemy schema | flow already ships `adapters/capture.py` — but it targets a **stale/assumed** capture schema | **Adopt via the public API, fix the adapter.** Rewrite the adapter onto `CaptureSession.load().actions()`; keep it an optional extra | **Medium** — current adapter reads a DB layout that no longer exists; it is dead/untested against real capture output | -| **openadapt-verifier** | Clinical/RWE **data-extraction** statistical validator | **None.** Shares only the word "verify" | **Leave standalone. Do not integrate.** | **N/A** — wrong tool; integrating would be a category error | +| Component | Current Flow integration | Lifecycle boundary | +| --- | --- | --- | +| `openadapt-capture` | Supported native capture input through `openadapt-flow[capture]` and `openadapt_flow.adapters.capture` | Capture is Beta. Each native or remote workflow still needs application-specific qualification. | +| `openadapt-types` | Optional interop boundary through `openadapt-flow[interop]` | The shared schema does not replace Flow's compiled IR or safety contracts. | +| `openadapt-privacy` | Optional source and artifact privacy controls through `openadapt-flow[privacy]` | A recording never becomes safe to upload only because it compiled. | +| `openadapt-grounding` | Optional grounding rung | It cannot authorize an action or prove an effect. | +| `openadapt-verifier` | No runtime integration | It is an unrelated research package and is not required by the product. | -**Sequenced order:** (1) types interop shim → (2) capture adapter fix → then defer/skip -verifier. Privacy and grounding (other workstreams) slot around these; see §5. +## Capture integration ---- +`openadapt-capture` is the canonical native screen, mouse, keyboard, timing, +window-scope, and media-capture component. Flow does not implement a second +native recorder. -## 1. Principles - -flow's value proposition is a **lean, auditable, standalone core**: a demonstration -compiler whose replay loop makes **zero model calls**, runs in CI with no OS permissions, -and whose entire trust surface (`ir.py` + the resolution/identity ladders) can be read in -an afternoon. That smallness is not an accident of youth — it is a **certification-moat -asset**. Every dependency pulled into the replay path is something a regulated buyer's -security review must also vet. - -The integration philosophy that follows from this: - -1. **Adopt shared *vocabulary*, not shared *machinery*.** Speaking `openadapt-types` - `Action` at the *boundaries* (import, export, interop with agents/evals) is pure - upside. Dissolving flow's internal `ir.py` into a monorepo type is not — flow's IR - carries compiler-specific evidence (`Anchor`, `Postcondition`, `IdentityCheck`) that - the canonical schema deliberately does not model. -2. **Close *real* gaps with dedicated packages.** Where the ecosystem genuinely does - something flow can't (cross-platform desktop **capture**; PHI **scrubbing**; - VLM **grounding**), integrate — behind an optional extra, off the default path. -3. **Never put a heavy dependency on the replay hot path.** The replay loop is the - moat. Recording, grounding, and privacy are *edges* (compile-time or opt-in); the - core stays `pydantic + opencv + rapidocr + pillow`. -4. **Prefer additive interop over swaps.** flow moves fast and its schema is load-bearing. - `ir.py` is imported by **44 files** and marked FROZEN in `DESIGN.md`. A schema swap has blast - radius across the whole compiler, runtime, and benchmark surface; an interop shim has - none. - -> One-line creed: **adopt the words, keep the core.** - ---- - -## 2. openadapt-types — canonical action schema - -### 2.1 What it provides - -`openadapt-types` (v0.1.0, PyPI) is a **pydantic-only, zero-heavy-dep** schema library -("No ML libraries, no heavy deps" — its README). Public surface: - -- `ActionType` (str-enum, 21 members): `CLICK, DOUBLE_CLICK, RIGHT_CLICK, DRAG, SCROLL, - HOVER, TYPE, KEY, HOTKEY, GOTO, BACK, FORWARD, REFRESH, OPEN_APP, CLOSE_APP, - WINDOW_FOCUS, WAIT, SCREENSHOT, DONE, FAIL, ANSWER`. -- `Action` (`type`, `target`, `text`, `key`, `modifiers`, `scroll_*`, `url`, …) with a - `model_validator` enforcing per-type required fields (TYPE⇒text, KEY⇒key, …). -- `ActionTarget` (`node_id` / `description` / `x,y,is_normalized`) — grounding priority - node_id > description > coords. -- `ActionResult` (`success`, `error`, `error_type∈{grounding_error, execution_error, - state_mismatch, timeout, permission_denied, infrastructure_error}`, `duration_ms`, - `changed_node_ids`, `resolved_coordinates`). -- `ComputerState` / `UINode` / `BoundingBox` / `ProcessInfo` — observation graph. -- `Episode` / `Step` — trajectory container (for evals/RL). -- Parsers: `parse_action` (auto JSON/DSL), `parse_action_dsl`, `parse_action_json`, - `from_benchmark_action` / `to_benchmark_action_dict` — all **fail-safe** (return - `Action(type=DONE)` on malformed input, never raise). - -It is a **passive data schema**: no executor, no resolver, no verification logic, -no anchors, no template crops, no OCR labels, no postconditions. - -### 2.2 What flow reimplements that overlaps - -flow's `openadapt_flow/ir.py` defines its own action vocabulary: - -- `ir.ActionKind` (`ir.py:33-39`): `CLICK, DOUBLE_CLICK, TYPE, KEY, WAIT, SCROLL` - (6 members). Used in **27 sites** across the compiler/runtime. -- `ir.Step` (`ir.py:157-...`) is flow's per-step record: `action`, `anchor`, `text`, - `param`, `key`, `scroll_dx/dy`, `expect`, `risk`, `identity_armed`. -- `ir.Workflow` (`ir.py:...`) is the compiled bundle root (`schema_version`, `name`, - `params`, `steps`, `save()/load()`). - -### 2.3 The DELTA - -**Maps 1:1 (flow → types):** every `ActionKind` member is a subset of `ActionType` -with identical string values: - -| flow `ActionKind` | value | types `ActionType` | -|---|---|---| -| `CLICK` | `"click"` | `CLICK` ✅ | -| `DOUBLE_CLICK` | `"double_click"` | `DOUBLE_CLICK` ✅ | -| `TYPE` | `"type"` | `TYPE` ✅ | -| `KEY` | `"key"` | `KEY` ✅ | -| `WAIT` | `"wait"` | `WAIT` ✅ | -| `SCROLL` | `"scroll"` | `SCROLL` ✅ | - -The string values are byte-identical, so a flow `Step` could emit/ingest an -`openadapt_types.Action` with a trivial field map (click_point → `ActionTarget(x,y)`, -`text` → `text`, `key` → `key`, `scroll_dx/dy` → `scroll_direction`/`scroll_amount`). - -**flow has that types lacks (the compiler-specific IR — net-new, keep):** - -- `Anchor` — redundant visual evidence: `template` crop path, `region`, `click_point`, - `ocr_text`, `context_text`, `structured_identity`, `identifier_crop/region`, - `landmarks`, `search_pad`. -- `Landmark` — geometry-rung stable-text offsets. -- `Postcondition` / `PostconditionKind` — `TEXT_PRESENT/ABSENT`, `REGION_STABLE` (phash), - `URL/TITLE_CHANGED`, `NEW_TAB_OPENED`. -- `Resolution` — which ladder rung resolved the target (`template/…/grounder`), point, - confidence, `elapsed_ms`. -- `IdentityCheck` — the pre-click same-entity verdict (`verified/mismatch/abstain/ - unreadable` × `structured/pixel/vlm/context/param`). This is the wrong-entity safety - core. -- `HealEvent`, `StepResult`, `RunReport`, `UnarmedStep` — audit/telemetry. -- `risk∈{reversible,irreversible}`, `identity_armed` — the halt-on-uncertainty gates. - -None of these exist anywhere in `openadapt-types`. The closest analogs are -`UINode.automation_id/xpath/css_selector` (string locators, not visual anchors) and -`ActionResult.error_type="state_mismatch"` (a bare enum literal, not a postcondition -engine). - -**types has that flow lacks (worth borrowing at the boundary, not the core):** - -- `ActionTarget.node_id` / `UINode` graph — a structured-element addressing model flow - only touches via its optional `IdentityBackend.structured_text_at`. -- Fail-safe **DSL/JSON parsers** — useful when flow emits skills/MCP tools that an agent - or evals harness must round-trip. -- `Episode`/`Step` trajectory container — the lingua franca of `openadapt-evals`; useful - when flow's `RunReport` needs to feed the eval/RL flywheel. -- A **wider `ActionType`** (right-click, drag, hotkey, goto) — flow will grow into some of - these; aligning values now avoids a later rename. - -### 2.4 Migration shape — **compatibility shim (adopt the vocabulary, keep the IR)** - -Do **not** replace `ir.ActionKind` / `ir.Step` with `openadapt_types.Action`. Reasons: - -- **Impedance mismatch.** flow's `Step` is a *compiled artifact* (anchor + postconditions - + identity gates). `openadapt_types.Action` is an *instantaneous intent*. They are - different layers; forcing flow's IR to be the canonical `Action` would either bloat the - canonical schema with compiler internals or strip flow's evidence out of its own IR. -- **Blast radius.** `ir` is imported by 44 files and 27 `ActionKind.` call-sites, and is - marked **FROZEN** in `DESIGN.md` (additive changes only). - A swap touches the compiler, every runtime rung, the healer, the benchmark harness, and - the emit/skill + MCP surfaces at once. That is a major refactor, not an additive dependency. - -**Recommended concrete shape** (optional, additive): +The adapter uses Capture's public API: +```python +session = CaptureSession.load(capture_dir) +actions = session.actions(include_moves=False) +frame = session.get_frame_at(timestamp) ``` -openadapt_flow/interop/types.py # new, optional - def step_to_action(step: ir.Step, resolution: ir.Resolution|None) -> "openadapt_types.Action" - def action_to_step_stub(a: "openadapt_types.Action") -> ir.Step # for ingest - def result_to_action_result(r: ir.StepResult) -> "openadapt_types.ActionResult" -``` - -- Guard the import (`try: import openadapt_types`) so the core never hard-depends on it. -- Expose it under an optional extra: `openadapt-flow[types]` (or fold into the umbrella - `openadapt[flow]` extra the other workstream owns). -- Use it in exactly two places where the *boundary* benefits: - 1. **emit/** (skill + MCP tool generation) — so a flow bundle can describe its steps in - the ecosystem's canonical action language. - 2. **benchmark/** — so flow's runs can serialize as `openadapt_types.Episode`/`Step` - for the evals/RL flywheel instead of a bespoke JSON. - -**Risk: Low.** Purely additive, no change to `ir.py`, no change to the replay hot path, -trivially revertible. The only ongoing cost is keeping the field map in sync if -`ActionType` gains members flow starts using — a one-function maintenance surface. - -> If a future release wants tighter alignment, the safe next step is to make -> `ir.ActionKind`'s **values** authoritative-compatible (they already are) and add a CI -> test asserting `set(ActionKind) ⊆ set(ActionType)`, so drift is caught without a swap. - -### 2.5 Priority - -**First.** It's the cheapest, lowest-risk, highest-leverage integration and it's the -"shared vocabulary" principle in action. It also unblocks clean interop with evals and -with whatever the umbrella-extra workstream assembles. Ship the shim before the capture -rewrite. ---- +It does not read Capture's private database schema. The adapter normalizes the +public actions and frames into Flow's recording contract. It preserves click, +double-click, drag, type, key, shortcut, and scroll semantics. It rejects an +unsupported action instead of dropping it. -## 3. openadapt-capture — cross-platform desktop recording +The required CI suite tests the released package API, timestamp and frame +alignment, coordinate scaling, secret exclusion, structural observations, +action vocabulary, and all desktop backend selectors. See +[`tests/test_capture_adapter.py`](../tests/test_capture_adapter.py) and +[`openadapt_flow/adapters/capture.py`](../openadapt_flow/adapters/capture.py). -### 3.1 What it provides +Install the native recording path with: -`openadapt-capture` (v0.5.1) is the ecosystem's **desktop recorder**: - -- `Recorder(capture_dir, task_description, …)` context manager — spawns reader threads - for mouse/keyboard/screen/window/browser (pynput + mss + PyAV), writes a per-capture - **`recording.db`** (SQLAlchemy) + action-gated `oa_recording-*.mp4` + optional audio. -- `CaptureSession.load(dir)` → `.actions(include_moves=False)` yields public `Action` - dataclasses (`.timestamp`, `.type` e.g. `mouse.singleclick`/`key.type`, `.x/.y/.dx/.dy`, - `.button/.text/.keys`, lazy `.screenshot` PIL frame via `get_frame_at`). Also - `.raw_events()`, `.browser_events()`, and metadata (`platform`, `screen_size`, - `pixel_ratio`, `duration`, `task_description`, `video_path`). -- Cross-platform input via **pynput** (macOS Quartz / Windows hooks / X11); screenshots - via **mss**; per-click element state and window geometry available but **off by - default** (`RECORD_WINDOW_DATA=False`, `RECORD_READ_ACTIVE_ELEMENT_STATE=False`). -- Import stays headless-safe (guarded so `Recorder=None` without a display); **actual - recording** pulls native deps and needs OS input-monitoring/screen-recording - permission. No OCR. Its own schema — **no dependency on openadapt-types.** - -### 3.2 What flow reimplements / already has - -flow's **native/desktop recording gap** is real: `recorder.py` records only through a -`Backend` (the Playwright reference backend); there is no cross-platform OS-level capture. -flow already anticipated this with `openadapt_flow/adapters/capture.py` + -`convert_capture()`, documented in `docs/desktop/PHASE1.md` — an adapter that converts a -capture session into flow's recording format (`meta.json` + `events.jsonl` + `frames/`). - -### 3.3 The DELTA — and a concrete defect - -**Maps well conceptually:** capture's derived action types line up with flow's event -kinds (`mouse.singleclick`→click, `mouse.doubleclick`→double_click, `key.type`→type, -`key.down`→key, `mouse.scroll`→scroll). The adapter already handles the hard parts: -logical-point→physical-pixel scaling via `pixel_ratio`, wheel-notch→pixel conversion, -param-value tagging, and loud rejection of untranslatable events (`mouse.drag`, -`key.shortcut`) so a demonstrated action is never silently dropped. - -**capture has that flow lacks:** genuine cross-platform desktop recording (flow can't -record outside a browser backend); per-event window geometry and element/a11y state -(opt-in); audio narration + Whisper word timestamps; browser semantic-element refs -(role/name/bbox/xpath/css). - -**flow has that capture lacks:** the perceptual-hash **settle wait** (capture is -action-gated video, so the adapter approximates settle by sampling a frame at -`t+settle_s`), OCR labels, template crops, and everything downstream in the compiler. - -**⚠️ Concrete defect (load-bearing finding):** `adapters/capture.py` reads the **wrong -schema**. It opens a raw `capture.db` with a flat `events(timestamp, type, data JSON)` -table and a `capture(screen_width, video_start_time, …)` metadata row. The **actual** -openadapt-capture 0.5.1 writes `recording.db` (SQLAlchemy `Recording`/`ActionEvent` -models) and exposes the public `CaptureSession.load(dir).actions()` API. The filenames -(`capture.db` vs `recording.db`, `video.mp4` vs `oa_recording-*.mp4`) and the access -pattern (raw SQL vs public API) do **not** match. The adapter was written against an -assumed/older/hypothetical capture layout and is effectively **dead code against real -capture output** — it will `FileNotFoundError` on `capture.db` the moment it meets a real -session. - -### 3.4 Migration shape — **adopt the dependency, rewrite the adapter onto the public API** - -- **Rewrite `convert_capture()` to consume `CaptureSession.load(dir).actions()`** instead - of hand-rolled SQL against a non-existent table. This deletes the fragile raw-schema - coupling, gets the `pixel_ratio` and frame extraction from capture's own tested code - (`Action.screenshot` / `get_frame_at`), and survives capture's future schema changes - (they'll be absorbed behind capture's public API). -- **Keep it an optional extra.** `openadapt-flow[capture]` pulling `openadapt-capture`. - The core replay path never imports it; only the `demo-record`-from-desktop compile edge - does. This preserves the permission-free, CI-friendly core. -- **Do not** replace flow's Playwright `Recorder` — that is the reference recorder that - keeps the whole loop runnable in CI with zero OS permissions. capture is the **desktop** - on-ramp, not a replacement. -- **Preserve the loud-rejection contract.** The current adapter's refusal to silently drop - `mouse.drag`/`key.shortcut` is exactly the right wrong-action-safety posture; keep it - when moving to the public API. - -**Risk: Medium.** The direction is clearly right (real gap, dedicated package, adapter -already scoped), but the existing adapter is **untested against real capture output** and -targets a schema that isn't there — so "integration" here is really "finish + correct a -half-built bridge," and it needs an integration test against an actual recorded session -(which needs OS permissions, i.e. not pure-CI). Until that test exists, treat desktop -capture as **experimental**. - -### 3.5 Priority - -**Second.** After the types shim, before anything speculative. It closes flow's single -biggest capability gap (desktop, not just browser) and the scaffolding already exists — -the work is correcting it, not greenfield. Gate it behind an extra so it never threatens -the core's leanness. Sequence it *after* privacy lands if capture is to feed PHI-bearing -desktop recordings (see §5), since desktop capture is exactly where PHI scrubbing matters. - ---- - -## 4. openadapt-verifier — **do not integrate** - -### 4.1 What it actually is - -Despite the name, `openadapt-verifier` (v0.1.0) is a **clinical / real-world-evidence -(RWE) data-extraction validator**. It scores already-extracted structured field -predictions against a human gold standard and emits an FDA-credibility-framed statistical -report (Kahn conformance/completeness/plausibility checks, Wilson/Clopper-Pearson -confidence intervals, Cohen/Fleiss kappa, `validate()`/`verify()`/`compare()` over a -`Dataset` of `FieldSpec`+gold+pred `Record`s). - -A grep of the package for `screenshot|ocr|template|ssim|pixel|dom|vlm|opencv|cv2` returns -**nothing**. It has **no dependency on openadapt-types**, no `ActionResult` coupling, no -images, no actions. It shares exactly one thing with flow's verification: the word -"verify." - -### 4.2 The DELTA — disjoint - -flow's "verification" is `runtime/identity.py` + `Postcondition` + the resolution ladder: -per-click same-entity checks, screen-state postconditions (phash/text/URL), halt-on- -uncertainty, wrong-action detection. openadapt-verifier's "verification" is offline -statistical scoring of extracted clinical fields. **Zero API overlap. Zero conceptual -overlap** beyond both caring about "correctness." - -### 4.3 Migration shape — **leave standalone (category mismatch)** - -Integrating openadapt-verifier into flow's replay/postcondition path would be a category -error — it does not verify GUI state and cannot. flow's postcondition/identity machinery -is *correctly* home-grown because **nothing in the ecosystem does GUI-state verification**; -this is genuinely flow-specific IP, and keeping it in `ir.py`/`runtime/` is the right call. - -**The one legitimate (future, out-of-scope) touchpoint:** if flow ever ships a -*benchmark-accuracy* report — "the compiled workflow wrote the correct field values into -the EMR across N runs, with confidence intervals" — then openadapt-verifier is the right -tool for **that** report (it's literally a clinical-extraction accuracy scorer with CIs and -regression `compare()`). That is a *benchmark/reporting* use, entirely separate from the -runtime, and not part of this roadmap. Do not couple the runtime to it. - -**Risk: N/A.** The recommendation is non-integration. - ---- - -## 5. Sequenced roadmap - -Ordering across **all** in-flight integrations (types/capture/verifier owned here; -privacy/grounding/umbrella owned by other workstreams): - -| # | Integration | Buys | Risks | Path-of-dependency | Verdict | -|---|---|---|---|---|---| -| **1** | **types interop shim** | Shared action vocabulary; clean evals/emit round-trip; future-proofs `ActionType` alignment | Low; field-map maintenance only | Boundary only (emit/, benchmark/); optional extra | **Do now** | -| **2** | **privacy** *(other workstream)* | PHI scrubbing on recordings/frames — prerequisite for any healthcare desktop capture | Medium; must scrub at capture/compile edge, never leak into bundles | Compile-time edge; optional extra | **Do (owned elsewhere); land before desktop capture ships to clinics** | -| **3** | **capture adapter fix** | Real cross-platform **desktop** recording (flow's biggest gap) | Medium; current adapter targets a dead schema; needs a permissioned integration test | Compile-time on-ramp; optional `[capture]` extra | **Do after 1–2** | -| **4** | **grounding eval** *(other workstream)* | Evidence for/against the optional grounder rung; keeps the ladder honest | Low if kept opt-in; High if grounder creeps onto the default path | Opt-in last ladder rung only | **Evaluate (owned elsewhere); keep OFF by default** | -| **5** | **umbrella `openadapt[flow]` extra** *(other workstream)* | One-line install of the composed stack | Low; packaging only — must not make heavy deps mandatory | Packaging metadata | **Do last; assemble the optional extras above** | -| **—** | **verifier** | Nothing for the runtime | — | — | **Do NOT integrate** (see §4) | - -### Why this order - -- **types first** because it's the cheapest and it's a *precondition for clean interop* on - everything downstream (emit, benchmark, evals). It also has zero effect on the hot path, - so it can't destabilize the current release. -- **privacy before desktop capture** because desktop capture is precisely where - PHI-bearing pixels and keystrokes enter the system. Recording a clinic desktop without a - scrub step is the one integration ordering that is *unsafe* to get wrong. -- **capture third** because the scaffolding exists but is broken against the real schema — - it's correction work with a real test dependency (OS permissions), so it shouldn't block - the cheap wins. -- **grounding evaluated, not adopted-by-default** — the grounder is the *last, optional* - ladder rung by design. The moment it becomes mandatory, flow loses its "$0, no model - calls, runs in CI" headline. Keep it opt-in behind `[grounder]`. -- **umbrella last** because it's the bow on top: it should compose the optional extras - above, and must never promote any of them to a mandatory core dependency. - ---- +```bash +python -m pip install 'openadapt-flow[capture]' +``` -## 6. Where staying standalone is the right call +The Capture dependency remains outside the browser and replay hot paths. A +native recording needs an interactive desktop plus the applicable operating +system permissions. An offline pixel recording cannot reconstruct structural +accessibility evidence. A workflow that needs UIA, Accessibility, or AT-SPI +identity must retain a live structural observation or receive that evidence +during qualification. -flow's standalone lean surface is a deliberate strength — a certification-moat asset, not -tech debt. Explicit "keep it home-grown" calls: +For RDP and Citrix, Capture observes the local client window. It does not claim +that a native accessibility tree crosses the remote boundary. Flow uses the +retained pixels, OCR, relational anchors, identity regions, and fresh-frame +checks for that external surface. -1. **`ir.py` stays the internal source of truth.** Interop with `openadapt-types` at the - boundary; never dissolve the compiler IR (Anchor/Postcondition/IdentityCheck) into the - canonical schema. Those are flow's differentiators and don't belong in a passive shared - type. -2. **The replay hot path stays dependency-minimal.** `pydantic + opencv + rapidocr + - pillow + numpy`. Grounding, capture, privacy, and types-interop are all **edges** - (compile-time or opt-in), never core. The property that "the whole loop runs in CI with - no OS permissions and zero model calls" is the product; protect it. -3. **Postcondition + identity verification stays home-grown.** Nothing in the ecosystem - does GUI-state verification (openadapt-verifier is a clinical scorer). This is - flow-specific IP; keep it in `runtime/`. -4. **The Playwright reference recorder stays.** capture is the desktop on-ramp, not a - replacement — the permission-free CI recorder is what keeps the whole test story cheap. -5. **The `Backend` protocol stays small.** New substrates (Windows/Parallels/RDP) - implement the tiny vision-only protocol; they do not pull the ecosystem into the - runtime. +## Types integration -The governing test for any future integration proposal: +Flow's compiled `Step` contains anchors, identity requirements, +postconditions, action risk, effect contracts, and audit state. A shared +`openadapt-types` action describes portable action intent. These are different +layers. -> **Does it add shared vocabulary or close a real capability gap — at an edge — without -> putting a new dependency on the replay hot path or dissolving the auditable core?** -> If yes, integrate behind an optional extra. If no, stay standalone. +The optional interop module converts supported actions at package boundaries. +Flow keeps its internal IR as the source of truth for compilation and runtime +safety. Consumers must negotiate a schema version. Dependency presence alone +does not upgrade an existing peer contract. ---- +## Privacy integration -## Appendix A — flow `ir` ↔ `openadapt-types` field map (for the shim) +Privacy controls run at source-time and artifact boundaries. Secret browser +fields are excluded before an event or frame persists. A sanitized derivative +is created from a copy, inventories every file, binds review to the exact +bytes, and preserves the original inside its trusted boundary. -| flow `ir` | openadapt-types | Note | -|---|---|---| -| `ActionKind.{CLICK,DOUBLE_CLICK,TYPE,KEY,WAIT,SCROLL}` | `ActionType.{same}` | identical string values; `set(ActionKind) ⊆ set(ActionType)` | -| `Step.action` | `Action.type` | 1:1 | -| `Anchor.click_point (x,y)` | `Action.target = ActionTarget(x,y,is_normalized=False)` | pixel coords | -| `Anchor.ocr_text` / `context_text` | `ActionTarget.description` (lossy) | flow's is richer; export only | -| `Step.text` / `Step.key` | `Action.text` / `Action.key` | 1:1; validators agree | -| `Step.scroll_dx/dy` | `Action.scroll_direction` + `scroll_amount` | sign/px→direction conversion | -| `StepResult{ok,error,elapsed_ms,resolution.point}` | `ActionResult{success,error,duration_ms,resolved_coordinates}` | export flow→types | -| `Anchor/Postcondition/Landmark/IdentityCheck/HealEvent` | *(none)* | net-new; **do not** try to map | -| `RunReport` | `Episode`(+`Step`) | for evals/RL flywheel only | +Live runtime observations can contain sensitive data again. They remain inside +the declared execution boundary. -## Appendix B — dependency-weight ledger +## Non-product research packages -| Package | Import weight | On flow's hot path? | -|---|---|---| -| openadapt-types | pydantic only (light) | No — boundary/optional | -| openadapt-capture | Native input/screen APIs plus separately provisioned external FFmpeg for video | No — compile-time recorder on-ramp, optional extra | -| openadapt-verifier | stdlib-only (light) | N/A — not integrated | -| flow core | pydantic, numpy, opencv-headless, pillow, rapidocr, playwright, httpx | This IS the hot path; keep it here | +`openadapt-verifier`, `openadapt-grounding`, `openadapt-retrieval`, and +`openadapt-viewer` are not required to record, compile, replay, or verify a +workflow. Keep a package integration only when a live product boundary consumes +it. Do not infer product maturity from a package name or from code presence. diff --git a/docs/PRODUCT_STATUS.md b/docs/PRODUCT_STATUS.md index 96d7ac19..ff0b27fb 100644 --- a/docs/PRODUCT_STATUS.md +++ b/docs/PRODUCT_STATUS.md @@ -39,11 +39,11 @@ and its generated view is [`VERIFICATION.md`](VERIFICATION.md). | System-of-record effect verification | **Experimental** | REST, FHIR, SQL, file, and document verifier contracts catch fault classes that screen-only verification misses. A deployment with multiple reviewed read boundaries selects and preflights the strongest evidence tier before input, retains that binding through durable resume, and never downgrades after an action. | Effects are not generally inferred; both authored effects and a configured verifier are required. A selected verifier that becomes unavailable halts or enters reconciliation. | | Lint and certification policies | **Beta** | The CLI reports coverage gaps and refuses bundles that violate a selected policy. | Certification is opt-in; `replay` remains the permissive tutorial path. Use fail-closed `run` for a deployment. | | Durable pause, approval, and resume | **Experimental** | Checkpoint, bundle-version binding, approval, stale-pause, and resume semantics are tested. | Operator identity is recorded, not integrated with an enterprise IdP; field operation is unmeasured. | -| Typed business decisions | **Experimental** | A typed qualification API adds or updates a finite decision node without manual manifest edits and invalidates stale certification. The graph runtime pauses at the certified choice, validates a supplied principal and role, retains a signed durable receipt, restores it after a crash, revalidates the live application, and permits only the certified successor branch. | The engine does not authenticate a user. Desktop, Cloud, or a customer-local identity route must supply an authenticated principal. A decision never replaces entity identity or effect verification. | +| Typed business decisions | **Beta** | A typed qualification API adds or updates a finite decision node without manual manifest edits and invalidates stale certification. The graph runtime pauses at the certified choice, validates a supplied principal and role, retains a signed durable receipt, restores it after a crash, revalidates the live application, and permits only the certified successor branch. | The engine does not authenticate a user. Desktop, Cloud, or a customer-local identity route must supply an authenticated principal. A decision never replaces entity identity or effect verification. | | Reviewed judgment cases | **Experimental** | Qualification binds typed facts, local evidence hashes, reviewer provenance, and the exact decision contract to reviewed examples or counterfactuals. It preserves permanent human authority, requires reciprocal contrasts for an automatic-rule candidate, and refuses certification when a case still needs evidence. | The case layer does not synthesize executable policy from one or more examples. A reviewed automatic rule must be authored and qualified through the normal program path. | | Qualified remote decision tasks | **Experimental** | An explicitly negotiated V2 task binds optional reviewed entity wording to the exact qualification, bundle, step, policy, and pause. V1 stays byte-compatible, and an unavailable or unrecognized class renders as the signed neutral `record` or `item` fallback. | V2 requires `openadapt-types` 0.10.x and a consumer that negotiates the schema. Actual entity identifiers and live revalidation stay inside the customer-controlled runner. | | On-prem / air-gapped deployment | **Beta foundation** | Local queue, fail-closed run gate, egress attestation, audit-chain verification, signed release verification, fresh-environment smoke/air-gap checks, atomic blue/green update, and rollback ship. | Site firewall, storage, keys, OS hardening, identity/effect integrations, and acceptance in the customer's environment remain deployment responsibilities. | -| Desktop GUI and tray | **Beta, separate repository** | Desktop `v0.9.0` ships installable Windows, macOS, and Linux artifacts with checksums; its frozen engine lifecycle and install/launch/uninstall contracts run in release CI. | The published installer evidence covers the embedded browser lifecycle. Native and remote substrate selection is independently qualified through Flow and remains bound to the selected target configuration. | +| Desktop GUI and tray | **Beta, separate repository** | Desktop `v0.15.0` ships installable Windows, macOS, and Linux artifacts with checksums; its frozen engine lifecycle and install/launch/uninstall contracts run in release CI. | The published installer evidence covers the embedded browser lifecycle. Native and remote substrate selection is independently qualified through Flow and remains bound to the selected target configuration. | | Hosted dashboard / control plane | **Live beta, separate repository** | The deployed service uses live Supabase, Stripe, and Modal dependencies for account and organization onboarding, checkout, exact-hash artifact ingest, attested browser workflow versions, structural reports, replacement activation, scheduling, entitlements, and metering. The reversible pre-payment contract passed 3/3 production trials. | The first genuine customer payment remains the acceptance event for the paid post-payment lifecycle. SLA, BAA, and compliance commitments apply only when included in reviewed written terms. | | Hosted execution | **Live beta — browser** | Production mode admits exact attested browser bundles, dispatches the configured runner, authenticates callbacks, and refuses mock fallback; development mock mode remains visibly synthetic. | Desktop, RDP, Citrix, and customer-controlled regulated execution are separately scoped deployment lanes rather than capabilities implied by the browser subscription. | | Offline update and rollback | **Beta** | The operator-pulled path verifies signed archives, installs into a fresh blue/green environment, runs smoke and air-gap checks, atomically swaps the active release, and records rollback state. | Signer trust, artifact transport, OS/container policy, backup, disaster recovery, and a customer-site rehearsal remain deployment responsibilities. | @@ -73,7 +73,7 @@ and its generated view is [`VERIFICATION.md`](VERIFICATION.md). - Benchmarks describe their task, environment, run count, success oracle, latency, model calls, cost assumptions, and caveats. They are bounded evidence, not general market proof. -- The nightly +- The weekly [`quickstart-lifecycle.yml`](../.github/workflows/quickstart-lifecycle.yml) installs the built wheel in a clean environment on Linux, macOS, and Windows; records, compiles, lints, certifies, replays, induces drift, inspects repair diff --git a/docs/SURFACES.md b/docs/SURFACES.md index 0570ed8d..06de164d 100644 --- a/docs/SURFACES.md +++ b/docs/SURFACES.md @@ -43,31 +43,41 @@ as before. Each surface has an equivalent record -> compile -> replay path: +| Workflow surface | Exact install | +|---|---| +| Browser | `pip install 'openadapt-flow[browser]'` | +| Native Windows | `pip install 'openadapt-flow[capture,windows]'` | +| Native macOS | `pip install 'openadapt-flow[capture,macos]'` | +| Native Linux | `pip install 'openadapt-flow[capture,linux]'` plus the AT-SPI system packages in [`desktop/LINUX_NATIVE.md`](desktop/LINUX_NATIVE.md) | +| Network RDP | Recorder: `pip install 'openadapt-flow[capture]'` inside the demonstrated session; runner: `pip install 'openadapt-flow[rdp]'` | +| Local RDP/Citrix client window | macOS host: `pip install 'openadapt-flow[capture,macos]'`; Windows host: `pip install 'openadapt-flow[capture,windows]'` | + ```bash # Browser (Playwright / Chromium) openadapt-flow record --backend web --url https://your.app --out rec openadapt-flow compile rec --out bundle --name my-task openadapt-flow replay bundle --url https://your.app -# Windows (native UI Automation via the in-guest WAA agent) -openadapt-flow record --backend windows --agent-url http://localhost:5001 --out rec +# Windows: Capture records the local window; the in-guest WAA agent replays it. +openadapt-flow record --backend windows --window "Target App" --out rec openadapt-flow compile rec --out bundle --name my-task openadapt-flow replay bundle --agent-url http://localhost:5001 -# macOS (accessibility, one app window) -openadapt-flow record --backend macos --macos-app TextEdit --out rec +# macOS: the app and title scope the local Capture window. +openadapt-flow record --backend macos --macos-app TextEdit \ + --macos-window-title notes.txt --out rec openadapt-flow compile rec --out bundle --name my-task -openadapt-flow replay bundle --macos-app TextEdit +openadapt-flow replay bundle --macos-app TextEdit \ + --macos-window-title notes.txt -# Linux (AT-SPI, one exact app window) -openadapt-flow record --backend linux --linux-app gedit \ - --linux-window-title "Untitled Document 1" --out rec +# Linux: Capture records the local desktop; AT-SPI selects the replay target. +openadapt-flow record --backend linux --out rec openadapt-flow compile rec --out bundle --name my-task openadapt-flow replay bundle --linux-app gedit \ --linux-window-title "Untitled Document 1" -# RDP (network session, or a local remote-desktop client window) -openadapt-flow record --backend rdp --rdp-host 10.0.0.5 --out rec +# Network RDP: run record inside the demonstrated remote session. +openadapt-flow record --backend rdp --out rec openadapt-flow compile rec --out bundle --name my-task openadapt-flow replay bundle --rdp-host 10.0.0.5 @@ -81,10 +91,14 @@ openadapt-flow replay bundle --rdp-window "Citrix Viewer" \ ``` The bound surface is the replay default, so `--backend` may be omitted on -`replay`/`run` for a bound bundle; the target flags (`--agent-url`, -`--macos-app`, ...) still name the concrete window/host. `run ... --config -deploy.yaml --profile standard|regulated` wires the same selection for a real -deployment. +`replay`/`run` for a bound bundle. During record, `--macos-app` / +`--macos-window-title` scope the macOS Capture window, and the local +RDP/Citrix flags `--rdp-window` / `--rdp-window-title` scope Capture and enter +the bundle's existing replay-binding metadata. `--agent-url`, `--linux-app`, +`--linux-window-title`, and `--rdp-host` are replay targets. The local Capture +session cannot control them, so `record` refuses them instead of accepting an +unused flag. Pass them to `replay`/`run`; `run ... --config deploy.yaml +--profile standard|regulated` wires the same selection for a real deployment. ## The two remote execution modes diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 1e1de28e..a34f294e 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -25,14 +25,14 @@ | Backing evidence | Kind | Gating / CI stage | Strength | Proves | |---|---|---|---|---| -| `tests/e2e/test_record_compile_replay.py` | test | ci (post-merge/nightly full suite) | supported | Records the MockMed browser demo once, compiles it, and replays it under baseline + theme/move/rename drift and parameter substitution through the headless-browser Backend. | +| `tests/e2e/test_record_compile_replay.py` | test | ci (required PR gate (e2e-browser)) | supported | Records the MockMed browser demo once, compiles it, and replays it under baseline + theme/move/rename drift and parameter substitution through the headless-browser Backend. | | `tests/test_mockmed.py` | test | ci (required PR gate (test)) | supported | The reference browser demo app and its drift screens render deterministically (no CSS transitions), so replay is repeatable. | | `tests/test_replayer.py` | test | ci (required PR gate (test)) | supported | The deterministic replayer resolves steps, substitutes parameters, enforces postconditions and the risk gate — no model in the loop. | **Caveats (honest limits):** - "Supported" is scoped to the reference headless-browser backend in this registry. Desktop and remote-display workflows use the separately scoped acceptance and code-qualified claims below. -- The full record->compile->replay e2e runs in the post-merge/nightly full suite; the fast required PR gate covers the replayer + demo units. +- The full record->compile->replay browser suite runs in the required e2e-browser PR gate and repeats in the weekly compatibility matrix. ### `deterministic-zero-model-replay` — supported — CI-proven today @@ -43,7 +43,7 @@ | Backing evidence | Kind | Gating / CI stage | Strength | Proves | |---|---|---|---|---| -| `tests/e2e/test_record_compile_replay.py` | test | ci (post-merge/nightly full suite) | supported | Baseline replay x3 resolves every step on the `template` rung with 0 heals and 0 model calls; healed bundles replay clean afterward. | +| `tests/e2e/test_record_compile_replay.py` | test | ci (required PR gate (e2e-browser)) | supported | Baseline replay x3 resolves every step on the `template` rung with 0 heals and 0 model calls; healed bundles replay clean afterward. | | `tests/test_replayer.py` | test | ci (required PR gate (test)) | supported | The replayer is a deterministic resolution ladder; the VLM rung is a fallback, not on the default path — the core runtime is model-free. | | `benchmark/BENCHMARK.md` | benchmark | artifact (doc/benchmark) | roadmap | CI-reproducible MockMed run artifact: 100/100 compiled replays at 4.9s p50 with zero model calls, measured 2026-07-08 on Flow 0.1.0 (pre-v0.2.0 source build); field/artifact reference. | diff --git a/docs/desktop/PHASE1.md b/docs/desktop/PHASE1.md index 9ae90be4..cf1f59fa 100644 --- a/docs/desktop/PHASE1.md +++ b/docs/desktop/PHASE1.md @@ -156,9 +156,10 @@ window events (out-of-window input refuses conversion — it targeted a different window), refuses sessions where the target window was resized (capture video and Flow recordings currently use one fixed viewport), verifies extracted frames have that exact viewport, and stamps the output `meta.json` -with `window_capture` provenance plus `backend_hints` (`rdp_window` / -`rdp_window_title`) naming the recorded target window for -`replay --backend rdp`. +with `window_capture` provenance. The `record --backend rdp|citrix` +orchestration adds `backend_hints` (`rdp_window` / `rdp_window_title`) naming +the recorded target window for remote replay. Native Windows and macOS window +recordings do not receive remote hints. **Frame selection.** For an event at wall-clock `T`: *before* = last video frame at/before `T`; *after* = frame at `T + settle_s` (default 1.0 s), diff --git a/docs/desktop/RECORDING.md b/docs/desktop/RECORDING.md index 5cf266c1..a05c3caf 100644 --- a/docs/desktop/RECORDING.md +++ b/docs/desktop/RECORDING.md @@ -128,10 +128,13 @@ demonstration recorded here is already in the pixel space the `rdp` backend replays in (`CaptureSession.window_capture`, `coordinate_space: window_pixels`). The capture adapter stamps the window identity into `meta.json` under `window_capture` (target + resolved owner/title, plus the -resolved `resolved_pid` / `resolved_window_id` OS handle where available) and -emits closed `backend_hints` (`backend`, `rdp_window`, `rdp_window_title`, and -optional `rdp_readiness_text`) so compile preserves the target and an unflagged -replay resolves the same client window. Capture selectors are intentionally +resolved `resolved_pid` / `resolved_window_id` OS handle where available). +When the operator selected RDP or Citrix, the desktop recording orchestration +also emits closed `backend_hints` (`backend`, `rdp_window`, +`rdp_window_title`, and optional `rdp_readiness_text`) so compile preserves the +target and an unflagged replay resolves the same client window. Native Windows +and macOS window recordings keep their native surface identity and do not get +remote hints. Capture selectors are intentionally substring-based; replay selectors are exact. Use `--window` to find the window during recording and `--rdp-window` / `--rdp-window-title` to pin the exact replay identity. The resolved exact owner/title is used when the explicit replay diff --git a/docs/verification.json b/docs/verification.json index d6adf1e8..f47e56f5 100644 --- a/docs/verification.json +++ b/docs/verification.json @@ -16,7 +16,7 @@ "strongest_evidence": "supported", "caveats": [ "\"Supported\" is scoped to the reference headless-browser backend in this registry. Desktop and remote-display workflows use the separately scoped acceptance and code-qualified claims below.", - "The full record->compile->replay e2e runs in the post-merge/nightly full suite; the fast required PR gate covers the replayer + demo units." + "The full record->compile->replay browser suite runs in the required e2e-browser PR gate and repeats in the weekly compatibility matrix." ], "evidence": [ { @@ -24,7 +24,7 @@ "kind": "test", "exists": true, "strength": "supported", - "gating": "ci (post-merge/nightly full suite)", + "gating": "ci (required PR gate (e2e-browser))", "node": null, "node_found": null, "junit_status": null, @@ -75,7 +75,7 @@ "kind": "test", "exists": true, "strength": "supported", - "gating": "ci (post-merge/nightly full suite)", + "gating": "ci (required PR gate (e2e-browser))", "node": null, "node_found": null, "junit_status": null, diff --git a/openadapt_flow/__main__.py b/openadapt_flow/__main__.py index ab63fb38..eb248e05 100644 --- a/openadapt_flow/__main__.py +++ b/openadapt_flow/__main__.py @@ -112,6 +112,141 @@ def _parse_identifier_region_arg( return (region[0], region[1], region[2], region[3]) +_RECORD_TARGET_FLAGS: tuple[tuple[str, str], ...] = ( + ("agent_url", "--agent-url"), + ("macos_app", "--macos-app"), + ("macos_window_title", "--macos-window-title"), + ("linux_app", "--linux-app"), + ("linux_window_title", "--linux-window-title"), + ("linux_allow_physical_input", "--linux-allow-physical-input"), + ("rdp_host", "--rdp-host"), + ("rdp_window", "--rdp-window"), + ("rdp_window_title", "--rdp-window-title"), + ("rdp_readiness_text", "--rdp-readiness-text"), +) + + +def _record_flag_is_set(args: argparse.Namespace, attr: str) -> bool: + value = getattr(args, attr, None) + if isinstance(value, bool): + return value + return value is not None + + +def _reject_unbound_record_target_flags(args: argparse.Namespace, backend: str) -> None: + """Refuse target flags that cannot affect this capture session. + + The backend flags are shared with replay/run. A record command must not + accept one unless it either scopes the live Capture session or is retained + by the existing recording schema as a replay binding. Otherwise the CLI + would appear to record the named app/host while actually recording an + unrelated local desktop. + """ + allowed: dict[str, set[str]] = { + "web": set(), + "windows": set(), + "macos": {"macos_app", "macos_window_title"}, + "linux": set(), + "rdp": {"rdp_window", "rdp_window_title", "rdp_readiness_text"}, + "citrix": {"rdp_window", "rdp_window_title", "rdp_readiness_text"}, + } + for attr, flag in _RECORD_TARGET_FLAGS: + if not _record_flag_is_set(args, attr) or attr in allowed[backend]: + continue + if backend == "windows" and attr == "agent_url": + reason = ( + "the local Capture session cannot bind to a WAA endpoint. " + "Scope the local recording with --window/--window-title, then " + "pass --agent-url to replay or run" + ) + elif backend == "linux" and attr in { + "linux_app", + "linux_window_title", + "linux_allow_physical_input", + }: + reason = ( + "the current Capture component has no Linux window-scoping " + "primitive. Record the local Linux desktop without this flag, " + "then pass it to replay or run" + ) + elif backend == "rdp" and attr == "rdp_host": + reason = ( + "Capture cannot connect to a network RDP endpoint. Record " + "inside the remote session, or scope a local client with " + "--window/--rdp-window, then pass --rdp-host to replay or run" + ) + else: + reason = f"this target flag does not apply to the {backend} recorder" + raise SystemExit( + f"record --backend {backend}: {flag} was not applied: {reason}. " + "Nothing was recorded." + ) + + +def _merge_record_window_selector( + generic_value: Optional[str], + target_value: Optional[str], + *, + generic_flag: str, + target_flag: str, + backend: str, +) -> Optional[str]: + """Merge generic and surface-specific Capture selectors without guessing.""" + if generic_value is None: + return target_value + if target_value is None: + return generic_value + if generic_value.strip().casefold() != target_value.strip().casefold(): + raise SystemExit( + f"record --backend {backend}: {generic_flag} and {target_flag} " + "name different capture targets. Give one target, or give the " + "same value to both. Nothing was recorded." + ) + return generic_value + + +def _resolve_record_capture_window( + args: argparse.Namespace, backend: str +) -> Optional[dict[str, Optional[str]]]: + """Resolve the exact window that the local Capture session will record.""" + _reject_unbound_record_target_flags(args, backend) + + owner = getattr(args, "window", None) + title = getattr(args, "window_title", None) + if backend == "linux" and (owner is not None or title is not None): + raise SystemExit( + "record --backend linux: --window/--window-title cannot be applied " + "because the current Capture component has no Linux " + "window-scoping primitive. Record the local Linux desktop without " + "these flags. Nothing was recorded." + ) + if backend == "macos": + owner = _merge_record_window_selector( + owner, + getattr(args, "macos_app", None), + generic_flag="--window", + target_flag="--macos-app", + backend=backend, + ) + title = _merge_record_window_selector( + title, + getattr(args, "macos_window_title", None), + generic_flag="--window-title", + target_flag="--macos-window-title", + backend=backend, + ) + elif backend in ("rdp", "citrix"): + # The capture selector is a local owner/title substring. The replay + # selector can be an exact process identity (for example ``wfica32``), + # so the two values can legitimately differ. A dedicated --window + # value controls Capture; otherwise the replay selector also scopes it. + owner = owner or getattr(args, "rdp_window", None) + title = title or getattr(args, "rdp_window_title", None) + if owner is None and title is None: + return None + return {"owner": owner, "title": title} + + def _replay_params( pairs: Sequence[str] | None, params_file: str | None = None, @@ -776,6 +911,7 @@ def _cmd_record(args: argparse.Namespace) -> int: "backends (--backend windows/macos/linux/rdp/citrix); --backend web " "records the Playwright page given by --url." ) + _reject_unbound_record_target_flags(args, "web") if not args.url: raise SystemExit( @@ -821,6 +957,23 @@ def _cmd_record_desktop(args: argparse.Namespace, backend: str) -> int: full-screen the client) so coordinates align; a cross-machine coordinate remap is a documented follow-up (docs/desktop/RECORDING.md). """ + if getattr(args, "url", None) is not None: + raise SystemExit( + f"record --backend {backend}: --url applies only to --backend web. " + "Nothing was recorded." + ) + if getattr(args, "headless", False): + raise SystemExit( + f"record --backend {backend}: --headless applies only to --backend " + "web. Nothing was recorded." + ) + + # Resolve and validate every target flag before importing or starting + # Capture. Surface-specific flags either scope this exact recording (macOS + # and local RDP/Citrix windows), enter the existing remote-display binding + # schema, or fail loud. None can be accepted and then ignored. + window = _resolve_record_capture_window(args, backend) + if args.secret: # Field-level secret redaction relies on DOM field geometry (the # browser recorder blacks out the field rect). A pixel/desktop capture @@ -846,17 +999,6 @@ def _cmd_record_desktop(args: argparse.Namespace, backend: str) -> int: getattr(args, "identifier", None) or (), backend=backend ) - # Window-scoping (optional): capture ONE window in its own pixel space. - # Selectors are case-insensitive substrings (owner app / window title), - # matching openadapt-capture's WindowTarget; None means full-screen capture. - window_owner = getattr(args, "window", None) or getattr(args, "rdp_window", None) - window_title = getattr(args, "window_title", None) or getattr( - args, "rdp_window_title", None - ) - window: Optional[dict[str, Optional[str]]] = None - if window_owner or window_title: - window = {"owner": window_owner, "title": window_title} - from openadapt_flow.desktop_record import record_desktop_capture task = args.task or f"openadapt-flow {backend} recording" @@ -3538,12 +3680,10 @@ def _add_backend_flags(p: argparse.ArgumentParser) -> None: default=None, help=( "Backend to drive: 'web' (default; Playwright/Chromium), 'windows' " - "(native Windows via the WAA HTTP agent — needs --agent-url), " - "'macos' (one native Mac app window — needs --macos-app), or " - "'linux' (one exact AT-SPI app window — needs --linux-app and " - "--linux-window-title), or " - "'rdp' (pixel-only network or local remote desktop — needs " - "--rdp-host or a configured rdp_window), or 'citrix' (the local " + "(native Windows via the WAA HTTP agent at replay), 'macos' (one " + "native Mac app window), 'linux' (one exact AT-SPI app window at " + "replay), 'rdp' (pixel-only network or local remote desktop), or " + "'citrix' (the local " "Citrix Workspace window; its owner defaults by host OS and a " "configured rdp_window may override it). Overrides backend.kind " "from --config." @@ -3555,7 +3695,9 @@ def _add_backend_flags(p: argparse.ArgumentParser) -> None: metavar="URL", help=( "Base URL of the in-guest Windows (WAA) agent for --backend windows " - "(e.g. http://localhost:5001). Overrides backend.agent_url." + "(e.g. http://localhost:5001). Replay/run only: desktop record " + "refuses this flag because local Capture cannot bind to a WAA " + "endpoint. Overrides backend.agent_url." ), ) p.add_argument( @@ -3563,8 +3705,9 @@ def _add_backend_flags(p: argparse.ArgumentParser) -> None: default=None, metavar="APP", help=( - "Owner application for --backend macos (e.g. TextEdit). Overrides " - "backend.macos_app." + "Owner application for --backend macos (e.g. TextEdit). During " + "record this scopes Capture to that local app window; during " + "replay/run it overrides backend.macos_app." ), ) p.add_argument( @@ -3573,7 +3716,8 @@ def _add_backend_flags(p: argparse.ArgumentParser) -> None: metavar="TITLE", help=( "Window-title substring for --backend macos. Ambiguous matches " - "are refused. Overrides backend.macos_window_title." + "are refused. During record this scopes Capture; during replay/run " + "it overrides backend.macos_window_title." ), ) p.add_argument( @@ -3582,7 +3726,8 @@ def _add_backend_flags(p: argparse.ArgumentParser) -> None: metavar="APP", help=( "Exact AT-SPI application name for --backend linux (e.g. gedit). " - "Overrides backend.linux_app." + "Replay/run only: the current Capture path records the local Linux " + "desktop and refuses this flag. Overrides backend.linux_app." ), ) p.add_argument( @@ -3591,7 +3736,9 @@ def _add_backend_flags(p: argparse.ArgumentParser) -> None: metavar="TITLE", help=( "Exact top-level window title for --backend linux. Zero or " - "multiple matches are refused. Overrides backend.linux_window_title." + "multiple matches are refused. Replay/run only: the current " + "Capture path records the local Linux desktop and refuses this " + "flag. Overrides backend.linux_window_title." ), ) p.add_argument( @@ -3599,7 +3746,8 @@ def _add_backend_flags(p: argparse.ArgumentParser) -> None: action="store_true", help=( "Explicitly allow window-bound X11 pointer/keyboard fallback for " - "--backend linux when native AT-SPI actuation is unavailable." + "--backend linux replay/run when native AT-SPI actuation is " + "unavailable. Record refuses this replay-only flag." ), ) p.add_argument( @@ -3608,8 +3756,9 @@ def _add_backend_flags(p: argparse.ArgumentParser) -> None: metavar="HOST", help=( "RDP host/IP for --backend rdp (network RDP via FreeRDP). Overrides " - "backend.rdp_host. For a local client window use --rdp-window " - "instead." + "backend.rdp_host. Replay/run only: record cannot connect local " + "Capture to this endpoint. Record inside the remote session, or " + "capture a local client with --window/--rdp-window." ), ) p.add_argument( @@ -3620,7 +3769,8 @@ def _add_backend_flags(p: argparse.ArgumentParser) -> None: "Exact local remote-display window owner/process for --backend " "rdp or citrix. On Windows this is the process basename (for " "example wfica32); on macOS it is the app owner (for example " - "'Citrix Viewer'). Overrides backend.rdp_window." + "'Citrix Viewer'). During record it also scopes local Capture to " + "that window; during replay/run it overrides backend.rdp_window." ), ) p.add_argument( @@ -3629,7 +3779,8 @@ def _add_backend_flags(p: argparse.ArgumentParser) -> None: metavar="TITLE", help=( "Exact local remote-display window title used to disambiguate " - "multiple matching RDP/Citrix client windows. Overrides " + "multiple matching RDP/Citrix client windows. During record it " + "also scopes local Capture; during replay/run it overrides " "backend.rdp_window_title." ), ) @@ -3814,8 +3965,8 @@ def build_parser() -> argparse.ArgumentParser: "record", help=( "Record YOUR workflow interactively: a headed browser " - "(--backend web --url), or a native Windows desktop " - "(--backend windows --agent-url) capturing the operator's real input" + "(--backend web --url), or the operator's real local desktop input " + "through openadapt-capture (--backend windows/macos/linux/rdp/citrix)" ), ) p.add_argument( diff --git a/openadapt_flow/_browser_setup.py b/openadapt_flow/_browser_setup.py index d3216074..17d5b097 100644 --- a/openadapt_flow/_browser_setup.py +++ b/openadapt_flow/_browser_setup.py @@ -26,9 +26,11 @@ import importlib.util import os +import re import subprocess import sys import threading +from pathlib import Path #: Environment variable that disables the auto-install (air-gapped / CI that #: pre-provisions the browser itself). Any non-empty value opts out. @@ -74,18 +76,34 @@ def _opted_out() -> bool: def _chromium_present() -> bool: """Return whether Playwright's Chromium browser binary is installed. - Playwright always reports the *expected* executable path for the pinned - browser revision (even when it has never been downloaded), so the presence - of the file on disk is the reliable signal -- we do not rely on catching a - launch error. Any failure to determine the path is treated as "not present" - so the (idempotent) install is attempted rather than wrongly skipped. + Ask Playwright's non-actuating CLI for the exact install locations and + require each completion marker. Do not start ``sync_playwright()`` only to + inspect ``chromium.executable_path``. Playwright 1.62 can leave its driver + connection task pending when that short-lived probe exits, which prints a + false-success-shaped ``TargetClosedError`` after an otherwise healthy CLI + command on Linux, macOS, and Windows. + + Any failure to determine the locations is treated as "not present" so the + idempotent install is attempted rather than wrongly skipped. """ require_browser_support() - from playwright.sync_api import sync_playwright - - with sync_playwright() as p: - path = p.chromium.executable_path - return bool(path) and os.path.exists(path) + result = subprocess.run( + [sys.executable, "-m", "playwright", "install", "--dry-run", "chromium"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + ) + locations = [ + Path(match.group(1).strip()) + for match in re.finditer( + r"(?m)^\s*Install location:\s*(.+?)\s*$", result.stdout + ) + ] + return bool(locations) and all( + (location / "INSTALLATION_COMPLETE").is_file() for location in locations + ) def _install_chromium() -> None: diff --git a/openadapt_flow/adapters/capture.py b/openadapt_flow/adapters/capture.py index ef700167..3586839a 100644 --- a/openadapt_flow/adapters/capture.py +++ b/openadapt_flow/adapters/capture.py @@ -88,14 +88,11 @@ silently lose a demonstrated action; keeping it would compile a wrong-target step); * stamps the output ``meta.json`` with the recorded scoping - (``window_capture``) plus ``backend_hints`` naming the recorded target - owner/title in ``BackendConfig`` terms (``rdp_window`` / - ``rdp_window_title``). Live ``record --backend rdp|citrix`` orchestration - seals these hints, the compiler carries them in ``Workflow.backend_hints``, - and replay uses them unless explicit deployment config or CLI flags - override them. Direct offline conversion cannot infer Citrix versus another - remote-display client, so it retains the generic ``backend: rdp`` identity - for backward compatibility. + (``window_capture``), but does not infer an execution surface from window + scope alone. Live ``record --backend rdp|citrix`` orchestration adds the + exact ``backend_hints`` (``rdp_window`` / ``rdp_window_title``). Native + Windows and macOS recording uses the same window primitive without remote + hints, so the compiler receives one consistent surface identity. A window-scoped session that declares a coordinate space this adapter does not understand is refused loudly rather than converted with guessed scaling. @@ -834,9 +831,10 @@ def convert_capture( pass through unscaled (they are already captured-frame pixels), the viewport/timeline/frame-size contract is verified, every mouse action is screened against it (an out-of-window action refuses conversion), and the - output ``meta.json`` additionally carries ``window_capture`` provenance - plus ``backend_hints`` (``rdp_window`` / ``rdp_window_title``) naming the - recorded target window for ``replay --backend rdp``. + output ``meta.json`` additionally carries ``window_capture`` provenance. + The orchestration layer adds replay hints only when the operator selected + an RDP or Citrix surface. A window alone does not prove that the captured + surface is remote. Args: capture_dir: An openadapt-capture session directory (contains @@ -994,14 +992,12 @@ def convert_capture( "task_description": session.task_description, } if window_capture is not None: - # Additive provenance + replay hints (the compiler ignores unknown - # meta.json keys; a non-window session's meta is unchanged). The - # hints carry the recorded TARGET owner/title substrings — the - # user's proven-to-resolve intent, stabler across replays than the - # resolved window's live title — in BackendConfig terms - # (rdp_window / rdp_window_title), so `replay --backend rdp` can - # resolve the same client window. Resolved values are kept as - # provenance and used only when the target carried neither field. + # Additive window provenance (the compiler ignores unknown + # meta.json keys; a non-window session's meta is unchanged). + # Do not infer an RDP backend from window scope. Native Windows and + # macOS recordings use the same Capture primitive. The desktop + # orchestration layer adds exact replay hints only for an explicit + # RDP or Citrix recording. assert window_selectors is not None owner, title, resolved_owner, resolved_title = window_selectors meta["window_capture"] = { @@ -1021,12 +1017,6 @@ def convert_capture( meta["window_capture"]["resolved_pid"] = resolved_pid if resolved_window_id is not None: meta["window_capture"]["resolved_window_id"] = resolved_window_id - hints: dict[str, Any] = {"backend": "rdp"} - if owner: - hints["rdp_window"] = owner - if title: - hints["rdp_window_title"] = title - meta["backend_hints"] = hints (out_dir / "meta.json").write_text(json.dumps(meta, indent=2)) return out_dir finally: diff --git a/public-artifacts.json b/public-artifacts.json index 9fe9bb54..383afae7 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -89,7 +89,7 @@ }, { "path": ".github/workflows/ci.yml", - "sha256": "34f47a2752dda1e3502e90d32f443f07d27c5452af8e89e78931b507323e9bf4" + "sha256": "6ababed7e1d90cbc63b681011712ddf5c335429044508836bdd47a113d7331f8" }, { "path": ".github/workflows/citrix-workspace-standin.yml", @@ -109,7 +109,7 @@ }, { "path": ".github/workflows/quickstart-lifecycle.yml", - "sha256": "01c504f7038b156cd2747fd8f5de385492633a42f61a62923406903ec7758dda" + "sha256": "34edc8580648b5aeeb3cbd1860681567694a6b5c9ea337b6bb504509ea644b07" }, { "path": ".github/workflows/release-health.yml", @@ -601,7 +601,7 @@ }, { "path": "claims.yaml", - "sha256": "0663daa2a910cf9844d094c8e62f5cfd85defa6a7787ba5d209c1a63bd054e65" + "sha256": "bf33b2ef84654c8563937828e38e389e065efcea5ec2654d4888423c1ede0142" }, { "path": "deploy/on-prem/docker-compose.yml", @@ -1809,7 +1809,7 @@ }, { "path": "docs/verification.json", - "sha256": "2a561c44799c49066619ef422a4384ef7cf2b9b80153c8bd70e743a7adaa92b1" + "sha256": "b27d6f6a848d1dd75149155bc5f33dc4b797634def65da5ffe4f62fe0053f44e" }, { "path": "openadapt_flow/console/static/console.css", diff --git a/scripts/quickstart_lifecycle.py b/scripts/quickstart_lifecycle.py index 32e61930..88adcf25 100644 --- a/scripts/quickstart_lifecycle.py +++ b/scripts/quickstart_lifecycle.py @@ -11,6 +11,7 @@ import argparse import glob +import hashlib import json import os import subprocess @@ -19,6 +20,13 @@ from pathlib import Path from typing import Sequence +_UNHANDLED_RUNTIME_MARKERS = ( + "Task was destroyed but it is pending!", + "Task exception was never retrieved", + "Future exception was never retrieved", + "TargetClosedError", +) + def _run( command: Sequence[str], @@ -50,6 +58,13 @@ def _run( print(result.stdout, end="", flush=True) log.parent.mkdir(parents=True, exist_ok=True) log.write_text(f"$ {printable}\n\n{result.stdout}", encoding="utf-8") + marker = next( + (item for item in _UNHANDLED_RUNTIME_MARKERS if item in result.stdout), None + ) + if marker is not None: + raise RuntimeError( + f"{printable} emitted an unhandled runtime error ({marker}); see {log}" + ) if result.returncode != expected: raise RuntimeError( f"{printable} exited {result.returncode}; expected {expected} (see {log})" @@ -231,6 +246,7 @@ def run_lifecycle( *, install_browser: bool, browser_with_deps: bool, + source_revision: str | None = None, ) -> dict[str, object]: """Run install through uninstall, returning the evidence summary.""" if work_dir.exists(): @@ -255,7 +271,9 @@ def run_lifecycle( installed = False summary: dict[str, object] = { "wheel": wheel.name, + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), "platform": sys.platform, + "source_revision": source_revision or "local-unbound", } try: @@ -282,10 +300,13 @@ def run_lifecycle( log=logs / "02-cli-help.log", ) - if install_browser: + # Linux needs host libraries that the ordinary unprivileged first-run + # download cannot install. Pre-provision them only in that lane. The + # macOS and Windows lanes leave Chromium absent here so the first Flow + # command proves the public lazy auto-install contract. + if browser_with_deps: browser_command = [str(python), "-m", "playwright", "install"] - if browser_with_deps: - browser_command.append("--with-deps") + browser_command.append("--with-deps") browser_command.append("chromium") _run( browser_command, @@ -422,12 +443,23 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--install-browser", action="store_true", - help="Install Playwright Chromium before running the lifecycle", + help=( + "Install the wheel's browser extra; Chromium remains lazy unless " + "--browser-with-deps pre-provisions it" + ), ) parser.add_argument( "--browser-with-deps", action="store_true", - help="Also install Linux browser system dependencies", + help=( + "Pre-provision Chromium and its Linux host dependencies; without " + "this flag the first Flow command must auto-install Chromium" + ), + ) + parser.add_argument( + "--source-revision", + default=None, + help="Exact source revision that produced the supplied wheel", ) return parser @@ -442,6 +474,7 @@ def main(argv: Sequence[str] | None = None) -> int: Path(args.work_dir).resolve(), install_browser=args.install_browser, browser_with_deps=args.browser_with_deps, + source_revision=args.source_revision, ) return 0 diff --git a/scripts/validate_claims.py b/scripts/validate_claims.py index 5cbf2acb..97a69f4b 100644 --- a/scripts/validate_claims.py +++ b/scripts/validate_claims.py @@ -270,7 +270,7 @@ def _classify_evidence( else: strength = STRENGTH_CI stage = ( - "post-merge/nightly full suite" + "required PR gate (e2e-browser)" if path.startswith("tests/e2e/") else "required PR gate (test)" ) diff --git a/tests/test_browser_setup.py b/tests/test_browser_setup.py index 1ca867cd..5504b8e7 100644 --- a/tests/test_browser_setup.py +++ b/tests/test_browser_setup.py @@ -55,6 +55,60 @@ def test_noop_when_browser_present(monkeypatch): assert calls == [] +def test_presence_probe_uses_non_actuating_cli_and_completion_markers( + monkeypatch, tmp_path +): + """The readiness probe must not start a Playwright driver connection.""" + chromium = tmp_path / "chromium-1234" + ffmpeg = tmp_path / "ffmpeg-1011" + shell = tmp_path / "chromium_headless_shell-1234" + for location in (chromium, ffmpeg, shell): + location.mkdir() + (location / "INSTALLATION_COMPLETE").write_text("", encoding="utf-8") + + calls = [] + + def fake_run(cmd, **kwargs): + calls.append((cmd, kwargs)) + return subprocess.CompletedProcess( + cmd, + 0, + stdout=( + f"Install location: {chromium}\n" + f" Install location: {ffmpeg}\n" + f"Install location: {shell}\n" + ), + ) + + monkeypatch.setattr(subprocess, "run", fake_run) + + assert bs._chromium_present() is True + assert calls[0][0][1:] == [ + "-m", + "playwright", + "install", + "--dry-run", + "chromium", + ] + assert calls[0][1]["stdout"] is subprocess.PIPE + assert calls[0][1]["stderr"] is subprocess.STDOUT + + +def test_presence_probe_refuses_a_partial_install(monkeypatch, tmp_path): + chromium = tmp_path / "chromium-1234" + chromium.mkdir() + + monkeypatch.setattr( + subprocess, + "run", + lambda cmd, **kwargs: subprocess.CompletedProcess( + cmd, 0, stdout=f"Install location: {chromium}\n" + ), + ) + + assert bs._chromium_present() is False + + def test_missing_browser_extra_refuses_before_network_or_subprocess(monkeypatch): """A non-browser base install gets one exact install action, not an import trace.""" monkeypatch.setattr(bs, "browser_support_installed", lambda: False) diff --git a/tests/test_capture_adapter.py b/tests/test_capture_adapter.py index 700a057b..a44fd6ac 100644 --- a/tests/test_capture_adapter.py +++ b/tests/test_capture_adapter.py @@ -25,6 +25,7 @@ import importlib.util import io import json +import shutil import zipfile from pathlib import Path from types import SimpleNamespace @@ -755,8 +756,10 @@ def test_window_mode_frames_taken_as_is(window_converted: Path) -> None: assert (window_converted / "frames" / "0000_before.png").is_file() -def test_window_mode_meta_stamps_backend_hints(window_converted: Path) -> None: - """meta.json carries the scoping provenance + rdp replay hints.""" +def test_window_mode_meta_stamps_provenance_without_guessing_surface( + window_converted: Path, +) -> None: + """Window scope alone does not imply that the target is RDP.""" import json meta = json.loads((window_converted / "meta.json").read_text()) @@ -771,9 +774,31 @@ def test_window_mode_meta_stamps_backend_hints(window_converted: Path) -> None: "resolved_pid": 4242, "resolved_window_id": 42, } - # The recorded TARGET had owner only (title=None): hints carry exactly the - # user's proven-to-resolve substrings, not the volatile resolved title. - assert meta["backend_hints"] == {"backend": "rdp", "rdp_window": WINDOW_OWNER} + assert "backend_hints" not in meta + + +@pytest.mark.parametrize("surface", ["windows", "macos"]) +def test_native_window_recording_compiles_with_its_exact_surface( + window_converted: Path, tmp_path: Path, surface: str +) -> None: + """The native CLI surface stamp cannot conflict with remote hints.""" + from openadapt_flow.compiler import compile_recording + + recording = tmp_path / f"{surface}-window-recording" + shutil.copytree(window_converted, recording) + meta_path = recording / "meta.json" + meta = json.loads(meta_path.read_text()) + meta["surface"] = surface + meta_path.write_text(json.dumps(meta, indent=2)) + + workflow = compile_recording( + recording, + tmp_path / f"{surface}-window-bundle", + name=f"{surface}-window", + ) + + assert workflow.surface == surface + assert workflow.backend_hints is None def test_window_mode_identity_omitted_when_absent(tmp_path: Path) -> None: diff --git a/tests/test_desktop_record.py b/tests/test_desktop_record.py index 5b86e9f9..ed9f7f5e 100644 --- a/tests/test_desktop_record.py +++ b/tests/test_desktop_record.py @@ -523,6 +523,150 @@ def test_cli_record_web_requires_url(tmp_path: Path) -> None: _run_cli(["record", "--out", str(tmp_path / "r")]) +@pytest.mark.parametrize( + ("backend", "flags", "message"), + [ + ( + "windows", + ["--agent-url", "http://localhost:5001"], + "local Capture session cannot bind to a WAA endpoint", + ), + ( + "linux", + ["--linux-app", "gedit"], + "no Linux window-scoping primitive", + ), + ( + "linux", + ["--linux-window-title", "Untitled Document 1"], + "no Linux window-scoping primitive", + ), + ( + "linux", + ["--linux-allow-physical-input"], + "no Linux window-scoping primitive", + ), + ( + "rdp", + ["--rdp-host", "10.0.0.5"], + "Capture cannot connect to a network RDP endpoint", + ), + ( + "windows", + ["--macos-app", "TextEdit"], + "does not apply to the windows recorder", + ), + ( + "web", + ["--url", "https://example.test", "--macos-app", "TextEdit"], + "does not apply to the web recorder", + ), + ( + "windows", + ["--url", "https://example.test"], + "--url applies only to --backend web", + ), + ( + "windows", + ["--headless"], + "--headless applies only to --backend web", + ), + ], +) +def test_cli_record_refuses_unapplied_flags_before_capture( + tmp_path: Path, + monkeypatch, + backend: str, + flags: list[str], + message: str, +) -> None: + """A record target flag must affect Capture/metadata or stop up front.""" + capture_started = False + + def fail_if_capture_starts(*args, **kwargs): + nonlocal capture_started + capture_started = True + raise AssertionError("Capture must not start") + + monkeypatch.setattr( + "openadapt_flow.desktop_record.record_desktop_capture", + fail_if_capture_starts, + ) + out = tmp_path / "rec" + with pytest.raises(SystemExit, match=message): + _run_cli(["record", "--backend", backend, "--out", str(out), *flags]) + assert capture_started is False + assert not out.exists() + + +def test_cli_record_macos_target_scopes_capture(tmp_path: Path, monkeypatch) -> None: + """macOS target flags bind the exact local Capture window.""" + captured: dict = {} + monkeypatch.setattr( + "openadapt_flow.desktop_record.record_desktop_capture", + _fake_desktop_record(captured), + ) + rc = _run_cli( + [ + "record", + "--backend", + "macos", + "--macos-app", + "TextEdit", + "--macos-window-title", + "notes.txt", + "--out", + str(tmp_path / "rec"), + ] + ) + assert rc == 0 + assert captured["window"] == {"owner": "TextEdit", "title": "notes.txt"} + + +@pytest.mark.parametrize( + ("backend", "generic_flags", "target_flags"), + [ + ("macos", ["--window", "Notes"], ["--macos-app", "TextEdit"]), + ( + "macos", + ["--window-title", "notes.txt"], + ["--macos-window-title", "draft.txt"], + ), + ], +) +def test_cli_record_refuses_conflicting_capture_targets_before_capture( + tmp_path: Path, + monkeypatch, + backend: str, + generic_flags: list[str], + target_flags: list[str], +) -> None: + capture_started = False + + def fail_if_capture_starts(*args, **kwargs): + nonlocal capture_started + capture_started = True + raise AssertionError("Capture must not start") + + monkeypatch.setattr( + "openadapt_flow.desktop_record.record_desktop_capture", + fail_if_capture_starts, + ) + with pytest.raises(SystemExit, match="name different capture targets"): + _run_cli( + [ + "record", + "--backend", + backend, + "--out", + str(tmp_path / "rec"), + *generic_flags, + *target_flags, + ] + ) + assert capture_started is False + + # -- Window-scoped recording (--window) -------------------------------------- diff --git a/tests/test_quickstart_lifecycle.py b/tests/test_quickstart_lifecycle.py index ed15545e..9dfdcd21 100644 --- a/tests/test_quickstart_lifecycle.py +++ b/tests/test_quickstart_lifecycle.py @@ -107,6 +107,37 @@ def fake_run(command, **kwargs): assert log.read_bytes().decode("utf-8").endswith("✓ UTF-8\n") +@pytest.mark.parametrize( + "marker", + [ + "Task was destroyed but it is pending!", + "Task exception was never retrieved", + "Future exception was never retrieved", + "TargetClosedError: Target page, context or browser has been closed", + ], +) +def test_run_rejects_unhandled_runtime_errors_on_a_zero_exit( + tmp_path, monkeypatch, marker +): + lifecycle = _module() + + monkeypatch.setattr( + lifecycle.subprocess, + "run", + lambda command, **kwargs: subprocess.CompletedProcess( + command, 0, stdout=f"VERIFIED\n{marker}\n" + ), + ) + + with pytest.raises(RuntimeError, match="unhandled runtime error"): + lifecycle._run( + ["openadapt-flow", "tutorial"], + cwd=tmp_path, + env={}, + log=tmp_path / "tutorial.log", + ) + + def test_inspect_artifacts_requires_reports_repairs_and_healed_bundle(tmp_path): lifecycle = _module() artifacts = tmp_path / "artifacts" diff --git a/tests/test_validate_claims.py b/tests/test_validate_claims.py index 0ce73c2e..94817aa0 100644 --- a/tests/test_validate_claims.py +++ b/tests/test_validate_claims.py @@ -131,6 +131,22 @@ def test_supported_with_ci_backing_passes() -> None: assert vc.validate_claim(_claim()).ok +def test_browser_e2e_evidence_names_its_required_pr_gate() -> None: + result = vc.validate_claim( + _claim( + evidence=[ + { + "path": "tests/e2e/test_record_compile_replay.py", + "proves": "x", + } + ] + ) + ) + + assert result.ok + assert result.evidence[0].gating == "ci (required PR gate (e2e-browser))" + + def test_field_result_cannot_be_supported() -> None: """A not-CI-reproducible field result may not be labeled `supported`, even with a real CI unit test attached."""