From c3bddcf9dc4cca2db62a64e5e99ee2c5c0f46cfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 17 Aug 2026 18:21:13 +0200 Subject: [PATCH 1/4] feat(web): add hover command for hover-gated UI (#1783) Add a first-class `hover [--settle]` verb, admitted on web only, that moves the pointer without pressing via the agent-browser backend (mouse move). It rides the existing targeted-touch pipeline (ref/selector/coordinate resolution, occlusion/off-screen guards, settle observation, response builder, recording) through a new optional Interactor/backend `hover` op that only the web provider implements. Touch platforms have no hover state: capabilities advertise it on web only and iOS/Android/Linux reject it at admission with a --platform web hint; longpress stays the mobile hold-gesture verb. Closes #1783 --- CHANGELOG.md | 1 + .../ad-script/src/internal/script-utils.ts | 6 +- packages/ad-script/src/internal/script.ts | 7 +- packages/contracts/src/client-gesture.ts | 5 ++ packages/contracts/src/facades/client.ts | 1 + packages/contracts/src/facades/interaction.ts | 2 + .../contracts/src/interaction-guarantees.ts | 6 +- packages/contracts/src/interaction.ts | 20 +++++ packages/contracts/src/interactor-types.ts | 6 ++ skills/agent-device/SKILL.md | 2 +- .../test-utils/property-arbitraries.ts | 1 + src/agent-device-client.ts | 1 + src/backend.ts | 1 + src/cli/parser/cli-help-overview.ts | 3 +- src/cli/parser/cli-help.ts | 4 +- src/client/client-types.ts | 2 + src/commands/interaction/index.ts | 37 +++++++++ src/commands/interaction/interactions.ts | 9 +++ src/commands/interaction/metadata.ts | 10 +++ src/commands/interaction/output.ts | 1 + src/commands/interaction/runtime/gestures.ts | 43 ++++++++++ src/commands/interaction/runtime/index.ts | 10 +++ .../interaction/runtime/interactions.ts | 4 +- .../interaction/runtime/resolution.ts | 3 + src/core/__tests__/capabilities.test.ts | 19 +++++ src/core/__tests__/web-interactor.test.ts | 16 ++++ src/core/capabilities.ts | 10 ++- .../__tests__/command-result.test.ts | 1 + .../__tests__/parity.test.ts | 1 + .../__tests__/post-action-observation.test.ts | 5 ++ .../__tests__/timeout-policy.test.ts | 1 + src/core/command-descriptor/command-result.ts | 2 + .../post-action-observation.ts | 3 + src/core/command-descriptor/registry.ts | 22 ++++++ src/core/dispatch-interactions.ts | 16 ++++ src/core/dispatch.ts | 2 + src/core/interactors/web.ts | 7 ++ src/daemon/__tests__/session-store.test.ts | 8 ++ .../__tests__/interaction-touch.test.ts | 78 ++++++++++++++++++- src/daemon/handlers/interaction-flags.ts | 2 +- src/daemon/handlers/interaction-runtime.ts | 12 +++ .../handlers/interaction-touch-payload.ts | 7 +- .../handlers/interaction-touch-policy.ts | 2 +- .../interaction-touch-press-admission.ts | 22 ++++-- .../handlers/interaction-touch-press.ts | 66 ++++++++++------ .../handlers/interaction-touch-response.ts | 9 ++- src/daemon/handlers/interaction-touch.ts | 2 + src/daemon/response-views.ts | 1 + src/daemon/session-event-action.ts | 4 + src/daemon/session-script-writer.ts | 4 +- src/mcp/command-output-schemas.ts | 6 ++ src/mcp/tool-ref-pins.ts | 1 + .../web/agent-browser-provider.test.ts | 3 + src/platforms/web/agent-browser-provider.ts | 13 +++- src/platforms/web/provider.ts | 2 + .../android-emulator-e2e/coverage-manifest.ts | 4 + .../ios-simulator-e2e/coverage-manifest.ts | 8 ++ website/docs/docs/commands.md | 6 +- 58 files changed, 500 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f418da12c2..ccf1f45521 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- New `hover ` command for `--platform web` (#1783). It moves the pointer over the target without pressing, so hover-gated UI — a message row's `...` toolbar, a menu that opens on pointer enter — becomes reachable through agent-device the way it already was through the underlying `agent-browser` backend (`mouse move`). It is a member of the targeted-touch family: same `@ref`/selector/coordinate targeting, occlusion and off-screen guards, and `--settle` (the settled diff carries the revealed controls with fresh refs, e.g. `+ @e4 [button] "Delete"`), but no `--verify`, since hover reveals rather than activates. `hover @ref` publishes as a portable selector line in recorded scripts, and the Node client exposes `interactions.hover`. Hover is a pointer state that touch platforms do not have, so `capabilities` advertises it on web only and iOS/Android/Linux reject it during admission with `UNSUPPORTED_OPERATION` and a hint naming `--platform web`; `longpress` remains the mobile hold-gesture verb. - Internal: session recording is now derived from the script-publication lifecycle instead of being stored beside it. `SessionState.recordSession` is removed; `isRecordingPublication` answers the question from the aggregate (ordinary authoring records only while ARMED, a repair transaction records for its whole lifetime). The stored flag was a second source of truth that handler surfaces set directly, which is how #1533's aborted-recording drift arose — no surface can now arm recording without moving the lifecycle that authorizes it, and the script writer's publication gate is answered entirely by the aggregate. Behavior-preserving: the derivation reproduces what the flag held at every transition. - Fixed: a script recording aborted by a second `open` is no longer published by a later bare `close` (#1533). `open --save-script` followed by a second successful `open` terminates the recording and warns "Script publication was aborted…", and `close --save-script` correctly refuses it with "Retry with plain close; it will tear down the session without writing." But when that second `open` itself carried `--save-script`, the flag re-armed recording behind the terminal status, and a bare `close` then wrote the full session log to disk — publishing a recording the caller had been told was aborted, and breaking the promise the refusal makes. An aborted authoring lifecycle is now terminal by construction: `--save-script` arms nothing on any surface that handles it — the re-open builder, the close finalizer, and the recorded-action ingress — and the script writer refuses the lifecycle from every path that reaches it (bare `close`, teardown, idle-reap, active publication). This also stops an aborted session from paying recording-time costs it can never publish: a re-opened aborted recording no longer keeps the direct iOS selector fast paths for `click` and `get` disabled. Armed recordings, published recordings, and every repair transaction are unaffected. - `agent-device mcp` now serves the stateless MCP `2026-07-28` revision alongside the handshake-based revisions it already spoke, as the spec's "dual-era server". Modern clients probe `server/discover`, which advertises the supported revisions, the tools capability, and server identity; their requests declare a protocol version in `_meta`, and their results carry `resultType: "complete"` plus `_meta["io.modelcontextprotocol/serverInfo"]`. `tools/list` and `server/discover` now return the `ttlMs`/`cacheScope` cache hints, so a client can cache the 55-tool, ~223KB tool list for an hour instead of re-fetching it on every start; the list was already emitted in a deterministic (sorted) order, which is the other half of what makes it cacheable. Each revision is answered on its own wire contract: a request declaring `2025-11-25` or `2025-06-18` through modern framing still gets the legacy result shape, and `initialize` never agrees to `2026-07-28`, which has no handshake to establish. A declared revision this server does not implement is rejected with `UnsupportedProtocolVersionError` (`-32022`) naming the ones it does, rather than being served under a version the client did not ask for, and modern framing that omits its required `protocolVersion`/`clientCapabilities` metadata — or supplies a `clientInfo` that is not a valid `Implementation` — is rejected as invalid params. `initialize` and `ping` were removed in `2026-07-28`, so a modern-framed call to either is answered `-32601` rather than served inside a `resultType: "complete"` envelope. Responses to legacy clients are unchanged byte-for-byte — `initialize` and `ping` are still served, and no cache, `resultType`, or `_meta` field is added to their results. Nothing here affects the CLI, Node, or daemon surfaces: the stdio transport, the tool set, and every tool's input/output schema are untouched. diff --git a/packages/ad-script/src/internal/script-utils.ts b/packages/ad-script/src/internal/script-utils.ts index 666ca5ec7d..edd755b58d 100644 --- a/packages/ad-script/src/internal/script-utils.ts +++ b/packages/ad-script/src/internal/script-utils.ts @@ -44,8 +44,10 @@ export function isClickLikeCommand(command: string): command is 'click' | 'press return command === 'click' || command === 'press'; } -export function isTouchTargetCommand(command: string): command is 'click' | 'press' | 'longpress' { - return isClickLikeCommand(command) || command === 'longpress'; +export function isTouchTargetCommand( + command: string, +): command is 'click' | 'press' | 'longpress' | 'hover' { + return isClickLikeCommand(command) || command === 'longpress' || command === 'hover'; } function isTypingCommand(command: string): command is 'type' | 'fill' { diff --git a/packages/ad-script/src/internal/script.ts b/packages/ad-script/src/internal/script.ts index 41ea7bff9d..4c6e74af9d 100644 --- a/packages/ad-script/src/internal/script.ts +++ b/packages/ad-script/src/internal/script.ts @@ -463,10 +463,11 @@ function parseReplayScriptLine(line: string): SessionAction | null { return action; } - // wait @ref [timeout] and longpress @ref [durationMs] flow through this - // generic branch: strip recorded generation pins like the branches above. + // wait @ref [timeout], longpress @ref [durationMs], and hover @ref flow + // through this generic branch: strip recorded generation pins like the + // branches above. action.positionals = - command === 'wait' || command === 'longpress' + command === 'wait' || command === 'longpress' || command === 'hover' ? args.map((token) => stripRecordedRefGeneration(token)) : args; return action; diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index faa3d5ca35..48a796ff42 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -64,6 +64,11 @@ export type LongPressOptions = DeviceCommandBaseOptions & durationMs?: number; }; +export type HoverOptions = DeviceCommandBaseOptions & + SelectorSnapshotCommandOptions & + InteractionTarget & + SettleCommandOptions; + export type SwipeOptions = DeviceCommandBaseOptions & { from: { x: number; y: number }; to: { x: number; y: number }; diff --git a/packages/contracts/src/facades/client.ts b/packages/contracts/src/facades/client.ts index ba8b99a466..c4fa838537 100644 --- a/packages/contracts/src/facades/client.ts +++ b/packages/contracts/src/facades/client.ts @@ -47,6 +47,7 @@ export type { FillOptions, FlingOptions, FocusOptions, + HoverOptions, LongPressOptions, PanOptions, PinchOptions, diff --git a/packages/contracts/src/facades/interaction.ts b/packages/contracts/src/facades/interaction.ts index f837902e80..44273de357 100644 --- a/packages/contracts/src/facades/interaction.ts +++ b/packages/contracts/src/facades/interaction.ts @@ -85,6 +85,8 @@ export type { FillCommandResponseData, FillCommandResult, FindCommandResponseData, + HoverCommandResponseData, + HoverCommandResult, InteractionEvidence, InteractionTarget, LongPressCommandResponseData, diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 5ed5ce4cd0..b0b41ef2d6 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -199,7 +199,7 @@ const RUNTIME_TREE_SHARED_GUARANTEES = { export const INTERACTION_DISPATCH_PATHS: Record = { 'runtime-selector': { description: 'Daemon tree capture, selector chain resolution, guarded coordinate tap.', - commands: ['press', 'click', 'fill', 'longpress'], + commands: ['press', 'click', 'fill', 'longpress', 'hover'], guarantees: { ...RUNTIME_TREE_SHARED_GUARANTEES, disambiguation: { @@ -221,7 +221,7 @@ export const INTERACTION_DISPATCH_PATHS: Record; + message?: string; + warning?: string; + settle?: SettleObservation; +}; + /** * Daemon response data for the `find` command. Read-only actions (`exists`, * `wait`, `get_text`, `get_attrs`) may issue a pinnable ref with diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index 69116edfec..2cade5fcbe 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -195,6 +195,12 @@ export type Interactor = { tapElementSelector?(selector: ElementSelectorTapOptions): Promise | void>; doubleTap(x: number, y: number): Promise | void>; longPress(x: number, y: number, durationMs?: number): Promise | void>; + /** + * Move the pointer to a point without pressing. Only pointer-driven + * platforms (web today) implement it; touch platforms have no hover state + * and leave it undefined, which the `hover` command reports as unsupported. + */ + hover?(x: number, y: number): Promise | void>; focus(x: number, y: number): Promise | void>; type(text: string, delayMs?: number): Promise; fillElementSelector?( diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index e8aafb0a48..18db7667bc 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -13,7 +13,7 @@ agent-device open --foreground That starts the session and returns the initial interactive snapshot with `@refs`. -Loop: act with `press|click|fill|longpress ... --settle`, `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation (`wait text "..."`, `is`, `get`, or `find`), then run `agent-device close`. +Loop: act with `press|click|fill|longpress ... --settle`, `hover --settle` (web only, reveals hover-gated UI), `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation (`wait text "..."`, `is`, `get`, or `find`), then run `agent-device close`. Copy refs byte-for-byte: `@e12`, `@e12~s4` — keep the `@` and any `~sN`. Prefer current refs, then `id`/`label`/`role` selectors; coordinates are a last resort. If snapshot reports sparse/AX-unavailable, its refs and selectors are invalid: run `agent-device screenshot`, inspect the image, use coordinates, then retry `snapshot -i` after navigating. Otherwise run `snapshot -i` only when the diff lacks the next target. diff --git a/src/__tests__/test-utils/property-arbitraries.ts b/src/__tests__/test-utils/property-arbitraries.ts index 97a662ee1c..0e283c5470 100644 --- a/src/__tests__/test-utils/property-arbitraries.ts +++ b/src/__tests__/test-utils/property-arbitraries.ts @@ -357,6 +357,7 @@ const REPLAY_SCRIPT_LINE_PLANS = { .map(([target, button]) => `press ${target} --button ${button}`), ), longpress: scriptTargetArb.map((target) => `longpress ${target}`), + hover: scriptTargetArb.map((target) => `hover ${target}`), wait: fc .tuple(scriptTargetArb, fc.integer({ min: 100, max: 5000 })) .map(([target, timeout]) => `wait ${target} ${timeout}`), diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index f79d35617b..49616f9cdb 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -404,6 +404,7 @@ export function createAgentDeviceClient( click: async (options) => await executeCommand('click', options), press: async (options) => await executeCommand('press', options), longPress: async (options) => await executeCommand('longpress', options), + hover: async (options) => await executeCommand('hover', options), swipe: async (options) => await executeCommand('swipe', options), pan: async (options) => await executeCommand('gesture', panGestureInput(options)), drag: async (options) => await executeCommand('gesture', dragGestureInput(options)), diff --git a/src/backend.ts b/src/backend.ts index 392a75f684..ea508feff1 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -499,6 +499,7 @@ export type AgentDeviceBackend = { point: Point, options?: BackendLongPressOptions, ): Promise; + hover?(context: BackendCommandContext, point: Point): Promise; scroll?( context: BackendCommandContext, target: BackendScrollTarget, diff --git a/src/cli/parser/cli-help-overview.ts b/src/cli/parser/cli-help-overview.ts index 67a4b777b4..59312fe788 100644 --- a/src/cli/parser/cli-help-overview.ts +++ b/src/cli/parser/cli-help-overview.ts @@ -21,6 +21,7 @@ Start: Loop: press|click|fill|longpress ... --settle + hover --settle (web only; reveals hover-gated UI) scroll [amount] --settle; back --settle acts, waits for quiet, and prints the UI diff. Continue from that diff. Run snapshot -i only when the diff lacks the next target or did not settle. @@ -37,7 +38,7 @@ Targets: Then screenshot, press , and re-snapshot on the changed screen. Rules: - --settle is only for press/click/fill/longpress/scroll/back; never open, + --settle is only for press/click/fill/longpress/hover/scroll/back; never open, snapshot, or close. type never accepts --settle. fill --settle replaces; type appends after focus. Late network/debounce result: wait text "Expected", not snapshot polling. diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 195b78de9e..9139ee6ce9 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -886,6 +886,7 @@ First-slice loop: agent-device is visible 'label="Welcome"' --platform web agent-device find text "Welcome" exists --platform web agent-device click @e12 --platform web + agent-device hover @e14 --settle --platform web agent-device fill @e13 "qa@example.com" --platform web agent-device wait text "Welcome" 3000 --platform web agent-device record start ./artifacts/web-flow.webm --platform web @@ -898,7 +899,8 @@ First-slice loop: agent-device close --platform web Supported in agent-device web sessions: - open , snapshot -i, get text/attrs, is visible/exists/text, find text/selector, click/press @ref or selector, fill/type @ref or selector, wait text/selector, network dump, audio probe, screenshot, record start/stop with WebM output, close, and replay scripts made from those commands. + open , snapshot -i, get text/attrs, is visible/exists/text, find text/selector, click/press @ref or selector, hover @ref or selector, fill/type @ref or selector, wait text/selector, network dump, audio probe, screenshot, record start/stop with WebM output, close, and replay scripts made from those commands. + hover moves the pointer without pressing so hover-gated UI (row toolbars, menus) appears; use --settle to read what it revealed, then act on the fresh refs. Web only: touch platforms have no hover state, so hover-gated flows there need a different entry point. Out of scope for agent-device web support: Browser runtime debugging, tabs/windows/devtools control, network routing/interception/HAR, storage/cookie management, arbitrary page scripting, downloads/uploads, multi-page orchestration, and agent-browser-specific diagnostics. Use agent-browser directly for those browser-specific workflows. diff --git a/src/client/client-types.ts b/src/client/client-types.ts index 08523b8f5f..49b9a120e5 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -92,6 +92,7 @@ import type { FlingOptions, FocusOptions, GetOptions, + HoverOptions, IsOptions, KeyboardCommandOptions, Lease, @@ -271,6 +272,7 @@ export type AgentDeviceClient = { click: (options: ClickOptions) => Promise>; press: (options: PressOptions) => Promise>; longPress: (options: LongPressOptions) => Promise>; + hover: (options: HoverOptions) => Promise>; swipe: (options: SwipeOptions) => Promise; pan: (options: PanOptions) => Promise; drag: (options: DragOptions) => Promise; diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 3e54f65e83..89473140eb 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -6,6 +6,7 @@ import type { FlingOptions, FocusOptions, GetOptions, + HoverOptions, IsOptions, LongPressOptions, PanOptions, @@ -39,6 +40,7 @@ import { type FillInput, type FlingInput, type GetInput, + type HoverInput, type LongPressInput, type PanInput, type PinchInput, @@ -95,6 +97,12 @@ const interactionCliSchemas = { allowsExtraPositionals: true, allowedFlags: [...postActionObservationCliFlags('longpress'), ...SELECTOR_SNAPSHOT_FLAGS], }, + hover: { + usageOverride: 'hover ', + positionalArgs: ['targetOrX', 'y?'], + allowsExtraPositionals: true, + allowedFlags: [...postActionObservationCliFlags('hover'), ...SELECTOR_SNAPSHOT_FLAGS], + }, swipe: { positionalArgs: ['x1', 'y1', 'x2', 'y2'], // Arity is enforced by swipePayloadFromPositionals (assertGestureArity), so @@ -155,6 +163,10 @@ const longPressCommandDefinition = defineExecutableCommand(metadata('longpress') client.interactions.longPress(toLongPressOptions(input)), ); +const hoverCommandDefinition = defineExecutableCommand(metadata('hover'), (client, input) => + client.interactions.hover(toHoverOptions(input)), +); + const swipeCommandDefinition = defineExecutableCommand(metadata('swipe'), (client, input) => client.interactions.swipe(input as SwipeOptions), ); @@ -259,6 +271,21 @@ const longPressCommandFacet = defineCommandFacet({ cliOutputFormatter: interactionCliOutputFormatters.longpress, }); +const hoverCommandFacet = defineCommandFacet({ + name: 'hover', + text: { + summary: 'Hover the pointer over a UI target (web only)', + cliDetail: + 'The pointer stays where hover left it: read the revealed UI (--settle or snapshot -i) and act on it before another click or hover moves the pointer away.', + }, + metadata: metadata('hover'), + definition: hoverCommandDefinition, + cliSchema: interactionCliSchemas.hover, + cliReader: interactionCliReaders.hover, + daemonWriter: interactionDaemonWriters.hover, + cliOutputFormatter: interactionCliOutputFormatters.hover, +}); + const swipeCommandFacet = defineCommandFacet({ name: 'swipe', text: { @@ -369,6 +396,7 @@ export const interactionCommandFamily = defineCommandFamilyFromFacets({ pressCommandFacet, fillCommandFacet, longPressCommandFacet, + hoverCommandFacet, swipeCommandFacet, focusCommandFacet, typeCommandFacet, @@ -434,6 +462,15 @@ function toLongPressOptions(input: LongPressInput): LongPressOptions { }; } +function toHoverOptions(input: HoverInput): HoverOptions { + return { + ...commonToClientOptions(input), + ...toClientInteractionTarget(input.target), + ...toSelectorSnapshotOptions(input), + ...toSettleOptions(input), + }; +} + function toSettleOptions(input: { settle?: boolean; settleQuietMs?: number; diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index 110eb85fff..c4067501cf 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -64,6 +64,12 @@ export const interactionCliReaders = { durationMs: decoded.durationMs, }; }, + hover: (positionals, flags) => ({ + ...commonInputFromFlags(flags), + ...selectorSnapshotInputFromFlags(flags), + ...settleInputFromFlags(flags), + target: targetInputFromClientTarget(readInteractionTargetFromPositionals(positionals)), + }), swipe: (positionals, flags) => ({ ...commonInputFromFlags(flags), ...swipePayloadFromPositionals(positionals, { @@ -128,6 +134,9 @@ export const interactionDaemonWriters = { longpress: direct(PUBLIC_COMMANDS.longPress, (input) => longPressPositionals(input as LongPressOptions), ), + hover: direct(PUBLIC_COMMANDS.hover, (input) => + interactionTargetPositionals(input as InteractionTarget), + ), swipe: (input) => { assertNoRemovedSwipeInput(input); return request(PUBLIC_COMMANDS.swipe, [], input, { diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index d732f3025e..49e66664bb 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -62,6 +62,8 @@ const interactionCommandDescriptions = { fill: 'Replace text in a UI input selected by snapshot ref, selector, or coordinates. Prefer refs or selectors after snapshot; use recordAs to keep sensitive text out of a recorded replay while sending it to the live app.', longpress: 'Hold a UI target by snapshot ref, selector, or coordinates to open a context menu or perform another hold gesture. Set durationMs when the default hold duration is unsuitable.', + hover: + 'Move the pointer over a UI target by snapshot ref, selector, or coordinates without pressing, to reveal hover-gated UI such as row toolbars or menus. Web only; touch platforms have no hover state. Use settle to observe what the hover revealed without a follow-up snapshot.', swipe: 'Quick coordinate fling with optional repeat pattern.', focus: 'Move input focus to explicit screen coordinates without entering text. Prefer semantic interactions when a snapshot ref or selector is available; use type or fill after focus.', @@ -109,6 +111,12 @@ const longPressFields = { ...postActionObservationFields('longpress'), }; +const hoverFields = { + target: requiredField(interactionTargetField()), + ...selectorSnapshotFields(), + ...postActionObservationFields('hover'), +}; + const swipeFields = { from: requiredField(pointField('Swipe start point.')), to: requiredField(pointField('Swipe end point.')), @@ -207,6 +215,7 @@ export type ClickInput = InferCommandInput; export type PressInput = InferCommandInput; export type FillInput = InferCommandInput; export type LongPressInput = InferCommandInput; +export type HoverInput = InferCommandInput; export type GetInput = InferCommandInput; export type PanInput = CommonCommandInput & PanGesturePayload; @@ -246,6 +255,7 @@ export const interactionCommandMetadata = [ readInput: (input) => readFieldInput(input, fillFields), }), defineInteractionCommandMetadata('longpress', longPressFields), + defineInteractionCommandMetadata('hover', hoverFields), defineInteractionCommandMetadata('swipe', swipeFields), defineInteractionCommandMetadata('focus', focusFields), defineInteractionCommandMetadata('type', typeFields), diff --git a/src/commands/interaction/output.ts b/src/commands/interaction/output.ts index d8ec940954..5240fbd818 100644 --- a/src/commands/interaction/output.ts +++ b/src/commands/interaction/output.ts @@ -84,6 +84,7 @@ export const interactionCliOutputFormatters = { press: resultOutput(tapCliOutput), fill: messageWithSettleOutput, longpress: messageWithSettleOutput, + hover: messageWithSettleOutput, scroll: messageWithSettleOutput, get: ({ input, result }) => getCliOutput({ diff --git a/src/commands/interaction/runtime/gestures.ts b/src/commands/interaction/runtime/gestures.ts index 4656be7486..1a9d1968ea 100644 --- a/src/commands/interaction/runtime/gestures.ts +++ b/src/commands/interaction/runtime/gestures.ts @@ -1,5 +1,6 @@ import type { DragGestureInput, + HoverCommandResult, LongPressCommandResult, ResolutionDisclosure, ScrollDirection, @@ -113,6 +114,14 @@ export type LongPressCommandOptions = CommandContext & { export type { LongPressCommandResult }; +export type HoverCommandOptions = CommandContext & { + target: InteractionTarget; + /** ADR 0012 step 4: replay-only post-resolution guard; see resolution.ts. */ + expectedResolvedTarget?: ExpectedResolvedTarget; +} & SettlePostActionObservationOptions; + +export type { HoverCommandResult }; + export type GestureDirection = ScrollDirection; // The input vocabulary lives in contracts/scroll-gesture.ts beside the other scroll vocabularies, // so the public API can declare `ScrollOptions` without depending on this command runtime. @@ -218,6 +227,40 @@ export const longPressCommand: RuntimeCommand< ); }; +export const hoverCommand: RuntimeCommand = async ( + runtime, + options, +): Promise => { + const observation = planPostActionObservation(options); + // Hover keeps the element it resolved (no hittable-ancestor promotion): the + // pointer only has to enter the matched node's box for its hover state to + // raise, and promoting could move it onto a sibling-owned region. + const resolved = await resolveInteractionTarget(runtime, options, { + action: 'hover', + requireInteractive: false, + pipeline: SELECTOR_PIPELINE_POLICIES.resolvedTarget, + captureEvidenceBaseline: observation.needsPreActionBaseline, + expectedResolvedTarget: options.expectedResolvedTarget, + }); + if (!runtime.backend.hover) { + throw new AppError('UNSUPPORTED_OPERATION', 'hover is not supported by this backend'); + } + const point = requireResolvedPoint(resolved); + const backendResult = await runtime.backend.hover(toBackendContext(runtime, options), point); + const formattedBackendResult = toBackendResult(backendResult); + return await applyPostActionObservation( + runtime, + options, + resolved, + { + ...resolved, + ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + ...successText(`Hovered (${point.x}, ${point.y})`), + }, + observation, + ); +}; + export const dragCommand: RuntimeCommand = async ( runtime, options, diff --git a/src/commands/interaction/runtime/index.ts b/src/commands/interaction/runtime/index.ts index 7259c3317c..969ba80039 100644 --- a/src/commands/interaction/runtime/index.ts +++ b/src/commands/interaction/runtime/index.ts @@ -4,6 +4,7 @@ import { clickCommand, fillCommand, focusCommand, + hoverCommand, longPressCommand, pressCommand, scrollCommand, @@ -13,6 +14,8 @@ import { type FillCommandResult, type FocusCommandOptions, type FocusCommandResult, + type HoverCommandOptions, + type HoverCommandResult, type InteractionTarget, type LongPressCommandOptions, type LongPressCommandResult, @@ -78,6 +81,7 @@ export type InteractionCommands = { typeText: RuntimeCommand; focus: RuntimeCommand; longPress: RuntimeCommand; + hover: RuntimeCommand; scroll: RuntimeCommand; gesture: RuntimeCommand; /** @@ -142,6 +146,10 @@ export type BoundInteractionCommands = { target: InteractionTarget, options?: Omit, ) => Promise; + hover: ( + target: InteractionTarget, + options?: Omit, + ) => Promise; scroll: BoundRuntimeCommand; gesture: BoundRuntimeCommand; settleObservation: BoundRuntimeCommand; @@ -166,6 +174,7 @@ export const interactionCommands: InteractionCommands = { typeText: typeTextCommand, focus: focusCommand, longPress: longPressCommand, + hover: hoverCommand, scroll: scrollCommand, gesture: gestureCommand, settleObservation: settleObservationCommand, @@ -197,6 +206,7 @@ export function bindInteractionCommands(runtime: AgentDeviceRuntime): BoundInter focus: (target, options = {}) => interactionCommands.focus(runtime, { ...options, target }), longPress: (target, options = {}) => interactionCommands.longPress(runtime, { ...options, target }), + hover: (target, options = {}) => interactionCommands.hover(runtime, { ...options, target }), scroll: (options) => interactionCommands.scroll(runtime, options), gesture: (options) => interactionCommands.gesture(runtime, options), settleObservation: (options) => interactionCommands.settleObservation(runtime, options), diff --git a/src/commands/interaction/runtime/interactions.ts b/src/commands/interaction/runtime/interactions.ts index a63df32294..dc31bd30ae 100644 --- a/src/commands/interaction/runtime/interactions.ts +++ b/src/commands/interaction/runtime/interactions.ts @@ -34,10 +34,12 @@ import { type InteractionTarget, } from './resolution.ts'; -export { focusCommand, longPressCommand, scrollCommand } from './gestures.ts'; +export { focusCommand, hoverCommand, longPressCommand, scrollCommand } from './gestures.ts'; export type { FocusCommandOptions, FocusCommandResult, + HoverCommandOptions, + HoverCommandResult, LongPressCommandOptions, LongPressCommandResult, ScrollCommandOptions, diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index b6ac2e207a..b51ae21b90 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -125,6 +125,7 @@ export type InteractionAction = | 'fill' | 'focus' | 'longPress' + | 'hover' | 'scroll' | 'swipe' | 'pinch' @@ -657,6 +658,8 @@ function interactionVerb(action: InteractionAction): string { return 'be focused'; case 'longPress': return 'be long-pressed'; + case 'hover': + return 'be hovered'; default: return 'be tapped'; } diff --git a/src/core/__tests__/capabilities.test.ts b/src/core/__tests__/capabilities.test.ts index dfc7191947..a0db66915b 100644 --- a/src/core/__tests__/capabilities.test.ts +++ b/src/core/__tests__/capabilities.test.ts @@ -206,6 +206,24 @@ test('viewport resizing is admitted only on web, where a backend exists', () => assert.equal(unsupportedHintForDevice('viewport', webDevice), undefined); }); +// #1783: hover is a pointer state, so it is admitted exactly where a pointer +// exists (the web backend's mouse move) and denied on every touch platform. +test('hover is admitted only on web, where a pointer backend exists', () => { + assertCommandSupport( + ['hover'], + [ + { device: webDevice, expected: true, label: 'on web' }, + { device: iosSimulator, expected: false, label: 'on iOS simulator' }, + { device: iosDevice, expected: false, label: 'on iOS device' }, + { device: macOsDevice, expected: false, label: 'on macOS' }, + { device: tvOsSimulator, expected: false, label: 'on tvOS simulator' }, + { device: androidDevice, expected: false, label: 'on Android device' }, + { device: androidEmulator, expected: false, label: 'on Android emulator' }, + { device: linuxDevice, expected: false, label: 'on linux' }, + ], + ); +}); + test('capabilities reject CoreDevice-only commands for XCTest-backed devices', () => { // Runtime-backed logs and record admission are proven from exact device facts in // their handler/runtime tests, never through this legacy matrix projection. @@ -336,6 +354,7 @@ test('web supports only the initial browser interaction slice', () => { 'focus', 'find', 'get', + 'hover', 'is', 'press', 'record', diff --git a/src/core/__tests__/web-interactor.test.ts b/src/core/__tests__/web-interactor.test.ts index 268990eeb9..4811962df1 100644 --- a/src/core/__tests__/web-interactor.test.ts +++ b/src/core/__tests__/web-interactor.test.ts @@ -30,6 +30,9 @@ test('web interactor delegates first-slice operations to the scoped provider', a async click(x, y) { calls.push(`click:${x}:${y}`); }, + async hover(x, y) { + calls.push(`hover:${x}:${y}`); + }, async fill(x, y, text, options) { calls.push(`fill:${x}:${y}:${text}:${options?.delayMs ?? 0}`); }, @@ -46,6 +49,7 @@ test('web interactor delegates first-slice operations to the scoped provider', a await interactor.open('app-shell', { url: 'https://example.test/deep' }); await interactor.close('app-shell'); await interactor.tap(10, 20); + await interactor.hover?.(30, 40); await interactor.focus(11, 21); await interactor.fill(12, 22, 'hello', 5); await interactor.type('world', 6); @@ -60,6 +64,7 @@ test('web interactor delegates first-slice operations to the scoped provider', a 'open:https://example.test/deep:https://example.test/deep', 'close:app-shell', 'click:10:20', + 'hover:30:40', 'click:11:21', 'fill:12:22:hello:5', 'type:world:6', @@ -73,6 +78,17 @@ test('web interactor delegates first-slice operations to the scoped provider', a assert.deepEqual(snapshot.nodes, [{ index: 0, role: 'button', label: 'Submit' }]); }); +test('web interactor reports hover unsupported when the provider lacks it', async () => { + const interactor = createWebInteractor(); + await assert.rejects( + () => withWebProvider(makeWebProvider(), async () => await interactor.hover?.(1, 2)), + (error: unknown) => + error instanceof AppError && + error.code === 'UNSUPPORTED_OPERATION' && + error.message === 'hover is not supported by this web provider', + ); +}); + test('web interactor reports unsupported operations explicitly', async () => { const interactor = createWebInteractor(); diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index a2e6f50755..64e3100180 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -68,7 +68,15 @@ const WEB_QUERY_COMMANDS = [ 'snapshot', 'wait', ] as const; -const WEB_INTERACTION_COMMANDS = ['click', 'fill', 'focus', 'press', 'scroll', 'type'] as const; +const WEB_INTERACTION_COMMANDS = [ + 'click', + 'fill', + 'focus', + 'hover', + 'press', + 'scroll', + 'type', +] as const; const WEB_SETTING_COMMANDS = ['viewport'] as const; const WEB_SUPPORTED_COMMANDS = new Set([ ...WEB_QUERY_COMMANDS, diff --git a/src/core/command-descriptor/__tests__/command-result.test.ts b/src/core/command-descriptor/__tests__/command-result.test.ts index 0329565830..ba46158845 100644 --- a/src/core/command-descriptor/__tests__/command-result.test.ts +++ b/src/core/command-descriptor/__tests__/command-result.test.ts @@ -139,6 +139,7 @@ test('CommandResultMap is seeded only from already-existing contract result type | 'click' | 'fill' | 'longpress' + | 'hover' | 'find' | 'boot' | 'shutdown' diff --git a/src/core/command-descriptor/__tests__/parity.test.ts b/src/core/command-descriptor/__tests__/parity.test.ts index d769328845..111f3705a5 100644 --- a/src/core/command-descriptor/__tests__/parity.test.ts +++ b/src/core/command-descriptor/__tests__/parity.test.ts @@ -423,6 +423,7 @@ test('targetIdentityVerification pins exactly the evidence-carrying command set ['fill', 'pre-dispatch'], ['gesture', 'pre-dispatch'], ['get', 'pre-dispatch'], + ['hover', 'pre-dispatch'], ['is', 'pre-dispatch'], ['longpress', 'pre-dispatch'], ['press', 'pre-dispatch'], diff --git a/src/core/command-descriptor/__tests__/post-action-observation.test.ts b/src/core/command-descriptor/__tests__/post-action-observation.test.ts index 4cbed78eb6..b31eca7576 100644 --- a/src/core/command-descriptor/__tests__/post-action-observation.test.ts +++ b/src/core/command-descriptor/__tests__/post-action-observation.test.ts @@ -14,6 +14,7 @@ const SETTLE_OBSERVATION_COMMANDS = [ PUBLIC_COMMANDS.click, PUBLIC_COMMANDS.fill, PUBLIC_COMMANDS.longPress, + PUBLIC_COMMANDS.hover, PUBLIC_COMMANDS.press, PUBLIC_COMMANDS.scroll, PUBLIC_COMMANDS.back, @@ -33,6 +34,10 @@ test('post-action observation descriptor traits are the source for settle comman assert.equal(resolveCommandPostActionObservationSupport('fill'), 'settle-and-verify'); assert.equal(resolveCommandPostActionObservationSupport('longpress'), 'settle'); assert.equal(commandSupportsVerifyEvidence('longpress'), false); + // #1783: hover reveals UI instead of activating a target, so the settled + // diff is the observation and there is no --verify digest. + assert.equal(resolveCommandPostActionObservationSupport('hover'), 'settle'); + assert.equal(commandSupportsVerifyEvidence('hover'), false); // #1638: the generic-route pair resolves no element, so there is nothing to // digest into --verify evidence — the settled diff IS the observation. assert.equal(resolveCommandPostActionObservationSupport('scroll'), 'settle'); diff --git a/src/core/command-descriptor/__tests__/timeout-policy.test.ts b/src/core/command-descriptor/__tests__/timeout-policy.test.ts index 287f21d180..ff164aa37d 100644 --- a/src/core/command-descriptor/__tests__/timeout-policy.test.ts +++ b/src/core/command-descriptor/__tests__/timeout-policy.test.ts @@ -79,6 +79,7 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { 'fill', 'find', 'get', + 'hover', 'is', 'longpress', 'press', diff --git a/src/core/command-descriptor/command-result.ts b/src/core/command-descriptor/command-result.ts index 3af279f050..0a40f61bd3 100644 --- a/src/core/command-descriptor/command-result.ts +++ b/src/core/command-descriptor/command-result.ts @@ -18,6 +18,7 @@ import type { FindCommandResponseData, HomeCommandResult, KeyboardCommandResult, + HoverCommandResponseData, LongPressCommandResponseData, OrientationCommandResult, PressCommandResponseData, @@ -52,6 +53,7 @@ export interface CommandResultMap { click: ClickCommandResponseData; fill: FillCommandResponseData; longpress: LongPressCommandResponseData; + hover: HoverCommandResponseData; find: FindCommandResponseData; boot: BootCommandResult; shutdown: ShutdownCommandResult; diff --git a/src/core/command-descriptor/post-action-observation.ts b/src/core/command-descriptor/post-action-observation.ts index 5938e1d8a7..fe066b2a42 100644 --- a/src/core/command-descriptor/post-action-observation.ts +++ b/src/core/command-descriptor/post-action-observation.ts @@ -11,6 +11,9 @@ const POST_ACTION_OBSERVATION_BY_COMMAND = { press: 'settle-and-verify', fill: 'settle-and-verify', longpress: 'settle', + // Hover reveals UI (toolbars, menus) rather than activating a target, so the + // settled diff is the observation; nothing to re-digest into --verify. + hover: 'settle', scroll: 'settle', back: 'settle', } as const satisfies Record; diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index 39480e13a3..f591252423 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -1015,6 +1015,28 @@ export const RAW_COMMAND_DESCRIPTORS = [ batchable: true, platformExecution: LEGACY_PLATFORM_EXECUTION, }, + { + name: 'hover', + ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/interaction/index.ts'] as const } : {}), + targetIdentityVerification: 'pre-dispatch', + catalog: { group: 'public' }, + recordsSessionAction: true, + recordingEffect: 'mutates-app', + daemon: { + route: 'interaction', + refFrameEffect: 'may-invalidate', + }, + dispatch: {}, + // Hover is a pointer-only state (#1783): the web provider moves the mouse + // without pressing. Touch platforms have no hover, so no device bucket + // admits it; `WEB_INTERACTION_COMMANDS` in src/core/capabilities.ts adds the + // web bucket, the same way `viewport` is web-only. + capability: { apple: {}, android: {}, linux: LINUX_NONE }, + timeoutPolicy: postActionObservationTimeoutPolicy('hover', PRESERVE_DAEMON_TIMEOUT_POLICY), + postActionObservation: postActionObservation('hover'), + batchable: true, + platformExecution: LEGACY_PLATFORM_EXECUTION, + }, { name: 'press', ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/interaction/index.ts'] as const } : {}), diff --git a/src/core/dispatch-interactions.ts b/src/core/dispatch-interactions.ts index 0f8062ac22..24846aa8ae 100644 --- a/src/core/dispatch-interactions.ts +++ b/src/core/dispatch-interactions.ts @@ -57,6 +57,22 @@ export async function handleLongPressCommand( return { x, y, durationMs, ...successText(`Long pressed (${x}, ${y})`) }; } +export async function handleHoverCommand( + interactor: Interactor, + positionals: string[], +): Promise> { + const { x, y } = readPoint(positionals, 'hover requires x y', { + hint: 'Direct platform hover requires coordinates. In an open daemon session, use agent-device hover @ref|selector; otherwise run snapshot -i, use the target rect center as x y, then retry hover x y.', + }); + if (!interactor.hover) { + throw new AppError('UNSUPPORTED_OPERATION', 'hover is not supported on this platform', { + hint: 'hover raises pointer hover state and is available on web targets only. On touch platforms use longpress for hold gestures.', + }); + } + await interactor.hover(x, y); + return { x, y, ...successText(`Hovered (${x}, ${y})`) }; +} + export async function handleFocusCommand( interactor: Interactor, positionals: string[], diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index b32d46c07e..934014b919 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -18,6 +18,7 @@ import type { DispatchContext } from './dispatch-context.ts'; import { handleFillCommand, handleFocusCommand, + handleHoverCommand, handleLongPressCommand, handlePressCommand, handleReadCommand, @@ -148,6 +149,7 @@ const DISPATCH_HANDLERS: Record = { press: ({ device, interactor, positionals, context }) => handlePressCommand(device, interactor, positionals, context), longpress: ({ interactor, positionals }) => handleLongPressCommand(interactor, positionals), + hover: ({ interactor, positionals }) => handleHoverCommand(interactor, positionals), focus: ({ interactor, positionals }) => handleFocusCommand(interactor, positionals), type: ({ interactor, positionals, context }) => handleTypeCommand(interactor, positionals, context), diff --git a/src/core/interactors/web.ts b/src/core/interactors/web.ts index 4d16a7cc76..03a07f7ad6 100644 --- a/src/core/interactors/web.ts +++ b/src/core/interactors/web.ts @@ -12,6 +12,13 @@ export function createWebInteractor(): Interactor { openDevice: () => provider().open('about:blank'), close: (target) => provider().close(target), tap: (x, y) => provider().click(x, y), + hover: async (x, y) => { + const hover = provider().hover; + if (!hover) { + throw new AppError('UNSUPPORTED_OPERATION', 'hover is not supported by this web provider'); + } + await hover(x, y); + }, focus: (x, y) => provider().click(x, y), type: (text, delayMs) => provider().typeText(text, { delayMs }), fill: (x, y, text, delayMs) => provider().fill(x, y, text, { delayMs }), diff --git a/src/daemon/__tests__/session-store.test.ts b/src/daemon/__tests__/session-store.test.ts index 331c4d5760..447f43482b 100644 --- a/src/daemon/__tests__/session-store.test.ts +++ b/src/daemon/__tests__/session-store.test.ts @@ -592,6 +592,13 @@ test('writeSessionLog optimizes selector chains and scopes fallback snapshots', durationMs: 800, }, }); + // #1783: hover @ref publishes as a portable selector line like click/longpress. + fixture.store.recordAction(fixture.session, { + command: 'hover', + positionals: ['@e4~s12'], + flags: { platform: 'web', settle: true }, + result: { selectorChain: ['text="Second message"', 'role=link'] }, + }); fixture.store.recordAction(fixture.session, { command: 'fill', positionals: ['@e2', 'hello world'], @@ -604,6 +611,7 @@ test('writeSessionLog optimizes selector chains and scopes fallback snapshots', assertScriptMatches(script, [ /click "text=\\"Continue\\" \|\| role=button" --count 2/, /longpress "label=\\"Last message\\" \|\| role=\\"statictext\\"" 800/, + /hover "text=\\"Second message\\" \|\| role=link"\n/, /snapshot -i -s "Email"/, /fill @e2 "Email" "hello world" --delay-ms 5/, ]); diff --git a/src/daemon/handlers/__tests__/interaction-touch.test.ts b/src/daemon/handlers/__tests__/interaction-touch.test.ts index 966720d00b..c53badfa05 100644 --- a/src/daemon/handlers/__tests__/interaction-touch.test.ts +++ b/src/daemon/handlers/__tests__/interaction-touch.test.ts @@ -1,6 +1,7 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; +import { WEB_DESKTOP_DEVICE } from '../../../__tests__/test-utils/device-fixtures.ts'; import { handleInteractionCommands } from '../interaction.ts'; import { contextFromFlags, @@ -11,7 +12,7 @@ import { } from './interaction-touch-fixtures.ts'; // Router ownership: one representative per touch command proves -// handleTouchInteractionCommands claims press/click/longpress/fill. +// handleTouchInteractionCommands claims press/click/longpress/hover/fill. const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ mockRunAppleRunnerCommand: vi.fn(), @@ -226,3 +227,78 @@ test('longpress @ref resolves the target and dispatches coordinate longpress', a expect(mockDispatch.mock.calls[0]?.[2]).toEqual(['60', '40', '800']); expect(sessionStore.get(sessionName)?.actions[0]?.command).toBe('longpress'); }); + +// #1783: hover is the pointer-only member of the targeted-touch family. It rides +// the same admission/resolution path as press/longpress but dispatches the +// core `hover` verb, and only web admits it. +test('hover @ref resolves the target and dispatches coordinate hover on web', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'hover-ref'; + const session = makeSession(sessionName); + session.device = WEB_DESKTOP_DEVICE; + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + role: 'listitem', + label: 'Last message', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'web', + }; + sessionStore.set(sessionName, session); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'hover', + positionals: ['@e1'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response).toMatchObject({ + ok: true, + data: { x: 60, y: 40, gesture: 'hover', message: expect.stringMatching(/Hovered @e1/) }, + }); + expect(mockDispatch.mock.calls).toEqual([ + [expect.anything(), 'hover', ['60', '40'], undefined, expect.anything()], + ]); + expect(sessionStore.get(sessionName)?.actions[0]?.command).toBe('hover'); +}); + +test('hover is refused by capability on touch platforms before any dispatch', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'hover-ios'; + sessionStore.set(sessionName, makeSession(sessionName)); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'hover', + positionals: ['100', '200'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response).toMatchObject({ + ok: false, + error: { + code: 'UNSUPPORTED_OPERATION', + message: expect.stringMatching(/--platform web/), + }, + }); + expect(mockDispatch).not.toHaveBeenCalled(); +}); diff --git a/src/daemon/handlers/interaction-flags.ts b/src/daemon/handlers/interaction-flags.ts index bd37f86e54..b511ebd3de 100644 --- a/src/daemon/handlers/interaction-flags.ts +++ b/src/daemon/handlers/interaction-flags.ts @@ -10,7 +10,7 @@ const REF_UNSUPPORTED_FLAG_MAP: ReadonlyArray<[keyof CommandFlags, string]> = [ ]; export function refSnapshotFlagGuardResponse( - command: 'press' | 'fill' | 'get' | 'longpress', + command: 'press' | 'fill' | 'get' | 'longpress' | 'hover', flags: CommandFlags | undefined, ): DaemonResponse | null { const unsupported = unsupportedRefSnapshotFlags(flags); diff --git a/src/daemon/handlers/interaction-runtime.ts b/src/daemon/handlers/interaction-runtime.ts index 3adeab08ee..78966b51e8 100644 --- a/src/daemon/handlers/interaction-runtime.ts +++ b/src/daemon/handlers/interaction-runtime.ts @@ -168,6 +168,18 @@ function createInteractionBackend( ), ); }, + hover: async (_context, point): Promise => { + expireRefFrame(session); + return toBackendActionResult( + await dispatchCommand( + session.device, + 'hover', + [String(point.x), String(point.y)], + req.flags?.out, + params.contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), + ), + ); + }, performGesture: async (_context, plan): Promise => { expireRefFrame(session); return toBackendActionResult( diff --git a/src/daemon/handlers/interaction-touch-payload.ts b/src/daemon/handlers/interaction-touch-payload.ts index 37845ab688..d70216180b 100644 --- a/src/daemon/handlers/interaction-touch-payload.ts +++ b/src/daemon/handlers/interaction-touch-payload.ts @@ -65,7 +65,9 @@ function buildPointTouchMessage( extra: Record | undefined, pointSuffix: string, ): string { - return extra?.gesture === 'longpress' ? `Long pressed${pointSuffix}` : `Tapped${pointSuffix}`; + if (extra?.gesture === 'longpress') return `Long pressed${pointSuffix}`; + if (extra?.gesture === 'hover') return `Hovered${pointSuffix}`; + return `Tapped${pointSuffix}`; } function buildTouchTargetMessage( @@ -77,6 +79,9 @@ function buildTouchTargetMessage( if (extra.gesture === 'longpress') { return `Long pressed ${label}${pointSuffix}`; } + if (extra.gesture === 'hover') { + return `Hovered ${label}${pointSuffix}`; + } if (button && button !== 'primary') { return `Clicked ${button} ${label}${pointSuffix}`; } diff --git a/src/daemon/handlers/interaction-touch-policy.ts b/src/daemon/handlers/interaction-touch-policy.ts index ba73ee4ffe..c586451875 100644 --- a/src/daemon/handlers/interaction-touch-policy.ts +++ b/src/daemon/handlers/interaction-touch-policy.ts @@ -4,7 +4,7 @@ import { errorResponse } from './response.ts'; export function unsupportedMacOsDesktopSurfaceInteraction( session: SessionState, - command: 'click' | 'press' | 'fill' | 'longpress', + command: 'click' | 'press' | 'fill' | 'longpress' | 'hover', ): DaemonResponse | null { if (!isMacOs(session.device)) { return null; diff --git a/src/daemon/handlers/interaction-touch-press-admission.ts b/src/daemon/handlers/interaction-touch-press-admission.ts index bf90911b47..66a18d572e 100644 --- a/src/daemon/handlers/interaction-touch-press-admission.ts +++ b/src/daemon/handlers/interaction-touch-press-admission.ts @@ -24,12 +24,18 @@ import { import { errorResponse, noActiveSessionError, requireCommandSupported } from './response.ts'; /** - * Whether a targeted `press`/`click`/`longpress` may act, and on what: macOS + * Whether a targeted `press`/`click`/`longpress`/`hover` may act, and on what: macOS * surface and capability policy, click-option validation, target parsing, and * `@ref` staleness plus mutation admission (ADR 0014). Nothing here dispatches. */ -export type TargetedTouchCommand = 'press' | 'click' | 'longpress'; +export type TargetedTouchCommand = 'press' | 'click' | 'longpress' | 'hover'; + +/** The family members that take `--button`; longpress and hover have no button. */ +const CLICK_BUTTON_COMMANDS: ReadonlySet = new Set(['press', 'click']); + +const HOVER_UNSUPPORTED_MESSAGE = + 'hover is not supported on this device: hover is a pointer state that only web targets have (--platform web). Touch platforms have no hover; use longpress for hold gestures.'; export type TargetedTouchParams = InteractionHandlerParams & { captureSnapshotForSession: CaptureSnapshotForSession; @@ -102,13 +108,17 @@ function targetedTouchPolicyResponse( commandLabel: TargetedTouchCommand, flags: CommandFlags | undefined, ): DaemonResponse | undefined { - const capabilityCommand = command === 'longpress' ? 'longpress' : 'press'; + const capabilityCommand = command === 'click' ? 'press' : command; const unsupportedSurfaceResponse = unsupportedMacOsDesktopSurfaceInteraction( session, commandLabel, ); if (unsupportedSurfaceResponse) return unsupportedSurfaceResponse; - const unsupported = requireCommandSupported(capabilityCommand, session.device); + const unsupported = requireCommandSupported( + capabilityCommand, + session.device, + command === 'hover' ? { message: HOVER_UNSUPPORTED_MESSAGE } : undefined, + ); if (unsupported) return unsupported; const invalidSettleFlags = settleFlagGuardResponse(command, flags); if (invalidSettleFlags) return invalidSettleFlags; @@ -122,7 +132,7 @@ function clickButtonValidationResponse( flags: CommandFlags | undefined, ): DaemonResponse | undefined { const clickButton = resolveClickButton(flags); - if (command === 'longpress' || clickButton === 'primary') return undefined; + if (!CLICK_BUTTON_COMMANDS.has(command) || clickButton === 'primary') return undefined; const validationError = getClickButtonValidationError({ commandLabel, platform: publicPlatformString(session.device), @@ -184,7 +194,7 @@ async function admitTargetedTouchRef( const { req } = params; if (parsedTarget.target.kind !== 'ref') return {}; const invalidRefFlagsResponse = params.refSnapshotFlagGuardResponse( - command === 'longpress' ? 'longpress' : 'press', + command === 'click' ? 'press' : command, req.flags, ); if (invalidRefFlagsResponse) return { response: invalidRefFlagsResponse }; diff --git a/src/daemon/handlers/interaction-touch-press.ts b/src/daemon/handlers/interaction-touch-press.ts index e78c12db1b..118d28ea02 100644 --- a/src/daemon/handlers/interaction-touch-press.ts +++ b/src/daemon/handlers/interaction-touch-press.ts @@ -26,7 +26,7 @@ import { dispatchRuntimeInteraction } from './interaction-touch-runtime.ts'; import { formatTouchTargetLabel } from './interaction-touch-targets.ts'; /** - * How an admitted targeted `press`/`click`/`longpress` executes: the direct-iOS + * How an admitted targeted `press`/`click`/`longpress`/`hover` executes: the direct-iOS * attempt, then the shared runtime dispatch it delegates to with the options * and payload projection this command family owns. */ @@ -75,7 +75,9 @@ function buildTargetedRuntimeOptions( const targetedExtra = command === 'longpress' ? { ...(durationMs !== undefined ? { durationMs } : {}), gesture: 'longpress' } - : resultButtonTag; + : command === 'hover' + ? { gesture: 'hover' } + : resultButtonTag; return { androidFreshnessBaseline: admitted.androidFreshnessBaseline, refContext: admitted.refContext, @@ -106,7 +108,7 @@ function buildTargetedRuntimeOptions( staleRefsWarning, publicData: transformTouchResponseData({ session, - command: command === 'longpress' ? undefined : command, + command: command === 'longpress' || command === 'hover' ? undefined : command, flags: req.flags, data: result.backendResult, }), @@ -116,7 +118,9 @@ function buildTargetedRuntimeOptions( ...(resultDurationMs !== undefined ? { durationMs: resultDurationMs } : {}), gesture: 'longpress', } - : resultButtonTag, + : command === 'hover' + ? { gesture: 'hover' } + : resultButtonTag, }); }, }; @@ -137,20 +141,43 @@ async function runTargetedTouchInteraction(params: { }): Promise { const { runtime, command, target, sessionName, requestId, flags, expectedResolvedTarget } = params; - const settle = readSettleRequest(flags); - if (command === 'longpress') { - return await runtime.interactions.longPress(target, { - session: sessionName, - requestId, - durationMs: params.durationMs, - settle, - expectedResolvedTarget, - }); - } - - const options = { + const shared = { session: sessionName, requestId, + settle: readSettleRequest(flags), + expectedResolvedTarget, + }; + switch (command) { + case 'longpress': + return await runtime.interactions.longPress(target, { + ...shared, + durationMs: params.durationMs, + }); + case 'hover': + return await runtime.interactions.hover(target, shared); + case 'click': + return await runtime.interactions.click(target, pressRuntimeOptions(params, shared)); + case 'press': + return await runtime.interactions.press(target, pressRuntimeOptions(params, shared)); + } +} + +function pressRuntimeOptions( + params: { + clickButton: ReturnType; + flags: CommandFlags | undefined; + preresolvedTarget?: PreresolvedInteractionTarget; + }, + shared: { + session: string; + requestId: string | undefined; + settle: ReturnType; + expectedResolvedTarget?: ReplayTargetGuardDenotation; + }, +) { + const { flags } = params; + return { + ...shared, button: params.clickButton, count: flags?.count, intervalMs: flags?.intervalMs, @@ -158,16 +185,11 @@ async function runTargetedTouchInteraction(params: { jitterPx: flags?.jitterPx, doubleTap: flags?.doubleTap, verify: flags?.verify, - settle, - expectedResolvedTarget, // Only click/press take it: `find` dispatches click and fill, never - // longpress, so declaring it on the longPress options above would be an + // longpress or hover, so declaring it on their options would be an // unconsumed claim (#1649 review). preresolvedTarget: params.preresolvedTarget, }; - return command === 'click' - ? await runtime.interactions.click(target, options) - : await runtime.interactions.press(target, options); } function readLongPressResultDuration(result: TargetedTouchResult): number | undefined { diff --git a/src/daemon/handlers/interaction-touch-response.ts b/src/daemon/handlers/interaction-touch-response.ts index 0e65e1a3f8..c81d2969af 100644 --- a/src/daemon/handlers/interaction-touch-response.ts +++ b/src/daemon/handlers/interaction-touch-response.ts @@ -2,6 +2,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import type { FillCommandResult, GestureReferenceFrame, + HoverCommandResult, LongPressCommandResult, PressCommandResult, RecordingTargetOverride, @@ -38,7 +39,11 @@ import { interactionResultExtra } from './interaction-touch-targets.ts'; * module. */ -type InteractionRuntimeResult = PressCommandResult | FillCommandResult | LongPressCommandResult; +type InteractionRuntimeResult = + | PressCommandResult + | FillCommandResult + | LongPressCommandResult + | HoverCommandResult; type InteractionResponseSourceBase = { publicData?: Record; @@ -275,7 +280,7 @@ function composeResponseWarning( } /** What `press`/`click`/`longpress` resolve to; `fill` carries its own result. */ -export type TargetedTouchResult = PressCommandResult | LongPressCommandResult; +export type TargetedTouchResult = PressCommandResult | LongPressCommandResult | HoverCommandResult; export async function buildTargetedTouchResponsePayloads(params: { params: InteractionHandlerParams & { diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index f313f0d3e8..112d2fdac3 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -19,6 +19,8 @@ export async function handleTouchInteractionCommands( return await dispatchTargetedTouchViaRuntime(params, 'click'); case 'longpress': return await dispatchTargetedTouchViaRuntime(params, 'longpress'); + case 'hover': + return await dispatchTargetedTouchViaRuntime(params, 'hover'); case 'fill': return await dispatchFillViaRuntime(params); default: diff --git a/src/daemon/response-views.ts b/src/daemon/response-views.ts index 608cc2198d..4ae7222549 100644 --- a/src/daemon/response-views.ts +++ b/src/daemon/response-views.ts @@ -271,5 +271,6 @@ export const RESPONSE_VIEWS: Record = { click: interactionDigestView, fill: interactionDigestView, longpress: interactionDigestView, + hover: interactionDigestView, network: networkView, }; diff --git a/src/daemon/session-event-action.ts b/src/daemon/session-event-action.ts index 39fc775a9c..f086cd4136 100644 --- a/src/daemon/session-event-action.ts +++ b/src/daemon/session-event-action.ts @@ -27,6 +27,8 @@ export function buildActionSummary(action: SessionAction): string { return `Tapped ${readActionTargetLabel(action) ?? 'target'}`; case PUBLIC_COMMANDS.longPress: return `Long pressed ${readActionTargetLabel(action) ?? 'target'}`; + case PUBLIC_COMMANDS.hover: + return `Hovered ${readActionTargetLabel(action) ?? 'target'}`; case PUBLIC_COMMANDS.fill: return `Filled ${readActionTargetLabel(action) ?? 'target'}`; case PUBLIC_COMMANDS.type: @@ -107,6 +109,7 @@ function isTargetActionCommand(command: string): boolean { command === PUBLIC_COMMANDS.click || command === PUBLIC_COMMANDS.press || command === PUBLIC_COMMANDS.longPress || + command === PUBLIC_COMMANDS.hover || command === PUBLIC_COMMANDS.focus || command === PUBLIC_COMMANDS.fill ); @@ -293,6 +296,7 @@ const SAFE_ACTION_FLAG_SPECS: Record = { [PUBLIC_COMMANDS.click]: touchSafeFlagSpec(), [PUBLIC_COMMANDS.press]: touchSafeFlagSpec(), [PUBLIC_COMMANDS.longPress]: touchSafeFlagSpec(), + [PUBLIC_COMMANDS.hover]: touchSafeFlagSpec(), [PUBLIC_COMMANDS.fill]: textEntrySafeFlagSpec(), [PUBLIC_COMMANDS.type]: textEntrySafeFlagSpec(), [PUBLIC_COMMANDS.scroll]: { diff --git a/src/daemon/session-script-writer.ts b/src/daemon/session-script-writer.ts index 63e3884fc4..0900321c98 100644 --- a/src/daemon/session-script-writer.ts +++ b/src/daemon/session-script-writer.ts @@ -380,7 +380,9 @@ function assertNoUnresolvedDragEndpoint(drag: { source: string; destination: str function optimizeSelectorChainAction(action: SessionAction): SessionAction | undefined { const selectorExpr = readSelectorChainExpression(action); if (!selectorExpr || !isSelectorTargetingCommand(action.command)) return undefined; - if (isClickLikeCommand(action.command)) return { ...action, positionals: [selectorExpr] }; + if (isClickLikeCommand(action.command) || action.command === 'hover') { + return { ...action, positionals: [selectorExpr] }; + } if (action.command === 'longpress') return optimizeLongPressAction(action, selectorExpr); if (action.command === 'fill') return optimizeFillAction(action, selectorExpr); return optimizeGetAction(action, selectorExpr); diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 547d9fef5c..aab49ed032 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -396,6 +396,12 @@ export const COMMAND_OUTPUT_SCHEMAS = { gesture: constSchema('longpress'), }, }), + hover: interactionResponseDataSchema({ + properties: { + settle: settleObservationSchema, + gesture: constSchema('hover'), + }, + }), find: objectSchema( { ref: stringSchema('Snapshot ref without the @ prefix when the find action returns one.'), diff --git a/src/mcp/tool-ref-pins.ts b/src/mcp/tool-ref-pins.ts index 6cbb7d2547..e32d86fce2 100644 --- a/src/mcp/tool-ref-pins.ts +++ b/src/mcp/tool-ref-pins.ts @@ -100,6 +100,7 @@ const TARGET_REF_TOOLS: ReadonlySet = new Set([ 'click', 'fill', 'longpress', + 'hover', 'get', ] as const); diff --git a/src/platforms/web/agent-browser-provider.test.ts b/src/platforms/web/agent-browser-provider.test.ts index db4cfc0e94..c3683bab02 100644 --- a/src/platforms/web/agent-browser-provider.test.ts +++ b/src/platforms/web/agent-browser-provider.test.ts @@ -46,6 +46,7 @@ test('agent-browser provider maps supported operations to session-scoped JSON co await provider.setViewport(1280, 900); await provider.click(10.4, 20.6); await provider.clickRef?.('@e3'); + await provider.hover?.(30.2, 40.7); await provider.fill(11, 22, 'Ada'); await provider.fillRef?.('@e2', 'Grace'); await provider.typeText('hello'); @@ -70,6 +71,8 @@ test('agent-browser provider maps supported operations to session-scoped JSON co ['mouse', 'down', '--json', '--session', 'web-session'], ['mouse', 'up', '--json', '--session', 'web-session'], ['click', '@e3', '--json', '--session', 'web-session'], + // hover is a bare pointer move: no button transition follows (#1783). + ['mouse', 'move', '30', '41', '--json', '--session', 'web-session'], ['mouse', 'move', '11', '22', '--json', '--session', 'web-session'], ['mouse', 'down', '--json', '--session', 'web-session'], ['mouse', 'up', '--json', '--session', 'web-session'], diff --git a/src/platforms/web/agent-browser-provider.ts b/src/platforms/web/agent-browser-provider.ts index 9e573fe3de..808fcea85c 100644 --- a/src/platforms/web/agent-browser-provider.ts +++ b/src/platforms/web/agent-browser-provider.ts @@ -69,6 +69,9 @@ export function createAgentBrowserWebProvider( async click(x, y) { await clickCoordinates(runJson, x, y); }, + async hover(x, y) { + await movePointer(runJson, x, y); + }, async clickRef(ref) { await runJson(['click', browserRefSelector(ref)]); }, @@ -164,11 +167,19 @@ async function clickCoordinates( x: number, y: number, ): Promise { - await runJson(['mouse', 'move', String(Math.round(x)), String(Math.round(y))]); + await movePointer(runJson, x, y); await runJson(['mouse', 'down']); await runJson(['mouse', 'up']); } +async function movePointer( + runJson: (args: string[]) => Promise, + x: number, + y: number, +): Promise { + await runJson(['mouse', 'move', String(Math.round(x)), String(Math.round(y))]); +} + async function captureAgentBrowserSnapshot( runJson: (args: string[]) => Promise, options: WebSnapshotOptions | undefined, diff --git a/src/platforms/web/provider.ts b/src/platforms/web/provider.ts index cda2edc021..dbbaf032d1 100644 --- a/src/platforms/web/provider.ts +++ b/src/platforms/web/provider.ts @@ -54,6 +54,8 @@ export type WebProvider = { setViewport(width: number, height: number): Promise; click(x: number, y: number): Promise; clickRef?(ref: string): Promise; + /** Move the pointer to a point without pressing, raising the page's hover state. */ + hover?(x: number, y: number): Promise; fill(x: number, y: number, text: string, options?: { delayMs?: number }): Promise; fillRef?(ref: string, text: string, options?: { delayMs?: number }): Promise; typeText(text: string, options?: { delayMs?: number }): Promise; diff --git a/test/integration/android-emulator-e2e/coverage-manifest.ts b/test/integration/android-emulator-e2e/coverage-manifest.ts index a6324d120f..13d69fab09 100644 --- a/test/integration/android-emulator-e2e/coverage-manifest.ts +++ b/test/integration/android-emulator-e2e/coverage-manifest.ts @@ -195,6 +195,10 @@ export const ANDROID_EMULATOR_E2E_COVERAGE = { level: 'capability-denial', }, [C.type]: live('smoke:form-input', 'typed suffix is read back from focused Android field'), + [C.hover]: { + assertion: 'Android emulator capability model rejects hover, a pointer-only web contract', + level: 'capability-denial', + }, [C.viewport]: { assertion: 'Android emulator capability model rejects standalone viewport control', level: 'capability-denial', diff --git a/test/integration/ios-simulator-e2e/coverage-manifest.ts b/test/integration/ios-simulator-e2e/coverage-manifest.ts index 5b1490c8a7..f594965938 100644 --- a/test/integration/ios-simulator-e2e/coverage-manifest.ts +++ b/test/integration/ios-simulator-e2e/coverage-manifest.ts @@ -195,6 +195,14 @@ export const IOS_SIMULATOR_E2E_COVERAGE = { 'smoke:form-input', 'AX-independent first-responder typing appends and is read back from a focused fixture field', ), + [C.hover]: { + assertion: 'iOS simulator capability model rejects hover, a pointer-only web contract', + level: 'capability-denial', + owner: { + path: 'test/integration/smoke-ios-simulator-coverage.test.ts', + test: 'capability classifications match executable simulator behavior', + }, + }, [C.viewport]: { assertion: 'iOS simulator capability model rejects viewport resizing, a web-only contract', level: 'capability-denial', diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index d18c55a599..216f998680 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -124,6 +124,7 @@ agent-device get text @e2 --platform web agent-device is visible 'label="Welcome"' --platform web agent-device find text "Welcome" exists --platform web agent-device click @e12 --platform web +agent-device hover @e14 --settle --platform web agent-device fill @e13 "test@example.com" --platform web agent-device wait text "Welcome" --platform web agent-device network dump 25 --include headers --platform web @@ -143,7 +144,8 @@ agent-device close --platform web - `web doctor` verifies the managed backend after setup. - The managed install respects `--state-dir` and `AGENT_DEVICE_STATE_DIR`. - Web automation requires Node 24+. -- Supported through `agent-device`: URL open, snapshot refs, `get text/attrs`, `is visible/exists/text`, `find text/selector`, click/press, fill/type, wait, `network dump`, `audio probe`, screenshot, close, and replay scripts composed from those commands. +- Supported through `agent-device`: URL open, snapshot refs, `get text/attrs`, `is visible/exists/text`, `find text/selector`, click/press, hover, fill/type, wait, `network dump`, `audio probe`, screenshot, close, and replay scripts composed from those commands. +- `hover <@ref|selector|x y>` moves the pointer without pressing so hover-gated UI (row toolbars, menus) appears. Add `--settle` to read what it revealed instead of taking another snapshot. - `audio probe start [durationSeconds] [bucketMs]` samples HTML media elements into compact RMS/peak dBFS buckets while the page keeps running. The first timing positional is seconds; the second is milliseconds. - URL-backed web media may be routed through the probe `AudioContext` while observed. Use `audio probe status` to poll partial buckets and `audio probe stop` to end the probe early. - Out of scope for `agent-device` web support: tab/window/devtools control, network routing/interception/HAR, cookies/storage, downloads/uploads, arbitrary page scripting, multi-page orchestration, and raw browser diagnostics. Use `agent-browser` directly for those browser-specific workflows. @@ -393,6 +395,7 @@ agent-device gesture fling right 200 420 180 agent-device gesture drag 'id="drag-source"' 'id="drop-target"' agent-device gesture drag @e4~s12 'label="Archive"' 700 600 200 agent-device longpress 300 500 800 +agent-device hover @e12 --settle # Web only: move the pointer without pressing agent-device scroll down 0.5 agent-device scroll down --pixels 320 agent-device gesture pinch 2.0 # zoom in 2x @@ -441,6 +444,7 @@ done ``` `longpress` is supported on iOS and Android. +`hover` is supported on web only. It moves the pointer over a target (`@ref`, selector, or coordinates) without pressing, so hover-gated UI such as row toolbars and menus appears; touch platforms have no hover state and reject it. Use `--settle` to get the diff of what the hover revealed and act on the fresh refs; use `longpress` for the mobile hold-gesture equivalent. `gesture pinch` is supported on Android and iOS simulator app sessions. `gesture rotate` is supported on Android and iOS simulator app sessions. Use `orientation` for device orientation. Two-finger `gesture pan` and `gesture transform` are supported on Android and iOS simulator app sessions. One-finger `gesture pan` keeps the broader platform support of ordinary coordinate drags. From fdebd8144a86cb5cc6e251bad3cd6f363b16f086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 17 Aug 2026 18:53:08 +0200 Subject: [PATCH 2/4] fix(hover): native hoverRef route for web @ref, android coverage pin, revert skill edit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on #1786: - hover @ref on web now dispatches through the provider's own element handle (agent-browser `hover `) via a new backend hoverTarget, mirroring click/fill's ADR 0011 native-ref path — web ref frames carry no rects, so the coordinate route could never resolve them. The shared preflight + exact-ref dispatch is extracted into dispatchNativeRefInteraction and used by tap/fill/hover; the guarantee matrix native-ref row now lists hover. - Daemon regression test is production-faithful: rect-less web ref frame, scoped provider, asserts no coordinate dispatch. Selector→coordinate and provider hoverRef tests added. - Android emulator coverage summary pin 2/53 → 3/54. - skills/agent-device/SKILL.md reverted (out of scope, AGENTS.md rule). - Docs/help disclose that --settle with @ref on web shares click's existing limitation; use a selector or coordinates for the settled diff. --- .../contracts/src/interaction-guarantees.ts | 12 +-- skills/agent-device/SKILL.md | 2 +- src/backend.ts | 4 + src/cli/parser/cli-help.ts | 2 +- src/commands/interaction/runtime/gestures.ts | 29 +++++++ .../interaction/runtime/interactions.ts | 53 +++++------- .../interaction/runtime/resolution.ts | 54 ++++++++++++- .../__tests__/interaction-touch.test.ts | 81 ++++++++++++++++--- src/daemon/handlers/interaction-runtime.ts | 9 ++- .../web/agent-browser-provider.test.ts | 2 + src/platforms/web/agent-browser-provider.ts | 3 + src/platforms/web/provider.ts | 2 + .../smoke-android-emulator-coverage.test.ts | 4 +- website/docs/docs/commands.md | 2 +- 14 files changed, 201 insertions(+), 58 deletions(-) diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index b0b41ef2d6..3244250f6c 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -361,13 +361,13 @@ export const INTERACTION_DISPATCH_PATHS: Record --foreground That starts the session and returns the initial interactive snapshot with `@refs`. -Loop: act with `press|click|fill|longpress ... --settle`, `hover --settle` (web only, reveals hover-gated UI), `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation (`wait text "..."`, `is`, `get`, or `find`), then run `agent-device close`. +Loop: act with `press|click|fill|longpress ... --settle`, `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation (`wait text "..."`, `is`, `get`, or `find`), then run `agent-device close`. Copy refs byte-for-byte: `@e12`, `@e12~s4` — keep the `@` and any `~sN`. Prefer current refs, then `id`/`label`/`role` selectors; coordinates are a last resort. If snapshot reports sparse/AX-unavailable, its refs and selectors are invalid: run `agent-device screenshot`, inspect the image, use coordinates, then retry `snapshot -i` after navigating. Otherwise run `snapshot -i` only when the diff lacks the next target. diff --git a/src/backend.ts b/src/backend.ts index ea508feff1..501509825c 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -500,6 +500,10 @@ export type AgentDeviceBackend = { options?: BackendLongPressOptions, ): Promise; hover?(context: BackendCommandContext, point: Point): Promise; + hoverTarget?( + context: BackendCommandContext, + target: BackendRefTarget, + ): Promise; scroll?( context: BackendCommandContext, target: BackendScrollTarget, diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 9139ee6ce9..808be759a4 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -900,7 +900,7 @@ First-slice loop: Supported in agent-device web sessions: open , snapshot -i, get text/attrs, is visible/exists/text, find text/selector, click/press @ref or selector, hover @ref or selector, fill/type @ref or selector, wait text/selector, network dump, audio probe, screenshot, record start/stop with WebM output, close, and replay scripts made from those commands. - hover moves the pointer without pressing so hover-gated UI (row toolbars, menus) appears; use --settle to read what it revealed, then act on the fresh refs. Web only: touch platforms have no hover state, so hover-gated flows there need a different entry point. + hover moves the pointer without pressing so hover-gated UI (row toolbars, menus) appears; use --settle to read what it revealed, then act on the fresh refs. hover @ref hovers the browser element handle directly; pair --settle with a selector or coordinates (web refs carry no geometry, as with click @ref --settle). Web only: touch platforms have no hover state, so hover-gated flows there need a different entry point. Out of scope for agent-device web support: Browser runtime debugging, tabs/windows/devtools control, network routing/interception/HAR, storage/cookie management, arbitrary page scripting, downloads/uploads, multi-page orchestration, and agent-browser-specific diagnostics. Use agent-browser directly for those browser-specific workflows. diff --git a/src/commands/interaction/runtime/gestures.ts b/src/commands/interaction/runtime/gestures.ts index 1a9d1968ea..99dda33076 100644 --- a/src/commands/interaction/runtime/gestures.ts +++ b/src/commands/interaction/runtime/gestures.ts @@ -42,6 +42,7 @@ import { import { assertSupportedInteractionSurface, captureInteractionSnapshot, + dispatchNativeRefInteraction, resolveInteractionTarget, type ExpectedResolvedTarget, type InteractionTarget, @@ -232,6 +233,10 @@ export const hoverCommand: RuntimeCommand => { const observation = planPostActionObservation(options); + const nativeRefHover = observation.needsPreActionBaseline + ? null + : await maybeHoverRefTarget(runtime, options); + if (nativeRefHover) return nativeRefHover; // Hover keeps the element it resolved (no hittable-ancestor promotion): the // pointer only has to enter the matched node's box for its hover state to // raise, and promoting could move it onto a sibling-owned region. @@ -261,6 +266,30 @@ export const hoverCommand: RuntimeCommand { + if (options.target.kind !== 'ref' || !runtime.backend.hoverTarget) return null; + if (options.expectedResolvedTarget) return null; + const { hoverTarget } = runtime.backend; + return await dispatchNativeRefInteraction( + runtime, + options, + options.target, + 'hover', + async (context, refTarget) => await hoverTarget(context, refTarget), + ); +} + export const dragCommand: RuntimeCommand = async ( runtime, options, diff --git a/src/commands/interaction/runtime/interactions.ts b/src/commands/interaction/runtime/interactions.ts index dc31bd30ae..763e9e8289 100644 --- a/src/commands/interaction/runtime/interactions.ts +++ b/src/commands/interaction/runtime/interactions.ts @@ -27,8 +27,7 @@ import { type PostActionObservationOptions, } from './post-action-observation.ts'; import { - EXACT_REF_RESOLUTION, - preflightNativeRefInteraction, + dispatchNativeRefInteraction, resolveInteractionTarget, type ExpectedResolvedTarget, type InteractionTarget, @@ -267,20 +266,14 @@ async function maybeTapRefTarget( // against the stored session snapshot node before the backend call (a // backend fast path can silently "succeed", so errors must be raised here). // No snapshot / no usable rect → no-op; never adds a capture round trip. - const preflight = await preflightNativeRefInteraction(runtime, options, options.target, action); - const backendResult = await runtime.backend.tapTarget(toBackendContext(runtime, options), { - kind: 'ref', - ref: options.target.ref, - ...(options.target.fallbackLabel ? { fallbackLabel: options.target.fallbackLabel } : {}), - }); - const formattedBackendResult = toBackendResult(backendResult); - return { - kind: 'ref', - target: { kind: 'ref', ref: options.target.ref }, - resolution: EXACT_REF_RESOLUTION, - ...preflight, - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - }; + const { tapTarget } = runtime.backend; + return await dispatchNativeRefInteraction( + runtime, + options, + options.target, + action, + async (context, refTarget) => await tapTarget(context, refTarget), + ); } async function maybeFillRefTarget( @@ -291,26 +284,16 @@ async function maybeFillRefTarget( // ADR 0012 step 4: guarded replay actions take the runtime path — see maybeTapRefTarget. if (options.expectedResolvedTarget) return null; // ADR 0011 native-ref preflight — see maybeTapRefTarget. - const preflight = await preflightNativeRefInteraction(runtime, options, options.target, 'fill'); - const backendResult = await runtime.backend.fillTarget( - toBackendContext(runtime, options), - { - kind: 'ref', - ref: options.target.ref, - ...(options.target.fallbackLabel ? { fallbackLabel: options.target.fallbackLabel } : {}), - }, - options.text, - { delayMs: options.delayMs }, + const { fillTarget } = runtime.backend; + const dispatched = await dispatchNativeRefInteraction( + runtime, + options, + options.target, + 'fill', + async (context, refTarget) => + await fillTarget(context, refTarget, options.text, { delayMs: options.delayMs }), ); - const formattedBackendResult = toBackendResult(backendResult); - return { - kind: 'ref', - target: { kind: 'ref', ref: options.target.ref }, - text: options.text, - resolution: EXACT_REF_RESOLUTION, - ...preflight, - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - }; + return { ...dispatched, text: options.text }; } function hasNonDefaultTapOptions(options: PressCommandOptions): boolean { diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index b51ae21b90..47c61f1fd5 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -48,7 +48,13 @@ import type { ResolutionDisclosure, ResolvedInteractionTarget, } from '@agent-device/contracts/interaction'; +import type { + BackendActionResult, + BackendCommandContext, + BackendRefTarget, +} from '../../../backend.ts'; import { now, toBackendContext } from '../../runtime-common.ts'; +import { toBackendResult } from '../../runtime-types.ts'; import { resolveInteractionTouchPoint } from '../../../core/interaction-touch-point.ts'; import { localIdentitiesEqual, @@ -453,7 +459,12 @@ function assertReplayTargetResolution( const RESOLUTION_DIAGNOSTIC_STRING_BYTE_CAP = 256; const MAX_RESOLUTION_ALTERNATIVES = 5; -/** A successful `@ref` lookup names exactly one node; label recovery discloses label-fallback instead. */ +/** + * A successful `@ref` lookup names exactly one node; label recovery discloses label-fallback instead. + * Exported as an ADR 0011 registry anchor: interaction-guarantees.ts cites it as a `via` + * symbol and the gate test imports it dynamically, which fallow cannot trace statically. + */ +// fallow-ignore-next-line unused-export export const EXACT_REF_RESOLUTION: ResolutionDisclosure = { source: 'ref', phase: 'pre-action', @@ -925,7 +936,12 @@ function scrollRevealClause(direction: OffscreenScrollDirection | null): string * a would-be off-screen refusal may spend one extra iOS runner round trip * (#1542's double-check) before erroring — cost only on the path that was * about to fail anyway. + * + * Exported as an ADR 0011 registry anchor (interaction-guarantees.ts `via` + * symbol, imported dynamically by the gate test); production callers reach + * it through `dispatchNativeRefInteraction`. */ +// fallow-ignore-next-line unused-export export async function preflightNativeRefInteraction( runtime: AgentDeviceRuntime, options: CommandContext, @@ -972,6 +988,42 @@ export async function preflightNativeRefInteraction( }; } +/** + * ADR 0011 native-ref dispatch, shared by click/fill/hover @ref: run the + * preflight guards against the stored node, hand the ref to the backend as + * its own element handle, and return the exact-ref result envelope. Callers + * decide WHEN the path applies (backend capability, no non-default options, + * no replay guard, no settle baseline); this owns only the dispatch itself so + * the three commands cannot drift on preflight or disclosure. + */ +export async function dispatchNativeRefInteraction( + runtime: AgentDeviceRuntime, + options: CommandContext, + target: Extract, + action: InteractionAction, + dispatch: ( + context: BackendCommandContext, + refTarget: BackendRefTarget, + ) => Promise, +): Promise< + Extract & { backendResult?: Record } +> { + const preflight = await preflightNativeRefInteraction(runtime, options, target, action); + const backendResult = await dispatch(toBackendContext(runtime, options), { + kind: 'ref', + ref: target.ref, + ...(target.fallbackLabel ? { fallbackLabel: target.fallbackLabel } : {}), + }); + const formattedBackendResult = toBackendResult(backendResult); + return { + kind: 'ref', + target: { kind: 'ref', ref: target.ref }, + resolution: EXACT_REF_RESOLUTION, + ...preflight, + ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + }; +} + // isNodeVisibleOnScreen (not the effective-viewport form): items inside an // off-screen scrollable container (closed drawer) must also count as // off-screen, not just items scrolled out of an on-screen container. diff --git a/src/daemon/handlers/__tests__/interaction-touch.test.ts b/src/daemon/handlers/__tests__/interaction-touch.test.ts index c53badfa05..b3a7bc7016 100644 --- a/src/daemon/handlers/__tests__/interaction-touch.test.ts +++ b/src/daemon/handlers/__tests__/interaction-touch.test.ts @@ -2,6 +2,7 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; import { WEB_DESKTOP_DEVICE } from '../../../__tests__/test-utils/device-fixtures.ts'; +import { withWebProvider, type WebProvider } from '../../../platforms/web/provider.ts'; import { handleInteractionCommands } from '../interaction.ts'; import { contextFromFlags, @@ -229,19 +230,66 @@ test('longpress @ref resolves the target and dispatches coordinate longpress', a }); // #1783: hover is the pointer-only member of the targeted-touch family. It rides -// the same admission/resolution path as press/longpress but dispatches the -// core `hover` verb, and only web admits it. -test('hover @ref resolves the target and dispatches coordinate hover on web', async () => { +// the same admission path as press/longpress, and only web admits it. On web +// the session's ref frame is minted WITHOUT rects (snapshot -i does not fetch +// boxes), so `hover @ref` cannot resolve to coordinates: like `click @ref` it +// must take the provider-native route (`hoverRef`, ADR 0011 native-ref path). +// This test therefore stores a rect-less web frame and a scoped provider — the +// production shape — and asserts no coordinate dispatch happens. +test('hover @ref on web dispatches through the provider hoverRef route, not coordinates', async () => { const sessionStore = makeSessionStore(); const sessionName = 'hover-ref'; const session = makeSession(sessionName); session.device = WEB_DESKTOP_DEVICE; session.snapshot = { + nodes: attachRefs([{ index: 0, role: 'link', label: 'Second message', enabled: true }]), + createdAt: Date.now(), + backend: 'web', + }; + sessionStore.set(sessionName, session); + const hoveredRefs: string[] = []; + const provider = makeWebProvider({ + hoverRef: async (ref) => { + hoveredRefs.push(ref); + }, + }); + + const response = await withWebProvider(provider, async () => + handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'hover', + positionals: ['@e1'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + }), + ); + + expect(response).toMatchObject({ + ok: true, + data: { ref: 'e1', gesture: 'hover', message: expect.stringMatching(/Hovered @e1/) }, + }); + expect(hoveredRefs).toEqual(['@e1']); + expect(mockDispatch).not.toHaveBeenCalled(); + expect(sessionStore.get(sessionName)?.actions[0]?.command).toBe('hover'); +}); + +test('hover selector on web resolves the target and dispatches coordinate hover', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'hover-selector'; + const session = makeSession(sessionName); + session.device = WEB_DESKTOP_DEVICE; + sessionStore.set(sessionName, session); + mockCaptureSnapshotForSession.mockResolvedValue({ nodes: attachRefs([ { index: 0, - role: 'listitem', - label: 'Last message', + role: 'link', + label: 'Second message', rect: { x: 10, y: 20, width: 100, height: 40 }, enabled: true, hittable: true, @@ -249,15 +297,14 @@ test('hover @ref resolves the target and dispatches coordinate hover on web', as ]), createdAt: Date.now(), backend: 'web', - }; - sessionStore.set(sessionName, session); + }); const response = await handleInteractionCommands({ req: { token: 't', session: sessionName, command: 'hover', - positionals: ['@e1'], + positionals: ['label="Second message"'], flags: {}, }, sessionName, @@ -267,12 +314,11 @@ test('hover @ref resolves the target and dispatches coordinate hover on web', as expect(response).toMatchObject({ ok: true, - data: { x: 60, y: 40, gesture: 'hover', message: expect.stringMatching(/Hovered @e1/) }, + data: { x: 60, y: 40, gesture: 'hover', message: expect.stringMatching(/Hovered label/) }, }); expect(mockDispatch.mock.calls).toEqual([ [expect.anything(), 'hover', ['60', '40'], undefined, expect.anything()], ]); - expect(sessionStore.get(sessionName)?.actions[0]?.command).toBe('hover'); }); test('hover is refused by capability on touch platforms before any dispatch', async () => { @@ -302,3 +348,18 @@ test('hover is refused by capability on touch platforms before any dispatch', as }); expect(mockDispatch).not.toHaveBeenCalled(); }); + +function makeWebProvider(overrides: Partial): WebProvider { + return { + open: async () => {}, + close: async () => {}, + snapshot: async () => ({ nodes: [] }), + screenshot: async () => {}, + setViewport: async () => {}, + click: async () => {}, + fill: async () => {}, + typeText: async () => {}, + scroll: async () => {}, + ...overrides, + }; +} diff --git a/src/daemon/handlers/interaction-runtime.ts b/src/daemon/handlers/interaction-runtime.ts index 78966b51e8..f9e87158e9 100644 --- a/src/daemon/handlers/interaction-runtime.ts +++ b/src/daemon/handlers/interaction-runtime.ts @@ -168,6 +168,13 @@ function createInteractionBackend( ), ); }, + hoverTarget: webProvider?.hoverRef + ? async (_context, target): Promise => { + expireRefFrame(session); + await webProvider.hoverRef?.(target.ref); + return { ref: stripAtPrefix(target.ref) }; + } + : undefined, hover: async (_context, point): Promise => { expireRefFrame(session); return toBackendActionResult( @@ -208,7 +215,7 @@ function createInteractionBackend( function resolveNativeWebInteractionProvider(session: SessionState): WebProvider | undefined { if (session.device.platform !== 'web') return undefined; const provider = resolveWebProvider(); - return provider.clickRef || provider.fillRef ? provider : undefined; + return provider.clickRef || provider.fillRef || provider.hoverRef ? provider : undefined; } function toBackendActionResult(data: unknown): BackendActionResult { diff --git a/src/platforms/web/agent-browser-provider.test.ts b/src/platforms/web/agent-browser-provider.test.ts index c3683bab02..85b6770054 100644 --- a/src/platforms/web/agent-browser-provider.test.ts +++ b/src/platforms/web/agent-browser-provider.test.ts @@ -47,6 +47,7 @@ test('agent-browser provider maps supported operations to session-scoped JSON co await provider.click(10.4, 20.6); await provider.clickRef?.('@e3'); await provider.hover?.(30.2, 40.7); + await provider.hoverRef?.('@e5'); await provider.fill(11, 22, 'Ada'); await provider.fillRef?.('@e2', 'Grace'); await provider.typeText('hello'); @@ -73,6 +74,7 @@ test('agent-browser provider maps supported operations to session-scoped JSON co ['click', '@e3', '--json', '--session', 'web-session'], // hover is a bare pointer move: no button transition follows (#1783). ['mouse', 'move', '30', '41', '--json', '--session', 'web-session'], + ['hover', '@e5', '--json', '--session', 'web-session'], ['mouse', 'move', '11', '22', '--json', '--session', 'web-session'], ['mouse', 'down', '--json', '--session', 'web-session'], ['mouse', 'up', '--json', '--session', 'web-session'], diff --git a/src/platforms/web/agent-browser-provider.ts b/src/platforms/web/agent-browser-provider.ts index 808fcea85c..dc35db5a49 100644 --- a/src/platforms/web/agent-browser-provider.ts +++ b/src/platforms/web/agent-browser-provider.ts @@ -72,6 +72,9 @@ export function createAgentBrowserWebProvider( async hover(x, y) { await movePointer(runJson, x, y); }, + async hoverRef(ref) { + await runJson(['hover', browserRefSelector(ref)]); + }, async clickRef(ref) { await runJson(['click', browserRefSelector(ref)]); }, diff --git a/src/platforms/web/provider.ts b/src/platforms/web/provider.ts index dbbaf032d1..a5605c5602 100644 --- a/src/platforms/web/provider.ts +++ b/src/platforms/web/provider.ts @@ -56,6 +56,8 @@ export type WebProvider = { clickRef?(ref: string): Promise; /** Move the pointer to a point without pressing, raising the page's hover state. */ hover?(x: number, y: number): Promise; + /** Hover a snapshot ref through the provider's own element handle. */ + hoverRef?(ref: string): Promise; fill(x: number, y: number, text: string, options?: { delayMs?: number }): Promise; fillRef?(ref: string, text: string, options?: { delayMs?: number }): Promise; typeText(text: string, options?: { delayMs?: number }): Promise; diff --git a/test/integration/smoke-android-emulator-coverage.test.ts b/test/integration/smoke-android-emulator-coverage.test.ts index d2a515a475..f4f1626d21 100644 --- a/test/integration/smoke-android-emulator-coverage.test.ts +++ b/test/integration/smoke-android-emulator-coverage.test.ts @@ -47,11 +47,11 @@ test('Android emulator coverage exhaustively classifies the public catalog', () test('Android coverage report summary accounts for every manifest classification', () => { const summary = ANDROID_EMULATOR_COVERAGE_CLASSIFICATION_SUMMARY; assert.deepEqual(summary, { - capabilityDenial: 2, + capabilityDenial: 3, contract: 10, gap: 0, live: 41, - total: 53, + total: 54, }); assert.equal( summary.live + summary.contract + summary.gap + summary.capabilityDenial, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 216f998680..69f9f6d769 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -145,7 +145,7 @@ agent-device close --platform web - The managed install respects `--state-dir` and `AGENT_DEVICE_STATE_DIR`. - Web automation requires Node 24+. - Supported through `agent-device`: URL open, snapshot refs, `get text/attrs`, `is visible/exists/text`, `find text/selector`, click/press, hover, fill/type, wait, `network dump`, `audio probe`, screenshot, close, and replay scripts composed from those commands. -- `hover <@ref|selector|x y>` moves the pointer without pressing so hover-gated UI (row toolbars, menus) appears. Add `--settle` to read what it revealed instead of taking another snapshot. +- `hover <@ref|selector|x y>` moves the pointer without pressing so hover-gated UI (row toolbars, menus) appears. Add `--settle` to read what it revealed instead of taking another snapshot. `hover @ref` hovers the browser's own element handle; like `click @ref --settle`, the `--settle` diff needs a selector or coordinate target on web because web refs carry no geometry. - `audio probe start [durationSeconds] [bucketMs]` samples HTML media elements into compact RMS/peak dBFS buckets while the page keeps running. The first timing positional is seconds; the second is milliseconds. - URL-backed web media may be routed through the probe `AudioContext` while observed. Use `audio probe status` to poll partial buckets and `audio probe stop` to end the probe early. - Out of scope for `agent-device` web support: tab/window/devtools control, network routing/interception/HAR, cookies/storage, downloads/uploads, arbitrary page scripting, multi-page orchestration, and raw browser diagnostics. Use `agent-browser` directly for those browser-specific workflows. From e95941d291096fd7bbe27f55bdfe64a5b2d9b4c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 17 Aug 2026 19:12:17 +0200 Subject: [PATCH 3/4] test: drive hover through the apple output guard; cover direct hover dispatch The provider-integration apple-leak guard partitions every public command into driven/skipped; hover was neither, which failed Integration Tests and took Coverage down with it. Drive it (it reaches the Apple capability refusal, which is scanned like any other error response). Also cover the direct-dispatch handleHoverCommand seam. --- src/core/__tests__/dispatch-hover.test.ts | 50 +++++++++++++++++++ .../apple-platform-output-guard.test.ts | 2 + 2 files changed, 52 insertions(+) create mode 100644 src/core/__tests__/dispatch-hover.test.ts diff --git a/src/core/__tests__/dispatch-hover.test.ts b/src/core/__tests__/dispatch-hover.test.ts new file mode 100644 index 0000000000..49153bc537 --- /dev/null +++ b/src/core/__tests__/dispatch-hover.test.ts @@ -0,0 +1,50 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import type { Interactor } from '@agent-device/contracts/interaction'; +import { handleHoverCommand } from '../dispatch-interactions.ts'; + +// #1783: the direct-dispatch hover handler (the core seam the daemon's web +// backend calls, and what `agent-device hover x y` reaches without a session). + +test('dispatch hover moves the pointer through the interactor and reports the point', async () => { + const calls: Array<[number, number]> = []; + const interactor = { + hover: async (x: number, y: number) => { + calls.push([x, y]); + }, + } as unknown as Interactor; + + const result = await handleHoverCommand(interactor, ['155', '172']); + + assert.deepEqual(calls, [[155, 172]]); + assert.equal(result.x, 155); + assert.equal(result.y, 172); + assert.match(String(result.message), /Hovered \(155, 172\)/); +}); + +test('dispatch hover refuses platforms whose interactor has no pointer', async () => { + const interactor = {} as unknown as Interactor; + + await assert.rejects( + () => handleHoverCommand(interactor, ['10', '10']), + (error: unknown) => + error instanceof AppError && + error.code === 'UNSUPPORTED_OPERATION' && + /hover is not supported on this platform/i.test(error.message) && + /web targets only/i.test(String(error.details?.hint)), + ); +}); + +test('dispatch hover explains the direct platform coordinate requirement', async () => { + const interactor = { hover: async () => {} } as unknown as Interactor; + + await assert.rejects( + () => handleHoverCommand(interactor, ['@e40']), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /hover requires x y/i.test(error.message) && + /open daemon session/i.test(String(error.details?.hint)), + ); +}); diff --git a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts index b7c82c2578..2599aead48 100644 --- a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts +++ b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts @@ -113,6 +113,8 @@ const DRIVEN_COMMANDS: Record = { [PUBLIC_COMMANDS.click]: () => one(['10', '10']), [PUBLIC_COMMANDS.fill]: () => one(['label=General', 'hello']), [PUBLIC_COMMANDS.longPress]: () => one(['10', '10']), + // Web-only: reaches the capability refusal on Apple; the error must not leak either. + [PUBLIC_COMMANDS.hover]: () => one(['10', '10']), [PUBLIC_COMMANDS.press]: () => one(['10', '10']), [PUBLIC_COMMANDS.type]: () => one(['hello']), [PUBLIC_COMMANDS.back]: () => one(), From 134115c1fd7630787c50107fb43c04b98bab777f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 17 Aug 2026 19:23:54 +0200 Subject: [PATCH 4/4] test(web): drive hover @ref in the provider-backed web scenario The integration-progress gate requires every public command to be referenced by a provider-backed scenario. Add hover @ref to the web desktop flow: it must reach the provider's hoverRef handle (never a coordinate) and be recorded on the session without fabricated x/y, like click @ref. --- .../provider-scenarios/web-desktop.test.ts | 32 +++++++++++++++++++ .../provider-scenarios/web-world.ts | 6 ++++ 2 files changed, 38 insertions(+) diff --git a/test/integration/provider-scenarios/web-desktop.test.ts b/test/integration/provider-scenarios/web-desktop.test.ts index e3970f1aa6..bcf98d04e6 100644 --- a/test/integration/provider-scenarios/web-desktop.test.ts +++ b/test/integration/provider-scenarios/web-desktop.test.ts @@ -82,6 +82,27 @@ test('Provider-backed integration web desktop flow uses semantic web provider ca positionals: ['text', 'Ready', '100'], expectData: { text: 'Ready' }, }, + { + // #1783: hover @ref on web rides the provider element handle + // (hoverRef), never a coordinate — web ref frames carry no rects. + name: 'hover submit ref', + command: 'hover', + positionals: ['@e4'], + expectData: { ref: 'e4', gesture: 'hover' }, + assert: (response) => { + const data = response.json?.result?.data; + assert.equal(data?.x, undefined); + assert.equal(data?.y, undefined); + assert.equal(data?.message, 'Hovered @e4'); + }, + }, + { + // ADR 0014: the ref hover above expired the frame; re-observe + // before the next ref mutation. + name: 're-observe before the ref click', + command: 'snapshot', + flags: { snapshotInteractiveOnly: true }, + }, { name: 'click submit ref', command: 'click', @@ -149,6 +170,16 @@ test('Provider-backed integration web desktop flow uses semantic web provider ca ]); const actions = daemon.session()?.actions ?? []; + assert.ok( + actions.some( + (action) => + action.command === 'hover' && + action.positionals.join(' ') === '@e4' && + action.result?.x === undefined && + action.result?.y === undefined, + ), + 'Expected ref hover action to be recorded on the session without fabricated coordinates', + ); assert.ok( actions.some( (action) => @@ -183,6 +214,7 @@ test('Provider-backed integration web desktop flow uses semantic web provider ca assertFlatToolCall(semanticCalls, ['web', 'open', WEB_URL, '']); assertFlatToolCall(semanticCalls, ['web', 'recordStart', recordingPath]); assertFlatToolCall(semanticCalls, ['web', 'snapshot', 'true', '']); + assertFlatToolCall(semanticCalls, ['web', 'hoverRef', '@e4']); assertFlatToolCall(semanticCalls, ['web', 'clickRef', '@e4']); assertFlatToolCall(semanticCalls, ['web', 'fillRef', '@e3', 'qa@example.test', '1']); assertFlatToolCall(semanticCalls, ['web', 'type', ' ok', '0']); diff --git a/test/integration/provider-scenarios/web-world.ts b/test/integration/provider-scenarios/web-world.ts index 9df49d28c8..ae4bd5eeb7 100644 --- a/test/integration/provider-scenarios/web-world.ts +++ b/test/integration/provider-scenarios/web-world.ts @@ -83,6 +83,12 @@ export async function createWebDesktopWorld(): Promise { state.statusText = 'Submitted'; } }, + hover: async (x, y) => { + semanticCalls.push(['web', 'hover', String(x), String(y)]); + }, + hoverRef: async (ref) => { + semanticCalls.push(['web', 'hoverRef', ref]); + }, fill: async (x, y, text, options) => { semanticCalls.push([ 'web',