From 428b8c9b62b30d202c06b5a3b568cb03045d4851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 21 Aug 2026 09:47:37 +0200 Subject: [PATCH] feat(ios): publish effective snapshot geometry --- .github/workflows/ios.yml | 4 + CHANGELOG.md | 4 + .../RunnerTests+SnapshotPresentation.swift | 209 ++++-------------- ...erTests+SnapshotPresentationGeometry.swift | 61 +++++ ...ts+SnapshotPresentationGeometryTests.swift | 30 +++ ...nnerTests+SnapshotPresentationModels.swift | 148 +++++++++++++ ...unnerTests+SnapshotPresentationTests.swift | 55 +++++ .../RunnerTests+SnapshotVisibilityFold.swift | 108 ++++----- ...nerTests+SnapshotVisibilityFoldTests.swift | 55 +++-- .../adr/0004-ios-snapshot-backend-strategy.md | 5 +- 10 files changed, 451 insertions(+), 228 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationGeometry.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationGeometryTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationModels.swift diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 6f2806b581..7e02d580c5 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -151,6 +151,7 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationPreservesCurrentWireShape \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationPublishesEffectiveRectWhileRawKeepsReportedFrame \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationOwnsBackendNeutralEligibility \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationOwnsScopeAndRelativeDepth \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPresentationRefusesAnAcquisitionCapturedForTheOtherProjection \ @@ -159,6 +160,9 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldKeepsWindowCarriersButNeverHittableOutsideClip \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldDropsSubPixelContentlessDecorationOnEveryBackend \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPlainViewportPolicyFoldsWithoutAncestorCursor \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPlainViewportPolicyDoesNotClipToScrollAncestor \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testEffectiveGeometryIntersectsViewportAndAncestorClip \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testEffectiveGeometryKeepsReportedOriginWhenFullyClipped \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationRoutesThroughVisibilityFold \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotScopePolicyMatchesGoldenParityTable \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollContainerTypeNamesMatchElementTypeSet \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 382f9bc05c..dc4d9c7134 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- iOS regular snapshot nodes now publish presentation-owned effective geometry through the existing + `rect` field: backend-reported frames remain available to acquisition, while regular output uses + the viewport and declared scroll-clip intersection. Raw snapshots and direct element reads retain + reported geometry (#1797). - Android recording-session commands now warn when blocking-dialog readiness inspection fails open. The requested command still runs, but the successful response discloses that readiness could not be inspected and preserves the inspection error's actionable hint, including `pnpm build:android` when the snapshot helper is unavailable (#1895). - Breaking (device selection): when a command needs one concrete device, carries no `--device`/`--udid`/`--serial`, and more than one candidate is equally preferred, agent-device now refuses with `AMBIGUOUS_MATCH` and the candidate list instead of quietly picking one. Established preferences are unchanged — virtual over physical, booted over offline, and the Apple kind/target ranking — so a single booted emulator beside offline ones still resolves, an existing session binding still resolves (its identity is already fixed), and explicit selectors still resolve. What no longer happens is choosing between two equally booted devices by discovery order or alphabetically: that produced a **successful response describing a device the caller never selected**, and reads are no safer than writes there. `devices` and other genuinely multi-device commands never enter singular resolution and are unaffected. The error carries the bounded candidate list in the declared `devices` details domain, so CLI and MCP print it, with a hint naming the right selector for the platform (`--serial` for Android/HarmonyOS, `--udid` for Apple, or `--device ""`). - `--udid` with `--platform android` (and `--serial` with an Apple platform) now fails as the flag mistake it is — `INVALID_ARGS` naming the right flag — instead of reaching device resolution and answering `No Apple device with UDID emulator-5580` for an explicitly Android request. `--udid` addresses Apple devices, `--serial` addresses Android and HarmonyOS; matching pairs and requests that name no platform are unchanged. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift index 6f9538cbe4..65f8e21ac6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentation.swift @@ -1,134 +1,5 @@ import Foundation -/// Backend-owned snapshot output before it crosses the presentation seam. -/// -/// The incremental #1797 migration still carries derived fields that later semantic layers move -/// behind `SnapshotPresentation`. Eligibility is no longer one of those backend-owned decisions. -struct RawAXNode { - let index: Int - let type: String - let label: String? - let identifier: String? - let value: String? - let rect: SnapshotRect - let enabled: Bool - let focused: Bool? - let selected: Bool? - let hittable: Bool - let depth: Int - let parentIndex: Int? - let hiddenContentAbove: Bool? - let hiddenContentBelow: Bool? - var actions: [String]? = nil - - var hasSemanticContent: Bool { - [label, identifier, value].contains { - !($0?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ?? true) - } - } -} - -/// The acquisition-facing view of a snapshot request, derived once by -/// `SnapshotPresentation.captureHint(for:)`. -/// -/// Backends read a hint, never `PresentationOptions`: presentation owns interpretation, and a hint -/// may narrow acquisition only where the backend can prove the narrowing complete for the requested -/// projection (#1797 conservatism). Everything else a hint carries is budget and ordering. -struct CaptureHint { - /// Which projection this acquisition must serve. Backends that cannot serve one are not planned - /// for it (`SnapshotBackendKind.supportsRawProjection`), and presentation refuses an acquisition - /// captured for the other projection rather than relabeling it, so a `--raw` request can never be - /// answered with regular-projection membership (#1797 D4). - enum Projection: String { - case regular - case raw - } - - let projection: Projection - /// Traversal-depth budget. - /// - /// Declared residue (#1797): complete for the raw projection, whose presented depth *is* - /// traversal depth. Regular presentation emits collapsed depth, so cutting the traversal at this - /// limit can still drop a node that would have presented within it — the open "visible-depth - /// frontier completeness" obligation, kept as-is here because the cut is also what keeps - /// `--depth 1` probes cheap. - let depth: Int? - /// Regular-projection acquisition budget. The raw projection is the acquired tree, so it never - /// carries this: `--raw -i` returns everything the backend serialized. - let interactiveOnly: Bool - let customActions: Bool - - var isRaw: Bool { projection == .raw } -} - -/// One backend attempt after acquisition and its current backend-specific interpretation. -/// -/// It is the only input snapshot presentation accepts, and it carries the hint it was captured -/// under so the two sides of the seam cannot disagree about which projection this is. Later #1797 -/// steps move the interpretation that still precedes this value — the clip fold and hittability — -/// into `SnapshotPresentation` without changing the capture-plan seam. -struct SnapshotAcquisition { - /// The hint this acquisition was captured under. Presentation compares it with the requested - /// projection instead of trusting the backend's label. - let hint: CaptureHint - let nodes: [RawAXNode] - let truncated: Bool - let effectiveDepth: Int? - var customActions: SnapshotCustomActionCoverage? = nil - /// Viewport the regular projection's clip fold runs against. `.infinite` disables the fold -- - /// legitimate only for raw acquisitions and depth-0 probes, where no fold applies. - let viewport: CGRect -} - -/// The only snapshot node shape accepted by response payload assembly. -/// -/// Its initializer is private so a backend cannot bypass `SnapshotPresentation`. The encoded shape -/// intentionally remains byte-for-byte compatible with the former `SnapshotNode` wire model. -struct PresentedNode: Codable { - let index: Int - let type: String - let label: String? - let identifier: String? - let value: String? - let rect: SnapshotRect - let enabled: Bool - let focused: Bool? - let selected: Bool? - let hittable: Bool - let depth: Int - let parentIndex: Int? - let hiddenContentAbove: Bool? - let hiddenContentBelow: Bool? - let actions: [String]? - - fileprivate init(presenting raw: RawAXNode) { - self.init( - presenting: raw, - index: raw.index, - depth: raw.depth, - parentIndex: raw.parentIndex - ) - } - - fileprivate init(presenting raw: RawAXNode, index: Int, depth: Int, parentIndex: Int?) { - self.index = index - type = raw.type - label = raw.label - identifier = raw.identifier - value = raw.value - rect = raw.rect - enabled = raw.enabled - focused = raw.focused - selected = raw.selected - hittable = raw.hittable - self.depth = depth - self.parentIndex = parentIndex - hiddenContentAbove = raw.hiddenContentAbove - hiddenContentBelow = raw.hiddenContentBelow - actions = raw.actions - } -} - enum SnapshotPresentation { private static let eligibleInteractiveTypes: Set = [ "Button", @@ -207,7 +78,12 @@ enum SnapshotPresentation { _ acquisition: SnapshotAcquisition, options: PresentationOptions ) -> SnapshotBackendCapture { - project(acquisition.nodes, acquisition: acquisition, options: options, projection: .raw) + project( + acquisition.nodes.map(SnapshotPresentationNode.reported), + acquisition: acquisition, + options: options, + projection: .raw + ) } /// Derives the one acquisition-facing view of a request, so no backend re-reads @@ -227,7 +103,7 @@ enum SnapshotPresentation { } private static func project( - _ projectionNodes: [RawAXNode], + _ projectionNodes: [SnapshotPresentationNode], acquisition: SnapshotAcquisition, options: PresentationOptions, projection: CaptureHint.Projection @@ -258,16 +134,17 @@ enum SnapshotPresentation { } private static func presentedNodes( - from rawNodes: [RawAXNode], + from rawNodes: [SnapshotPresentationNode], projection: CaptureHint.Projection ) -> [PresentedNode] { if projection == .raw { - return rawNodes.map(PresentedNode.init(presenting:)) + return rawNodes.map { PresentedNode(presenting: $0) } } var nodes: [PresentedNode] = [] var nearestPresentedNodeByRawIndex: [Int: (index: Int, depth: Int)] = [:] - for raw in rawNodes { + for node in rawNodes { + let raw = node.raw let presentedParent = raw.parentIndex.flatMap { nearestPresentedNodeByRawIndex[$0] } @@ -284,6 +161,7 @@ enum SnapshotPresentation { nodes.append( PresentedNode( presenting: raw, + rect: node.effectiveRect, index: presentedIndex, depth: presentedDepth, parentIndex: presentedParent?.index @@ -294,17 +172,18 @@ enum SnapshotPresentation { } private static func applyScope( - to rawNodes: [RawAXNode], + to rawNodes: [SnapshotPresentationNode], options: PresentationOptions, projection: CaptureHint.Projection - ) -> [RawAXNode] { + ) -> [SnapshotPresentationNode] { switch SnapshotScopePolicy.select( fromPreorder: rawNodes, scope: options.scope, - depth: \.depth, - semanticValues: { [$0.label, $0.identifier, $0.value] }, + depth: { $0.raw.depth }, + semanticValues: { [$0.raw.label, $0.raw.identifier, $0.raw.value] }, subtreeContributes: { range in - projection == .raw || rawNodes[range].contains(where: isEligibleForRegularPresentation) + projection == .raw + || rawNodes[range].contains { isEligibleForRegularPresentation($0.raw) } } ) { case .unscoped: @@ -312,39 +191,47 @@ enum SnapshotPresentation { case .missing: return [] case .matched(let startIndex): - let startDepth = rawNodes[startIndex].depth + let startDepth = rawNodes[startIndex].raw.depth let range = SnapshotScopePolicy.subtreeRange( from: startIndex, in: rawNodes, - depth: \.depth + depth: { $0.raw.depth } ) let maxDepth = options.depth ?? Int.max return reindex( - Array(rawNodes[range]).filter { $0.depth - startDepth <= maxDepth }, + Array(rawNodes[range]).filter { $0.raw.depth - startDepth <= maxDepth }, depthOffset: startDepth ) } } - private static func reindex(_ rawNodes: [RawAXNode], depthOffset: Int) -> [RawAXNode] { - let indexMap = Dictionary(uniqueKeysWithValues: rawNodes.enumerated().map { ($0.element.index, $0.offset) }) - return rawNodes.enumerated().map { offset, raw in - RawAXNode( - index: offset, - type: raw.type, - label: raw.label, - identifier: raw.identifier, - value: raw.value, - rect: raw.rect, - enabled: raw.enabled, - focused: raw.focused, - selected: raw.selected, - hittable: raw.hittable, - depth: max(0, raw.depth - depthOffset), - parentIndex: raw.parentIndex.flatMap { indexMap[$0] }, - hiddenContentAbove: raw.hiddenContentAbove, - hiddenContentBelow: raw.hiddenContentBelow, - actions: raw.actions + private static func reindex( + _ nodes: [SnapshotPresentationNode], + depthOffset: Int + ) -> [SnapshotPresentationNode] { + let indexMap = Dictionary( + uniqueKeysWithValues: nodes.enumerated().map { ($0.element.raw.index, $0.offset) }) + return nodes.enumerated().map { offset, node in + let raw = node.raw + return SnapshotPresentationNode( + raw: RawAXNode( + index: offset, + type: raw.type, + label: raw.label, + identifier: raw.identifier, + value: raw.value, + rect: raw.rect, + enabled: raw.enabled, + focused: raw.focused, + selected: raw.selected, + hittable: raw.hittable, + depth: max(0, raw.depth - depthOffset), + parentIndex: raw.parentIndex.flatMap { indexMap[$0] }, + hiddenContentAbove: raw.hiddenContentAbove, + hiddenContentBelow: raw.hiddenContentBelow, + actions: raw.actions + ), + effectiveRect: node.effectiveRect ) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationGeometry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationGeometry.swift new file mode 100644 index 0000000000..50e8664258 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationGeometry.swift @@ -0,0 +1,61 @@ +import Foundation + +/// The typed carrier between acquisition facts and snapshot wire projection. +/// +/// `raw.rect` remains the backend-reported frame for runner-internal traversal and deduplication. +/// `effectiveRect` is the geometry after the regular presentation policy. Raw projection sets both +/// to the reported frame, so callers cannot accidentally publish an unclassified geometry source. +struct SnapshotPresentationNode { + let raw: RawAXNode + let effectiveRect: SnapshotRect + + init(raw: RawAXNode, effectiveRect: SnapshotRect) { + self.raw = raw + self.effectiveRect = effectiveRect + } + + static func reported(_ raw: RawAXNode) -> Self { + Self(raw: raw, effectiveRect: raw.rect) + } +} + +/// Geometry conversion owned by presentation. Acquisition reports frames; this type computes the +/// only rectangle that can cross the regular snapshot wire boundary. +enum SnapshotGeometry { + static func effectiveFrame( + reportedFrame: CGRect, + viewport: CGRect, + ancestorClip: CGRect? + ) -> CGRect { + var frame = reportedFrame + if !viewport.isInfinite { + frame = clipped(frame, to: viewport) + } + if let ancestorClip { + frame = clipped(frame, to: ancestorClip) + } + return frame + } + + static func snapshotRect(from frame: CGRect, reportedFrame: CGRect) -> SnapshotRect { + guard !frame.isNull, !frame.isEmpty else { + return SnapshotRect( + x: Double(reportedFrame.minX), y: Double(reportedFrame.minY), width: 0, height: 0) + } + return SnapshotRect( + x: Double(frame.origin.x), + y: Double(frame.origin.y), + width: Double(max(0, frame.size.width)), + height: Double(max(0, frame.size.height)) + ) + } + + private static func clipped(_ frame: CGRect, to clip: CGRect) -> CGRect { + guard !frame.isNull, !frame.isEmpty else { return frame } + let intersection = frame.intersection(clip) + guard !intersection.isNull, !intersection.isEmpty else { + return CGRect(x: frame.minX, y: frame.minY, width: 0, height: 0) + } + return intersection + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationGeometryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationGeometryTests.swift new file mode 100644 index 0000000000..20d46a2619 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationGeometryTests.swift @@ -0,0 +1,30 @@ +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +import XCTest + +extension RunnerTests { + func testEffectiveGeometryIntersectsViewportAndAncestorClip() { + let effective = SnapshotGeometry.effectiveFrame( + reportedFrame: CGRect(x: 350, y: 80, width: 100, height: 100), + viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + ancestorClip: CGRect(x: 300, y: 100, width: 80, height: 80) + ) + + XCTAssertEqual(effective, CGRect(x: 350, y: 100, width: 30, height: 80)) + } + + func testEffectiveGeometryKeepsReportedOriginWhenFullyClipped() { + let reported = CGRect(x: 500, y: 120, width: 100, height: 44) + let effective = SnapshotGeometry.effectiveFrame( + reportedFrame: reported, + viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + ancestorClip: nil + ) + + let rect = SnapshotGeometry.snapshotRect(from: effective, reportedFrame: reported) + XCTAssertEqual(rect.x, 500) + XCTAssertEqual(rect.y, 120) + XCTAssertEqual(rect.width, 0) + XCTAssertEqual(rect.height, 0) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationModels.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationModels.swift new file mode 100644 index 0000000000..6f85eb76ed --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationModels.swift @@ -0,0 +1,148 @@ +import Foundation + +/// Backend-owned snapshot output before it crosses the presentation seam. +/// +/// The incremental #1797 migration still carries derived fields that later semantic layers move +/// behind `SnapshotPresentation`. Eligibility is no longer one of those backend-owned decisions. +struct RawAXNode { + let index: Int + let type: String + let label: String? + let identifier: String? + let value: String? + let rect: SnapshotRect + let enabled: Bool + let focused: Bool? + let selected: Bool? + let hittable: Bool + let depth: Int + let parentIndex: Int? + let hiddenContentAbove: Bool? + let hiddenContentBelow: Bool? + var actions: [String]? = nil + + var hasSemanticContent: Bool { + [label, identifier, value].contains { + !($0?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ?? true) + } + } +} + +/// The acquisition-facing view of a snapshot request, derived once by +/// `SnapshotPresentation.captureHint(for:)`. +/// +/// Backends read a hint, never `PresentationOptions`: presentation owns interpretation, and a hint +/// may narrow acquisition only where the backend can prove the narrowing complete for the requested +/// projection (#1797 conservatism). Everything else a hint carries is budget and ordering. +struct CaptureHint { + /// Which projection this acquisition must serve. Backends that cannot serve one are not planned + /// for it, and presentation refuses an acquisition captured for the other projection rather + /// than relabeling it. + enum Projection: String { + case regular + case raw + } + + let projection: Projection + /// Traversal-depth budget. Regular presentation's collapsed depth retains the visible-depth + /// frontier residue described by #1797; this cut stays cheap for depth probes. + let depth: Int? + /// Regular-projection acquisition budget. Raw projection is the acquired tree and never carries + /// this narrowing. + let interactiveOnly: Bool + let customActions: Bool + + var isRaw: Bool { projection == .raw } +} + +/// One backend attempt after acquisition and its current backend-specific interpretation. +/// +/// It is the only input snapshot presentation accepts, and it carries the hint it was captured +/// under so the two sides of the seam cannot disagree about which projection this is. +struct SnapshotAcquisition { + let hint: CaptureHint + let nodes: [RawAXNode] + let truncated: Bool + let effectiveDepth: Int? + var customActions: SnapshotCustomActionCoverage? = nil + /// Viewport the regular projection's clip fold runs against. `.infinite` disables the fold for + /// raw acquisitions and depth-0 probes. + let viewport: CGRect +} + +/// The only snapshot node shape accepted by response payload assembly. +/// +/// It has no memberwise initializer, so response assembly has to choose one of the explicit +/// presentation constructors. The encoded shape intentionally remains byte-for-byte compatible +/// with the former `SnapshotNode` wire model. +struct PresentedNode: Codable { + let index: Int + let type: String + let label: String? + let identifier: String? + let value: String? + let rect: SnapshotRect + let enabled: Bool + let focused: Bool? + let selected: Bool? + let hittable: Bool + let depth: Int + let parentIndex: Int? + let hiddenContentAbove: Bool? + let hiddenContentBelow: Bool? + let actions: [String]? + + init(presenting raw: RawAXNode) { + self.init( + presenting: raw, + rect: raw.rect, + index: raw.index, + depth: raw.depth, + parentIndex: raw.parentIndex + ) + } + + init(presenting node: SnapshotPresentationNode) { + self.init( + presenting: node.raw, + rect: node.effectiveRect, + index: node.raw.index, + depth: node.raw.depth, + parentIndex: node.raw.parentIndex + ) + } + + init(presenting raw: RawAXNode, index: Int, depth: Int, parentIndex: Int?) { + self.init( + presenting: raw, + rect: raw.rect, + index: index, + depth: depth, + parentIndex: parentIndex + ) + } + + init( + presenting raw: RawAXNode, + rect: SnapshotRect, + index: Int, + depth: Int, + parentIndex: Int? + ) { + self.index = index + type = raw.type + label = raw.label + identifier = raw.identifier + value = raw.value + self.rect = rect + enabled = raw.enabled + focused = raw.focused + selected = raw.selected + hittable = raw.hittable + self.depth = depth + self.parentIndex = parentIndex + hiddenContentAbove = raw.hiddenContentAbove + hiddenContentBelow = raw.hiddenContentBelow + actions = raw.actions + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationTests.swift index 0bae0f2247..d97b2ad453 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotPresentationTests.swift @@ -182,6 +182,61 @@ extension RunnerTests { XCTAssertEqual(presented?.compactMap(\.label), ["App"]) } + func testRegularPresentationPublishesEffectiveRectWhileRawKeepsReportedFrame() throws { + let acquired = [ + RawAXNode( + index: 0, type: "Application", label: "App", identifier: nil, value: nil, + rect: SnapshotRect(x: 0, y: 0, width: 402, height: 874), enabled: true, + focused: nil, selected: nil, hittable: false, depth: 0, parentIndex: nil, + hiddenContentAbove: nil, hiddenContentBelow: nil), + RawAXNode( + index: 1, type: "Button", label: "Partially clipped", identifier: nil, value: nil, + rect: SnapshotRect(x: 350, y: 120, width: 100, height: 44), enabled: true, + focused: nil, selected: nil, hittable: true, depth: 1, parentIndex: 0, + hiddenContentAbove: nil, hiddenContentBelow: nil), + ] + let viewport = CGRect(x: 0, y: 0, width: 402, height: 874) + let regularOptions = PresentationOptions( + interactiveOnly: false, depth: nil, scope: nil, raw: false) + let regular = try XCTUnwrap( + SnapshotPresentation.presentRegular( + SnapshotAcquisition( + hint: SnapshotPresentation.captureHint(for: regularOptions), + nodes: acquired, + truncated: false, + effectiveDepth: nil, + viewport: viewport + ), + options: regularOptions + ).payload.nodes + ) + let regularButton = try XCTUnwrap(regular.first { $0.label == "Partially clipped" }) + XCTAssertEqual(regularButton.rect.x, 350) + XCTAssertEqual(regularButton.rect.y, 120) + XCTAssertEqual(regularButton.rect.width, 52) + XCTAssertEqual(regularButton.rect.height, 44) + + let rawOptions = PresentationOptions( + interactiveOnly: false, depth: nil, scope: nil, raw: true) + let raw = try XCTUnwrap( + SnapshotPresentation.presentRaw( + SnapshotAcquisition( + hint: SnapshotPresentation.captureHint(for: rawOptions), + nodes: acquired, + truncated: false, + effectiveDepth: nil, + viewport: .infinite + ), + options: rawOptions + ).payload.nodes + ) + let rawButton = try XCTUnwrap(raw.first { $0.label == "Partially clipped" }) + XCTAssertEqual(rawButton.rect.x, 350) + XCTAssertEqual(rawButton.rect.y, 120) + XCTAssertEqual(rawButton.rect.width, 100) + XCTAssertEqual(rawButton.rect.height, 44) + } + func testSnapshotPresentationOwnsScopeAndRelativeDepth() throws { // Non-vacuity: disconnecting applyScope produces eight scope, depth, and raw-projection failures. func node( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFold.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFold.swift index 69dd658c40..3bab16e8d3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFold.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFold.swift @@ -83,7 +83,7 @@ enum SnapshotVisibilityFold { viewport: CGRect, interactiveOnly: Bool, policy: Policy - ) -> [RawAXNode] { + ) -> [SnapshotPresentationNode] { var hasChildren = [Bool](repeating: false, count: nodes.count) for node in nodes { if let parentIndex = node.parentIndex, parentIndex >= 0, parentIndex < nodes.count { @@ -92,17 +92,22 @@ enum SnapshotVisibilityFold { } var states = [BranchState?](repeating: nil, count: nodes.count) - var kept: [RawAXNode] = [] + var kept: [SnapshotPresentationNode] = [] var hints: [Int: (above: Bool, below: Bool)] = [:] for (offset, node) in nodes.enumerated() { let parentState = node.parentIndex.flatMap { states[$0] } let parentCursor = parentState?.cursor ?? .root - let parentAnchor = parentState?.anchor + let parentAnchor = policy == .cursorProjected ? parentState?.anchor : nil let rect = CGRect( x: node.rect.x, y: node.rect.y, width: node.rect.width, height: node.rect.height ) - let intersects = intersectsClip(rect, viewport: viewport, scrollAnchor: parentAnchor) + let effectiveFrame = SnapshotGeometry.effectiveFrame( + reportedFrame: rect, + viewport: viewport, + ancestorClip: parentAnchor?.rect + ) + let intersects = !effectiveFrame.isNull && !effectiveFrame.isEmpty let transition = projectionTransition( frame: rect, intersectsClip: intersects, @@ -136,22 +141,26 @@ enum SnapshotVisibilityFold { let outIndex = kept.count let outDepth = keptDepth + 1 kept.append( - RawAXNode( - index: outIndex, - type: node.type, - label: node.label, - identifier: node.identifier, - value: node.value, - rect: node.rect, - enabled: node.enabled, - focused: node.focused, - selected: node.selected, - hittable: node.hittable && intersects, - depth: outDepth, - parentIndex: keptIndex, - hiddenContentAbove: node.hiddenContentAbove, - hiddenContentBelow: node.hiddenContentBelow, - actions: node.actions + SnapshotPresentationNode( + raw: RawAXNode( + index: outIndex, + type: node.type, + label: node.label, + identifier: node.identifier, + value: node.value, + rect: node.rect, + enabled: node.enabled, + focused: node.focused, + selected: node.selected, + hittable: node.hittable && intersects, + depth: outDepth, + parentIndex: keptIndex, + hiddenContentAbove: node.hiddenContentAbove, + hiddenContentBelow: node.hiddenContentBelow, + actions: node.actions + ), + effectiveRect: SnapshotGeometry.snapshotRect( + from: effectiveFrame, reportedFrame: rect) ) ) keptIndex = outIndex @@ -159,12 +168,13 @@ enum SnapshotVisibilityFold { } var anchor = parentAnchor - if include, + if policy == .cursorProjected, + include, let newAnchor = scrollContainerAnchor( forTypeName: node.type, hasChildren: hasChildren[offset], visible: intersects, - frame: rect, + frame: effectiveFrame, nodeIndex: keptIndex ) { @@ -224,16 +234,6 @@ enum SnapshotVisibilityFold { return visibilityExemptCarrierTypes.contains(node.type) || visible } - private static func intersectsClip( - _ rect: CGRect, - viewport: CGRect, - scrollAnchor: (index: Int, rect: CGRect)? - ) -> Bool { - guard !rect.isNull, !rect.isEmpty, rect.intersects(viewport) else { return false } - guard let scrollAnchor else { return true } - return rect.intersects(scrollAnchor.rect) - } - private static func scrollContainerAnchor( forTypeName typeName: String, hasChildren: Bool, @@ -267,27 +267,31 @@ enum SnapshotVisibilityFold { private static func applyHiddenContentHints( _ hints: [Int: (above: Bool, below: Bool)], - to nodes: [RawAXNode] - ) -> [RawAXNode] { + to nodes: [SnapshotPresentationNode] + ) -> [SnapshotPresentationNode] { if hints.isEmpty { return nodes } - return nodes.map { node in - guard let hint = hints[node.index] else { return node } - return RawAXNode( - index: node.index, - type: node.type, - label: node.label, - identifier: node.identifier, - value: node.value, - rect: node.rect, - enabled: node.enabled, - focused: node.focused, - selected: node.selected, - hittable: node.hittable, - depth: node.depth, - parentIndex: node.parentIndex, - hiddenContentAbove: node.hiddenContentAbove == true || hint.above ? true : nil, - hiddenContentBelow: node.hiddenContentBelow == true || hint.below ? true : nil, - actions: node.actions + return nodes.map { presentationNode in + let node = presentationNode.raw + guard let hint = hints[node.index] else { return presentationNode } + return SnapshotPresentationNode( + raw: RawAXNode( + index: node.index, + type: node.type, + label: node.label, + identifier: node.identifier, + value: node.value, + rect: node.rect, + enabled: node.enabled, + focused: node.focused, + selected: node.selected, + hittable: node.hittable, + depth: node.depth, + parentIndex: node.parentIndex, + hiddenContentAbove: node.hiddenContentAbove == true || hint.above ? true : nil, + hiddenContentBelow: node.hiddenContentBelow == true || hint.below ? true : nil, + actions: node.actions + ), + effectiveRect: presentationNode.effectiveRect ) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFoldTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFoldTests.swift index aa12fce447..f57639d4fa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFoldTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotVisibilityFoldTests.swift @@ -23,7 +23,7 @@ extension RunnerTests { viewport: CGRect, interactiveOnly: Bool = false, policy: SnapshotVisibilityFold.Policy = .cursorProjected - ) -> [RawAXNode] { + ) -> [SnapshotPresentationNode] { SnapshotVisibilityFold.fold( nodes, viewport: viewport, interactiveOnly: interactiveOnly, policy: policy) } @@ -54,12 +54,14 @@ extension RunnerTests { let folded = Self.folded(nodes, viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) XCTAssertEqual( - folded.map(\.type), ["Application", "ScrollView", "Cell", "StaticText", "StaticText"]) + folded.map { $0.raw.type }, + ["Application", "ScrollView", "Cell", "StaticText", "StaticText"]) XCTAssertEqual( - folded.compactMap(\.label), ["App", "Visible row", "Detail", "Visible overlay"]) - XCTAssertEqual(folded.map(\.depth), [0, 1, 2, 3, 2]) - XCTAssertEqual(folded.map(\.parentIndex), [nil, 0, 1, 2, 1]) - XCTAssertEqual(folded.first { $0.type == "ScrollView" }?.hiddenContentBelow, true) + folded.compactMap { $0.raw.label }, + ["App", "Visible row", "Detail", "Visible overlay"]) + XCTAssertEqual(folded.map { $0.raw.depth }, [0, 1, 2, 3, 2]) + XCTAssertEqual(folded.map { $0.raw.parentIndex }, [nil, 0, 1, 2, 1]) + XCTAssertEqual(folded.first { $0.raw.type == "ScrollView" }?.raw.hiddenContentBelow, true) } func testRegularFoldKeepsWindowCarriersButNeverHittableOutsideClip() { @@ -75,9 +77,13 @@ extension RunnerTests { ] let folded = Self.folded(nodes, viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) - XCTAssertEqual(folded.map(\.type), ["Application", "Window"]) - XCTAssertEqual(folded.last?.hittable, false) - XCTAssertFalse(folded.contains { $0.label == "Gone" }) + XCTAssertEqual(folded.map { $0.raw.type }, ["Application", "Window"]) + XCTAssertEqual(folded.last?.raw.hittable, false) + XCTAssertEqual(folded.last?.effectiveRect.x, 402) + XCTAssertEqual(folded.last?.effectiveRect.y, 0) + XCTAssertEqual(folded.last?.effectiveRect.width, 0) + XCTAssertEqual(folded.last?.effectiveRect.height, 0) + XCTAssertFalse(folded.contains { $0.raw.label == "Gone" }) } func testRegularFoldDropsSubPixelContentlessDecorationOnEveryBackend() { @@ -95,9 +101,11 @@ extension RunnerTests { ] let folded = Self.folded(nodes, viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) - XCTAssertEqual(folded.compactMap(\.label), ["App", "Hairline caption", "Frameless semantics"]) - XCTAssertFalse(folded.contains { $0.type == "Button" }) - XCTAssertEqual(folded.first { $0.label == "Frameless semantics" }?.hittable, false) + XCTAssertEqual( + folded.compactMap { $0.raw.label }, + ["App", "Hairline caption", "Frameless semantics"]) + XCTAssertFalse(folded.contains { $0.raw.type == "Button" }) + XCTAssertEqual(folded.first { $0.raw.label == "Frameless semantics" }?.raw.hittable, false) } func testPlainViewportPolicyFoldsWithoutAncestorCursor() { @@ -114,11 +122,30 @@ extension RunnerTests { let viewport = CGRect(x: 0, y: 0, width: 800, height: 600) let regular = Self.folded(nodes, viewport: viewport, policy: .plainViewport) - XCTAssertEqual(regular.compactMap(\.label), ["App", "Offscreen window", "Clamped child"]) + XCTAssertEqual( + regular.compactMap { $0.raw.label }, ["App", "Offscreen window", "Clamped child"]) let interactive = Self.folded( nodes, viewport: viewport, interactiveOnly: true, policy: .plainViewport) - XCTAssertFalse(interactive.contains { $0.label == "Offscreen window" }) + XCTAssertFalse(interactive.contains { $0.raw.label == "Offscreen window" }) + } + + func testPlainViewportPolicyDoesNotClipToScrollAncestor() { + let nodes = [ + Self.foldNode(0, type: "Application", label: "App", + rect: SnapshotRect(x: 0, y: 0, width: 800, height: 600), depth: 0, parentIndex: nil), + Self.foldNode(1, type: "ScrollView", label: "Scroll", + rect: SnapshotRect(x: 0, y: 100, width: 800, height: 100), depth: 1, parentIndex: 0), + Self.foldNode(2, type: "StaticText", label: "Outside scroll clip", + rect: SnapshotRect(x: 16, y: 240, width: 100, height: 20), depth: 2, parentIndex: 1), + ] + let viewport = CGRect(x: 0, y: 0, width: 800, height: 600) + + let plain = Self.folded(nodes, viewport: viewport, policy: .plainViewport) + XCTAssertTrue(plain.contains { $0.raw.label == "Outside scroll clip" }) + + let cursorProjected = Self.folded(nodes, viewport: viewport, policy: .cursorProjected) + XCTAssertFalse(cursorProjected.contains { $0.raw.label == "Outside scroll clip" }) } func testScrollContainerTypeNamesMatchElementTypeSet() { diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 706a9f053b..e8a2ad6949 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -145,7 +145,10 @@ requested label. The fourth semantic layer moves the clip fold itself into presentation. Acquisition backends are fact serializers: every traversed node is emitted at raw traversal depth with its reported frame, -and `SnapshotAcquisition` carries the viewport. `presentRegular` runs the one visibility +and `SnapshotAcquisition` carries the viewport. The fold returns a typed carrier with both values: +`raw.rect` remains runner-internal reported geometry, while regular presentation writes the +carrier's effective rectangle through the existing wire `rect` field; raw projections and direct +single-element reads retain reported geometry. `presentRegular` runs the one visibility interpreter for every backend — viewport ∩ scroll-container clip, the ancestor projection cursor (an out-of-clip Cell or scroll container hides descendants whose clamped frames would otherwise leak back into the viewport), the sub-pixel decoration rule, hidden-content hints booked onto