From f1ec9b1a32eb9d5e3cf7bc13faaee20aff238ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 18 Aug 2026 08:41:49 +0200 Subject: [PATCH 1/4] fix(android): let a sibling hide only what its content covers (#1806) pruneAndroidCoveredSubtrees credited a higher drawing-order sibling with painting its whole box as soon as it had any content anywhere inside it, or a label of its own. A full-screen DoraemonKit drag surface holding one 189px floating icon therefore condemned the entire app subtree, and an empty labelled match_parent placeholder did the same. Occlusion is now spatial. A subtree's footprint is the bounding box of what it presents (agent targets and labelled leaves); a sibling is covered when its footprint lies under a candidate's footprint. A node's own label is no longer paint evidence: a container's content-desc describes its children and an empty labelled View draws nothing. Only a touch target still hides its full box (scrims). Comparing footprint to footprint keeps stacked screens with matching margins registering as covered. Live on a Pixel 9 Pro XL API 37 emulator with a DoKit-shaped overlay added to the test app: snapshot -i went from 2 nodes + the sparse hint to the full app; helper-XML A/B across home/catalog/form/product-detail recovered every label with none lost, and non-overlay screens are byte-identical. --- .../android/__tests__/ui-hierarchy.test.ts | 75 ++++++++++- src/platforms/android/ui-hierarchy.ts | 122 +++++++++++++----- 2 files changed, 161 insertions(+), 36 deletions(-) diff --git a/src/platforms/android/__tests__/ui-hierarchy.test.ts b/src/platforms/android/__tests__/ui-hierarchy.test.ts index ae362b14b..17d4ace5b 100644 --- a/src/platforms/android/__tests__/ui-hierarchy.test.ts +++ b/src/platforms/android/__tests__/ui-hierarchy.test.ts @@ -269,7 +269,7 @@ test('parseUiHierarchy reads an omitted clickable attribute the same as clickabl const tree = (clickable: string) => ` - + @@ -329,10 +329,15 @@ test('parseUiHierarchy prunes descendants of Android nodes that are not visible }); test('parseUiHierarchy prunes lower drawing-order subtrees covered by a foreground sibling', () => { + // A pushed screen (header + full-width rows) drawn above a still-attached drawer surface. The + // pushed screen's presented content lies over the drawer's content, so the drawer is covered. const xml = ` - + + + + @@ -353,6 +358,69 @@ test('parseUiHierarchy prunes lower drawing-order subtrees covered by a foregrou ); }); +test('parseUiHierarchy keeps app content under a full-screen overlay holding one floating icon (#1806)', () => { + // DoraemonKit injects a full-screen FrameLayout above the whole Activity purely as a drag surface + // for a small floating icon. It presents only the icon, so it can only hide what sits under it. + const xml = ` + + + + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.deepEqual( + result.nodes.filter((node) => node.label).map((node) => node.label), + ['Editor', 'Save', 'dokit_contentview_id_DokitFrameLayout[1]', 'DoKit'], + ); +}); + +test('parseUiHierarchy keeps app content beside an empty labelled full-screen placeholder (#1806)', () => { + // A match_parent placeholder container that stays VISIBLE and only receives a fragment later. + // Its content-desc is an announcement, not paint: it draws nothing over the app. + const xml = ` + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.label === 'Toolbar action'), + true, + ); +}); + +test('parseUiHierarchy compares presented footprints so a sparse overlay never condemns a rich sibling', () => { + // The overlay's only content is a corner badge; the sibling's content spans the screen. Box + // geometry alone (overlay box ⊇ sibling box) would call this covered. + const xml = ` + + + + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.deepEqual( + result.nodes.filter((node) => node.label).map((node) => node.label), + ['Top action', 'Bottom action', 'Badge'], + ); +}); + test('parseUiHierarchy keeps visible identifier-only markers beside covering content', () => { const xml = ` @@ -556,9 +624,10 @@ test('parseUiHierarchy keeps an overlapped text leaf drawn inside a composite wi }); test('parseUiHierarchy still condemns a clickable leaf covered by a foreground sibling', () => { + // A tap-to-dismiss scrim is itself a touch target, so it hides its whole box. const xml = ` - + diff --git a/src/platforms/android/ui-hierarchy.ts b/src/platforms/android/ui-hierarchy.ts index 07c7a7cd4..e2083594d 100644 --- a/src/platforms/android/ui-hierarchy.ts +++ b/src/platforms/android/ui-hierarchy.ts @@ -452,13 +452,20 @@ type AndroidNodeInclusionInfo = { isVisual: boolean; }; +type AndroidFootprint = { + /** Bounding box of what the subtree presents (agent targets and labelled leaves), if anything. */ + rect: Rect | null; + hasAgentTarget: boolean; +}; + type AndroidTreePruneState = { - actionableContentMemo: WeakMap; + footprintMemo: WeakMap; }; -type AndroidCoveringCandidate = AndroidNode & { - rect: Rect; +type AndroidCoveringCandidate = { + node: AndroidNode; drawingOrder: number; + footprint: Rect; }; const ANDROID_WINDOW_TYPE_APPLICATION = 1; @@ -525,7 +532,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy { discardInactiveAndroidApplicationWindows(root); // UiAutomation can expose covered React Native navigation surfaces in the same accessibility // window. If a higher drawing-order sibling covers them, agents should see the foreground surface. - pruneAndroidCoveredSubtrees(root, { actionableContentMemo: new WeakMap() }); + pruneAndroidCoveredSubtrees(root, { footprintMemo: new WeakMap() }); applyAndroidScrollActionHints(root); return root; } @@ -550,22 +557,59 @@ function hasSemanticContent(node: AndroidNode): boolean { return hasMeaningfulLabel(node) || hasMeaningfulIdentifier(node); } -/** Focusability is traversal, not paint, so it is not evidence a node hides anything (#1733). */ +/** + * Focusability is traversal, not paint (#1733), and a label is an announcement, not paint (#1806): + * a container's content-desc describes its children and an empty labelled View draws nothing. Only + * a touch target is direct evidence that a node hides what lies under its box. + */ function hasDirectOcclusionEvidence(node: AndroidNode): boolean { - return node.visibleToUser !== false && (isTouchTarget(node) || hasSemanticContent(node)); + return node.visibleToUser !== false && isTouchTarget(node); } /** Evidence the node is a real surface because it contains something an agent could drive. */ function hasDescendantOcclusionEvidence(node: AndroidNode, state: AndroidTreePruneState): boolean { - const cached = state.actionableContentMemo.get(node); - if (cached !== undefined) return cached; - const result = node.children.some( - (child) => - child.visibleToUser !== false && - (isAgentTarget(child) || hasDescendantOcclusionEvidence(child, state)), + return node.children.some( + (child) => child.visibleToUser !== false && subtreeFootprint(child, state).hasAgentTarget, ); - state.actionableContentMemo.set(node, result); - return result; +} + +/** + * Where a subtree visibly presents something: the bounding box of its agent targets and labelled + * leaves. A full-screen debug overlay holding one floating icon presents only that icon, so it can + * only hide what sits under the icon, never the whole app behind it (#1806). + */ +function subtreeFootprint(node: AndroidNode, state: AndroidTreePruneState): AndroidFootprint { + const cached = state.footprintMemo.get(node); + if (cached !== undefined) return cached; + let hasAgentTarget = isAgentTarget(node); + let rect: Rect | null = + (isAgentTarget(node) || isLabelledLeaf(node)) && hasPositiveRect(node) ? node.rect : null; + for (const child of node.children) { + if (child.visibleToUser === false) continue; + const childFootprint = subtreeFootprint(child, state); + hasAgentTarget ||= childFootprint.hasAgentTarget; + rect = unionRect(rect, childFootprint.rect); + } + const footprint = { rect, hasAgentTarget }; + state.footprintMemo.set(node, footprint); + return footprint; +} + +function isLabelledLeaf(node: AndroidNode): boolean { + return node.children.length === 0 && hasMeaningfulLabel(node); +} + +function unionRect(left: Rect | null, right: Rect | null): Rect | null { + if (!left) return right; + if (!right) return left; + const x = Math.min(left.x, right.x); + const y = Math.min(left.y, right.y); + return { + x, + y, + width: Math.max(left.x + left.width, right.x + right.width) - x, + height: Math.max(left.y + left.height, right.y + right.height) - y, + }; } /** @@ -598,32 +642,45 @@ function pruneAndroidCoveredSubtrees(node: AndroidNode, state: AndroidTreePruneS return; } const siblings = node.children; - const coveringCandidates = siblings.filter((sibling) => canCoverSibling(sibling, state)); + const coveringCandidates = siblings + .map((sibling) => coveringCandidateOf(sibling, state)) + .filter((candidate) => candidate !== null); if (coveringCandidates.length === 0) return; - node.children = siblings.filter((child) => shouldKeepAndroidSibling(child, coveringCandidates)); + node.children = siblings.filter((child) => + shouldKeepAndroidSibling(child, coveringCandidates, state), + ); } function shouldKeepAndroidSibling( node: AndroidNode, coveringCandidates: AndroidCoveringCandidate[], + state: AndroidTreePruneState, ): boolean { return ( - isPresentationLeaf(node) || !isCoveredByHigherDrawingOrderSibling(node, coveringCandidates) + isPresentationLeaf(node) || + !isCoveredByHigherDrawingOrderSibling(node, coveringCandidates, state) ); } +/** + * Covered means the sibling's presented content lies under the candidate's presented content. + * Comparing footprints rather than boxes lets two stacked screens with the same layout margins + * still register as covered, while a sparse overlay never condemns a rich screen. + */ function isCoveredByHigherDrawingOrderSibling( node: AndroidNode, coveringCandidates: AndroidCoveringCandidate[], + state: AndroidTreePruneState, ): boolean { if (node.visibleToUser === false || node.drawingOrder === undefined || !hasPositiveRect(node)) { return false; } - for (const sibling of coveringCandidates) { - if (sibling === node || sibling.drawingOrder <= node.drawingOrder) { + const coveredRect = subtreeFootprint(node, state).rect ?? node.rect; + for (const candidate of coveringCandidates) { + if (candidate.node === node || candidate.drawingOrder <= node.drawingOrder) { continue; } - if (rectCoverage(sibling.rect, node.rect) >= 0.9) { + if (rectCoverage(candidate.footprint, coveredRect) >= 0.9) { return true; } } @@ -635,21 +692,20 @@ function hasMeaningfulIdentifier(node: AndroidNode): boolean { return Boolean(identifier && !isGenericAndroidId(identifier)); } -function canCoverSibling( +/** The single occlusion classification. Covering is never re-derived from a raw attribute. */ +function coveringCandidateOf( node: AndroidNode, state: AndroidTreePruneState, -): node is AndroidCoveringCandidate { - return ( - node.visibleToUser !== false && - node.drawingOrder !== undefined && - hasPositiveRect(node) && - hasOcclusionEvidence(node, state) - ); -} - -/** The single occlusion classification. Covering is never re-derived from a raw attribute. */ -function hasOcclusionEvidence(node: AndroidNode, state: AndroidTreePruneState): boolean { - return hasDirectOcclusionEvidence(node) || hasDescendantOcclusionEvidence(node, state); +): AndroidCoveringCandidate | null { + const { drawingOrder } = node; + if (node.visibleToUser === false || drawingOrder === undefined || !hasPositiveRect(node)) { + return null; + } + if (!hasDirectOcclusionEvidence(node) && !hasDescendantOcclusionEvidence(node, state)) { + return null; + } + const footprint = subtreeFootprint(node, state).rect; + return footprint ? { node, drawingOrder, footprint } : null; } function hasMeaningfulLabel(node: AndroidNode): boolean { From 51addee665fe6478fda22a38f32e97f043d7ab0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 18 Aug 2026 11:15:47 +0200 Subject: [PATCH 2/4] fix(android): measure occlusion by overlapped area, not bounding box Review on #1808: a bounding box of two corner controls spans the viewport, so a transparent overlay with a control in each corner still acquired a full-screen footprint and could prune the app beneath it. Footprints now keep their presented rects apart, and coverage is the overlapped area of the two unions (coordinate-compressed cell sweep). Scrollables count as presenting their box: they consume touches over it, which is what lets a real pushed screen (header, scrollable body, footer) still cover a drawer surface. Adds the disconnected-corner regression. --- .../android/__tests__/ui-hierarchy.test.ts | 33 ++++- src/platforms/android/ui-hierarchy.ts | 121 ++++++++++++------ 2 files changed, 109 insertions(+), 45 deletions(-) diff --git a/src/platforms/android/__tests__/ui-hierarchy.test.ts b/src/platforms/android/__tests__/ui-hierarchy.test.ts index 17d4ace5b..b5e0f5acb 100644 --- a/src/platforms/android/__tests__/ui-hierarchy.test.ts +++ b/src/platforms/android/__tests__/ui-hierarchy.test.ts @@ -329,14 +329,16 @@ test('parseUiHierarchy prunes descendants of Android nodes that are not visible }); test('parseUiHierarchy prunes lower drawing-order subtrees covered by a foreground sibling', () => { - // A pushed screen (header + full-width rows) drawn above a still-attached drawer surface. The - // pushed screen's presented content lies over the drawer's content, so the drawer is covered. + // A pushed screen (header, scrollable body, footer) drawn above a still-attached drawer surface. + // The pushed screen's presented content lies over the drawer's content, so the drawer is covered. const xml = ` - + + + @@ -399,6 +401,31 @@ test('parseUiHierarchy keeps app content beside an empty labelled full-screen pl ); }); +test('parseUiHierarchy keeps app content under an overlay whose only controls sit in opposite corners', () => { + // Two floating controls in opposite corners present two corners, not the screen between them. + // A bounding box of the presented content would span the viewport and condemn the whole app. + const xml = ` + + + + + + + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.deepEqual( + result.nodes.filter((node) => node.label).map((node) => node.label), + ['Editor', 'Save', 'Debug menu', 'Frame stats'], + ); +}); + test('parseUiHierarchy compares presented footprints so a sparse overlay never condemns a rich sibling', () => { // The overlay's only content is a corner badge; the sibling's content spans the screen. Box // geometry alone (overlay box ⊇ sibling box) would call this covered. diff --git a/src/platforms/android/ui-hierarchy.ts b/src/platforms/android/ui-hierarchy.ts index e2083594d..5e64eb8ea 100644 --- a/src/platforms/android/ui-hierarchy.ts +++ b/src/platforms/android/ui-hierarchy.ts @@ -2,7 +2,6 @@ import type { RawSnapshotNode, Rect, SnapshotOptions } from '@agent-device/kerne import { parseBounds } from '@agent-device/kernel/bounds'; import { decodeXmlCharacterReferences } from '@agent-device/xml'; import { isScrollableType } from '@agent-device/contracts/snapshot'; -import { intersectArea } from '../../utils/screenshot-geometry.ts'; import { type AndroidSystemChromeProvenance, isAndroidSystemChromeWindowResourceId, @@ -453,8 +452,8 @@ type AndroidNodeInclusionInfo = { }; type AndroidFootprint = { - /** Bounding box of what the subtree presents (agent targets and labelled leaves), if anything. */ - rect: Rect | null; + /** Boxes of what the subtree presents: touch/focus targets, scrollables and labelled leaves. */ + rects: Rect[]; hasAgentTarget: boolean; }; @@ -465,7 +464,7 @@ type AndroidTreePruneState = { type AndroidCoveringCandidate = { node: AndroidNode; drawingOrder: number; - footprint: Rect; + footprint: Rect[]; }; const ANDROID_WINDOW_TYPE_APPLICATION = 1; @@ -574,42 +573,84 @@ function hasDescendantOcclusionEvidence(node: AndroidNode, state: AndroidTreePru } /** - * Where a subtree visibly presents something: the bounding box of its agent targets and labelled - * leaves. A full-screen debug overlay holding one floating icon presents only that icon, so it can - * only hide what sits under the icon, never the whole app behind it (#1806). + * Where a subtree visibly presents something: the boxes of its touch-consuming surfaces (touch + * targets, scrollables), focus targets and labelled leaves. A full-screen debug overlay holding one + * floating icon presents only that icon, so it can only hide what sits under the icon, never the + * whole app behind it (#1806). The rects are kept apart rather than merged into one bounding box: + * two controls in opposite corners present two corners, not the screen between them. */ function subtreeFootprint(node: AndroidNode, state: AndroidTreePruneState): AndroidFootprint { const cached = state.footprintMemo.get(node); if (cached !== undefined) return cached; - let hasAgentTarget = isAgentTarget(node); - let rect: Rect | null = - (isAgentTarget(node) || isLabelledLeaf(node)) && hasPositiveRect(node) ? node.rect : null; - for (const child of node.children) { - if (child.visibleToUser === false) continue; - const childFootprint = subtreeFootprint(child, state); - hasAgentTarget ||= childFootprint.hasAgentTarget; - rect = unionRect(rect, childFootprint.rect); + let footprint: AndroidFootprint; + if (presentsOwnBox(node) && hasPositiveRect(node)) { + // Its box is presented as a whole; whatever it contains lies inside that box. + footprint = { rects: [node.rect], hasAgentTarget: isAgentTarget(node) }; + } else { + footprint = { rects: [], hasAgentTarget: isAgentTarget(node) }; + for (const child of node.children) { + if (child.visibleToUser === false) continue; + const childFootprint = subtreeFootprint(child, state); + footprint.hasAgentTarget ||= childFootprint.hasAgentTarget; + footprint.rects.push(...childFootprint.rects); + } } - const footprint = { rect, hasAgentTarget }; state.footprintMemo.set(node, footprint); return footprint; } -function isLabelledLeaf(node: AndroidNode): boolean { - return node.children.length === 0 && hasMeaningfulLabel(node); +function presentsOwnBox(node: AndroidNode): boolean { + return ( + isAgentTarget(node) || + node.scrollable === true || + (node.children.length === 0 && hasMeaningfulLabel(node)) + ); } -function unionRect(left: Rect | null, right: Rect | null): Rect | null { - if (!left) return right; - if (!right) return left; - const x = Math.min(left.x, right.x); - const y = Math.min(left.y, right.y); - return { - x, - y, - width: Math.max(left.x + left.width, right.x + right.width) - x, - height: Math.max(left.y + left.height, right.y + right.height) - y, - }; +/** Fraction of the covered rects' union that lies under the covering rects' union. */ +function unionCoverage(coveringRects: Rect[], coveredRects: Rect[]): number { + const xs = compressedEdges([...coveringRects, ...coveredRects], (rect) => [ + rect.x, + rect.x + rect.width, + ]); + const ys = compressedEdges([...coveringRects, ...coveredRects], (rect) => [ + rect.y, + rect.y + rect.height, + ]); + const covering = markCells(coveringRects, xs, ys); + const covered = markCells(coveredRects, xs, ys); + let coveredArea = 0; + let overlapArea = 0; + for (let column = 0; column < xs.length - 1; column += 1) { + const width = xs[column + 1]! - xs[column]!; + for (let row = 0; row < ys.length - 1; row += 1) { + const cell = column * (ys.length - 1) + row; + if (!covered[cell]) continue; + const area = width * (ys[row + 1]! - ys[row]!); + coveredArea += area; + if (covering[cell]) overlapArea += area; + } + } + return coveredArea <= 0 ? 0 : overlapArea / coveredArea; +} + +function compressedEdges(rects: Rect[], edgesOf: (rect: Rect) => [number, number]): number[] { + return [...new Set(rects.flatMap(edgesOf))].sort((left, right) => left - right); +} + +function markCells(rects: Rect[], xs: number[], ys: number[]): Uint8Array { + const rows = ys.length - 1; + const cells = new Uint8Array((xs.length - 1) * rows); + for (const rect of rects) { + const firstColumn = xs.indexOf(rect.x); + const lastColumn = xs.indexOf(rect.x + rect.width); + const firstRow = ys.indexOf(rect.y); + const lastRow = ys.indexOf(rect.y + rect.height); + for (let column = firstColumn; column < lastColumn; column += 1) { + cells.fill(1, column * rows + firstRow, column * rows + lastRow); + } + } + return cells; } /** @@ -663,9 +704,10 @@ function shouldKeepAndroidSibling( } /** - * Covered means the sibling's presented content lies under the candidate's presented content. - * Comparing footprints rather than boxes lets two stacked screens with the same layout margins - * still register as covered, while a sparse overlay never condemns a rich screen. + * Covered means the sibling's presented content lies under the candidate's presented content, by + * actual overlapped area. Comparing footprints rather than boxes lets two stacked screens with the + * same layout margins still register as covered, while a sparse overlay never condemns a rich + * screen however far apart its controls sit. */ function isCoveredByHigherDrawingOrderSibling( node: AndroidNode, @@ -675,12 +717,13 @@ function isCoveredByHigherDrawingOrderSibling( if (node.visibleToUser === false || node.drawingOrder === undefined || !hasPositiveRect(node)) { return false; } - const coveredRect = subtreeFootprint(node, state).rect ?? node.rect; + const footprint = subtreeFootprint(node, state).rects; + const coveredRects = footprint.length > 0 ? footprint : [node.rect]; for (const candidate of coveringCandidates) { if (candidate.node === node || candidate.drawingOrder <= node.drawingOrder) { continue; } - if (rectCoverage(candidate.footprint, coveredRect) >= 0.9) { + if (unionCoverage(candidate.footprint, coveredRects) >= 0.9) { return true; } } @@ -704,8 +747,8 @@ function coveringCandidateOf( if (!hasDirectOcclusionEvidence(node) && !hasDescendantOcclusionEvidence(node, state)) { return null; } - const footprint = subtreeFootprint(node, state).rect; - return footprint ? { node, drawingOrder, footprint } : null; + const footprint = subtreeFootprint(node, state).rects; + return footprint.length > 0 ? { node, drawingOrder, footprint } : null; } function hasMeaningfulLabel(node: AndroidNode): boolean { @@ -717,12 +760,6 @@ function hasPositiveRect(node: AndroidNode): node is AndroidNode & { rect: Rect return Boolean(node.rect && node.rect.width > 0 && node.rect.height > 0); } -function rectCoverage(coveringRect: Rect, targetRect: Rect): number { - const targetArea = targetRect.width * targetRect.height; - if (targetArea <= 0) return 0; - return intersectArea(coveringRect, targetRect) / targetArea; -} - function applyAndroidScrollActionHints(root: AndroidUiHierarchy): void { const stack = [...root.children]; while (stack.length > 0) { From 169c95b705cca119903aaef02ba9a15ccad37c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 18 Aug 2026 12:34:40 +0200 Subject: [PATCH 3/4] fix(android): count what a covered sibling shows, not only what it paints Fuzzing random sibling trees old-vs-new surfaced the one direction the footprint model could still regress: a container whose only painted content is small (one corner icon) but which also carries labelled containers or testID-only markers was condemned as soon as a touch surface covered that icon, since markers and container labels are not paint and never entered the footprint. Footprints now carry two rect sets. `paints` (touch targets, scrollables, labelled leaves) is what a candidate can cover with; it still excludes identifiers and container labels, or the DoKit fix would unwind. `shows` adds every labelled or identified node and is what a covered sibling must lose in full. Focusable-only nodes no longer paint their box either, matching #1733 for descendants as well as siblings. Adds the marker regression. Re-fuzzed 20k trees: new-prunes-more is down to 0.14 %, all of the class where everything the target shows lies under a higher touch/scroll surface. Live captures unchanged. --- .../android/__tests__/ui-hierarchy.test.ts | 23 ++++++ src/platforms/android/ui-hierarchy.ts | 81 ++++++++++++------- 2 files changed, 77 insertions(+), 27 deletions(-) diff --git a/src/platforms/android/__tests__/ui-hierarchy.test.ts b/src/platforms/android/__tests__/ui-hierarchy.test.ts index b5e0f5acb..4e1aab607 100644 --- a/src/platforms/android/__tests__/ui-hierarchy.test.ts +++ b/src/platforms/android/__tests__/ui-hierarchy.test.ts @@ -426,6 +426,29 @@ test('parseUiHierarchy keeps app content under an overlay whose only controls si ); }); +test('parseUiHierarchy counts identifier-only markers toward what a covered sibling shows', () => { + // A screen container carrying testID markers whose only painted content is one corner icon, + // under a clickable header bar. The bar covers the icon, but the agent would also lose the + // markers, which sit well outside the bar — so the container is not covered. + const xml = ` + + + + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.identifier === 'home-body'), + true, + ); +}); + test('parseUiHierarchy compares presented footprints so a sparse overlay never condemns a rich sibling', () => { // The overlay's only content is a corner badge; the sibling's content spans the screen. Box // geometry alone (overlay box ⊇ sibling box) would call this covered. diff --git a/src/platforms/android/ui-hierarchy.ts b/src/platforms/android/ui-hierarchy.ts index 5e64eb8ea..ebfbf9986 100644 --- a/src/platforms/android/ui-hierarchy.ts +++ b/src/platforms/android/ui-hierarchy.ts @@ -452,8 +452,10 @@ type AndroidNodeInclusionInfo = { }; type AndroidFootprint = { - /** Boxes of what the subtree presents: touch/focus targets, scrollables and labelled leaves. */ - rects: Rect[]; + /** Boxes of what the subtree paints: touch targets, scrollables and labelled leaves. */ + paints: Rect[]; + /** Boxes of what an agent would see of the subtree: `paints` plus labelled/identified nodes. */ + shows: Rect[]; hasAgentTarget: boolean; }; @@ -573,35 +575,60 @@ function hasDescendantOcclusionEvidence(node: AndroidNode, state: AndroidTreePru } /** - * Where a subtree visibly presents something: the boxes of its touch-consuming surfaces (touch - * targets, scrollables), focus targets and labelled leaves. A full-screen debug overlay holding one - * floating icon presents only that icon, so it can only hide what sits under the icon, never the - * whole app behind it (#1806). The rects are kept apart rather than merged into one bounding box: - * two controls in opposite corners present two corners, not the screen between them. + * What a subtree paints and what it shows. Paint is the boxes of its touch-consuming surfaces + * (touch targets, scrollables) and labelled leaves: a full-screen debug overlay + * holding one floating icon paints only that icon, so it can only hide what sits under the icon, + * never the whole app behind it (#1806). Rects are kept apart rather than merged into one bounding + * box: two controls in opposite corners paint two corners, not the screen between them. + * + * Shows adds every labelled or identified node — a testID marker or a described container paints + * nothing, so it never helps a candidate cover, but an agent would still lose it, so it always + * counts toward what a covered sibling has. */ function subtreeFootprint(node: AndroidNode, state: AndroidTreePruneState): AndroidFootprint { const cached = state.footprintMemo.get(node); if (cached !== undefined) return cached; - let footprint: AndroidFootprint; - if (presentsOwnBox(node) && hasPositiveRect(node)) { - // Its box is presented as a whole; whatever it contains lies inside that box. - footprint = { rects: [node.rect], hasAgentTarget: isAgentTarget(node) }; - } else { - footprint = { rects: [], hasAgentTarget: isAgentTarget(node) }; - for (const child of node.children) { - if (child.visibleToUser === false) continue; - const childFootprint = subtreeFootprint(child, state); - footprint.hasAgentTarget ||= childFootprint.hasAgentTarget; - footprint.rects.push(...childFootprint.rects); - } - } + const footprint = hasPositiveRect(node) + ? footprintWithinBox(node, node.rect, state) + : childrenFootprint(node, state); state.footprintMemo.set(node, footprint); return footprint; } -function presentsOwnBox(node: AndroidNode): boolean { +function footprintWithinBox( + node: AndroidNode, + ownBox: Rect, + state: AndroidTreePruneState, +): AndroidFootprint { + if (paintsOwnBox(node)) { + // The whole box is painted; whatever it contains lies inside that box. + return { paints: [ownBox], shows: [ownBox], hasAgentTarget: isAgentTarget(node) }; + } + const footprint = childrenFootprint(node, state); + if (hasSemanticContent(node)) footprint.shows.push(ownBox); + return footprint; +} + +function childrenFootprint(node: AndroidNode, state: AndroidTreePruneState): AndroidFootprint { + const footprint: AndroidFootprint = { + paints: [], + shows: [], + hasAgentTarget: isAgentTarget(node), + }; + for (const child of node.children) { + if (child.visibleToUser === false) continue; + const childFootprint = subtreeFootprint(child, state); + footprint.hasAgentTarget ||= childFootprint.hasAgentTarget; + footprint.paints.push(...childFootprint.paints); + footprint.shows.push(...childFootprint.shows); + } + return footprint; +} + +/** Focusability is traversal, not paint (#1733); a container's label describes its children. */ +function paintsOwnBox(node: AndroidNode): boolean { return ( - isAgentTarget(node) || + isTouchTarget(node) || node.scrollable === true || (node.children.length === 0 && hasMeaningfulLabel(node)) ); @@ -704,8 +731,8 @@ function shouldKeepAndroidSibling( } /** - * Covered means the sibling's presented content lies under the candidate's presented content, by - * actual overlapped area. Comparing footprints rather than boxes lets two stacked screens with the + * Covered means everything an agent would see of the sibling lies under what the candidate paints, + * by actual overlapped area. Comparing footprints rather than boxes lets two stacked screens with the * same layout margins still register as covered, while a sparse overlay never condemns a rich * screen however far apart its controls sit. */ @@ -717,8 +744,8 @@ function isCoveredByHigherDrawingOrderSibling( if (node.visibleToUser === false || node.drawingOrder === undefined || !hasPositiveRect(node)) { return false; } - const footprint = subtreeFootprint(node, state).rects; - const coveredRects = footprint.length > 0 ? footprint : [node.rect]; + const shows = subtreeFootprint(node, state).shows; + const coveredRects = shows.length > 0 ? shows : [node.rect]; for (const candidate of coveringCandidates) { if (candidate.node === node || candidate.drawingOrder <= node.drawingOrder) { continue; @@ -747,7 +774,7 @@ function coveringCandidateOf( if (!hasDirectOcclusionEvidence(node) && !hasDescendantOcclusionEvidence(node, state)) { return null; } - const footprint = subtreeFootprint(node, state).rects; + const footprint = subtreeFootprint(node, state).paints; return footprint.length > 0 ? { node, drawingOrder, footprint } : null; } From 189990a76aa063cc898723af3a35e7fc8868411f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 18 Aug 2026 14:14:39 +0200 Subject: [PATCH 4/4] test(android): pin that focusability never paints a covering candidate's box A full-screen focusable wrapper holding one clickable icon is a covering candidate; the lower app content must survive. Fails when paintsOwnBox counts focus targets again. --- .../android/__tests__/ui-hierarchy.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/platforms/android/__tests__/ui-hierarchy.test.ts b/src/platforms/android/__tests__/ui-hierarchy.test.ts index 4e1aab607..ea14510b3 100644 --- a/src/platforms/android/__tests__/ui-hierarchy.test.ts +++ b/src/platforms/android/__tests__/ui-hierarchy.test.ts @@ -426,6 +426,29 @@ test('parseUiHierarchy keeps app content under an overlay whose only controls si ); }); +test('parseUiHierarchy keeps app content under a focusable full-screen overlay holding one clickable icon', () => { + // The Telegram wrapper from #1733, but with one floating clickable icon inside it. The icon makes + // the wrapper a covering candidate; its focusability must still not paint the box, or the wrapper + // condemns the whole app under it exactly as it did before #1733. + const xml = ` + + + + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.deepEqual( + result.nodes.filter((node) => node.label).map((node) => node.label), + ['Your phone number', '208 379 7171', 'Attach'], + ); +}); + test('parseUiHierarchy counts identifier-only markers toward what a covered sibling shows', () => { // A screen container carrying testID markers whose only painted content is one corner icon, // under a clickable header bar. The bar covers the icon, but the agent would also lose the