From 99d209e1745f537f1cc8465c9b74edae4a979c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 17 Aug 2026 19:38:54 +0200 Subject: [PATCH 1/2] test(android): restore full-tier lifecycle and observability scenarios (#1781 A1) The nightly Android job has failed on `click id="automation-request-microphone"` since the full tier landed: `settings permission reset microphone` runs `pm revoke`, and revoking a *granted* runtime permission kills the app process, so the round after an accept clicked into an empty launcher surface. Reproduced on the pinned CI image (android-36 google_apis_playstore, Pixel 7): pm revoke leaves pid 4259 alive when the permission is denied and kills it when it is granted, with NexusLauncher resumed afterwards. Fixing that exposed the rest of a scenario that had never executed end to end: the post-revoke readback cold-started on the tabs home instead of Automation lab, the relaunched Automation lab needed its controls revealed, the Form tab does not exist on the Automation root route, that section needs the system IME back, and the IME diagnostic sits above the bottom of the form. The observability scenario then failed the same way (reveal distances tuned for a taller device) plus an event-timeline walk whose page was smaller than the events each page read appends. Validated live against a local Pixel_7_CI emulator (API 36, same profile as the lane): the full tier now runs bootstrap -> inventory -> automation-system -> form-input -> keyboard-ime -> capture-close -> lifecycle-system -> observability-artifacts and stops only in full:fixture-replays. --- .../live-form-scenario.ts | 2 +- .../live-lifecycle-scenario.ts | 55 +++++++++++++------ .../live-observability-scenario.ts | 13 ++++- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/test/integration/android-emulator-e2e/live-form-scenario.ts b/test/integration/android-emulator-e2e/live-form-scenario.ts index 25201347f..267160fc1 100644 --- a/test/integration/android-emulator-e2e/live-form-scenario.ts +++ b/test/integration/android-emulator-e2e/live-form-scenario.ts @@ -14,7 +14,7 @@ import { import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; const C = PUBLIC_COMMANDS; -const ANDROID_TEST_IME_PACKAGE = 'com.callstack.agentdevice.imehelper'; +export const ANDROID_TEST_IME_PACKAGE = 'com.callstack.agentdevice.imehelper'; const ANDROID_TEST_IME_SERVICE = `${ANDROID_TEST_IME_PACKAGE}/.TestInputMethodService`; const KEYBOARD_VISIBILITY_TIMEOUT_MS = 10_000; const KEYBOARD_VISIBILITY_POLL_MS = 250; diff --git a/test/integration/android-emulator-e2e/live-lifecycle-scenario.ts b/test/integration/android-emulator-e2e/live-lifecycle-scenario.ts index 1fd7573ca..b8cf3f0eb 100644 --- a/test/integration/android-emulator-e2e/live-lifecycle-scenario.ts +++ b/test/integration/android-emulator-e2e/live-lifecycle-scenario.ts @@ -2,27 +2,31 @@ import assert from 'node:assert/strict'; import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; import { assertElementText, assertWaitText } from './live-assertions.ts'; +import { ANDROID_TEST_IME_PACKAGE } from './live-form-scenario.ts'; import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; const C = PUBLIC_COMMANDS; const AUTOMATION_DEEP_LINK = 'agent-device-test-app:///automation?event=full.lifecycle&payload=%7B%22source%22%3A%22android-nightly%22%7D'; +const TABS_DEEP_LINK = 'agent-device-test-app:///'; export const ANDROID_PERMISSION_PROMPT_COMMAND = ['alert', 'wait', '10000'] as const; export async function assertLifecycleAndSystem(context: LiveContext): Promise { - await runStep(context, 'open Android fixture lifecycle route', [ - 'open', - context.appId, - '--relaunch', - AUTOMATION_DEEP_LINK, - ]); - await assertWaitText(context, 'Automation lab'); + await openAutomationLab(context, 'open Android fixture lifecycle route'); await assertPermissionRecovery(context); await assertPushBroadcast(context); await assertImeRecovery(context); } +// Leaves the fixture on Automation lab with its permission controls on screen: Android +// snapshots only carry on-screen nodes, and a relaunch always lands above the controls. +async function openAutomationLab(context: LiveContext, step: string): Promise { + await runStep(context, step, ['open', context.appId, '--relaunch', AUTOMATION_DEEP_LINK]); + await assertWaitText(context, 'Automation lab'); + await runStep(context, `${step} (reveal automation controls)`, ['scroll', 'bottom']); +} + async function assertPermissionRecovery(context: LiveContext): Promise { await resetAndRequestMicrophonePermission(context, 'accept'); await assertElementText(context, 'id="automation-microphone-permission"', 'granted'); @@ -39,9 +43,7 @@ async function assertPermissionRecovery(context: LiveContext): Promise { 'deny', 'microphone', ]); - await runStep(context, 'background fixture before revoked permission readback', ['home']); - await runStep(context, 'restore fixture after microphone revocation', ['open', context.appId]); - await assertWaitText(context, 'Automation lab'); + await openAutomationLab(context, 'restore fixture after microphone revocation'); await assertElementText(context, 'id="automation-microphone-permission"', 'denied'); verifyCommand( context, @@ -51,10 +53,13 @@ async function assertPermissionRecovery(context: LiveContext): Promise { verifyBehavior( context, 'runtime-permission-recovery', - 'fixture observed native prompt accept and deny, then app-active readback after pm revoke', + 'fixture observed native prompt accept and deny, then relaunched readback after the pm revoke killed the app', ); } +// Android kills the app process whenever a *granted* runtime permission is revoked, and +// `settings permission reset` revokes. Every reset therefore relaunches the fixture so the +// following steps always run against a live Automation lab. async function resetAndRequestMicrophonePermission( context: LiveContext, action: 'accept' | 'deny' | 'accept after denial', @@ -65,10 +70,7 @@ async function resetAndRequestMicrophonePermission( 'reset', 'microphone', ]); - await runStep(context, `scroll to microphone permission control before ${action}`, [ - 'scroll', - 'bottom', - ]); + await openAutomationLab(context, `restore fixture after reset before ${action}`); await runStep(context, `request Android microphone permission for ${action}`, [ 'click', 'id="automation-request-microphone"', @@ -109,6 +111,18 @@ async function assertPushBroadcast(context: LiveContext): Promise { } async function assertImeRecovery(context: LiveContext): Promise { + // Automation lab is a root route without the tab bar, so the tabs have to be restored + // before the Form tab can be clicked. This section observes the system IME, and only + // closing the session puts the test IME back, so the reopen follows a close. + await runStep(context, 'release Android test IME before IME recovery', ['close']); + await runStep(context, 'return to Android fixture tabs before IME recovery', [ + 'open', + context.appId, + '--relaunch', + '--no-test-ime', + TABS_DEEP_LINK, + ]); + await assertWaitText(context, 'Agent Device Tester'); await runStep(context, 'open Android fixture form for IME recovery', ['click', 'label="Form"']); await assertWaitText(context, 'Checkout form'); await runStep(context, 'fill Android IME recovery field', [ @@ -121,8 +135,17 @@ async function assertImeRecovery(context: LiveContext): Promise { 'status', ]); assert.equal(status.json?.data?.visible, true, JSON.stringify(status.json)); + // Names what `visible: true` only implies: the injecting test IME draws nothing, so reading + // it here would mean the reopen above failed to hand input back to the system keyboard. + assert.notEqual( + status.json?.data?.inputMethodPackage, + ANDROID_TEST_IME_PACKAGE, + JSON.stringify(status.json), + ); await runStep(context, 'dismiss Android IME after fill recovery', ['keyboard', 'dismiss']); - await runStep(context, 'scroll to Android IME diagnostic', ['scroll', 'bottom']); + // The diagnostic sits between the IME field and the delivery choices, so scrolling to the + // bottom of the form scrolls past it. + await runStep(context, 'scroll to Android IME diagnostic', ['scroll', 'down', '0.3']); await assertWaitText(context, 'Android fill input was captured by the active keyboard'); await assertElementText(context, 'id="field-ime-capture-target"', 'agent-device'); verifyBehavior( diff --git a/test/integration/android-emulator-e2e/live-observability-scenario.ts b/test/integration/android-emulator-e2e/live-observability-scenario.ts index ba9de674b..1d5ea1703 100644 --- a/test/integration/android-emulator-e2e/live-observability-scenario.ts +++ b/test/integration/android-emulator-e2e/live-observability-scenario.ts @@ -65,7 +65,9 @@ async function assertLogs(context: LiveContext): Promise { async function assertTraceAndRecording(context: LiveContext): Promise { const tracePath = path.join(context.artifactDir, 'fixture.adtrace'); - await runStep(context, 'reveal Android quick actions before trace', ['scroll', 'down', '0.7']); + // 0.7 of a viewport scrolls the quick actions off the top on the pixel_7 profile the + // nightly lane pins; half a viewport puts the whole card on screen. + await runStep(context, 'reveal Android quick actions before trace', ['scroll', 'down', '0.5']); await runStep(context, 'start Android interaction trace', ['trace', 'start', tracePath]); await runStep(context, 'trace Android visible mutation', ['press', 'id="home-open-catalog"']); await runStep(context, 'stop Android interaction trace', ['trace', 'stop', tracePath]); @@ -104,6 +106,9 @@ async function assertBatchAndEvents(context: LiveContext): Promise { await runStep(context, 'return to Android fixture home before batch', ['click', 'label="Home"']); await runStep(context, 'restore Android fixture home title before batch', ['scroll', 'top']); await assertWaitText(context, 'Agent Device Tester'); + // The release notice sits below the gesture lab card on the pixel_7 profile, and Android + // snapshots only carry on-screen nodes, so both batch targets are revealed first. + await runStep(context, 'reveal Android release notice before batch', ['scroll', 'down', '0.3']); const batch = await runStep(context, 'run Android nested semantic read batch', [ 'batch', '--steps', @@ -114,7 +119,7 @@ async function assertBatchAndEvents(context: LiveContext): Promise { }, { command: 'is', - input: { predicate: 'visible', selector: 'id="home-title"' }, + input: { predicate: 'visible', selector: 'id="release-notice"' }, }, ]), ]); @@ -135,7 +140,9 @@ async function assertBatchAndEvents(context: LiveContext): Promise { const result = await runStep( context, cursor === undefined ? 'read Android event timeline' : `read Android events from ${cursor}`, - cursor === undefined ? ['events', '4'] : ['events', '4', cursor], + // Each page read appends its own request events, so a page smaller than that tail + // never catches up with a full-tier session's timeline. + cursor === undefined ? ['events', '50'] : ['events', '50', cursor], ); return (result.json?.data ?? {}) as EventTimelinePage; }); From 8d36ad8da92348782978c58d54f1fe4ecd70a983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 18 Aug 2026 08:19:09 +0200 Subject: [PATCH 2/2] test(android): repair the drifted fixture replays and pin the catalog canary (#1781 A1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up. The nested batch regression now checks a sibling card instead of the notice that owns `dismiss-notice`: resolving a child already proves its parent is present, so the old target could not fail on its own. Confirmed on a Pixel 7 / API 36 emulator that `gesture-lab-card` and `dismiss-notice` are on screen together at the scenario's existing 0.3 reveal (both present at 0.2-0.4; the card is gone by 0.5). Getting a full-tier run to complete then required repairing what the lane had never executed: - `01-navigation-scroll.ad` clicked `label="Catalog, 0 new notifications"`. #1543 made the cart badge conditional, so the live label is `Catalog` — what the iOS twin already used. - The catalog scroll canary lives inside the scrolling content, and Android accessibility snapshots carry on-screen nodes only, so every state except the initial `top` was unobservable: `wait "Catalog scroll: down|bottom|up"` could never pass, whatever the swipe coordinates were. `stickyHeaderIndices` pins that one line, which makes all four states readable at any offset on both platforms rather than tuning the .ad around a canary that scrolls away. - `gesture-lab-android.ad` started its multi-pointer gestures at y=1040, inside the target when the file was last repaired but 90px from its top edge after #1567 moved the card (targets now span y=949-1525). The second pointer landed outside the view, which reads as "the gesture did nothing". Multi-pointer gestures now start at the target centre, and the header comment records the geometry they depend on. Evidence: the lane's own command (`AGENT_DEVICE_ANDROID_E2E_TIER=full` over smoke-android-emulator.test.ts) passes end to end on a Pixel 7 / API 36 AVD with a CI-equivalent fixture APK (cached native + head JS through the same repack the workflow runs): 9/9 scenarios, 153s. --- examples/test-app/replays/gesture-lab-android.ad | 16 +++++++++------- examples/test-app/src/screens/CatalogScreen.tsx | 4 ++++ .../live-observability-scenario.ts | 11 +++++++---- .../android/fixture/01-navigation-scroll.ad | 2 +- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/examples/test-app/replays/gesture-lab-android.ad b/examples/test-app/replays/gesture-lab-android.ad index c4e57183e..53b8787d3 100644 --- a/examples/test-app/replays/gesture-lab-android.ad +++ b/examples/test-app/replays/gesture-lab-android.ad @@ -1,5 +1,7 @@ -# Coordinates target the CI emulator profile: pixel_7, 1080x2400 @ 420 dpi -# (gesture card spans y=754-1329 there). Another size/density moves the card +# Coordinates target the CI emulator profile: pixel_7, 1080x2400 @ 420 dpi, where the gesture +# targets span y=949-1525 (the two-pointer target is the left half, x=91-540). Multi-pointer +# gestures start at the target's center: a start near an edge puts the second pointer outside +# the view, which reads as "the gesture did nothing". Another size/density moves the targets # and the canary waits below fail honestly rather than silently passing. context platform=android kind=emulator timeout=60000 @@ -12,10 +14,10 @@ wait "Gesture lab" 30000 wait "gesture canary ready" 5000 wait "two-pointer pan activations 0" 5000 -gesture pan 220 1040 180 0 500 +gesture pan 300 1237 180 0 500 wait "two-pointer pan activations 0" 5000 -gesture pan 220 1040 180 0 500 --pointer-count 2 +gesture pan 300 1237 180 0 500 --pointer-count 2 wait "two-pointer pan activations 1" 5000 gesture fling right 700 1040 300 @@ -29,7 +31,7 @@ react-native dismiss-overlay wait "gesture canary ready" 5000 wait "pan changed no, pinch changed no, rotate changed no" 30000 -gesture pinch 1.5 540 1040 +gesture pinch 1.5 540 1237 wait "pan changed no, pinch changed yes, rotate changed no" 5000 open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" @@ -37,7 +39,7 @@ react-native dismiss-overlay wait "gesture canary ready" 5000 wait "pan changed no, pinch changed no, rotate changed no" 30000 -gesture rotate 35 540 1040 +gesture rotate 35 540 1237 wait "pan changed no, pinch changed no, rotate changed yes" 5000 open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}" @@ -45,7 +47,7 @@ react-native dismiss-overlay wait "gesture canary ready" 5000 wait "pan changed no, pinch changed no, rotate changed no" 30000 -gesture transform 540 1040 60 0 1.75 30 600 +gesture transform 540 1237 60 0 1.75 30 600 wait "pan changed yes, pinch changed yes, rotate changed yes" 5000 close diff --git a/examples/test-app/src/screens/CatalogScreen.tsx b/examples/test-app/src/screens/CatalogScreen.tsx index a3bf322ea..086f47ea4 100644 --- a/examples/test-app/src/screens/CatalogScreen.tsx +++ b/examples/test-app/src/screens/CatalogScreen.tsx @@ -62,6 +62,10 @@ export function CatalogScreen(props: CatalogScreenProps) { onScroll={updateScrollState} scrollEventThrottle={16} showsVerticalScrollIndicator={false} + // Pins the scroll-state line (child index 1) so every state it reports stays on screen. + // Android accessibility snapshots carry on-screen nodes only, so a canary that scrolls + // away can only ever be read as "top" — the state it holds before anything scrolls. + stickyHeaderIndices={[1]} > { await runStep(context, 'return to Android fixture home before batch', ['click', 'label="Home"']); await runStep(context, 'restore Android fixture home title before batch', ['scroll', 'top']); await assertWaitText(context, 'Agent Device Tester'); - // The release notice sits below the gesture lab card on the pixel_7 profile, and Android - // snapshots only carry on-screen nodes, so both batch targets are revealed first. - await runStep(context, 'reveal Android release notice before batch', ['scroll', 'down', '0.3']); + // Android snapshots only carry on-screen nodes, so both batch targets are revealed first. + // 0.3 of a viewport is the band where the gesture lab card and the release notice below it + // are on screen together on the pixel_7 profile the lane pins; by 0.4 the card is gone. + await runStep(context, 'reveal Android batch targets', ['scroll', 'down', '0.3']); const batch = await runStep(context, 'run Android nested semantic read batch', [ 'batch', '--steps', @@ -119,7 +120,9 @@ async function assertBatchAndEvents(context: LiveContext): Promise { }, { command: 'is', - input: { predicate: 'visible', selector: 'id="release-notice"' }, + // A sibling card, not the notice that owns `dismiss-notice`: resolving a child already + // proves its parent is present, so a parent target could not fail on its own. + input: { predicate: 'visible', selector: 'id="gesture-lab-card"' }, }, ]), ]); diff --git a/test/integration/replays/android/fixture/01-navigation-scroll.ad b/test/integration/replays/android/fixture/01-navigation-scroll.ad index a1010764f..8f651c37e 100644 --- a/test/integration/replays/android/fixture/01-navigation-scroll.ad +++ b/test/integration/replays/android/fixture/01-navigation-scroll.ad @@ -5,7 +5,7 @@ env APP_TARGET="com.callstack.agentdevicelab" open "${APP_TARGET}" --relaunch wait "Agent Device Tester" 30000 -click "label=\"Catalog, 0 new notifications\"" +click "label=\"Catalog\"" wait "Catalog scroll: top" 5000 swipe 220 700 220 380 wait "Catalog scroll: down" 5000