From a839a7a6f939356076ebabb072c0385c38d5479b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 5 Jun 2026 14:56:23 +0200 Subject: [PATCH] fix: retry iOS runner prepare launch --- src/__tests__/cli-help.test.ts | 11 + .../__tests__/runner-command-retry.test.ts | 129 ++++++++++ src/platforms/ios/runner-lifecycle.ts | 236 +++++++++++++----- src/utils/cli-command-overrides.ts | 2 +- src/utils/cli-help.ts | 3 +- website/docs/docs/commands.md | 14 ++ 6 files changed, 335 insertions(+), 60 deletions(-) diff --git a/src/__tests__/cli-help.test.ts b/src/__tests__/cli-help.test.ts index ffd74ba10..54b1dbfe6 100644 --- a/src/__tests__/cli-help.test.ts +++ b/src/__tests__/cli-help.test.ts @@ -40,6 +40,15 @@ test('appstate --help prints command help and skips daemon dispatch', async () = assert.match(result.stdout, /Global flags:/); }); +test('prepare help documents iOS runner CI setup', async () => { + const result = await runCliCapture(['help', 'prepare']); + assert.equal(result.code, 0); + assert.equal(result.calls.length, 0); + assert.match(result.stdout, /prepare ios-runner --platform ios\|macos/); + assert.match(result.stdout, /health-checks the XCTest runner/); + assert.match(result.stdout, /after boot\/install and before replay\/test/); +}); + test('connect help documents cloud auth environment origins', async () => { const result = await runCliCapture(['help', 'connect']); assert.equal(result.code, 0); @@ -91,6 +100,8 @@ test('help workflow preserves known device workaround guidance', async () => { assert.match(result.stdout, /agent-device clipboard write "some text"/); assert.match(result.stdout, /provider-native text injection when available/); assert.match(result.stdout, /Do not switch to raw adb, clipboard, or paste as an agent fallback/); + assert.match(result.stdout, /exact key that includes the agent-device package and Xcode version/); + assert.match(result.stdout, /Avoid broad restore-key fallbacks/); }); test('help unknown command prints error plus global usage and skips daemon dispatch', async () => { diff --git a/src/platforms/ios/__tests__/runner-command-retry.test.ts b/src/platforms/ios/__tests__/runner-command-retry.test.ts index 62ae12041..ee85813ef 100644 --- a/src/platforms/ios/__tests__/runner-command-retry.test.ts +++ b/src/platforms/ios/__tests__/runner-command-retry.test.ts @@ -1,6 +1,7 @@ import { beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { IOS_SIMULATOR } from '../../../__tests__/test-utils/index.ts'; +import { clearRequestCanceled, markRequestCanceled } from '../../../daemon/request-cancel.ts'; import { AppError } from '../../../utils/errors.ts'; import type { RunnerSession } from '../runner-session-types.ts'; @@ -128,6 +129,134 @@ test('prepareIosRunner invalidates rebuilt sessions when bad-cache recovery heal ]); }); +test('prepareIosRunner retries a fresh launch session when the health check cannot connect', async () => { + const stuckSession = makeRunnerSession({ port: 8100 }); + const relaunchedSession = makeRunnerSession({ port: 8101 }); + + mockEnsureRunnerSession.mockResolvedValueOnce(stuckSession).mockResolvedValueOnce(relaunchedSession); + mockExecuteRunnerCommandWithSession + .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'Runner did not accept connection')) + .mockResolvedValueOnce({ uptimeMs: 42 }); + + const result = await prepareIosRunner(IOS_SIMULATOR, { + healthTimeoutMs: 90_000, + buildTimeoutMs: 300_000, + }); + + assert.deepEqual(result.runner, { uptimeMs: 42 }); + assert.equal(result.recoveryReason, 'Runner did not accept connection'); + assert.equal(mockEnsureRunnerSession.mock.calls[0]?.[1]?.cleanStaleBundles, undefined); + assert.deepEqual(mockInvalidateRunnerSession.mock.calls, [ + [stuckSession, 'prepare_runner_health_retry'], + ]); + assert.equal(mockEnsureRunnerSession.mock.calls.length, 2); + assert.equal(mockEnsureRunnerSession.mock.calls[1]?.[1]?.cleanStaleBundles, true); + assert.equal(mockEnsureRunnerSession.mock.calls[1]?.[1]?.forceRunnerXctestrunRebuild, undefined); + assert.equal(mockExecuteRunnerCommandWithSession.mock.calls[1]?.[1], relaunchedSession); + assert.deepEqual( + mockEmitDiagnostic.mock.calls.find( + ([event]) => event.phase === 'ios_runner_prepare_health_retry', + )?.[0].data, + { + command: 'uptime', + commandId: mockExecuteRunnerCommandWithSession.mock.calls[0]?.[2].commandId, + sessionId: stuckSession.sessionId, + attempt: 1, + maxAttempts: 2, + reason: 'Runner did not accept connection', + }, + ); +}); + +test('prepareIosRunner does not force a rebuild when the relaunched fresh session still cannot connect', async () => { + const missArtifact = makeRunnerArtifact({ + xctestrunPath: '/tmp/miss.xctestrun', + cache: 'miss', + artifact: 'valid', + }); + const exactArtifact = makeRunnerArtifact({ + xctestrunPath: '/tmp/exact.xctestrun', + cache: 'exact', + artifact: 'valid', + }); + const stuckSession = makeRunnerSession({ + port: 8100, + xctestrunPath: missArtifact.xctestrunPath, + xctestrunArtifact: missArtifact, + }); + const relaunchedSession = makeRunnerSession({ + port: 8101, + xctestrunPath: exactArtifact.xctestrunPath, + xctestrunArtifact: exactArtifact, + }); + + mockEnsureRunnerSession.mockResolvedValueOnce(stuckSession).mockResolvedValueOnce(relaunchedSession); + mockExecuteRunnerCommandWithSession + .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'Runner did not accept connection')) + .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'Runner did not accept connection')); + + await assert.rejects( + () => + prepareIosRunner(IOS_SIMULATOR, { + healthTimeoutMs: 90_000, + forceRunnerXctestrunRebuild: false, + }), + /Runner did not accept connection/, + ); + + assert.deepEqual(mockInvalidateRunnerSession.mock.calls, [ + [stuckSession, 'prepare_runner_health_retry'], + [relaunchedSession, 'prepare_runner_health_failed'], + ]); + assert.equal(mockMarkRunnerXctestrunArtifactBadForRun.mock.calls.length, 0); + assert.equal(mockEnsureRunnerSession.mock.calls.length, 2); + assert.equal(mockEnsureRunnerSession.mock.calls[0]?.[1]?.cleanStaleBundles, undefined); + assert.equal(mockEnsureRunnerSession.mock.calls[0]?.[1]?.forceRunnerXctestrunRebuild, false); + assert.equal(mockEnsureRunnerSession.mock.calls[1]?.[1]?.cleanStaleBundles, true); + assert.equal(mockEnsureRunnerSession.mock.calls[1]?.[1]?.forceRunnerXctestrunRebuild, false); +}); + +test('prepareIosRunner does not relaunch after non-retryable runner startup failures', async () => { + const failedSession = makeRunnerSession({ port: 8100 }); + + mockEnsureRunnerSession.mockResolvedValueOnce(failedSession); + mockExecuteRunnerCommandWithSession.mockRejectedValueOnce( + new AppError('COMMAND_FAILED', 'xcodebuild exited early'), + ); + + await assert.rejects( + () => prepareIosRunner(IOS_SIMULATOR, { healthTimeoutMs: 90_000 }), + /xcodebuild exited early/, + ); + + assert.equal(mockEnsureRunnerSession.mock.calls.length, 1); + assert.equal(mockInvalidateRunnerSession.mock.calls.length, 0); + assert.equal(mockMarkRunnerXctestrunArtifactBadForRun.mock.calls.length, 0); +}); + +test('prepareIosRunner does not relaunch after request cancellation', async () => { + const requestId = 'prepare-canceled-before-retry'; + const stuckSession = makeRunnerSession({ port: 8100 }); + + mockEnsureRunnerSession.mockResolvedValueOnce(stuckSession); + mockExecuteRunnerCommandWithSession.mockImplementationOnce(() => { + markRequestCanceled(requestId); + throw new AppError('COMMAND_FAILED', 'Runner did not accept connection'); + }); + + try { + await assert.rejects( + () => prepareIosRunner(IOS_SIMULATOR, { healthTimeoutMs: 90_000, requestId }), + /request canceled/, + ); + } finally { + clearRequestCanceled(requestId); + } + + assert.equal(mockEnsureRunnerSession.mock.calls.length, 1); + assert.equal(mockInvalidateRunnerSession.mock.calls.length, 0); +}); + test('mutating commands restart stale ready sessions when the preflight probe never reaches the runner', async () => { const staleSession = makeRunnerSession({ port: 8100, ready: true }); const freshSession = makeRunnerSession({ port: 8101, ready: false }); diff --git a/src/platforms/ios/runner-lifecycle.ts b/src/platforms/ios/runner-lifecycle.ts index 8e55ed4b0..30ee09450 100644 --- a/src/platforms/ios/runner-lifecycle.ts +++ b/src/platforms/ios/runner-lifecycle.ts @@ -1,7 +1,7 @@ import { AppError } from '../../utils/errors.ts'; import type { DeviceInfo } from '../../utils/device.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; -import { getRequestSignal } from '../../daemon/request-cancel.ts'; +import { getRequestSignal, isRequestCanceledError } from '../../daemon/request-cancel.ts'; import { RUNNER_COMMAND_TIMEOUT_MS, RUNNER_STARTUP_TIMEOUT_MS } from './runner-transport.ts'; import { type RunnerSession, @@ -28,6 +28,8 @@ import { handleRunnerTransportErrorAfterCommandSend } from './runner-command-rec export type PrepareIosRunnerOptions = AppleRunnerPrepareOptions; export type PrepareIosRunnerResult = AppleRunnerPrepareResult; +const PREPARE_RUNNER_HEALTH_MAX_SESSION_ATTEMPTS = 2; + export async function prepareLocalIosRunner( device: DeviceInfo, options: PrepareIosRunnerOptions, @@ -35,69 +37,191 @@ export async function prepareLocalIosRunner( assertRunnerRequestActive(options.requestId); const signal = getRequestSignal(options.requestId); const command = withRunnerCommandId({ command: 'uptime' }); - let session: RunnerSession | undefined; + let recoveryReason: string | undefined; + for (let attempt = 1; attempt <= PREPARE_RUNNER_HEALTH_MAX_SESSION_ATTEMPTS; attempt += 1) { + const result = await runPrepareAttempt({ + device, + command, + options, + signal, + attempt, + recoveryReason, + }); + if (result.kind === 'prepared') return result.result; + recoveryReason = result.recoveryReason; + } + + // Unreachable while PREPARE_RUNNER_HEALTH_MAX_SESSION_ATTEMPTS is positive. + throw new AppError('COMMAND_FAILED', 'iOS runner prepare failed'); +} + +type PrepareAttemptResult = + | { kind: 'prepared'; result: PrepareIosRunnerResult } + | { kind: 'retry'; recoveryReason: string }; + +async function runPrepareAttempt(params: { + device: DeviceInfo; + command: RunnerCommand; + options: PrepareIosRunnerOptions; + signal: AbortSignal | undefined; + attempt: number; + recoveryReason: string | undefined; +}): Promise { + const { device, command, options, signal, attempt, recoveryReason } = params; + const connectStartedAt = Date.now(); + const session = await ensureRunnerSession(device, { + ...options, + cleanStaleBundles: attempt > 1 ? true : options.cleanStaleBundles, + }); + const connectMs = Date.now() - connectStartedAt; try { - const connectStartedAt = Date.now(); - session = await ensureRunnerSession(device, options); - const connectMs = Date.now() - connectStartedAt; - return recordPrepareResult( + const result = await runPrepareHealthCheck(device, session, command, options, signal, connectMs, { + recoveryReason, + }); + return { kind: 'prepared', result: recordPrepareResult(device, result) }; + } catch (error) { + return await handlePrepareHealthFailure({ device, - await runPrepareHealthCheck(device, session, command, options, signal, connectMs), - ); - } catch (err) { - const appErr = err instanceof AppError ? err : new AppError('COMMAND_FAILED', String(err)); - if (!session || !shouldRecoverBadCachedRunnerArtifact(appErr, session)) { - throw err; - } - const reason = appErr.message || 'runner_health_failed'; - await invalidateRunnerSession(session, 'prepare_cached_runner_health_failed'); - await markRunnerXctestrunArtifactBadForRun(session.xctestrunArtifact, reason); - const connectStartedAt = Date.now(); - const rebuiltSession = await ensureRunnerSession(device, { - ...options, - cleanStaleBundles: true, - forceRunnerXctestrunRebuild: true, + session, + command, + options, + signal, + attempt, + error, }); - const connectMs = Date.now() - connectStartedAt; - try { - const recovered = await runPrepareHealthCheck( + } +} + +async function handlePrepareHealthFailure(params: { + device: DeviceInfo; + session: RunnerSession; + command: RunnerCommand; + options: PrepareIosRunnerOptions; + signal: AbortSignal | undefined; + attempt: number; + error: unknown; +}): Promise { + const { device, session, command, options, signal, attempt, error } = params; + const appErr = error instanceof AppError ? error : new AppError('COMMAND_FAILED', String(error)); + if (attempt === 1 && shouldRecoverBadCachedRunnerArtifact(appErr, session)) { + return { + kind: 'prepared', + result: await recoverBadCachedRunnerArtifact({ device, - rebuiltSession, + session, command, options, signal, - connectMs, - { recoveryReason: reason }, - ); - emitDiagnostic({ - level: 'info', - phase: 'ios_runner_prepare_bad_cache_recovered', - data: { - command: command.command, - commandId: command.commandId, - sessionId: rebuiltSession.sessionId, - xctestrunPath: rebuiltSession.xctestrunArtifact?.xctestrunPath, - reason, - }, - }); - return recordPrepareResult(device, recovered); - } catch (retryErr) { - await invalidateRunnerSession(rebuiltSession, 'prepare_rebuilt_runner_health_failed'); - const wrapped = wrapPrepareHealthFailure(retryErr, rebuiltSession, reason); - emitPrepareDiagnostic(device, { - cache: rebuiltSession.xctestrunArtifact?.cache, - artifact: rebuiltSession.xctestrunArtifact?.artifact, - buildMs: rebuiltSession.xctestrunArtifact?.buildMs, - connectMs, - healthCheckMs: 0, + error: appErr, + }), + }; + } + if (!shouldRetryPrepareRunnerHealthFailure(appErr)) { + throw error; + } + const reason = appErr.message || 'runner_health_failed'; + if (attempt >= PREPARE_RUNNER_HEALTH_MAX_SESSION_ATTEMPTS) { + await invalidateRunnerSessionBestEffort(session, 'prepare_runner_health_failed'); + throw error; + } + + assertRunnerRequestActive(options.requestId); + await invalidateRunnerSession(session, 'prepare_runner_health_retry'); + emitDiagnostic({ + level: 'warn', + phase: 'ios_runner_prepare_health_retry', + data: { + command: command.command, + commandId: command.commandId, + sessionId: session.sessionId, + attempt, + maxAttempts: PREPARE_RUNNER_HEALTH_MAX_SESSION_ATTEMPTS, + reason, + }, + }); + return { kind: 'retry', recoveryReason: reason }; +} + +async function recoverBadCachedRunnerArtifact(params: { + device: DeviceInfo; + session: RunnerSession & { + xctestrunArtifact: NonNullable; + }; + command: RunnerCommand; + options: PrepareIosRunnerOptions; + signal: AbortSignal | undefined; + error: AppError; +}): Promise { + const { device, session, command, options, signal, error } = params; + const reason = error.message || 'runner_health_failed'; + await invalidateRunnerSession(session, 'prepare_cached_runner_health_failed'); + await markRunnerXctestrunArtifactBadForRun(session.xctestrunArtifact, reason); + const connectStartedAt = Date.now(); + const rebuiltSession = await ensureRunnerSession(device, { + ...options, + cleanStaleBundles: true, + forceRunnerXctestrunRebuild: true, + }); + const connectMs = Date.now() - connectStartedAt; + try { + const recovered = await runPrepareHealthCheck( + device, + rebuiltSession, + command, + options, + signal, + connectMs, + { recoveryReason: reason }, + ); + emitDiagnostic({ + level: 'info', + phase: 'ios_runner_prepare_bad_cache_recovered', + data: { + command: command.command, + commandId: command.commandId, + sessionId: rebuiltSession.sessionId, xctestrunPath: rebuiltSession.xctestrunArtifact?.xctestrunPath, - failureReason: wrapped.message, - }); - throw wrapped; - } + reason, + }, + }); + return recordPrepareResult(device, recovered); + } catch (retryErr) { + await invalidateRunnerSessionBestEffort( + rebuiltSession, + 'prepare_rebuilt_runner_health_failed', + ); + const wrapped = wrapPrepareHealthFailure(retryErr, rebuiltSession, reason); + emitPrepareDiagnostic(device, { + cache: rebuiltSession.xctestrunArtifact?.cache, + artifact: rebuiltSession.xctestrunArtifact?.artifact, + buildMs: rebuiltSession.xctestrunArtifact?.buildMs, + connectMs, + healthCheckMs: 0, + xctestrunPath: rebuiltSession.xctestrunArtifact?.xctestrunPath, + failureReason: wrapped.message, + }); + throw wrapped; } } +async function invalidateRunnerSessionBestEffort( + session: RunnerSession, + reason: Parameters[1], +): Promise { + try { + await invalidateRunnerSession(session, reason); + } catch {} +} + +function shouldRetryPrepareRunnerHealthFailure(error: AppError): boolean { + if (isRequestCanceledError(error)) return false; + return ( + isRetryableRunnerError(error) || + shouldRetryRunnerConnectError(error) || + isPrepareHealthTimeout(error) + ); +} + // fallow-ignore-next-line complexity export async function executeRunnerCommand( device: DeviceInfo, @@ -260,11 +384,7 @@ function shouldRecoverBadCachedRunnerArtifact( } { const artifact = session.xctestrunArtifact; if (!artifact || artifact.cache === 'miss') return false; - return ( - isRetryableRunnerError(error) || - shouldRetryRunnerConnectError(error) || - isPrepareHealthTimeout(error) - ); + return shouldRetryPrepareRunnerHealthFailure(error); } function isPrepareHealthTimeout(error: AppError): boolean { diff --git a/src/utils/cli-command-overrides.ts b/src/utils/cli-command-overrides.ts index 220973472..9eff15bbd 100644 --- a/src/utils/cli-command-overrides.ts +++ b/src/utils/cli-command-overrides.ts @@ -69,7 +69,7 @@ const CLI_COMMAND_OVERRIDES = { usageOverride: 'prepare ios-runner --platform ios|macos [--timeout ]', listUsageOverride: 'prepare ios-runner --platform ios|macos', helpDescription: - 'Prepare platform helper infrastructure. ios-runner builds/reuses and starts the XCTest runner so later Apple snapshots and interactions do not pay first-use startup cost.', + 'Prepare platform helper infrastructure. ios-runner builds/reuses, starts, and health-checks the XCTest runner so later Apple snapshots and interactions do not pay first-use startup cost. In CI, run it after boot/install and before replay/test.', summary: 'Prepare platform helpers', positionalArgs: ['ios-runner'], allowedFlags: ['timeoutMs'], diff --git a/src/utils/cli-help.ts b/src/utils/cli-help.ts index b0fac6d59..0687b2554 100644 --- a/src/utils/cli-help.ts +++ b/src/utils/cli-help.ts @@ -122,7 +122,8 @@ Bootstrap: agent-device prepare ios-runner --platform ios --timeout 240000 If app id is unknown, plan devices, apps, then open . Discovery is not enough when the task asks to open/start the app. Install arguments are app/package id then artifact path. If the task says install, use install; use reinstall only when explicitly requested. Fresh runtime state is open --relaunch after install. - In Apple CI, run prepare ios-runner after boot/install and before replay/test. prepare ios-runner builds/reuses the XCTest runner and proves it can answer a lightweight command before the first snapshot pays that setup cost. + In Apple CI, run prepare ios-runner after boot/install and before replay/test. prepare ios-runner builds/reuses the XCTest runner, health-checks it with a lightweight command, and retries one stuck/non-connecting runner launch before the first snapshot pays that setup cost. + CI may cache ~/.agent-device/ios-runner/derived with an exact key that includes the agent-device package and Xcode version. Avoid broad restore-key fallbacks; prepare ios-runner already recovers bad restored runner artifacts and one retryable non-connecting runner launch. Do not open artifact paths or invent package ids. If apps lookup misses the target and no URL/artifact is provided, ask or stop. Snapshots and refs: diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 58cfd138a..9a0ee76be 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -123,6 +123,20 @@ agent-device devices --platform android --android-device-allowlist emulator-5554 - Use `--platform` to narrow discovery to Apple-family (`ios`, `tvOS`, `macOS`) or Android targets. - Use `--ios-simulator-device-set` and `--android-device-allowlist` when you need tenant- or lab-scoped discovery. +## Prepare Apple runner + +```bash +agent-device prepare ios-runner --platform ios --timeout 240000 +``` + +- `prepare ios-runner` is intended for Apple-platform CI setup before `snapshot`, `replay`, or `test`. +- Run it after the simulator/device is booted and the app is installed, but before the first snapshot, replay, or test command. +- It builds or reuses the local XCTest runner, starts a runner session, and verifies that the runner can answer a lightweight health command. +- If health checking exposes a bad restored runner artifact, Agent Device marks that artifact bad and rebuilds once. +- If a fresh runner launch gets stuck before accepting connections, Agent Device invalidates that runner session and launches it once more without forcing a rebuild. +- CI may cache `~/.agent-device/ios-runner/derived` when the cache key includes the exact Agent Device package contents and selected Xcode version. +- Avoid broad `restore-keys` fallbacks for runner caches. Reusing runner artifacts across Agent Device or Xcode versions can restore stale `.xctestrun` products; `prepare ios-runner` already handles bad exact-cache artifacts and one retryable non-connecting runner launch. + ## TV targets ```bash