ci(ios): run the full XCTest suite nightly and check the PR test list (#1781 A7)

.github/workflows/ci.yml

@@ -177,6 +177,16 @@ jobs:
         uses: ./.github/actions/run-gate
         with: { gate: gate-manifest }
 
+      # Same family as the manifest above — a CI selection that has stopped selecting what
+      # it claims. ios.yml runs a hand-written subset of the runner XCTest methods through an
+      # `-only-testing:` list, and xcodebuild treats an identifier that matches nothing as
+      # an empty selection rather than an error, so a rename drops a test with no signal —
+      # in both directions, since a typo in xctest-nightly.yml's `-skip-testing:` entry
+      # re-arms a 24-hour hang. Parse-only, no Xcode (#1781 A7).
+      - name: Check the PR XCTest selection still names real tests
+        uses: ./.github/actions/run-gate
+        with: { gate: xctest-selection }
+
   maestro-conformance:
     name: Maestro Conformance Oracle
     runs-on: ubuntu-latest

.github/workflows/xctest-nightly.yml

@@ -0,0 +1,180 @@
+name: XCTest Nightly
+
+# The full iOS runner XCTest suite (#1781 A7). The PR lane (ios.yml) names a hand-written
+# subset of the target's methods in an `-only-testing:` list; everything outside that list ran
+# nowhere at all. This lane drops the filter and runs the test plan whole, skipping only
+# `testCommand` — which is the runner's server entry point rather than a test (see the step
+# below).
+#
+# No count is quoted here on purpose. `pnpm check:xctest-selection` prints the live split
+# (declared / PR-selected / skipped / nightly-only) and is the only place those numbers are
+# derived rather than remembered. A hand-copied figure in a comment is the same class of
+# defect this lane exists to close: between this file being written and first review, the
+# suite gained three methods and the PR list gained five entries, so every number originally
+# written here was already wrong.
+#
+# Under the lane rule in #1781:
+#
+#   - Catches: regressions in the runner-semantics tests the PR list leaves out — selector
+#     matching, snapshot traversal, command journal, sequence execution, tv remote, keyboard,
+#     scroll/gesture policy. Nothing else executes them. Note the reachable set is smaller
+#     than the declared set: `testCommand` is skipped, and the `…OnTvOS` tests sit behind
+#     `#if os(tvOS)` and cannot execute on this lane's iOS Simulator destination.
+#   - Evidence: the list has never been re-derived since it was written by hand; the tests it
+#     omits have been dark for their whole life, so their current state is unmeasured. The
+#     first red night is the evidence this lane exists to produce.
+#   - Cost: this is a public repository, so nothing here is billed. The real cost is one macOS
+#     concurrency slot per night. The Apple runner build shares ios.yml's cache (identical
+#     inputs and build variant), but scheduled runs only see DEFAULT-BRANCH caches, so the hit
+#     depends on ios.yml's `push: main` run having populated the cache at the same source
+#     hash — on a miss this lane also pays a full runner build. Adds nothing to any PR.
+#   - Kill criterion: when A7's classification lands — pure Swift decision tests moved to a
+#     fast unit target, stale tests deleted, the rest kept on the simulator — this lane goes
+#     if the surviving set is small enough to run on every PR.
+#
+# Scheduled + manual only. It is deliberately NOT a PR gate: ios.yml is already over half of
+# this repo's CI spend (#1781 A9), and moving the unrun set onto it would make that worse for
+# tests whose flakiness on a shared runner is unknown.
+#
+# Note on `concurrency` below: it is keyed on `github.ref`, so a manual dispatch on the same
+# ref cancels an in-flight scheduled run. That is the intended trade — an operator asking for
+# a run wants the fresh one — but it means "dispatch while the nightly is running" loses the
+# nightly's result rather than queueing behind it.
+
+on:
+  schedule:
+    - cron: '30 4 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  full-suite:
+    name: iOS Runner Full XCTest Suite
+    runs-on: macos-26
+    # Measured, not guessed. A full local run of this exact command on matching toolchain
+    # (Xcode 26.2, iOS 26.2 Simulator) executed 154 tests in 120s wall — the suite itself is
+    # ~2 minutes, not the bulk of the job. What dominates the budget is the runner build on a
+    # derived-data cache MISS, plus checkout and simulator boot. ios.yml carries 80 minutes
+    # while doing strictly more than this lane (same build and boot, plus replays, prepare,
+    # and the fixture E2E), so half of that is generous headroom for a cache-miss night and
+    # still caps a wedged run at 45 minutes instead of two hours of held macOS slot.
+    timeout-minutes: 45
+    env:
+      # Must match ios.yml. The runtime version rides in the derived-data cache key, and
+      # AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS is BOTH an input to the cache's build-variant
+      # hash and the `-D AGENT_DEVICE_RUNNER_UNIT_TESTS` compile flag that makes the RunnerTests
+      # methods exist at all. Dropping it would build a different variant (cache miss) whose
+      # bundle contains no tests, and `xcodebuild` reports running zero tests as success — the
+      # "silent 0-test green" this lane would then be. The test-count assertion below is the
+      # backstop for exactly that.
+      IOS_RUNTIME_VERSION: '26.2'
+      AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1'
+      AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
+      RESULT_BUNDLE_PATH: ${{ github.workspace }}/.tmp/xctest-nightly/RunnerTests.xcresult
+      RESULT_SUMMARY_PATH: ${{ github.workspace }}/.tmp/xctest-nightly/test-results-summary.json
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup toolchain
+        uses: ./.github/actions/setup-node-pnpm
+        with:
+          cache-dependency-path: |
+            pnpm-lock.yaml
+            examples/test-app/pnpm-lock.yaml
+
+      # Byte-identical inputs to ios.yml's step, so both lanes restore the same cache entry
+      # and only the first one to run on a commit pays for the build.
+      - name: Restore and build iOS XCTest runner
+        uses: ./.github/actions/setup-apple-runner-build
+        with:
+          derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }}
+          cache-key-prefix: ios-runner-prebuilt
+          cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}
+          gate: swift-runner-ios
+          xcuitest-platform: ios
+          xcuitest-destination: generic/platform=iOS Simulator
+
+      - name: Boot iOS test simulator
+        id: ios-simulator
+        uses: ./.github/actions/boot-ios-test-simulator
+        with:
+          runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
+          preferred-device-name: iPhone 17 Pro
+
+      # Same command as ios.yml's targeted step minus every `-only-testing:` flag, so the
+      # xctestrun's own test plan decides what runs — with one exception.
+      #
+      # `RunnerTests/testCommand` is not a test. It is the runner's server entry point: it
+      # opens an NWListener and blocks in `XCTWaiter.wait(timeout: 24 * 60 * 60)` until a
+      # client closes the session, which is exactly how production drives it
+      # (src/platforms/apple/core/runner/runner-session.ts always passes it as the sole
+      # `-only-testing:`). It compiles unconditionally — the `#if AGENT_DEVICE_RUNNER_UNIT_TESTS`
+      # block ends well above it — so an unfiltered run reaches it in alphabetical order and
+      # hangs the job until `timeout-minutes`. Its one escape hatch,
+      # AGENT_DEVICE_RUNNER_NOOP_STARTUP, is an environment variable, and the comment above
+      # the flag records that env plumbing into a simulator test process is not dependable;
+      # `-skip-testing:` is the lever that works from here.
+      #
+      # A typo in that identifier silently re-arms the hang, so
+      # `pnpm check:xctest-selection` validates `-skip-testing:` exactly like `-only-testing:`.
+      - name: Run the full iOS runner XCTest suite
+        run: |
+          set -euo pipefail
+          XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)"
+          test -n "$XCTESTRUN_PATH"
+          mkdir -p "$(dirname "$RESULT_BUNDLE_PATH")"
+          xcodebuild test-without-building \
+            -xctestrun "$XCTESTRUN_PATH" \
+            -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \
+            -skip-testing:AgentDeviceRunnerUITests/RunnerTests/testCommand \
+            -resultBundlePath "$RESULT_BUNDLE_PATH"
+
+      # Best-effort and never the job's verdict on its own; the step below is what asserts.
+      # `--compact` first because a red night's summary is the large one, and the job summary
+      # has a 1 MiB cap — the plain form is the fallback if an Xcode version rejects the flag.
+      - name: Extract the test-results summary
+        if: always()
+        run: |
+          set -uo pipefail
+          [ -d "$RESULT_BUNDLE_PATH" ] || exit 0
+          xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" --compact \
+            > "$RESULT_SUMMARY_PATH" 2>/dev/null ||
+            xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" \
+              > "$RESULT_SUMMARY_PATH" 2>/dev/null ||
+            rm -f "$RESULT_SUMMARY_PATH"
+          exit 0
+
+      # Reports the run AND is the lane's own liveness check. `xcodebuild` exits 0 when a
+      # selection matches nothing, so "green" and "ran no tests" are the same log tail: a
+      # build variant without the unit-test compile flag, an empty test plan, or a renamed
+      # target would all read as a healthy night. Asserting the run executed tests is what
+      # tells those apart. The failure list is capped so a badly red night cannot blow the
+      # 1 MiB job-summary limit and lose the headline with it.
+      - name: Report the run and assert it executed tests
+        if: always()
+        run: |
+          set -euo pipefail
+          if [ ! -s "$RESULT_SUMMARY_PATH" ]; then
+            echo 'No usable test-results summary was produced, so nothing here can show a test ran.'
+            echo 'The suite step above failed before or during the run; read its log first.'
+            echo '### iOS runner full XCTest suite' >> "$GITHUB_STEP_SUMMARY"
+            echo 'No result-bundle summary; see the run log and the uploaded bundle.' >> "$GITHUB_STEP_SUMMARY"
+            exit 1
+          fi
+          node --experimental-strip-types scripts/xctest-run-summary.ts
+
+      - name: Upload result bundle
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: xctest-nightly-results-${{ github.run_id }}-${{ github.run_attempt }}
+          path: .tmp/xctest-nightly
+          if-no-files-found: warn

docs/agents/testing.md

@@ -37,6 +37,7 @@ The mapping it encodes, for when you need to run a gate directly or reason about
 | Platform/device response — anything emitting `platform`/`appleOs` on the wire, or shaping a daemon response | `pnpm test:integration:provider` **and** `pnpm test:coverage` |
 | Cross-platform behavior | `pnpm test:integration` |
 | Apple runner / Swift | Build the changed target with `pnpm build:xcuitest:<platform>`; use `pnpm build:xcuitest` only for shared iOS/macOS changes |
+| Runner XCTest methods (`apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/**`) | `pnpm check:xctest-selection`, which prints the live declared/PR-selected/skipped/nightly-only split — the counts move often enough that quoting one here would rot. `ios.yml` runs a hand-written subset through an `-only-testing:` list and `xctest-nightly.yml` excludes one method via `-skip-testing:`; `xcodebuild` treats an identifier matching nothing as an empty selection rather than an error, in both directions — a rename drops a test from the PR lane silently, or re-admits `RunnerTests/testCommand`, the runner's 24-hour server entry point, into the nightly and hangs it. The check fails only on a listed name no source declares; a test in no list is expected, because the nightly runs the suite whole |
 | CLI help/guidance (`src/cli/parser/cli-help.ts`, `src/cli-schema/`) | `pnpm exec vitest run src/cli/parser/__tests__ src/cli-schema/command-schema-guards.test.ts scripts/__tests__` — the `scripts/__tests__` gates enforce help-topic benchmark coverage and pin the bench's quoted CLI samples to the real renderers |
 | Help benchmark cases (`scripts/help-conformance-*.mjs`) | `pnpm exec vitest run scripts/__tests__` (deterministic gates); model-backed: `pnpm bench:help-conformance` (paid LLM calls, local only) |
 | `.ad` grammar (`src/replay/script.ts`, gesture arity, replay vars) | `pnpm exec vitest run --project unit-core test/replay-compat` — the frozen replay-compat corpus asserts which released script surfaces still parse; a flipped verdict is edited in `test/replay-compat/manifest.ts`, never in the script. Adding or re-pinning a corpus entry also runs `pnpm check:replay-compat`, which re-derives each entry from its release tag in git history |

package.json

@@ -143,14 +143,15 @@
     "check:replay-compat": "node --experimental-strip-types scripts/check-replay-compat-provenance.ts",
     "check:daemon-wire-compat": "node --experimental-strip-types scripts/wire-compat/run.ts",
     "check:daemon-wire-compat:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/wire-compat/model.test.ts",
+    "check:xctest-selection": "node --experimental-strip-types scripts/check-xctest-selection.ts",
     "check:tmpdir-leaks": "node --experimental-strip-types scripts/check-tmpdir-leaks.ts",
     "check:tmpdir-leaks:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/check-tmpdir-leaks-model.test.ts scripts/vitest-tmpdir-global-setup.test.ts scripts/node-test-tmpdir.test.ts scripts/swift-toolchain-tmpdir.test.ts",
     "check:freerange": "fr",
     "check:quick": "pnpm lint && pnpm typecheck",
     "sync:mcp-metadata": "node scripts/sync-mcp-metadata.mjs",
     "check:mcp-metadata": "node scripts/sync-mcp-metadata.mjs --check",
     "version": "pnpm sync:mcp-metadata && git add server.json",
-    "check:tooling": "pnpm format:check && pnpm lint && pnpm typecheck && pnpm check:layering && pnpm depgraph:test && pnpm check:gate-manifest:test && pnpm check:gate-manifest && pnpm check:production-exports && pnpm check:tmpdir-leaks:test && pnpm check:mcp-metadata && pnpm build && pnpm check:bundle-owner-files && pnpm check:package",
+    "check:tooling": "pnpm format:check && pnpm lint && pnpm typecheck && pnpm check:layering && pnpm depgraph:test && pnpm check:gate-manifest:test && pnpm check:gate-manifest && pnpm check:production-exports && pnpm check:tmpdir-leaks:test && pnpm check:xctest-selection && pnpm check:mcp-metadata && pnpm build && pnpm check:bundle-owner-files && pnpm check:package",
     "check:unit": "pnpm check:contention-retry && pnpm test:unit && pnpm check:tmpdir-leaks && pnpm test:smoke",
     "check": "pnpm check:tooling && pnpm check:fallow && pnpm check:unit",
     "prepack": "pnpm check:mcp-metadata && pnpm package:npm",

scripts/__tests__/xctest-run-summary.test.ts

@@ -0,0 +1,117 @@
+// The nightly XCTest lane's reporter is also its liveness check, and both halves only ever
+// execute on a macOS runner at 04:30 — so they are proven here instead.
+
+import { describe, expect, test } from 'vitest';
+import {
+  livenessFailure,
+  MAX_FAILURE_TEXT,
+  MAX_LISTED_FAILURES,
+  renderSummary,
+  type ResultSummary,
+} from '../xctest-run-summary.ts';
+
+function failures(count: number, text = 'assertion failed') {
+  return Array.from({ length: count }, (_, index) => ({
+    testName: `testCase${index}()`,
+    failureText: text,
+  }));
+}
+
+describe('the liveness check', () => {
+  test('a run that executed no tests fails, because xcodebuild calls that success', () => {
+    // The lane's whole reason to assert: a build without the unit-test compile flag, an
+    // empty test plan, or a `-skip-testing:` entry that swallowed the suite all exit 0.
+    const failure = livenessFailure({ result: 'Passed', totalTestCount: 0 });
+    expect(failure).toContain('executed no tests');
+    expect(failure).toContain('AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS');
+    expect(failure).toContain('-skip-testing');
+  });
+
+  test('a summary missing the count entirely is treated as no tests, not as unknown', () => {
+    expect(livenessFailure({})).not.toBeNull();
+  });
+
+  test('a run that executed tests passes, red or green', () => {
+    expect(livenessFailure({ totalTestCount: 153, result: 'Failed', failedTests: 9 })).toBeNull();
+    expect(livenessFailure({ totalTestCount: 1, result: 'Passed' })).toBeNull();
+  });
+});
+
+describe('the job summary', () => {
+  const green: ResultSummary = {
+    result: 'Passed',
+    totalTestCount: 153,
+    passedTests: 153,
+    failedTests: 0,
+    skippedTests: 0,
+    expectedFailures: 0,
+    startTime: 1000,
+    finishTime: 1450,
+  };
+
+  test('leads with the headline a reader needs', () => {
+    const rendered = renderSummary(green);
+    expect(rendered).toContain('- result: **Passed**');
+    expect(rendered).toContain('- executed: **153** (passed 153, failed 0, skipped 0');
+    expect(rendered).toContain('- duration: 450s');
+    expect(rendered).not.toContain('#### Failures');
+  });
+
+  test('reports an unknown duration rather than NaN when the times are absent', () => {
+    expect(renderSummary({ totalTestCount: 1 })).toContain('- duration: unknown');
+  });
+
+  test('lists failures and names each one', () => {
+    const rendered = renderSummary({ ...green, result: 'Failed', testFailures: failures(3) });
+    expect(rendered).toContain('#### Failures');
+    expect(rendered).toContain('`testCase0()`');
+    expect(rendered).toContain('assertion failed');
+  });
+
+  test('caps the list, and says how many it dropped', () => {
+    // The night this lane matters most is the night the failure list is longest, and the
+    // job summary is capped at 1 MiB — losing the headline to the tail would be the worst
+    // possible trade.
+    const rendered = renderSummary({
+      ...green,
+      result: 'Failed',
+      testFailures: failures(MAX_LISTED_FAILURES + 7),
+    });
+    const listed = rendered.split('\n').filter((line) => line.startsWith('- `test')).length;
+    expect(listed).toBe(MAX_LISTED_FAILURES);
+    expect(rendered).toContain('…and 7 more');
+  });
+
+  test('truncates one enormous failure message instead of letting it dominate', () => {
+    const rendered = renderSummary({
+      ...green,
+      result: 'Failed',
+      testFailures: failures(1, 'x'.repeat(5000)),
+    });
+    expect(rendered).toContain('x'.repeat(MAX_FAILURE_TEXT));
+    expect(rendered).not.toContain('x'.repeat(MAX_FAILURE_TEXT + 1));
+  });
+
+  test('flattens newlines so a multi-line stack cannot forge markdown structure', () => {
+    const rendered = renderSummary({
+      ...green,
+      result: 'Failed',
+      testFailures: [{ testName: 'testX()', failureText: 'line one\n#### Injected\nline two' }],
+    });
+    // Markdown headings only bind at the start of a line, so flattening is what disarms
+    // the injection: the text survives verbatim, but it can no longer open a section.
+    expect(rendered.split('\n').filter((line) => line.startsWith('#'))).toEqual([
+      '### iOS runner full XCTest suite',
+      '#### Failures',
+    ]);
+    expect(rendered).toContain('line one #### Injected line two');
+  });
+
+  test('falls back to the identifier when a failure carries no test name', () => {
+    const rendered = renderSummary({
+      ...green,
+      testFailures: [{ testIdentifierString: 'RunnerTests/testY()', failureText: 'boom' }],
+    });
+    expect(rendered).toContain('`RunnerTests/testY()`');
+  });
+});