diff --git a/.github/workflows/replays-manual.yml b/.github/workflows/replays-manual.yml new file mode 100644 index 000000000..14706da13 --- /dev/null +++ b/.github/workflows/replays-manual.yml @@ -0,0 +1,197 @@ +name: Replay Manual + +# Parked here by #1781 A1. Both suites failed every scheduled run from 2026-07-24 on — the +# Android suite inside full-tier scenarios that had never executed end to end, the iOS suite +# on varying steps — so on a schedule they only produced noise. Manual dispatch keeps them +# runnable while they are being repaired; `pnpm check:gate-manifest` reports the checks they +# declare as manual-only (scripts/gate/declarations.ts) so the missing owner stays visible. +# Move these jobs back into replays-nightly.yml once a dispatch run is green. + +on: + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + manual-android: + name: Android Full Emulator Suite + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/android-e2e-state + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + + - name: Setup Android replay host + id: android-replay-host + uses: ./.github/actions/setup-android-replay-host + with: + package-helpers: 'true' + + - name: Restore Android fixture APK + id: fixture-app + uses: ./.github/actions/setup-fixture-app + with: + platform: android + install: 'false' + require-artifact: 'true' + + - name: Mark Android emulator setup complete + id: android-setup + run: echo "seconds=$(date +%s)" >> "$GITHUB_OUTPUT" + + - name: Run Android full emulator suite + uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0 + with: + api-level: 36 + arch: x86_64 + profile: pixel_7 + target: google_apis_playstore + emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -no-metrics + script: | + # android-emulator-runner executes this with /usr/bin/sh, which is dash on + # ubuntu runners and rejects `-o pipefail`. The serial assignment below is + # validated by the `test -n` guard instead. + set -eu + ANDROID_SERIAL="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1; exit }')" + test -n "$ANDROID_SERIAL" + BOOT_FINISHED_AT="$(date +%s)" + pnpm gate build + pnpm clean:daemon + adb -s "$ANDROID_SERIAL" install -r "${{ steps.fixture-app.outputs.apk-path }}" + node --experimental-strip-types src/bin.ts replay examples/test-app/replays/drag-android.ad --platform android --serial "$ANDROID_SERIAL" --session nightly-android-target-drag --json + pnpm clean:daemon + FULL_STARTED_AT="$(date +%s)" + AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_E2E_TIER=full AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="${{ steps.fixture-app.outputs.apk-path }}" AGENT_DEVICE_FIXTURE_APP_ID="${{ steps.fixture-app.outputs.app-id }}" node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-android-emulator.test.ts + FULL_FINISHED_AT="$(date +%s)" + echo "Android setup before emulator: $((BOOT_FINISHED_AT - ${{ steps.android-setup.outputs.seconds }}))s" + echo "Android full scenario wall time: $((FULL_FINISHED_AT - FULL_STARTED_AT))s" + pnpm gate replay-android --retries 2 --artifacts-dir test/artifacts/replays-android --report-junit test/artifacts/replays-android.junit.xml + + - name: Upload Android artifacts + if: always() + uses: ./.github/actions/upload-agent-device-artifacts + with: + artifact-name: replay-nightly-android-artifacts + agent-state-dir: ${{ env.AGENT_DEVICE_STATE_DIR }} + + manual-ios: + name: iOS Replay Suite + runs-on: macos-26 + timeout-minutes: 80 + env: + IOS_RUNTIME_VERSION: '26.2' + AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state + AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived + AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000' + AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + with: + cache-dependency-path: | + pnpm-lock.yaml + examples/test-app/pnpm-lock.yaml + + - name: Restore and build iOS XCTest runner + uses: ./.github/actions/setup-apple-runner-build + with: + derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} + cache-key-prefix: ios-runner-prebuilt + cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }} + gate: swift-runner-ios + xcuitest-platform: ios + xcuitest-destination: generic/platform=iOS Simulator + + - name: Boot iOS test simulator + id: ios-simulator + uses: ./.github/actions/boot-ios-test-simulator + with: + runtime-version: ${{ env.IOS_RUNTIME_VERSION }} + preferred-device-name: iPhone 17 Pro + + - name: Preflight iOS runner through public CLI + run: | + pnpm clean:daemon + node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json + pnpm clean:daemon + + - name: Run iOS simulator replay suite + uses: ./.github/actions/run-gate + with: + gate: replay-ios + args: | + --udid + ${{ steps.ios-simulator.outputs.simulator-udid }} + --retries + 2 + --artifacts-dir + test/artifacts/replays-ios-simulator + --report-junit + test/artifacts/replays-ios-simulator.junit.xml + + - name: Fetch current fixture app + id: fixture-app + uses: ./.github/actions/setup-fixture-app + with: + install: 'false' + + - name: Report fixture cache source + run: | + echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" + + - name: Prove selector drag reaches its destination on iOS + run: | + xcrun simctl install "${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.fixture-app.outputs.app-path }}" + pnpm clean:daemon + node --experimental-strip-types src/bin.ts replay examples/test-app/replays/drag.ad --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --session nightly-ios-target-drag --json + pnpm clean:daemon + + - name: Run full fixture-backed iOS simulator E2E + env: + AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} + AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} + AGENT_DEVICE_IOS_E2E: '1' + AGENT_DEVICE_IOS_E2E_TIER: full + AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + run: | + pnpm gate build + node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts + + - name: Run iOS physical device replay suite + if: env.IOS_UDID != '' + env: + IOS_UDID: ${{ vars.IOS_UDID }} + uses: ./.github/actions/run-gate + with: + gate: replay-ios-device + args: | + --udid + ${{ env.IOS_UDID }} + --retries + 2 + --artifacts-dir + test/artifacts/replays-ios-device + --report-junit + test/artifacts/replays-ios-device.junit.xml + + - name: Upload iOS artifacts + if: always() + uses: ./.github/actions/upload-agent-device-artifacts + with: + artifact-name: replay-nightly-ios-artifacts + agent-state-dir: ${{ env.AGENT_DEVICE_STATE_DIR }} + runner-derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} diff --git a/.github/workflows/replays-nightly.yml b/.github/workflows/replays-nightly.yml index 83f6cd378..623f3be33 100644 --- a/.github/workflows/replays-nightly.yml +++ b/.github/workflows/replays-nightly.yml @@ -1,5 +1,9 @@ name: Replay Nightly +# The device replay suites this workflow used to carry moved to replays-manual.yml (#1781 A1) +# after failing every scheduled run from 2026-07-24 on. What stays here is what a schedule can +# still keep honest: the device-free parser fuzz lane. + on: schedule: - cron: '0 3 * * *' @@ -99,180 +103,3 @@ jobs: done [ "$found" = 1 ] || echo 'No run envelope was produced — the lane failed before it could run.' } >> "$GITHUB_STEP_SUMMARY" - - nightly-android: - name: Android Full Emulator Suite - runs-on: ubuntu-latest - timeout-minutes: 15 - env: - AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/android-e2e-state - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup toolchain - uses: ./.github/actions/setup-node-pnpm - - - name: Setup Android replay host - id: android-replay-host - uses: ./.github/actions/setup-android-replay-host - with: - package-helpers: 'true' - - - name: Restore Android fixture APK - id: fixture-app - uses: ./.github/actions/setup-fixture-app - with: - platform: android - install: 'false' - require-artifact: 'true' - - - name: Mark Android emulator setup complete - id: android-setup - run: echo "seconds=$(date +%s)" >> "$GITHUB_OUTPUT" - - - name: Run Android full emulator suite - uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0 - with: - api-level: 36 - arch: x86_64 - profile: pixel_7 - target: google_apis_playstore - emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -no-metrics - script: | - # android-emulator-runner executes this with /usr/bin/sh, which is dash on - # ubuntu runners and rejects `-o pipefail`. The serial assignment below is - # validated by the `test -n` guard instead. - set -eu - ANDROID_SERIAL="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1; exit }')" - test -n "$ANDROID_SERIAL" - BOOT_FINISHED_AT="$(date +%s)" - pnpm gate build - pnpm clean:daemon - adb -s "$ANDROID_SERIAL" install -r "${{ steps.fixture-app.outputs.apk-path }}" - node --experimental-strip-types src/bin.ts replay examples/test-app/replays/drag-android.ad --platform android --serial "$ANDROID_SERIAL" --session nightly-android-target-drag --json - pnpm clean:daemon - FULL_STARTED_AT="$(date +%s)" - AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_E2E_TIER=full AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="${{ steps.fixture-app.outputs.apk-path }}" AGENT_DEVICE_FIXTURE_APP_ID="${{ steps.fixture-app.outputs.app-id }}" node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-android-emulator.test.ts - FULL_FINISHED_AT="$(date +%s)" - echo "Android setup before emulator: $((BOOT_FINISHED_AT - ${{ steps.android-setup.outputs.seconds }}))s" - echo "Android full scenario wall time: $((FULL_FINISHED_AT - FULL_STARTED_AT))s" - pnpm gate replay-android --retries 2 --artifacts-dir test/artifacts/replays-android --report-junit test/artifacts/replays-android.junit.xml - - - name: Upload Android artifacts - if: always() - uses: ./.github/actions/upload-agent-device-artifacts - with: - artifact-name: replay-nightly-android-artifacts - agent-state-dir: ${{ env.AGENT_DEVICE_STATE_DIR }} - - nightly-ios: - name: iOS Replay Suite - runs-on: macos-26 - timeout-minutes: 80 - env: - IOS_RUNTIME_VERSION: '26.2' - AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state - AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived - AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000' - AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload} - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Setup toolchain - uses: ./.github/actions/setup-node-pnpm - with: - cache-dependency-path: | - pnpm-lock.yaml - examples/test-app/pnpm-lock.yaml - - - name: Restore and build iOS XCTest runner - uses: ./.github/actions/setup-apple-runner-build - with: - derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} - cache-key-prefix: ios-runner-prebuilt - cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }} - gate: swift-runner-ios - xcuitest-platform: ios - xcuitest-destination: generic/platform=iOS Simulator - - - name: Boot iOS test simulator - id: ios-simulator - uses: ./.github/actions/boot-ios-test-simulator - with: - runtime-version: ${{ env.IOS_RUNTIME_VERSION }} - preferred-device-name: iPhone 17 Pro - - - name: Preflight iOS runner through public CLI - run: | - pnpm clean:daemon - node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json - pnpm clean:daemon - - - name: Run iOS simulator replay suite - uses: ./.github/actions/run-gate - with: - gate: replay-ios - args: | - --udid - ${{ steps.ios-simulator.outputs.simulator-udid }} - --retries - 2 - --artifacts-dir - test/artifacts/replays-ios-simulator - --report-junit - test/artifacts/replays-ios-simulator.junit.xml - - - name: Fetch current fixture app - id: fixture-app - uses: ./.github/actions/setup-fixture-app - with: - install: 'false' - - - name: Report fixture cache source - run: | - echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" - - - name: Prove selector drag reaches its destination on iOS - run: | - xcrun simctl install "${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.fixture-app.outputs.app-path }}" - pnpm clean:daemon - node --experimental-strip-types src/bin.ts replay examples/test-app/replays/drag.ad --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --session nightly-ios-target-drag --json - pnpm clean:daemon - - - name: Run full fixture-backed iOS simulator E2E - env: - AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} - AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} - AGENT_DEVICE_IOS_E2E: '1' - AGENT_DEVICE_IOS_E2E_TIER: full - AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} - run: | - pnpm gate build - node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts - - - name: Run iOS physical device replay suite - if: env.IOS_UDID != '' - env: - IOS_UDID: ${{ vars.IOS_UDID }} - uses: ./.github/actions/run-gate - with: - gate: replay-ios-device - args: | - --udid - ${{ env.IOS_UDID }} - --retries - 2 - --artifacts-dir - test/artifacts/replays-ios-device - --report-junit - test/artifacts/replays-ios-device.junit.xml - - - name: Upload iOS artifacts - if: always() - uses: ./.github/actions/upload-agent-device-artifacts - with: - artifact-name: replay-nightly-ios-artifacts - agent-state-dir: ${{ env.AGENT_DEVICE_STATE_DIR }} - runner-derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 0b060a062..91c72745f 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -421,8 +421,19 @@ The iOS lane combines three evidence layers instead of treating a catalog mentio - pull requests run a short JSON-asserting fixture smoke against the real built CLI, daemon, XCTest runner, and simulator; -- the scheduled/manual nightly workflow adds device lifecycle, system UI, recording/trace, and - fixture replay scenarios without putting those slower operations on the pull-request merge gate; +- `Replay Manual` (`.github/workflows/replays-manual.yml`) adds device lifecycle, system UI, + recording/trace, and fixture replay scenarios without putting those slower operations on the + pull-request merge gate. It is **`workflow_dispatch` only** since #1781 A1 — the suite failed + every scheduled run from 2026-07-24 on — so `replay-ios`, `replay-ios-device`, and + `replay-android` run when someone dispatches the workflow and at no other time. That gap is + declared in `scripts/gate/declarations.ts` (`MANUAL_ONLY_OWNERS`) and printed by + `pnpm check:gate-manifest` on every run. Each entry names the dispatch lane that still runs it, + and the audit resolves that name: deleting the parked job, putting it back on a schedule, or + removing its `run-gate` step fails the manifest instead of leaving the check listed as merely + parked. `replay-android` is marked `opaque` because its gate sits inside the third-party + emulator action's `script:`, which the loader does not read (#1429), so the job's existence is + the whole attestation. Put the jobs back on a schedule once a dispatch run is green and delete + their entries; - command-contract, workflow-live, and capability-denial rows explicitly own functionality that requires remote sources, unavailable host permissions, or CI setup outside the app session. @@ -471,7 +482,7 @@ AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE='agent-device-test-app:///automation?eve node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts ``` -Use `AGENT_DEVICE_IOS_E2E_TIER=full` for the nightly subset. Step history, coverage reports, +Use `AGENT_DEVICE_IOS_E2E_TIER=full` for the `Replay Manual` subset. Step history, coverage reports, screenshots, recordings, traces, and failure context are written below `test/artifacts/ios-simulator/` and uploaded by the existing shared artifact action. The six Settings replays remain additive OS-chrome coverage and are not modified by this suite. diff --git a/scripts/gate/audit.test.ts b/scripts/gate/audit.test.ts index b610160f6..4d5339c29 100644 --- a/scripts/gate/audit.test.ts +++ b/scripts/gate/audit.test.ts @@ -6,8 +6,10 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; +import { CHECK_CATALOG } from '../check-affected/checks.ts'; import { audit, formatFailures } from './audit.ts'; -import { loadModel, type Model } from './model.ts'; +import { MANUAL_ONLY_OWNERS } from './declarations.ts'; +import { covered, loadModel, type Model } from './model.ts'; import { loadLanes } from './workflows.ts'; const repoRoot = path.resolve(import.meta.dirname, '../..'); @@ -68,6 +70,117 @@ jobs: ); }); +// The exemption is what keeps the tree green while these lanes are parked, so nothing else +// would notice it going stale: a re-scheduled lane would silently keep its "nothing runs this" +// declaration, and the next parked check would be waved through under a name that no longer +// describes it. +test('every manual-only declaration names a registered check no qualifying lane owns', () => { + for (const id of Object.keys(MANUAL_ONLY_OWNERS)) { + const spec = CHECK_CATALOG.find((entry) => entry.id === id); + assert.ok(spec, `manual-only declaration "${id}" names no registered check`); + assert.equal( + covered(spec, null, base).covered, + false, + `"${id}" is owned by a pull_request/schedule lane again — drop its MANUAL_ONLY_OWNERS entry`, + ); + } +}); + +/** The live model with the parked dispatch lanes rewritten, to age the declaration on purpose. */ +function withManualLanes(rewrite: (lane: Model['lanes'][number]) => Model['lanes'][number] | null) { + const parked = new Set(Object.values(MANUAL_ONLY_OWNERS).map((owner) => owner.lane)); + return { + ...base, + lanes: base.lanes.flatMap((lane) => { + if (!parked.has(lane.label)) return [lane]; + const rewritten = rewrite(lane); + return rewritten ? [rewritten] : []; + }), + }; +} + +test('deleting a manual-only declaration reports its check as unowned', () => { + const failures = audit(base, { manualOnly: {}, unprovable: {} }); + for (const id of Object.keys(MANUAL_ONLY_OWNERS)) { + assert.ok( + failures.some( + (failure) => failure.assertion === 'owned' && failure.message.includes(`"${id}"`), + ), + `dropping the declaration for "${id}" must surface it as unowned, not as wired`, + ); + } +}); + +// The failure this record exists to prevent: parked coverage quietly becoming deleted coverage. +test('deleting a parked job fails instead of reading as still parked', () => { + const failures = audit(withManualLanes(() => null)); + for (const id of Object.keys(MANUAL_ONLY_OWNERS)) { + assert.ok( + failures.some( + (failure) => + failure.assertion === 'manual-only' && + failure.message.includes(`"${id}"`) && + failure.message.includes('no workflow defines'), + ), + `deleting the lane "${MANUAL_ONLY_OWNERS[id]?.lane}" must fail the manifest for "${id}"`, + ); + } +}); + +// `qualifying` alone cannot carry this: swapping `workflow_dispatch` for `push` keeps the lane +// non-qualifying, so without the trigger kinds the audit would stay green while the run nobody +// starts by hand became a run nobody starts by hand *or* reviews. +test('a parked lane re-triggered by push fails even though push does not qualify', () => { + const failures = audit( + withManualLanes((lane) => ({ ...lane, qualifying: false, triggers: ['push'] })), + ); + assert.ok( + failures.some( + (failure) => + failure.assertion === 'manual-only' && failure.message.includes('is triggered by push'), + ), + 'replacing workflow_dispatch with another non-qualifying trigger must fail', + ); +}); + +test('a parked lane that loses workflow_dispatch entirely fails', () => { + const failures = audit(withManualLanes((lane) => ({ ...lane, triggers: [] }))); + assert.ok( + failures.some( + (failure) => + failure.assertion === 'manual-only' && failure.message.includes('no trigger at all'), + ), + ); +}); + +test('a parked lane back on a schedule fails until its declaration is deleted', () => { + const failures = audit(withManualLanes((lane) => ({ ...lane, qualifying: true }))); + assert.ok( + failures.some( + (failure) => + failure.assertion === 'manual-only' && + /runs on pull_request\/schedule again/.test(failure.message), + ), + ); +}); + +// Android is the reason `opaque` exists: its gate sits inside a third-party action's `script:`, +// so the lane's own steps can never show it and the job's existence is the whole attestation. +test('a parked lane that stops declaring its gate fails unless the gate is opaque', () => { + const failures = audit(withManualLanes((lane) => ({ ...lane, gates: [] }))); + assert.ok( + failures.some( + (failure) => + failure.assertion === 'manual-only' && failure.message.includes('no longer declares gate'), + ), + 'a visible parked lane losing its run-gate step must fail', + ); + assert.ok( + !failures.some((failure) => failure.message.includes('gate "replay-android"')), + 'the opaque Android entry cannot assert a step the loader never reads', + ); +}); + test('unknown assertion kinds remain visible in the report', () => { const report = formatFailures([ { assertion: 'owned', message: 'a' }, diff --git a/scripts/gate/audit.ts b/scripts/gate/audit.ts index 43318282f..2955f6460 100644 --- a/scripts/gate/audit.ts +++ b/scripts/gate/audit.ts @@ -1,13 +1,31 @@ // Structural owners, path reachability, and suite registration over the derived model. import { CHECK_CATALOG } from '../check-affected/checks.ts'; -import { REPORTING_SCRIPTS, UNPROVABLE_OWNERS } from './declarations.ts'; +import type { CheckId } from '../check-affected/model.ts'; +import { + MANUAL_ONLY_OWNERS, + type ManualOnlyOwner, + REPORTING_SCRIPTS, + UNPROVABLE_OWNERS, +} from './declarations.ts'; import { categories, checkUnits, covered, scriptUnits, type Model } from './model.ts'; export type Failure = { readonly assertion: string; readonly message: string }; +/** The hand-written half of the manifest, injectable so its own removal can be tested. */ +export type GateDeclarations = { + readonly manualOnly: Readonly>; + readonly unprovable: Readonly>; +}; + +const DECLARED: GateDeclarations = { + manualOnly: MANUAL_ONLY_OWNERS, + unprovable: UNPROVABLE_OWNERS, +}; + const HEADINGS: Readonly> = { owned: 'Registered checks no lane declares', + 'manual-only': 'Manual-only declarations no dispatch lane backs', gate: 'Gate ids that name no registered check', surface: 'Execution surfaces the manifest does not model', 'path-coverage': 'Paths whose selected checks no triggered lane runs', @@ -37,14 +55,15 @@ function fail(assertion: string, message: string): Failure { return { assertion, message }; } -// Every registered check is declared by some qualifying lane, unit by unit. -function unowned( - model: Model, - unprovable: Readonly> = UNPROVABLE_OWNERS, -): Failure[] { +// Every registered check is declared by some qualifying lane, unit by unit. The two declared +// exemptions differ in kind: UNPROVABLE_OWNERS says "it runs, this loader cannot see it", +// MANUAL_ONLY_OWNERS says "nothing runs it automatically" — and check.ts reports the latter by +// name so the gap is read as a gap. +function unowned(model: Model, declarations: GateDeclarations): Failure[] { + const exempt = { ...declarations.unprovable, ...declarations.manualOnly }; return CHECK_CATALOG.flatMap((spec) => { const result = covered(spec, null, model); - if (result.covered || spec.id in unprovable) return []; + if (result.covered || spec.id in exempt) return []; const missing = result.missing.length > 0 ? result.missing.join(', ') : '(no units resolved)'; return [ fail( @@ -56,6 +75,74 @@ function unowned( }); } +/** + * `workflow_dispatch` and nothing else. Read from the trigger names rather than from + * `qualifying`, which only says "not pull_request/schedule": `push`, `release` and friends are + * non-qualifying too, and every one of them starts the run without a human. + */ +function dispatchOnly(lane: Model['lanes'][number]): boolean { + return ( + lane.triggers.length > 0 && lane.triggers.every((trigger) => trigger === 'workflow_dispatch') + ); +} + +function describeTriggers(lane: Model['lanes'][number]): string { + return lane.triggers.length > 0 ? lane.triggers.join(', ') : 'no trigger at all'; +} + +// A manual-only declaration is an attestation about a lane, so the lane has to back it: the +// job still exists, still runs on dispatch and nothing else, and (unless its gate lives inside +// a surface the loader cannot open) still declares the gate. Without this, deleting the parked +// job — or quietly re-triggering it — would leave the manifest green and still printing the +// check as manual-only: parked coverage silently turned into deleted or unattested coverage. +function manualOnly(model: Model, declared: Readonly>): Failure[] { + return Object.entries(declared).flatMap(([id, owner]) => { + if (!REGISTERED.has(id)) { + return [fail('manual-only', `"${id}" names no registered check. Drop the declaration.`)]; + } + const lane = model.lanes.find((candidate) => candidate.label === owner.lane); + if (!lane) { + return [ + fail( + 'manual-only', + `"${id}" is declared manual-only on lane "${owner.lane}", which no workflow defines. ` + + `Nothing runs this check at all — restore the job, or drop the check.`, + ), + ]; + } + if (lane.qualifying) { + return [ + fail( + 'manual-only', + `"${id}" is declared manual-only, but "${owner.lane}" runs on pull_request/schedule ` + + `again. Delete the MANUAL_ONLY_OWNERS entry so the check counts as wired.`, + ), + ]; + } + if (!dispatchOnly(lane)) { + return [ + fail( + 'manual-only', + `"${owner.lane}" is triggered by ${describeTriggers(lane)}, so "${id}" is not ` + + `manual-only — "manual" is a claim about who starts the run, and every trigger ` + + `other than workflow_dispatch starts it without them. Restore a dispatch-only ` + + `workflow, or declare what actually runs the check.`, + ), + ]; + } + if (!owner.opaque && !lane.gates.includes(id as CheckId)) { + return [ + fail( + 'manual-only', + `"${owner.lane}" no longer declares gate "${id}", so the manual-only declaration ` + + `attests to a step that is gone. Restore the run-gate step, or drop the check.`, + ), + ]; + } + return []; + }); +} + function gateIds(model: Model): Failure[] { return model.lanes .filter((lane) => lane.qualifying) @@ -141,9 +228,10 @@ function orphanProjects(model: Model): Failure[] { .map((name) => fail('registered', `Vitest project "${name}" is run by no registered check.`)); } -export function audit(model: Model): Failure[] { +export function audit(model: Model, declarations: GateDeclarations = DECLARED): Failure[] { return [ - ...unowned(model), + ...unowned(model, declarations), + ...manualOnly(model, declarations.manualOnly), ...gateIds(model), ...laneSurfaces(model), ...pathCoverage(model), diff --git a/scripts/gate/check.ts b/scripts/gate/check.ts index c984f943b..91a460da9 100644 --- a/scripts/gate/check.ts +++ b/scripts/gate/check.ts @@ -10,7 +10,7 @@ import { pathToFileURL } from 'node:url'; import { runCmdSync } from '../../src/utils/exec.ts'; import { audit, formatFailures } from './audit.ts'; -import { UNPROVABLE_OWNERS } from './declarations.ts'; +import { MANUAL_ONLY_OWNERS, UNPROVABLE_OWNERS } from './declarations.ts'; import { loadModel } from './model.ts'; function main(): number { @@ -25,10 +25,14 @@ function main(): number { // Unprovable owners are reported rather than folded into the count: a check whose lane // this tree cannot show running should not read the same as one it can. const unprovable = Object.keys(UNPROVABLE_OWNERS).length; + // Manual-only checks are named, not counted: "3 manual-only" reads like a tally, while the + // ids read like the list of things nothing runs until someone dispatches them. + const manual = Object.keys(MANUAL_ONLY_OWNERS).sort(); process.stdout.write( `gate manifest: ok — ${new Set(gates).size} checks wired across ` + `${model.lanes.filter((lane) => lane.gates.length > 0).length} lanes` + - `${unprovable > 0 ? `, ${unprovable} declared unprovable` : ''}.\n`, + `${unprovable > 0 ? `, ${unprovable} declared unprovable` : ''}` + + `${manual.length > 0 ? `, manual-only: ${manual.join(', ')}` : ''}.\n`, ); return 0; } diff --git a/scripts/gate/declarations.ts b/scripts/gate/declarations.ts index 04ae7ffd5..064e2a236 100644 --- a/scripts/gate/declarations.ts +++ b/scripts/gate/declarations.ts @@ -1,4 +1,4 @@ -// The three small facts the manifest cannot derive from package scripts and workflow YAML. +// The four small facts the manifest cannot derive from package scripts and workflow YAML. export const OPAQUE_RUNNERS: Readonly> = { // This wrapper runs Vitest over every project, then retries owned contention failures. @@ -19,12 +19,60 @@ export const REPORTING_SCRIPTS: Readonly> = { ].join(' '), }; -export const UNPROVABLE_OWNERS: Readonly> = { - 'replay-android': [ - 'Replay Nightly / Android Replay Suite runs `pnpm gate replay-android`, but inside the', - '`script:` input of `reactivecircus/android-emulator-runner` — shell handed to a', - 'third-party action, which this loader does not read. The suite executes; the manifest', - 'cannot see it. Routing the emulator lane through steps it can read is the open item', - 'named in #1429.', - ].join(' '), +// A check whose lane runs, but through a surface the loader cannot read. Empty right now: +// the one entry moved to MANUAL_ONLY_OWNERS when its lane stopped running automatically. +export const UNPROVABLE_OWNERS: Readonly> = {}; + +/** + * A check whose only lane is a `workflow_dispatch` workflow. + * + * `lane` is the load-bearing field, not documentation: the audit resolves it against the + * derived model and fails when the named lane is gone, has become a pull_request/schedule + * lane again, or has stopped declaring the gate. Without it the record would be a plain + * allowlist, and deleting the parked job would read as "parked" forever — parked coverage + * quietly becoming deleted coverage. + */ +export type ManualOnlyOwner = { + /** Lane label, exactly as the loader builds it: ` / `. */ + readonly lane: string; + /** + * Set when the lane's own steps cannot show the gate, so the lane's existence is the whole + * attestation the model can make. Every opaque entry needs the surface named in `reason`. + */ + readonly opaque?: true; + readonly reason: string; +}; + +// Checks nothing runs on the way in or on a schedule — *unowned by design and temporarily*, +// the opposite of the claim UNPROVABLE_OWNERS makes. `check.ts` prints them by name on every +// run rather than folding them into the wired count, because a check that quietly loses its +// owner reads exactly like a green build. Delete the entry when the lane goes back on +// `pull_request` or `schedule`; the audit fails if an entry outlives its parking, and fails +// the other way too if a declaration is deleted while its lane is still dispatch-only. +export const MANUAL_ONLY_OWNERS: Readonly> = { + 'replay-android': { + lane: 'Replay Manual / Android Full Emulator Suite', + opaque: true, + reason: [ + 'Parked on `workflow_dispatch` by #1781 A1. The lane runs `pnpm gate replay-android`', + 'inside the `script:` input of `reactivecircus/android-emulator-runner` — shell handed', + 'to a third-party action, which this loader does not read — so the job existing is all', + 'the model can attest, and the gate would be invisible even back on a schedule. Routing', + 'the emulator lane through steps the loader opens is the open item named in #1429.', + ].join(' '), + }, + 'replay-ios': { + lane: 'Replay Manual / iOS Replay Suite', + reason: [ + 'Parked on `workflow_dispatch` by #1781 A1 after the suite failed every scheduled run', + 'from 2026-07-24 on.', + ].join(' '), + }, + 'replay-ios-device': { + lane: 'Replay Manual / iOS Replay Suite', + reason: [ + 'Parked on `workflow_dispatch` by #1781 A1 with the simulator suite it shares a job', + 'with; the step is additionally skipped unless the `IOS_UDID` repository variable is set.', + ].join(' '), + }, }; diff --git a/scripts/gate/workflows.test.ts b/scripts/gate/workflows.test.ts index 35906c161..2ac09ef9f 100644 --- a/scripts/gate/workflows.test.ts +++ b/scripts/gate/workflows.test.ts @@ -88,6 +88,42 @@ test('lanes carry the workflow spelling the catalog used, and only real triggers assert.equal(deploy?.qualifying, false, 'a push-only lane gates nothing on the way in'); }); +test('a workflow_dispatch-only lane owns nothing, however many gates it declares', () => { + const [lane] = planted({ + '.github/workflows/planted.yml': `name: Planted +on: + workflow_dispatch: +jobs: + planted: + steps: + - uses: ./.github/actions/run-gate + with: + gate: replay-ios`, + }); + assert.deepEqual(lane?.gates, ['replay-ios'], 'the gate is still read'); + assert.equal(lane?.qualifying, false, 'but nothing dispatches itself, so it owns nothing'); + assert.deepEqual(lane?.triggers, ['workflow_dispatch'], 'the trigger kind survives the model'); +}); + +// `qualifying` collapses every trigger into one bit, and two very different lanes share the +// `false` side of it: one a human starts, one that starts itself on every push. +test('trigger kinds survive the model, not just whether they qualify', () => { + const [push] = planted({ + '.github/workflows/planted.yml': `name: Planted +on: + push: + branches: [main] +jobs: + planted: + steps: + - run: echo hi`, + }); + assert.equal(push?.qualifying, false); + assert.deepEqual(push?.triggers, ['push']); + const nightly = model.lanes.find((lane) => lane.workflow === 'replays-nightly.yml'); + assert.deepEqual(nightly?.triggers, ['schedule', 'workflow_dispatch']); +}); + test('a gate invoked from inside a composite action belongs to the calling lane', () => { const android = model.lanes.find((lane) => lane.label === 'Android / Smoke Tests'); assert.ok( diff --git a/scripts/gate/workflows.ts b/scripts/gate/workflows.ts index befa1042a..f6b93b999 100644 --- a/scripts/gate/workflows.ts +++ b/scripts/gate/workflows.ts @@ -13,6 +13,12 @@ export type Lane = { readonly workflow: string; readonly label: string; readonly qualifying: boolean; + /** + * The workflow's trigger names, kept rather than collapsed into `qualifying`: "not a + * pull_request/schedule lane" and "a lane only a human can start" are different facts, and + * a declaration that means the second cannot be checked against the first. + */ + readonly triggers: readonly string[]; readonly gates: readonly CheckId[]; readonly verbatim: readonly string[]; readonly paths: readonly string[]; @@ -103,6 +109,7 @@ function workflowLanes( workflow: file, label: laneLabel(doc.name ?? file, job.name ?? jobId), qualifying, + triggers: Object.keys(on), gates: [...new Set(declaredGates(job.steps ?? [], root))], verbatim: (job.steps ?? []).flatMap((step) => typeof step.run === 'string' ? verbatimScripts(step.run, scripts) : [],