fix: inspect windows whose title changes during the read

dist/build.js

@@ -39,6 +39,7 @@ const patches = [
     "peekaboo-sheet-window.patch",
     "peekaboo-sheet-focus.patch",
     "peekaboo-restore.patch",
+    "peekaboo-title.patch",
 ].map((name) => join(native, "patches", name));
 function spawn(command, env) {
     return spawnSync(command[0], command.slice(1), { encoding: "utf8", env: env ?? process.env });

native/patches/README.md

@@ -208,6 +208,15 @@ requested-state mismatch for `restoreWindow` is read again, at most 8 times 100
 posts no input and repeats the full identity, process-generation and bounds validation. Identity
 failures, lookup errors and a window that stays minimized remain indeterminate.
 
+`peekaboo-title.patch` lets an Accessibility observation succeed while the window's title changes,
+found in the [Ace desktop study](https://github.com/githubnext/ace/issues/427). The resolver reads the
+title from WindowServer; the traversal reads it again from Accessibility, so a page with a live title
+failed binding validation with `window-context window title`. A window context that names a window is
+already pinned by that exact window ID, its WindowServer receipt and bounds, and one process generation,
+and the traversal selects its AX window by that ID, so its title is no longer compared. A context without
+a window ID still must match the title. The resolved target keeps its title; the client reports the newer
+traversal title. Input actions bind by window ID, receipt and focused-element identity, never the title.
+
 `desktop-tools-build` resolves only `Package.resolved` versions, checks the pin and checkout revision,
 and assembles the ordered patch stack in a private Git index. A build compares the checkout
 with each complete ordered prefix, since later patches can change earlier patch contexts.

native/patches/peekaboo-title.patch

@@ -0,0 +1,138 @@
+diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift
+index 3b700e0c6..dbca4edfe 100644
+--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift
++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift
+@@ -643,7 +643,10 @@ enum PeekabooBridgeDesktopObservationBinding {
+             if let windowID = context.windowID, windowID != targetWindow.windowID {
+                 return "window-context window ID"
+             }
+-            if let title = context.windowTitle, title != targetWindow.title {
++            // The window ID, receipt, bounds and process generation pin a context to the target.
++            // Its title is read later during traversal and may be newer, so it is evidence only
++            // when the context names no window.
++            if context.windowID == nil, let title = context.windowTitle, title != targetWindow.title {
+                 return "window-context window title"
+             }
+             if let bounds = context.windowBounds, bounds != targetWindow.bounds {
+diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeObservationTitleChurnTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeObservationTitleChurnTests.swift
+new file mode 100644
+index 000000000..06741d227
+--- /dev/null
++++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeObservationTitleChurnTests.swift
+@@ -0,0 +1,116 @@
++import CoreGraphics
++import Foundation
++import PeekabooFoundation
++import Testing
++@testable import PeekabooAutomationKit
++@testable import PeekabooBridge
++
++@Suite(.serialized)
++struct PeekabooBridgeObservationTitleChurnTests: DesktopObservationBindingFixtureProviding {
++    private static let detection = DesktopDetectionOptions(
++        mode: .accessibility,
++        requiresFreshAccessibilityTree: true)
++    private static let request = DesktopObservationRequest(
++        target: .pid(42, window: .id(73)),
++        detection: Self.detection)
++
++    @Test
++    @MainActor
++    func `a title that changes during the traversal binds by window ID and reports the newer title`() async throws {
++        let observed = try Self.observation(traversedTitle: "Inbox (3)", traversedWindowID: 73)
++
++        #expect(PeekabooBridgeDesktopObservationBinding.mismatch(
++            request: Self.request,
++            result: observed.result,
++            requireSelectorResolutionProof: true) == nil)
++        let handled = try await PeekabooBridgeRequestContext.$usesAttestedOperationResultSemantics.withValue(true) {
++            try await Self.server(provider: ObservationProvider(result: observed.result)).handleAuthorized(
++                .desktopObservation(Self.request),
++                peer: nil,
++                permissions: Self.permissions)
++        }
++        guard case let .desktopObservation(response) = handled.response else {
++            Issue.record("Expected a desktop observation response")
++            return
++        }
++        #expect(response.target.window?.title == "Inbox (2)")
++        #expect(response.elements?.metadata.windowContext?.windowTitle == "Inbox (3)")
++
++        let signed = try await Self.makeBundle(
++            request: .desktopObservation(Self.request),
++            response: .desktopObservation(observed.result),
++            target: .window(observed.identity))
++        try signed.validateIntegrity()
++    }
++
++    @Test
++    func `a traversed title still cannot stand in for a missing or different window ID`() throws {
++        let unnamed = try Self.observation(traversedTitle: "Inbox (3)", traversedWindowID: nil)
++        #expect(PeekabooBridgeDesktopObservationBinding.mismatch(
++            request: Self.request,
++            result: unnamed.result) == "window-context window title")
++
++        let other = try Self.observation(traversedTitle: "Inbox (2)", traversedWindowID: 74)
++        #expect(PeekabooBridgeDesktopObservationBinding.mismatch(
++            request: Self.request,
++            result: other.result) == "window-context window ID")
++    }
++
++    private static func observation(
++        traversedTitle: String,
++        traversedWindowID: Int?) throws -> (result: DesktopObservationResult, identity: WindowMutationIdentity)
++    {
++        let fixture = Self.windowResult(.init(
++            processIdentifier: 42,
++            generation: 1001,
++            bundleIdentifier: "dev.peekaboo.fixture",
++            applicationName: "Fixture",
++            windowID: 73,
++            title: "Inbox (2)",
++            index: 0,
++            windowSelector: .id(73)))
++        let original = fixture.result.target
++        let windowProof = try #require(original.selectorResolutionProofs?.last)
++        let target = ResolvedObservationTarget(
++            kind: original.kind,
++            app: original.app,
++            window: original.window,
++            bounds: original.bounds,
++            detectionContext: original.detectionContext,
++            captureScaleHint: original.captureScaleHint,
++            selectorResolutionProofs: [windowProof],
++            mutationTargetIdentity: original.mutationTargetIdentity)
++        let resolved = try #require(original.detectionContext)
++        let traversed = WindowContext(
++            applicationName: resolved.applicationName,
++            applicationBundleId: resolved.applicationBundleId,
++            applicationProcessId: resolved.applicationProcessId,
++            applicationProcessStartIdentity: fixture.identity.ownerProcessStartIdentity,
++            windowTitle: traversedTitle,
++            windowID: traversedWindowID,
++            windowBounds: resolved.windowBounds,
++            windowMutationIdentity: resolved.windowMutationIdentity,
++            shouldFocusWebContent: Self.detection.allowWebFocusFallback,
++            includeMenuBarElements: Self.detection.includeMenuBarElements,
++            traversalBudget: Self.detection.traversalBudget,
++            requiresFreshAccessibilityTree: true)
++        let base = fixture.result
++        let result = DesktopObservationResult(
++            target: target,
++            capture: base.capture,
++            elements: ElementDetectionResult(
++                snapshotId: "title-churn",
++                screenshotPath: "",
++                elements: DetectedElements(),
++                metadata: DetectionMetadata(
++                    detectionTime: 0,
++                    elementCount: 0,
++                    method: "AXorcist",
++                    windowContext: traversed)),
++            files: base.files,
++            timings: base.timings,
++            diagnostics: base.diagnostics,
++            captureContentDigest: base.captureContentDigest)
++        return (result, fixture.identity)
++    }
++}

native/sources/client/client.swift

@@ -179,7 +179,8 @@ private enum Client {
 			inspection_mode: accessibility ? InspectionMode.accessibility.rawValue : InspectionMode.pixels.rawValue,
 			note: accessibility ? nil : "Pixel snapshot: no Accessibility elements or focused control. Its snapshot_id authorizes only screenshot-point clicks, point scrolls, and drags in this exact window; element, text, key, and insertion actions are refused.",
 			application_name: result.target.app?.name,
-			window_title: result.target.window?.title,
+			// The traversal reads the title after target resolution, so a live title is newer there.
+			window_title: result.elements?.metadata.windowContext?.windowTitle ?? result.target.window?.title,
 			snapshot_id: result.files.publishedSnapshotID,
 			element_count: result.elements?.metadata.elementCount ?? 0,
 			ui_elements: result.elements?.elements.all.map(Element.init) ?? [],

src/build.ts

@@ -40,6 +40,7 @@ const patches = [
 	"peekaboo-sheet-window.patch",
 	"peekaboo-sheet-focus.patch",
 	"peekaboo-restore.patch",
+	"peekaboo-title.patch",
 ].map((name) => join(native, "patches", name));
 
 function spawn(command: string[], env?: NodeJS.ProcessEnv) {