From 6eb4fc88c74c39e2e15027a3de32ddf1e93a8af6 Mon Sep 17 00:00:00 2001 From: iamnbutler Date: Thu, 15 Oct 2026 17:24:41 -0400 Subject: [PATCH] fix: inspect windows whose title changes during the read desktop_inspect in Accessibility mode on a Chrome window whose page title changes every 2 s always failed with "The desktop observation provider returned inconsistent response evidence: window-context window title." (githubnext/ace#427, D7 and R1). The observation resolver takes the window's title from WindowServer, and the detached Accessibility traversal reads kAXTitle again later; the Bridge binding validator required the two to be equal, so any live title (unread counts, timers, chat) made the window uninspectable for input. Pixels mode has no traversal and passed. A new last patch, peekaboo-title.patch, compares a window context's title only when the context names no window. A context with a window ID is already pinned to the target by that exact ID, its WindowServer receipt (owner PID, process generation, bounds) and bounds, and the traversal selects its AX window by the same ID. The resolved target keeps its title, and the client now reports the traversal's newer title. Input actions bind by window ID, receipt and focused-element identity, never the window title, so they need no change. Why: native/patches/peekaboo-title.patch | A window context's title is skipped only when its window ID matches the target; a context without a window ID must still match the title, and window ID, receipt, bounds and process generation stay exact. --- dist/build.js | 1 + native/patches/README.md | 9 ++ native/patches/peekaboo-title.patch | 138 ++++++++++++++++++++++++++++ native/sources/client/client.swift | 3 +- src/build.ts | 1 + 5 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 native/patches/peekaboo-title.patch diff --git a/dist/build.js b/dist/build.js index a988205..393da78 100644 --- a/dist/build.js +++ b/dist/build.js @@ -39,6 +39,7 @@ const patches = [ "peekaboo-sheet-window.patch", "peekaboo-sheet-focus.patch", "peekaboo-restore.patch", + "peekaboo-title.patch", ].map((name) => join(native, "patches", name)); function spawn(command, env) { return spawnSync(command[0], command.slice(1), { encoding: "utf8", env: env ?? process.env }); diff --git a/native/patches/README.md b/native/patches/README.md index cc81564..0d67e05 100644 --- a/native/patches/README.md +++ b/native/patches/README.md @@ -208,6 +208,15 @@ requested-state mismatch for `restoreWindow` is read again, at most 8 times 100 posts no input and repeats the full identity, process-generation and bounds validation. Identity failures, lookup errors and a window that stays minimized remain indeterminate. +`peekaboo-title.patch` lets an Accessibility observation succeed while the window's title changes, +found in the [Ace desktop study](https://github.com/githubnext/ace/issues/427). The resolver reads the +title from WindowServer; the traversal reads it again from Accessibility, so a page with a live title +failed binding validation with `window-context window title`. A window context that names a window is +already pinned by that exact window ID, its WindowServer receipt and bounds, and one process generation, +and the traversal selects its AX window by that ID, so its title is no longer compared. A context without +a window ID still must match the title. The resolved target keeps its title; the client reports the newer +traversal title. Input actions bind by window ID, receipt and focused-element identity, never the title. + `desktop-tools-build` resolves only `Package.resolved` versions, checks the pin and checkout revision, and assembles the ordered patch stack in a private Git index. A build compares the checkout with each complete ordered prefix, since later patches can change earlier patch contexts. diff --git a/native/patches/peekaboo-title.patch b/native/patches/peekaboo-title.patch new file mode 100644 index 0000000..4f2103f --- /dev/null +++ b/native/patches/peekaboo-title.patch @@ -0,0 +1,138 @@ +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift +index 3b700e0c6..dbca4edfe 100644 +--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift +@@ -643,7 +643,10 @@ enum PeekabooBridgeDesktopObservationBinding { + if let windowID = context.windowID, windowID != targetWindow.windowID { + return "window-context window ID" + } +- if let title = context.windowTitle, title != targetWindow.title { ++ // The window ID, receipt, bounds and process generation pin a context to the target. ++ // Its title is read later during traversal and may be newer, so it is evidence only ++ // when the context names no window. ++ if context.windowID == nil, let title = context.windowTitle, title != targetWindow.title { + return "window-context window title" + } + if let bounds = context.windowBounds, bounds != targetWindow.bounds { +diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeObservationTitleChurnTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeObservationTitleChurnTests.swift +new file mode 100644 +index 000000000..06741d227 +--- /dev/null ++++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeObservationTitleChurnTests.swift +@@ -0,0 +1,116 @@ ++import CoreGraphics ++import Foundation ++import PeekabooFoundation ++import Testing ++@testable import PeekabooAutomationKit ++@testable import PeekabooBridge ++ ++@Suite(.serialized) ++struct PeekabooBridgeObservationTitleChurnTests: DesktopObservationBindingFixtureProviding { ++ private static let detection = DesktopDetectionOptions( ++ mode: .accessibility, ++ requiresFreshAccessibilityTree: true) ++ private static let request = DesktopObservationRequest( ++ target: .pid(42, window: .id(73)), ++ detection: Self.detection) ++ ++ @Test ++ @MainActor ++ func `a title that changes during the traversal binds by window ID and reports the newer title`() async throws { ++ let observed = try Self.observation(traversedTitle: "Inbox (3)", traversedWindowID: 73) ++ ++ #expect(PeekabooBridgeDesktopObservationBinding.mismatch( ++ request: Self.request, ++ result: observed.result, ++ requireSelectorResolutionProof: true) == nil) ++ let handled = try await PeekabooBridgeRequestContext.$usesAttestedOperationResultSemantics.withValue(true) { ++ try await Self.server(provider: ObservationProvider(result: observed.result)).handleAuthorized( ++ .desktopObservation(Self.request), ++ peer: nil, ++ permissions: Self.permissions) ++ } ++ guard case let .desktopObservation(response) = handled.response else { ++ Issue.record("Expected a desktop observation response") ++ return ++ } ++ #expect(response.target.window?.title == "Inbox (2)") ++ #expect(response.elements?.metadata.windowContext?.windowTitle == "Inbox (3)") ++ ++ let signed = try await Self.makeBundle( ++ request: .desktopObservation(Self.request), ++ response: .desktopObservation(observed.result), ++ target: .window(observed.identity)) ++ try signed.validateIntegrity() ++ } ++ ++ @Test ++ func `a traversed title still cannot stand in for a missing or different window ID`() throws { ++ let unnamed = try Self.observation(traversedTitle: "Inbox (3)", traversedWindowID: nil) ++ #expect(PeekabooBridgeDesktopObservationBinding.mismatch( ++ request: Self.request, ++ result: unnamed.result) == "window-context window title") ++ ++ let other = try Self.observation(traversedTitle: "Inbox (2)", traversedWindowID: 74) ++ #expect(PeekabooBridgeDesktopObservationBinding.mismatch( ++ request: Self.request, ++ result: other.result) == "window-context window ID") ++ } ++ ++ private static func observation( ++ traversedTitle: String, ++ traversedWindowID: Int?) throws -> (result: DesktopObservationResult, identity: WindowMutationIdentity) ++ { ++ let fixture = Self.windowResult(.init( ++ processIdentifier: 42, ++ generation: 1001, ++ bundleIdentifier: "dev.peekaboo.fixture", ++ applicationName: "Fixture", ++ windowID: 73, ++ title: "Inbox (2)", ++ index: 0, ++ windowSelector: .id(73))) ++ let original = fixture.result.target ++ let windowProof = try #require(original.selectorResolutionProofs?.last) ++ let target = ResolvedObservationTarget( ++ kind: original.kind, ++ app: original.app, ++ window: original.window, ++ bounds: original.bounds, ++ detectionContext: original.detectionContext, ++ captureScaleHint: original.captureScaleHint, ++ selectorResolutionProofs: [windowProof], ++ mutationTargetIdentity: original.mutationTargetIdentity) ++ let resolved = try #require(original.detectionContext) ++ let traversed = WindowContext( ++ applicationName: resolved.applicationName, ++ applicationBundleId: resolved.applicationBundleId, ++ applicationProcessId: resolved.applicationProcessId, ++ applicationProcessStartIdentity: fixture.identity.ownerProcessStartIdentity, ++ windowTitle: traversedTitle, ++ windowID: traversedWindowID, ++ windowBounds: resolved.windowBounds, ++ windowMutationIdentity: resolved.windowMutationIdentity, ++ shouldFocusWebContent: Self.detection.allowWebFocusFallback, ++ includeMenuBarElements: Self.detection.includeMenuBarElements, ++ traversalBudget: Self.detection.traversalBudget, ++ requiresFreshAccessibilityTree: true) ++ let base = fixture.result ++ let result = DesktopObservationResult( ++ target: target, ++ capture: base.capture, ++ elements: ElementDetectionResult( ++ snapshotId: "title-churn", ++ screenshotPath: "", ++ elements: DetectedElements(), ++ metadata: DetectionMetadata( ++ detectionTime: 0, ++ elementCount: 0, ++ method: "AXorcist", ++ windowContext: traversed)), ++ files: base.files, ++ timings: base.timings, ++ diagnostics: base.diagnostics, ++ captureContentDigest: base.captureContentDigest) ++ return (result, fixture.identity) ++ } ++} diff --git a/native/sources/client/client.swift b/native/sources/client/client.swift index e1971d8..99f2adb 100644 --- a/native/sources/client/client.swift +++ b/native/sources/client/client.swift @@ -179,7 +179,8 @@ private enum Client { inspection_mode: accessibility ? InspectionMode.accessibility.rawValue : InspectionMode.pixels.rawValue, note: accessibility ? nil : "Pixel snapshot: no Accessibility elements or focused control. Its snapshot_id authorizes only screenshot-point clicks, point scrolls, and drags in this exact window; element, text, key, and insertion actions are refused.", application_name: result.target.app?.name, - window_title: result.target.window?.title, + // The traversal reads the title after target resolution, so a live title is newer there. + window_title: result.elements?.metadata.windowContext?.windowTitle ?? result.target.window?.title, snapshot_id: result.files.publishedSnapshotID, element_count: result.elements?.metadata.elementCount ?? 0, ui_elements: result.elements?.elements.all.map(Element.init) ?? [], diff --git a/src/build.ts b/src/build.ts index da484a7..7ebb553 100644 --- a/src/build.ts +++ b/src/build.ts @@ -40,6 +40,7 @@ const patches = [ "peekaboo-sheet-window.patch", "peekaboo-sheet-focus.patch", "peekaboo-restore.patch", + "peekaboo-title.patch", ].map((name) => join(native, "patches", name)); function spawn(command: string[], env?: NodeJS.ProcessEnv) {