diff --git a/dist/build.js b/dist/build.js index a988205..393da78 100644 --- a/dist/build.js +++ b/dist/build.js @@ -39,6 +39,7 @@ const patches = [ "peekaboo-sheet-window.patch", "peekaboo-sheet-focus.patch", "peekaboo-restore.patch", + "peekaboo-title.patch", ].map((name) => join(native, "patches", name)); function spawn(command, env) { return spawnSync(command[0], command.slice(1), { encoding: "utf8", env: env ?? process.env }); diff --git a/native/patches/README.md b/native/patches/README.md index cc81564..0d67e05 100644 --- a/native/patches/README.md +++ b/native/patches/README.md @@ -208,6 +208,15 @@ requested-state mismatch for `restoreWindow` is read again, at most 8 times 100 posts no input and repeats the full identity, process-generation and bounds validation. Identity failures, lookup errors and a window that stays minimized remain indeterminate. +`peekaboo-title.patch` lets an Accessibility observation succeed while the window's title changes, +found in the [Ace desktop study](https://github.com/githubnext/ace/issues/427). The resolver reads the +title from WindowServer; the traversal reads it again from Accessibility, so a page with a live title +failed binding validation with `window-context window title`. A window context that names a window is +already pinned by that exact window ID, its WindowServer receipt and bounds, and one process generation, +and the traversal selects its AX window by that ID, so its title is no longer compared. A context without +a window ID still must match the title. The resolved target keeps its title; the client reports the newer +traversal title. Input actions bind by window ID, receipt and focused-element identity, never the title. + `desktop-tools-build` resolves only `Package.resolved` versions, checks the pin and checkout revision, and assembles the ordered patch stack in a private Git index. A build compares the checkout with each complete ordered prefix, since later patches can change earlier patch contexts. diff --git a/native/patches/peekaboo-title.patch b/native/patches/peekaboo-title.patch new file mode 100644 index 0000000..4f2103f --- /dev/null +++ b/native/patches/peekaboo-title.patch @@ -0,0 +1,138 @@ +diff --git a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift +index 3b700e0c6..dbca4edfe 100644 +--- a/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift ++++ b/Core/PeekabooCore/Sources/PeekabooBridge/PeekabooBridgeDesktopObservationBinding.swift +@@ -643,7 +643,10 @@ enum PeekabooBridgeDesktopObservationBinding { + if let windowID = context.windowID, windowID != targetWindow.windowID { + return "window-context window ID" + } +- if let title = context.windowTitle, title != targetWindow.title { ++ // The window ID, receipt, bounds and process generation pin a context to the target. ++ // Its title is read later during traversal and may be newer, so it is evidence only ++ // when the context names no window. ++ if context.windowID == nil, let title = context.windowTitle, title != targetWindow.title { + return "window-context window title" + } + if let bounds = context.windowBounds, bounds != targetWindow.bounds { +diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeObservationTitleChurnTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeObservationTitleChurnTests.swift +new file mode 100644 +index 000000000..06741d227 +--- /dev/null ++++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooBridgeObservationTitleChurnTests.swift +@@ -0,0 +1,116 @@ ++import CoreGraphics ++import Foundation ++import PeekabooFoundation ++import Testing ++@testable import PeekabooAutomationKit ++@testable import PeekabooBridge ++ ++@Suite(.serialized) ++struct PeekabooBridgeObservationTitleChurnTests: DesktopObservationBindingFixtureProviding { ++ private static let detection = DesktopDetectionOptions( ++ mode: .accessibility, ++ requiresFreshAccessibilityTree: true) ++ private static let request = DesktopObservationRequest( ++ target: .pid(42, window: .id(73)), ++ detection: Self.detection) ++ ++ @Test ++ @MainActor ++ func `a title that changes during the traversal binds by window ID and reports the newer title`() async throws { ++ let observed = try Self.observation(traversedTitle: "Inbox (3)", traversedWindowID: 73) ++ ++ #expect(PeekabooBridgeDesktopObservationBinding.mismatch( ++ request: Self.request, ++ result: observed.result, ++ requireSelectorResolutionProof: true) == nil) ++ let handled = try await PeekabooBridgeRequestContext.$usesAttestedOperationResultSemantics.withValue(true) { ++ try await Self.server(provider: ObservationProvider(result: observed.result)).handleAuthorized( ++ .desktopObservation(Self.request), ++ peer: nil, ++ permissions: Self.permissions) ++ } ++ guard case let .desktopObservation(response) = handled.response else { ++ Issue.record("Expected a desktop observation response") ++ return ++ } ++ #expect(response.target.window?.title == "Inbox (2)") ++ #expect(response.elements?.metadata.windowContext?.windowTitle == "Inbox (3)") ++ ++ let signed = try await Self.makeBundle( ++ request: .desktopObservation(Self.request), ++ response: .desktopObservation(observed.result), ++ target: .window(observed.identity)) ++ try signed.validateIntegrity() ++ } ++ ++ @Test ++ func `a traversed title still cannot stand in for a missing or different window ID`() throws { ++ let unnamed = try Self.observation(traversedTitle: "Inbox (3)", traversedWindowID: nil) ++ #expect(PeekabooBridgeDesktopObservationBinding.mismatch( ++ request: Self.request, ++ result: unnamed.result) == "window-context window title") ++ ++ let other = try Self.observation(traversedTitle: "Inbox (2)", traversedWindowID: 74) ++ #expect(PeekabooBridgeDesktopObservationBinding.mismatch( ++ request: Self.request, ++ result: other.result) == "window-context window ID") ++ } ++ ++ private static func observation( ++ traversedTitle: String, ++ traversedWindowID: Int?) throws -> (result: DesktopObservationResult, identity: WindowMutationIdentity) ++ { ++ let fixture = Self.windowResult(.init( ++ processIdentifier: 42, ++ generation: 1001, ++ bundleIdentifier: "dev.peekaboo.fixture", ++ applicationName: "Fixture", ++ windowID: 73, ++ title: "Inbox (2)", ++ index: 0, ++ windowSelector: .id(73))) ++ let original = fixture.result.target ++ let windowProof = try #require(original.selectorResolutionProofs?.last) ++ let target = ResolvedObservationTarget( ++ kind: original.kind, ++ app: original.app, ++ window: original.window, ++ bounds: original.bounds, ++ detectionContext: original.detectionContext, ++ captureScaleHint: original.captureScaleHint, ++ selectorResolutionProofs: [windowProof], ++ mutationTargetIdentity: original.mutationTargetIdentity) ++ let resolved = try #require(original.detectionContext) ++ let traversed = WindowContext( ++ applicationName: resolved.applicationName, ++ applicationBundleId: resolved.applicationBundleId, ++ applicationProcessId: resolved.applicationProcessId, ++ applicationProcessStartIdentity: fixture.identity.ownerProcessStartIdentity, ++ windowTitle: traversedTitle, ++ windowID: traversedWindowID, ++ windowBounds: resolved.windowBounds, ++ windowMutationIdentity: resolved.windowMutationIdentity, ++ shouldFocusWebContent: Self.detection.allowWebFocusFallback, ++ includeMenuBarElements: Self.detection.includeMenuBarElements, ++ traversalBudget: Self.detection.traversalBudget, ++ requiresFreshAccessibilityTree: true) ++ let base = fixture.result ++ let result = DesktopObservationResult( ++ target: target, ++ capture: base.capture, ++ elements: ElementDetectionResult( ++ snapshotId: "title-churn", ++ screenshotPath: "", ++ elements: DetectedElements(), ++ metadata: DetectionMetadata( ++ detectionTime: 0, ++ elementCount: 0, ++ method: "AXorcist", ++ windowContext: traversed)), ++ files: base.files, ++ timings: base.timings, ++ diagnostics: base.diagnostics, ++ captureContentDigest: base.captureContentDigest) ++ return (result, fixture.identity) ++ } ++} diff --git a/native/sources/client/client.swift b/native/sources/client/client.swift index e1971d8..99f2adb 100644 --- a/native/sources/client/client.swift +++ b/native/sources/client/client.swift @@ -179,7 +179,8 @@ private enum Client { inspection_mode: accessibility ? InspectionMode.accessibility.rawValue : InspectionMode.pixels.rawValue, note: accessibility ? nil : "Pixel snapshot: no Accessibility elements or focused control. Its snapshot_id authorizes only screenshot-point clicks, point scrolls, and drags in this exact window; element, text, key, and insertion actions are refused.", application_name: result.target.app?.name, - window_title: result.target.window?.title, + // The traversal reads the title after target resolution, so a live title is newer there. + window_title: result.elements?.metadata.windowContext?.windowTitle ?? result.target.window?.title, snapshot_id: result.files.publishedSnapshotID, element_count: result.elements?.metadata.elementCount ?? 0, ui_elements: result.elements?.elements.all.map(Element.init) ?? [], diff --git a/src/build.ts b/src/build.ts index da484a7..7ebb553 100644 --- a/src/build.ts +++ b/src/build.ts @@ -40,6 +40,7 @@ const patches = [ "peekaboo-sheet-window.patch", "peekaboo-sheet-focus.patch", "peekaboo-restore.patch", + "peekaboo-title.patch", ].map((name) => join(native, "patches", name)); function spawn(command: string[], env?: NodeJS.ProcessEnv) {