From d9c37e86312c688de84a3fff1168bf1d8cf6a6df Mon Sep 17 00:00:00 2001 From: iamnbutler Date: Sat, 24 Oct 2026 20:11:11 -0400 Subject: [PATCH 1/2] native2: point clicks reach Chrome's vertical tabs under its empty overlay groups (#202) Chrome's vertical tab strip lays childless, actionless AXGroups over its tabs. The Accessibility hit test returns one of them, and neither it nor its ancestors can be pressed, so a point click on a tab was refused. When the hit is such an empty group, search its parent's and grandparent's elements at the point (frames checked before children, as for #147) and press the smallest pressable one. Fixes #202 Why: native2/Sources/DesktopMac/Pointer.swift#beneath | Chrome's vertical tab strip covers its tabs with empty AXGroups that the AX hit test returns; only such childless, actionless groups give way to what is beneath --- CHANGELOG.md | 2 + native2/Sources/DesktopMac/Pointer.swift | 29 +++++++++++++- .../RuntimeTests/OverlayClickTests.swift | 38 +++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 native2/Tests/RuntimeTests/OverlayClickTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index d5f47a3..af856e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ Notable changes to Desktop Tools, newest first. Each release's GitHub release no - `desktop_inspect` takes its screenshot after reading the window's Accessibility tree, not during it. Reading some SwiftUI lists makes them load rows and grow, which moved content after the screenshot was taken, so a point read off it could click whatever moved under it. An inspection takes about as long as its tree read longer. ([#159](https://github.com/githubnext/desktop-tools/issues/159)) - `return` is accepted as a key name wherever keys are named (`desktop_key`, `desktop_system_key`), as an alias of `enter`. `desktop_system_key` results name the app that was frontmost when the keys were sent (`frontmost`), and `desktop_wait`'s description states its argument rules: `element_appears` and `element_gone` need an element with a role plus a title, value or identifier, and `timeout_ms` is an integer from 100 to 45000. ([#199](https://github.com/githubnext/desktop-tools/issues/199)) +- A background point click on a tab in Chrome's vertical tab strip switches to that tab. Chrome lays empty groups over its vertical tabs, and the Accessibility hit test returns one of those, so the click was refused with "No pressable accessibility element". When the element at the point is such an empty group, the click now presses the smallest pressable element beneath it, such as the tab or its close button. ([#202](https://github.com/githubnext/desktop-tools/issues/202)) + - An open menu's window, such as a SwiftUI `MenuBarExtra` status item's NSMenu or one of its submenus, can be inspected: `desktop_inspect` reads the open `AXMenu` and lists its items with `checked` for a checkmark, so they can be clicked by element. `desktop_menubar_click` on an item whose menu is open closes the menu without choosing an item (`closed_menu`), and `desktop_element_action` `AXCancel` on the menu closes it too. A window opened by a status item is `inspectable` only when it has Accessibility elements to read. ([#193](https://github.com/githubnext/desktop-tools/issues/193)) - Typing refused because secure keyboard entry is on now names the app and window holding it (`secure_input_holder`: pid, name, bundle ID, window title), read from the window server's session, such as a system password dialog behind the target, and asks to have the user dismiss it. ([#176](https://github.com/githubnext/desktop-tools/issues/176)) - `desktop_hover` and foreground `desktop_drag` no longer count a window that lets the pointer through, such as a screen recorder's frame, as covering the target: the cover check asks the window server which window takes pointer events at the point. An unverified hover or a cover refusal now names the window found there. `desktop_focus` and the other window and app tools say which `target` field is missing or wrong, such as a pid and window passed at the top level, and a window tool no longer asks for an application's target. ([#196](https://github.com/githubnext/desktop-tools/issues/196)) diff --git a/native2/Sources/DesktopMac/Pointer.swift b/native2/Sources/DesktopMac/Pointer.swift index 5841656..57b484b 100644 --- a/native2/Sources/DesktopMac/Pointer.swift +++ b/native2/Sources/DesktopMac/Pointer.swift @@ -164,12 +164,14 @@ extension Mac { throw Refusal("ELEMENT_NOT_FOUND", "No Accessibility element is at that point in the window.") } var candidates = [hit] + descendants(of: hit, containing: point) + var ancestors: [AXUIElement] = [] var parent = hit for _ in 0..<8 { guard let next = AX.element(AX.value(parent, kAXParentAttribute)) else { break } - candidates.append(next) + ancestors.append(next) parent = next } + candidates += beneathOverlay(hit, ancestors: ancestors, containing: point) + ancestors let roles = candidates.map { AX.string($0, kAXRoleAttribute) ?? "" } let enabled = candidates.map { (AX.value($0, kAXEnabledAttribute) as? Bool) != false } let pick: (AXUIElement, String)? = { @@ -379,6 +381,31 @@ extension Mac { return AXUIElementCopyActionNames(element, &names) == .success ? names as? [String] ?? [] : [] } + /// Elements under an empty overlay at the point, smallest first. Chrome's vertical tab strip lays childless, + /// actionless groups over its tabs, and the Accessibility hit test returns one of those, whose ancestors hold + /// nothing pressable (#202). When the hit is such a group, the tree of its parent, then grandparent, is searched + /// for elements at the point instead, so the tab or its close button is found. + private static func beneathOverlay(_ hit: AXUIElement, ancestors: [AXUIElement], containing point: CGPoint) -> [AXUIElement] { + beneath( + hit, ancestors: ancestors, role: { AX.string($0, kAXRoleAttribute) ?? "" }, + isLeaf: { (AX.value($0, kAXChildrenAttribute) as? [AXUIElement] ?? []).isEmpty }, actions: actions, + atPoint: { descendants(of: $0, containing: point) }, area: { AX.frame($0).map { $0.width * $0.height } ?? .greatestFiniteMagnitude }, + same: { CFEqual($0, $1) } + ) + } + + static func beneath( + _ hit: Node, ancestors: [Node], role: (Node) -> String, isLeaf: (Node) -> Bool, actions: (Node) -> [String], + atPoint: (Node) -> [Node], area: (Node) -> CGFloat, same: (Node, Node) -> Bool + ) -> [Node] { + guard role(hit) == "AXGroup", isLeaf(hit), actions(hit).isEmpty else { return [] } + for root in ancestors.prefix(2) { + let found = atPoint(root).filter { !same($0, hit) && actions($0).contains(kAXPressAction) } + if !found.isEmpty { return found.sorted { area($0) < area($1) } } + } + return [] + } + /// Up to 256 descendants, 8 levels deep, whose frames contain the point, breadth first. private static func descendants(of root: AXUIElement, containing point: CGPoint) -> [AXUIElement] { var found: [AXUIElement] = [] diff --git a/native2/Tests/RuntimeTests/OverlayClickTests.swift b/native2/Tests/RuntimeTests/OverlayClickTests.swift new file mode 100644 index 0000000..8012409 --- /dev/null +++ b/native2/Tests/RuntimeTests/OverlayClickTests.swift @@ -0,0 +1,38 @@ +@testable import DesktopMac +import CoreGraphics +import Testing + +/// Chrome's vertical tab strip lays empty groups over its tabs; a point click must find the tab beneath (#202). +struct OverlayClickTests { + struct Node { + let name: String + var role = "AXGroup" + var actions: [String] = [] + var area: CGFloat = 100 + var leaf = true + } + + static func beneath(_ hit: Node, under: [Node], ancestors: Int = 1) -> [String] { + Mac.beneath( + hit, ancestors: (0.. Date: Sat, 24 Oct 2026 20:11:20 -0400 Subject: [PATCH 2/2] lane: confirm Pointer.swift notes --- ...5R6B8TAZ57JZG7CAA2WW9TZ-chrome-s-vertical-tab-strip.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5R6B8TAZ57JZG7CAA2WW9TZ-chrome-s-vertical-tab-strip.md diff --git a/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5R6B8TAZ57JZG7CAA2WW9TZ-chrome-s-vertical-tab-strip.md b/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5R6B8TAZ57JZG7CAA2WW9TZ-chrome-s-vertical-tab-strip.md new file mode 100644 index 0000000..d1cd4ea --- /dev/null +++ b/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5R6B8TAZ57JZG7CAA2WW9TZ-chrome-s-vertical-tab-strip.md @@ -0,0 +1,8 @@ +--- +id: 01M5R6B8TAZ57JZG7CAA2WW9TZ +anchor: beneath +created: 2026-10-25T00:11:11Z +norm: '1' +--- + +Chrome's vertical tab strip covers its tabs with empty AXGroups that the AX hit test returns; only such childless, actionless groups give way to what is beneath