From b73dbabc84951eb0da1a29b4b3ab96d9348b7a99 Mon Sep 17 00:00:00 2001 From: iamnbutler Date: Sat, 24 Oct 2026 19:45:32 -0400 Subject: [PATCH 1/2] native2: modifier click checks held input at the hardware level and names it (#198) The held-input check read the combined session state, which also keeps keys and buttons posted down to the session and never released, or whose release a secure field swallowed, until they are pressed again; every modifier click was then refused while nothing was held. Read held keys and buttons from the HID state, and name what is held in the refusal. Fixes #198 Why: native2/Sources/DesktopMac/ForegroundPointer.swift#current | held keys and buttons come from hidSystemState; combinedSessionState keeps stale session-posted downs, which refused every modifier click (#198) --- CHANGELOG.md | 1 + .../DesktopMac/ForegroundPointer.swift | 37 ++++++++++++++++--- native2/Sources/DesktopMac/Keyboard.swift | 2 +- .../Tests/RuntimeTests/HeldInputTests.swift | 28 ++++++++++++++ 4 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 native2/Tests/RuntimeTests/HeldInputTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e730ad..a2c6592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ Notable changes to Desktop Tools, newest first. Each release's GitHub release no - An agent run as a background job, with no terminal or app above it, is asked about as the nearest program signed by a developer team that runs it, such as Claude Code, and the prompt says it's a program, not an app. When there is no such program, the refusal says so and why, instead of only that the caller couldn't be identified. A caller's team counts only when Apple issued its certificate. Setup docs note that background jobs don't load nvm. ([#194](https://github.com/githubnext/desktop-tools/issues/194)) - A background click on a web menu row, button or tab in a WebKit app, such as a WKWebView or Electrobun app or Safari, no longer presses a control nested inside it. WebKit's Accessibility press goes to the first native button inside the element, so a click on a project row in Ace pressed its hover-only remove button. Such an element now gets a pointer click at the point when its window is frontmost and is refused otherwise, and every press result names what was pressed in `pressed`. ([#195](https://github.com/githubnext/desktop-tools/issues/195)) - `desktop_read` with a `scope` walks that element's subtree directly, so a busy window can't spend the read's budget before reaching it, and returns the subtree's full text. A top-level `snapshot` and `element`, which desktop_read never had, are read as that scope instead of being ignored and reading the whole window; an `element` alone is refused with how to pass a scope. ([#204](https://github.com/githubnext/desktop-tools/issues/204)) +- A modifier click is no longer refused for held input that only the session state remembers, such as a key some process posted down and never released, or whose release a secure field swallowed; only keys and buttons held at the hardware level count. A refusal now names what it found held, such as `Shift (key 56)` or `mouse button 3`. ([#198](https://github.com/githubnext/desktop-tools/issues/198)) ## [0.17.1] - 2026-10-20 diff --git a/native2/Sources/DesktopMac/ForegroundPointer.swift b/native2/Sources/DesktopMac/ForegroundPointer.swift index 08adc17..c3bf1e6 100644 --- a/native2/Sources/DesktopMac/ForegroundPointer.swift +++ b/native2/Sources/DesktopMac/ForegroundPointer.swift @@ -20,12 +20,22 @@ struct InputActivity: Equatable { var hasHeldInput: Bool { heldKeys.contains(true) || heldButtons.contains(true) } + /// The keys and buttons held, named as a person would read them, such as "Shift (key 56)" or "mouse button 3". + var heldNames: [String] { + heldKeys.indices.filter { heldKeys[$0] }.map { keyName(CGKeyCode($0)) } + + heldButtons.indices.filter { heldButtons[$0] }.map { ["left mouse button", "right mouse button", "middle mouse button"].indices.contains($0) ? ["left mouse button", "right mouse button", "middle mouse button"][$0] : "mouse button \($0)" } + } + static func current() -> InputActivity { let state = CGEventSourceStateID.combinedSessionState func count(_ type: CGEventType) -> UInt32 { CGEventSource.counterForEventType(state, eventType: type) } - // Caps Lock's key state is its latch, not a held key. - let keys = (CGKeyCode(0)..<128).map { $0 != 0x39 && CGEventSource.keyState(state, key: $0) } - let buttons = (UInt32(0)..<32).map { CGMouseButton(rawValue: $0).map { CGEventSource.buttonState(state, button: $0) } ?? false } + // Held means held at the hardware level. The combined session state also keeps keys and buttons that some + // process posted down to the session and never released, or whose release a secure field swallowed; those + // stay "down" until pressed again and refused every modifier click while nothing was held (#198). Caps Lock's + // key state is its latch, not a held key. + let held = CGEventSourceStateID.hidSystemState + let keys = (CGKeyCode(0)..<128).map { $0 != 0x39 && CGEventSource.keyState(held, key: $0) } + let buttons = (UInt32(0)..<32).map { CGMouseButton(rawValue: $0).map { CGEventSource.buttonState(held, button: $0) } ?? false } return InputActivity( moved: count(.mouseMoved), leftDragged: count(.leftMouseDragged), rightDragged: count(.rightMouseDragged), otherDragged: count(.otherMouseDragged), leftDown: count(.leftMouseDown), leftUp: count(.leftMouseUp), @@ -51,6 +61,21 @@ struct InputActivity: Equatable { } } +/// A key's name for a refusal: its modifier or contract name, with its virtual key code. +func keyName(_ code: CGKeyCode) -> String { + let modifiers: [CGKeyCode: String] = [ + 0x37: "Command", 0x36: "Right Command", 0x38: "Shift", 0x3C: "Right Shift", 0x3A: "Option", 0x3D: "Right Option", + 0x3B: "Control", 0x3E: "Right Control", 0x3F: "Fn", + ] + let name = modifiers[code] ?? namedKeys.filter { $0.value == code }.keys.min().map { $0.prefix(1).uppercased() + $0.dropFirst() } + return name.map { "\($0) (key \(code))" } ?? "key \(code)" +} + +/// "A", "A and B", "A, B and C". +func spokenList(_ items: [String]) -> String { + items.count < 2 ? items.joined() : items.dropLast().joined(separator: ", ") + " and " + items.last! +} + /// The frontmost app and its focused window, which a foreground action takes and then gives back. struct FocusState: Equatable { let pid: pid_t @@ -267,9 +292,11 @@ extension Mac { } let initial = InputActivity.current() guard !initial.hasHeldInput else { + let held = initial.heldNames throw Refusal( - "DESKTOP_ACTION_FAILED", "Modifier-click requires every autorepeating key or held mouse button to be released before focus.", - hint: "Release held input and retry from a fresh observation." + "DESKTOP_ACTION_FAILED", + "Modifier-click requires every key and mouse button to be released before focus, but the keyboard and mouse report \(spokenList(held)) held.", + hint: "Release \(held.count == 1 ? "it" : "them") and retry from a fresh observation. If nothing is held, ask the user to press and release \(held.count == 1 ? "it" : "each") once, which clears a stuck state." ) } let targetState = FocusState(pid: target.pid, generation: target.generation, window: target) diff --git a/native2/Sources/DesktopMac/Keyboard.swift b/native2/Sources/DesktopMac/Keyboard.swift index b3b43ea..e06a40d 100644 --- a/native2/Sources/DesktopMac/Keyboard.swift +++ b/native2/Sources/DesktopMac/Keyboard.swift @@ -10,7 +10,7 @@ import Foundation // window and element must already be in the exact window, which is checked before every event. /// Virtual key codes by this contract's key names (ANSI). -private let namedKeys: [String: CGKeyCode] = [ +let namedKeys: [String: CGKeyCode] = [ "enter": 0x24, "return": 0x24, "tab": 0x30, "space": 0x31, "backspace": 0x33, "escape": 0x35, "delete": 0x75, "help": 0x72, "home": 0x73, "pageup": 0x74, "end": 0x77, "pagedown": 0x79, "left": 0x7B, "right": 0x7C, "down": 0x7D, "up": 0x7E, diff --git a/native2/Tests/RuntimeTests/HeldInputTests.swift b/native2/Tests/RuntimeTests/HeldInputTests.swift new file mode 100644 index 0000000..f935bfe --- /dev/null +++ b/native2/Tests/RuntimeTests/HeldInputTests.swift @@ -0,0 +1,28 @@ +@testable import DesktopMac +import Testing + +/// A modifier click refused for held input names what it found held (#198). +struct HeldInputTests { + static func activity(keys: [Int] = [], buttons: [Int] = []) -> InputActivity { + InputActivity( + moved: 0, leftDragged: 0, rightDragged: 0, otherDragged: 0, leftDown: 0, leftUp: 0, rightDown: 0, rightUp: 0, + otherDown: 0, otherUp: 0, keyDown: 0, keyUp: 0, flagsChanged: 0, scrollWheel: 0, + heldKeys: (0..<128).map { keys.contains($0) }, heldButtons: (0..<32).map { buttons.contains($0) } + ) + } + + @Test + func `held keys and buttons are named, modifiers and contract keys by name`() { + let held = Self.activity(keys: [0x38, 0x35, 0x00], buttons: [0, 4]) + #expect(held.hasHeldInput) + #expect(held.heldNames == ["key 0", "Escape (key 53)", "Shift (key 56)", "left mouse button", "mouse button 4"]) + #expect(spokenList(held.heldNames) == "key 0, Escape (key 53), Shift (key 56), left mouse button and mouse button 4") + } + + @Test + func `nothing held names nothing`() { + #expect(!Self.activity().hasHeldInput) + #expect(Self.activity().heldNames.isEmpty) + #expect(spokenList(["Fn (key 63)"]) == "Fn (key 63)") + } +} From ad9d32fe9d0887282866ac11ee45c57260f50898 Mon Sep 17 00:00:00 2001 From: iamnbutler Date: Sat, 24 Oct 2026 19:45:39 -0400 Subject: [PATCH 2/2] lane: confirm Keyboard.swift note --- ...R4WA2CVGZHCYQA8ZRPJW1Y-held-keys-and-buttons-come-f.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .lane/memory/native2/Sources/DesktopMac/ForegroundPointer.swift/01M5R4WA2CVGZHCYQA8ZRPJW1Y-held-keys-and-buttons-come-f.md diff --git a/.lane/memory/native2/Sources/DesktopMac/ForegroundPointer.swift/01M5R4WA2CVGZHCYQA8ZRPJW1Y-held-keys-and-buttons-come-f.md b/.lane/memory/native2/Sources/DesktopMac/ForegroundPointer.swift/01M5R4WA2CVGZHCYQA8ZRPJW1Y-held-keys-and-buttons-come-f.md new file mode 100644 index 0000000..f2022ab --- /dev/null +++ b/.lane/memory/native2/Sources/DesktopMac/ForegroundPointer.swift/01M5R4WA2CVGZHCYQA8ZRPJW1Y-held-keys-and-buttons-come-f.md @@ -0,0 +1,8 @@ +--- +id: 01M5R4WA2CVGZHCYQA8ZRPJW1Y +anchor: current +created: 2026-10-24T23:45:32Z +norm: '1' +--- + +held keys and buttons come from hidSystemState; combinedSessionState keeps stale session-posted downs, which refused every modifier click (#198)