native2: modifier click checks held input at the hardware level and names it (#198)

.lane/memory/native2/Sources/DesktopMac/ForegroundPointer.swift/01M5R4WA2CVGZHCYQA8ZRPJW1Y-held-keys-and-buttons-come-f.md

@@ -0,0 +1,8 @@
+---
+id: 01M5R4WA2CVGZHCYQA8ZRPJW1Y
+anchor: current
+created: 2026-10-24T23:45:32Z
+norm: '1'
+---
+
+held keys and buttons come from hidSystemState; combinedSessionState keeps stale session-posted downs, which refused every modifier click (#198)

CHANGELOG.md

@@ -29,6 +29,7 @@ Notable changes to Desktop Tools, newest first. Each release's GitHub release no
 - An agent run as a background job, with no terminal or app above it, is asked about as the nearest program signed by a developer team that runs it, such as Claude Code, and the prompt says it's a program, not an app. When there is no such program, the refusal says so and why, instead of only that the caller couldn't be identified. A caller's team counts only when Apple issued its certificate. Setup docs note that background jobs don't load nvm. ([#194](https://github.com/githubnext/desktop-tools/issues/194))
 - A background click on a web menu row, button or tab in a WebKit app, such as a WKWebView or Electrobun app or Safari, no longer presses a control nested inside it. WebKit's Accessibility press goes to the first native button inside the element, so a click on a project row in Ace pressed its hover-only remove button. Such an element now gets a pointer click at the point when its window is frontmost and is refused otherwise, and every press result names what was pressed in `pressed`. ([#195](https://github.com/githubnext/desktop-tools/issues/195))
 - `desktop_read` with a `scope` walks that element's subtree directly, so a busy window can't spend the read's budget before reaching it, and returns the subtree's full text. A top-level `snapshot` and `element`, which desktop_read never had, are read as that scope instead of being ignored and reading the whole window; an `element` alone is refused with how to pass a scope. ([#204](https://github.com/githubnext/desktop-tools/issues/204))
+- A modifier click is no longer refused for held input that only the session state remembers, such as a key some process posted down and never released, or whose release a secure field swallowed; only keys and buttons held at the hardware level count. A refusal now names what it found held, such as `Shift (key 56)` or `mouse button 3`. ([#198](https://github.com/githubnext/desktop-tools/issues/198))
 
 ## [0.17.1] - 2026-10-20
 

native2/Sources/DesktopMac/ForegroundPointer.swift

@@ -20,12 +20,22 @@ struct InputActivity: Equatable {
 
 	var hasHeldInput: Bool { heldKeys.contains(true) || heldButtons.contains(true) }
 
+	/// The keys and buttons held, named as a person would read them, such as "Shift (key 56)" or "mouse button 3".
+	var heldNames: [String] {
+		heldKeys.indices.filter { heldKeys[$0] }.map { keyName(CGKeyCode($0)) }
+			+ heldButtons.indices.filter { heldButtons[$0] }.map { ["left mouse button", "right mouse button", "middle mouse button"].indices.contains($0) ? ["left mouse button", "right mouse button", "middle mouse button"][$0] : "mouse button \($0)" }
+	}
+
 	static func current() -> InputActivity {
 		let state = CGEventSourceStateID.combinedSessionState
 		func count(_ type: CGEventType) -> UInt32 { CGEventSource.counterForEventType(state, eventType: type) }
-		// Caps Lock's key state is its latch, not a held key.
-		let keys = (CGKeyCode(0)..<128).map { $0 != 0x39 && CGEventSource.keyState(state, key: $0) }
-		let buttons = (UInt32(0)..<32).map { CGMouseButton(rawValue: $0).map { CGEventSource.buttonState(state, button: $0) } ?? false }
+		// Held means held at the hardware level. The combined session state also keeps keys and buttons that some
+		// process posted down to the session and never released, or whose release a secure field swallowed; those
+		// stay "down" until pressed again and refused every modifier click while nothing was held (#198). Caps Lock's
+		// key state is its latch, not a held key.
+		let held = CGEventSourceStateID.hidSystemState
+		let keys = (CGKeyCode(0)..<128).map { $0 != 0x39 && CGEventSource.keyState(held, key: $0) }
+		let buttons = (UInt32(0)..<32).map { CGMouseButton(rawValue: $0).map { CGEventSource.buttonState(held, button: $0) } ?? false }
 		return InputActivity(
 			moved: count(.mouseMoved), leftDragged: count(.leftMouseDragged), rightDragged: count(.rightMouseDragged),
 			otherDragged: count(.otherMouseDragged), leftDown: count(.leftMouseDown), leftUp: count(.leftMouseUp),
@@ -51,6 +61,21 @@ struct InputActivity: Equatable {
 	}
 }
 
+/// A key's name for a refusal: its modifier or contract name, with its virtual key code.
+func keyName(_ code: CGKeyCode) -> String {
+	let modifiers: [CGKeyCode: String] = [
+		0x37: "Command", 0x36: "Right Command", 0x38: "Shift", 0x3C: "Right Shift", 0x3A: "Option", 0x3D: "Right Option",
+		0x3B: "Control", 0x3E: "Right Control", 0x3F: "Fn",
+	]
+	let name = modifiers[code] ?? namedKeys.filter { $0.value == code }.keys.min().map { $0.prefix(1).uppercased() + $0.dropFirst() }
+	return name.map { "\($0) (key \(code))" } ?? "key \(code)"
+}
+
+/// "A", "A and B", "A, B and C".
+func spokenList(_ items: [String]) -> String {
+	items.count < 2 ? items.joined() : items.dropLast().joined(separator: ", ") + " and " + items.last!
+}
+
 /// The frontmost app and its focused window, which a foreground action takes and then gives back.
 struct FocusState: Equatable {
 	let pid: pid_t
@@ -267,9 +292,11 @@ extension Mac {
 		}
 		let initial = InputActivity.current()
 		guard !initial.hasHeldInput else {
+			let held = initial.heldNames
 			throw Refusal(
-				"DESKTOP_ACTION_FAILED", "Modifier-click requires every autorepeating key or held mouse button to be released before focus.",
-				hint: "Release held input and retry from a fresh observation."
+				"DESKTOP_ACTION_FAILED",
+				"Modifier-click requires every key and mouse button to be released before focus, but the keyboard and mouse report \(spokenList(held)) held.",
+				hint: "Release \(held.count == 1 ? "it" : "them") and retry from a fresh observation. If nothing is held, ask the user to press and release \(held.count == 1 ? "it" : "each") once, which clears a stuck state."
 			)
 		}
 		let targetState = FocusState(pid: target.pid, generation: target.generation, window: target)

native2/Sources/DesktopMac/Keyboard.swift

@@ -10,7 +10,7 @@ import Foundation
 // window and element must already be in the exact window, which is checked before every event.
 
 /// Virtual key codes by this contract's key names (ANSI).
-private let namedKeys: [String: CGKeyCode] = [
+let namedKeys: [String: CGKeyCode] = [
 	"enter": 0x24, "return": 0x24, "tab": 0x30, "space": 0x31, "backspace": 0x33, "escape": 0x35, "delete": 0x75,
 	"help": 0x72, "home": 0x73, "pageup": 0x74, "end": 0x77, "pagedown": 0x79,
 	"left": 0x7B, "right": 0x7C, "down": 0x7D, "up": 0x7E,

native2/Tests/RuntimeTests/HeldInputTests.swift

@@ -0,0 +1,28 @@
+@testable import DesktopMac
+import Testing
+
+/// A modifier click refused for held input names what it found held (#198).
+struct HeldInputTests {
+	static func activity(keys: [Int] = [], buttons: [Int] = []) -> InputActivity {
+		InputActivity(
+			moved: 0, leftDragged: 0, rightDragged: 0, otherDragged: 0, leftDown: 0, leftUp: 0, rightDown: 0, rightUp: 0,
+			otherDown: 0, otherUp: 0, keyDown: 0, keyUp: 0, flagsChanged: 0, scrollWheel: 0,
+			heldKeys: (0..<128).map { keys.contains($0) }, heldButtons: (0..<32).map { buttons.contains($0) }
+		)
+	}
+
+	@Test
+	func `held keys and buttons are named, modifiers and contract keys by name`() {
+		let held = Self.activity(keys: [0x38, 0x35, 0x00], buttons: [0, 4])
+		#expect(held.hasHeldInput)
+		#expect(held.heldNames == ["key 0", "Escape (key 53)", "Shift (key 56)", "left mouse button", "mouse button 4"])
+		#expect(spokenList(held.heldNames) == "key 0, Escape (key 53), Shift (key 56), left mouse button and mouse button 4")
+	}
+
+	@Test
+	func `nothing held names nothing`() {
+		#expect(!Self.activity().hasHeldInput)
+		#expect(Self.activity().heldNames.isEmpty)
+		#expect(spokenList(["Fn (key 63)"]) == "Fn (key 63)")
+	}
+}