From 91111313232fe4106549fca0b0b17a9e2be86dce Mon Sep 17 00:00:00 2001 From: masnwilliams <43387599+masnwilliams@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:19:14 +0000 Subject: [PATCH 1/9] Make live view usable on touch devices Add a gesture layer for touch input: tap clicks, one-finger swipes scroll with momentum, long-press right-clicks or drags, and two-finger pinch zooms and pans the video locally. Raise the soft keyboard from the tap when the remote cursor is a text cursor, type Android input and composition events, and send Shift for upper case letters from soft keyboards. Replace the small keyboard icon with a labeled button inside the safe area and fix the layout below 360px and under mobile browser toolbars. --- .../chromium-headful/client/public/index.html | 2 +- images/chromium-headful/client/src/app.vue | 5 +- .../client/src/components/video.vue | 586 ++++++++++++++++-- .../chromium-headful/client/src/neko/base.ts | 2 + .../chromium-headful/client/src/neko/index.ts | 13 +- .../client/src/utils/cursor-shape.ts | 100 +++ .../client/src/utils/text-input.ts | 38 ++ .../client/src/utils/touch-gestures.ts | 223 +++++++ .../client/src/utils/zoom-pan.ts | 67 ++ .../client/tests/touch-input.test.ts | 131 ++++ 10 files changed, 1115 insertions(+), 52 deletions(-) create mode 100644 images/chromium-headful/client/src/utils/cursor-shape.ts create mode 100644 images/chromium-headful/client/src/utils/text-input.ts create mode 100644 images/chromium-headful/client/src/utils/touch-gestures.ts create mode 100644 images/chromium-headful/client/src/utils/zoom-pan.ts create mode 100644 images/chromium-headful/client/tests/touch-input.test.ts diff --git a/images/chromium-headful/client/public/index.html b/images/chromium-headful/client/public/index.html index 6869c600b..e327bfa9b 100644 --- a/images/chromium-headful/client/public/index.html +++ b/images/chromium-headful/client/public/index.html @@ -3,7 +3,7 @@ - + Kernel diff --git a/images/chromium-headful/client/src/app.vue b/images/chromium-headful/client/src/app.vue index 3d7a83039..cf25d9983 100644 --- a/images/chromium-headful/client/src/app.vue +++ b/images/chromium-headful/client/src/app.vue @@ -56,11 +56,12 @@ bottom: 0; max-width: 100vw; max-height: 100vh; + max-height: 100dvh; flex-direction: row; display: flex; .neko-main { - min-width: 360px; + min-width: 0; max-width: 100%; flex-grow: 1; flex-direction: column; @@ -144,10 +145,12 @@ .neko-main { height: 100vh; + height: 100dvh; } .neko-menu { height: 100vh; + height: 100dvh; width: 100% !important; } } diff --git a/images/chromium-headful/client/src/components/video.vue b/images/chromium-headful/client/src/components/video.vue index 6b0355651..d44d51238 100644 --- a/images/chromium-headful/client/src/components/video.vue +++ b/images/chromium-headful/client/src/components/video.vue @@ -12,9 +12,12 @@ ref="overlay" class="overlay" spellcheck="false" + autocapitalize="off" + autocorrect="off" + autocomplete="off" tabindex="0" data-gramm="false" - :style="{ pointerEvents: hosting ? 'auto' : 'none' }" + :style="{ pointerEvents: hosting || is_touch_device ? 'auto' : 'none' }" @click.stop.prevent @contextmenu.stop.prevent @mousemove.stop.prevent="onMouseMove" @@ -22,12 +25,16 @@ @mouseup.stop.prevent="onMouseUp" @mouseenter.stop.prevent="onMouseEnter" @mouseleave.stop.prevent="onMouseLeave" - @touchmove.stop.prevent="onTouchHandler" - @touchstart.stop.prevent="onTouchHandler" - @touchend.stop.prevent="onTouchHandler" + @touchstart.stop.prevent="onTouchStart" + @touchmove.stop.prevent="onTouchMove" + @touchend.stop.prevent="onTouchEnd" + @touchcancel.stop.prevent="onTouchCancel" + @input="onOverlayInput" + @compositionstart="onCompositionStart" + @compositionend="onCompositionEnd" @paste.stop.prevent="onPaste" @focus="onOverlayFocus" - @blur="resetKeyboard" + @blur="onOverlayBlur" /> -
  • +
  • - -
    • + + + @@ -97,6 +127,44 @@ justify-content: center; align-items: center; background: #000; + overflow: hidden; + touch-action: none; + + .touch-button { + position: absolute; + z-index: 2; + display: flex; + align-items: center; + gap: 8px; + min-height: 44px; + padding: 0 16px; + border: 0; + border-radius: 22px; + background: rgba($color: #000, $alpha: 0.7); + color: #fff; + font-size: 15px; + font-weight: 600; + white-space: nowrap; + cursor: pointer; + touch-action: manipulation; + -webkit-tap-highlight-color: transparent; + } + + .keyboard-button { + right: calc(12px + env(safe-area-inset-right)); + bottom: calc(12px + env(safe-area-inset-bottom)); + } + + .zoom-chip { + left: calc(12px + env(safe-area-inset-left)); + top: calc(12px + env(safe-area-inset-top)); + } + + .type-chip { + transform: translateX(-50%); + background: #81b300; + color: #000; + } .video-menu { position: absolute; @@ -153,6 +221,7 @@ position: relative; width: 100%; max-width: calc(16 / 9 * 100vh); + transform-origin: 0 0; video { position: absolute; @@ -205,8 +274,12 @@ outline: 0; border: 0; color: transparent; + caret-color: transparent; background: transparent; resize: none; + touch-action: none; + // iOS zooms the page when focusing a field with a smaller font + font-size: 16px; } .player-aspect { @@ -223,6 +296,18 @@ import ResizeObserver from 'resize-observer-polyfill' import { elementRequestFullscreen, onFullscreenChange, isFullscreen, lockKeyboard, unlockKeyboard } from '~/utils' import { isClipboardReadGranted } from '~/utils/clipboard' + import { TouchGestures, Point } from '~/utils/touch-gestures' + import { ZoomPan, Box } from '~/utils/zoom-pan' + import { CursorImage, CursorKind, cachedCursorKind, classifyCursor } from '~/utils/cursor-shape' + import { + XK_BACKSPACE, + XK_RETURN, + XK_SHIFT_L, + charToKeysym, + keysymNeedsShift, + needsShift, + textDiff, + } from '~/utils/text-input' import Emote from './emote.vue' import Resolution from './resolution.vue' @@ -235,6 +320,20 @@ const SCROLL_SENSITIVITY_BASE = 10 const INT16_MAX = 32767 + // wheel units per remote pixel of finger travel, so content tracks the finger + const TOUCH_SCROLL_UNITS_PER_PX = 1.1 + const DOUBLE_TAP_MS = 300 + const DOUBLE_TAP_SLOP = 40 + // a cursor shape unchanged this long after the pointer moved belongs to the new position + const CURSOR_SETTLE_MS = 200 + // how long a late cursor update can still act on the tap that caused it + const CURSOR_WAIT_MS = 800 + const TYPE_CHIP_MS = 4000 + // kept in the overlay so soft keyboards that only emit input events still report backspace + const INPUT_SENTINEL = ' ' + + type TapKeyboardMode = 'auto' | 'chip' | 'always' | 'off' + @Component({ name: 'neko-video', components: { @@ -271,6 +370,25 @@ private mutedOverlay = true private isVideoSyncing = false + private gestures!: TouchGestures + private zoom!: ZoomPan + private zoomScale = 1 + private keyboardOpen = false + private typeChip: Point | null = null + private typeChipTimer = 0 + private cursorKind: CursorKind = 'unknown' + private cursorChangedAt = 0 + private touchBeganAt = 0 + private lastTap: { p: Point; at: number } | null = null + private pendingTap: { p: Point; at: number } | null = null + private pendingTapTimer = 0 + private scrollRemainder = { x: 0, y: 0 } + private composing = false + private composed = '' + private lastCompositionEnd = { data: '', at: 0 } + private lastKeydownAt = 0 + private shiftWrapped = new Set() + get admin() { return this.$accessor.user.admin } @@ -399,13 +517,31 @@ ) } + get showTouchControls() { + return this.hosting && this.is_touch_device && !this.hideControls && !this.fullscreen + } + + get zoomed() { + return this.zoomScale > 1.01 + } + + // ?tapKeyboard=auto (default) raises the keyboard when a tap lands on text, + // chip offers a "tap to type" button there instead, always raises it on + // every tap, off leaves it to the keyboard button + get tapKeyboard(): TapKeyboardMode { + const value = new URL(location.href).searchParams.get('tapKeyboard') + return value === 'chip' || value === 'always' || value === 'off' ? value : 'auto' + } + @Watch('width') onWidthChanged() { + this.resetZoom() this.onResize() } @Watch('height') onHeightChanged() { + this.resetZoom() this.onResize() } @@ -491,6 +627,22 @@ } mounted() { + this.zoom = new ZoomPan(this.containerBox, this.playerBox, this.maxZoom) + this.gestures = new TouchGestures({ + onTouchBegin: this.onGestureBegin, + onTap: this.onGestureTap, + onLongPress: this.onGestureLongPress, + onLongPressRelease: this.onGestureLongPressRelease, + onDragStart: this.onGestureDragStart, + onDragMove: this.onGestureDragMove, + onDragEnd: this.onGestureDragEnd, + onScroll: this.onGestureScroll, + onPinchStart: this.onGesturePinchStart, + onPinchMove: this.onGesturePinchMove, + onPinchEnd: this.onGesturePinchEnd, + }) + this.$client.on('cursor', this.onCursorImage) + this._container.addEventListener('resize', this.onResize) this.onVolumeChanged(this.volume) this.onMutedChanged(this.muted) @@ -570,6 +722,12 @@ return true } + this.lastKeydownAt = performance.now() + if (this.is_touch_device && keysymNeedsShift(key) && !this.keyboard.modifiers.shift) { + this.shiftWrapped.add(key) + this.$client.sendData('keydown', { key: XK_SHIFT_L }) + } + this.$client.sendData('keydown', { key: this.keyMap(key) }) } this.keyboard.onkeyup = (key: number) => { @@ -583,11 +741,18 @@ } this.$client.sendData('keyup', { key: this.keyMap(key) }) + if (this.shiftWrapped.delete(key)) { + this.$client.sendData('keyup', { key: XK_SHIFT_L }) + } } this.keyboard.listenTo(this._overlay) } beforeDestroy() { + this.gestures.destroy() + this.$client.off('cursor', this.onCursorImage) + window.clearTimeout(this.typeChipTimer) + window.clearTimeout(this.pendingTapTimer) if (this._wheelHandler) { document.removeEventListener('wheel', this._wheelHandler, { capture: true }) this._wheelHandler = null @@ -797,43 +962,362 @@ } } - onTouchHandler(e: TouchEvent) { - let first = e.changedTouches[0] - let type = '' - switch (e.type) { - case 'touchstart': - type = 'mousedown' - break - case 'touchmove': - type = 'mousemove' - break - case 'touchend': - type = 'mouseup' - break - default: - return + onTouchStart(e: TouchEvent) { + this.gestures.touchStart(e) + } + + onTouchMove(e: TouchEvent) { + this.gestures.touchMove(e) + } + + onTouchEnd(e: TouchEvent) { + this.gestures.touchEnd(e) + } + + onTouchCancel() { + this.gestures.touchCancel() + } + + canSendInput() { + return this.hosting && !this.locked + } + + // maps a client point on the (possibly zoomed) video to remote screen pixels + remotePoint(p: Point) { + const { w, h } = this.$accessor.video.resolution + const rect = this._overlay.getBoundingClientRect() + return { + x: Math.max(0, Math.min(w - 1, Math.round((w / rect.width) * (p.x - rect.left)))), + y: Math.max(0, Math.min(h - 1, Math.round((h / rect.height) * (p.y - rect.top)))), + } + } + + sendPointer(p: Point) { + this.$client.sendData('mousemove', this.remotePoint(p)) + } + + clickAt(p: Point, button: number) { + this.sendPointer(p) + this.$client.sendData('mousedown', { key: button }) + this.$client.sendData('mouseup', { key: button }) + } + + onGestureBegin(p: Point) { + this.unmuteOnInteraction() + this.hideTypeChip() + this.touchBeganAt = performance.now() + + if (!this.controlling && this.implicitHosting && !this.locked) { + this.$accessor.remote.request() + } + if (!this.hosting) { + this.$emit('control-attempt') + } + if (this.canSendInput()) { + this.sendPointer(p) + } + } + + onGestureTap(p: Point) { + const now = performance.now() + const last = this.lastTap + this.lastTap = { p, at: now } + + if ( + this.zoomed && + last && + now - last.at < DOUBLE_TAP_MS && + Math.hypot(p.x - last.p.x, p.y - last.p.y) < DOUBLE_TAP_SLOP + ) { + this.lastTap = null + this.resetZoom() + return } - const simulatedEvent = new MouseEvent(type, { - bubbles: true, - cancelable: true, - view: window, - screenX: first.screenX, - screenY: first.screenY, - clientX: first.clientX, - clientY: first.clientY, + if (!this.canSendInput()) return + this.clickAt(p, 1) + this.keyboardAfterTap(p) + } + + onGestureLongPress() { + if (!this.canSendInput()) return + if (navigator.vibrate) navigator.vibrate(15) + } + + onGestureLongPressRelease(p: Point) { + if (!this.canSendInput()) return + this.clickAt(p, 3) + } + + onGestureDragStart(p: Point) { + if (!this.canSendInput()) return + this.sendPointer(p) + this.pressedMouseButtons.add(1) + this.$client.sendData('mousedown', { key: 1 }) + } + + onGestureDragMove(p: Point) { + if (!this.canSendInput()) return + this.sendPointer(p) + } + + onGestureDragEnd(p: Point) { + if (!this.pressedMouseButtons.has(1)) return + this.sendPointer(p) + this.pressedMouseButtons.delete(1) + this.$client.sendData('mouseup', { key: 1 }) + } + + onGestureScroll(dx: number, dy: number) { + if (!this.canSendInput()) { + this.gestures.stopFling() + return + } + + // finger travel in remote pixels; content follows the finger, so scroll the opposite way + const remotePerClient = this.width / this._overlay.getBoundingClientRect().width + const units = TOUCH_SCROLL_UNITS_PER_PX * remotePerClient + this.scrollRemainder.x -= dx * units + this.scrollRemainder.y -= dy * units + + const x = Math.trunc(this.scrollRemainder.x) + const y = Math.trunc(this.scrollRemainder.y) + if (x === 0 && y === 0) return + this.scrollRemainder.x -= x + this.scrollRemainder.y -= y + this.$client.sendData('wheel', { + x: Math.max(-INT16_MAX, Math.min(INT16_MAX, x)), + y: Math.max(-INT16_MAX, Math.min(INT16_MAX, y)), }) - first.target.dispatchEvent(simulatedEvent) } - focusOverlay(e: MouseEvent) { - // Touch input is translated into an untrusted mouse event above. Keep the - // existing mobile-keyboard behavior while allowing a real mouse to focus - // the overlay on touch-capable devices. - if (this.is_touch_device && !e.isTrusted) { + onGesturePinchStart(mid: Point, distance: number) { + this.hideTypeChip() + this.zoom.pinchStart(mid, distance) + } + + onGesturePinchMove(mid: Point, distance: number) { + this.zoom.pinchMove(mid, distance) + this.applyZoom(false) + } + + onGesturePinchEnd() { + if (!this.zoom.zoomed) this.resetZoom() + } + + // the container box before the zoom transform, in client coordinates + containerBox(): Box { + const player = this._player.getBoundingClientRect() + return { + left: player.left + this._container.offsetLeft, + top: player.top + this._container.offsetTop, + width: this._container.offsetWidth, + height: this._container.offsetHeight, + } + } + + playerBox(): Box { + const { left, top, width, height } = this._player.getBoundingClientRect() + return { left, top, width, height } + } + + maxZoom() { + const fit = this.width / Math.max(this._container.offsetWidth, 1) + return Math.min(8, Math.max(3, fit * 1.5)) + } + + applyZoom(animate: boolean) { + this._container.style.transition = animate ? 'transform 150ms ease-out' : '' + this._container.style.transform = this.zoom.transform + this.zoomScale = this.zoom.scale + } + + resetZoom() { + if (!this.zoom) return + this.zoom.reset() + this.applyZoom(true) + } + + // Runs inside touchend, the only place iOS lets focus() raise the keyboard. + // The remote cursor shape is the signal: Chromium shows an I-beam over text + // fields (and over selectable page text, which is the false positive), and + // the pointer was moved to the tap position on touchstart. + keyboardAfterTap(p: Point) { + window.clearTimeout(this.pendingTapTimer) + this.pendingTap = null + + if (this.tapKeyboard === 'off') return + if (this.tapKeyboard === 'always') { + this.focusForTyping() + return + } + + const now = performance.now() + const fresh = this.cursorChangedAt >= this.touchBeganAt + const settled = now - this.touchBeganAt >= CURSOR_SETTLE_MS + if (fresh || settled) { + this.applyCursorToKeyboard(p, true) + return + } + + // the cursor update for this position may still be in flight + this.pendingTap = { p, at: now } + this.pendingTapTimer = window.setTimeout(() => { + if (!this.pendingTap) return + const tap = this.pendingTap + this.pendingTap = null + this.applyCursorToKeyboard(tap.p, false) + }, CURSOR_WAIT_MS) + } + + applyCursorToKeyboard(p: Point, inGesture: boolean) { + if (this.cursorKind === 'text') { + if (this.keyboardOpen) return + if (inGesture && this.tapKeyboard === 'auto') { + this.focusForTyping() + } else { + this.showTypeChip(p) + } + } else if (this.cursorKind === 'other' && this.keyboardOpen) { + this._overlay.blur() + } + } + + onCursorImage(image: CursorImage) { + const apply = (kind: CursorKind) => { + this.cursorKind = kind + this.cursorChangedAt = performance.now() + + const tap = this.pendingTap + if (tap && performance.now() - tap.at < CURSOR_WAIT_MS) { + window.clearTimeout(this.pendingTapTimer) + this.pendingTap = null + this.applyCursorToKeyboard(tap.p, false) + } + } + + const cached = cachedCursorKind(image) + if (cached) { + apply(cached) + } else { + classifyCursor(image).then(apply) + } + } + + showTypeChip(p: Point) { + const player = this._player.getBoundingClientRect() + this.typeChip = { + x: Math.max(80, Math.min(player.width - 80, p.x - player.left)), + y: Math.max(8, Math.min(player.height - 52, p.y - player.top - 64)), + } + window.clearTimeout(this.typeChipTimer) + this.typeChipTimer = window.setTimeout(this.hideTypeChip, TYPE_CHIP_MS) + } + + hideTypeChip() { + window.clearTimeout(this.typeChipTimer) + this.typeChip = null + } + + onTypeChip() { + this.hideTypeChip() + this.focusForTyping() + } + + focusForTyping() { + this.resetInputSentinel() + this._overlay.focus({ preventScroll: true }) + } + + toggleMobileKeyboard() { + if (this.keyboardOpen) { + this._overlay.blur() + } else { + this.focusForTyping() + } + } + + resetInputSentinel() { + if (!this.is_touch_device) return + this._overlay.value = INPUT_SENTINEL + this._overlay.setSelectionRange(INPUT_SENTINEL.length, INPUT_SENTINEL.length) + } + + // Soft keyboards that report keyCode 229 (Android) deliver text only through + // input events, which the Guacamole keyboard does not handle. + onOverlayInput(e: Event) { + if (!this.is_touch_device) return + const event = e as InputEvent + + if (!this.canSendInput()) { + if (!this.composing) this.resetInputSentinel() + return + } + + if (event.inputType === 'insertCompositionText') { + this.updateComposition(event.data || '') return } + // the Guacamole keyboard already sent this key + const handledByKeydown = performance.now() - this.lastKeydownAt < 50 + const echoOfComposition = + event.data === this.lastCompositionEnd.data && performance.now() - this.lastCompositionEnd.at < 50 + + if (!handledByKeydown && !echoOfComposition) { + switch (event.inputType) { + case 'insertText': + case 'insertReplacementText': + this.typeText(event.data || '') + break + case 'insertLineBreak': + case 'insertParagraph': + this.pressKey(XK_RETURN) + break + case 'deleteContentBackward': + this.pressKey(XK_BACKSPACE) + break + } + } + + if (!this.composing) this.resetInputSentinel() + } + + onCompositionStart() { + this.composing = true + this.composed = '' + } + + onCompositionEnd(e: CompositionEvent) { + if (this.canSendInput()) this.updateComposition(e.data || '') + this.lastCompositionEnd = { data: e.data || '', at: performance.now() } + this.composing = false + this.composed = '' + this.resetInputSentinel() + } + + updateComposition(text: string) { + const { deletes, insert } = textDiff(this.composed, text) + for (let i = 0; i < deletes; i++) this.pressKey(XK_BACKSPACE) + this.typeText(insert) + this.composed = text + } + + typeText(text: string) { + for (const ch of Array.from(text)) { + const shift = needsShift(ch) && !this.keyboard.modifiers.shift + if (shift) this.$client.sendData('keydown', { key: XK_SHIFT_L }) + this.pressKey(charToKeysym(ch)) + if (shift) this.$client.sendData('keyup', { key: XK_SHIFT_L }) + } + } + + pressKey(key: number) { + this.$client.sendData('keydown', { key }) + this.$client.sendData('keyup', { key }) + } + + focusOverlay() { const focus = () => { if (this.hosting && !this.locked) { this._overlay.focus() @@ -859,7 +1343,7 @@ return } - this.focusOverlay(e) + this.focusOverlay() this.sendMousePos(e) this.pressedMouseButtons.add(e.button + 1) @@ -871,7 +1355,7 @@ return } - this.focusOverlay(e) + this.focusOverlay() this.sendMousePos(e) this.pressedMouseButtons.delete(e.button + 1) this.$client.sendData('mouseup', { key: e.button + 1 }) @@ -978,13 +1462,22 @@ } onOverlayFocus() { + this.keyboardOpen = true if (this.hosting) { this.syncClipboard() } } + onOverlayBlur() { + this.keyboardOpen = false + this.resetKeyboard() + } + onResize() { const { offsetWidth, offsetHeight } = !this.fullscreen ? this._component : document.body + if (`${offsetWidth}px` !== this._player.style.width || `${offsetHeight}px` !== this._player.style.height) { + this.resetZoom() + } this._player.style.width = `${offsetWidth}px` this._player.style.height = `${offsetHeight}px` const aspectPreservingMaxWidth = (this.horizontal / this.vertical) * offsetHeight @@ -1008,10 +1501,5 @@ this._overlay.focus() } } - - openMobileKeyboard() { - // focus opens the keyboard on mobile - this._overlay.focus() - } } diff --git a/images/chromium-headful/client/src/neko/base.ts b/images/chromium-headful/client/src/neko/base.ts index 5bacc96fe..80c68a7f7 100644 --- a/images/chromium-headful/client/src/neko/base.ts +++ b/images/chromium-headful/client/src/neko/base.ts @@ -1,6 +1,7 @@ import EventEmitter from 'eventemitter3' import { OPCODE } from './data' import { EVENT, WebSocketEvents } from './events' +import { CursorImage } from '~/utils/cursor-shape' import { WebSocketMessages, @@ -16,6 +17,7 @@ export interface BaseEvents { warn: (...message: any[]) => void debug: (...message: any[]) => void error: (error: Error) => void + cursor: (image: CursorImage) => void } export abstract class BaseClient extends EventEmitter { diff --git a/images/chromium-headful/client/src/neko/index.ts b/images/chromium-headful/client/src/neko/index.ts index 3571bb88e..5a68ec105 100644 --- a/images/chromium-headful/client/src/neko/index.ts +++ b/images/chromium-headful/client/src/neko/index.ts @@ -4,6 +4,7 @@ import { BaseClient, BaseEvents } from './base' import { Member } from './types' import { EVENT } from './events' import { accessor } from '~/store' +import { parseCursorImage } from '~/utils/cursor-shape' import { SystemMessagePayload, @@ -131,7 +132,17 @@ export class NekoClient extends BaseClient implements EventEmitter { this.$accessor.video.setStream(0) } - protected [EVENT.DATA]() {} + protected [EVENT.DATA](data: ArrayBuffer | Blob) { + if (data instanceof Blob) { + data.arrayBuffer().then((buffer) => this[EVENT.DATA](buffer)) + return + } + + const image = parseCursorImage(data) + if (image) { + this.emit('cursor', image) + } + } ///////////////////////////// // System Events diff --git a/images/chromium-headful/client/src/utils/cursor-shape.ts b/images/chromium-headful/client/src/utils/cursor-shape.ts new file mode 100644 index 000000000..ca0666d6d --- /dev/null +++ b/images/chromium-headful/client/src/utils/cursor-shape.ts @@ -0,0 +1,100 @@ +// Classifies the remote X cursor image that neko pushes over the data channel. +// Chromium switches to the text (I-beam) cursor over editable fields, which is +// the only signal the client has that a tap landed on something typeable. + +export type CursorKind = 'text' | 'other' | 'unknown' + +export interface CursorImage { + width: number + height: number + xhot: number + yhot: number + png: Uint8Array +} + +// data channel opcode for cursor images (neko server/internal/webrtc/payload/send.go) +const OP_CURSOR_IMAGE = 0x02 +const HEADER_SIZE = 3 +const CURSOR_HEADER_SIZE = 8 + +export function parseCursorImage(buffer: ArrayBuffer): CursorImage | undefined { + if (buffer.byteLength < HEADER_SIZE + CURSOR_HEADER_SIZE) return + const view = new DataView(buffer) + if (view.getUint8(0) !== OP_CURSOR_IMAGE) return + return { + width: view.getUint16(3), + height: view.getUint16(5), + xhot: view.getUint16(7), + yhot: view.getUint16(9), + png: new Uint8Array(buffer, HEADER_SIZE + CURSOR_HEADER_SIZE), + } +} + +const cache = new Map() + +function cacheKey(img: CursorImage) { + // FNV-1a over the image bytes; cursor images are small + let hash = 0x811c9dc5 + for (let i = 0; i < img.png.length; i++) { + hash ^= img.png[i] + hash = Math.imul(hash, 0x01000193) + } + return `${img.width}x${img.height}@${img.xhot},${img.yhot}:${img.png.length}:${hash >>> 0}` +} + +export function cachedCursorKind(img: CursorImage): CursorKind | undefined { + return cache.get(cacheKey(img)) +} + +export async function classifyCursor(img: CursorImage): Promise { + const key = cacheKey(img) + const cached = cache.get(key) + if (cached) return cached + + let kind: CursorKind = 'unknown' + try { + const bitmap = await createImageBitmap(new Blob([img.png], { type: 'image/png' })) + const canvas = document.createElement('canvas') + canvas.width = bitmap.width + canvas.height = bitmap.height + const ctx = canvas.getContext('2d') + if (ctx) { + ctx.drawImage(bitmap, 0, 0) + kind = classifyPixels(ctx.getImageData(0, 0, bitmap.width, bitmap.height), img.xhot, img.yhot) + } + bitmap.close() + } catch (e) { + kind = 'unknown' + } + + cache.set(key, kind) + return kind +} + +function classifyPixels(data: ImageData, xhot: number, yhot: number): CursorKind { + let minX = Infinity + let minY = Infinity + let maxX = -Infinity + let maxY = -Infinity + for (let y = 0; y < data.height; y++) { + for (let x = 0; x < data.width; x++) { + if (data.data[(y * data.width + x) * 4 + 3] > 64) { + minX = Math.min(minX, x) + maxX = Math.max(maxX, x) + minY = Math.min(minY, y) + maxY = Math.max(maxY, y) + } + } + } + // an empty image means the cursor is hidden + if (maxX < 0) return 'unknown' + + const width = maxX - minX + 1 + const height = maxY - minY + 1 + // I-beam: tall and narrow, hotspot in the middle of the shape. Arrows and + // hands have their hotspot at the tip, near the top edge. + const narrow = height >= 8 && width / height <= 0.6 + const centeredX = Math.abs(xhot - (minX + maxX) / 2) <= Math.max(2, width * 0.35) + const middleY = yhot > minY + height * 0.2 && yhot < maxY - height * 0.2 + return narrow && centeredX && middleY ? 'text' : 'other' +} diff --git a/images/chromium-headful/client/src/utils/text-input.ts b/images/chromium-headful/client/src/utils/text-input.ts new file mode 100644 index 000000000..a9c6f6220 --- /dev/null +++ b/images/chromium-headful/client/src/utils/text-input.ts @@ -0,0 +1,38 @@ +// Helpers for turning text from mobile soft keyboards into X keysyms. +// Android keyboards report keyCode 229 for most keys, which the Guacamole +// keyboard ignores, so their text only arrives through input events. + +export const XK_SHIFT_L = 0xffe1 +export const XK_BACKSPACE = 0xff08 +export const XK_RETURN = 0xff0d + +export function charToKeysym(ch: string): number { + if (ch === '\n' || ch === '\r') return XK_RETURN + const cp = ch.codePointAt(0) as number + // Latin-1 keysyms equal their code points; everything else uses the Unicode range + if ((cp >= 0x20 && cp <= 0x7e) || (cp >= 0xa0 && cp <= 0xff)) return cp + return 0x01000000 | cp +} + +// Neko maps a keysym to a keycode using the current modifier state. An upper +// case letter sent without Shift lands on the lower case key, so soft keyboards +// that never report Shift type the wrong case. +export function needsShift(ch: string) { + return ch !== ch.toLowerCase() && ch === ch.toUpperCase() +} + +export function keysymNeedsShift(keysym: number) { + if (keysym >= 0x41 && keysym <= 0x5a) return true + if (keysym >= 0xc0 && keysym <= 0xde && keysym !== 0xd7) return true + if ((keysym & 0xff000000) === 0x01000000) return needsShift(String.fromCodePoint(keysym & 0x00ffffff)) + return false +} + +// Minimal edit that turns prev into next: delete from the end, then insert. +export function textDiff(prev: string, next: string) { + const a = Array.from(prev) + const b = Array.from(next) + let common = 0 + while (common < a.length && common < b.length && a[common] === b[common]) common++ + return { deletes: a.length - common, insert: b.slice(common).join('') } +} diff --git a/images/chromium-headful/client/src/utils/touch-gestures.ts b/images/chromium-headful/client/src/utils/touch-gestures.ts new file mode 100644 index 000000000..2656dfbdd --- /dev/null +++ b/images/chromium-headful/client/src/utils/touch-gestures.ts @@ -0,0 +1,223 @@ +// Turns raw touch events on the live view overlay into semantic gestures: +// tap, long-press (release = right click, move = drag), one-finger scroll with +// momentum, and two-finger pinch/pan. Coordinates are client (CSS) pixels. + +export interface Point { + x: number + y: number +} + +export interface GestureHandlers { + // first finger down; called before any other handler for the sequence + onTouchBegin(p: Point): void + // called synchronously from touchend, so it runs inside the user gesture + onTap(p: Point): void + onLongPress(p: Point): void + onLongPressRelease(p: Point): void + onDragStart(p: Point): void + onDragMove(p: Point): void + onDragEnd(p: Point): void + // finger delta since the previous call; positive means the finger moved right/down + onScroll(dx: number, dy: number, anchor: Point): void + onPinchStart(mid: Point, distance: number): void + onPinchMove(mid: Point, distance: number): void + onPinchEnd(): void +} + +const TAP_SLOP = 10 +const LONG_PRESS_MS = 500 +const VELOCITY_WINDOW_MS = 100 +// a finger that rested this long before lifting does not fling +const FLING_REST_MS = 60 +// per-millisecond velocity decay, close to iOS UIScrollView's "normal" rate +const FLING_DECAY = 0.9975 +const FLING_MIN_START = 0.15 +const FLING_MIN_STOP = 0.02 + +type Mode = 'idle' | 'pending' | 'scroll' | 'held' | 'drag' | 'pinch' + +interface Sample extends Point { + t: number +} + +function point(t: Touch): Point { + return { x: t.clientX, y: t.clientY } +} + +function distance(a: Point, b: Point) { + return Math.hypot(a.x - b.x, a.y - b.y) +} + +function midpoint(a: Point, b: Point): Point { + return { x: (a.x + b.x) / 2, y: (a.y + b.y) / 2 } +} + +export class TouchGestures { + private mode: Mode = 'idle' + private start: Point = { x: 0, y: 0 } + private last: Point = { x: 0, y: 0 } + private samples: Sample[] = [] + private longPressTimer?: ReturnType + private flingFrame?: number + private interruptedFling = false + + constructor(private handlers: GestureHandlers) {} + + get flinging() { + return this.flingFrame !== undefined + } + + touchStart(e: TouchEvent) { + if (e.touches.length === 1 && this.mode === 'idle') { + this.interruptedFling = this.stopFling() + const p = point(e.touches[0]) + this.start = p + this.last = p + this.samples = [{ ...p, t: e.timeStamp }] + this.mode = 'pending' + this.handlers.onTouchBegin(p) + this.longPressTimer = setTimeout(() => { + this.longPressTimer = undefined + if (this.mode === 'pending') { + this.mode = 'held' + this.handlers.onLongPress(this.start) + } + }, LONG_PRESS_MS) + return + } + + if (e.touches.length >= 2 && this.mode !== 'pinch') { + this.clearLongPress() + this.stopFling() + if (this.mode === 'drag') this.handlers.onDragEnd(this.last) + this.mode = 'pinch' + const a = point(e.touches[0]) + const b = point(e.touches[1]) + this.handlers.onPinchStart(midpoint(a, b), distance(a, b)) + } + } + + touchMove(e: TouchEvent) { + if (this.mode === 'pinch') { + if (e.touches.length >= 2) { + const a = point(e.touches[0]) + const b = point(e.touches[1]) + this.handlers.onPinchMove(midpoint(a, b), distance(a, b)) + } + return + } + + if (e.touches.length !== 1) return + const p = point(e.touches[0]) + + switch (this.mode) { + case 'pending': + if (distance(p, this.start) <= TAP_SLOP) return + this.clearLongPress() + this.mode = 'scroll' + this.handlers.onScroll(p.x - this.last.x, p.y - this.last.y, this.start) + break + case 'scroll': + this.handlers.onScroll(p.x - this.last.x, p.y - this.last.y, this.start) + break + case 'held': + if (distance(p, this.start) <= TAP_SLOP) return + this.mode = 'drag' + this.handlers.onDragStart(this.start) + this.handlers.onDragMove(p) + break + case 'drag': + this.handlers.onDragMove(p) + break + } + + this.last = p + this.samples.push({ ...p, t: e.timeStamp }) + while (this.samples.length > 2 && e.timeStamp - this.samples[0].t > VELOCITY_WINDOW_MS) { + this.samples.shift() + } + } + + touchEnd(e: TouchEvent) { + // wait for every finger to lift; a leftover finger after a pinch must not scroll + if (e.touches.length > 0) return + this.clearLongPress() + + const mode = this.mode + this.mode = 'idle' + + switch (mode) { + case 'pending': + if (!this.interruptedFling) this.handlers.onTap(this.last) + break + case 'held': + this.handlers.onLongPressRelease(this.last) + break + case 'drag': + this.handlers.onDragEnd(this.last) + break + case 'scroll': + this.startFling(e.timeStamp) + break + case 'pinch': + this.handlers.onPinchEnd() + break + } + } + + touchCancel() { + this.clearLongPress() + if (this.mode === 'drag') this.handlers.onDragEnd(this.last) + if (this.mode === 'pinch') this.handlers.onPinchEnd() + this.mode = 'idle' + } + + // returns true if a fling was running + stopFling() { + if (this.flingFrame === undefined) return false + cancelAnimationFrame(this.flingFrame) + this.flingFrame = undefined + return true + } + + destroy() { + this.clearLongPress() + this.stopFling() + } + + private clearLongPress() { + if (this.longPressTimer !== undefined) { + clearTimeout(this.longPressTimer) + this.longPressTimer = undefined + } + } + + private startFling(endTime: number) { + const first = this.samples[0] + const last = this.samples[this.samples.length - 1] + if (!first || !last || endTime - last.t > FLING_REST_MS) return + const dt = last.t - first.t + if (dt <= 0) return + + let vx = (last.x - first.x) / dt + let vy = (last.y - first.y) / dt + if (Math.hypot(vx, vy) < FLING_MIN_START) return + + const anchor = this.start + let prev = performance.now() + const step = (now: number) => { + const elapsed = Math.min(now - prev, 50) + prev = now + const decay = Math.pow(FLING_DECAY, elapsed) + vx *= decay + vy *= decay + this.handlers.onScroll(vx * elapsed, vy * elapsed, anchor) + if (Math.hypot(vx, vy) < FLING_MIN_STOP) { + this.flingFrame = undefined + return + } + this.flingFrame = requestAnimationFrame(step) + } + this.flingFrame = requestAnimationFrame(step) + } +} diff --git a/images/chromium-headful/client/src/utils/zoom-pan.ts b/images/chromium-headful/client/src/utils/zoom-pan.ts new file mode 100644 index 000000000..2e63bb053 --- /dev/null +++ b/images/chromium-headful/client/src/utils/zoom-pan.ts @@ -0,0 +1,67 @@ +// Local zoom and pan of the live view video. The remote screen is unchanged; +// this only scales what the viewer sees so a large stream is usable on a phone. + +import { Point } from './touch-gestures' + +export interface Box { + left: number + top: number + width: number + height: number +} + +export class ZoomPan { + scale = 1 + tx = 0 + ty = 0 + + private startScale = 1 + private startDistance = 1 + private anchor: Point = { x: 0, y: 0 } + + // element: the video box before the transform. viewport: the visible area the + // zoomed video may fill, which includes any letterbox bars around it. + constructor(private element: () => Box, private viewport: () => Box, private maxScale: () => number) {} + + get zoomed() { + return this.scale > 1.01 + } + + pinchStart(mid: Point, distance: number) { + const e = this.element() + this.startScale = this.scale + this.startDistance = Math.max(distance, 1) + // content point (in unscaled element px) under the fingers + this.anchor = { + x: (mid.x - e.left - this.tx) / this.scale, + y: (mid.y - e.top - this.ty) / this.scale, + } + } + + pinchMove(mid: Point, distance: number) { + const e = this.element() + const scale = Math.min(this.maxScale(), Math.max(1, (this.startScale * distance) / this.startDistance)) + this.scale = scale + this.tx = clampAxis(mid.x - e.left - this.anchor.x * scale, e.left, e.width * scale, this.viewport(), 'x') + this.ty = clampAxis(mid.y - e.top - this.anchor.y * scale, e.top, e.height * scale, this.viewport(), 'y') + } + + reset() { + this.scale = 1 + this.tx = 0 + this.ty = 0 + } + + get transform() { + if (!this.zoomed) return '' + return `translate(${this.tx}px, ${this.ty}px) scale(${this.scale})` + } +} + +// Content larger than the viewport must cover it; smaller content stays centered. +function clampAxis(offset: number, origin: number, size: number, viewport: Box, axis: 'x' | 'y') { + const start = axis === 'x' ? viewport.left : viewport.top + const length = axis === 'x' ? viewport.width : viewport.height + if (size <= length) return start + (length - size) / 2 - origin + return Math.min(start - origin, Math.max(start + length - size - origin, offset)) +} diff --git a/images/chromium-headful/client/tests/touch-input.test.ts b/images/chromium-headful/client/tests/touch-input.test.ts new file mode 100644 index 000000000..744e9be7f --- /dev/null +++ b/images/chromium-headful/client/tests/touch-input.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, test } from 'bun:test' +import { charToKeysym, keysymNeedsShift, needsShift, textDiff, XK_RETURN } from '../src/utils/text-input' +import { ZoomPan } from '../src/utils/zoom-pan' +import { GestureHandlers, Point, TouchGestures } from '../src/utils/touch-gestures' + +describe('text input', () => { + test('maps characters to keysyms', () => { + expect(charToKeysym('a')).toBe(0x61) + expect(charToKeysym('é')).toBe(0xe9) + expect(charToKeysym('€')).toBe(0x010020ac) + expect(charToKeysym('\n')).toBe(XK_RETURN) + }) + + test('detects letters that need shift', () => { + expect(needsShift('A')).toBe(true) + expect(needsShift('a')).toBe(false) + expect(needsShift('!')).toBe(false) + expect(keysymNeedsShift(0x41)).toBe(true) + expect(keysymNeedsShift(0x61)).toBe(false) + expect(keysymNeedsShift(0xc9)).toBe(true) + expect(keysymNeedsShift(0x01000416)).toBe(true) + }) + + test('diffs composition updates', () => { + expect(textDiff('', 'te')).toEqual({ deletes: 0, insert: 'te' }) + expect(textDiff('tez', 'tes')).toEqual({ deletes: 1, insert: 's' }) + expect(textDiff('hello', 'help')).toEqual({ deletes: 2, insert: 'p' }) + }) +}) + +describe('zoom and pan', () => { + const element = { left: 0, top: 200, width: 400, height: 225 } + const viewport = { left: 0, top: 0, width: 400, height: 625 } + + test('keeps the content point under the fingers while scaling', () => { + const zoom = new ZoomPan( + () => element, + () => viewport, + () => 8, + ) + zoom.pinchStart({ x: 200, y: 312 }, 100) + zoom.pinchMove({ x: 200, y: 312 }, 300) + expect(zoom.scale).toBe(3) + // content point (200, 112) in element px stays at client (200, 312) + expect(element.left + zoom.tx + 200 * zoom.scale).toBeCloseTo(200) + expect(element.top + zoom.ty + 112 * zoom.scale).toBeCloseTo(312) + }) + + test('clamps scale and keeps the viewport covered', () => { + const zoom = new ZoomPan( + () => element, + () => viewport, + () => 4, + ) + zoom.pinchStart({ x: 10, y: 210 }, 100) + zoom.pinchMove({ x: 390, y: 600 }, 1000) + expect(zoom.scale).toBe(4) + expect(element.left + zoom.tx).toBeLessThanOrEqual(viewport.left) + expect(element.left + zoom.tx + element.width * 4).toBeGreaterThanOrEqual(viewport.left + viewport.width) + expect(element.top + zoom.ty).toBeLessThanOrEqual(viewport.top) + expect(element.top + zoom.ty + element.height * 4).toBeGreaterThanOrEqual(viewport.top + viewport.height) + }) + + test('never zooms out below the fitted size', () => { + const zoom = new ZoomPan( + () => element, + () => viewport, + () => 4, + ) + zoom.pinchStart({ x: 200, y: 312 }, 300) + zoom.pinchMove({ x: 200, y: 312 }, 100) + expect(zoom.scale).toBe(1) + expect(zoom.transform).toBe('') + }) +}) + +describe('touch gestures', () => { + const touches = (...points: Point[]) => points.map((p) => ({ clientX: p.x, clientY: p.y })) as unknown as TouchList + const event = (timeStamp: number, ...points: Point[]) => + ({ timeStamp, touches: touches(...points) } as unknown as TouchEvent) + + const recorder = () => { + const calls: string[] = [] + const handlers = new Proxy({} as GestureHandlers, { + get: + (_, name: string) => + (...args: unknown[]) => + calls.push(`${name} ${JSON.stringify(args)}`), + }) + return { calls, handlers } + } + + test('a short touch without movement is a tap', () => { + const { calls, handlers } = recorder() + const g = new TouchGestures(handlers) + g.touchStart(event(0, { x: 10, y: 10 })) + g.touchMove(event(20, { x: 12, y: 11 })) + g.touchEnd(event(60)) + g.destroy() + expect(calls.map((c) => c.split(' ')[0])).toEqual(['onTouchBegin', 'onTap']) + }) + + test('movement past the slop scrolls by the finger delta', () => { + const { calls, handlers } = recorder() + const g = new TouchGestures(handlers) + g.touchStart(event(0, { x: 100, y: 300 })) + g.touchMove(event(16, { x: 100, y: 280 })) + g.touchMove(event(32, { x: 100, y: 250 })) + g.touchEnd(event(200)) + g.destroy() + const scrolls = calls.filter((c) => c.startsWith('onScroll')) + expect(scrolls.length).toBe(2) + expect(scrolls[0]).toContain('[0,-20,') + expect(scrolls[1]).toContain('[0,-30,') + expect(calls.some((c) => c.startsWith('onTap'))).toBe(false) + }) + + test('two fingers pinch and never scroll or tap', () => { + const { calls, handlers } = recorder() + const g = new TouchGestures(handlers) + g.touchStart(event(0, { x: 100, y: 300 })) + g.touchStart(event(10, { x: 100, y: 300 }, { x: 200, y: 300 })) + g.touchMove(event(26, { x: 80, y: 300 }, { x: 220, y: 300 })) + g.touchEnd(event(40, { x: 220, y: 300 })) + g.touchMove(event(56, { x: 260, y: 300 })) + g.touchEnd(event(70)) + g.destroy() + const names = calls.map((c) => c.split(' ')[0]) + expect(names).toEqual(['onTouchBegin', 'onPinchStart', 'onPinchMove', 'onPinchEnd']) + }) +}) From 52d6c145b56e218ee203e78e6a5e7ea35bdce740 Mon Sep 17 00:00:00 2001 From: masnwilliams <43387599+masnwilliams@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:08:02 +0000 Subject: [PATCH 2/9] Keep the keyboard button above the soft keyboard and ignore stale cursor shapes Lift the keyboard button by the height the visual viewport loses to an overlaid soft keyboard, and only apply the classification of the newest cursor image. --- .../client/src/components/video.vue | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/images/chromium-headful/client/src/components/video.vue b/images/chromium-headful/client/src/components/video.vue index d44d51238..a892eb770 100644 --- a/images/chromium-headful/client/src/components/video.vue +++ b/images/chromium-headful/client/src/components/video.vue @@ -86,6 +86,7 @@ + ref="controls" + :mode="touchLayout.mode" + :position="controlPosition" + :keyboard-open="keyboardOpen" + :area-width="playerWidth" + :area-height="playerHeight" + :keyboard-inset="keyboardInset" + @toggle="toggleMobileKeyboard" + @move="onControlMove" + />