diff --git a/images/chromium-headful/client/public/index.html b/images/chromium-headful/client/public/index.html
index 6869c600b..e327bfa9b 100644
--- a/images/chromium-headful/client/public/index.html
+++ b/images/chromium-headful/client/public/index.html
@@ -3,7 +3,7 @@
-
+
Kernel
diff --git a/images/chromium-headful/client/src/app.vue b/images/chromium-headful/client/src/app.vue
index 3d7a83039..cf25d9983 100644
--- a/images/chromium-headful/client/src/app.vue
+++ b/images/chromium-headful/client/src/app.vue
@@ -56,11 +56,12 @@
bottom: 0;
max-width: 100vw;
max-height: 100vh;
+ max-height: 100dvh;
flex-direction: row;
display: flex;
.neko-main {
- min-width: 360px;
+ min-width: 0;
max-width: 100%;
flex-grow: 1;
flex-direction: column;
@@ -144,10 +145,12 @@
.neko-main {
height: 100vh;
+ height: 100dvh;
}
.neko-menu {
height: 100vh;
+ height: 100dvh;
width: 100% !important;
}
}
diff --git a/images/chromium-headful/client/src/components/touch-controls.vue b/images/chromium-headful/client/src/components/touch-controls.vue
new file mode 100644
index 000000000..69e33665c
--- /dev/null
+++ b/images/chromium-headful/client/src/components/touch-controls.vue
@@ -0,0 +1,228 @@
+
+
+
+
+
+
+
+
+
diff --git a/images/chromium-headful/client/src/components/video.vue b/images/chromium-headful/client/src/components/video.vue
index 6b0355651..ab1724296 100644
--- a/images/chromium-headful/client/src/components/video.vue
+++ b/images/chromium-headful/client/src/components/video.vue
@@ -12,9 +12,12 @@
ref="overlay"
class="overlay"
spellcheck="false"
+ autocapitalize="off"
+ autocorrect="off"
+ autocomplete="off"
tabindex="0"
data-gramm="false"
- :style="{ pointerEvents: hosting ? 'auto' : 'none' }"
+ :style="{ pointerEvents: hosting || is_touch_device ? 'auto' : 'none' }"
@click.stop.prevent
@contextmenu.stop.prevent
@mousemove.stop.prevent="onMouseMove"
@@ -22,12 +25,16 @@
@mouseup.stop.prevent="onMouseUp"
@mouseenter.stop.prevent="onMouseEnter"
@mouseleave.stop.prevent="onMouseLeave"
- @touchmove.stop.prevent="onTouchHandler"
- @touchstart.stop.prevent="onTouchHandler"
- @touchend.stop.prevent="onTouchHandler"
+ @touchstart.stop.prevent="onTouchStart"
+ @touchmove.stop.prevent="onTouchMove"
+ @touchend.stop.prevent="onTouchEnd"
+ @touchcancel.stop.prevent="onTouchCancel"
+ @input="onOverlayInput"
+ @compositionstart="onCompositionStart"
+ @compositionend="onCompositionEnd"
@paste.stop.prevent="onPaste"
@focus="onOverlayFocus"
- @blur="resetKeyboard"
+ @blur="onOverlayBlur"
/>
-
+
-
-
+
+
+
@@ -93,10 +127,44 @@
.player {
position: absolute;
+ box-sizing: border-box;
display: flex;
justify-content: center;
align-items: center;
background: #000;
+ overflow: hidden;
+ touch-action: none;
+
+ .touch-button {
+ position: absolute;
+ z-index: 2;
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ min-height: 44px;
+ padding: 0 16px;
+ border: 0;
+ border-radius: 22px;
+ background: rgba($color: #000, $alpha: 0.7);
+ color: #fff;
+ font-size: 15px;
+ font-weight: 600;
+ white-space: nowrap;
+ cursor: pointer;
+ touch-action: manipulation;
+ -webkit-tap-highlight-color: transparent;
+ }
+
+ .zoom-chip {
+ left: calc(12px + env(safe-area-inset-left));
+ top: calc(12px + env(safe-area-inset-top));
+ }
+
+ .type-chip {
+ transform: translateX(-50%);
+ background: #81b300;
+ color: #000;
+ }
.video-menu {
position: absolute;
@@ -153,6 +221,7 @@
position: relative;
width: 100%;
max-width: calc(16 / 9 * 100vh);
+ transform-origin: 0 0;
video {
position: absolute;
@@ -205,8 +274,12 @@
outline: 0;
border: 0;
color: transparent;
+ caret-color: transparent;
background: transparent;
resize: none;
+ touch-action: none;
+ // iOS zooms the page when focusing a field with a smaller font
+ font-size: 16px;
}
.player-aspect {
@@ -223,10 +296,32 @@
import ResizeObserver from 'resize-observer-polyfill'
import { elementRequestFullscreen, onFullscreenChange, isFullscreen, lockKeyboard, unlockKeyboard } from '~/utils'
import { isClipboardReadGranted } from '~/utils/clipboard'
+ import { TouchGestures, Point } from '~/utils/touch-gestures'
+ import { ZoomPan, Box } from '~/utils/zoom-pan'
+ import {
+ ControlLayout,
+ ControlPosition,
+ SafeArea,
+ controlLayout,
+ decodePosition,
+ encodePosition,
+ } from '~/utils/touch-controls'
+ import { get, set } from '~/utils/localstorage'
+ import { CursorImage, CursorKind, cachedCursorKind, classifyCursor } from '~/utils/cursor-shape'
+ import {
+ XK_BACKSPACE,
+ XK_RETURN,
+ XK_SHIFT_L,
+ charToKeysym,
+ keysymNeedsShift,
+ needsShift,
+ textDiff,
+ } from '~/utils/text-input'
import Emote from './emote.vue'
import Resolution from './resolution.vue'
import Clipboard from './clipboard.vue'
+ import TouchControls from './touch-controls.vue'
// @ts-ignore
import GuacamoleKeyboard from '~/utils/guacamole-keyboard.ts'
@@ -235,12 +330,33 @@
const SCROLL_SENSITIVITY_BASE = 10
const INT16_MAX = 32767
+ // wheel units per remote pixel of finger travel, so content tracks the finger
+ const TOUCH_SCROLL_UNITS_PER_PX = 1.1
+ const DOUBLE_TAP_MS = 300
+ const DOUBLE_TAP_SLOP = 40
+ // how long after a tap a text cursor can still offer the keyboard
+ const CURSOR_WAIT_MS = 800
+ // a tap this close (remote px) to where a text cursor was seen raises the keyboard
+ const CURSOR_SAME_SPOT = 16
+ const TYPE_CHIP_MS = 4000
+ // kept in the overlay so soft keyboards that only emit input events still report backspace
+ const INPUT_SENTINEL = ' '
+
+ function nearSpot(a: Point, b: Point) {
+ return Math.hypot(a.x - b.x, a.y - b.y) <= CURSOR_SAME_SPOT
+ }
+
+ type TapKeyboardMode = 'auto' | 'chip' | 'always' | 'off'
+
+ const CONTROL_POSITION_KEY = 'touch_control_position'
+
@Component({
name: 'neko-video',
components: {
'neko-emote': Emote,
'neko-resolution': Resolution,
'neko-clipboard': Clipboard,
+ 'neko-touch-controls': TouchControls,
},
})
export default class extends Vue {
@@ -254,6 +370,7 @@
private _wheelHandler: ((e: WheelEvent) => void) | null = null
@Ref('clipboard') readonly _clipboard!: Clipboard
+ @Ref('controls') readonly _controls?: TouchControls
// all controls are hidden (e.g. for cast mode)
@Prop(Boolean) readonly hideControls!: boolean
@@ -271,6 +388,39 @@
private mutedOverlay = true
private isVideoSyncing = false
+ private gestures!: TouchGestures
+ private zoom!: ZoomPan
+ private zoomScale = 1
+ private keyboardOpen = false
+ private typeChip: Point | null = null
+ private typeChipTimer = 0
+ private cursorKind: CursorKind = 'unknown'
+ private cursorChangedAt = 0
+ private cursorSeq = 0
+ // remote pointer position: last sent, and where a text cursor was last seen
+ private pointer: Point | null = null
+ private textSpot: Point | null = null
+ // when the pointer last moved to a new spot, and the time before that
+ private pointerJumpAt = 0
+ private previousPointerJumpAt = 0
+ private keyboardInset = 0
+ private touchLayout: ControlLayout = { mode: 'overlay', band: 0 }
+ private controlPosition: ControlPosition = decodePosition(get(CONTROL_POSITION_KEY, ''))
+ private playerWidth = 0
+ private playerHeight = 0
+ private safeAreaProbe: HTMLElement | null = null
+ private safeArea: SafeArea = { top: 0, bottom: 0, left: 0, right: 0 }
+ private touchBeganAt = 0
+ private lastTap: { p: Point; at: number } | null = null
+ private pendingTap: { p: Point; here: Point; at: number } | null = null
+ private pendingTapTimer = 0
+ private scrollRemainder = { x: 0, y: 0 }
+ private composing = false
+ private composed = ''
+ private lastCompositionEnd = { data: '', at: 0 }
+ private lastKeydownAt = 0
+ private shiftWrapped = new Set()
+
get admin() {
return this.$accessor.user.admin
}
@@ -399,13 +549,31 @@
)
}
+ get showTouchControls() {
+ return this.hosting && this.is_touch_device && !this.hideControls && !this.fullscreen
+ }
+
+ get zoomed() {
+ return this.zoomScale > 1.01
+ }
+
+ // ?tapKeyboard=auto (default) raises the keyboard when a tap lands on text,
+ // chip offers a "tap to type" button there instead, always raises it on
+ // every tap, off leaves it to the keyboard button
+ get tapKeyboard(): TapKeyboardMode {
+ const value = new URL(location.href).searchParams.get('tapKeyboard')
+ return value === 'chip' || value === 'always' || value === 'off' ? value : 'auto'
+ }
+
@Watch('width')
onWidthChanged() {
+ this.resetZoom()
this.onResize()
}
@Watch('height')
onHeightChanged() {
+ this.resetZoom()
this.onResize()
}
@@ -491,6 +659,22 @@
}
mounted() {
+ this.zoom = new ZoomPan(this.containerBox, this.playerBox, this.maxZoom)
+ this.gestures = new TouchGestures({
+ onTouchBegin: this.onGestureBegin,
+ onTap: this.onGestureTap,
+ onLongPress: this.onGestureLongPress,
+ onLongPressRelease: this.onGestureLongPressRelease,
+ onDragStart: this.onGestureDragStart,
+ onDragMove: this.onGestureDragMove,
+ onDragEnd: this.onGestureDragEnd,
+ onScroll: this.onGestureScroll,
+ onPinchStart: this.onGesturePinchStart,
+ onPinchMove: this.onGesturePinchMove,
+ onPinchEnd: this.onGesturePinchEnd,
+ })
+ this.$client.on('cursor', this.onCursorImage)
+
this._container.addEventListener('resize', this.onResize)
this.onVolumeChanged(this.volume)
this.onMutedChanged(this.muted)
@@ -548,6 +732,8 @@
}
document.addEventListener('wheel', this._wheelHandler, { passive: false, capture: true })
window.addEventListener('blur', this.resetKeyboard)
+ window.visualViewport?.addEventListener('resize', this.updateKeyboardInset)
+ window.visualViewport?.addEventListener('scroll', this.updateKeyboardInset)
window.addEventListener('pagehide', this.resetKeyboard)
document.addEventListener('visibilitychange', this.resetKeyboardWhenHidden)
@@ -570,6 +756,12 @@
return true
}
+ this.lastKeydownAt = performance.now()
+ if (this.is_touch_device && keysymNeedsShift(key) && !this.keyboard.modifiers.shift) {
+ this.shiftWrapped.add(key)
+ this.$client.sendData('keydown', { key: XK_SHIFT_L })
+ }
+
this.$client.sendData('keydown', { key: this.keyMap(key) })
}
this.keyboard.onkeyup = (key: number) => {
@@ -583,16 +775,26 @@
}
this.$client.sendData('keyup', { key: this.keyMap(key) })
+ if (this.shiftWrapped.delete(key)) {
+ this.$client.sendData('keyup', { key: XK_SHIFT_L })
+ }
}
this.keyboard.listenTo(this._overlay)
}
beforeDestroy() {
+ this.gestures.destroy()
+ this.$client.off('cursor', this.onCursorImage)
+ window.clearTimeout(this.typeChipTimer)
+ window.clearTimeout(this.pendingTapTimer)
+ if (this.safeAreaProbe) this.safeAreaProbe.remove()
if (this._wheelHandler) {
document.removeEventListener('wheel', this._wheelHandler, { capture: true })
this._wheelHandler = null
}
window.removeEventListener('blur', this.resetKeyboard)
+ window.visualViewport?.removeEventListener('resize', this.updateKeyboardInset)
+ window.visualViewport?.removeEventListener('scroll', this.updateKeyboardInset)
window.removeEventListener('pagehide', this.resetKeyboard)
document.removeEventListener('visibilitychange', this.resetKeyboardWhenHidden)
this.observer.disconnect()
@@ -797,43 +999,405 @@
}
}
- onTouchHandler(e: TouchEvent) {
- let first = e.changedTouches[0]
- let type = ''
- switch (e.type) {
- case 'touchstart':
- type = 'mousedown'
- break
- case 'touchmove':
- type = 'mousemove'
- break
- case 'touchend':
- type = 'mouseup'
- break
- default:
- return
+ onTouchStart(e: TouchEvent) {
+ this.gestures.touchStart(e)
+ }
+
+ onTouchMove(e: TouchEvent) {
+ this.gestures.touchMove(e)
+ }
+
+ onTouchEnd(e: TouchEvent) {
+ this.gestures.touchEnd(e)
+ }
+
+ onTouchCancel() {
+ this.gestures.touchCancel()
+ }
+
+ canSendInput() {
+ return this.hosting && !this.locked
+ }
+
+ // maps a client point on the (possibly zoomed) video to remote screen pixels
+ remotePoint(p: Point) {
+ const { w, h } = this.$accessor.video.resolution
+ const rect = this._overlay.getBoundingClientRect()
+ return {
+ x: Math.max(0, Math.min(w - 1, Math.round((w / rect.width) * (p.x - rect.left)))),
+ y: Math.max(0, Math.min(h - 1, Math.round((h / rect.height) * (p.y - rect.top)))),
}
+ }
- const simulatedEvent = new MouseEvent(type, {
- bubbles: true,
- cancelable: true,
- view: window,
- screenX: first.screenX,
- screenY: first.screenY,
- clientX: first.clientX,
- clientY: first.clientY,
+ sendPointer(p: Point) {
+ const next = this.remotePoint(p)
+ if (!this.pointer || !nearSpot(next, this.pointer)) {
+ this.previousPointerJumpAt = this.pointerJumpAt
+ this.pointerJumpAt = performance.now()
+ }
+ this.pointer = next
+ this.$client.sendData('mousemove', next)
+ }
+
+ // The remote pointer position a new cursor image belongs to, or null when
+ // an update for an earlier position could still be in flight.
+ cursorImageOwner() {
+ return this.pointerJumpAt - this.previousPointerJumpAt >= CURSOR_WAIT_MS ? this.pointer : null
+ }
+
+ clickAt(p: Point, button: number) {
+ this.sendPointer(p)
+ this.$client.sendData('mousedown', { key: button })
+ this.$client.sendData('mouseup', { key: button })
+ }
+
+ onGestureBegin(p: Point) {
+ this.unmuteOnInteraction()
+ this.hideTypeChip()
+ if (this._controls) this._controls.wake()
+ this.touchBeganAt = performance.now()
+
+ if (!this.controlling && this.implicitHosting && !this.locked) {
+ this.$accessor.remote.request()
+ }
+ if (!this.hosting) {
+ this.$emit('control-attempt')
+ }
+ if (this.canSendInput()) {
+ this.sendPointer(p)
+ }
+ }
+
+ onGestureTap(p: Point) {
+ const now = performance.now()
+ const last = this.lastTap
+ this.lastTap = { p, at: now }
+
+ if (
+ this.zoomed &&
+ last &&
+ now - last.at < DOUBLE_TAP_MS &&
+ Math.hypot(p.x - last.p.x, p.y - last.p.y) < DOUBLE_TAP_SLOP
+ ) {
+ this.lastTap = null
+ this.resetZoom()
+ return
+ }
+
+ if (!this.canSendInput()) return
+ this.clickAt(p, 1)
+ this.keyboardAfterTap(p)
+ }
+
+ onGestureLongPress() {
+ if (!this.canSendInput()) return
+ if (navigator.vibrate) navigator.vibrate(15)
+ }
+
+ onGestureLongPressRelease(p: Point) {
+ if (!this.canSendInput()) return
+ this.clickAt(p, 3)
+ }
+
+ onGestureDragStart(p: Point) {
+ if (!this.canSendInput()) return
+ this.sendPointer(p)
+ this.pressedMouseButtons.add(1)
+ this.$client.sendData('mousedown', { key: 1 })
+ }
+
+ onGestureDragMove(p: Point) {
+ if (!this.canSendInput()) return
+ this.sendPointer(p)
+ }
+
+ onGestureDragEnd(p: Point) {
+ if (!this.pressedMouseButtons.has(1)) return
+ this.sendPointer(p)
+ this.pressedMouseButtons.delete(1)
+ this.$client.sendData('mouseup', { key: 1 })
+ }
+
+ onGestureScroll(dx: number, dy: number) {
+ if (!this.canSendInput()) {
+ this.gestures.stopFling()
+ return
+ }
+
+ // finger travel in remote pixels; content follows the finger, so scroll the opposite way
+ const remotePerClient = this.width / this._overlay.getBoundingClientRect().width
+ const units = TOUCH_SCROLL_UNITS_PER_PX * remotePerClient
+ this.scrollRemainder.x -= dx * units
+ this.scrollRemainder.y -= dy * units
+
+ const x = Math.trunc(this.scrollRemainder.x)
+ const y = Math.trunc(this.scrollRemainder.y)
+ if (x === 0 && y === 0) return
+ this.scrollRemainder.x -= x
+ this.scrollRemainder.y -= y
+ this.$client.sendData('wheel', {
+ x: Math.max(-INT16_MAX, Math.min(INT16_MAX, x)),
+ y: Math.max(-INT16_MAX, Math.min(INT16_MAX, y)),
})
- first.target.dispatchEvent(simulatedEvent)
}
- focusOverlay(e: MouseEvent) {
- // Touch input is translated into an untrusted mouse event above. Keep the
- // existing mobile-keyboard behavior while allowing a real mouse to focus
- // the overlay on touch-capable devices.
- if (this.is_touch_device && !e.isTrusted) {
+ onGesturePinchStart(mid: Point, distance: number) {
+ this.hideTypeChip()
+ this.zoom.pinchStart(mid, distance)
+ }
+
+ onGesturePinchMove(mid: Point, distance: number) {
+ this.zoom.pinchMove(mid, distance)
+ this.applyZoom(false)
+ }
+
+ onGesturePinchEnd() {
+ if (!this.zoom.zoomed) this.resetZoom()
+ }
+
+ // the container box before the zoom transform, in client coordinates
+ containerBox(): Box {
+ const player = this._player.getBoundingClientRect()
+ return {
+ left: player.left + this._container.offsetLeft,
+ top: player.top + this._container.offsetTop,
+ width: this._container.offsetWidth,
+ height: this._container.offsetHeight,
+ }
+ }
+
+ // the area the zoomed video may fill: the player minus the control band
+ playerBox(): Box {
+ const { left, top, width, height } = this._player.getBoundingClientRect()
+ const { mode, band } = this.touchLayout
+ if (mode === 'bottom') return { left, top, width, height: height - band }
+ if (mode === 'left') return { left: left + band, top, width: width - band, height }
+ if (mode === 'right') return { left, top, width: width - band, height }
+ return { left, top, width, height }
+ }
+
+ maxZoom() {
+ const fit = this.width / Math.max(this._container.offsetWidth, 1)
+ return Math.min(8, Math.max(3, fit * 1.5))
+ }
+
+ applyZoom(animate: boolean) {
+ this._container.style.transition = animate ? 'transform 150ms ease-out' : ''
+ this._container.style.transform = this.zoom.transform
+ this.zoomScale = this.zoom.scale
+ }
+
+ resetZoom() {
+ if (!this.zoom) return
+ this.zoom.reset()
+ this.applyZoom(true)
+ }
+
+ // Runs inside touchend, the only place iOS lets focus() raise the keyboard.
+ // The remote cursor shape is the signal: Chromium shows an I-beam over text
+ // fields (and over selectable page text, which is the false positive), and
+ // the pointer was moved to the tap position on touchstart. Over a real
+ // network the update usually arrives after the finger lifts, so the keyboard
+ // opens here only if a text cursor showed up during this touch or was seen
+ // at this spot before; otherwise a later text cursor offers the chip.
+ keyboardAfterTap(p: Point) {
+ window.clearTimeout(this.pendingTapTimer)
+ this.pendingTap = null
+
+ if (this.tapKeyboard === 'off') return
+ if (this.tapKeyboard === 'always') {
+ this.focusForTyping()
+ return
+ }
+
+ const here = this.remotePoint(p)
+ const textDuringTouch = this.cursorKind === 'text' && this.cursorChangedAt >= this.touchBeganAt
+ if (textDuringTouch || (this.textSpot && nearSpot(here, this.textSpot))) {
+ this.offerKeyboard(p, true)
return
}
+ // Pages often flash another cursor before the I-beam, so any text cursor
+ // within the window counts; only the end of the window means "not text".
+ this.pendingTap = { p, here, at: performance.now() }
+ this.pendingTapTimer = window.setTimeout(this.endPendingTap, CURSOR_WAIT_MS)
+ }
+
+ offerKeyboard(p: Point, inGesture: boolean) {
+ if (this.keyboardOpen) return
+ if (inGesture && this.tapKeyboard === 'auto') {
+ this.focusForTyping()
+ } else {
+ this.showTypeChip(p)
+ }
+ }
+
+ // No new text cursor showed up for the tap. An I-beam that was already
+ // showing stays put when the tap lands on text again, so only a non-text
+ // cursor means the tap was not on a text field.
+ endPendingTap() {
+ const tap = this.pendingTap
+ if (!tap) return
+ this.pendingTap = null
+ if (this.cursorKind === 'text') {
+ if (this.pointer && nearSpot(this.pointer, tap.here)) this.textSpot = tap.here
+ this.offerKeyboard(tap.p, false)
+ return
+ }
+ if (this.textSpot && nearSpot(tap.here, this.textSpot)) this.textSpot = null
+ if (this.keyboardOpen) this._overlay.blur()
+ }
+
+ onCursorImage(image: CursorImage) {
+ // classification is async; only the newest cursor image may apply
+ const seq = ++this.cursorSeq
+ const at = this.cursorImageOwner()
+ const apply = (kind: CursorKind) => {
+ if (seq !== this.cursorSeq) return
+ this.cursorKind = kind
+ this.cursorChangedAt = performance.now()
+ if (kind !== 'text') return
+
+ if (at) this.textSpot = at
+ const tap = this.pendingTap
+ if (tap && performance.now() - tap.at < CURSOR_WAIT_MS) {
+ window.clearTimeout(this.pendingTapTimer)
+ this.pendingTap = null
+ this.offerKeyboard(tap.p, false)
+ }
+ }
+
+ const cached = cachedCursorKind(image)
+ if (cached) {
+ apply(cached)
+ } else {
+ classifyCursor(image).then(apply)
+ }
+ }
+
+ showTypeChip(p: Point) {
+ const player = this._player.getBoundingClientRect()
+ this.typeChip = {
+ x: Math.max(80, Math.min(player.width - 80, p.x - player.left)),
+ y: Math.max(8, Math.min(player.height - 52, p.y - player.top - 64)),
+ }
+ window.clearTimeout(this.typeChipTimer)
+ this.typeChipTimer = window.setTimeout(this.hideTypeChip, TYPE_CHIP_MS)
+ }
+
+ hideTypeChip() {
+ window.clearTimeout(this.typeChipTimer)
+ this.typeChip = null
+ }
+
+ onTypeChip() {
+ this.hideTypeChip()
+ this.focusForTyping()
+ }
+
+ focusForTyping() {
+ this.resetInputSentinel()
+ this._overlay.focus({ preventScroll: true })
+ }
+
+ // iOS overlays the soft keyboard on the layout viewport instead of resizing
+ // it, so lift the keyboard button by the part of the player it covers
+ updateKeyboardInset() {
+ const viewport = window.visualViewport
+ if (!viewport) return
+ const covered = this._player.getBoundingClientRect().bottom - (viewport.offsetTop + viewport.height)
+ this.keyboardInset = Math.max(0, Math.round(covered))
+ }
+
+ toggleMobileKeyboard() {
+ if (this.keyboardOpen) {
+ this._overlay.blur()
+ } else {
+ this.focusForTyping()
+ }
+ }
+
+ resetInputSentinel() {
+ if (!this.is_touch_device) return
+ this._overlay.value = INPUT_SENTINEL
+ this._overlay.setSelectionRange(INPUT_SENTINEL.length, INPUT_SENTINEL.length)
+ }
+
+ // Soft keyboards that report keyCode 229 (Android) deliver text only through
+ // input events, which the Guacamole keyboard does not handle.
+ onOverlayInput(e: Event) {
+ if (!this.is_touch_device) return
+ const event = e as InputEvent
+
+ if (!this.canSendInput()) {
+ if (!this.composing) this.resetInputSentinel()
+ return
+ }
+
+ if (event.inputType === 'insertCompositionText') {
+ this.updateComposition(event.data || '')
+ return
+ }
+
+ // the Guacamole keyboard already sent this key
+ const handledByKeydown = performance.now() - this.lastKeydownAt < 50
+ const echoOfComposition =
+ event.data === this.lastCompositionEnd.data && performance.now() - this.lastCompositionEnd.at < 50
+
+ if (!handledByKeydown && !echoOfComposition) {
+ switch (event.inputType) {
+ case 'insertText':
+ case 'insertReplacementText':
+ this.typeText(event.data || '')
+ break
+ case 'insertLineBreak':
+ case 'insertParagraph':
+ this.pressKey(XK_RETURN)
+ break
+ case 'deleteContentBackward':
+ this.pressKey(XK_BACKSPACE)
+ break
+ }
+ }
+
+ if (!this.composing) this.resetInputSentinel()
+ }
+
+ onCompositionStart() {
+ this.composing = true
+ this.composed = ''
+ }
+
+ onCompositionEnd(e: CompositionEvent) {
+ if (this.canSendInput()) this.updateComposition(e.data || '')
+ this.lastCompositionEnd = { data: e.data || '', at: performance.now() }
+ this.composing = false
+ this.composed = ''
+ this.resetInputSentinel()
+ }
+
+ updateComposition(text: string) {
+ const { deletes, insert } = textDiff(this.composed, text)
+ for (let i = 0; i < deletes; i++) this.pressKey(XK_BACKSPACE)
+ this.typeText(insert)
+ this.composed = text
+ }
+
+ typeText(text: string) {
+ for (const ch of Array.from(text)) {
+ const shift = needsShift(ch) && !this.keyboard.modifiers.shift
+ if (shift) this.$client.sendData('keydown', { key: XK_SHIFT_L })
+ this.pressKey(charToKeysym(ch))
+ if (shift) this.$client.sendData('keyup', { key: XK_SHIFT_L })
+ }
+ }
+
+ pressKey(key: number) {
+ this.$client.sendData('keydown', { key })
+ this.$client.sendData('keyup', { key })
+ }
+
+ focusOverlay() {
const focus = () => {
if (this.hosting && !this.locked) {
this._overlay.focus()
@@ -859,7 +1423,7 @@
return
}
- this.focusOverlay(e)
+ this.focusOverlay()
this.sendMousePos(e)
this.pressedMouseButtons.add(e.button + 1)
@@ -871,7 +1435,7 @@
return
}
- this.focusOverlay(e)
+ this.focusOverlay()
this.sendMousePos(e)
this.pressedMouseButtons.delete(e.button + 1)
this.$client.sendData('mouseup', { key: e.button + 1 })
@@ -978,22 +1542,81 @@
}
onOverlayFocus() {
+ this.keyboardOpen = true
if (this.hosting) {
this.syncClipboard()
}
}
+ onOverlayBlur() {
+ this.keyboardOpen = false
+ this.resetKeyboard()
+ }
+
onResize() {
const { offsetWidth, offsetHeight } = !this.fullscreen ? this._component : document.body
+ if (`${offsetWidth}px` !== this._player.style.width || `${offsetHeight}px` !== this._player.style.height) {
+ this.resetZoom()
+ }
this._player.style.width = `${offsetWidth}px`
this._player.style.height = `${offsetHeight}px`
- const aspectPreservingMaxWidth = (this.horizontal / this.vertical) * offsetHeight
- this._container.style.maxWidth = `${
- !this.fullscreen ? Math.min(this.width, aspectPreservingMaxWidth) : aspectPreservingMaxWidth
- }px`
+ this.playerWidth = offsetWidth
+ this.playerHeight = offsetHeight
+
+ const aspect = this.horizontal / this.vertical
+ const maxWidth = (height: number) => (!this.fullscreen ? Math.min(this.width, aspect * height) : aspect * height)
+
+ // Reserve a band outside the video for the touch controls, shrinking the
+ // video slightly if needed; with no room they overlay the stream instead.
+ const videoWidth = Math.min(offsetWidth, maxWidth(offsetHeight))
+ this.safeArea = this.safeAreaInsets()
+ this.touchLayout = this.showTouchControls
+ ? controlLayout(
+ offsetWidth,
+ offsetHeight,
+ videoWidth,
+ videoWidth / aspect,
+ this.safeArea,
+ this.controlPosition.side,
+ )
+ : { mode: 'overlay', band: 0 }
+ const { mode, band } = this.touchLayout
+ this._player.style.paddingBottom = mode === 'bottom' ? `${band}px` : ''
+ this._player.style.paddingLeft = mode === 'left' ? `${band}px` : ''
+ this._player.style.paddingRight = mode === 'right' ? `${band}px` : ''
+
+ this._container.style.maxWidth = `${maxWidth(offsetHeight - (mode === 'bottom' ? band : 0))}px`
this._aspect.style.paddingBottom = `${(this.vertical / this.horizontal) * 100}%`
}
+ safeAreaInsets(): SafeArea {
+ if (!this.safeAreaProbe) {
+ this.safeAreaProbe = document.createElement('div')
+ this.safeAreaProbe.style.cssText =
+ 'position:fixed;visibility:hidden;pointer-events:none;' +
+ 'padding:env(safe-area-inset-top) env(safe-area-inset-right) env(safe-area-inset-bottom) env(safe-area-inset-left)'
+ document.body.appendChild(this.safeAreaProbe)
+ }
+ const style = getComputedStyle(this.safeAreaProbe)
+ return {
+ top: parseFloat(style.paddingTop) || 0,
+ bottom: parseFloat(style.paddingBottom) || 0,
+ left: parseFloat(style.paddingLeft) || 0,
+ right: parseFloat(style.paddingRight) || 0,
+ }
+ }
+
+ onControlMove(position: ControlPosition) {
+ this.controlPosition = position
+ set(CONTROL_POSITION_KEY, encodePosition(position))
+ this.onResize()
+ }
+
+ @Watch('showTouchControls')
+ onShowTouchControls() {
+ this.$nextTick(this.onResize)
+ }
+
@Watch('focused')
@Watch('hosting')
@Watch('locked')
@@ -1008,10 +1631,5 @@
this._overlay.focus()
}
}
-
- openMobileKeyboard() {
- // focus opens the keyboard on mobile
- this._overlay.focus()
- }
}
diff --git a/images/chromium-headful/client/src/neko/base.ts b/images/chromium-headful/client/src/neko/base.ts
index 5bacc96fe..80c68a7f7 100644
--- a/images/chromium-headful/client/src/neko/base.ts
+++ b/images/chromium-headful/client/src/neko/base.ts
@@ -1,6 +1,7 @@
import EventEmitter from 'eventemitter3'
import { OPCODE } from './data'
import { EVENT, WebSocketEvents } from './events'
+import { CursorImage } from '~/utils/cursor-shape'
import {
WebSocketMessages,
@@ -16,6 +17,7 @@ export interface BaseEvents {
warn: (...message: any[]) => void
debug: (...message: any[]) => void
error: (error: Error) => void
+ cursor: (image: CursorImage) => void
}
export abstract class BaseClient extends EventEmitter {
diff --git a/images/chromium-headful/client/src/neko/index.ts b/images/chromium-headful/client/src/neko/index.ts
index 3571bb88e..5a68ec105 100644
--- a/images/chromium-headful/client/src/neko/index.ts
+++ b/images/chromium-headful/client/src/neko/index.ts
@@ -4,6 +4,7 @@ import { BaseClient, BaseEvents } from './base'
import { Member } from './types'
import { EVENT } from './events'
import { accessor } from '~/store'
+import { parseCursorImage } from '~/utils/cursor-shape'
import {
SystemMessagePayload,
@@ -131,7 +132,17 @@ export class NekoClient extends BaseClient implements EventEmitter {
this.$accessor.video.setStream(0)
}
- protected [EVENT.DATA]() {}
+ protected [EVENT.DATA](data: ArrayBuffer | Blob) {
+ if (data instanceof Blob) {
+ data.arrayBuffer().then((buffer) => this[EVENT.DATA](buffer))
+ return
+ }
+
+ const image = parseCursorImage(data)
+ if (image) {
+ this.emit('cursor', image)
+ }
+ }
/////////////////////////////
// System Events
diff --git a/images/chromium-headful/client/src/utils/cursor-shape.ts b/images/chromium-headful/client/src/utils/cursor-shape.ts
new file mode 100644
index 000000000..ca0666d6d
--- /dev/null
+++ b/images/chromium-headful/client/src/utils/cursor-shape.ts
@@ -0,0 +1,100 @@
+// Classifies the remote X cursor image that neko pushes over the data channel.
+// Chromium switches to the text (I-beam) cursor over editable fields, which is
+// the only signal the client has that a tap landed on something typeable.
+
+export type CursorKind = 'text' | 'other' | 'unknown'
+
+export interface CursorImage {
+ width: number
+ height: number
+ xhot: number
+ yhot: number
+ png: Uint8Array
+}
+
+// data channel opcode for cursor images (neko server/internal/webrtc/payload/send.go)
+const OP_CURSOR_IMAGE = 0x02
+const HEADER_SIZE = 3
+const CURSOR_HEADER_SIZE = 8
+
+export function parseCursorImage(buffer: ArrayBuffer): CursorImage | undefined {
+ if (buffer.byteLength < HEADER_SIZE + CURSOR_HEADER_SIZE) return
+ const view = new DataView(buffer)
+ if (view.getUint8(0) !== OP_CURSOR_IMAGE) return
+ return {
+ width: view.getUint16(3),
+ height: view.getUint16(5),
+ xhot: view.getUint16(7),
+ yhot: view.getUint16(9),
+ png: new Uint8Array(buffer, HEADER_SIZE + CURSOR_HEADER_SIZE),
+ }
+}
+
+const cache = new Map()
+
+function cacheKey(img: CursorImage) {
+ // FNV-1a over the image bytes; cursor images are small
+ let hash = 0x811c9dc5
+ for (let i = 0; i < img.png.length; i++) {
+ hash ^= img.png[i]
+ hash = Math.imul(hash, 0x01000193)
+ }
+ return `${img.width}x${img.height}@${img.xhot},${img.yhot}:${img.png.length}:${hash >>> 0}`
+}
+
+export function cachedCursorKind(img: CursorImage): CursorKind | undefined {
+ return cache.get(cacheKey(img))
+}
+
+export async function classifyCursor(img: CursorImage): Promise {
+ const key = cacheKey(img)
+ const cached = cache.get(key)
+ if (cached) return cached
+
+ let kind: CursorKind = 'unknown'
+ try {
+ const bitmap = await createImageBitmap(new Blob([img.png], { type: 'image/png' }))
+ const canvas = document.createElement('canvas')
+ canvas.width = bitmap.width
+ canvas.height = bitmap.height
+ const ctx = canvas.getContext('2d')
+ if (ctx) {
+ ctx.drawImage(bitmap, 0, 0)
+ kind = classifyPixels(ctx.getImageData(0, 0, bitmap.width, bitmap.height), img.xhot, img.yhot)
+ }
+ bitmap.close()
+ } catch (e) {
+ kind = 'unknown'
+ }
+
+ cache.set(key, kind)
+ return kind
+}
+
+function classifyPixels(data: ImageData, xhot: number, yhot: number): CursorKind {
+ let minX = Infinity
+ let minY = Infinity
+ let maxX = -Infinity
+ let maxY = -Infinity
+ for (let y = 0; y < data.height; y++) {
+ for (let x = 0; x < data.width; x++) {
+ if (data.data[(y * data.width + x) * 4 + 3] > 64) {
+ minX = Math.min(minX, x)
+ maxX = Math.max(maxX, x)
+ minY = Math.min(minY, y)
+ maxY = Math.max(maxY, y)
+ }
+ }
+ }
+ // an empty image means the cursor is hidden
+ if (maxX < 0) return 'unknown'
+
+ const width = maxX - minX + 1
+ const height = maxY - minY + 1
+ // I-beam: tall and narrow, hotspot in the middle of the shape. Arrows and
+ // hands have their hotspot at the tip, near the top edge.
+ const narrow = height >= 8 && width / height <= 0.6
+ const centeredX = Math.abs(xhot - (minX + maxX) / 2) <= Math.max(2, width * 0.35)
+ const middleY = yhot > minY + height * 0.2 && yhot < maxY - height * 0.2
+ return narrow && centeredX && middleY ? 'text' : 'other'
+}
diff --git a/images/chromium-headful/client/src/utils/text-input.ts b/images/chromium-headful/client/src/utils/text-input.ts
new file mode 100644
index 000000000..a9c6f6220
--- /dev/null
+++ b/images/chromium-headful/client/src/utils/text-input.ts
@@ -0,0 +1,38 @@
+// Helpers for turning text from mobile soft keyboards into X keysyms.
+// Android keyboards report keyCode 229 for most keys, which the Guacamole
+// keyboard ignores, so their text only arrives through input events.
+
+export const XK_SHIFT_L = 0xffe1
+export const XK_BACKSPACE = 0xff08
+export const XK_RETURN = 0xff0d
+
+export function charToKeysym(ch: string): number {
+ if (ch === '\n' || ch === '\r') return XK_RETURN
+ const cp = ch.codePointAt(0) as number
+ // Latin-1 keysyms equal their code points; everything else uses the Unicode range
+ if ((cp >= 0x20 && cp <= 0x7e) || (cp >= 0xa0 && cp <= 0xff)) return cp
+ return 0x01000000 | cp
+}
+
+// Neko maps a keysym to a keycode using the current modifier state. An upper
+// case letter sent without Shift lands on the lower case key, so soft keyboards
+// that never report Shift type the wrong case.
+export function needsShift(ch: string) {
+ return ch !== ch.toLowerCase() && ch === ch.toUpperCase()
+}
+
+export function keysymNeedsShift(keysym: number) {
+ if (keysym >= 0x41 && keysym <= 0x5a) return true
+ if (keysym >= 0xc0 && keysym <= 0xde && keysym !== 0xd7) return true
+ if ((keysym & 0xff000000) === 0x01000000) return needsShift(String.fromCodePoint(keysym & 0x00ffffff))
+ return false
+}
+
+// Minimal edit that turns prev into next: delete from the end, then insert.
+export function textDiff(prev: string, next: string) {
+ const a = Array.from(prev)
+ const b = Array.from(next)
+ let common = 0
+ while (common < a.length && common < b.length && a[common] === b[common]) common++
+ return { deletes: a.length - common, insert: b.slice(common).join('') }
+}
diff --git a/images/chromium-headful/client/src/utils/touch-controls.ts b/images/chromium-headful/client/src/utils/touch-controls.ts
new file mode 100644
index 000000000..d85635c95
--- /dev/null
+++ b/images/chromium-headful/client/src/utils/touch-controls.ts
@@ -0,0 +1,150 @@
+// Layout math for the touch controls. The control prefers a band outside the
+// video (shrinking the video a little to make one if needed) and only
+// overlays the stream when there is no room.
+
+// a 44px target plus a 4px margin on each side
+export const BAND_SIZE = 52
+// how much the video may shrink to make room for a band
+export const MAX_VIDEO_SHRINK = 0.1
+export const EDGE_MARGIN = 8
+
+export type ControlSide = 'left' | 'right'
+export type ControlMode = 'bottom' | ControlSide | 'overlay'
+
+export interface ControlLayout {
+ mode: ControlMode
+ // reserved band thickness in px, including the safe-area inset; 0 for overlay
+ band: number
+}
+
+export interface ControlPosition {
+ side: ControlSide
+ // 0..1 along the bottom band
+ x: number
+ // 0..1 along a side strip or screen edge
+ y: number
+}
+
+export interface Insets {
+ bottom: number
+ left: number
+ right: number
+}
+
+export const DEFAULT_POSITION: ControlPosition = { side: 'left', x: 0, y: 0.5 }
+
+// videoWidth/videoHeight: the size the video gets in the whole area, before any band is reserved
+export function controlLayout(
+ areaWidth: number,
+ areaHeight: number,
+ videoWidth: number,
+ videoHeight: number,
+ insets: Insets,
+ side: ControlSide,
+): ControlLayout {
+ const bottom = BAND_SIZE + insets.bottom
+ const strip = BAND_SIZE + insets[side]
+
+ if (areaHeight - videoHeight >= bottom) return { mode: 'bottom', band: bottom }
+ if (areaWidth - videoWidth >= strip) return { mode: side, band: strip }
+
+ // scale the video would need to fit next to each band
+ const fit = (width: number, height: number) => Math.min(1, width / videoWidth, height / videoHeight)
+ const bottomScale = fit(areaWidth, areaHeight - bottom)
+ const stripScale = fit(areaWidth - strip, areaHeight)
+ const best = Math.max(bottomScale, stripScale)
+ if (best < 1 - MAX_VIDEO_SHRINK) return { mode: 'overlay', band: 0 }
+ return bottomScale >= stripScale ? { mode: 'bottom', band: bottom } : { mode: side, band: strip }
+}
+
+export interface Size {
+ width: number
+ height: number
+}
+
+export interface SafeArea extends Insets {
+ top: number
+}
+
+export interface ControlFrame {
+ area: Size
+ control: Size
+ safe: SafeArea
+ // height of the soft keyboard covering the bottom of the area
+ keyboardInset: number
+}
+
+// gap between the control and the edge it is docked to
+const DOCK_GAP = { band: 4, overlay: 6 }
+
+interface Track {
+ start: number
+ length: number
+}
+
+// the range the control's leading edge can move along
+function track(mode: ControlMode, frame: ControlFrame): Track {
+ const { area, control, safe, keyboardInset } = frame
+ if (mode === 'bottom') {
+ const start = safe.left + EDGE_MARGIN
+ return { start, length: Math.max(0, area.width - safe.right - EDGE_MARGIN - control.width - start) }
+ }
+ const start = safe.top + EDGE_MARGIN
+ const end = area.height - keyboardInset - safe.bottom - EDGE_MARGIN - control.height
+ return { start, length: Math.max(0, end - start) }
+}
+
+// top-left corner of the control in area coordinates
+export function placeControl(mode: ControlMode, position: ControlPosition, frame: ControlFrame) {
+ const { area, control, safe, keyboardInset } = frame
+ const t = track(mode, frame)
+ if (mode === 'bottom') {
+ return {
+ x: t.start + position.x * t.length,
+ y: area.height - keyboardInset - safe.bottom - DOCK_GAP.band - control.height,
+ }
+ }
+ const side = mode === 'overlay' ? position.side : mode
+ const gap = mode === 'overlay' ? DOCK_GAP.overlay : DOCK_GAP.band
+ return {
+ x: side === 'left' ? safe.left + gap : area.width - safe.right - gap - control.width,
+ y: t.start + position.y * t.length,
+ }
+}
+
+function fraction(offset: number, t: Track) {
+ return t.length > 0 ? Math.min(1, Math.max(0, (offset - t.start) / t.length)) : 0.5
+}
+
+// Where a control dropped with its top-left corner at (x, y) ends up. Overlay
+// and side strips snap to the nearest side; the bottom band only moves along
+// its length.
+export function dropControl(
+ mode: ControlMode,
+ position: ControlPosition,
+ x: number,
+ y: number,
+ frame: ControlFrame,
+): ControlPosition {
+ const t = track(mode, frame)
+ if (mode === 'bottom') return { ...position, x: fraction(x, t) }
+ const center = x + frame.control.width / 2
+ return { ...position, side: center < frame.area.width / 2 ? 'left' : 'right', y: fraction(y, t) }
+}
+
+export function encodePosition(position: ControlPosition) {
+ return `${position.side}:${Math.round(position.x * 1000)}:${Math.round(position.y * 1000)}`
+}
+
+export function decodePosition(value: string): ControlPosition {
+ const [side, x, y] = value.split(':')
+ const unit = (v: string, def: number) => {
+ const n = parseInt(v, 10) / 1000
+ return isNaN(n) ? def : Math.min(1, Math.max(0, n))
+ }
+ return {
+ side: side === 'right' ? 'right' : 'left',
+ x: unit(x, DEFAULT_POSITION.x),
+ y: unit(y, DEFAULT_POSITION.y),
+ }
+}
diff --git a/images/chromium-headful/client/src/utils/touch-gestures.ts b/images/chromium-headful/client/src/utils/touch-gestures.ts
new file mode 100644
index 000000000..2656dfbdd
--- /dev/null
+++ b/images/chromium-headful/client/src/utils/touch-gestures.ts
@@ -0,0 +1,223 @@
+// Turns raw touch events on the live view overlay into semantic gestures:
+// tap, long-press (release = right click, move = drag), one-finger scroll with
+// momentum, and two-finger pinch/pan. Coordinates are client (CSS) pixels.
+
+export interface Point {
+ x: number
+ y: number
+}
+
+export interface GestureHandlers {
+ // first finger down; called before any other handler for the sequence
+ onTouchBegin(p: Point): void
+ // called synchronously from touchend, so it runs inside the user gesture
+ onTap(p: Point): void
+ onLongPress(p: Point): void
+ onLongPressRelease(p: Point): void
+ onDragStart(p: Point): void
+ onDragMove(p: Point): void
+ onDragEnd(p: Point): void
+ // finger delta since the previous call; positive means the finger moved right/down
+ onScroll(dx: number, dy: number, anchor: Point): void
+ onPinchStart(mid: Point, distance: number): void
+ onPinchMove(mid: Point, distance: number): void
+ onPinchEnd(): void
+}
+
+const TAP_SLOP = 10
+const LONG_PRESS_MS = 500
+const VELOCITY_WINDOW_MS = 100
+// a finger that rested this long before lifting does not fling
+const FLING_REST_MS = 60
+// per-millisecond velocity decay, close to iOS UIScrollView's "normal" rate
+const FLING_DECAY = 0.9975
+const FLING_MIN_START = 0.15
+const FLING_MIN_STOP = 0.02
+
+type Mode = 'idle' | 'pending' | 'scroll' | 'held' | 'drag' | 'pinch'
+
+interface Sample extends Point {
+ t: number
+}
+
+function point(t: Touch): Point {
+ return { x: t.clientX, y: t.clientY }
+}
+
+function distance(a: Point, b: Point) {
+ return Math.hypot(a.x - b.x, a.y - b.y)
+}
+
+function midpoint(a: Point, b: Point): Point {
+ return { x: (a.x + b.x) / 2, y: (a.y + b.y) / 2 }
+}
+
+export class TouchGestures {
+ private mode: Mode = 'idle'
+ private start: Point = { x: 0, y: 0 }
+ private last: Point = { x: 0, y: 0 }
+ private samples: Sample[] = []
+ private longPressTimer?: ReturnType
+ private flingFrame?: number
+ private interruptedFling = false
+
+ constructor(private handlers: GestureHandlers) {}
+
+ get flinging() {
+ return this.flingFrame !== undefined
+ }
+
+ touchStart(e: TouchEvent) {
+ if (e.touches.length === 1 && this.mode === 'idle') {
+ this.interruptedFling = this.stopFling()
+ const p = point(e.touches[0])
+ this.start = p
+ this.last = p
+ this.samples = [{ ...p, t: e.timeStamp }]
+ this.mode = 'pending'
+ this.handlers.onTouchBegin(p)
+ this.longPressTimer = setTimeout(() => {
+ this.longPressTimer = undefined
+ if (this.mode === 'pending') {
+ this.mode = 'held'
+ this.handlers.onLongPress(this.start)
+ }
+ }, LONG_PRESS_MS)
+ return
+ }
+
+ if (e.touches.length >= 2 && this.mode !== 'pinch') {
+ this.clearLongPress()
+ this.stopFling()
+ if (this.mode === 'drag') this.handlers.onDragEnd(this.last)
+ this.mode = 'pinch'
+ const a = point(e.touches[0])
+ const b = point(e.touches[1])
+ this.handlers.onPinchStart(midpoint(a, b), distance(a, b))
+ }
+ }
+
+ touchMove(e: TouchEvent) {
+ if (this.mode === 'pinch') {
+ if (e.touches.length >= 2) {
+ const a = point(e.touches[0])
+ const b = point(e.touches[1])
+ this.handlers.onPinchMove(midpoint(a, b), distance(a, b))
+ }
+ return
+ }
+
+ if (e.touches.length !== 1) return
+ const p = point(e.touches[0])
+
+ switch (this.mode) {
+ case 'pending':
+ if (distance(p, this.start) <= TAP_SLOP) return
+ this.clearLongPress()
+ this.mode = 'scroll'
+ this.handlers.onScroll(p.x - this.last.x, p.y - this.last.y, this.start)
+ break
+ case 'scroll':
+ this.handlers.onScroll(p.x - this.last.x, p.y - this.last.y, this.start)
+ break
+ case 'held':
+ if (distance(p, this.start) <= TAP_SLOP) return
+ this.mode = 'drag'
+ this.handlers.onDragStart(this.start)
+ this.handlers.onDragMove(p)
+ break
+ case 'drag':
+ this.handlers.onDragMove(p)
+ break
+ }
+
+ this.last = p
+ this.samples.push({ ...p, t: e.timeStamp })
+ while (this.samples.length > 2 && e.timeStamp - this.samples[0].t > VELOCITY_WINDOW_MS) {
+ this.samples.shift()
+ }
+ }
+
+ touchEnd(e: TouchEvent) {
+ // wait for every finger to lift; a leftover finger after a pinch must not scroll
+ if (e.touches.length > 0) return
+ this.clearLongPress()
+
+ const mode = this.mode
+ this.mode = 'idle'
+
+ switch (mode) {
+ case 'pending':
+ if (!this.interruptedFling) this.handlers.onTap(this.last)
+ break
+ case 'held':
+ this.handlers.onLongPressRelease(this.last)
+ break
+ case 'drag':
+ this.handlers.onDragEnd(this.last)
+ break
+ case 'scroll':
+ this.startFling(e.timeStamp)
+ break
+ case 'pinch':
+ this.handlers.onPinchEnd()
+ break
+ }
+ }
+
+ touchCancel() {
+ this.clearLongPress()
+ if (this.mode === 'drag') this.handlers.onDragEnd(this.last)
+ if (this.mode === 'pinch') this.handlers.onPinchEnd()
+ this.mode = 'idle'
+ }
+
+ // returns true if a fling was running
+ stopFling() {
+ if (this.flingFrame === undefined) return false
+ cancelAnimationFrame(this.flingFrame)
+ this.flingFrame = undefined
+ return true
+ }
+
+ destroy() {
+ this.clearLongPress()
+ this.stopFling()
+ }
+
+ private clearLongPress() {
+ if (this.longPressTimer !== undefined) {
+ clearTimeout(this.longPressTimer)
+ this.longPressTimer = undefined
+ }
+ }
+
+ private startFling(endTime: number) {
+ const first = this.samples[0]
+ const last = this.samples[this.samples.length - 1]
+ if (!first || !last || endTime - last.t > FLING_REST_MS) return
+ const dt = last.t - first.t
+ if (dt <= 0) return
+
+ let vx = (last.x - first.x) / dt
+ let vy = (last.y - first.y) / dt
+ if (Math.hypot(vx, vy) < FLING_MIN_START) return
+
+ const anchor = this.start
+ let prev = performance.now()
+ const step = (now: number) => {
+ const elapsed = Math.min(now - prev, 50)
+ prev = now
+ const decay = Math.pow(FLING_DECAY, elapsed)
+ vx *= decay
+ vy *= decay
+ this.handlers.onScroll(vx * elapsed, vy * elapsed, anchor)
+ if (Math.hypot(vx, vy) < FLING_MIN_STOP) {
+ this.flingFrame = undefined
+ return
+ }
+ this.flingFrame = requestAnimationFrame(step)
+ }
+ this.flingFrame = requestAnimationFrame(step)
+ }
+}
diff --git a/images/chromium-headful/client/src/utils/zoom-pan.ts b/images/chromium-headful/client/src/utils/zoom-pan.ts
new file mode 100644
index 000000000..2e63bb053
--- /dev/null
+++ b/images/chromium-headful/client/src/utils/zoom-pan.ts
@@ -0,0 +1,67 @@
+// Local zoom and pan of the live view video. The remote screen is unchanged;
+// this only scales what the viewer sees so a large stream is usable on a phone.
+
+import { Point } from './touch-gestures'
+
+export interface Box {
+ left: number
+ top: number
+ width: number
+ height: number
+}
+
+export class ZoomPan {
+ scale = 1
+ tx = 0
+ ty = 0
+
+ private startScale = 1
+ private startDistance = 1
+ private anchor: Point = { x: 0, y: 0 }
+
+ // element: the video box before the transform. viewport: the visible area the
+ // zoomed video may fill, which includes any letterbox bars around it.
+ constructor(private element: () => Box, private viewport: () => Box, private maxScale: () => number) {}
+
+ get zoomed() {
+ return this.scale > 1.01
+ }
+
+ pinchStart(mid: Point, distance: number) {
+ const e = this.element()
+ this.startScale = this.scale
+ this.startDistance = Math.max(distance, 1)
+ // content point (in unscaled element px) under the fingers
+ this.anchor = {
+ x: (mid.x - e.left - this.tx) / this.scale,
+ y: (mid.y - e.top - this.ty) / this.scale,
+ }
+ }
+
+ pinchMove(mid: Point, distance: number) {
+ const e = this.element()
+ const scale = Math.min(this.maxScale(), Math.max(1, (this.startScale * distance) / this.startDistance))
+ this.scale = scale
+ this.tx = clampAxis(mid.x - e.left - this.anchor.x * scale, e.left, e.width * scale, this.viewport(), 'x')
+ this.ty = clampAxis(mid.y - e.top - this.anchor.y * scale, e.top, e.height * scale, this.viewport(), 'y')
+ }
+
+ reset() {
+ this.scale = 1
+ this.tx = 0
+ this.ty = 0
+ }
+
+ get transform() {
+ if (!this.zoomed) return ''
+ return `translate(${this.tx}px, ${this.ty}px) scale(${this.scale})`
+ }
+}
+
+// Content larger than the viewport must cover it; smaller content stays centered.
+function clampAxis(offset: number, origin: number, size: number, viewport: Box, axis: 'x' | 'y') {
+ const start = axis === 'x' ? viewport.left : viewport.top
+ const length = axis === 'x' ? viewport.width : viewport.height
+ if (size <= length) return start + (length - size) / 2 - origin
+ return Math.min(start - origin, Math.max(start + length - size - origin, offset))
+}
diff --git a/images/chromium-headful/client/tests/touch-controls.test.ts b/images/chromium-headful/client/tests/touch-controls.test.ts
new file mode 100644
index 000000000..7c8801cec
--- /dev/null
+++ b/images/chromium-headful/client/tests/touch-controls.test.ts
@@ -0,0 +1,82 @@
+import { describe, expect, test } from 'bun:test'
+import {
+ BAND_SIZE,
+ ControlFrame,
+ DEFAULT_POSITION,
+ controlLayout,
+ decodePosition,
+ dropControl,
+ encodePosition,
+ placeControl,
+} from '../src/utils/touch-controls'
+
+const noInsets = { bottom: 0, left: 0, right: 0 }
+
+describe('control layout', () => {
+ test('uses the bottom band when the video leaves vertical room', () => {
+ // a 16:9 stream on a portrait phone
+ expect(controlLayout(390, 664, 390, 219, noInsets, 'left')).toEqual({ mode: 'bottom', band: BAND_SIZE })
+ })
+
+ test('uses a side strip on the preferred side when only horizontal room is left', () => {
+ // a portrait stream pillarboxed on a shorter screen
+ expect(controlLayout(390, 664, 307, 664, noInsets, 'right')).toEqual({ mode: 'right', band: BAND_SIZE })
+ })
+
+ test('includes the safe-area inset in the band', () => {
+ const insets = { bottom: 34, left: 0, right: 0 }
+ expect(controlLayout(390, 664, 390, 219, insets, 'left')).toEqual({ mode: 'bottom', band: BAND_SIZE + 34 })
+ // 52 + 34 does not fit in a 60px band, so the strip or shrinking decides instead
+ expect(controlLayout(390, 844, 390, 784, insets, 'left').mode).not.toBe('overlay')
+ })
+
+ test('shrinks the video slightly rather than overlaying it', () => {
+ // a stream that exactly fills its frame: shrinking by 52px is about 6%
+ expect(controlLayout(390, 844, 390, 844, noInsets, 'left')).toEqual({ mode: 'bottom', band: BAND_SIZE })
+ })
+
+ test('overlays the stream when making room would shrink it too much', () => {
+ expect(controlLayout(300, 200, 300, 200, noInsets, 'left')).toEqual({ mode: 'overlay', band: 0 })
+ })
+})
+
+const frame = (overrides: Partial = {}): ControlFrame => ({
+ area: { width: 390, height: 844 },
+ control: { width: 44, height: 44 },
+ safe: { top: 0, bottom: 0, left: 0, right: 0 },
+ keyboardInset: 0,
+ ...overrides,
+})
+
+describe('placing and dragging the control', () => {
+ test('a dropped control snaps to the nearest side edge and keeps its height', () => {
+ const dropped = dropControl('overlay', DEFAULT_POSITION, 330, 380, frame())
+ expect(dropped.side).toBe('right')
+ expect(placeControl('overlay', dropped, frame())).toEqual({ x: 390 - 6 - 44, y: 380 })
+ expect(dropControl('left', dropped, 20, -50, frame())).toEqual({ ...dropped, side: 'left', y: 0 })
+ })
+
+ test('drops a wide control where it was released, not where a 44px one would be', () => {
+ const wide = frame({ control: { width: 120, height: 44 } })
+ const dropped = dropControl('bottom', DEFAULT_POSITION, 150, 790, wide)
+ expect(placeControl('bottom', dropped, wide).x).toBeCloseTo(150)
+ // the bottom band only moves sideways
+ expect(placeControl('bottom', dropped, wide).y).toBe(844 - 4 - 44)
+ })
+
+ test('keeps the control out of the safe areas and above the soft keyboard', () => {
+ const notched = frame({ safe: { top: 47, bottom: 34, left: 0, right: 0 }, keyboardInset: 300 })
+ const top = placeControl('overlay', { ...DEFAULT_POSITION, y: 0 }, notched)
+ const bottom = placeControl('overlay', { ...DEFAULT_POSITION, y: 1 }, notched)
+ expect(top.y).toBe(47 + 8)
+ expect(bottom.y + 44).toBe(844 - 300 - 34 - 8)
+ expect(placeControl('bottom', DEFAULT_POSITION, notched).y).toBe(844 - 300 - 34 - 4 - 44)
+ })
+
+ test('round-trips the stored position and falls back to the default', () => {
+ const position = { side: 'right' as const, x: 0.25, y: 0.75 }
+ expect(decodePosition(encodePosition(position))).toEqual(position)
+ expect(decodePosition('')).toEqual(DEFAULT_POSITION)
+ expect(decodePosition('up:abc:2000')).toEqual({ side: 'left', x: DEFAULT_POSITION.x, y: 1 })
+ })
+})
diff --git a/images/chromium-headful/client/tests/touch-input.test.ts b/images/chromium-headful/client/tests/touch-input.test.ts
new file mode 100644
index 000000000..744e9be7f
--- /dev/null
+++ b/images/chromium-headful/client/tests/touch-input.test.ts
@@ -0,0 +1,131 @@
+import { describe, expect, test } from 'bun:test'
+import { charToKeysym, keysymNeedsShift, needsShift, textDiff, XK_RETURN } from '../src/utils/text-input'
+import { ZoomPan } from '../src/utils/zoom-pan'
+import { GestureHandlers, Point, TouchGestures } from '../src/utils/touch-gestures'
+
+describe('text input', () => {
+ test('maps characters to keysyms', () => {
+ expect(charToKeysym('a')).toBe(0x61)
+ expect(charToKeysym('é')).toBe(0xe9)
+ expect(charToKeysym('€')).toBe(0x010020ac)
+ expect(charToKeysym('\n')).toBe(XK_RETURN)
+ })
+
+ test('detects letters that need shift', () => {
+ expect(needsShift('A')).toBe(true)
+ expect(needsShift('a')).toBe(false)
+ expect(needsShift('!')).toBe(false)
+ expect(keysymNeedsShift(0x41)).toBe(true)
+ expect(keysymNeedsShift(0x61)).toBe(false)
+ expect(keysymNeedsShift(0xc9)).toBe(true)
+ expect(keysymNeedsShift(0x01000416)).toBe(true)
+ })
+
+ test('diffs composition updates', () => {
+ expect(textDiff('', 'te')).toEqual({ deletes: 0, insert: 'te' })
+ expect(textDiff('tez', 'tes')).toEqual({ deletes: 1, insert: 's' })
+ expect(textDiff('hello', 'help')).toEqual({ deletes: 2, insert: 'p' })
+ })
+})
+
+describe('zoom and pan', () => {
+ const element = { left: 0, top: 200, width: 400, height: 225 }
+ const viewport = { left: 0, top: 0, width: 400, height: 625 }
+
+ test('keeps the content point under the fingers while scaling', () => {
+ const zoom = new ZoomPan(
+ () => element,
+ () => viewport,
+ () => 8,
+ )
+ zoom.pinchStart({ x: 200, y: 312 }, 100)
+ zoom.pinchMove({ x: 200, y: 312 }, 300)
+ expect(zoom.scale).toBe(3)
+ // content point (200, 112) in element px stays at client (200, 312)
+ expect(element.left + zoom.tx + 200 * zoom.scale).toBeCloseTo(200)
+ expect(element.top + zoom.ty + 112 * zoom.scale).toBeCloseTo(312)
+ })
+
+ test('clamps scale and keeps the viewport covered', () => {
+ const zoom = new ZoomPan(
+ () => element,
+ () => viewport,
+ () => 4,
+ )
+ zoom.pinchStart({ x: 10, y: 210 }, 100)
+ zoom.pinchMove({ x: 390, y: 600 }, 1000)
+ expect(zoom.scale).toBe(4)
+ expect(element.left + zoom.tx).toBeLessThanOrEqual(viewport.left)
+ expect(element.left + zoom.tx + element.width * 4).toBeGreaterThanOrEqual(viewport.left + viewport.width)
+ expect(element.top + zoom.ty).toBeLessThanOrEqual(viewport.top)
+ expect(element.top + zoom.ty + element.height * 4).toBeGreaterThanOrEqual(viewport.top + viewport.height)
+ })
+
+ test('never zooms out below the fitted size', () => {
+ const zoom = new ZoomPan(
+ () => element,
+ () => viewport,
+ () => 4,
+ )
+ zoom.pinchStart({ x: 200, y: 312 }, 300)
+ zoom.pinchMove({ x: 200, y: 312 }, 100)
+ expect(zoom.scale).toBe(1)
+ expect(zoom.transform).toBe('')
+ })
+})
+
+describe('touch gestures', () => {
+ const touches = (...points: Point[]) => points.map((p) => ({ clientX: p.x, clientY: p.y })) as unknown as TouchList
+ const event = (timeStamp: number, ...points: Point[]) =>
+ ({ timeStamp, touches: touches(...points) } as unknown as TouchEvent)
+
+ const recorder = () => {
+ const calls: string[] = []
+ const handlers = new Proxy({} as GestureHandlers, {
+ get:
+ (_, name: string) =>
+ (...args: unknown[]) =>
+ calls.push(`${name} ${JSON.stringify(args)}`),
+ })
+ return { calls, handlers }
+ }
+
+ test('a short touch without movement is a tap', () => {
+ const { calls, handlers } = recorder()
+ const g = new TouchGestures(handlers)
+ g.touchStart(event(0, { x: 10, y: 10 }))
+ g.touchMove(event(20, { x: 12, y: 11 }))
+ g.touchEnd(event(60))
+ g.destroy()
+ expect(calls.map((c) => c.split(' ')[0])).toEqual(['onTouchBegin', 'onTap'])
+ })
+
+ test('movement past the slop scrolls by the finger delta', () => {
+ const { calls, handlers } = recorder()
+ const g = new TouchGestures(handlers)
+ g.touchStart(event(0, { x: 100, y: 300 }))
+ g.touchMove(event(16, { x: 100, y: 280 }))
+ g.touchMove(event(32, { x: 100, y: 250 }))
+ g.touchEnd(event(200))
+ g.destroy()
+ const scrolls = calls.filter((c) => c.startsWith('onScroll'))
+ expect(scrolls.length).toBe(2)
+ expect(scrolls[0]).toContain('[0,-20,')
+ expect(scrolls[1]).toContain('[0,-30,')
+ expect(calls.some((c) => c.startsWith('onTap'))).toBe(false)
+ })
+
+ test('two fingers pinch and never scroll or tap', () => {
+ const { calls, handlers } = recorder()
+ const g = new TouchGestures(handlers)
+ g.touchStart(event(0, { x: 100, y: 300 }))
+ g.touchStart(event(10, { x: 100, y: 300 }, { x: 200, y: 300 }))
+ g.touchMove(event(26, { x: 80, y: 300 }, { x: 220, y: 300 }))
+ g.touchEnd(event(40, { x: 220, y: 300 }))
+ g.touchMove(event(56, { x: 260, y: 300 }))
+ g.touchEnd(event(70))
+ g.destroy()
+ const names = calls.map((c) => c.split(' ')[0])
+ expect(names).toEqual(['onTouchBegin', 'onPinchStart', 'onPinchMove', 'onPinchEnd'])
+ })
+})