From 7e0354704e55ab23fa1f1ada1bba05fb3100831d Mon Sep 17 00:00:00 2001 From: rainyflash <1317811579@qq.com> Date: Sat, 17 Oct 2026 10:26:34 -0400 Subject: [PATCH 1/8] feat(editor): add keystroke telemetry helpers and shortcut HUD timeline lane (#1317) --- electron/ai-edition/agent-tools.ts | 170 ++++++++- .../ai-edition/deep-agent/service.test.ts | 2 + electron/ai-edition/deep-agent/service.ts | 8 + src/components/ai-edition/NewEditorShell.tsx | 20 +- src/components/ai-edition/Preview.tsx | 4 + src/components/ai-edition/PreviewCanvas.tsx | 111 ++++++ .../ai-edition/v4/FloatingInspector.tsx | 117 ++++++ src/components/ai-edition/v4/V4Timeline.tsx | 38 +- src/lib/ai-edition/document/timeline.ts | 22 +- src/lib/ai-edition/schema/index.ts | 39 ++ src/lib/ai-edition/store/regionClipboard.ts | 1 + src/lib/ai-edition/store/useTimeline.test.ts | 5 + src/lib/ai-edition/store/useTimeline.ts | 98 +++++ src/lib/keystrokes/keystrokeTelemetry.test.ts | 262 +++++++++++++ src/lib/keystrokes/keystrokeTelemetry.ts | 353 ++++++++++++++++++ src/native/sceneDescription.ts | 44 +++ 16 files changed, 1288 insertions(+), 6 deletions(-) create mode 100644 src/lib/keystrokes/keystrokeTelemetry.test.ts create mode 100644 src/lib/keystrokes/keystrokeTelemetry.ts diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 50bb5de41..0c7b3a781 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -47,11 +47,16 @@ import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcrip import { type AxcutAnnotationRegion, type AxcutDocument, + type AxcutKeystrokeRange, type AxcutShaderEffect, type AxcutShaderEffectRange, blurDataSchema, clipCropRegionSchema, figureDataSchema, + keystrokePlatformSchema, + keystrokePositionSchema, + keystrokeSizeSchema, + keystrokeThemeSchema, shaderEffectParamSchema, shaderEffectSchema, shaderEffectValueSchema, @@ -96,6 +101,7 @@ import { transitionCutsMs, zoomTransitions, } from "../../src/lib/ai-edition/timeline/zoom-transitions"; +import { parseShortcutKeys } from "../../src/lib/keystrokes/keystrokeTelemetry"; import { SETTING_BOUNDS } from "../../src/lib/projectDefaults"; import type { SegmentationSupport } from "../../src/native/contracts"; import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription"; @@ -150,6 +156,8 @@ function modifierIds(document: AxcutDocument, kind: ModifierKind): string[] { return [...new Set(document.audioTracks.map(trackGroupId))]; case "shaderEffect": return (document.shaderEffectRanges ?? []).map((region) => region.id); + case "keystroke": + return (document.keystrokeRanges ?? []).map((region) => region.id); } } @@ -606,6 +614,7 @@ function modifierIdsOf(document: AxcutDocument): string[] { ...speedRegions.map((r) => r.id), ...cameraFullscreenRegions.map((r) => r.id), ...(document.shaderEffectRanges ?? []).map((r) => r.id), + ...(document.keystrokeRanges ?? []).map((r) => r.id), ]; } @@ -1048,6 +1057,33 @@ export const setCameraFullscreenArgs = z.object({ endSec: secondsSchema.optional(), }); +const shortcutKeysInputSchema = z + .union([z.string().min(1), z.array(z.string().min(1)).min(1).max(8)]) + .describe('Shortcut keys, e.g. "Cmd+Shift+P" or ["Cmd", "K"].'); + +export const addKeystrokeArgs = z.object({ + startSec: secondsSchema, + endSec: secondsSchema, + keys: shortcutKeysInputSchema, + label: z.string().max(120).optional().describe("Optional action label, e.g. 'Command Palette'."), + position: keystrokePositionSchema.default("bottom-center"), + theme: keystrokeThemeSchema.default("dark"), + size: keystrokeSizeSchema.default("md"), + platform: keystrokePlatformSchema.default("win32"), +}); + +export const setKeystrokeArgs = z.object({ + keystrokeId: z.string().min(1), + startSec: secondsSchema.optional(), + endSec: secondsSchema.optional(), + keys: shortcutKeysInputSchema.optional(), + label: z.string().max(120).optional(), + position: keystrokePositionSchema.optional(), + theme: keystrokeThemeSchema.optional(), + size: keystrokeSizeSchema.optional(), + platform: keystrokePlatformSchema.optional(), +}); + export const getTranscriptWordsArgs = z.object({ assetId: z.string().min(1).optional(), startSec: secondsSchema.optional(), @@ -1117,6 +1153,8 @@ export const OPENSCREEN_TOOL_NAMES = [ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addKeystroke", + "setKeystroke", "addEffect", "setEffect", "setEffectDefinition", @@ -1199,6 +1237,8 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addKeystroke", + "setKeystroke", "addEffect", "setEffect", "setEffectDefinition", @@ -1470,6 +1510,18 @@ export function documentSnapshotForModel( startSec: roundSec(c.startMs), endSec: roundSec(c.endMs), })), + keystrokeRanges: coalesceForAgent(document.keystrokeRanges ?? []).map((k) => ({ + id: k.id, + startSec: roundSec(k.startMs), + endSec: roundSec(k.endMs), + keys: k.keys, + ...(k.label ? { label: k.label } : {}), + position: k.position, + theme: k.theme, + size: k.size, + platform: k.platform, + origin: k.origin, + })), // Where each effect is posed, and how. The code is not here: getEffect reads it. effects: coalesceForAgent(document.shaderEffectRanges ?? []).map((r) => ({ id: r.id, @@ -3608,6 +3660,121 @@ export function executeAgentTool( }; } + case "addKeystroke": { + const parsed = addKeystrokeArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const tokens = parseShortcutKeys(parsed.data.keys); + if (tokens.length === 0) return failure("Keystroke shortcut cannot be empty."); + const canonicalKeys = tokens.join("+"); + const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec)); + const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec)); + const region: AxcutKeystrokeRange = { + id: createId("keys"), + startMs, + endMs, + keys: canonicalKeys, + ...(parsed.data.label ? { label: parsed.data.label } : {}), + position: parsed.data.position, + theme: parsed.data.theme, + size: parsed.data.size, + platform: parsed.data.platform, + origin: "agent", + }; + const placed = anchorForAgent(region, document, "keys"); + const landing = landingOf(placed, document); + if (!landing.anchored) { + return coversNoClip("keystroke callout", startMs / 1000, endMs / 1000, document); + } + const next: AxcutDocument = { + ...document, + keystrokeRanges: [...(document.keystrokeRanges ?? []), ...placed], + }; + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + keystrokeId: landing.ids[0], + keys: canonicalKeys, + ...(parsed.data.label ? { label: parsed.data.label } : {}), + position: parsed.data.position, + theme: parsed.data.theme, + size: parsed.data.size, + platform: parsed.data.platform, + ...landingReport(landing, startMs / 1000, endMs / 1000), + }), + summary: + `shortcut [${canonicalKeys}] ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + + landingSuffix(landing, startMs / 1000, endMs / 1000), + }; + } + + case "setKeystroke": { + const parsed = setKeystrokeArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const prev = document.keystrokeRanges ?? []; + const existing = prev.find((r) => r.id === parsed.data.keystrokeId); + if (!existing) return failure(`Unknown keystroke region: ${parsed.data.keystrokeId}`); + const nextKeys = + parsed.data.keys !== undefined ? parseShortcutKeys(parsed.data.keys).join("+") : undefined; + if (nextKeys !== undefined && !nextKeys) { + return failure("Keystroke shortcut cannot be empty."); + } + const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec); + const blocked = refuseIfBlocked("keystroke callout", prev, parsed.data.keystrokeId, { + startMs, + endMs, + }); + if (blocked) return blocked; + const keystrokePill = new Set(resolvePillIds(prev, parsed.data.keystrokeId)); + const patched = prev.map((r) => + keystrokePill.has(r.id) + ? { + ...r, + ...(nextKeys !== undefined ? { keys: nextKeys } : {}), + ...(parsed.data.label !== undefined ? { label: parsed.data.label || undefined } : {}), + ...(parsed.data.position !== undefined ? { position: parsed.data.position } : {}), + ...(parsed.data.theme !== undefined ? { theme: parsed.data.theme } : {}), + ...(parsed.data.size !== undefined ? { size: parsed.data.size } : {}), + ...(parsed.data.platform !== undefined ? { platform: parsed.data.platform } : {}), + } + : r, + ); + const rebuilt = replacePillSpan( + patched, + parsed.data.keystrokeId, + startMs, + endMs, + document.timeline.clips, + () => createId("keys"), + ); + const landing = landingAfterPillEdit(prev, rebuilt, keystrokePill, document); + if (!landing.anchored) { + return coversNoClip("keystroke callout", startMs / 1000, endMs / 1000, document); + } + const next: AxcutDocument = { + ...document, + keystrokeRanges: rebuilt, + }; + const landed = rebuilt.find((r) => r.id === landing.ids[0]); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + keystrokeId: landing.ids[0] ?? parsed.data.keystrokeId, + keys: landed?.keys ?? existing.keys, + ...(landed?.label ? { label: landed.label } : {}), + position: landed?.position ?? existing.position, + theme: landed?.theme ?? existing.theme, + size: landed?.size ?? existing.size, + platform: landed?.platform ?? existing.platform, + ...landingReport(landing, startMs / 1000, endMs / 1000), + }), + summary: + `updated shortcut [${landed?.keys ?? existing.keys}] ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + + landingSuffix(landing, startMs / 1000, endMs / 1000), + }; + } + case "addAudio": { const parsed = addAudioArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); @@ -3802,9 +3969,10 @@ export function executeAgentTool( else if (cameraFullscreenRegions.some((c) => c.id === id)) kind = "cameraFullscreen"; else if (document.audioTracks.some((t) => trackGroupId(t) === id)) kind = "audio"; else if ((document.shaderEffectRanges ?? []).some((r) => r.id === id)) kind = "shaderEffect"; + else if ((document.keystrokeRanges ?? []).some((r) => r.id === id)) kind = "keystroke"; if (!kind) { return failure( - `No zoom / speed / annotation / full-camera / audio / effect modifier with id ${id}. ` + + `No zoom / speed / annotation / full-camera / keystroke / audio / effect modifier with id ${id}. ` + `For a trim use removeTrim; for a clip use removeClip.`, ); } diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 0a368e95b..6c9746266 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -83,6 +83,8 @@ const ARGS: Record = { setAnnotation: { annotationId: "ann_nope" }, addCameraFullscreen: { startSec: 1, endSec: 2 }, setCameraFullscreen: { cameraFullscreenId: "cam_nope" }, + addKeystroke: { startSec: 1, endSec: 2, keys: ["Cmd", "K"] }, + setKeystroke: { keystrokeId: "keys_nope" }, // No checker in these tests: the effect is stored unchecked, and the result says so. addEffect: { startSec: 1, diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 00fc819ee..e51abc631 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -57,6 +57,7 @@ import { addAutoZoomsArgs, addCameraFullscreenArgs, addEffectArgs, + addKeystrokeArgs, addSpeedArgs, addTrimArgs, addTrimsArgs, @@ -85,6 +86,7 @@ import { setClipRangeArgs, setEffectArgs, setEffectDefinitionArgs, + setKeystrokeArgs, setSpeedArgs, setTrimArgs, setWordTextArgs, @@ -274,6 +276,10 @@ export const TOOL_DESCRIPTIONS: Record = { "Read one effect of the project in full: its WGSL code, parameters, description, cost and the regions that use it. getCurrentDocument lists effects without their code.", setCameraFullscreen: "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.", + addKeystroke: + "Place a stylized keyboard shortcut HUD callout over a span of the edited timeline (virtual seconds). Pass `keys` as an array of key tokens (e.g. ['Cmd', 'K'] or ['Ctrl', 'Shift', 'P']), plus optional `label` (e.g. 'Command Palette'), `position` ('bottom-center', 'bottom-left', 'bottom-right', 'top-center', 'top-left', 'top-right'), `theme` ('dark', 'light', 'glass'), `size` ('sm', 'md', 'lg'), and `platform` ('auto', 'mac', 'win').", + setKeystroke: + "Move, resize, or restyle an existing keyboard shortcut HUD callout by `keystrokeId` (virtual-timeline seconds). Only the fields you pass are changed.", addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them. If there is none, a file has to be imported first (importMedia, where it is offered): never guess an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`, setAudio: `Move, resize, re-level, re-lane, mute, loop, fade or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to fade it in or out (fadeInMs / fadeOutMs, 0 to ${AUDIO_FADE_MAX_MS} ms), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.`, removeTrim: @@ -558,6 +564,8 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ ["setAnnotation", setAnnotationArgs], ["addCameraFullscreen", addCameraFullscreenArgs], ["setCameraFullscreen", setCameraFullscreenArgs], + ["addKeystroke", addKeystrokeArgs], + ["setKeystroke", setKeystrokeArgs], ["addEffect", addEffectArgs], ["setEffect", setEffectArgs], ["setEffectDefinition", setEffectDefinitionArgs], diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index 13e5c1c47..4bbcb3779 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -1075,7 +1075,9 @@ export function NewEditorShell() { ? "ann" : snapshot.kind === "shaderEffect" ? "fxr" - : snapshot.kind; + : snapshot.kind === "keystroke" + ? "key" + : snapshot.kind; // Audio re-ventilates through its own anchorer, which advances each // fragment's source offset — the generic one would copy the offset into @@ -1142,6 +1144,17 @@ export function NewEditorShell() { return; } await saveDocument(next, { history: true }); + } else if (snapshot.kind === "keystroke") { + await saveDocument( + { + ...doc, + keystrokeRanges: [ + ...(doc.keystrokeRanges ?? []), + ...anchored, + ] as typeof doc.keystrokeRanges, + }, + { history: true }, + ); } else { // speed and cameraFullscreen are both plain spans on legacyEditor. const key = snapshot.kind === "speed" ? "speedRegions" : "cameraFullscreenRegions"; @@ -1227,7 +1240,9 @@ export function NewEditorShell() { ? tl.annotationRegions : sel.kind === "speed" ? tl.speedRegions - : tl.cameraFullscreenRegions; + : sel.kind === "keystroke" + ? tl.keystrokeRanges + : tl.cameraFullscreenRegions; const region = wholePillOf( source as Array<{ id: string; startMs: number; endMs: number }>, sel.id, @@ -1644,6 +1659,7 @@ export function NewEditorShell() { zoomRegions={tl.zoomRegions} speedRegions={tl.speedRegions} cameraFullscreenRegions={tl.cameraFullscreenRegions} + keystrokeRanges={tl.keystrokeRanges} trimRanges={tl.trimRanges} selectedZoomRegionId={tl.selection?.kind === "zoom" ? tl.selection.id : null} onZoomFocusChange={tl.updateZoomFocusLive} diff --git a/src/components/ai-edition/Preview.tsx b/src/components/ai-edition/Preview.tsx index f4ad0a5c5..d48dd1d69 100644 --- a/src/components/ai-edition/Preview.tsx +++ b/src/components/ai-edition/Preview.tsx @@ -5,6 +5,7 @@ import type { AxcutAnnotationRegion, AxcutAudioTrack, AxcutClip, + AxcutKeystrokeRange, AxcutTranscript, AxcutTrimRange, AxcutZoomRegion, @@ -38,6 +39,7 @@ interface PreviewProps { zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; + keystrokeRanges?: AxcutKeystrokeRange[]; trimRanges?: AxcutTrimRange[]; transcripts?: AxcutTranscript[]; selectedZoomRegionId?: string | null; @@ -73,6 +75,7 @@ export function Preview({ zoomRegions, speedRegions, cameraFullscreenRegions, + keystrokeRanges, trimRanges, transcripts, selectedZoomRegionId, @@ -216,6 +219,7 @@ export function Preview({ zoomRegions={zoomRegions} speedRegions={speedRegions} cameraFullscreenRegions={cameraFullscreenRegions} + keystrokeRanges={keystrokeRanges} trimRanges={trimRanges} transcripts={transcripts} selectedZoomRegionId={selectedZoomRegionId} diff --git a/src/components/ai-edition/PreviewCanvas.tsx b/src/components/ai-edition/PreviewCanvas.tsx index c817fde3c..ca74ba15e 100644 --- a/src/components/ai-edition/PreviewCanvas.tsx +++ b/src/components/ai-edition/PreviewCanvas.tsx @@ -41,6 +41,7 @@ import type { AxcutAnnotationRegion, AxcutAudioTrack, AxcutClip, + AxcutKeystrokeRange, AxcutTranscript, AxcutTrimRange, AxcutZoomRegion, @@ -57,6 +58,7 @@ import { resolveWebcamLayoutPreset, type WebcamCompositeLayout, } from "@/lib/compositeLayout"; +import { formatShortcutKeycaps } from "@/lib/keystrokes/keystrokeTelemetry"; import { webcamAnchorAt } from "@/lib/projectDefaults"; import { wallpaperStyle } from "@/lib/wallpaper"; import { getCssClipPath } from "@/lib/webcamMaskShapes"; @@ -86,6 +88,7 @@ interface PreviewCanvasProps { zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; + keystrokeRanges?: AxcutKeystrokeRange[]; trimRanges?: AxcutTrimRange[]; /** Relayed to `VirtualPreview` so a clip carrying added words plays their extensions. */ transcripts?: AxcutTranscript[]; @@ -512,6 +515,114 @@ export function PreviewCanvas(props: PreviewCanvasProps) { onCommit={props.onAnnotationCommit} /> ) : null} + {props.keystrokeRanges && props.keystrokeRanges.length > 0 ? ( + + ) : null} + + ); +} + +function KeystrokeHudLayer({ + ranges, + currentTimeSec, +}: { + ranges: AxcutKeystrokeRange[]; + currentTimeSec: number; +}) { + const nowMs = currentTimeSec * 1000; + const active = ranges.find((r) => r.startMs <= nowMs && nowMs < r.endMs) ?? null; + if (!active) return null; + + const keycaps = formatShortcutKeycaps(active.keys, active.platform); + if (keycaps.length === 0) return null; + + const posStyle: React.CSSProperties = { + position: "absolute", + zIndex: 8, + pointerEvents: "none", + display: "flex", + alignItems: "center", + gap: active.size === "sm" ? 6 : active.size === "lg" ? 10 : 8, + padding: active.size === "sm" ? "6px 10px" : active.size === "lg" ? "12px 18px" : "8px 14px", + borderRadius: 14, + ...(active.position.startsWith("bottom") ? { bottom: "6%" } : { top: "6%" }), + ...(active.position.endsWith("center") + ? { left: "50%", transform: "translateX(-50%)" } + : active.position.endsWith("left") + ? { left: "4%" } + : { right: "4%" }), + ...(active.theme === "light" + ? { + background: "rgba(255, 255, 255, 0.92)", + border: "1px solid rgba(0, 0, 0, 0.12)", + boxShadow: "0 10px 28px rgba(0, 0, 0, 0.22)", + color: "#111827", + } + : active.theme === "glass" + ? { + background: "rgba(24, 24, 27, 0.58)", + backdropFilter: "blur(16px)", + border: "1px solid rgba(255, 255, 255, 0.22)", + boxShadow: "0 12px 32px rgba(0, 0, 0, 0.38)", + color: "#f9fafb", + } + : { + background: "rgba(17, 17, 20, 0.92)", + border: "1px solid rgba(255, 255, 255, 0.14)", + boxShadow: "0 12px 32px rgba(0, 0, 0, 0.48)", + color: "#f9fafb", + }), + }; + + const keycapFontPx = active.size === "sm" ? 13 : active.size === "lg" ? 20 : 16; + const keycapPad = + active.size === "sm" ? "3px 8px" : active.size === "lg" ? "6px 13px" : "4px 10px"; + + return ( +
+
+ {keycaps.map((cap, idx) => ( + + {cap} + + ))} +
+ {active.label ? ( + + {active.label} + + ) : null}
); } diff --git a/src/components/ai-edition/v4/FloatingInspector.tsx b/src/components/ai-edition/v4/FloatingInspector.tsx index b82a0e9cd..5ad93392a 100644 --- a/src/components/ai-edition/v4/FloatingInspector.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.tsx @@ -14,6 +14,7 @@ import { EyeOff, FileText, ImageIcon, + Keyboard, type LucideIcon, Maximize2, MousePointer2, @@ -59,6 +60,10 @@ import { resolveAspectRatioValue } from "@/lib/ai-edition/document/outputFormat" import type { AxcutAnnotationRegion, AxcutClip, + AxcutKeystrokePlatform, + AxcutKeystrokePosition, + AxcutKeystrokeSize, + AxcutKeystrokeTheme, AxcutShaderEffect, AxcutShaderEffectParam, AxcutShaderEffectRange, @@ -1313,6 +1318,118 @@ function SelectionPane({ tl, onClose }: { tl: TimelineApi; onClose: () => void } ); } + if (selection.kind === "keystroke") { + const region = tl.keystrokeRanges.find((r) => r.id === selection.id); + if (!region) return null; + return ( +
+ {paneHeader(, "Shortcut Callout", onClose, tc("actions.close"))} +
+
+ + Shortcut Keys + + { + const nextKeys = e.target.value.trim(); + if (nextKeys) { + void tl.updateKeystroke(region.id, { keys: nextKeys }); + } + }} + style={{ + width: "100%", + padding: "7px 10px", + borderRadius: 10, + border: "1px solid var(--border)", + background: "var(--surface-2)", + color: "var(--fg)", + fontSize: 13, + }} + /> +
+
+ + Action Label (optional) + + + void tl.updateKeystroke(region.id, { + label: e.target.value || undefined, + }) + } + style={{ + width: "100%", + padding: "7px 10px", + borderRadius: 10, + border: "1px solid var(--border)", + background: "var(--surface-2)", + color: "var(--fg)", + fontSize: 13, + }} + /> +
+ + label="Position" + columns={2} + options={[ + { value: "bottom-center", label: "Bottom Center" }, + { value: "bottom-left", label: "Bottom Left" }, + { value: "bottom-right", label: "Bottom Right" }, + { value: "top-center", label: "Top Center" }, + { value: "top-left", label: "Top Left" }, + { value: "top-right", label: "Top Right" }, + ]} + value={region.position} + onChange={(position) => void tl.updateKeystroke(region.id, { position })} + /> + + label="Theme" + columns={3} + options={[ + { value: "dark", label: "Dark" }, + { value: "light", label: "Light" }, + { value: "glass", label: "Glass" }, + ]} + value={region.theme} + onChange={(theme) => void tl.updateKeystroke(region.id, { theme })} + /> + + label="Size" + columns={3} + options={[ + { value: "sm", label: "S" }, + { value: "md", label: "M" }, + { value: "lg", label: "L" }, + ]} + value={region.size} + onChange={(size) => void tl.updateKeystroke(region.id, { size })} + /> + + label="Symbols" + columns={3} + options={[ + { value: "darwin", label: "macOS (⌘)" }, + { value: "win32", label: "Windows" }, + { value: "linux", label: "Linux" }, + ]} + value={region.platform} + onChange={(platform) => void tl.updateKeystroke(region.id, { platform })} + /> + +
+
+ ); + } + // trim — a trim ventilated across a clip boundary is 2+ DSL rows that render // as one coalesced pill (see V4Timeline's trimPills), so the DURATION shown has to be // the group's, not the clicked row's. Deleting no longer needs the same expansion here: diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index bb026fcd8..6170fa26b 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -3,6 +3,7 @@ import { Clock, Crosshair, Eraser, + Keyboard, Loader2, Maximize2, MessageSquare, @@ -596,7 +597,14 @@ const AudioLanePill = memo(function AudioLanePill({ interface LanePill { id: string; - kind: "annotation" | "speed" | "trim" | "zoom" | "cameraFullscreen" | "shaderEffect"; + kind: + | "annotation" + | "speed" + | "trim" + | "zoom" + | "cameraFullscreen" + | "shaderEffect" + | "keystroke"; start: number; end: number; label: string; @@ -776,6 +784,14 @@ export function V4Timeline({ label: effectNames.get(p.member.effectId) ?? "", sourceIds: p.ids, })); + const keystrokePills: LanePill[] = coalesceRegionsForRuler(tl.keystrokeRanges).map((p) => ({ + id: p.ids[0], + kind: "keystroke", + start: p.start, + end: p.end, + label: p.member.label ? `${p.member.keys} · ${p.member.label}` : p.member.keys, + sourceIds: p.ids, + })); const zoomRuns = coalesceRegionsForRuler(tl.zoomRegions); const zoomPills: LanePill[] = zoomRuns.map((p) => ({ id: p.ids[0], @@ -950,6 +966,7 @@ export function V4Timeline({ zoom: tl.zoomRegions, cameraFullscreen: tl.cameraFullscreenRegions, shaderEffect: tl.shaderEffectRanges, + keystroke: tl.keystrokeRanges, // Nothing to edit on a trim but its span, and a span edit moves the whole pill. trim: [], }; @@ -1027,6 +1044,8 @@ export function V4Timeline({ await tl.updateCameraFullscreenSpan(grabbed, s * 1000, en * 1000); else if (pill.kind === "shaderEffect") await tl.updateShaderEffectSpan(grabbed, s * 1000, en * 1000); + else if (pill.kind === "keystroke") + await tl.updateKeystrokeSpan(grabbed, s * 1000, en * 1000); else { // Trims are stored in source-time per asset but manipulated on the // timeline like every other pill. Ventilate the new span across the @@ -1456,7 +1475,7 @@ export function V4Timeline({ ? styles.laneTrim : kind === "cameraFullscreen" ? styles.laneCameraFullscreen - : kind === "shaderEffect" + : kind === "shaderEffect" || kind === "keystroke" ? styles.laneShaderEffect : styles.laneZoom; const pillIcon = (kind: LanePill["kind"]) => @@ -1470,6 +1489,8 @@ export function V4Timeline({ ) : kind === "shaderEffect" ? ( + ) : kind === "keystroke" ? ( + ) : ( ); @@ -2177,6 +2198,16 @@ export function V4Timeline({ ) : null} + + + {/* Last, behind a divider: every button before it adds a region, this one clears them. Absent with its divider, not greyed out, when there is nothing to clear. One write in the store, so one Ctrl+Z restores every @@ -2330,6 +2361,9 @@ export function V4Timeline({ {tl.shaderEffectRanges.length > 0 ? (
{renderPills(shaderEffectPills, "")}
) : null} + {tl.keystrokeRanges.length > 0 ? ( +
{renderPills(keystrokePills, "")}
+ ) : null} {/* Imported audio tracks (issue #350). Always shown, like every other lane — "Add audio" is a toolbar peer of the region tools now (and has a keyboard shortcut), so an empty lane advertises the shortcut diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index 2b8366ba3..ea9d2ae7c 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -51,7 +51,8 @@ export type RegionKind = | "speed" | "cameraFullscreen" | "audio" - | "shaderEffect"; + | "shaderEffect" + | "keystroke"; /** Length a clip is given before its media has been probed. Lives here, in the pure * document layer, because that layer decides which clips are still waiting for a real @@ -397,6 +398,12 @@ export function mapAllRegionCollections( (document.shaderEffectRanges ?? []) as unknown as StoredRegion[], "fxr", ) as unknown as AxcutDocument["shaderEffectRanges"], + keystrokeRanges: document.keystrokeRanges + ? (fn( + document.keystrokeRanges as unknown as StoredRegion[], + "key", + ) as unknown as AxcutDocument["keystrokeRanges"]) + : undefined, annotations: fn( document.annotations as unknown as StoredRegion[], "ann", @@ -748,6 +755,7 @@ function anchoredRegionsOf(document: AxcutDocument): Array<{ id: string; clipId: (legacy?.speedRegions as StoredRegion[] | undefined) ?? [], (legacy?.cameraFullscreenRegions as StoredRegion[] | undefined) ?? [], (document.shaderEffectRanges ?? []) as unknown as StoredRegion[], + (document.keystrokeRanges ?? []) as unknown as StoredRegion[], ]; return collections .flat() @@ -1217,6 +1225,14 @@ export function removeRegion(document: AxcutDocument, kind: RegionKind, id: stri ...document, shaderEffectRanges: dropPillById(document.shaderEffectRanges ?? [], id), }); + case "keystroke": + return { + ...document, + keystrokeRanges: dropPillById( + document.keystrokeRanges ?? [], + id, + ) as AxcutDocument["keystrokeRanges"], + }; default: { // ponytail: exhaustive — TS errors here if a new RegionKind is added. const exhaustive: never = kind; @@ -1265,6 +1281,10 @@ const EDIT_REGIONS: Record< count: (d) => (d.shaderEffectRanges ?? []).length, clear: (d) => ({ ...d, shaderEffectRanges: [], shaderEffects: [] }), }, + keystroke: { + count: (d) => (d.keystrokeRanges ?? []).length, + clear: (d) => ({ ...d, keystrokeRanges: [] }), + }, }; /** How many edit regions the document holds, stored rows and not pills. Pure. */ diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index c3422b597..80aeb4897 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -190,6 +190,7 @@ export const assetSchema = z.object({ // no schema-version bump (an older build simply drops the key on save). transcriptionFailure: assetTranscriptionFailureSchema.nullish(), cameraTrack: cameraTrackSchema, + keystrokeTelemetryPath: z.string().optional(), }); // A crop is a sub-rectangle of the source video, expressed as fractions @@ -700,6 +701,37 @@ export const audioTrackSchema = endGteStart( "startMs", ); +export const keystrokePositionSchema = z.enum([ + "bottom-center", + "bottom-left", + "bottom-right", + "top-center", + "top-left", + "top-right", +]); + +export const keystrokeThemeSchema = z.enum(["dark", "light", "glass"]); +export const keystrokeSizeSchema = z.enum(["sm", "md", "lg"]); +export const keystrokePlatformSchema = z.enum(["darwin", "win32", "linux"]); + +export const keystrokeRangeSchema = endGteStart( + z.object({ + id: z.string().min(1), + startMs: z.number().nonnegative(), + endMs: z.number().nonnegative(), + ...clipAnchorShape, + keys: z.string().min(1), + label: z.string().max(120).optional(), + position: keystrokePositionSchema.default("bottom-center"), + theme: keystrokeThemeSchema.default("dark"), + size: keystrokeSizeSchema.default("md"), + platform: keystrokePlatformSchema.default("win32"), + origin: z.enum(["system", "agent", "user"]).default("user"), + }), + "endMs", + "startMs", +); + // Legacy OpenScreen appearance / export settings that the v3 schema doesn't // normalize into the timeline / assets model. They are applied at export time // by the existing pipeline (see technical-documentation/architecture/document-model.md). @@ -740,6 +772,8 @@ const documentSchemaShape = z.object({ // an absent list as empty. An older build drops both keys when it saves. shaderEffects: z.array(shaderEffectSchema).optional(), shaderEffectRanges: z.array(shaderEffectRangeSchema).optional(), + // Keystroke / shortcut HUD callout regions (issue #1317). + keystrokeRanges: z.array(keystrokeRangeSchema).optional(), // Imported audio tracks (issue #350). Defaulted so every document written // before this loads unchanged; an older build simply strips the key on save. audioTracks: z.array(audioTrackSchema).default([]), @@ -1379,6 +1413,11 @@ export type AxcutShaderEffect = z.infer; export type AxcutShaderEffectParam = z.infer; export type AxcutShaderEffectRange = z.infer; export type AxcutShaderEffectValue = z.infer; +export type AxcutKeystrokeRange = z.infer; +export type AxcutKeystrokePosition = z.infer; +export type AxcutKeystrokeTheme = z.infer; +export type AxcutKeystrokeSize = z.infer; +export type AxcutKeystrokePlatform = z.infer; export type AxcutLegacyEditor = z.infer; export type AxcutDocument = z.infer; export type AxcutDocumentInput = z.input; diff --git a/src/lib/ai-edition/store/regionClipboard.ts b/src/lib/ai-edition/store/regionClipboard.ts index 9ae66a8ab..e8075da87 100644 --- a/src/lib/ai-edition/store/regionClipboard.ts +++ b/src/lib/ai-edition/store/regionClipboard.ts @@ -19,6 +19,7 @@ export type RegionSnapshot = // An effect region carries its definition: pasted into another project, the definition is // recreated there, since the region alone would name an effect that project does not have. | { kind: "shaderEffect"; region: Record; definition: AxcutShaderEffect } + | { kind: "keystroke"; region: Record } // A trim carries no user-visible properties, so all there is to copy is how // LONG it was — `{ durationSec }`. That is not a special case so much as the // general one made obvious: every paste keeps the copied properties and takes diff --git a/src/lib/ai-edition/store/useTimeline.test.ts b/src/lib/ai-edition/store/useTimeline.test.ts index f8e6ba8a3..07262316d 100644 --- a/src/lib/ai-edition/store/useTimeline.test.ts +++ b/src/lib/ai-edition/store/useTimeline.test.ts @@ -889,6 +889,7 @@ describe("useTimeline.clearTimeline", () => { { history: true }, ); }, + keystroke: (tl) => tl.addKeystroke(), }; const editKinds = Object.keys(addOf) as EditKind[]; @@ -901,6 +902,7 @@ describe("useTimeline.clearTimeline", () => { speed: doc ? readSpeedRegions(doc).length : 0, cameraFullscreen: legacy.cameraFullscreenRegions?.length ?? 0, shaderEffect: doc?.shaderEffectRanges?.length ?? 0, + keystroke: doc?.keystrokeRanges?.length ?? 0, }; }; const noRegions: Record = { @@ -910,6 +912,7 @@ describe("useTimeline.clearTimeline", () => { speed: 0, cameraFullscreen: 0, shaderEffect: 0, + keystroke: 0, }; // A project holding everything that is NOT an edit: two clips, a webcam, an imported @@ -1070,6 +1073,7 @@ describe("useTimeline.clearTimeline", () => { speed: 1, cameraFullscreen: 1, shaderEffect: 1, + keystroke: 1, }); await act(async () => { @@ -1110,6 +1114,7 @@ describe("useTimeline.clearTimeline", () => { speed: 1, cameraFullscreen: 1, shaderEffect: 1, + keystroke: 1, }); act(() => { diff --git a/src/lib/ai-edition/store/useTimeline.ts b/src/lib/ai-edition/store/useTimeline.ts index 7d650ed85..0b91987ea 100644 --- a/src/lib/ai-edition/store/useTimeline.ts +++ b/src/lib/ai-edition/store/useTimeline.ts @@ -41,6 +41,7 @@ import type { AxcutAudioTrack, AxcutClipCropRegion, AxcutDocument, + AxcutKeystrokeRange, AxcutShaderEffectRange, } from "../schema"; import { @@ -614,6 +615,44 @@ export function useTimeline() { [document, saveDocument], ); + const addKeystroke = useCallback( + async ( + durationSec = DEFAULT_NEW_REGION_SEC, + seed?: Partial>, + ) => { + if (!document) return; + const timeMs = Math.round(playheadSec() * 1000); + const endMs = timeMs + Math.round(durationSec * 1000); + const id = createId("key"); + const nextRegion: AxcutKeystrokeRange = { + id, + startMs: timeMs, + endMs, + keys: seed?.keys ?? "Cmd+Shift+P", + ...(seed?.label ? { label: seed.label } : {}), + position: seed?.position ?? "bottom-center", + theme: seed?.theme ?? "dark", + size: seed?.size ?? "md", + platform: seed?.platform ?? "win32", + origin: seed?.origin ?? "user", + }; + const anchored = anchorRegionsWithDerivedMs([nextRegion], document.timeline.clips, () => + createId("key"), + ) as AxcutKeystrokeRange[]; + const next: AxcutDocument = { + ...document, + keystrokeRanges: [...(document.keystrokeRanges ?? []), ...anchored], + }; + if (!(await saveDocument(next, { history: true }))) return; + const placedId = anchored[0]?.id ?? id; + setSelection({ kind: "keystroke", id: placedId }); + setMultiSelection([{ kind: "keystroke", id: placedId }]); + setClipSelection(null); + setSelectedAudioTrackId(null); + }, + [document, saveDocument, setSelectedAudioTrackId], + ); + // Like updateTrimRange but also re-attaches the trim to a (possibly different) CLIP — // needed when a trim is dragged across a clip boundary, whether or not the landing clip // is backed by another asset. Re-pointing `clipId` as well as `assetId` is what makes a @@ -1164,6 +1203,51 @@ export function useTimeline() { } }, [saveDocument]); + const updateKeystrokeSpan = useCallback( + async (id: string, startMs: number, endMs: number) => { + if (!document) return; + const s = finiteMs(startMs); + const e = finiteMs(endMs); + const next: AxcutDocument = { + ...document, + keystrokeRanges: replacePillSpan( + document.keystrokeRanges ?? [], + id, + Math.min(s, e), + Math.max(s, e), + document.timeline.clips, + () => createId("key"), + ) as AxcutDocument["keystrokeRanges"], + }; + await saveDocument(next, { history: true }); + }, + [document, saveDocument], + ); + + const updateKeystroke = useCallback( + async ( + id: string, + patch: Partial< + Omit< + AxcutKeystrokeRange, + "id" | "startMs" | "endMs" | "clipId" | "sourceStartSec" | "sourceEndSec" + > + >, + ) => { + if (!document) return; + const next: AxcutDocument = { + ...document, + keystrokeRanges: patchRegionById( + document.keystrokeRanges ?? [], + id, + patch, + ) as AxcutDocument["keystrokeRanges"], + }; + await saveDocument(next, { history: true }); + }, + [document, saveDocument], + ); + const updateSpeedValue = useCallback( async (id: string, speed: number) => { if (!document) return; @@ -1215,6 +1299,7 @@ export function useTimeline() { const shaderEffectIds = new Set( handles.filter((h) => h.kind === "shaderEffect").map((h) => h.id), ); + const keystrokeIds = new Set(handles.filter((h) => h.kind === "keystroke").map((h) => h.id)); const legacy = (document.legacyEditor as Record) ?? {}; const prevSpeed = dropPillsByIds( (legacy.speedRegions as Array<{ id: string; startMs: number; endMs: number }>) ?? [], @@ -1237,6 +1322,14 @@ export function useTimeline() { ), } : {}), + ...(keystrokeIds.size > 0 + ? { + keystrokeRanges: dropPillsByIds( + document.keystrokeRanges ?? [], + keystrokeIds, + ) as AxcutDocument["keystrokeRanges"], + } + : {}), timeline: { ...document.timeline, // Whole-pill delete, same as the zoom/annotation lines above — a trim grown @@ -1556,6 +1649,7 @@ export function useTimeline() { // One region per effect, never two at once; each names its definition. const shaderEffectRanges = hasDoc ? (document.shaderEffectRanges ?? []) : []; const shaderEffects = hasDoc ? (document.shaderEffects ?? []) : []; + const keystrokeRanges = hasDoc ? (document.keystrokeRanges ?? []) : []; // --- Timeline audio tracks (issue #350) ------------------------------------- // CLIP-ANCHORED like every region above: one user-visible track is one pill @@ -1744,6 +1838,7 @@ export function useTimeline() { cameraFullscreenRegions, shaderEffectRanges, shaderEffects, + keystrokeRanges, clips: document?.timeline.clips ?? [], assets: document?.assets ?? [], // The timeline marks where the user has ADDED words — text with no audio behind it. @@ -1760,6 +1855,7 @@ export function useTimeline() { addAnnotation, addSpeed, addCameraFullscreen, + addKeystroke, removeRegion, removeRegions, hasEditRegions, @@ -1801,6 +1897,8 @@ export function useTimeline() { updateShaderEffect, updateShaderEffectLive, commitShaderEffectChange, + updateKeystrokeSpan, + updateKeystroke, // T19 — drives the preview video during trim-edge resize. setCurrentTime: useProjectStore((s) => s.setCurrentTime), }; diff --git a/src/lib/keystrokes/keystrokeTelemetry.test.ts b/src/lib/keystrokes/keystrokeTelemetry.test.ts new file mode 100644 index 000000000..3d6beb424 --- /dev/null +++ b/src/lib/keystrokes/keystrokeTelemetry.test.ts @@ -0,0 +1,262 @@ +import { describe, expect, it } from "vitest"; +import { executeAgentTool } from "../../../electron/ai-edition/agent-tools"; +import { buildSceneDescription } from "../../native/sceneDescription"; +import { duplicateClip, removeClip } from "../ai-edition/document/timeline"; +import { type AxcutDocument, createEmptyDocument, documentSchema } from "../ai-edition/schema"; +import { + buildKeystrokeTrack, + coalesceKeystrokeSamplesToRegions, + formatKeycapLabel, + formatShortcutKeycaps, + getKeystrokeTelemetryCandidates, + isShortcutSample, + normalizeKeyToken, + parseShortcutKeys, +} from "./keystrokeTelemetry"; + +const FIXTURE_CREATED_AT = "2026-01-01T00:00:00.000Z"; + +function createTestDocument(): AxcutDocument { + const base = createEmptyDocument({ + title: "Keystroke Test", + projectId: "proj_keys", + createdAt: FIXTURE_CREATED_AT, + }); + return documentSchema.parse({ + ...base, + project: { ...base.project, primaryAssetId: "asset_1" }, + assets: [ + { + id: "asset_1", + kind: "video", + label: "Screen Recording", + originalPath: "/recordings/demo.mp4", + durationSec: 20, + }, + ], + timeline: { + ...base.timeline, + clips: [ + { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 20, + timelineStartSec: 0, + timelineEndSec: 20, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + trimRanges: [ + { + id: "trim_1", + assetId: "asset_1", + clipId: "clip_1", + startSec: 8, + endSec: 10, + reason: "pause", + origin: "user", + }, + ], + }, + }); +} + +describe("keystrokeTelemetry helpers", () => { + it("normalizes and parses shortcut key strings and arrays", () => { + expect(normalizeKeyToken("command")).toBe("Cmd"); + expect(normalizeKeyToken("control")).toBe("Ctrl"); + expect(normalizeKeyToken("opt")).toBe("Option"); + expect(normalizeKeyToken("escape")).toBe("Esc"); + expect(normalizeKeyToken("f12")).toBe("F12"); + expect(normalizeKeyToken("k")).toBe("K"); + + expect(parseShortcutKeys("cmd + shift + p")).toEqual(["Cmd", "Shift", "P"]); + expect(parseShortcutKeys(["ctrl", "alt", "delete"])).toEqual(["Ctrl", "Alt", "Delete"]); + expect(parseShortcutKeys("Cmd++")).toEqual(["Cmd", "+"]); + }); + + it("formats platform-specific keycap labels for macOS and Windows", () => { + expect(formatShortcutKeycaps(["Cmd", "Shift", "P"], "darwin")).toEqual(["⌘", "⇧", "P"]); + expect(formatShortcutKeycaps(["Option", "Backspace"], "darwin")).toEqual(["⌥", "⌫"]); + expect(formatShortcutKeycaps(["Cmd", "Shift", "P"], "win32")).toEqual(["Ctrl", "Shift", "P"]); + expect(formatKeycapLabel("Enter", "darwin")).toBe("↵"); + expect(formatKeycapLabel("Up", "win32")).toBe("↑"); + }); + + it("distinguishes intentional shortcuts from plain typing for privacy safety", () => { + expect(isShortcutSample({ timeMs: 1000, keys: ["Cmd", "K"] })).toBe(true); + expect(isShortcutSample({ timeMs: 1200, keys: ["Ctrl", "Shift", "P"] })).toBe(true); + expect(isShortcutSample({ timeMs: 1400, keys: ["F5"] })).toBe(true); + expect(isShortcutSample({ timeMs: 1500, keys: ["Esc"] })).toBe(true); + // Plain typing (letters, numbers, Shift+letter, Space, Backspace) must be ignored by default + expect(isShortcutSample({ timeMs: 1600, keys: ["a"] })).toBe(false); + expect(isShortcutSample({ timeMs: 1700, keys: ["Shift", "a"] })).toBe(true); + expect(isShortcutSample({ timeMs: 1800, keys: ["Space"] })).toBe(false); + expect(isShortcutSample({ timeMs: 1900, keys: ["Shift"] })).toBe(false); + // Explicit override + expect(isShortcutSample({ timeMs: 2000, keys: ["Space"], isShortcut: true })).toBe(true); + }); + + it("returns candidate sidecar paths for a video path", () => { + const candidates = getKeystrokeTelemetryCandidates("/recordings/take-1.mp4"); + expect(candidates).toEqual([ + "/recordings/take-1.mp4.keys.json", + "/recordings/take-1.keys.json", + ]); + expect(getKeystrokeTelemetryCandidates("")).toEqual([]); + }); + + it("coalesces rapid repeated shortcuts and filters out prose keystrokes", () => { + const spans = coalesceKeystrokeSamplesToRegions( + [ + { timeMs: 1000, keys: ["Cmd", "S"], label: "Save" }, + { timeMs: 1300, keys: ["Cmd", "S"] }, // within 600ms merge window + { timeMs: 1500, keys: ["x"] }, // plain key — filtered out + { timeMs: 3000, keys: ["Cmd", "Shift", "P"], label: "Command Palette" }, + ], + { holdMs: 1500, mergeWindowMs: 500 }, + ); + + expect(spans).toEqual([ + { + startMs: 1000, + endMs: 2800, + keys: "Cmd+S", + label: "Save", + }, + { + startMs: 3000, + endMs: 4500, + keys: "Cmd+Shift+P", + label: "Command Palette", + }, + ]); + }); + + it("projects recorded keystrokes onto the edited timeline and flags trimmed spans", () => { + const doc = createTestDocument(); + const track = buildKeystrokeTrack( + [ + { timeMs: 2000, keys: ["Cmd", "K"], label: "Quick Open" }, + { timeMs: 9000, keys: ["Cmd", "B"], label: "Toggle Sidebar" }, // inside trim 8..10s + ], + doc.timeline.clips, + doc.timeline.trimRanges, + { assetId: "asset_1" }, + ); + + expect(track.assetId).toBe("asset_1"); + expect(track.eventCount).toBe(2); + expect(track.events[0]).toMatchObject({ + atSec: 2, + virtualSec: 2, + keys: "Cmd+K", + label: "Quick Open", + isShortcut: true, + }); + expect(track.events[1]).toMatchObject({ + atSec: 9, + virtualSec: 9, + keys: "Cmd+B", + trimmed: true, + }); + }); +}); + +describe("keystrokeRanges document & agent tool integration", () => { + it("adds, updates, projects, duplicates, and removes keystroke HUD callouts", () => { + const doc = createTestDocument(); + + const added = executeAgentTool( + doc, + "addKeystroke", + JSON.stringify({ + startSec: 2, + endSec: 4, + keys: ["Cmd", "Shift", "P"], + label: "Command Palette", + position: "bottom-center", + theme: "glass", + size: "lg", + platform: "darwin", + }), + ); + expect(added.ok).toBe(true); + const addedDoc = added.document as AxcutDocument; + expect(addedDoc.keystrokeRanges).toHaveLength(1); + const keystrokeId = addedDoc.keystrokeRanges?.[0]?.id ?? ""; + + // Verify sceneDescription projection + const scene = buildSceneDescription(addedDoc); + expect(scene.keystrokeRegions).toHaveLength(1); + expect(scene.keystrokeRegions?.[0]).toMatchObject({ + id: keystrokeId, + startSec: 2, + endSec: 4, + keys: "Cmd+Shift+P", + label: "Command Palette", + position: "bottom-center", + theme: "glass", + size: "lg", + platform: "darwin", + }); + + // Update via setKeystroke + const updated = executeAgentTool( + addedDoc, + "setKeystroke", + JSON.stringify({ + keystrokeId, + startSec: 3, + endSec: 5.5, + keys: ["Cmd", "K"], + label: "Search", + position: "top-right", + }), + ); + expect(updated.ok).toBe(true); + const updatedDoc = updated.document as AxcutDocument; + expect(updatedDoc.keystrokeRanges?.[0]).toMatchObject({ + startMs: 3000, + endMs: 5500, + keys: "Cmd+K", + label: "Search", + position: "top-right", + theme: "glass", + }); + + // Verify getCurrentDocument includes keystrokeRanges + const snapshotExec = executeAgentTool(updatedDoc, "getCurrentDocument", "{}"); + expect(snapshotExec.ok).toBe(true); + const snapshot = JSON.parse(snapshotExec.resultJson); + expect(snapshot.keystrokeRanges).toHaveLength(1); + expect(snapshot.keystrokeRanges[0]).toMatchObject({ + id: updatedDoc.keystrokeRanges?.[0]?.id, + startSec: 3, + endSec: 5.5, + keys: "Cmd+K", + label: "Search", + }); + + // Duplicating a clip copies its anchored keystroke ranges + const duplicated = duplicateClip(updatedDoc, "clip_1"); + expect(duplicated.keystrokeRanges).toHaveLength(2); + + // Removing a clip drops its anchored keystroke ranges + const afterClipRemoved = removeClip(duplicated, "clip_1"); + expect(afterClipRemoved.keystrokeRanges).toHaveLength(1); + + // Removing via removeModifier + const remainingId = afterClipRemoved.keystrokeRanges?.[0]?.id ?? ""; + const removed = executeAgentTool( + afterClipRemoved, + "removeModifier", + JSON.stringify({ id: remainingId }), + ); + expect(removed.ok).toBe(true); + expect(removed.document?.keystrokeRanges).toHaveLength(0); + }); +}); diff --git a/src/lib/keystrokes/keystrokeTelemetry.ts b/src/lib/keystrokes/keystrokeTelemetry.ts new file mode 100644 index 000000000..da6205e43 --- /dev/null +++ b/src/lib/keystrokes/keystrokeTelemetry.ts @@ -0,0 +1,353 @@ +import type { AxcutClip, AxcutTrimRange } from "../ai-edition/schema"; +import { locateSourcePosition } from "../ai-edition/timeline/virtual-preview"; + +export type KeystrokePlatform = "darwin" | "win32" | "linux"; + +export interface KeystrokeTelemetrySample { + /** Milliseconds from the start of the recording source. */ + timeMs: number; + /** Key chord tokens in order, e.g. ["Cmd", "Shift", "P"] or ["Ctrl", "C"]. */ + keys: string[]; + /** Optional human-readable command label, e.g. "Command Palette" or "Copy". */ + label?: string; + /** Explicit shortcut flag; when omitted, inferred from modifiers and special keys. */ + isShortcut?: boolean; +} + +export interface KeystrokeTelemetryFile { + version: 1; + platform?: KeystrokePlatform; + samples: KeystrokeTelemetrySample[]; +} + +export interface CoalescedKeystrokeSpan { + startMs: number; + endMs: number; + /** Canonical "+"-joined key combo string, e.g. "Cmd+Shift+P". */ + keys: string; + label?: string; +} + +export interface KeystrokeTrackEvent { + /** Source timestamp in seconds on the asset's recording clock. */ + atSec: number; + /** Corresponding timestamp on the edited timeline, or null if not carried by any clip. */ + virtualSec: number | null; + keys: string; + label?: string; + isShortcut: boolean; + trimmed?: true; +} + +export interface KeystrokeTrack { + assetId: string; + sampleCount: number; + eventCount: number; + events: KeystrokeTrackEvent[]; +} + +const MODIFIER_CANONICAL: Record = { + cmd: "Cmd", + command: "Cmd", + meta: "Cmd", + super: "Win", + win: "Win", + windows: "Win", + "⌘": "Cmd", + ctrl: "Ctrl", + control: "Ctrl", + "⌃": "Ctrl", + alt: "Alt", + opt: "Option", + option: "Option", + "⌥": "Option", + shift: "Shift", + "⇧": "Shift", + commandorcontrol: "CmdOrCtrl", + cmdorctrl: "CmdOrCtrl", +}; + +const SPECIAL_KEY_CANONICAL: Record = { + enter: "Enter", + return: "Enter", + "↵": "Enter", + esc: "Esc", + escape: "Esc", + tab: "Tab", + "⇥": "Tab", + space: "Space", + " ": "Space", + backspace: "Backspace", + "⌫": "Backspace", + delete: "Delete", + del: "Delete", + "⌦": "Delete", + up: "Up", + arrowup: "Up", + "↑": "Up", + down: "Down", + arrowdown: "Down", + "↓": "Down", + left: "Left", + arrowleft: "Left", + "←": "Left", + right: "Right", + arrowright: "Right", + "→": "Right", + home: "Home", + end: "End", + pageup: "PageUp", + pgup: "PageUp", + pagedown: "PageDown", + pgdn: "PageDown", + plus: "+", +}; + +const MODIFIER_SET = new Set(["Cmd", "Win", "Ctrl", "Alt", "Option", "Shift", "CmdOrCtrl"]); + +const STANDALONE_SHORTCUT_KEY_SET = new Set([ + "Enter", + "Esc", + "Tab", + "F1", + "F2", + "F3", + "F4", + "F5", + "F6", + "F7", + "F8", + "F9", + "F10", + "F11", + "F12", +]); + +/** + * Normalize a single key token into a canonical form ("Cmd", "Ctrl", "Shift", "P", "Enter", etc.). + */ +export function normalizeKeyToken(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed) return ""; + const lower = trimmed.toLowerCase(); + if (MODIFIER_CANONICAL[lower]) return MODIFIER_CANONICAL[lower]; + if (SPECIAL_KEY_CANONICAL[lower]) return SPECIAL_KEY_CANONICAL[lower]; + if (/^f([1-9]|1[0-2])$/i.test(trimmed)) { + return trimmed.toUpperCase(); + } + if (trimmed.length === 1) { + return trimmed.toUpperCase(); + } + return trimmed; +} + +/** + * Parse either a "+"-separated shortcut string (e.g. "Cmd + Shift + P", "Ctrl++") + * or an array of key strings into canonical key tokens. + */ +export function parseShortcutKeys(input: string | readonly string[]): string[] { + if (Array.isArray(input)) { + return input.map((k) => normalizeKeyToken(String(k))).filter(Boolean); + } + const raw = String(input).trim(); + if (!raw) return []; + // Handle trailing "+" key (e.g. "Cmd++" or "Ctrl + +") + const EndsWithPlusKey = raw.endsWith("++") || /\+\s*\+$/.test(raw); + const base = EndsWithPlusKey ? raw.replace(/\+\s*\+$/, "+Plus") : raw; + return base + .split("+") + .map((part) => normalizeKeyToken(part)) + .filter(Boolean); +} + +/** + * Format a canonical key token for display inside a keycap HUD badge on the given platform. + */ +export function formatKeycapLabel(token: string, platform: KeystrokePlatform = "win32"): string { + const norm = normalizeKeyToken(token); + if (platform === "darwin") { + switch (norm) { + case "Cmd": + case "Win": + case "CmdOrCtrl": + return "⌘"; + case "Shift": + return "⇧"; + case "Option": + case "Alt": + return "⌥"; + case "Ctrl": + return "⌃"; + case "Enter": + return "↵"; + case "Backspace": + return "⌫"; + case "Delete": + return "⌦"; + case "Tab": + return "⇥"; + case "Esc": + return "⎋"; + case "Up": + return "↑"; + case "Down": + return "↓"; + case "Left": + return "←"; + case "Right": + return "→"; + default: + return norm; + } + } + switch (norm) { + case "CmdOrCtrl": + case "Cmd": + return "Ctrl"; + case "Option": + return "Alt"; + case "Up": + return "↑"; + case "Down": + return "↓"; + case "Left": + return "←"; + case "Right": + return "→"; + default: + return norm; + } +} + +/** + * Format a shortcut combo string or token list into platform-aware keycap display labels. + */ +export function formatShortcutKeycaps( + input: string | readonly string[], + platform: KeystrokePlatform = "win32", +): string[] { + return parseShortcutKeys(input).map((token) => formatKeycapLabel(token, platform)); +} + +/** + * Determine whether a telemetry sample represents an intentional shortcut / command key + * rather than plain prose / password typing (privacy-safe filtering). + */ +export function isShortcutSample(sample: KeystrokeTelemetrySample): boolean { + if (typeof sample.isShortcut === "boolean") { + return sample.isShortcut; + } + const tokens = parseShortcutKeys(sample.keys); + if (tokens.length === 0) return false; + const hasModifier = tokens.some((t) => MODIFIER_SET.has(t)); + const hasNonModifier = tokens.some((t) => !MODIFIER_SET.has(t)); + if (hasModifier && hasNonModifier) return true; + if (tokens.length === 1 && STANDALONE_SHORTCUT_KEY_SET.has(tokens[0])) return true; + return false; +} + +/** + * Candidate sidecar paths for a recorded video file (`