feat(editor): add keystroke telemetry helpers & shortcut HUD visualization (#1317)

electron/ai-edition/agent-tools.test.ts

@@ -195,6 +195,7 @@ describe("the mutating-tool table", () => {
 				"addAudio",
 				"addCameraFullscreen",
 				"addEffect",
+				"addKeystroke",
 				"addSpeed",
 				"addTrim",
 				"addTrims",
@@ -217,6 +218,7 @@ describe("the mutating-tool table", () => {
 				"setEffect",
 				"setEffectDefinition",
 				"setEditorSettings",
+				"setKeystroke",
 				"setSpeed",
 				"setTrim",
 				"setWordText",

electron/ai-edition/agent-tools.ts

@@ -47,11 +47,16 @@ import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcrip
 import {
 	type AxcutAnnotationRegion,
 	type AxcutDocument,
+	type AxcutKeystrokeRange,
 	type AxcutShaderEffect,
 	type AxcutShaderEffectRange,
 	blurDataSchema,
 	clipCropRegionSchema,
 	figureDataSchema,
+	keystrokePlatformSchema,
+	keystrokePositionSchema,
+	keystrokeSizeSchema,
+	keystrokeThemeSchema,
 	shaderEffectParamSchema,
 	shaderEffectSchema,
 	shaderEffectValueSchema,
@@ -96,6 +101,7 @@ import {
 	transitionCutsMs,
 	zoomTransitions,
 } from "../../src/lib/ai-edition/timeline/zoom-transitions";
+import { parseShortcutKeys } from "../../src/lib/keystrokes/keystrokeTelemetry";
 import { SETTING_BOUNDS } from "../../src/lib/projectDefaults";
 import type { SegmentationSupport } from "../../src/native/contracts";
 import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription";
@@ -150,6 +156,8 @@ function modifierIds(document: AxcutDocument, kind: ModifierKind): string[] {
 			return [...new Set(document.audioTracks.map(trackGroupId))];
 		case "shaderEffect":
 			return (document.shaderEffectRanges ?? []).map((region) => region.id);
+		case "keystroke":
+			return (document.keystrokeRanges ?? []).map((region) => region.id);
 	}
 }
 
@@ -606,6 +614,7 @@ function modifierIdsOf(document: AxcutDocument): string[] {
 		...speedRegions.map((r) => r.id),
 		...cameraFullscreenRegions.map((r) => r.id),
 		...(document.shaderEffectRanges ?? []).map((r) => r.id),
+		...(document.keystrokeRanges ?? []).map((r) => r.id),
 	];
 }
 
@@ -1048,6 +1057,33 @@ export const setCameraFullscreenArgs = z.object({
 	endSec: secondsSchema.optional(),
 });
 
+const shortcutKeysInputSchema = z
+	.union([z.string().min(1), z.array(z.string().min(1)).min(1).max(8)])
+	.describe('Shortcut keys, e.g. "Cmd+Shift+P" or ["Cmd", "K"].');
+
+export const addKeystrokeArgs = z.object({
+	startSec: secondsSchema,
+	endSec: secondsSchema,
+	keys: shortcutKeysInputSchema,
+	label: z.string().max(120).optional().describe("Optional action label, e.g. 'Command Palette'."),
+	position: keystrokePositionSchema.default("bottom-center"),
+	theme: keystrokeThemeSchema.default("dark"),
+	size: keystrokeSizeSchema.default("md"),
+	platform: keystrokePlatformSchema.default("win32"),
+});
+
+export const setKeystrokeArgs = z.object({
+	keystrokeId: z.string().min(1),
+	startSec: secondsSchema.optional(),
+	endSec: secondsSchema.optional(),
+	keys: shortcutKeysInputSchema.optional(),
+	label: z.string().max(120).optional(),
+	position: keystrokePositionSchema.optional(),
+	theme: keystrokeThemeSchema.optional(),
+	size: keystrokeSizeSchema.optional(),
+	platform: keystrokePlatformSchema.optional(),
+});
+
 export const getTranscriptWordsArgs = z.object({
 	assetId: z.string().min(1).optional(),
 	startSec: secondsSchema.optional(),
@@ -1117,6 +1153,8 @@ export const OPENSCREEN_TOOL_NAMES = [
 	"setAnnotation",
 	"addCameraFullscreen",
 	"setCameraFullscreen",
+	"addKeystroke",
+	"setKeystroke",
 	"addEffect",
 	"setEffect",
 	"setEffectDefinition",
@@ -1199,6 +1237,8 @@ export const MUTATING_TOOL_NAMES: ReadonlySet<string> = new Set([
 	"setAnnotation",
 	"addCameraFullscreen",
 	"setCameraFullscreen",
+	"addKeystroke",
+	"setKeystroke",
 	"addEffect",
 	"setEffect",
 	"setEffectDefinition",
@@ -1470,6 +1510,18 @@ export function documentSnapshotForModel(
 			startSec: roundSec(c.startMs),
 			endSec: roundSec(c.endMs),
 		})),
+		keystrokeRanges: coalesceForAgent(document.keystrokeRanges ?? []).map((k) => ({
+			id: k.id,
+			startSec: roundSec(k.startMs),
+			endSec: roundSec(k.endMs),
+			keys: k.keys,
+			...(k.label ? { label: k.label } : {}),
+			position: k.position,
+			theme: k.theme,
+			size: k.size,
+			platform: k.platform,
+			origin: k.origin,
+		})),
 		// Where each effect is posed, and how. The code is not here: getEffect reads it.
 		effects: coalesceForAgent(document.shaderEffectRanges ?? []).map((r) => ({
 			id: r.id,
@@ -3608,6 +3660,121 @@ export function executeAgentTool(
 			};
 		}
 
+		case "addKeystroke": {
+			const parsed = addKeystrokeArgs.safeParse(args);
+			if (!parsed.success) return failure(parsed.error.message);
+			const tokens = parseShortcutKeys(parsed.data.keys);
+			if (tokens.length === 0) return failure("Keystroke shortcut cannot be empty.");
+			const canonicalKeys = tokens.join("+");
+			const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec));
+			const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec));
+			const region: AxcutKeystrokeRange = {
+				id: createId("keys"),
+				startMs,
+				endMs,
+				keys: canonicalKeys,
+				...(parsed.data.label ? { label: parsed.data.label } : {}),
+				position: parsed.data.position,
+				theme: parsed.data.theme,
+				size: parsed.data.size,
+				platform: parsed.data.platform,
+				origin: "agent",
+			};
+			const placed = anchorForAgent(region, document, "keys");
+			const landing = landingOf(placed, document);
+			if (!landing.anchored) {
+				return coversNoClip("keystroke callout", startMs / 1000, endMs / 1000, document);
+			}
+			const next: AxcutDocument = {
+				...document,
+				keystrokeRanges: [...(document.keystrokeRanges ?? []), ...placed],
+			};
+			return {
+				ok: true,
+				document: next,
+				resultJson: JSON.stringify({
+					keystrokeId: landing.ids[0],
+					keys: canonicalKeys,
+					...(parsed.data.label ? { label: parsed.data.label } : {}),
+					position: parsed.data.position,
+					theme: parsed.data.theme,
+					size: parsed.data.size,
+					platform: parsed.data.platform,
+					...landingReport(landing, startMs / 1000, endMs / 1000),
+				}),
+				summary:
+					`shortcut [${canonicalKeys}] ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
+					landingSuffix(landing, startMs / 1000, endMs / 1000),
+			};
+		}
+
+		case "setKeystroke": {
+			const parsed = setKeystrokeArgs.safeParse(args);
+			if (!parsed.success) return failure(parsed.error.message);
+			const prev = document.keystrokeRanges ?? [];
+			const existing = prev.find((r) => r.id === parsed.data.keystrokeId);
+			if (!existing) return failure(`Unknown keystroke region: ${parsed.data.keystrokeId}`);
+			const nextKeys =
+				parsed.data.keys !== undefined ? parseShortcutKeys(parsed.data.keys).join("+") : undefined;
+			if (nextKeys !== undefined && !nextKeys) {
+				return failure("Keystroke shortcut cannot be empty.");
+			}
+			const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec);
+			const blocked = refuseIfBlocked("keystroke callout", prev, parsed.data.keystrokeId, {
+				startMs,
+				endMs,
+			});
+			if (blocked) return blocked;
+			const keystrokePill = new Set(resolvePillIds(prev, parsed.data.keystrokeId));
+			const patched = prev.map((r) =>
+				keystrokePill.has(r.id)
+					? {
+							...r,
+							...(nextKeys !== undefined ? { keys: nextKeys } : {}),
+							...(parsed.data.label !== undefined ? { label: parsed.data.label || undefined } : {}),
+							...(parsed.data.position !== undefined ? { position: parsed.data.position } : {}),
+							...(parsed.data.theme !== undefined ? { theme: parsed.data.theme } : {}),
+							...(parsed.data.size !== undefined ? { size: parsed.data.size } : {}),
+							...(parsed.data.platform !== undefined ? { platform: parsed.data.platform } : {}),
+						}
+					: r,
+			);
+			const rebuilt = replacePillSpan(
+				patched,
+				parsed.data.keystrokeId,
+				startMs,
+				endMs,
+				document.timeline.clips,
+				() => createId("keys"),
+			);
+			const landing = landingAfterPillEdit(prev, rebuilt, keystrokePill, document);
+			if (!landing.anchored) {
+				return coversNoClip("keystroke callout", startMs / 1000, endMs / 1000, document);
+			}
+			const next: AxcutDocument = {
+				...document,
+				keystrokeRanges: rebuilt,
+			};
+			const landed = rebuilt.find((r) => r.id === landing.ids[0]);
+			return {
+				ok: true,
+				document: next,
+				resultJson: JSON.stringify({
+					keystrokeId: landing.ids[0] ?? parsed.data.keystrokeId,
+					keys: landed?.keys ?? existing.keys,
+					...(landed?.label ? { label: landed.label } : {}),
+					position: landed?.position ?? existing.position,
+					theme: landed?.theme ?? existing.theme,
+					size: landed?.size ?? existing.size,
+					platform: landed?.platform ?? existing.platform,
+					...landingReport(landing, startMs / 1000, endMs / 1000),
+				}),
+				summary:
+					`updated shortcut [${landed?.keys ?? existing.keys}] ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` +
+					landingSuffix(landing, startMs / 1000, endMs / 1000),
+			};
+		}
+
 		case "addAudio": {
 			const parsed = addAudioArgs.safeParse(args);
 			if (!parsed.success) return failure(parsed.error.message);
@@ -3802,9 +3969,10 @@ export function executeAgentTool(
 			else if (cameraFullscreenRegions.some((c) => c.id === id)) kind = "cameraFullscreen";
 			else if (document.audioTracks.some((t) => trackGroupId(t) === id)) kind = "audio";
 			else if ((document.shaderEffectRanges ?? []).some((r) => r.id === id)) kind = "shaderEffect";
+			else if ((document.keystrokeRanges ?? []).some((r) => r.id === id)) kind = "keystroke";
 			if (!kind) {
 				return failure(
-					`No zoom / speed / annotation / full-camera / audio / effect modifier with id ${id}. ` +
+					`No zoom / speed / annotation / full-camera / keystroke / audio / effect modifier with id ${id}. ` +
 						`For a trim use removeTrim; for a clip use removeClip.`,
 				);
 			}

electron/ai-edition/deep-agent/service.test.ts

@@ -83,6 +83,8 @@ const ARGS: Record<string, unknown> = {
 	setAnnotation: { annotationId: "ann_nope" },
 	addCameraFullscreen: { startSec: 1, endSec: 2 },
 	setCameraFullscreen: { cameraFullscreenId: "cam_nope" },
+	addKeystroke: { startSec: 1, endSec: 2, keys: ["Cmd", "K"] },
+	setKeystroke: { keystrokeId: "keys_nope" },
 	// No checker in these tests: the effect is stored unchecked, and the result says so.
 	addEffect: {
 		startSec: 1,

electron/ai-edition/deep-agent/service.ts

@@ -57,6 +57,7 @@ import {
 	addAutoZoomsArgs,
 	addCameraFullscreenArgs,
 	addEffectArgs,
+	addKeystrokeArgs,
 	addSpeedArgs,
 	addTrimArgs,
 	addTrimsArgs,
@@ -85,6 +86,7 @@ import {
 	setClipRangeArgs,
 	setEffectArgs,
 	setEffectDefinitionArgs,
+	setKeystrokeArgs,
 	setSpeedArgs,
 	setTrimArgs,
 	setWordTextArgs,
@@ -274,12 +276,16 @@ export const TOOL_DESCRIPTIONS: Record<string, string> = {
 		"Read one effect of the project in full: its WGSL code, parameters, description, cost and the regions that use it. getCurrentDocument lists effects without their code.",
 	setCameraFullscreen:
 		"Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.",
+	addKeystroke:
+		"Place a stylized keyboard shortcut HUD callout over a span of the edited timeline (virtual seconds). Pass `keys` as an array of key tokens or '+'-separated string (e.g. ['Cmd', 'K'] or 'Ctrl+Shift+P'), plus optional `label` (e.g. 'Command Palette'), `position` ('bottom-center', 'bottom-left', 'bottom-right', 'top-center', 'top-left', 'top-right'), `theme` ('dark', 'light', 'glass'), `size` ('sm', 'md', 'lg'), and `platform` ('darwin', 'win32', 'linux', default 'win32').",
+	setKeystroke:
+		"Move, resize, or restyle an existing keyboard shortcut HUD callout by `keystrokeId` (virtual-timeline seconds). Only the fields you pass are changed.",
 	addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them. If there is none, a file has to be imported first (importMedia, where it is offered): never guess an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`,
 	setAudio: `Move, resize, re-level, re-lane, mute, loop, fade or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to fade it in or out (fadeInMs / fadeOutMs, 0 to ${AUDIO_FADE_MAX_MS} ms), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.`,
 	removeTrim:
 		"Delete a trim range by id — the cut is undone and that span plays/exports again. This is how you 'remove a trim'; never re-add a trim to undo one.",
 	removeModifier:
-		"Delete a modifier (zoom / speed / annotation / camera-fullscreen / audio / effect) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.",
+		"Delete a modifier (zoom / speed / annotation / camera-fullscreen / keystroke / audio / effect) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.",
 	removeClip:
 		"Delete a placed clip by id; remaining clips close the gap and effects anchored to it are dropped. Use only when the user asks to remove a clip — to shorten one, use setClipRange.",
 	getCaptions:
@@ -558,6 +564,8 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray<readonly [string, z.ZodObject]> = [
 	["setAnnotation", setAnnotationArgs],
 	["addCameraFullscreen", addCameraFullscreenArgs],
 	["setCameraFullscreen", setCameraFullscreenArgs],
+	["addKeystroke", addKeystrokeArgs],
+	["setKeystroke", setKeystrokeArgs],
 	["addEffect", addEffectArgs],
 	["setEffect", setEffectArgs],
 	["setEffectDefinition", setEffectDefinitionArgs],