diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index c105d2e4b..c666abaeb 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -195,6 +195,7 @@ describe("the mutating-tool table", () => { "addAudio", "addCameraFullscreen", "addEffect", + "addKeystroke", "addSpeed", "addTrim", "addTrims", @@ -217,6 +218,7 @@ describe("the mutating-tool table", () => { "setEffect", "setEffectDefinition", "setEditorSettings", + "setKeystroke", "setSpeed", "setTrim", "setWordText", diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 50bb5de41..0c7b3a781 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -47,11 +47,16 @@ import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcrip import { type AxcutAnnotationRegion, type AxcutDocument, + type AxcutKeystrokeRange, type AxcutShaderEffect, type AxcutShaderEffectRange, blurDataSchema, clipCropRegionSchema, figureDataSchema, + keystrokePlatformSchema, + keystrokePositionSchema, + keystrokeSizeSchema, + keystrokeThemeSchema, shaderEffectParamSchema, shaderEffectSchema, shaderEffectValueSchema, @@ -96,6 +101,7 @@ import { transitionCutsMs, zoomTransitions, } from "../../src/lib/ai-edition/timeline/zoom-transitions"; +import { parseShortcutKeys } from "../../src/lib/keystrokes/keystrokeTelemetry"; import { SETTING_BOUNDS } from "../../src/lib/projectDefaults"; import type { SegmentationSupport } from "../../src/native/contracts"; import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription"; @@ -150,6 +156,8 @@ function modifierIds(document: AxcutDocument, kind: ModifierKind): string[] { return [...new Set(document.audioTracks.map(trackGroupId))]; case "shaderEffect": return (document.shaderEffectRanges ?? []).map((region) => region.id); + case "keystroke": + return (document.keystrokeRanges ?? []).map((region) => region.id); } } @@ -606,6 +614,7 @@ function modifierIdsOf(document: AxcutDocument): string[] { ...speedRegions.map((r) => r.id), ...cameraFullscreenRegions.map((r) => r.id), ...(document.shaderEffectRanges ?? []).map((r) => r.id), + ...(document.keystrokeRanges ?? []).map((r) => r.id), ]; } @@ -1048,6 +1057,33 @@ export const setCameraFullscreenArgs = z.object({ endSec: secondsSchema.optional(), }); +const shortcutKeysInputSchema = z + .union([z.string().min(1), z.array(z.string().min(1)).min(1).max(8)]) + .describe('Shortcut keys, e.g. "Cmd+Shift+P" or ["Cmd", "K"].'); + +export const addKeystrokeArgs = z.object({ + startSec: secondsSchema, + endSec: secondsSchema, + keys: shortcutKeysInputSchema, + label: z.string().max(120).optional().describe("Optional action label, e.g. 'Command Palette'."), + position: keystrokePositionSchema.default("bottom-center"), + theme: keystrokeThemeSchema.default("dark"), + size: keystrokeSizeSchema.default("md"), + platform: keystrokePlatformSchema.default("win32"), +}); + +export const setKeystrokeArgs = z.object({ + keystrokeId: z.string().min(1), + startSec: secondsSchema.optional(), + endSec: secondsSchema.optional(), + keys: shortcutKeysInputSchema.optional(), + label: z.string().max(120).optional(), + position: keystrokePositionSchema.optional(), + theme: keystrokeThemeSchema.optional(), + size: keystrokeSizeSchema.optional(), + platform: keystrokePlatformSchema.optional(), +}); + export const getTranscriptWordsArgs = z.object({ assetId: z.string().min(1).optional(), startSec: secondsSchema.optional(), @@ -1117,6 +1153,8 @@ export const OPENSCREEN_TOOL_NAMES = [ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addKeystroke", + "setKeystroke", "addEffect", "setEffect", "setEffectDefinition", @@ -1199,6 +1237,8 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addKeystroke", + "setKeystroke", "addEffect", "setEffect", "setEffectDefinition", @@ -1470,6 +1510,18 @@ export function documentSnapshotForModel( startSec: roundSec(c.startMs), endSec: roundSec(c.endMs), })), + keystrokeRanges: coalesceForAgent(document.keystrokeRanges ?? []).map((k) => ({ + id: k.id, + startSec: roundSec(k.startMs), + endSec: roundSec(k.endMs), + keys: k.keys, + ...(k.label ? { label: k.label } : {}), + position: k.position, + theme: k.theme, + size: k.size, + platform: k.platform, + origin: k.origin, + })), // Where each effect is posed, and how. The code is not here: getEffect reads it. effects: coalesceForAgent(document.shaderEffectRanges ?? []).map((r) => ({ id: r.id, @@ -3608,6 +3660,121 @@ export function executeAgentTool( }; } + case "addKeystroke": { + const parsed = addKeystrokeArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const tokens = parseShortcutKeys(parsed.data.keys); + if (tokens.length === 0) return failure("Keystroke shortcut cannot be empty."); + const canonicalKeys = tokens.join("+"); + const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec)); + const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec)); + const region: AxcutKeystrokeRange = { + id: createId("keys"), + startMs, + endMs, + keys: canonicalKeys, + ...(parsed.data.label ? { label: parsed.data.label } : {}), + position: parsed.data.position, + theme: parsed.data.theme, + size: parsed.data.size, + platform: parsed.data.platform, + origin: "agent", + }; + const placed = anchorForAgent(region, document, "keys"); + const landing = landingOf(placed, document); + if (!landing.anchored) { + return coversNoClip("keystroke callout", startMs / 1000, endMs / 1000, document); + } + const next: AxcutDocument = { + ...document, + keystrokeRanges: [...(document.keystrokeRanges ?? []), ...placed], + }; + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + keystrokeId: landing.ids[0], + keys: canonicalKeys, + ...(parsed.data.label ? { label: parsed.data.label } : {}), + position: parsed.data.position, + theme: parsed.data.theme, + size: parsed.data.size, + platform: parsed.data.platform, + ...landingReport(landing, startMs / 1000, endMs / 1000), + }), + summary: + `shortcut [${canonicalKeys}] ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + + landingSuffix(landing, startMs / 1000, endMs / 1000), + }; + } + + case "setKeystroke": { + const parsed = setKeystrokeArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const prev = document.keystrokeRanges ?? []; + const existing = prev.find((r) => r.id === parsed.data.keystrokeId); + if (!existing) return failure(`Unknown keystroke region: ${parsed.data.keystrokeId}`); + const nextKeys = + parsed.data.keys !== undefined ? parseShortcutKeys(parsed.data.keys).join("+") : undefined; + if (nextKeys !== undefined && !nextKeys) { + return failure("Keystroke shortcut cannot be empty."); + } + const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec); + const blocked = refuseIfBlocked("keystroke callout", prev, parsed.data.keystrokeId, { + startMs, + endMs, + }); + if (blocked) return blocked; + const keystrokePill = new Set(resolvePillIds(prev, parsed.data.keystrokeId)); + const patched = prev.map((r) => + keystrokePill.has(r.id) + ? { + ...r, + ...(nextKeys !== undefined ? { keys: nextKeys } : {}), + ...(parsed.data.label !== undefined ? { label: parsed.data.label || undefined } : {}), + ...(parsed.data.position !== undefined ? { position: parsed.data.position } : {}), + ...(parsed.data.theme !== undefined ? { theme: parsed.data.theme } : {}), + ...(parsed.data.size !== undefined ? { size: parsed.data.size } : {}), + ...(parsed.data.platform !== undefined ? { platform: parsed.data.platform } : {}), + } + : r, + ); + const rebuilt = replacePillSpan( + patched, + parsed.data.keystrokeId, + startMs, + endMs, + document.timeline.clips, + () => createId("keys"), + ); + const landing = landingAfterPillEdit(prev, rebuilt, keystrokePill, document); + if (!landing.anchored) { + return coversNoClip("keystroke callout", startMs / 1000, endMs / 1000, document); + } + const next: AxcutDocument = { + ...document, + keystrokeRanges: rebuilt, + }; + const landed = rebuilt.find((r) => r.id === landing.ids[0]); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + keystrokeId: landing.ids[0] ?? parsed.data.keystrokeId, + keys: landed?.keys ?? existing.keys, + ...(landed?.label ? { label: landed.label } : {}), + position: landed?.position ?? existing.position, + theme: landed?.theme ?? existing.theme, + size: landed?.size ?? existing.size, + platform: landed?.platform ?? existing.platform, + ...landingReport(landing, startMs / 1000, endMs / 1000), + }), + summary: + `updated shortcut [${landed?.keys ?? existing.keys}] ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + + landingSuffix(landing, startMs / 1000, endMs / 1000), + }; + } + case "addAudio": { const parsed = addAudioArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); @@ -3802,9 +3969,10 @@ export function executeAgentTool( else if (cameraFullscreenRegions.some((c) => c.id === id)) kind = "cameraFullscreen"; else if (document.audioTracks.some((t) => trackGroupId(t) === id)) kind = "audio"; else if ((document.shaderEffectRanges ?? []).some((r) => r.id === id)) kind = "shaderEffect"; + else if ((document.keystrokeRanges ?? []).some((r) => r.id === id)) kind = "keystroke"; if (!kind) { return failure( - `No zoom / speed / annotation / full-camera / audio / effect modifier with id ${id}. ` + + `No zoom / speed / annotation / full-camera / keystroke / audio / effect modifier with id ${id}. ` + `For a trim use removeTrim; for a clip use removeClip.`, ); } diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 0a368e95b..6c9746266 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -83,6 +83,8 @@ const ARGS: Record = { setAnnotation: { annotationId: "ann_nope" }, addCameraFullscreen: { startSec: 1, endSec: 2 }, setCameraFullscreen: { cameraFullscreenId: "cam_nope" }, + addKeystroke: { startSec: 1, endSec: 2, keys: ["Cmd", "K"] }, + setKeystroke: { keystrokeId: "keys_nope" }, // No checker in these tests: the effect is stored unchecked, and the result says so. addEffect: { startSec: 1, diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 00fc819ee..c62dc550d 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -57,6 +57,7 @@ import { addAutoZoomsArgs, addCameraFullscreenArgs, addEffectArgs, + addKeystrokeArgs, addSpeedArgs, addTrimArgs, addTrimsArgs, @@ -85,6 +86,7 @@ import { setClipRangeArgs, setEffectArgs, setEffectDefinitionArgs, + setKeystrokeArgs, setSpeedArgs, setTrimArgs, setWordTextArgs, @@ -274,12 +276,16 @@ export const TOOL_DESCRIPTIONS: Record = { "Read one effect of the project in full: its WGSL code, parameters, description, cost and the regions that use it. getCurrentDocument lists effects without their code.", setCameraFullscreen: "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.", + addKeystroke: + "Place a stylized keyboard shortcut HUD callout over a span of the edited timeline (virtual seconds). Pass `keys` as an array of key tokens or '+'-separated string (e.g. ['Cmd', 'K'] or 'Ctrl+Shift+P'), plus optional `label` (e.g. 'Command Palette'), `position` ('bottom-center', 'bottom-left', 'bottom-right', 'top-center', 'top-left', 'top-right'), `theme` ('dark', 'light', 'glass'), `size` ('sm', 'md', 'lg'), and `platform` ('darwin', 'win32', 'linux', default 'win32').", + setKeystroke: + "Move, resize, or restyle an existing keyboard shortcut HUD callout by `keystrokeId` (virtual-timeline seconds). Only the fields you pass are changed.", addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them. If there is none, a file has to be imported first (importMedia, where it is offered): never guess an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`, setAudio: `Move, resize, re-level, re-lane, mute, loop, fade or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to fade it in or out (fadeInMs / fadeOutMs, 0 to ${AUDIO_FADE_MAX_MS} ms), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.`, removeTrim: "Delete a trim range by id — the cut is undone and that span plays/exports again. This is how you 'remove a trim'; never re-add a trim to undo one.", removeModifier: - "Delete a modifier (zoom / speed / annotation / camera-fullscreen / audio / effect) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.", + "Delete a modifier (zoom / speed / annotation / camera-fullscreen / keystroke / audio / effect) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.", removeClip: "Delete a placed clip by id; remaining clips close the gap and effects anchored to it are dropped. Use only when the user asks to remove a clip — to shorten one, use setClipRange.", getCaptions: @@ -558,6 +564,8 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ ["setAnnotation", setAnnotationArgs], ["addCameraFullscreen", addCameraFullscreenArgs], ["setCameraFullscreen", setCameraFullscreenArgs], + ["addKeystroke", addKeystrokeArgs], + ["setKeystroke", setKeystrokeArgs], ["addEffect", addEffectArgs], ["setEffect", setEffectArgs], ["setEffectDefinition", setEffectDefinitionArgs], diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.ts index 52e1bb97e..119a4214e 100644 --- a/electron/ai-edition/document-service.ts +++ b/electron/ai-edition/document-service.ts @@ -27,6 +27,11 @@ import { type StylePresetAppearance, stylePresetLegacyEditor, } from "../../src/lib/ai-edition/stylePresets"; +import { + getKeystrokeTelemetryCandidates, + parseKeystrokeSidecar, + seedKeystrokeRangesFromTelemetry, +} from "../../src/lib/keystrokes/keystrokeTelemetry"; import { renameSyncWithFallback, renameWithRetry } from "../atomic-rename"; import { ensureDocumentExtensions } from "../media/extensionClip"; import { relinkProjectMedia } from "../media/projectMediaRelinker"; @@ -248,6 +253,8 @@ export class DocumentService { private readonly writeQueues = new Map>(); /** Bumped as each read or save of a project starts — see saveProjectIfUnchanged. */ private readonly generations = new Map(); + /** Projects that have already completed initial keystroke sidecar seeding on import. */ + private readonly keystrokeSeededProjects = new Set(); // `mediaRegistryDir` is where the media-links registry file lives // (RECORDINGS_DIR in production) — see getProject. Injected for the same @@ -522,7 +529,18 @@ export class DocumentService { } async saveProject(document: AxcutDocument): Promise { - const stamped = stampForSave(document); + let nextDoc = document; + if ((nextDoc.keystrokeRanges?.length ?? 0) > 0) { + this.keystrokeSeededProjects.add(nextDoc.project.id); + } else if ( + !this.keystrokeSeededProjects.has(nextDoc.project.id) && + nextDoc.timeline.clips.length === 1 && + nextDoc.timeline.clips[0]?.reason === "Auto-imported recording" + ) { + this.keystrokeSeededProjects.add(nextDoc.project.id); + nextDoc = await this.seedKeystrokeSidecarIfPresent(nextDoc); + } + const stamped = stampForSave(nextDoc); this.touch(stamped.project.id); await this.writeProject(stamped); // After the write, never before: a derived file is not worth delaying the user's edit @@ -531,6 +549,37 @@ export class DocumentService { return stamped; } + private async seedKeystrokeSidecarIfPresent(doc: AxcutDocument): Promise { + const primary = + doc.assets.find((a) => a.id === doc.project.primaryAssetId) ?? + doc.assets.find((a) => a.kind !== "audio"); + if (!primary) return doc; + const candidates = [ + ...(primary.keystrokeTelemetryPath ? [primary.keystrokeTelemetryPath] : []), + ...(primary.originalPath ? getKeystrokeTelemetryCandidates(primary.originalPath) : []), + ]; + for (const candidate of candidates) { + try { + const raw = await fs.readFile(candidate, "utf8"); + const sidecar = parseKeystrokeSidecar(raw); + if (sidecar.samples.length > 0) { + const seeded = seedKeystrokeRangesFromTelemetry( + doc, + sidecar.samples, + () => createId("key"), + sidecar.platform, + ); + if ((seeded.keystrokeRanges?.length ?? 0) > 0) { + return seeded; + } + } + } catch { + // Sidecar not present at this candidate path; try the next one. + } + } + return doc; + } + /** * `saveProject` for a writer that does not hold the project open (the MCP * server), unless anything read or saved the project since the @@ -590,6 +639,18 @@ export class DocumentService { } catch { sizeBytes = undefined; } + let keystrokeTelemetryPath: string | undefined; + if ((input.kind ?? "video") === "video") { + for (const candidate of getKeystrokeTelemetryCandidates(absolutePath)) { + try { + await fs.access(candidate); + keystrokeTelemetryPath = candidate; + break; + } catch { + // Not found at this candidate path. + } + } + } const asset: AxcutAsset = { id: createId("asset"), kind: input.kind ?? "video", @@ -597,6 +658,7 @@ export class DocumentService { originalPath: absolutePath, sizeBytes, cameraTrack: null, + ...(keystrokeTelemetryPath ? { keystrokeTelemetryPath } : {}), }; return this.saveProject(documentWithAsset(doc, asset)); } diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index 13e5c1c47..7c95d5ffd 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -1075,7 +1075,9 @@ export function NewEditorShell() { ? "ann" : snapshot.kind === "shaderEffect" ? "fxr" - : snapshot.kind; + : snapshot.kind === "keystroke" + ? "key" + : snapshot.kind; // Audio re-ventilates through its own anchorer, which advances each // fragment's source offset — the generic one would copy the offset into @@ -1142,6 +1144,18 @@ export function NewEditorShell() { return; } await saveDocument(next, { history: true }); + } else if (snapshot.kind === "keystroke") { + const saved = await saveDocument( + { + ...doc, + keystrokeRanges: [ + ...(doc.keystrokeRanges ?? []), + ...anchored, + ] as typeof doc.keystrokeRanges, + }, + { history: true }, + ); + if (!saved) return; } else { // speed and cameraFullscreen are both plain spans on legacyEditor. const key = snapshot.kind === "speed" ? "speedRegions" : "cameraFullscreenRegions"; @@ -1227,7 +1241,9 @@ export function NewEditorShell() { ? tl.annotationRegions : sel.kind === "speed" ? tl.speedRegions - : tl.cameraFullscreenRegions; + : sel.kind === "keystroke" + ? tl.keystrokeRanges + : tl.cameraFullscreenRegions; const region = wholePillOf( source as Array<{ id: string; startMs: number; endMs: number }>, sel.id, @@ -1644,6 +1660,7 @@ export function NewEditorShell() { zoomRegions={tl.zoomRegions} speedRegions={tl.speedRegions} cameraFullscreenRegions={tl.cameraFullscreenRegions} + keystrokeRanges={tl.keystrokeRanges} trimRanges={tl.trimRanges} selectedZoomRegionId={tl.selection?.kind === "zoom" ? tl.selection.id : null} onZoomFocusChange={tl.updateZoomFocusLive} diff --git a/src/components/ai-edition/Preview.tsx b/src/components/ai-edition/Preview.tsx index f4ad0a5c5..d48dd1d69 100644 --- a/src/components/ai-edition/Preview.tsx +++ b/src/components/ai-edition/Preview.tsx @@ -5,6 +5,7 @@ import type { AxcutAnnotationRegion, AxcutAudioTrack, AxcutClip, + AxcutKeystrokeRange, AxcutTranscript, AxcutTrimRange, AxcutZoomRegion, @@ -38,6 +39,7 @@ interface PreviewProps { zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; + keystrokeRanges?: AxcutKeystrokeRange[]; trimRanges?: AxcutTrimRange[]; transcripts?: AxcutTranscript[]; selectedZoomRegionId?: string | null; @@ -73,6 +75,7 @@ export function Preview({ zoomRegions, speedRegions, cameraFullscreenRegions, + keystrokeRanges, trimRanges, transcripts, selectedZoomRegionId, @@ -216,6 +219,7 @@ export function Preview({ zoomRegions={zoomRegions} speedRegions={speedRegions} cameraFullscreenRegions={cameraFullscreenRegions} + keystrokeRanges={keystrokeRanges} trimRanges={trimRanges} transcripts={transcripts} selectedZoomRegionId={selectedZoomRegionId} diff --git a/src/components/ai-edition/PreviewCanvas.tsx b/src/components/ai-edition/PreviewCanvas.tsx index c817fde3c..ca74ba15e 100644 --- a/src/components/ai-edition/PreviewCanvas.tsx +++ b/src/components/ai-edition/PreviewCanvas.tsx @@ -41,6 +41,7 @@ import type { AxcutAnnotationRegion, AxcutAudioTrack, AxcutClip, + AxcutKeystrokeRange, AxcutTranscript, AxcutTrimRange, AxcutZoomRegion, @@ -57,6 +58,7 @@ import { resolveWebcamLayoutPreset, type WebcamCompositeLayout, } from "@/lib/compositeLayout"; +import { formatShortcutKeycaps } from "@/lib/keystrokes/keystrokeTelemetry"; import { webcamAnchorAt } from "@/lib/projectDefaults"; import { wallpaperStyle } from "@/lib/wallpaper"; import { getCssClipPath } from "@/lib/webcamMaskShapes"; @@ -86,6 +88,7 @@ interface PreviewCanvasProps { zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; + keystrokeRanges?: AxcutKeystrokeRange[]; trimRanges?: AxcutTrimRange[]; /** Relayed to `VirtualPreview` so a clip carrying added words plays their extensions. */ transcripts?: AxcutTranscript[]; @@ -512,6 +515,114 @@ export function PreviewCanvas(props: PreviewCanvasProps) { onCommit={props.onAnnotationCommit} /> ) : null} + {props.keystrokeRanges && props.keystrokeRanges.length > 0 ? ( + + ) : null} + + ); +} + +function KeystrokeHudLayer({ + ranges, + currentTimeSec, +}: { + ranges: AxcutKeystrokeRange[]; + currentTimeSec: number; +}) { + const nowMs = currentTimeSec * 1000; + const active = ranges.find((r) => r.startMs <= nowMs && nowMs < r.endMs) ?? null; + if (!active) return null; + + const keycaps = formatShortcutKeycaps(active.keys, active.platform); + if (keycaps.length === 0) return null; + + const posStyle: React.CSSProperties = { + position: "absolute", + zIndex: 8, + pointerEvents: "none", + display: "flex", + alignItems: "center", + gap: active.size === "sm" ? 6 : active.size === "lg" ? 10 : 8, + padding: active.size === "sm" ? "6px 10px" : active.size === "lg" ? "12px 18px" : "8px 14px", + borderRadius: 14, + ...(active.position.startsWith("bottom") ? { bottom: "6%" } : { top: "6%" }), + ...(active.position.endsWith("center") + ? { left: "50%", transform: "translateX(-50%)" } + : active.position.endsWith("left") + ? { left: "4%" } + : { right: "4%" }), + ...(active.theme === "light" + ? { + background: "rgba(255, 255, 255, 0.92)", + border: "1px solid rgba(0, 0, 0, 0.12)", + boxShadow: "0 10px 28px rgba(0, 0, 0, 0.22)", + color: "#111827", + } + : active.theme === "glass" + ? { + background: "rgba(24, 24, 27, 0.58)", + backdropFilter: "blur(16px)", + border: "1px solid rgba(255, 255, 255, 0.22)", + boxShadow: "0 12px 32px rgba(0, 0, 0, 0.38)", + color: "#f9fafb", + } + : { + background: "rgba(17, 17, 20, 0.92)", + border: "1px solid rgba(255, 255, 255, 0.14)", + boxShadow: "0 12px 32px rgba(0, 0, 0, 0.48)", + color: "#f9fafb", + }), + }; + + const keycapFontPx = active.size === "sm" ? 13 : active.size === "lg" ? 20 : 16; + const keycapPad = + active.size === "sm" ? "3px 8px" : active.size === "lg" ? "6px 13px" : "4px 10px"; + + return ( +
+
+ {keycaps.map((cap, idx) => ( + + {cap} + + ))} +
+ {active.label ? ( + + {active.label} + + ) : null}
); } diff --git a/src/components/ai-edition/v4/FloatingInspector.test.tsx b/src/components/ai-edition/v4/FloatingInspector.test.tsx index 090f87fbc..718075ab3 100644 --- a/src/components/ai-edition/v4/FloatingInspector.test.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.test.tsx @@ -94,6 +94,8 @@ describe("FloatingInspector", () => { tl: { selection: null, clearSelection: vi.fn(), + selectRegion: vi.fn(), + keystrokeRanges: [], selectedAudioTrackId: null, selectAudioTrack: vi.fn(), } as unknown as React.ComponentProps["tl"], diff --git a/src/components/ai-edition/v4/FloatingInspector.tsx b/src/components/ai-edition/v4/FloatingInspector.tsx index b82a0e9cd..5f4064781 100644 --- a/src/components/ai-edition/v4/FloatingInspector.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.tsx @@ -14,6 +14,7 @@ import { EyeOff, FileText, ImageIcon, + Keyboard, type LucideIcon, Maximize2, MousePointer2, @@ -59,6 +60,11 @@ import { resolveAspectRatioValue } from "@/lib/ai-edition/document/outputFormat" import type { AxcutAnnotationRegion, AxcutClip, + AxcutKeystrokePlatform, + AxcutKeystrokePosition, + AxcutKeystrokeRange, + AxcutKeystrokeSize, + AxcutKeystrokeTheme, AxcutShaderEffect, AxcutShaderEffectParam, AxcutShaderEffectRange, @@ -71,6 +77,7 @@ import type { ShaderEffectPatch, useTimeline } from "@/lib/ai-edition/store/useT import { formatSeconds } from "@/lib/ai-edition/timeline/format"; import { coalescedTrimGroups } from "@/lib/ai-edition/timeline/trim-mapping"; import { zoomLevelFor } from "@/lib/ai-edition/timeline/zoom-scale"; +import { parseShortcutKeys } from "@/lib/keystrokes/keystrokeTelemetry"; import { clampToBound } from "@/lib/projectDefaults"; import { checkShaderEffectCode } from "@/native/compositorViewClient"; import { annotationFootageRect, zoomScaleLimit } from "@/native/sceneDescription"; @@ -205,6 +212,28 @@ export function FloatingInspector({ ))} + {(tl.keystrokeRanges?.length ?? 0) > 0 ? ( + + + + ) : null} {/* A portal, like the timeline's menus: drawn inside the stage, the list was clipped by the stage's edge, and the rows past it sat under the timeline, out of reach. */} 1} onOpenChange={setClipPickerOpen}> @@ -1313,6 +1342,21 @@ function SelectionPane({ tl, onClose }: { tl: TimelineApi; onClose: () => void } ); } + if (selection.kind === "keystroke") { + const region = tl.keystrokeRanges.find((r) => r.id === selection.id); + if (!region) return null; + return ( + + ); + } + // trim — a trim ventilated across a clip boundary is 2+ DSL rows that render // as one coalesced pill (see V4Timeline's trimPills), so the DURATION shown has to be // the group's, not the clicked row's. Deleting no longer needs the same expansion here: @@ -1554,6 +1598,175 @@ function ShaderEffectPane({ ); } +function KeystrokeInspectorPane({ + tl, + region, + onClose, + onDelete, + bodyStyle, +}: { + tl: TimelineApi; + region: AxcutKeystrokeRange; + onClose: () => void; + onDelete: () => void; + bodyStyle: React.CSSProperties; +}) { + const tc = useScopedT("common"); + const te = useScopedT("editor"); + const keysInputId = useId(); + const labelInputId = useId(); + const [keysDraft, setKeysDraft] = useState(region.keys); + const [labelDraft, setLabelDraft] = useState(region.label ?? ""); + + useEffect(() => { + setKeysDraft(region.keys); + }, [region.keys]); + + useEffect(() => { + setLabelDraft(region.label ?? ""); + }, [region.label]); + + const commitKeys = () => { + const canonical = parseShortcutKeys(keysDraft).join("+"); + if (!canonical) { + setKeysDraft(region.keys); + return; + } + setKeysDraft(canonical); + if (canonical !== region.keys) { + void tl.updateKeystroke(region.id, { keys: canonical }); + } + }; + + const commitLabel = () => { + const trimmed = labelDraft.trim().slice(0, 120); + const nextLabel = trimmed || undefined; + if (nextLabel !== region.label) { + void tl.updateKeystroke(region.id, { label: nextLabel }); + } + }; + + return ( +
+ {paneHeader(, "Shortcut Callout", onClose, tc("actions.close"))} +
+
+ + setKeysDraft(e.target.value)} + onBlur={commitKeys} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.currentTarget.blur(); + } + }} + style={{ + width: "100%", + padding: "7px 10px", + borderRadius: 10, + border: "1px solid var(--border)", + background: "var(--surface-2)", + color: "var(--fg)", + fontSize: 13, + }} + /> +
+
+ + setLabelDraft(e.target.value)} + onBlur={commitLabel} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.currentTarget.blur(); + } + }} + style={{ + width: "100%", + padding: "7px 10px", + borderRadius: 10, + border: "1px solid var(--border)", + background: "var(--surface-2)", + color: "var(--fg)", + fontSize: 13, + }} + /> +
+ + label="Position" + columns={2} + options={[ + { value: "bottom-center", label: "Bottom Center" }, + { value: "bottom-left", label: "Bottom Left" }, + { value: "bottom-right", label: "Bottom Right" }, + { value: "top-center", label: "Top Center" }, + { value: "top-left", label: "Top Left" }, + { value: "top-right", label: "Top Right" }, + ]} + value={region.position} + onChange={(position) => void tl.updateKeystroke(region.id, { position })} + /> + + label="Theme" + columns={3} + options={[ + { value: "dark", label: "Dark" }, + { value: "light", label: "Light" }, + { value: "glass", label: "Glass" }, + ]} + value={region.theme} + onChange={(theme) => void tl.updateKeystroke(region.id, { theme })} + /> + + label="Size" + columns={3} + options={[ + { value: "sm", label: "S" }, + { value: "md", label: "M" }, + { value: "lg", label: "L" }, + ]} + value={region.size} + onChange={(size) => void tl.updateKeystroke(region.id, { size })} + /> + + label="Symbols" + columns={3} + options={[ + { value: "darwin", label: "macOS (⌘)" }, + { value: "win32", label: "Windows" }, + { value: "linux", label: "Linux" }, + ]} + value={region.platform} + onChange={(platform) => void tl.updateKeystroke(region.id, { platform })} + /> + +
+
+ ); +} + function FacetBody({ facet, onCollapse, diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index bb026fcd8..0c553e74e 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -3,6 +3,7 @@ import { Clock, Crosshair, Eraser, + Keyboard, Loader2, Maximize2, MessageSquare, @@ -596,7 +597,14 @@ const AudioLanePill = memo(function AudioLanePill({ interface LanePill { id: string; - kind: "annotation" | "speed" | "trim" | "zoom" | "cameraFullscreen" | "shaderEffect"; + kind: + | "annotation" + | "speed" + | "trim" + | "zoom" + | "cameraFullscreen" + | "shaderEffect" + | "keystroke"; start: number; end: number; label: string; @@ -776,6 +784,14 @@ export function V4Timeline({ label: effectNames.get(p.member.effectId) ?? "", sourceIds: p.ids, })); + const keystrokePills: LanePill[] = coalesceRegionsForRuler(tl.keystrokeRanges ?? []).map((p) => ({ + id: p.ids[0], + kind: "keystroke", + start: p.start, + end: p.end, + label: p.member.label ? `${p.member.keys} · ${p.member.label}` : p.member.keys, + sourceIds: p.ids, + })); const zoomRuns = coalesceRegionsForRuler(tl.zoomRegions); const zoomPills: LanePill[] = zoomRuns.map((p) => ({ id: p.ids[0], @@ -950,6 +966,7 @@ export function V4Timeline({ zoom: tl.zoomRegions, cameraFullscreen: tl.cameraFullscreenRegions, shaderEffect: tl.shaderEffectRanges, + keystroke: tl.keystrokeRanges ?? [], // Nothing to edit on a trim but its span, and a span edit moves the whole pill. trim: [], }; @@ -1027,6 +1044,8 @@ export function V4Timeline({ await tl.updateCameraFullscreenSpan(grabbed, s * 1000, en * 1000); else if (pill.kind === "shaderEffect") await tl.updateShaderEffectSpan(grabbed, s * 1000, en * 1000); + else if (pill.kind === "keystroke") + await tl.updateKeystrokeSpan(grabbed, s * 1000, en * 1000); else { // Trims are stored in source-time per asset but manipulated on the // timeline like every other pill. Ventilate the new span across the @@ -1456,7 +1475,7 @@ export function V4Timeline({ ? styles.laneTrim : kind === "cameraFullscreen" ? styles.laneCameraFullscreen - : kind === "shaderEffect" + : kind === "shaderEffect" || kind === "keystroke" ? styles.laneShaderEffect : styles.laneZoom; const pillIcon = (kind: LanePill["kind"]) => @@ -1470,6 +1489,8 @@ export function V4Timeline({ ) : kind === "shaderEffect" ? ( + ) : kind === "keystroke" ? ( + ) : ( ); @@ -2161,6 +2182,16 @@ export function V4Timeline({ + + + {/* Absent, not greyed out, in a project with no camera: nothing to show full screen. */} {hasAnyCamera ? ( 0 ? (
{renderPills(shaderEffectPills, "")}
) : null} + {(tl.keystrokeRanges?.length ?? 0) > 0 ? ( +
{renderPills(keystrokePills, "")}
+ ) : null} {/* Imported audio tracks (issue #350). Always shown, like every other lane — "Add audio" is a toolbar peer of the region tools now (and has a keyboard shortcut), so an empty lane advertises the shortcut diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index 143f19e57..345e7e4f2 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "ثقيل", "effectTierVeryHeavy": "ثقيل جدًا", "effectCode": "الشيفرة", - "copyEffectCode": "نسخ الشيفرة" + "copyEffectCode": "نسخ الشيفرة", + "keystrokeTitle": "اختصار لوحة المفاتيح" }, "chat": { "changeProvider": "تغيير المزوّد", diff --git a/src/i18n/locales/ar/timeline.json b/src/i18n/locales/ar/timeline.json index 21e5d5fae..f2cdb5672 100644 --- a/src/i18n/locales/ar/timeline.json +++ b/src/i18n/locales/ar/timeline.json @@ -7,7 +7,8 @@ "addTrim": "إضافة قص: يزيل جزءًا من الفيديو عند الوقت الحالي", "addAnnotation": "إضافة نص أو سهم أو صورة أو تمويه عند الوقت الحالي", "addSpeed": "إضافة تغيير في السرعة عند الوقت الحالي", - "addCameraFullscreen": "إضافة كاميرا كاملة الشاشة: تعرض الكاميرا بملء الشاشة عند الوقت الحالي" + "addCameraFullscreen": "إضافة كاميرا كاملة الشاشة: تعرض الكاميرا بملء الشاشة عند الوقت الحالي", + "addKeystroke": "إضافة وسيلة إيضاح لاختصار لوحة المفاتيح" }, "hints": { "pressZoom": "اضغط {{key}} لإضافة تكبير", diff --git a/src/i18n/locales/cs/editor.json b/src/i18n/locales/cs/editor.json index 53ba12667..637943da0 100644 --- a/src/i18n/locales/cs/editor.json +++ b/src/i18n/locales/cs/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "Těžký", "effectTierVeryHeavy": "Velmi těžký", "effectCode": "Kód", - "copyEffectCode": "Kopírovat kód" + "copyEffectCode": "Kopírovat kód", + "keystrokeTitle": "Klávesová zkratka" }, "chat": { "changeProvider": "Změnit poskytovatele", diff --git a/src/i18n/locales/cs/timeline.json b/src/i18n/locales/cs/timeline.json index 9e93b4eb7..fddf832b0 100644 --- a/src/i18n/locales/cs/timeline.json +++ b/src/i18n/locales/cs/timeline.json @@ -7,7 +7,8 @@ "addTrim": "Přidat oříznutí: odstraní část videa v aktuálním čase", "addAnnotation": "Přidat text, šipku, obrázek nebo rozmazání v aktuálním čase", "addSpeed": "Přidat změnu rychlosti v aktuálním čase", - "addCameraFullscreen": "Přidat Celou kameru: zobrazí kameru na celou obrazovku v aktuálním čase" + "addCameraFullscreen": "Přidat Celou kameru: zobrazí kameru na celou obrazovku v aktuálním čase", + "addKeystroke": "Přidat popisek klávesové zkratky" }, "hints": { "pressZoom": "Stiskněte {{key}} pro přidání přiblížení", diff --git a/src/i18n/locales/de/editor.json b/src/i18n/locales/de/editor.json index 196e06f51..28ee728cb 100644 --- a/src/i18n/locales/de/editor.json +++ b/src/i18n/locales/de/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "Schwer", "effectTierVeryHeavy": "Sehr schwer", "effectCode": "Code", - "copyEffectCode": "Code kopieren" + "copyEffectCode": "Code kopieren", + "keystrokeTitle": "Tastenkombinationen" }, "chat": { "changeProvider": "Anbieter wechseln", diff --git a/src/i18n/locales/de/timeline.json b/src/i18n/locales/de/timeline.json index fa18db607..78972eae9 100644 --- a/src/i18n/locales/de/timeline.json +++ b/src/i18n/locales/de/timeline.json @@ -7,7 +7,8 @@ "addTrim": "Trim hinzufügen: entfernt einen Abschnitt des Videos zum aktuellen Zeitpunkt", "addAnnotation": "Text, Pfeil, Bild oder Weichzeichner zum aktuellen Zeitpunkt hinzufügen", "addSpeed": "Tempowechsel zum aktuellen Zeitpunkt hinzufügen", - "addCameraFullscreen": "Vollbild-Kamera hinzufügen: zeigt die Kamera im Vollbild zum aktuellen Zeitpunkt" + "addCameraFullscreen": "Vollbild-Kamera hinzufügen: zeigt die Kamera im Vollbild zum aktuellen Zeitpunkt", + "addKeystroke": "Tastenkombination hinzufügen" }, "hints": { "pressZoom": "Drücke {{key}}, um einen Zoom hinzuzufügen", diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index fb0cb8917..6009a14d0 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "Heavy", "effectTierVeryHeavy": "Very heavy", "effectCode": "Code", - "copyEffectCode": "Copy code" + "copyEffectCode": "Copy code", + "keystrokeTitle": "Shortcut Callout" }, "chat": { "changeProvider": "Change provider", diff --git a/src/i18n/locales/en/timeline.json b/src/i18n/locales/en/timeline.json index ec4b8d28f..7d9c3bf9c 100644 --- a/src/i18n/locales/en/timeline.json +++ b/src/i18n/locales/en/timeline.json @@ -7,7 +7,8 @@ "addTrim": "Add a trim: removes a section of the video at the current time", "addAnnotation": "Add text, an arrow, an image or a blur at the current time", "addSpeed": "Add a speed change at the current time", - "addCameraFullscreen": "Add Full Camera: shows the camera full screen at the current time" + "addCameraFullscreen": "Add Full Camera: shows the camera full screen at the current time", + "addKeystroke": "Add Shortcut Callout" }, "hints": { "pressZoom": "Press {{key}} to add zoom", diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index 89bf7d390..d03077792 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "Pesado", "effectTierVeryHeavy": "Muy pesado", "effectCode": "Código", - "copyEffectCode": "Copiar código" + "copyEffectCode": "Copiar código", + "keystrokeTitle": "Atajo de teclado" }, "chat": { "changeProvider": "Cambiar proveedor", diff --git a/src/i18n/locales/es/timeline.json b/src/i18n/locales/es/timeline.json index f4fcd4374..7360e2fde 100644 --- a/src/i18n/locales/es/timeline.json +++ b/src/i18n/locales/es/timeline.json @@ -7,7 +7,8 @@ "addTrim": "Agregar un recorte: quita una parte del video en el momento actual", "addAnnotation": "Agregar texto, una flecha, una imagen o un desenfoque en el momento actual", "addSpeed": "Agregar un cambio de velocidad en el momento actual", - "addCameraFullscreen": "Agregar Cámara a pantalla completa: muestra la cámara a pantalla completa en el momento actual" + "addCameraFullscreen": "Agregar Cámara a pantalla completa: muestra la cámara a pantalla completa en el momento actual", + "addKeystroke": "Añadir atajo de teclado" }, "hints": { "pressZoom": "Presiona {{key}} para agregar zoom", diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 92ec22bcb..f597d3b3b 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "Lourd", "effectTierVeryHeavy": "Très lourd", "effectCode": "Code", - "copyEffectCode": "Copier le code" + "copyEffectCode": "Copier le code", + "keystrokeTitle": "Raccourci clavier" }, "chat": { "changeProvider": "Changer de fournisseur", diff --git a/src/i18n/locales/fr/timeline.json b/src/i18n/locales/fr/timeline.json index 4fd7b3123..cfdaaade9 100644 --- a/src/i18n/locales/fr/timeline.json +++ b/src/i18n/locales/fr/timeline.json @@ -7,7 +7,8 @@ "addTrim": "Ajouter une coupe : retire une partie de la vidéo à l'instant actuel", "addAnnotation": "Ajouter du texte, une flèche, une image ou un flou à l'instant actuel", "addSpeed": "Ajouter un changement de vitesse à l'instant actuel", - "addCameraFullscreen": "Ajouter Caméra plein écran : affiche la caméra en plein écran à l'instant actuel" + "addCameraFullscreen": "Ajouter Caméra plein écran : affiche la caméra en plein écran à l'instant actuel", + "addKeystroke": "Ajouter un raccourci clavier" }, "hints": { "pressZoom": "Appuyez sur {{key}} pour ajouter un zoom", diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index e9a61562e..104e42037 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "Pesante", "effectTierVeryHeavy": "Molto pesante", "effectCode": "Codice", - "copyEffectCode": "Copia codice" + "copyEffectCode": "Copia codice", + "keystrokeTitle": "Scorciatoia da tastiera" }, "chat": { "changeProvider": "Cambia provider", diff --git a/src/i18n/locales/it/timeline.json b/src/i18n/locales/it/timeline.json index e73a04db9..5c0d2e3a6 100644 --- a/src/i18n/locales/it/timeline.json +++ b/src/i18n/locales/it/timeline.json @@ -7,7 +7,8 @@ "addTrim": "Aggiungi un taglio: rimuove una parte del video nel momento attuale", "addAnnotation": "Aggiungi testo, una freccia, un'immagine o una sfocatura nel momento attuale", "addSpeed": "Aggiungi un cambio di velocità nel momento attuale", - "addCameraFullscreen": "Aggiungi Camera a schermo intero: mostra la fotocamera a schermo intero nel momento attuale" + "addCameraFullscreen": "Aggiungi Camera a schermo intero: mostra la fotocamera a schermo intero nel momento attuale", + "addKeystroke": "Aggiungi scorciatoia da tastiera" }, "hints": { "pressZoom": "Premi {{key}} per aggiungere zoom", diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index 10d8f7aab..3c2f46f3c 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "重い", "effectTierVeryHeavy": "非常に重い", "effectCode": "コード", - "copyEffectCode": "コードをコピー" + "copyEffectCode": "コードをコピー", + "keystrokeTitle": "ショートカット表示" }, "chat": { "changeProvider": "プロバイダーを変更", diff --git a/src/i18n/locales/ja-JP/timeline.json b/src/i18n/locales/ja-JP/timeline.json index 0f41b8fc9..0646f238d 100644 --- a/src/i18n/locales/ja-JP/timeline.json +++ b/src/i18n/locales/ja-JP/timeline.json @@ -7,7 +7,8 @@ "addTrim": "トリムを追加：現在の再生位置で動画の一部を削除します", "addAnnotation": "現在の再生位置にテキスト、矢印、画像、ぼかしを追加", "addSpeed": "現在の再生位置に再生速度の変更を追加", - "addCameraFullscreen": "フルスクリーンカメラを追加：現在の再生位置でカメラを全画面表示します" + "addCameraFullscreen": "フルスクリーンカメラを追加：現在の再生位置でカメラを全画面表示します", + "addKeystroke": "ショートカット表示を追加" }, "hints": { "pressZoom": "{{key}}キーを押してズームを追加", diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index fa1dbc148..7b51b28dd 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "무거움", "effectTierVeryHeavy": "매우 무거움", "effectCode": "코드", - "copyEffectCode": "코드 복사" + "copyEffectCode": "코드 복사", + "keystrokeTitle": "단축키 콜아웃" }, "chat": { "changeProvider": "제공업체 변경", diff --git a/src/i18n/locales/ko-KR/timeline.json b/src/i18n/locales/ko-KR/timeline.json index 64670c712..7244490d7 100644 --- a/src/i18n/locales/ko-KR/timeline.json +++ b/src/i18n/locales/ko-KR/timeline.json @@ -7,7 +7,8 @@ "addTrim": "트림 추가: 현재 시점에서 영상의 한 구간을 제거합니다", "addAnnotation": "현재 시점에 텍스트, 화살표, 이미지 또는 블러 추가", "addSpeed": "현재 시점에 속도 변경 추가", - "addCameraFullscreen": "전체 화면 카메라 추가: 현재 시점에 카메라를 전체 화면으로 표시합니다" + "addCameraFullscreen": "전체 화면 카메라 추가: 현재 시점에 카메라를 전체 화면으로 표시합니다", + "addKeystroke": "단축키 콜아웃 추가" }, "hints": { "pressZoom": "{{key}}를 눌러 줌 추가", diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index 8dcaff24a..751b7f940 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "Pesado", "effectTierVeryHeavy": "Muito pesado", "effectCode": "Código", - "copyEffectCode": "Copiar código" + "copyEffectCode": "Copiar código", + "keystrokeTitle": "Atalho de teclado" }, "chat": { "changeProvider": "Trocar provedor", diff --git a/src/i18n/locales/pt-BR/timeline.json b/src/i18n/locales/pt-BR/timeline.json index fee6e6933..3b7f5ef96 100644 --- a/src/i18n/locales/pt-BR/timeline.json +++ b/src/i18n/locales/pt-BR/timeline.json @@ -7,7 +7,8 @@ "addTrim": "Adicionar um recorte: remove uma parte do vídeo no momento atual", "addAnnotation": "Adicionar texto, seta, imagem ou desfoque no momento atual", "addSpeed": "Adicionar uma mudança de velocidade no momento atual", - "addCameraFullscreen": "Adicionar Câmera em Tela Cheia: mostra a câmera em tela cheia no momento atual" + "addCameraFullscreen": "Adicionar Câmera em Tela Cheia: mostra a câmera em tela cheia no momento atual", + "addKeystroke": "Adicionar atalho de teclado" }, "hints": { "pressZoom": "Pressione {{key}} para adicionar zoom", diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index 827488c31..3ed46b1c6 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "Тяжёлый", "effectTierVeryHeavy": "Очень тяжёлый", "effectCode": "Код", - "copyEffectCode": "Копировать код" + "copyEffectCode": "Копировать код", + "keystrokeTitle": "Сочетание клавиш" }, "chat": { "changeProvider": "Сменить провайдера", diff --git a/src/i18n/locales/ru/timeline.json b/src/i18n/locales/ru/timeline.json index f7ad39905..acc7ad3fa 100644 --- a/src/i18n/locales/ru/timeline.json +++ b/src/i18n/locales/ru/timeline.json @@ -7,7 +7,8 @@ "addTrim": "Добавить обрезку: удаляет фрагмент видео в текущий момент", "addAnnotation": "Добавить текст, стрелку, изображение или размытие в текущий момент", "addSpeed": "Добавить изменение скорости в текущий момент", - "addCameraFullscreen": "Добавить камеру на весь экран: показывает камеру на весь экран в текущий момент" + "addCameraFullscreen": "Добавить камеру на весь экран: показывает камеру на весь экран в текущий момент", + "addKeystroke": "Добавить сочетание клавиш" }, "hints": { "pressZoom": "Нажмите {{key}} для добавления масштабирования", diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index 78efe59a1..b040be631 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "Ağır", "effectTierVeryHeavy": "Çok ağır", "effectCode": "Kod", - "copyEffectCode": "Kodu kopyala" + "copyEffectCode": "Kodu kopyala", + "keystrokeTitle": "Kısayol Çağrısı" }, "chat": { "changeProvider": "Sağlayıcıyı değiştir", diff --git a/src/i18n/locales/tr/timeline.json b/src/i18n/locales/tr/timeline.json index d0dae7ceb..9fb1de221 100644 --- a/src/i18n/locales/tr/timeline.json +++ b/src/i18n/locales/tr/timeline.json @@ -7,7 +7,8 @@ "addTrim": "Kırpma ekle: videonun bir bölümünü şu anki zamanda çıkarır", "addAnnotation": "Şu anki zamanda metin, ok, resim veya bulanıklık ekle", "addSpeed": "Şu anki zamanda hız değişikliği ekle", - "addCameraFullscreen": "Tam Ekran Kamera ekle: kamerayı şu anki zamanda tam ekran gösterir" + "addCameraFullscreen": "Tam Ekran Kamera ekle: kamerayı şu anki zamanda tam ekran gösterir", + "addKeystroke": "Kısayol Çağrısı Ekle" }, "hints": { "pressZoom": "Yakınlaştırma eklemek için {{key}} tuşuna basın", diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index 5341e7021..c7f311d01 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "Nặng", "effectTierVeryHeavy": "Rất nặng", "effectCode": "Mã", - "copyEffectCode": "Sao chép mã" + "copyEffectCode": "Sao chép mã", + "keystrokeTitle": "Chú thích phím tắt" }, "chat": { "changeProvider": "Đổi nhà cung cấp", diff --git a/src/i18n/locales/vi/timeline.json b/src/i18n/locales/vi/timeline.json index 9baf8a115..a4fcaac61 100644 --- a/src/i18n/locales/vi/timeline.json +++ b/src/i18n/locales/vi/timeline.json @@ -7,7 +7,8 @@ "addTrim": "Thêm cắt: xóa một đoạn video tại thời điểm hiện tại", "addAnnotation": "Thêm văn bản, mũi tên, hình ảnh hoặc làm mờ tại thời điểm hiện tại", "addSpeed": "Thêm thay đổi tốc độ tại thời điểm hiện tại", - "addCameraFullscreen": "Thêm Camera Toàn màn hình: hiển thị camera toàn màn hình tại thời điểm hiện tại" + "addCameraFullscreen": "Thêm Camera Toàn màn hình: hiển thị camera toàn màn hình tại thời điểm hiện tại", + "addKeystroke": "Thêm chú thích phím tắt" }, "hints": { "pressZoom": "Nhấn {{key}} để thêm thu phóng", diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index a50819a0f..30aff6ceb 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "重", "effectTierVeryHeavy": "很重", "effectCode": "代码", - "copyEffectCode": "复制代码" + "copyEffectCode": "复制代码", + "keystrokeTitle": "快捷键提示" }, "chat": { "changeProvider": "更换提供方", diff --git a/src/i18n/locales/zh-CN/timeline.json b/src/i18n/locales/zh-CN/timeline.json index 149a56a61..16ca6cbb2 100644 --- a/src/i18n/locales/zh-CN/timeline.json +++ b/src/i18n/locales/zh-CN/timeline.json @@ -7,7 +7,8 @@ "addTrim": "在当前时间添加删减区间，移除这段视频", "addAnnotation": "在当前时间添加文字、箭头、图片或模糊效果", "addSpeed": "在当前时间添加变速效果", - "addCameraFullscreen": "添加全屏摄像头：在当前时间全屏显示摄像头画面" + "addCameraFullscreen": "添加全屏摄像头：在当前时间全屏显示摄像头画面", + "addKeystroke": "添加快捷键提示" }, "hints": { "pressZoom": "按 {{key}} 添加缩放", diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index f0a7c9703..1e0bbfed1 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -241,7 +241,8 @@ "effectTierHeavy": "重", "effectTierVeryHeavy": "很重", "effectCode": "程式碼", - "copyEffectCode": "複製程式碼" + "copyEffectCode": "複製程式碼", + "keystrokeTitle": "快速鍵提示" }, "chat": { "changeProvider": "更換提供者", diff --git a/src/i18n/locales/zh-TW/timeline.json b/src/i18n/locales/zh-TW/timeline.json index ed255420b..9cf16cd44 100644 --- a/src/i18n/locales/zh-TW/timeline.json +++ b/src/i18n/locales/zh-TW/timeline.json @@ -7,7 +7,8 @@ "addTrim": "新增剪輯：在目前時間移除一段影片", "addAnnotation": "在目前時間新增文字、箭頭、圖片或模糊", "addSpeed": "在目前時間新增速度變化", - "addCameraFullscreen": "新增全螢幕攝影機：在目前時間全螢幕顯示攝影機" + "addCameraFullscreen": "新增全螢幕攝影機：在目前時間全螢幕顯示攝影機", + "addKeystroke": "新增快速鍵提示" }, "hints": { "pressZoom": "按 {{key}} 新增縮放", diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index 2b8366ba3..ea9d2ae7c 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -51,7 +51,8 @@ export type RegionKind = | "speed" | "cameraFullscreen" | "audio" - | "shaderEffect"; + | "shaderEffect" + | "keystroke"; /** Length a clip is given before its media has been probed. Lives here, in the pure * document layer, because that layer decides which clips are still waiting for a real @@ -397,6 +398,12 @@ export function mapAllRegionCollections( (document.shaderEffectRanges ?? []) as unknown as StoredRegion[], "fxr", ) as unknown as AxcutDocument["shaderEffectRanges"], + keystrokeRanges: document.keystrokeRanges + ? (fn( + document.keystrokeRanges as unknown as StoredRegion[], + "key", + ) as unknown as AxcutDocument["keystrokeRanges"]) + : undefined, annotations: fn( document.annotations as unknown as StoredRegion[], "ann", @@ -748,6 +755,7 @@ function anchoredRegionsOf(document: AxcutDocument): Array<{ id: string; clipId: (legacy?.speedRegions as StoredRegion[] | undefined) ?? [], (legacy?.cameraFullscreenRegions as StoredRegion[] | undefined) ?? [], (document.shaderEffectRanges ?? []) as unknown as StoredRegion[], + (document.keystrokeRanges ?? []) as unknown as StoredRegion[], ]; return collections .flat() @@ -1217,6 +1225,14 @@ export function removeRegion(document: AxcutDocument, kind: RegionKind, id: stri ...document, shaderEffectRanges: dropPillById(document.shaderEffectRanges ?? [], id), }); + case "keystroke": + return { + ...document, + keystrokeRanges: dropPillById( + document.keystrokeRanges ?? [], + id, + ) as AxcutDocument["keystrokeRanges"], + }; default: { // ponytail: exhaustive — TS errors here if a new RegionKind is added. const exhaustive: never = kind; @@ -1265,6 +1281,10 @@ const EDIT_REGIONS: Record< count: (d) => (d.shaderEffectRanges ?? []).length, clear: (d) => ({ ...d, shaderEffectRanges: [], shaderEffects: [] }), }, + keystroke: { + count: (d) => (d.keystrokeRanges ?? []).length, + clear: (d) => ({ ...d, keystrokeRanges: [] }), + }, }; /** How many edit regions the document holds, stored rows and not pills. Pure. */ diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index c3422b597..80aeb4897 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -190,6 +190,7 @@ export const assetSchema = z.object({ // no schema-version bump (an older build simply drops the key on save). transcriptionFailure: assetTranscriptionFailureSchema.nullish(), cameraTrack: cameraTrackSchema, + keystrokeTelemetryPath: z.string().optional(), }); // A crop is a sub-rectangle of the source video, expressed as fractions @@ -700,6 +701,37 @@ export const audioTrackSchema = endGteStart( "startMs", ); +export const keystrokePositionSchema = z.enum([ + "bottom-center", + "bottom-left", + "bottom-right", + "top-center", + "top-left", + "top-right", +]); + +export const keystrokeThemeSchema = z.enum(["dark", "light", "glass"]); +export const keystrokeSizeSchema = z.enum(["sm", "md", "lg"]); +export const keystrokePlatformSchema = z.enum(["darwin", "win32", "linux"]); + +export const keystrokeRangeSchema = endGteStart( + z.object({ + id: z.string().min(1), + startMs: z.number().nonnegative(), + endMs: z.number().nonnegative(), + ...clipAnchorShape, + keys: z.string().min(1), + label: z.string().max(120).optional(), + position: keystrokePositionSchema.default("bottom-center"), + theme: keystrokeThemeSchema.default("dark"), + size: keystrokeSizeSchema.default("md"), + platform: keystrokePlatformSchema.default("win32"), + origin: z.enum(["system", "agent", "user"]).default("user"), + }), + "endMs", + "startMs", +); + // Legacy OpenScreen appearance / export settings that the v3 schema doesn't // normalize into the timeline / assets model. They are applied at export time // by the existing pipeline (see technical-documentation/architecture/document-model.md). @@ -740,6 +772,8 @@ const documentSchemaShape = z.object({ // an absent list as empty. An older build drops both keys when it saves. shaderEffects: z.array(shaderEffectSchema).optional(), shaderEffectRanges: z.array(shaderEffectRangeSchema).optional(), + // Keystroke / shortcut HUD callout regions (issue #1317). + keystrokeRanges: z.array(keystrokeRangeSchema).optional(), // Imported audio tracks (issue #350). Defaulted so every document written // before this loads unchanged; an older build simply strips the key on save. audioTracks: z.array(audioTrackSchema).default([]), @@ -1379,6 +1413,11 @@ export type AxcutShaderEffect = z.infer; export type AxcutShaderEffectParam = z.infer; export type AxcutShaderEffectRange = z.infer; export type AxcutShaderEffectValue = z.infer; +export type AxcutKeystrokeRange = z.infer; +export type AxcutKeystrokePosition = z.infer; +export type AxcutKeystrokeTheme = z.infer; +export type AxcutKeystrokeSize = z.infer; +export type AxcutKeystrokePlatform = z.infer; export type AxcutLegacyEditor = z.infer; export type AxcutDocument = z.infer; export type AxcutDocumentInput = z.input; diff --git a/src/lib/ai-edition/store/documentWriteAudit.test.ts b/src/lib/ai-edition/store/documentWriteAudit.test.ts index 575d96f5b..36ef6bd4b 100644 --- a/src/lib/ai-edition/store/documentWriteAudit.test.ts +++ b/src/lib/ai-edition/store/documentWriteAudit.test.ts @@ -168,6 +168,7 @@ const DECLARED: WritePath[] = [ w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), + w("src/components/ai-edition/NewEditorShell.tsx", "pasteRegion", "save", "gesture"), // The window is closing and the user answered "save". w("src/components/ai-edition/NewEditorShell.tsx", "unsubSaveBeforeClose", "save", "gesture"), @@ -254,6 +255,7 @@ const DECLARED: WritePath[] = [ w("src/lib/ai-edition/store/useTimeline.ts", "applyClipEdit", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "addAnnotation", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "addCameraFullscreen", "save", "gesture"), + w("src/lib/ai-edition/store/useTimeline.ts", "addKeystroke", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "addSpeed", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "addTrim", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "addZoom", "save", "gesture"), @@ -300,6 +302,8 @@ const DECLARED: WritePath[] = [ w("src/lib/ai-edition/store/useTimeline.ts", "updateAnnotationLive", "set", "automatic"), w("src/lib/ai-edition/store/useTimeline.ts", "updateAnnotationSpan", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "updateCameraFullscreenSpan", "save", "gesture"), + w("src/lib/ai-edition/store/useTimeline.ts", "updateKeystroke", "save", "gesture"), + w("src/lib/ai-edition/store/useTimeline.ts", "updateKeystrokeSpan", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "updateShaderEffect", "save", "gesture"), w("src/lib/ai-edition/store/useTimeline.ts", "updateShaderEffectLive", "set", "automatic"), w("src/lib/ai-edition/store/useTimeline.ts", "updateShaderEffectSpan", "save", "gesture"), diff --git a/src/lib/ai-edition/store/regionClipboard.ts b/src/lib/ai-edition/store/regionClipboard.ts index 9ae66a8ab..e8075da87 100644 --- a/src/lib/ai-edition/store/regionClipboard.ts +++ b/src/lib/ai-edition/store/regionClipboard.ts @@ -19,6 +19,7 @@ export type RegionSnapshot = // An effect region carries its definition: pasted into another project, the definition is // recreated there, since the region alone would name an effect that project does not have. | { kind: "shaderEffect"; region: Record; definition: AxcutShaderEffect } + | { kind: "keystroke"; region: Record } // A trim carries no user-visible properties, so all there is to copy is how // LONG it was — `{ durationSec }`. That is not a special case so much as the // general one made obvious: every paste keeps the copied properties and takes diff --git a/src/lib/ai-edition/store/useTimeline.test.ts b/src/lib/ai-edition/store/useTimeline.test.ts index f8e6ba8a3..07262316d 100644 --- a/src/lib/ai-edition/store/useTimeline.test.ts +++ b/src/lib/ai-edition/store/useTimeline.test.ts @@ -889,6 +889,7 @@ describe("useTimeline.clearTimeline", () => { { history: true }, ); }, + keystroke: (tl) => tl.addKeystroke(), }; const editKinds = Object.keys(addOf) as EditKind[]; @@ -901,6 +902,7 @@ describe("useTimeline.clearTimeline", () => { speed: doc ? readSpeedRegions(doc).length : 0, cameraFullscreen: legacy.cameraFullscreenRegions?.length ?? 0, shaderEffect: doc?.shaderEffectRanges?.length ?? 0, + keystroke: doc?.keystrokeRanges?.length ?? 0, }; }; const noRegions: Record = { @@ -910,6 +912,7 @@ describe("useTimeline.clearTimeline", () => { speed: 0, cameraFullscreen: 0, shaderEffect: 0, + keystroke: 0, }; // A project holding everything that is NOT an edit: two clips, a webcam, an imported @@ -1070,6 +1073,7 @@ describe("useTimeline.clearTimeline", () => { speed: 1, cameraFullscreen: 1, shaderEffect: 1, + keystroke: 1, }); await act(async () => { @@ -1110,6 +1114,7 @@ describe("useTimeline.clearTimeline", () => { speed: 1, cameraFullscreen: 1, shaderEffect: 1, + keystroke: 1, }); act(() => { diff --git a/src/lib/ai-edition/store/useTimeline.ts b/src/lib/ai-edition/store/useTimeline.ts index 7d650ed85..a72aaf165 100644 --- a/src/lib/ai-edition/store/useTimeline.ts +++ b/src/lib/ai-edition/store/useTimeline.ts @@ -41,6 +41,7 @@ import type { AxcutAudioTrack, AxcutClipCropRegion, AxcutDocument, + AxcutKeystrokeRange, AxcutShaderEffectRange, } from "../schema"; import { @@ -614,6 +615,44 @@ export function useTimeline() { [document, saveDocument], ); + const addKeystroke = useCallback( + async ( + durationSec = DEFAULT_NEW_REGION_SEC, + seed?: Partial>, + ) => { + if (!document) return; + const timeMs = Math.round(playheadSec() * 1000); + const endMs = timeMs + Math.round(durationSec * 1000); + const id = createId("key"); + const nextRegion: AxcutKeystrokeRange = { + id, + startMs: timeMs, + endMs, + keys: seed?.keys ?? "Cmd+Shift+P", + ...(seed?.label ? { label: seed.label } : {}), + position: seed?.position ?? "bottom-center", + theme: seed?.theme ?? "dark", + size: seed?.size ?? "md", + platform: seed?.platform ?? "win32", + origin: seed?.origin ?? "user", + }; + const anchored = anchorRegionsWithDerivedMs([nextRegion], document.timeline.clips, () => + createId("key"), + ) as AxcutKeystrokeRange[]; + const next: AxcutDocument = { + ...document, + keystrokeRanges: [...(document.keystrokeRanges ?? []), ...anchored], + }; + if (!(await saveDocument(next, { history: true }))) return; + const placedId = anchored[0]?.id ?? id; + setSelection({ kind: "keystroke", id: placedId }); + setMultiSelection([{ kind: "keystroke", id: placedId }]); + setClipSelection(null); + setSelectedAudioTrackId(null); + }, + [document, saveDocument, setSelectedAudioTrackId], + ); + // Like updateTrimRange but also re-attaches the trim to a (possibly different) CLIP — // needed when a trim is dragged across a clip boundary, whether or not the landing clip // is backed by another asset. Re-pointing `clipId` as well as `assetId` is what makes a @@ -1164,6 +1203,54 @@ export function useTimeline() { } }, [saveDocument]); + const updateKeystrokeSpan = useCallback( + async (id: string, startMs: number, endMs: number) => { + if (!document) return; + const s = finiteMs(startMs); + const e = finiteMs(endMs); + const next: AxcutDocument = { + ...document, + keystrokeRanges: replacePillSpan( + document.keystrokeRanges ?? [], + id, + Math.min(s, e), + Math.max(s, e), + document.timeline.clips, + () => createId("key"), + ) as AxcutDocument["keystrokeRanges"], + }; + await saveDocument(next, { history: true }); + }, + [document, saveDocument], + ); + + const updateKeystroke = useCallback( + async ( + id: string, + patch: Partial< + Omit< + AxcutKeystrokeRange, + "id" | "startMs" | "endMs" | "clipId" | "sourceStartSec" | "sourceEndSec" + > + >, + ) => { + await enqueueZoomWrite(async () => { + const current = useProjectStore.getState().document; + if (!current) return false; + const next: AxcutDocument = { + ...current, + keystrokeRanges: patchRegionById( + current.keystrokeRanges ?? [], + id, + patch, + ) as AxcutDocument["keystrokeRanges"], + }; + return saveDocument(next, { history: true }); + }); + }, + [enqueueZoomWrite, saveDocument], + ); + const updateSpeedValue = useCallback( async (id: string, speed: number) => { if (!document) return; @@ -1215,6 +1302,7 @@ export function useTimeline() { const shaderEffectIds = new Set( handles.filter((h) => h.kind === "shaderEffect").map((h) => h.id), ); + const keystrokeIds = new Set(handles.filter((h) => h.kind === "keystroke").map((h) => h.id)); const legacy = (document.legacyEditor as Record) ?? {}; const prevSpeed = dropPillsByIds( (legacy.speedRegions as Array<{ id: string; startMs: number; endMs: number }>) ?? [], @@ -1237,6 +1325,14 @@ export function useTimeline() { ), } : {}), + ...(keystrokeIds.size > 0 + ? { + keystrokeRanges: dropPillsByIds( + document.keystrokeRanges ?? [], + keystrokeIds, + ) as AxcutDocument["keystrokeRanges"], + } + : {}), timeline: { ...document.timeline, // Whole-pill delete, same as the zoom/annotation lines above — a trim grown @@ -1556,6 +1652,7 @@ export function useTimeline() { // One region per effect, never two at once; each names its definition. const shaderEffectRanges = hasDoc ? (document.shaderEffectRanges ?? []) : []; const shaderEffects = hasDoc ? (document.shaderEffects ?? []) : []; + const keystrokeRanges = hasDoc ? (document.keystrokeRanges ?? []) : []; // --- Timeline audio tracks (issue #350) ------------------------------------- // CLIP-ANCHORED like every region above: one user-visible track is one pill @@ -1744,6 +1841,7 @@ export function useTimeline() { cameraFullscreenRegions, shaderEffectRanges, shaderEffects, + keystrokeRanges, clips: document?.timeline.clips ?? [], assets: document?.assets ?? [], // The timeline marks where the user has ADDED words — text with no audio behind it. @@ -1760,6 +1858,7 @@ export function useTimeline() { addAnnotation, addSpeed, addCameraFullscreen, + addKeystroke, removeRegion, removeRegions, hasEditRegions, @@ -1801,6 +1900,8 @@ export function useTimeline() { updateShaderEffect, updateShaderEffectLive, commitShaderEffectChange, + updateKeystrokeSpan, + updateKeystroke, // T19 — drives the preview video during trim-edge resize. setCurrentTime: useProjectStore((s) => s.setCurrentTime), }; diff --git a/src/lib/keystrokes/keystrokeTelemetry.test.ts b/src/lib/keystrokes/keystrokeTelemetry.test.ts new file mode 100644 index 000000000..15840e293 --- /dev/null +++ b/src/lib/keystrokes/keystrokeTelemetry.test.ts @@ -0,0 +1,288 @@ +import { describe, expect, it } from "vitest"; +import { executeAgentTool } from "../../../electron/ai-edition/agent-tools"; +import { buildSceneDescription } from "../../native/sceneDescription"; +import { duplicateClip, removeClip } from "../ai-edition/document/timeline"; +import { type AxcutDocument, createEmptyDocument, documentSchema } from "../ai-edition/schema"; +import { + buildKeystrokeTrack, + coalesceKeystrokeSamplesToRegions, + formatKeycapLabel, + formatShortcutKeycaps, + getKeystrokeTelemetryCandidates, + isShortcutSample, + normalizeKeyToken, + parseKeystrokeSidecarJson, + parseShortcutKeys, + seedKeystrokeRangesFromTelemetry, +} from "./keystrokeTelemetry"; + +const FIXTURE_CREATED_AT = "2026-01-01T00:00:00.000Z"; + +function createTestDocument(): AxcutDocument { + const base = createEmptyDocument({ + title: "Keystroke Test", + projectId: "proj_keys", + createdAt: FIXTURE_CREATED_AT, + }); + return documentSchema.parse({ + ...base, + project: { ...base.project, primaryAssetId: "asset_1" }, + assets: [ + { + id: "asset_1", + kind: "video", + label: "Screen Recording", + originalPath: "/recordings/demo.mp4", + durationSec: 20, + }, + ], + timeline: { + ...base.timeline, + clips: [ + { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 20, + timelineStartSec: 0, + timelineEndSec: 20, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + trimRanges: [ + { + id: "trim_1", + assetId: "asset_1", + clipId: "clip_1", + startSec: 8, + endSec: 10, + reason: "pause", + origin: "user", + }, + ], + }, + }); +} + +describe("keystrokeTelemetry helpers", () => { + it("normalizes and parses shortcut key strings and arrays", () => { + expect(normalizeKeyToken("command")).toBe("Cmd"); + expect(normalizeKeyToken("control")).toBe("Ctrl"); + expect(normalizeKeyToken("opt")).toBe("Option"); + expect(normalizeKeyToken("escape")).toBe("Esc"); + expect(normalizeKeyToken("f12")).toBe("F12"); + expect(normalizeKeyToken("k")).toBe("K"); + + expect(parseShortcutKeys("cmd + shift + p")).toEqual(["Cmd", "Shift", "P"]); + expect(parseShortcutKeys(["ctrl", "alt", "delete"])).toEqual(["Ctrl", "Alt", "Delete"]); + expect(parseShortcutKeys("Cmd++")).toEqual(["Cmd", "+"]); + }); + + it("formats platform-specific keycap labels for macOS and Windows", () => { + expect(formatShortcutKeycaps(["Cmd", "Shift", "P"], "darwin")).toEqual(["⌘", "⇧", "P"]); + expect(formatShortcutKeycaps(["Option", "Backspace"], "darwin")).toEqual(["⌥", "⌫"]); + expect(formatShortcutKeycaps(["Cmd", "Shift", "P"], "win32")).toEqual(["Ctrl", "Shift", "P"]); + expect(formatKeycapLabel("Enter", "darwin")).toBe("↵"); + expect(formatKeycapLabel("Up", "win32")).toBe("↑"); + }); + + it("distinguishes intentional shortcuts from plain typing for privacy safety", () => { + expect(isShortcutSample({ timeMs: 1000, keys: ["Cmd", "K"] })).toBe(true); + expect(isShortcutSample({ timeMs: 1200, keys: ["Ctrl", "Shift", "P"] })).toBe(true); + expect(isShortcutSample({ timeMs: 1400, keys: ["F5"] })).toBe(true); + expect(isShortcutSample({ timeMs: 1500, keys: ["Esc"] })).toBe(true); + // Plain typing (letters, numbers, Shift+letter, Space, Backspace) must be ignored by default + expect(isShortcutSample({ timeMs: 1600, keys: ["a"] })).toBe(false); + expect(isShortcutSample({ timeMs: 1700, keys: ["Shift", "a"] })).toBe(false); + expect(isShortcutSample({ timeMs: 1750, keys: ["Shift", "Tab"] })).toBe(true); + expect(isShortcutSample({ timeMs: 1800, keys: ["Space"] })).toBe(false); + expect(isShortcutSample({ timeMs: 1900, keys: ["Shift"] })).toBe(false); + // Explicit override + expect(isShortcutSample({ timeMs: 2000, keys: ["Space"], isShortcut: true })).toBe(true); + }); + + it("returns candidate sidecar paths for a video path", () => { + const candidates = getKeystrokeTelemetryCandidates("/recordings/take-1.mp4"); + expect(candidates).toEqual([ + "/recordings/take-1.mp4.keys.json", + "/recordings/take-1.keys.json", + ]); + expect(getKeystrokeTelemetryCandidates("")).toEqual([]); + }); + + it("coalesces rapid repeated shortcuts and filters out prose keystrokes", () => { + const spans = coalesceKeystrokeSamplesToRegions( + [ + { timeMs: 1000, keys: ["Cmd", "S"], label: "Save" }, + { timeMs: 1300, keys: ["Cmd", "S"] }, // within 600ms merge window + { timeMs: 1500, keys: ["x"] }, // plain key — filtered out + { timeMs: 3000, keys: ["Cmd", "Shift", "P"], label: "Command Palette" }, + ], + { holdMs: 1500, mergeWindowMs: 500 }, + ); + + expect(spans).toEqual([ + { + startMs: 1000, + endMs: 2800, + keys: "Cmd+S", + label: "Save", + }, + { + startMs: 3000, + endMs: 4500, + keys: "Cmd+Shift+P", + label: "Command Palette", + }, + ]); + }); + + it("projects recorded keystrokes onto the edited timeline and flags trimmed spans", () => { + const doc = createTestDocument(); + const track = buildKeystrokeTrack( + [ + { timeMs: 2000, keys: ["Cmd", "K"], label: "Quick Open" }, + { timeMs: 9000, keys: ["Cmd", "B"], label: "Toggle Sidebar" }, // inside trim 8..10s + ], + doc.timeline.clips, + doc.timeline.trimRanges, + { assetId: "asset_1" }, + ); + + expect(track.assetId).toBe("asset_1"); + expect(track.eventCount).toBe(2); + expect(track.events[0]).toMatchObject({ + atSec: 2, + virtualSec: 2, + keys: "Cmd+K", + label: "Quick Open", + isShortcut: true, + }); + expect(track.events[1]).toMatchObject({ + atSec: 9, + virtualSec: 9, + keys: "Cmd+B", + trimmed: true, + }); + }); +}); + +describe("keystrokeRanges document & agent tool integration", () => { + it("adds, updates, projects, duplicates, and removes keystroke HUD callouts", () => { + const doc = createTestDocument(); + + const added = executeAgentTool( + doc, + "addKeystroke", + JSON.stringify({ + startSec: 2, + endSec: 4, + keys: ["Cmd", "Shift", "P"], + label: "Command Palette", + position: "bottom-center", + theme: "glass", + size: "lg", + platform: "darwin", + }), + ); + expect(added.ok).toBe(true); + const addedDoc = added.document as AxcutDocument; + expect(addedDoc.keystrokeRanges).toHaveLength(1); + const keystrokeId = addedDoc.keystrokeRanges?.[0]?.id ?? ""; + + // Verify sceneDescription projection + const scene = buildSceneDescription(addedDoc); + expect(scene.keystrokeRegions).toHaveLength(1); + expect(scene.keystrokeRegions?.[0]).toMatchObject({ + id: keystrokeId, + startSec: 2, + endSec: 4, + keys: "Cmd+Shift+P", + label: "Command Palette", + position: "bottom-center", + theme: "glass", + size: "lg", + platform: "darwin", + }); + + // Update via setKeystroke + const updated = executeAgentTool( + addedDoc, + "setKeystroke", + JSON.stringify({ + keystrokeId, + startSec: 3, + endSec: 5.5, + keys: ["Cmd", "K"], + label: "Search", + position: "top-right", + }), + ); + expect(updated.ok).toBe(true); + const updatedDoc = updated.document as AxcutDocument; + expect(updatedDoc.keystrokeRanges?.[0]).toMatchObject({ + startMs: 3000, + endMs: 5500, + keys: "Cmd+K", + label: "Search", + position: "top-right", + theme: "glass", + }); + + // Verify getCurrentDocument includes keystrokeRanges + const snapshotExec = executeAgentTool(updatedDoc, "getCurrentDocument", "{}"); + expect(snapshotExec.ok).toBe(true); + const snapshot = JSON.parse(snapshotExec.resultJson); + expect(snapshot.keystrokeRanges).toHaveLength(1); + expect(snapshot.keystrokeRanges[0]).toMatchObject({ + id: updatedDoc.keystrokeRanges?.[0]?.id, + startSec: 3, + endSec: 5.5, + keys: "Cmd+K", + label: "Search", + }); + + // Duplicating a clip copies its anchored keystroke ranges + const duplicated = duplicateClip(updatedDoc, "clip_1"); + expect(duplicated.keystrokeRanges).toHaveLength(2); + + // Removing a clip drops its anchored keystroke ranges + const afterClipRemoved = removeClip(duplicated, "clip_1"); + expect(afterClipRemoved.keystrokeRanges).toHaveLength(1); + + // Removing via removeModifier + const remainingId = afterClipRemoved.keystrokeRanges?.[0]?.id ?? ""; + const removed = executeAgentTool( + afterClipRemoved, + "removeModifier", + JSON.stringify({ id: remainingId }), + ); + expect(removed.ok).toBe(true); + expect(removed.document?.keystrokeRanges).toHaveLength(0); + }); + + it("parses .keys.json sidecar samples and seeds document.keystrokeRanges on import", () => { + const doc = createTestDocument(); + const rawSidecar = JSON.stringify({ + version: 1, + samples: [ + { timeMs: 1200, keys: ["Cmd", "Shift", "P"], label: "Command Palette" }, + { timeMs: 1500, keys: ["Cmd", "Shift", "P"] }, + { timeMs: 2500, keys: ["a"] }, // plain typing ignored + ], + }); + const samples = parseKeystrokeSidecarJson(rawSidecar); + expect(samples).toHaveLength(3); + + const seeded = seedKeystrokeRangesFromTelemetry(doc, samples, (i) => `key_seed_${i}`); + expect(seeded.keystrokeRanges).toHaveLength(1); + expect(seeded.keystrokeRanges?.[0]).toMatchObject({ + id: "key_seed_0", + keys: "Cmd+Shift+P", + label: "Command Palette", + origin: "system", + }); + }); +}); diff --git a/src/lib/keystrokes/keystrokeTelemetry.ts b/src/lib/keystrokes/keystrokeTelemetry.ts new file mode 100644 index 000000000..8fe2ff208 --- /dev/null +++ b/src/lib/keystrokes/keystrokeTelemetry.ts @@ -0,0 +1,468 @@ +import type { + AxcutClip, + AxcutDocument, + AxcutKeystrokeRange, + AxcutTrimRange, +} from "../ai-edition/schema"; +import { anchorRegionsWithDerivedMs } from "../ai-edition/timeline/timelineMap"; +import { locateSourcePosition } from "../ai-edition/timeline/virtual-preview"; + +export type KeystrokePlatform = "darwin" | "win32" | "linux"; + +export interface KeystrokeTelemetrySample { + /** Milliseconds from the start of the recording source. */ + timeMs: number; + /** Key chord tokens in order, e.g. ["Cmd", "Shift", "P"] or ["Ctrl", "C"]. */ + keys: string[]; + /** Optional human-readable command label, e.g. "Command Palette" or "Copy". */ + label?: string; + /** Explicit shortcut flag; when omitted, inferred from modifiers and special keys. */ + isShortcut?: boolean; +} + +export interface KeystrokeTelemetryFile { + version: 1; + platform?: KeystrokePlatform; + samples: KeystrokeTelemetrySample[]; +} + +export interface CoalescedKeystrokeSpan { + startMs: number; + endMs: number; + /** Canonical "+"-joined key combo string, e.g. "Cmd+Shift+P". */ + keys: string; + label?: string; +} + +export interface KeystrokeTrackEvent { + /** Source timestamp in seconds on the asset's recording clock. */ + atSec: number; + /** Corresponding timestamp on the edited timeline, or null if not carried by any clip. */ + virtualSec: number | null; + keys: string; + label?: string; + isShortcut: boolean; + trimmed?: true; +} + +export interface KeystrokeTrack { + assetId: string; + sampleCount: number; + eventCount: number; + events: KeystrokeTrackEvent[]; +} + +const MODIFIER_CANONICAL: Record = { + cmd: "Cmd", + command: "Cmd", + meta: "Cmd", + super: "Win", + win: "Win", + windows: "Win", + "⌘": "Cmd", + ctrl: "Ctrl", + control: "Ctrl", + "⌃": "Ctrl", + alt: "Alt", + opt: "Option", + option: "Option", + "⌥": "Option", + shift: "Shift", + "⇧": "Shift", + commandorcontrol: "CmdOrCtrl", + cmdorctrl: "CmdOrCtrl", +}; + +const SPECIAL_KEY_CANONICAL: Record = { + enter: "Enter", + return: "Enter", + "↵": "Enter", + esc: "Esc", + escape: "Esc", + tab: "Tab", + "⇥": "Tab", + space: "Space", + " ": "Space", + backspace: "Backspace", + "⌫": "Backspace", + delete: "Delete", + del: "Delete", + "⌦": "Delete", + up: "Up", + arrowup: "Up", + "↑": "Up", + down: "Down", + arrowdown: "Down", + "↓": "Down", + left: "Left", + arrowleft: "Left", + "←": "Left", + right: "Right", + arrowright: "Right", + "→": "Right", + home: "Home", + end: "End", + pageup: "PageUp", + pgup: "PageUp", + pagedown: "PageDown", + pgdn: "PageDown", + plus: "+", +}; + +const MODIFIER_SET = new Set(["Cmd", "Win", "Ctrl", "Alt", "Option", "Shift", "CmdOrCtrl"]); + +const STANDALONE_SHORTCUT_KEY_SET = new Set([ + "Enter", + "Esc", + "Tab", + "F1", + "F2", + "F3", + "F4", + "F5", + "F6", + "F7", + "F8", + "F9", + "F10", + "F11", + "F12", +]); + +/** + * Normalize a single key token into a canonical form ("Cmd", "Ctrl", "Shift", "P", "Enter", etc.). + */ +export function normalizeKeyToken(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed) return ""; + const lower = trimmed.toLowerCase(); + if (MODIFIER_CANONICAL[lower]) return MODIFIER_CANONICAL[lower]; + if (SPECIAL_KEY_CANONICAL[lower]) return SPECIAL_KEY_CANONICAL[lower]; + if (/^f([1-9]|1[0-2])$/i.test(trimmed)) { + return trimmed.toUpperCase(); + } + if (trimmed.length === 1) { + return trimmed.toUpperCase(); + } + return trimmed; +} + +/** + * Parse either a "+"-separated shortcut string (e.g. "Cmd + Shift + P", "Ctrl++") + * or an array of key strings into canonical key tokens. + */ +export function parseShortcutKeys(input: string | readonly string[]): string[] { + if (Array.isArray(input)) { + return input.map((k) => normalizeKeyToken(String(k))).filter(Boolean); + } + const raw = String(input).trim(); + if (!raw) return []; + // Handle trailing "+" key (e.g. "Cmd++" or "Ctrl + +") + const EndsWithPlusKey = raw.endsWith("++") || /\+\s*\+$/.test(raw); + const base = EndsWithPlusKey ? raw.replace(/\+\s*\+$/, "+Plus") : raw; + return base + .split("+") + .map((part) => normalizeKeyToken(part)) + .filter(Boolean); +} + +/** + * Format a canonical key token for display inside a keycap HUD badge on the given platform. + */ +export function formatKeycapLabel(token: string, platform: KeystrokePlatform = "win32"): string { + const norm = normalizeKeyToken(token); + if (platform === "darwin") { + switch (norm) { + case "Cmd": + case "Win": + case "CmdOrCtrl": + return "⌘"; + case "Shift": + return "⇧"; + case "Option": + case "Alt": + return "⌥"; + case "Ctrl": + return "⌃"; + case "Enter": + return "↵"; + case "Backspace": + return "⌫"; + case "Delete": + return "⌦"; + case "Tab": + return "⇥"; + case "Esc": + return "⎋"; + case "Up": + return "↑"; + case "Down": + return "↓"; + case "Left": + return "←"; + case "Right": + return "→"; + default: + return norm; + } + } + switch (norm) { + case "CmdOrCtrl": + case "Cmd": + return "Ctrl"; + case "Option": + return "Alt"; + case "Up": + return "↑"; + case "Down": + return "↓"; + case "Left": + return "←"; + case "Right": + return "→"; + default: + return norm; + } +} + +/** + * Format a shortcut combo string or token list into platform-aware keycap display labels. + */ +export function formatShortcutKeycaps( + input: string | readonly string[], + platform: KeystrokePlatform = "win32", +): string[] { + return parseShortcutKeys(input).map((token) => formatKeycapLabel(token, platform)); +} + +/** + * Determine whether a telemetry sample represents an intentional shortcut / command key + * rather than plain prose / password typing (privacy-safe filtering). + */ +export function isShortcutSample(sample: KeystrokeTelemetrySample): boolean { + if (typeof sample.isShortcut === "boolean") { + return sample.isShortcut; + } + const tokens = parseShortcutKeys(sample.keys); + if (tokens.length === 0) return false; + const nonModifiers = tokens.filter((t) => !MODIFIER_SET.has(t)); + if (nonModifiers.length === 0) return false; + const hasPrimaryModifier = tokens.some((t) => MODIFIER_SET.has(t) && t !== "Shift"); + if (hasPrimaryModifier) return true; + const hasShift = tokens.includes("Shift"); + if (hasShift && nonModifiers.some((t) => t.length > 1)) return true; + if (tokens.length === 1 && STANDALONE_SHORTCUT_KEY_SET.has(tokens[0])) return true; + return false; +} + +/** + * Candidate sidecar paths for a recorded video file (`