From dc725cdbfc5c810657cf721c5a7c590b6053960a Mon Sep 17 00:00:00 2001 From: rainyflash <1317811579@qq.com> Date: Sat, 17 Oct 2026 09:37:06 -0400 Subject: [PATCH 1/6] feat(editor): add multi-video overlay and split-screen layout regions (#1316) --- electron/ai-edition/agent-tools.ts | 192 +++++++++++++++++- .../ai-edition/deep-agent/service.test.ts | 2 + electron/ai-edition/deep-agent/service.ts | 8 + src/components/ai-edition/NewEditorShell.tsx | 20 +- src/components/ai-edition/Preview.tsx | 4 + src/components/ai-edition/PreviewCanvas.tsx | 138 +++++++++++++ .../ai-edition/v4/FloatingInspector.tsx | 148 ++++++++++++++ src/components/ai-edition/v4/V4Timeline.tsx | 40 +++- src/lib/ai-edition/document/timeline.ts | 21 +- .../ai-edition/document/videoOverlay.test.ts | 152 ++++++++++++++ src/lib/ai-edition/schema/index.ts | 46 +++++ src/lib/ai-edition/store/regionClipboard.ts | 1 + src/lib/ai-edition/store/useTimeline.test.ts | 5 + src/lib/ai-edition/store/useTimeline.ts | 116 +++++++++++ src/native/sceneDescription.ts | 59 ++++++ 15 files changed, 946 insertions(+), 6 deletions(-) create mode 100644 src/lib/ai-edition/document/videoOverlay.test.ts diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 50bb5de41..da03888d6 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -78,6 +78,7 @@ import { coalesceRegionsForRuler, replacePillSpan, resolvePillIds, + resolveRegionIds, } from "../../src/lib/ai-edition/timeline/timelineMap"; import { trimAppliesToClip, @@ -150,6 +151,8 @@ function modifierIds(document: AxcutDocument, kind: ModifierKind): string[] { return [...new Set(document.audioTracks.map(trackGroupId))]; case "shaderEffect": return (document.shaderEffectRanges ?? []).map((region) => region.id); + case "videoOverlay": + return (document.videoOverlayRanges ?? []).map((region) => region.id); } } @@ -1048,6 +1051,41 @@ export const setCameraFullscreenArgs = z.object({ endSec: secondsSchema.optional(), }); +export const addVideoOverlayArgs = z.object({ + assetId: z.string().min(1), + startSec: secondsSchema, + endSec: secondsSchema, + sourceOffsetSec: secondsSchema.default(0), + layout: z.enum(["side-by-side", "top-bottom", "pip", "wipe-compare"]).default("side-by-side"), + primaryPosition: z.enum(["left", "right", "top", "bottom"]).default("left"), + splitRatio: z.number().min(0.2).max(0.8).default(0.5), + pipAnchor: z + .enum(["top-left", "top-right", "bottom-left", "bottom-right"]) + .default("bottom-right"), + pipScale: z.number().min(0.15).max(0.5).default(0.3), + gapPx: z.number().min(0).max(64).default(12), + cornerRadiusPx: z.number().min(0).max(64).default(12), + muted: z.boolean().default(true), + label: z.string().default(""), +}); + +export const setVideoOverlayArgs = z.object({ + videoOverlayId: z.string().min(1), + assetId: z.string().min(1).optional(), + startSec: secondsSchema.optional(), + endSec: secondsSchema.optional(), + sourceOffsetSec: secondsSchema.optional(), + layout: z.enum(["side-by-side", "top-bottom", "pip", "wipe-compare"]).optional(), + primaryPosition: z.enum(["left", "right", "top", "bottom"]).optional(), + splitRatio: z.number().min(0.2).max(0.8).optional(), + pipAnchor: z.enum(["top-left", "top-right", "bottom-left", "bottom-right"]).optional(), + pipScale: z.number().min(0.15).max(0.5).optional(), + gapPx: z.number().min(0).max(64).optional(), + cornerRadiusPx: z.number().min(0).max(64).optional(), + muted: z.boolean().optional(), + label: z.string().optional(), +}); + export const getTranscriptWordsArgs = z.object({ assetId: z.string().min(1).optional(), startSec: secondsSchema.optional(), @@ -1117,6 +1155,8 @@ export const OPENSCREEN_TOOL_NAMES = [ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addVideoOverlay", + "setVideoOverlay", "addEffect", "setEffect", "setEffectDefinition", @@ -1199,6 +1239,8 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "setAnnotation", "addCameraFullscreen", "setCameraFullscreen", + "addVideoOverlay", + "setVideoOverlay", "addEffect", "setEffect", "setEffectDefinition", @@ -1470,6 +1512,26 @@ export function documentSnapshotForModel( startSec: roundSec(c.startMs), endSec: roundSec(c.endMs), })), + ...((document.videoOverlayRanges ?? []).length > 0 + ? { + videoOverlays: coalesceForAgent(document.videoOverlayRanges ?? []).map((v) => ({ + id: v.id, + assetId: v.assetId, + startSec: roundSec(v.startMs), + endSec: roundSec(v.endMs), + sourceOffsetSec: v.sourceOffsetSec, + layout: v.layout, + primaryPosition: v.primaryPosition, + splitRatio: v.splitRatio, + pipAnchor: v.pipAnchor, + pipScale: v.pipScale, + gapPx: v.gapPx, + cornerRadiusPx: v.cornerRadiusPx, + muted: v.muted, + label: v.label, + })), + } + : {}), // Where each effect is posed, and how. The code is not here: getEffect reads it. effects: coalesceForAgent(document.shaderEffectRanges ?? []).map((r) => ({ id: r.id, @@ -3771,6 +3833,133 @@ export function executeAgentTool( }; } + case "addVideoOverlay": { + const parsed = addVideoOverlayArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const asset = document.assets.find((a) => a.id === parsed.data.assetId); + if (!asset || asset.kind !== "video") { + return failure( + `Unknown video asset: ${parsed.data.assetId}. Pass a video assetId from getCurrentDocument.assets.`, + ); + } + const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec)); + const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec)); + if (endMs <= startMs) { + return failure("endSec must be greater than startSec."); + } + const id = createId("vov"); + const anchored = anchorRegionsWithDerivedMs( + [ + { + id, + startMs, + endMs, + assetId: parsed.data.assetId, + sourceOffsetSec: parsed.data.sourceOffsetSec, + layout: parsed.data.layout, + primaryPosition: parsed.data.primaryPosition, + splitRatio: parsed.data.splitRatio, + pipAnchor: parsed.data.pipAnchor, + pipScale: parsed.data.pipScale, + gapPx: parsed.data.gapPx, + cornerRadiusPx: parsed.data.cornerRadiusPx, + muted: parsed.data.muted, + label: parsed.data.label || asset.label || "", + origin: "agent" as const, + }, + ], + document.timeline.clips, + () => createId("vov"), + ); + if (anchored.length === 0) { + return coversNoClip("videoOverlay", startMs / 1000, endMs / 1000, document); + } + const next: AxcutDocument = { + ...document, + videoOverlayRanges: [ + ...(document.videoOverlayRanges ?? []), + ...anchored, + ] as AxcutDocument["videoOverlayRanges"], + }; + const first = anchored[0]; + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + videoOverlayId: first.id, + assetId: first.assetId, + layout: first.layout, + splitRatio: first.splitRatio, + startSec: roundSec(first.startMs), + endSec: roundSec(anchored[anchored.length - 1].endMs), + }), + summary: `added ${first.layout} video overlay (${asset.label || first.assetId}) ${formatSec(roundSec(first.startMs))} – ${formatSec(roundSec(anchored[anchored.length - 1].endMs))}`, + }; + } + + case "setVideoOverlay": { + const parsed = setVideoOverlayArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { videoOverlayId, startSec, endSec, ...patchFields } = parsed.data; + const currentList = document.videoOverlayRanges ?? []; + const target = currentList.find((r) => r.id === videoOverlayId); + if (!target) { + return failure(`Unknown videoOverlay: ${videoOverlayId}`); + } + if (patchFields.assetId !== undefined) { + const asset = document.assets.find((a) => a.id === patchFields.assetId); + if (!asset || asset.kind !== "video") { + return failure(`Unknown video asset: ${patchFields.assetId}`); + } + } + let updatedList = currentList; + const definedPatch = Object.fromEntries( + Object.entries(patchFields).filter(([, v]) => v !== undefined), + ); + if (Object.keys(definedPatch).length > 0) { + const groupIds = new Set(resolveRegionIds(currentList, videoOverlayId)); + updatedList = currentList.map((r) => (groupIds.has(r.id) ? { ...r, ...definedPatch } : r)); + } + if (startSec !== undefined || endSec !== undefined) { + const pill = coalesceForAgent(updatedList).find((p) => p.id === videoOverlayId) ?? target; + const { startMs, endMs } = resolveSpanMs(pill, startSec, endSec); + if (endMs <= startMs) return failure("endSec must be greater than startSec."); + const respan = replacePillSpan( + updatedList, + videoOverlayId, + startMs, + endMs, + document.timeline.clips, + () => createId("vov"), + ) as AxcutDocument["videoOverlayRanges"]; + if (!respan || respan.length === 0) { + return coversNoClip("videoOverlay", startMs / 1000, endMs / 1000, document); + } + updatedList = respan; + } + const next: AxcutDocument = { + ...document, + videoOverlayRanges: updatedList, + }; + const after = + coalesceForAgent(next.videoOverlayRanges ?? []).find((p) => p.id === videoOverlayId) ?? + (next.videoOverlayRanges ?? [])[0] ?? + target; + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + videoOverlayId: after.id, + assetId: after.assetId, + layout: after.layout, + splitRatio: after.splitRatio, + startSec: roundSec(after.startMs), + endSec: roundSec(after.endMs), + }), + summary: `updated video overlay ${after.id} (${after.layout})`, + }; + } + case "removeTrim": { const parsed = removeTrimArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); @@ -3802,9 +3991,10 @@ export function executeAgentTool( else if (cameraFullscreenRegions.some((c) => c.id === id)) kind = "cameraFullscreen"; else if (document.audioTracks.some((t) => trackGroupId(t) === id)) kind = "audio"; else if ((document.shaderEffectRanges ?? []).some((r) => r.id === id)) kind = "shaderEffect"; + else if ((document.videoOverlayRanges ?? []).some((r) => r.id === id)) kind = "videoOverlay"; if (!kind) { return failure( - `No zoom / speed / annotation / full-camera / audio / effect modifier with id ${id}. ` + + `No zoom / speed / annotation / full-camera / audio / effect / videoOverlay modifier with id ${id}. ` + `For a trim use removeTrim; for a clip use removeClip.`, ); } diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 0a368e95b..6f1790a1e 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -83,6 +83,8 @@ const ARGS: Record = { setAnnotation: { annotationId: "ann_nope" }, addCameraFullscreen: { startSec: 1, endSec: 2 }, setCameraFullscreen: { cameraFullscreenId: "cam_nope" }, + addVideoOverlay: { assetId: "asset_1", startSec: 1, endSec: 2, layout: "side-by-side" }, + setVideoOverlay: { videoOverlayId: "vov_nope" }, // No checker in these tests: the effect is stored unchecked, and the result says so. addEffect: { startSec: 1, diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 00fc819ee..61dfea973 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -60,6 +60,7 @@ import { addSpeedArgs, addTrimArgs, addTrimsArgs, + addVideoOverlayArgs, addZoomArgs, addZoomsArgs, type CursorTelemetryLoad, @@ -87,6 +88,7 @@ import { setEffectDefinitionArgs, setSpeedArgs, setTrimArgs, + setVideoOverlayArgs, setWordTextArgs, setZoomArgs, shaderEffectCheckLookup, @@ -274,6 +276,10 @@ export const TOOL_DESCRIPTIONS: Record = { "Read one effect of the project in full: its WGSL code, parameters, description, cost and the regions that use it. getCurrentDocument lists effects without their code.", setCameraFullscreen: "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.", + addVideoOverlay: + "Present a second video asset simultaneously with the primary recording over a span of the edited timeline (virtual seconds) in a split-screen, picture-in-picture, or wipe-compare layout. `assetId` must name a video asset in `getCurrentDocument.assets`. `layout` selects `'side-by-side'`, `'top-bottom'`, `'pip'`, or `'wipe-compare'`; `primaryPosition` (`'left'|'right'|'top'|'bottom'`) picks which side the primary track occupies; `splitRatio` (0.2–0.8, default 0.5) sets the fraction of the screen allocated to the primary video.", + setVideoOverlay: + "Move, resize, or change the layout, splitRatio, primaryPosition, pipAnchor, pipScale, sourceOffsetSec, or muted state of an existing video overlay region by id (virtual-timeline seconds). Only the fields you pass are changed.", addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them. If there is none, a file has to be imported first (importMedia, where it is offered): never guess an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`, setAudio: `Move, resize, re-level, re-lane, mute, loop, fade or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to fade it in or out (fadeInMs / fadeOutMs, 0 to ${AUDIO_FADE_MAX_MS} ms), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.`, removeTrim: @@ -558,6 +564,8 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ ["setAnnotation", setAnnotationArgs], ["addCameraFullscreen", addCameraFullscreenArgs], ["setCameraFullscreen", setCameraFullscreenArgs], + ["addVideoOverlay", addVideoOverlayArgs], + ["setVideoOverlay", setVideoOverlayArgs], ["addEffect", addEffectArgs], ["setEffect", setEffectArgs], ["setEffectDefinition", setEffectDefinitionArgs], diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index 13e5c1c47..683b89fd7 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -1075,7 +1075,9 @@ export function NewEditorShell() { ? "ann" : snapshot.kind === "shaderEffect" ? "fxr" - : snapshot.kind; + : snapshot.kind === "videoOverlay" + ? "vov" + : snapshot.kind; // Audio re-ventilates through its own anchorer, which advances each // fragment's source offset — the generic one would copy the offset into @@ -1142,6 +1144,17 @@ export function NewEditorShell() { return; } await saveDocument(next, { history: true }); + } else if (snapshot.kind === "videoOverlay") { + await saveDocument( + { + ...doc, + videoOverlayRanges: [ + ...(doc.videoOverlayRanges ?? []), + ...anchored, + ] as typeof doc.videoOverlayRanges, + }, + { history: true }, + ); } else { // speed and cameraFullscreen are both plain spans on legacyEditor. const key = snapshot.kind === "speed" ? "speedRegions" : "cameraFullscreenRegions"; @@ -1227,7 +1240,9 @@ export function NewEditorShell() { ? tl.annotationRegions : sel.kind === "speed" ? tl.speedRegions - : tl.cameraFullscreenRegions; + : sel.kind === "videoOverlay" + ? tl.videoOverlayRanges + : tl.cameraFullscreenRegions; const region = wholePillOf( source as Array<{ id: string; startMs: number; endMs: number }>, sel.id, @@ -1644,6 +1659,7 @@ export function NewEditorShell() { zoomRegions={tl.zoomRegions} speedRegions={tl.speedRegions} cameraFullscreenRegions={tl.cameraFullscreenRegions} + videoOverlayRanges={tl.videoOverlayRanges} trimRanges={tl.trimRanges} selectedZoomRegionId={tl.selection?.kind === "zoom" ? tl.selection.id : null} onZoomFocusChange={tl.updateZoomFocusLive} diff --git a/src/components/ai-edition/Preview.tsx b/src/components/ai-edition/Preview.tsx index f4ad0a5c5..e89a3f30c 100644 --- a/src/components/ai-edition/Preview.tsx +++ b/src/components/ai-edition/Preview.tsx @@ -7,6 +7,7 @@ import type { AxcutClip, AxcutTranscript, AxcutTrimRange, + AxcutVideoOverlayRange, AxcutZoomRegion, } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; @@ -38,6 +39,7 @@ interface PreviewProps { zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; + videoOverlayRanges?: AxcutVideoOverlayRange[]; trimRanges?: AxcutTrimRange[]; transcripts?: AxcutTranscript[]; selectedZoomRegionId?: string | null; @@ -73,6 +75,7 @@ export function Preview({ zoomRegions, speedRegions, cameraFullscreenRegions, + videoOverlayRanges, trimRanges, transcripts, selectedZoomRegionId, @@ -216,6 +219,7 @@ export function Preview({ zoomRegions={zoomRegions} speedRegions={speedRegions} cameraFullscreenRegions={cameraFullscreenRegions} + videoOverlayRanges={videoOverlayRanges} trimRanges={trimRanges} transcripts={transcripts} selectedZoomRegionId={selectedZoomRegionId} diff --git a/src/components/ai-edition/PreviewCanvas.tsx b/src/components/ai-edition/PreviewCanvas.tsx index c817fde3c..b90f22c87 100644 --- a/src/components/ai-edition/PreviewCanvas.tsx +++ b/src/components/ai-edition/PreviewCanvas.tsx @@ -43,6 +43,7 @@ import type { AxcutClip, AxcutTranscript, AxcutTrimRange, + AxcutVideoOverlayRange, AxcutZoomRegion, } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; @@ -86,6 +87,7 @@ interface PreviewCanvasProps { zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; cameraFullscreenRegions?: CameraFullscreenRegion[]; + videoOverlayRanges?: AxcutVideoOverlayRange[]; trimRanges?: AxcutTrimRange[]; /** Relayed to `VirtualPreview` so a clip carrying added words plays their extensions. */ transcripts?: AxcutTranscript[]; @@ -492,6 +494,14 @@ export function PreviewCanvas(props: PreviewCanvasProps) { ) : null} + {props.videoOverlayRanges && props.videoOverlayRanges.length > 0 ? ( + + ) : null} {/* Last, so a selected annotation over the camera takes the pointer before the camera's drag hitbox does. It spans the frame: text, images and arrows move anywhere in it, over the padding too. */} @@ -516,6 +526,134 @@ export function PreviewCanvas(props: PreviewCanvasProps) { ); } +function VideoOverlayPreviewLayer({ + ranges, + sources, + currentTimeSec, + isPlaying, +}: { + ranges: AxcutVideoOverlayRange[]; + sources: VideoSource[]; + currentTimeSec: number; + isPlaying: boolean; +}) { + const videoRef = useRef(null); + const nowMs = currentTimeSec * 1000; + const active = ranges.find((r) => r.startMs <= nowMs && nowMs < r.endMs) ?? null; + const srcUrl = active ? (sources.find((s) => s.id === active.assetId)?.src ?? null) : null; + const targetTimeSec = active + ? Math.max(0, currentTimeSec - active.startMs / 1000 + active.sourceOffsetSec) + : 0; + + useEffect(() => { + const el = videoRef.current; + if (!el || !active || !srcUrl) return; + if (Math.abs(el.currentTime - targetTimeSec) > 0.12) { + el.currentTime = targetTimeSec; + } + if (isPlaying && el.paused) { + void el.play().catch(() => undefined); + } else if (!isPlaying && !el.paused) { + el.pause(); + } + }, [active, srcUrl, targetTimeSec, isPlaying]); + + if (!active || !srcUrl) return null; + + const ratioPct = Math.round(active.splitRatio * 100); + const secondaryPct = 100 - ratioPct; + const halfGapPx = active.gapPx / 2; + + let boxStyle: React.CSSProperties; + if (active.layout === "side-by-side") { + const isSecondaryRight = active.primaryPosition !== "right"; + boxStyle = { + position: "absolute", + top: active.gapPx, + bottom: active.gapPx, + left: isSecondaryRight ? `calc(${ratioPct}% + ${halfGapPx}px)` : `${active.gapPx}px`, + width: `calc(${secondaryPct}% - ${active.gapPx + halfGapPx}px)`, + borderRadius: `${active.cornerRadiusPx}px`, + overflow: "hidden", + boxShadow: "0 12px 32px rgba(0, 0, 0, 0.35)", + background: "#000", + pointerEvents: "none", + zIndex: 4, + }; + } else if (active.layout === "top-bottom") { + const isSecondaryBottom = active.primaryPosition !== "bottom"; + boxStyle = { + position: "absolute", + left: active.gapPx, + right: active.gapPx, + top: isSecondaryBottom ? `calc(${ratioPct}% + ${halfGapPx}px)` : `${active.gapPx}px`, + height: `calc(${secondaryPct}% - ${active.gapPx + halfGapPx}px)`, + borderRadius: `${active.cornerRadiusPx}px`, + overflow: "hidden", + boxShadow: "0 12px 32px rgba(0, 0, 0, 0.35)", + background: "#000", + pointerEvents: "none", + zIndex: 4, + }; + } else if (active.layout === "pip") { + const inset = Math.max(12, active.gapPx); + boxStyle = { + position: "absolute", + width: `${Math.round(active.pipScale * 100)}%`, + aspectRatio: "16 / 9", + ...(active.pipAnchor.includes("top") ? { top: inset } : { bottom: inset }), + ...(active.pipAnchor.includes("left") ? { left: inset } : { right: inset }), + borderRadius: `${active.cornerRadiusPx}px`, + overflow: "hidden", + boxShadow: "0 12px 32px rgba(0, 0, 0, 0.45)", + background: "#000", + pointerEvents: "none", + zIndex: 4, + }; + } else { + // wipe-compare + boxStyle = { + position: "absolute", + inset: 0, + clipPath: `inset(0 0 0 ${ratioPct}%)`, + borderRadius: `${active.cornerRadiusPx}px`, + overflow: "hidden", + pointerEvents: "none", + zIndex: 4, + }; + } + + return ( + <> +
+
+ {active.layout === "wipe-compare" ? ( +
+ ) : null} + + ); +} + // Screen stage: rectangle from the composite layout, converted to percentages // of the canvas. Positions/sizes the zoom overlay hosted inside it and the // CSS-hidden