feat(editor): multi-video overlay and split-screen layout regions (#1316)

electron/ai-edition/agent-tools.test.ts

@@ -198,6 +198,7 @@ describe("the mutating-tool table", () => {
 				"addSpeed",
 				"addTrim",
 				"addTrims",
+				"addVideoOverlay",
 				"addZoom",
 				"addZooms",
 				"addAutoZooms",
@@ -219,6 +220,7 @@ describe("the mutating-tool table", () => {
 				"setEditorSettings",
 				"setSpeed",
 				"setTrim",
+				"setVideoOverlay",
 				"setWordText",
 				"setZoom",
 				"setCaptionSettings",

electron/ai-edition/agent-tools.ts

@@ -78,6 +78,7 @@ import {
 	coalesceRegionsForRuler,
 	replacePillSpan,
 	resolvePillIds,
+	resolveRegionIds,
 } from "../../src/lib/ai-edition/timeline/timelineMap";
 import {
 	trimAppliesToClip,
@@ -150,6 +151,8 @@ function modifierIds(document: AxcutDocument, kind: ModifierKind): string[] {
 			return [...new Set(document.audioTracks.map(trackGroupId))];
 		case "shaderEffect":
 			return (document.shaderEffectRanges ?? []).map((region) => region.id);
+		case "videoOverlay":
+			return (document.videoOverlayRanges ?? []).map((region) => region.id);
 	}
 }
 
@@ -606,6 +609,7 @@ function modifierIdsOf(document: AxcutDocument): string[] {
 		...speedRegions.map((r) => r.id),
 		...cameraFullscreenRegions.map((r) => r.id),
 		...(document.shaderEffectRanges ?? []).map((r) => r.id),
+		...(document.videoOverlayRanges ?? []).map((r) => r.id),
 	];
 }
 
@@ -1048,6 +1052,43 @@ export const setCameraFullscreenArgs = z.object({
 	endSec: secondsSchema.optional(),
 });
 
+export const addVideoOverlayArgs = z.object({
+	assetId: z.string().min(1),
+	startSec: secondsSchema,
+	endSec: secondsSchema,
+	sourceOffsetSec: secondsSchema.default(0),
+	layout: z.enum(["side-by-side", "top-bottom", "pip", "wipe-compare"]).default("side-by-side"),
+	primaryPosition: z.enum(["left", "right", "top", "bottom"]).default("left"),
+	splitRatio: z.number().min(0.2).max(0.8).default(0.5),
+	pipAnchor: z
+		.enum(["top-left", "top-right", "bottom-left", "bottom-right"])
+		.default("bottom-right"),
+	pipScale: z.number().min(0.15).max(0.5).default(0.3),
+	gapPx: z.number().min(0).max(64).default(12),
+	cornerRadiusPx: z.number().min(0).max(64).default(12),
+	muted: z.boolean().default(true),
+	gainDb: z.number().min(-60).max(12).default(0),
+	label: z.string().default(""),
+});
+
+export const setVideoOverlayArgs = z.object({
+	videoOverlayId: z.string().min(1),
+	assetId: z.string().min(1).optional(),
+	startSec: secondsSchema.optional(),
+	endSec: secondsSchema.optional(),
+	sourceOffsetSec: secondsSchema.optional(),
+	layout: z.enum(["side-by-side", "top-bottom", "pip", "wipe-compare"]).optional(),
+	primaryPosition: z.enum(["left", "right", "top", "bottom"]).optional(),
+	splitRatio: z.number().min(0.2).max(0.8).optional(),
+	pipAnchor: z.enum(["top-left", "top-right", "bottom-left", "bottom-right"]).optional(),
+	pipScale: z.number().min(0.15).max(0.5).optional(),
+	gapPx: z.number().min(0).max(64).optional(),
+	cornerRadiusPx: z.number().min(0).max(64).optional(),
+	muted: z.boolean().optional(),
+	gainDb: z.number().min(-60).max(12).optional(),
+	label: z.string().optional(),
+});
+
 export const getTranscriptWordsArgs = z.object({
 	assetId: z.string().min(1).optional(),
 	startSec: secondsSchema.optional(),
@@ -1117,6 +1158,8 @@ export const OPENSCREEN_TOOL_NAMES = [
 	"setAnnotation",
 	"addCameraFullscreen",
 	"setCameraFullscreen",
+	"addVideoOverlay",
+	"setVideoOverlay",
 	"addEffect",
 	"setEffect",
 	"setEffectDefinition",
@@ -1199,6 +1242,8 @@ export const MUTATING_TOOL_NAMES: ReadonlySet<string> = new Set([
 	"setAnnotation",
 	"addCameraFullscreen",
 	"setCameraFullscreen",
+	"addVideoOverlay",
+	"setVideoOverlay",
 	"addEffect",
 	"setEffect",
 	"setEffectDefinition",
@@ -1470,6 +1515,27 @@ export function documentSnapshotForModel(
 			startSec: roundSec(c.startMs),
 			endSec: roundSec(c.endMs),
 		})),
+		...((document.videoOverlayRanges ?? []).length > 0
+			? {
+					videoOverlays: coalesceForAgent(document.videoOverlayRanges ?? []).map((v) => ({
+						id: v.id,
+						assetId: v.assetId,
+						startSec: roundSec(v.startMs),
+						endSec: roundSec(v.endMs),
+						sourceOffsetSec: v.sourceOffsetSec,
+						layout: v.layout,
+						primaryPosition: v.primaryPosition,
+						splitRatio: v.splitRatio,
+						pipAnchor: v.pipAnchor,
+						pipScale: v.pipScale,
+						gapPx: v.gapPx,
+						cornerRadiusPx: v.cornerRadiusPx,
+						muted: v.muted,
+						gainDb: v.gainDb ?? 0,
+						label: v.label,
+					})),
+				}
+			: {}),
 		// Where each effect is posed, and how. The code is not here: getEffect reads it.
 		effects: coalesceForAgent(document.shaderEffectRanges ?? []).map((r) => ({
 			id: r.id,
@@ -3771,6 +3837,160 @@ export function executeAgentTool(
 			};
 		}
 
+		case "addVideoOverlay": {
+			const parsed = addVideoOverlayArgs.safeParse(args);
+			if (!parsed.success) return failure(parsed.error.message);
+			const asset = document.assets.find((a) => a.id === parsed.data.assetId);
+			if (!asset || asset.kind !== "video") {
+				return failure(
+					`Unknown video asset: ${parsed.data.assetId}. Pass a video assetId from getCurrentDocument.assets.`,
+				);
+			}
+			const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec));
+			const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec));
+			if (endMs <= startMs) {
+				return failure("endSec must be greater than startSec.");
+			}
+			const id = createId("vov");
+			const anchored = anchorRegionsWithDerivedMs(
+				[
+					{
+						id,
+						startMs,
+						endMs,
+						assetId: parsed.data.assetId,
+						sourceOffsetSec: parsed.data.sourceOffsetSec,
+						layout: parsed.data.layout,
+						primaryPosition: parsed.data.primaryPosition,
+						splitRatio: parsed.data.splitRatio,
+						pipAnchor: parsed.data.pipAnchor,
+						pipScale: parsed.data.pipScale,
+						gapPx: parsed.data.gapPx,
+						cornerRadiusPx: parsed.data.cornerRadiusPx,
+						muted: parsed.data.muted,
+						gainDb: parsed.data.gainDb,
+						label: parsed.data.label || asset.label || "",
+						origin: "agent" as const,
+					},
+				],
+				document.timeline.clips,
+				() => createId("vov"),
+			);
+			const landing = landingOf(anchored, document);
+			if (!landing.anchored) {
+				return coversNoClip("videoOverlay", startMs / 1000, endMs / 1000, document);
+			}
+			const next: AxcutDocument = {
+				...document,
+				videoOverlayRanges: [
+					...(document.videoOverlayRanges ?? []),
+					...anchored,
+				] as AxcutDocument["videoOverlayRanges"],
+			};
+			const first = anchored[0];
+			return {
+				ok: true,
+				document: next,
+				resultJson: JSON.stringify({
+					videoOverlayId: first.id,
+					assetId: first.assetId,
+					layout: first.layout,
+					splitRatio: first.splitRatio,
+					gainDb: first.gainDb,
+					startSec: landing.startSec,
+					endSec: landing.endSec,
+				}),
+				summary: `added ${first.layout} video overlay (${asset.label || first.assetId}) ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}`,
+			};
+		}
+
+		case "setVideoOverlay": {
+			const parsed = setVideoOverlayArgs.safeParse(args);
+			if (!parsed.success) return failure(parsed.error.message);
+			const { videoOverlayId, startSec, endSec, ...patchFields } = parsed.data;
+			const currentList = document.videoOverlayRanges ?? [];
+			const target = currentList.find((r) => r.id === videoOverlayId);
+			if (!target) {
+				return failure(`Unknown videoOverlay: ${videoOverlayId}`);
+			}
+			if (patchFields.assetId !== undefined) {
+				const asset = document.assets.find((a) => a.id === patchFields.assetId);
+				if (!asset || asset.kind !== "video") {
+					return failure(`Unknown video asset: ${patchFields.assetId}`);
+				}
+			}
+			const origGroupIds = new Set(resolveRegionIds(currentList, videoOverlayId));
+			const pill = coalesceForAgent(currentList).find((p) => origGroupIds.has(p.id)) ?? target;
+			let updatedList = currentList;
+			let targetGroupIds = origGroupIds;
+			if (startSec !== undefined || endSec !== undefined) {
+				const { startMs, endMs } = resolveSpanMs(pill, startSec, endSec);
+				if (endMs <= startMs) return failure("endSec must be greater than startSec.");
+				const respan = (replacePillSpan(
+					currentList,
+					videoOverlayId,
+					startMs,
+					endMs,
+					document.timeline.clips,
+					() => createId("vov"),
+				) ?? []) as NonNullable<AxcutDocument["videoOverlayRanges"]>;
+				const landing = landingAfterPillEdit(currentList, respan, origGroupIds, document);
+				if (!landing.anchored) {
+					return coversNoClip("videoOverlay", startMs / 1000, endMs / 1000, document);
+				}
+				updatedList = respan;
+				targetGroupIds = new Set(landing.ids);
+			}
+			const definedPatch = Object.fromEntries(
+				Object.entries(patchFields).filter(([, v]) => v !== undefined),
+			);
+			if (Object.keys(definedPatch).length > 0) {
+				const { sourceOffsetSec: headOffset, ...flatPatch } = definedPatch as {
+					sourceOffsetSec?: number;
+				} & Record<string, unknown>;
+				const headStartMs = Math.min(
+					...updatedList.filter((r) => targetGroupIds.has(r.id)).map((r) => r.startMs),
+				);
+				updatedList = updatedList.map((r) =>
+					targetGroupIds.has(r.id)
+						? {
+								...r,
+								...flatPatch,
+								...(headOffset !== undefined
+									? {
+											sourceOffsetSec:
+												Math.round(
+													(headOffset + Math.max(0, r.startMs - headStartMs) / 1000) * 1000,
+												) / 1000,
+										}
+									: {}),
+							}
+						: r,
+				);
+			}
+			const next: AxcutDocument = {
+				...document,
+				videoOverlayRanges: updatedList,
+			};
+			const after =
+				coalesceForAgent(next.videoOverlayRanges ?? []).find((p) => targetGroupIds.has(p.id)) ??
+				target;
+			return {
+				ok: true,
+				document: next,
+				resultJson: JSON.stringify({
+					videoOverlayId: after.id,
+					assetId: after.assetId,
+					layout: after.layout,
+					splitRatio: after.splitRatio,
+					gainDb: after.gainDb ?? 0,
+					startSec: roundSec(after.startMs),
+					endSec: roundSec(after.endMs),
+				}),
+				summary: `updated video overlay ${after.id} (${after.layout})`,
+			};
+		}
+
 		case "removeTrim": {
 			const parsed = removeTrimArgs.safeParse(args);
 			if (!parsed.success) return failure(parsed.error.message);
@@ -3802,9 +4022,10 @@ export function executeAgentTool(
 			else if (cameraFullscreenRegions.some((c) => c.id === id)) kind = "cameraFullscreen";
 			else if (document.audioTracks.some((t) => trackGroupId(t) === id)) kind = "audio";
 			else if ((document.shaderEffectRanges ?? []).some((r) => r.id === id)) kind = "shaderEffect";
+			else if ((document.videoOverlayRanges ?? []).some((r) => r.id === id)) kind = "videoOverlay";
 			if (!kind) {
 				return failure(
-					`No zoom / speed / annotation / full-camera / audio / effect modifier with id ${id}. ` +
+					`No zoom / speed / annotation / full-camera / audio / effect / videoOverlay modifier with id ${id}. ` +
 						`For a trim use removeTrim; for a clip use removeClip.`,
 				);
 			}

electron/ai-edition/deep-agent/service.test.ts

@@ -83,6 +83,8 @@ const ARGS: Record<string, unknown> = {
 	setAnnotation: { annotationId: "ann_nope" },
 	addCameraFullscreen: { startSec: 1, endSec: 2 },
 	setCameraFullscreen: { cameraFullscreenId: "cam_nope" },
+	addVideoOverlay: { assetId: "asset_1", startSec: 1, endSec: 2, layout: "side-by-side" },
+	setVideoOverlay: { videoOverlayId: "vov_nope" },
 	// No checker in these tests: the effect is stored unchecked, and the result says so.
 	addEffect: {
 		startSec: 1,

electron/ai-edition/deep-agent/service.ts

@@ -60,6 +60,7 @@ import {
 	addSpeedArgs,
 	addTrimArgs,
 	addTrimsArgs,
+	addVideoOverlayArgs,
 	addZoomArgs,
 	addZoomsArgs,
 	type CursorTelemetryLoad,
@@ -87,6 +88,7 @@ import {
 	setEffectDefinitionArgs,
 	setSpeedArgs,
 	setTrimArgs,
+	setVideoOverlayArgs,
 	setWordTextArgs,
 	setZoomArgs,
 	shaderEffectCheckLookup,
@@ -274,12 +276,16 @@ export const TOOL_DESCRIPTIONS: Record<string, string> = {
 		"Read one effect of the project in full: its WGSL code, parameters, description, cost and the regions that use it. getCurrentDocument lists effects without their code.",
 	setCameraFullscreen:
 		"Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.",
+	addVideoOverlay:
+		"Present a second video asset simultaneously with the primary recording over a span of the edited timeline (virtual seconds) in a split-screen, picture-in-picture, or wipe-compare layout. `assetId` must name a video asset in `getCurrentDocument.assets`. `layout` selects `'side-by-side'`, `'top-bottom'`, `'pip'`, or `'wipe-compare'`; `primaryPosition` (`'left'|'right'|'top'|'bottom'`) picks which side the primary track occupies; `splitRatio` (0.2–0.8, default 0.5) sets the fraction of the screen allocated to the primary video; `gainDb` (−60 to 12, default 0) sets the secondary video audio level when `muted` is false.",
+	setVideoOverlay:
+		"Move, resize, or change the layout, splitRatio, primaryPosition, pipAnchor, pipScale, sourceOffsetSec, muted, or gainDb state of an existing video overlay region by id (virtual-timeline seconds). Only the fields you pass are changed.",
 	addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them. If there is none, a file has to be imported first (importMedia, where it is offered): never guess an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`,
 	setAudio: `Move, resize, re-level, re-lane, mute, loop, fade or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to fade it in or out (fadeInMs / fadeOutMs, 0 to ${AUDIO_FADE_MAX_MS} ms), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.`,
 	removeTrim:
 		"Delete a trim range by id — the cut is undone and that span plays/exports again. This is how you 'remove a trim'; never re-add a trim to undo one.",
 	removeModifier:
-		"Delete a modifier (zoom / speed / annotation / camera-fullscreen / audio / effect) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.",
+		"Delete a modifier (zoom / speed / annotation / camera-fullscreen / video-overlay / audio / effect) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.",
 	removeClip:
 		"Delete a placed clip by id; remaining clips close the gap and effects anchored to it are dropped. Use only when the user asks to remove a clip — to shorten one, use setClipRange.",
 	getCaptions:
@@ -558,6 +564,8 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray<readonly [string, z.ZodObject]> = [
 	["setAnnotation", setAnnotationArgs],
 	["addCameraFullscreen", addCameraFullscreenArgs],
 	["setCameraFullscreen", setCameraFullscreenArgs],
+	["addVideoOverlay", addVideoOverlayArgs],
+	["setVideoOverlay", setVideoOverlayArgs],
 	["addEffect", addEffectArgs],
 	["setEffect", setEffectArgs],
 	["setEffectDefinition", setEffectDefinitionArgs],