diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index e1ae63d1..10d62c22 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "cut-as-code-editor", - "version": "0.1.0+codex.20260823221007", + "version": "0.1.0+codex.20260924000100", "description": "Local Cut as Code video project review and editing runtime.", "author": { "name": "WhiteTowerAI", diff --git a/runtime/protocol_service.py b/runtime/protocol_service.py index df9a4245..17dce016 100644 --- a/runtime/protocol_service.py +++ b/runtime/protocol_service.py @@ -769,6 +769,7 @@ def audio_index(clip_id): source = video(command["clip_id"]) if source.get("audio_mode", "embedded") == "detached": raise ValueError("audio is already detached") + muted = source.get("audio_mode") == "muted" source["audio_mode"] = "detached" audio_clips.append({ "id": f"{source['id']}:audio", @@ -777,7 +778,7 @@ def audio_index(clip_id): "speed": source.get("speed", 1.0), "source_video_clip_id": source["id"], "linked": True, - "muted": False, + "muted": muted, **({"source_asset_id": source["source_asset_id"]} if source.get("source_asset_id") else {}), }) elif command_type == "attach-audio": @@ -786,7 +787,7 @@ def audio_index(clip_id): if source.get("audio_mode") != "detached" or len(matches) != 1 or not matches[0].get("linked"): raise ValueError("only linked detached audio can be attached") audio_clips.remove(matches[0]) - source["audio_mode"] = "embedded" + source["audio_mode"] = "muted" if matches[0].get("muted") else "embedded" elif command_type == "mute-video-audio": source = video(command["clip_id"]) if source.get("audio_mode") == "detached" or not isinstance(command["muted"], bool): diff --git a/runtime/tests/test_protocol_service.py b/runtime/tests/test_protocol_service.py index c0426b96..b7c58e42 100644 --- a/runtime/tests/test_protocol_service.py +++ b/runtime/tests/test_protocol_service.py @@ -644,6 +644,32 @@ def test_timeline_edit_deletes_and_restores_the_final_clip(self): self.assertEqual(["clip-1"], [clip["id"] for clip in timeline["clips"]]) self.assertEqual({"start_s": 0.0, "end_s": 1.0}, timeline["clips"][0]["program_range"]) + def test_audio_mute_survives_detach_attach_and_reload(self): + self._configure_cut_project() + opened = self.service.handle_request({"verb": "open_project", "project_root": str(self.root)}) + snapshot = opened["snapshot"] + for command, mode, muted in [ + ({"type": "mute-video-audio", "clip_id": "clip-1", "muted": True}, "muted", None), + ({"type": "detach-audio", "clip_id": "clip-1"}, "detached", True), + ({"type": "attach-audio", "clip_id": "clip-1"}, "muted", None), + ({"type": "mute-video-audio", "clip_id": "clip-1", "muted": False}, "embedded", None), + ({"type": "detach-audio", "clip_id": "clip-1"}, "detached", False), + ]: + with self.subTest(command=command): + result = self.service.handle_request({ + "verb": "timeline.edit", "project_id": opened["project_id"], + "read_set": self._timeline_read_set(snapshot), "command": command, + }) + self.assertTrue(result["ok"], result) + reloaded = self.service.handle_request({"verb": "open_project", "project_root": str(self.root)}) + snapshot = reloaded["snapshot"] + timeline = json.loads(self.timeline.read_text(encoding="utf-8")) + self.assertEqual(mode, timeline["clips"][0]["audio_mode"]) + if muted is not None: + self.assertEqual(muted, timeline["audio_clips"][0]["muted"]) + else: + self.assertEqual([], timeline.get("audio_clips", [])) + def test_detached_audio_commands_are_atomic_and_linked_edits_sync(self): self._configure_cut_project() timeline = json.loads(self.timeline.read_text(encoding="utf-8")) diff --git a/skills/video-understand/scripts/render_project.py b/skills/video-understand/scripts/render_project.py index f254938d..ab654b4b 100644 --- a/skills/video-understand/scripts/render_project.py +++ b/skills/video-understand/scripts/render_project.py @@ -222,8 +222,8 @@ def _build(plan, project_root, plan_dir): audio_chain = "" if abs(speed - 1.0) < 1e-9 else atempo_chain(speed) + "," label = f"timeline-audio-{segment_index}" graph.append( - f"[{next_input}:a:0]{audio_chain}aresample=48000,asetpts=PTS-STARTPTS+" - f"{float(segment['program_range']['start_s']):.9f}/TB[{label}]" + f"[{next_input}:a:0]{audio_chain}aresample=48000,asetpts=PTS-STARTPTS," + f"adelay={round(float(segment['program_range']['start_s']) * 48000)}S:all=1[{label}]" ) segment_labels.append(label) next_input += 1 diff --git a/skills/video-understand/tests/test_render_project.py b/skills/video-understand/tests/test_render_project.py index 40b931ad..a7b1c7ce 100644 --- a/skills/video-understand/tests/test_render_project.py +++ b/skills/video-understand/tests/test_render_project.py @@ -138,6 +138,25 @@ def test_detached_audio_renders_once_without_embedded_audio_doubling(self): self.assertTrue(output.is_file()) self.assertLess(abs(_mean_volume(output) - _mean_volume(source)), 1.5) + audio = timeline["audio_clips"][0] + audio.update({"linked": False, "source_range": {"start_s": 0, "end_s": 0.25}, + "program_range": {"start_s": 0.5, "end_s": 0.75}}) + (root / "work" / "timeline.json").write_text(json.dumps(timeline), encoding="utf-8") + render_project.render(plan, root) + self.assertLess(max(map(abs, _audio_samples(output, 0.05, 0.2))), 10) + self.assertGreater(max(map(abs, _audio_samples(output, 0.55, 0.15))), 1000) + + audio["muted"] = True + (root / "work" / "timeline.json").write_text(json.dumps(timeline), encoding="utf-8") + render_project.render(plan, root) + self.assertLess(max(map(abs, _audio_samples(output, 0.55, 0.15))), 10) + + timeline["audio_clips"] = [] + timeline["clips"][0]["audio_mode"] = "muted" + (root / "work" / "timeline.json").write_text(json.dumps(timeline), encoding="utf-8") + render_project.render(plan, root) + self.assertLess(max(map(abs, _audio_samples(output, 0.1))), 10) + def test_overlay_editor_transform_scales_and_positions_from_normalized_center(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) diff --git a/ui/src/editor/EditorShell.tsx b/ui/src/editor/EditorShell.tsx index 24d5a98a..94aea31a 100644 --- a/ui/src/editor/EditorShell.tsx +++ b/ui/src/editor/EditorShell.tsx @@ -686,7 +686,7 @@ export function projectFromSnapshot(base: EditorProjectView | null, snapshot: Ru muted: clip.muted, implicit: false, })) ?? [] - const implicitAudioClips = hasAudio ? videoClips.filter((clip) => !detachedAudioIds.has(clip.id)).map((clip) => ({ + const implicitAudioClips = hasAudio ? videoClips.filter((clip) => clip.audioMode !== 'detached' && !detachedAudioIds.has(clip.id)).map((clip) => ({ ...clip, id: `${clip.id}:embedded-audio`, trackId: 'track-audio', diff --git a/ui/src/editor/TimelinePanel.tsx b/ui/src/editor/TimelinePanel.tsx index 97850eb4..1329efad 100644 --- a/ui/src/editor/TimelinePanel.tsx +++ b/ui/src/editor/TimelinePanel.tsx @@ -51,6 +51,7 @@ import { trimAudioSourceAtProgramDelta, trimSourceAtProgramDelta, moveAudioStartAtProgramDelta, + type TimelineEditCommand, type TimelineTrimEdge, } from './timeline-edit' @@ -244,6 +245,7 @@ function Clip({ trimming, onTrimStart, onAudioMoveStart, + onMute, onContextMenu, }: { track: TrackView @@ -261,6 +263,7 @@ function Clip({ trimming: TimelineTrimEdge | null onTrimStart: (event: PointerEvent, clip: ClipView, edge: TimelineTrimEdge, kind?: 'video' | 'audio') => void onAudioMoveStart: (event: PointerEvent, clip: ClipView) => void + onMute: (clip: ClipView) => void onContextMenu: (event: ReactMouseEvent, track: TrackView, clip: ClipView) => void }) { const kind = track.kind @@ -298,9 +301,11 @@ function Clip({ className={`timeline-clip timeline-clip--${kind}`} data-timeline-clip={id} {...(kind === 'audio' && !clip.implicit ? { 'data-linked': String(Boolean(clip.linked)) } : {})} + data-muted={kind === 'audio' ? Boolean(clip.muted) : undefined} aria-label={`${track.name} ${kind === 'video' || kind === 'audio' ? 'clip' : 'cue'}`} aria-pressed={selected} onPointerDown={() => select({ kind, id })} + onClick={() => select({ kind, id })} onPointerDownCapture={(event) => { if (kind === 'audio' && !clip.implicit && !clip.linked && event.button === 0) onAudioMoveStart(event, clip) }} @@ -392,11 +397,17 @@ function Clip({ > )} - {kind === 'audio' && !clip.implicit && ( - - {clip.linked ? : } - {clip.muted ? : null} - + {kind === 'audio' && clip.implicit !== undefined && ( +
+ + {clip.linked ? : } + + +
)} {kind === 'video' && trimming && ( @@ -505,13 +516,27 @@ export function TimelinePanel({ store }: TimelinePanelProps) { const selectedVideo = selection?.kind === 'video' ? project?.tracks.find((track) => track.kind === 'video')?.clips?.find((clip) => clip.id === selection.id) : undefined + const selectedAudio = selection?.kind === 'audio' + ? project?.tracks.find((track) => track.kind === 'audio')?.clips?.find((clip) => clip.id === selection.id) + : undefined + const deleteCommand: TimelineEditCommand | undefined = selectedVideo + ? { type: 'delete', clipId: selectedVideo.id } + : selectedAudio && !selectedAudio.implicit && !selectedAudio.linked + ? { type: 'delete-audio', audioClipId: selectedAudio.id } : undefined const splitEnabled = Boolean( timelineEditable && selectedVideo && canSplitClip(project!, selectedVideo.id, currentTimeS) && !timelinePending, ) const deleteEnabled = Boolean( - timelineEditable && selectedVideo && !timelinePending, + timelineEditable && deleteCommand && !timelinePending, ) + function toggleAudioMute(clip: ClipView) { + select({ kind: 'audio', id: clip.id }) + if (clip.implicit && clip.linkedClipId) { + void editTimeline({ type: 'mute-video-audio', clipId: clip.linkedClipId, muted: !clip.muted }) + } else void editTimeline({ type: 'mute-audio', audioClipId: clip.id, muted: !clip.muted }) + } + useEffect(() => { if (!deleteNotice) return const timeout = window.setTimeout(() => setDeleteNotice(null), 5000) @@ -536,7 +561,8 @@ export function TimelinePanel({ store }: TimelinePanelProps) { useEffect(() => { const handleKeyDown = (event: KeyboardEvent) => { const target = event.target as HTMLElement | null - if (target?.matches('input, textarea, [contenteditable="true"]')) return + if (event.defaultPrevented || event.isComposing || event.altKey + || target?.closest('input, textarea, select, [contenteditable]:not([contenteditable="false"]), [role="menu"], [role="dialog"]')) return const modifier = event.ctrlKey || event.metaKey if (modifier && event.key.toLowerCase() === 'z') { event.preventDefault() @@ -544,14 +570,14 @@ export function TimelinePanel({ store }: TimelinePanelProps) { else void undoTimeline() return } - if ((event.key === 'Delete' || event.key === 'Backspace') && deleteEnabled && selectedVideo) { + if ((event.key === 'Delete' || event.key === 'Backspace') && !modifier && !event.repeat && deleteEnabled && deleteCommand) { event.preventDefault() - void editTimeline({ type: 'delete', clipId: selectedVideo.id }) + void editTimeline(deleteCommand) } } window.addEventListener('keydown', handleKeyDown) return () => window.removeEventListener('keydown', handleKeyDown) - }, [deleteEnabled, editTimeline, redoTimeline, selectedVideo, undoTimeline]) + }, [deleteEnabled, editTimeline, redoTimeline, selectedVideo, selectedAudio, undoTimeline]) useEffect(() => { if (!trimDrag || !project) return @@ -1188,10 +1214,7 @@ export function TimelinePanel({ store }: TimelinePanelProps) { label: clip.muted ? 'Unmute audio' : 'Mute audio', icon: clip.muted ? Volume2 : VolumeX, disabled: !editable, - onSelect: () => { - if (clip.implicit && video) void editTimeline({ type: 'mute-video-audio', clipId: video.id, muted: !clip.muted }) - else void editTimeline({ type: 'mute-audio', audioClipId: clip.id, muted: !clip.muted }) - }, + onSelect: () => toggleAudioMute(clip), }, { id: 'copy-audio-range', @@ -1354,7 +1377,7 @@ export function TimelinePanel({ store }: TimelinePanelProps) {
{!runtime && } - + {(hasMedia || timelinePast.length > 0 || timelineFuture.length > 0) && ( <> @@ -1368,14 +1391,19 @@ export function TimelinePanel({ store }: TimelinePanelProps) { {!runtime && } {!runtime && } {!runtime && } - + )} {timelinePending && Saving timeline…} {!timelinePending && timelineError && {timelineError}} - + + {Math.round(timelineZoom * 100)}%
@@ -1498,13 +1526,14 @@ export function TimelinePanel({ store }: TimelinePanelProps) { presentationInsetPx={presentationInsetPx} selection={selection} select={select} - editable={timelineEditable && ( + editable={timelineEditable && !timelinePending && ( track.kind === 'video' - || (track.kind === 'audio' && !clip.implicit && !clip.linked) + || track.kind === 'audio' )} trimming={trimDrag?.clipId === clip.id ? trimDrag.edge : null} onTrimStart={startTrim} onAudioMoveStart={startAudioMove} + onMute={toggleAudioMute} onContextMenu={openClipContextMenu} /> ))} diff --git a/ui/src/editor/ViewerPanel.tsx b/ui/src/editor/ViewerPanel.tsx index fc6c3c64..299bc3f7 100644 --- a/ui/src/editor/ViewerPanel.tsx +++ b/ui/src/editor/ViewerPanel.tsx @@ -227,7 +227,8 @@ function supportsNativePlaybackRate(playbackRate: number) { } } -function applyClipPlaybackRate(video: HTMLVideoElement, clip: ClipView) { +function applyClipPlaybackSettings(video: HTMLVideoElement, clip: ClipView) { + video.muted = clip.audioMode === 'muted' || clip.audioMode === 'detached' const playbackRate = playbackRateForClip(clip) if (playbackRate !== null && Math.abs(video.playbackRate - playbackRate) > 0.001) { try { @@ -445,6 +446,41 @@ const aspectOptions = [ type ViewerAspect = 'Original' | '16:9' | '9:16' | '1:1' | '4:5' | '4:3' +function TimelineAudio({ clip, url, timeS, playing }: { + clip: ClipView; url?: string; timeS: number; playing: boolean +}) { + const mediaRef = useRef(null) + const [ready, setReady] = useState(false) + const [error, setError] = useState(false) + const active = timeS >= clip.programRange.startS && timeS < clip.programRange.endS + useEffect(() => { + const audio = mediaRef.current + if (!audio) return + if (!ready || error) { audio.pause(); return } + if (!active || clip.muted) { audio.pause(); return } + const rate = playbackRateForClip(clip) + if (rate === null) return + try { audio.playbackRate = rate } catch { audio.pause(); setError(true); return } + const sourceS = clip.sourceRange.startS + (timeS - clip.programRange.startS) * rate + // ponytail: native clocks resync above 80 ms; use Web Audio for sample-accurate preview. + if (audio.paused || !playing || Math.abs(audio.currentTime - sourceS) > 0.08 * rate) audio.currentTime = sourceS + if (!playing) { audio.pause(); return } + if (audio.paused) void audio.play().catch((reason: unknown) => { + if (!(reason instanceof DOMException && reason.name === 'AbortError')) setError(true) + }) + }, [active, clip, error, playing, ready, timeS]) + useEffect(() => { + const audio = mediaRef.current + return () => { audio?.pause() } + }, []) + return <> +