diff --git a/README.md b/README.md index 3ff36d9..c1b19b8 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Most editors keep a project in a format built for their own app, not for you or - **Fades**: fade a clip up from black and down to black, its sound with it, or make a fade through black between two clips. Titles, logos and music fade in and out the same way, and the preview plays each fade as it will export. - **Colour**: brightness, contrast, saturation and temperature on any clip, logo or picture-in-picture, and one click to give every clip the same look, for flat log footage or two cameras that do not match. The preview draws it with the same maths the export renders. - **Transitions**: put a dissolve, a fade through black or white, a wipe, a slide, a blur or a pixelate on any cut, from the mark between two clips on the timeline, or on every cut at once. A transition is centred on its cut and moves no clip, so titles and music stay where you put them, and the preview draws each one frame for frame as it exports. -- **Captions**: switch them on for the whole video, pick one of twelve languages and one style. The words come from each clip's transcript, so captions follow every trim, split and reorder. +- **Captions**: switch them on for the whole video, pick one of twelve languages and one style. The words come from each clip's transcript, so captions follow every trim, split and reorder. Correct a misheard word or a name's spelling once and every video cut from that clip shows it. Download any video's transcript as text, with the time of each line. - **AI, in one conversation**: add your clips and say what you want beside the cut, in your own words: it builds the cut from them ("a 30 second video from my clips") or changes the one you have. It remembers the conversation, so "shorter" or "a square one too" knows what it refers to, and it asks back instead of guessing when a request could mean two things. A request that would watch a lot of footage says how much first and, on a yes, runs in the background: keep editing or close the tab, and it never lands on top of a change you made meanwhile. The model calls a fixed set of checked operations, so the edit is always a valid document. One request, one undo. - **Projects hold videos**: a project is your footage and every video you make from it (the full cut, short clips, a teaser), each with its own timeline, shapes and export, listed together in the project. - **Clips from a long video**: "5 vertical clips with a strong hook from this interview" reads what is said, picks the moments that stand on their own, opens each on its strongest line, and makes each its own 9:16 video inside the project, with captions, framed on the person speaking. Shots it is unsure how to frame keep the whole picture, and it tells you which. "Frame it for vertical" does the same for the cut you have. diff --git a/agent.md b/agent.md index 7c721ec..f15ae75 100644 --- a/agent.md +++ b/agent.md @@ -351,6 +351,8 @@ in line, and asks Drive again now for the clips waiting on it. `DELETE /api/proj | GET | `/api/assets/{id}/playback` | For long footage: `{ ready, hls, thumbnail, duration }` | | GET | `/api/assets/{id}/frame?t=` | One frame of long footage, as an image | | GET | `/api/assets/{id}/transcript?lang=en` | The clip's transcript for captions → `{ status, vtt? }` | +| GET | `/api/assets/{id}/transcript/text` | The transcript as a text file, one `[m:ss] words` line per cue; 409 with the reason while there is none | +| PUT | `/api/assets/{id}/transcript/cues` | `{ lang, start, text }` corrects what the cue starting at `start` seconds says → `{ status, vtt }` | | DELETE | `/api/assets/{id}` | Remove from the library; 409 `in_use`, naming the projects, while any project uses it | | PUT | `/api/drive/folder` | `{ folderId }` limits browsing to one folder, `null` clears it | | GET | `/api/projects` | List projects | @@ -553,9 +555,10 @@ guests are stacked in the vertical version: **Captions** are a project setting, not overlays: an optional `captions` block, `{ "enabled": true, "lang": "en", "style": { "size": 0.055, "position": "bottom", "margin": 0.08, "background": true, "outline": false, "color": -"#ffffff", "maxChars": 32 } }` (`size` and `margin` are shares of the frame's -height; `outline` draws a black outline sized to the text, with or without -the box). The words come +"#ffffff", "maxChars": 32 } }` (`size` is a share of the frame's shorter +side, 0.035 to 0.07 in the editor; `margin` is a share of its height; +`outline` draws a black outline sized to the text, with or without the box; +`maxChars` is cut down where it would not fit on two lines). The words come from each clip's transcript and the part of it the clip plays, so captions follow every trim, split and reorder; nothing is stored per caption. Only footage on the media service (uploaded in the editor, or imported from Drive) has a transcript: poll @@ -563,6 +566,10 @@ footage on the media service (uploaded in the editor, or imported from Drive) ha silent clip, `unavailable` for footage in app storage). Languages: en, it, es, fr, de, nl, pt, pl, cs, ru, ja, ko. Use text overlays for titles and anything placed by hand; don't add captions on footage that already shows subtitles. +To fix a misheard word or a name's spelling, correct the cue +(`PUT /api/assets/{id}/transcript/cues`): the fix is kept in the clip's +transcript, so every video cut from that clip shows it. Asking for the +transcript in another language replaces it, corrections included. ### Worked examples (read → transform → save) diff --git a/src/client/edit.tsx b/src/client/edit.tsx index 2f26aee..74312c0 100644 --- a/src/client/edit.tsx +++ b/src/client/edit.tsx @@ -21,6 +21,7 @@ import { keepEdgeFades, splitClip } from "../shared/split"; import { drawnStroke, maxStrokeWidth, outlineShadow } from "../shared/outline"; import { DEFAULT_CAPTIONS, + captionChars, captionText, captionTimeline, type CaptionLine, @@ -548,6 +549,8 @@ function mainDur(el: MainElement, srcDur: (src: string) => number | undefined): * around its start (`before` / `after` the cut), shortened to fit as the * export does (shared/transition.ts). */ +type Segment = ReturnType[number]; + function mainSegments(edl: Edl, srcDur: (src: string) => number | undefined) { const els = edl.main.elements; const { placed } = layOut( @@ -1339,7 +1342,7 @@ export function EditEditor({ () => [...new Set(edl.main.elements.filter((e) => e.type === "video" && e.src.startsWith("asset:")).map((e) => e.src.slice(6)))], [edl.main.elements], ); - const transcripts = useTranscripts(captionIds, edl.captions?.lang ?? "en", !!edl.captions?.enabled || tab === "captions"); + const [transcripts, setTranscript] = useTranscripts(captionIds, edl.captions?.lang ?? "en", !!edl.captions?.enabled || tab === "captions"); const captionLayer = useMemo(() => { const cfg = edl.captions; if (!cfg?.enabled) return null; @@ -1348,8 +1351,8 @@ export function EditEditor({ .map((sg) => ({ src: sg.el.src, start: sg.start, dur: sg.dur, trimStart: (sg.el as MainVideo).trimStart ?? 0 })); const cues = new Map(); for (const [src, t] of transcripts) if (t.status === "ready") cues.set(src, t.cues); - return { lines: captionTimeline(placed, cues, cfg.style.maxChars), style: cfg.style }; - }, [edl.captions, segments, transcripts]); + return { lines: captionTimeline(placed, cues, captionChars(cfg.style, view.output)), style: cfg.style }; + }, [edl.captions, view.output, segments, transcripts]); // Distinct video clips on the main track, in timeline order — the unit // Auto-cut operates on (the arrangement is the user's intent). @@ -1688,6 +1691,9 @@ export function EditEditor({ edl={view} update={update} transcripts={transcripts} + setTranscript={setTranscript} + segments={segments} + seek={seek} comments={comments} onShowComment={showComment} /> @@ -1900,6 +1906,9 @@ function LeftPanel({ edl, update, transcripts, + setTranscript, + segments, + seek, comments, onShowComment, }: { @@ -1912,6 +1921,9 @@ function LeftPanel({ edl: Edl; update: (fn: (d: Edl) => void, coalesce?: boolean) => void; transcripts: Map; + setTranscript: (src: string, vtt: string) => void; + segments: Segment[]; + seek: (t: number) => void; assets: Asset[]; setAssets: React.Dispatch>; onAdd: (a: Asset) => void; @@ -1935,6 +1947,8 @@ function LeftPanel({ const { ready: mediaReady, ingesting: mediaIngesting, failed: mediaFailed } = useMediaReady(library); const [deleting, setDeleting] = useState(null); const [deleteErr, setDeleteErr] = useState(""); + /** Why a transcript did not download, e.g. it is still being made. */ + const [transcriptNote, setTranscriptNote] = useState(""); const deleteAsset = async (a: Asset) => { setDeleteErr(""); @@ -1960,7 +1974,10 @@ function LeftPanel({ /* The rail is surface-sunken; the canvas beside it stays white. The step between them is small, and the border carries the separation. */
-
+ {/* On a short screen the last tabs were below the fold with no way to + reach them: there the tabs are icons only, and the column scrolls if + even those do not fit. */} +
{( [ ["media", Film, "Media"], @@ -1975,12 +1992,13 @@ function LeftPanel({ key={key} onClick={() => setTab(key)} aria-pressed={tab === key} - className={`relative w-11 py-2 rounded-sm flex flex-col items-center gap-1 text-fine ${ + title={label} + className={`relative w-11 py-2 shrink-0 rounded-sm flex flex-col items-center gap-1 text-fine ${ tab === key ? "bg-surface text-foreground shadow-raised" : "text-muted hover:text-foreground" }`} > - {label} + {label} {key === "comments" && comments.open > 0 && ( , @@ -2003,6 +2021,9 @@ function LeftPanel({ edl={edl} update={update} transcripts={transcripts} + setTranscript={setTranscript} + segments={segments} + seek={seek} clips={[...new Set(edl.main.elements.filter((e) => e.type === "video" && e.src.startsWith("asset:")).map((e) => e.src.slice(6)))] .map((id) => assets.find((x) => x.id === id)) .filter((x): x is Asset => !!x)} @@ -2056,6 +2077,7 @@ function LeftPanel({ /> )} {deleteErr &&
{deleteErr}
} + {transcriptNote &&
{transcriptNote}
} {deleting && (
)} -
{a.name}
+
{a.name}
+ {a.media_uid && isVideoAsset(a) && !preparing && ( + + )} {/* Shown on hover or keyboard focus. Screens without hover and agents (data-hover-only) always see it. */}
+ + +
+ ); +} + +/** + * What the captions say, cue by cue, for the speech on the timeline. A cue is + * corrected in the clip's transcript, so the fix shows in every video cut + * from that clip. + */ +function CaptionWords({ + lang, + segments, + transcripts, + setTranscript, + seek, +}: { + lang: string; + segments: Segment[]; + transcripts: Map; + setTranscript: (src: string, vtt: string) => void; + seek: (t: number) => void; +}) { + const [error, setError] = useState(""); + const rows = segments.flatMap((sg) => { + if (sg.el.type !== "video" || !sg.el.src.startsWith("asset:")) return []; + const src = sg.el.src; + const from = (sg.el as MainVideo).trimStart ?? 0; + const cues = transcripts.get(src)?.cues ?? []; + return cues + .filter((cue) => cue.end > from && cue.start < from + sg.dur) + .map((cue) => ({ src, cue, at: sg.start + Math.max(0, cue.start - from) })); + }); + if (rows.length === 0) return null; + + const save = async (src: string, cue: Cue, text: string) => { + const next = text.replace(/\s+/g, " ").trim(); + if (!next || next === cue.text) return; + setError(""); + try { + const r = await api.send<{ vtt: string }>("PUT", `/api/assets/${src.slice(6)}/transcript/cues`, { lang, start: cue.start, text: next }); + setTranscript(src, r.vtt); + } catch (e) { + setError(`That correction was not saved: ${(e as Error).message}`); + } + }; + + return ( +
+ Words +

Fix a misheard word or a name's spelling. It changes in every video cut from that clip.

+ {error &&

{error}

} +
    + {rows.map(({ src, cue, at }, n) => ( +
  • + +