From 670b7db210d581128f433bf3eb89128ee2ff309f Mon Sep 17 00:00:00 2001 From: Riccardo Pallaoro <95774148+pallaoro@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:53:33 +0000 Subject: [PATCH 1/3] The hook is a 3 second cut at the start, and the clips proposal says what it does --- src/server/ask-jobs.ts | 7 +++++++ src/server/clips.ts | 32 +++++++++++++++++--------------- src/server/instruct.ts | 7 ++++++- test/clips.test.ts | 13 ++++++++++--- test/e2e/run.mjs | 1 + 5 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/server/ask-jobs.ts b/src/server/ask-jobs.ts index 673189b..980c298 100644 --- a/src/server/ask-jobs.ts +++ b/src/server/ask-jobs.ts @@ -118,6 +118,13 @@ function madeSoFar(state: InstructState): { id: string; name: string }[] { /** How the proposal reads: what it would watch, and that it runs in the background. */ export function proposalText(need: NeedsBackground): string { + if (need.makes) { + const minutes = Number.isFinite(need.seconds) ? Math.max(1, Math.round(need.seconds / 60)) : null; + return ( + `This reads${minutes ? ` about ${minutes} minute${minutes === 1 ? "" : "s"} of` : ""} the video, picks the moments, and cuts and frames each one as a new video in this project, ` + + "so it runs in the background for a few minutes: you can keep working or close the tab. This cut stays as it is. Run it?" + ); + } const clips = need.clips === 1 ? "a clip" : `${need.clips} clips`; const minutes = Number.isFinite(need.seconds) ? Math.max(1, Math.round(need.seconds / 60)) : null; const amount = minutes ? `, about ${minutes} minute${minutes === 1 ? "" : "s"} of footage` : ""; diff --git a/src/server/clips.ts b/src/server/clips.ts index 6528354..7785809 100644 --- a/src/server/clips.ts +++ b/src/server/clips.ts @@ -25,8 +25,8 @@ import { snapToCues, stamp } from "./highlights"; export const MAX_CLIPS = 8; /** How long a clip runs, in seconds, unless the request says otherwise. */ export const CLIP_SECONDS = { min: 10, usual: 45, max: 90 }; -/** A hook is one line: the first seconds of the clip. */ -const HOOK_SECONDS = { min: 0.8, max: 6 }; +/** The hook: a cut this long at the start of the clip, from its strongest line. */ +export const HOOK_SECONDS = 3; /** The most stretches one task has analysed: a cut of about four hours. */ export const MAX_PARTS = 48; /** A task that has waited this long on the media service gives up on what is left. */ @@ -141,13 +141,12 @@ export const CLIPS_SCHEMA = { items: { type: "object", additionalProperties: false, - required: ["title", "start", "end", "hook_start", "hook_end", "why"], + required: ["title", "start", "end", "hook_start", "why"], properties: { title: { type: "string", description: "a short label for the clip, at most 8 words, in the language spoken" }, start: { type: "string", description: "where the clip starts, on the transcript's clock (M:SS.s)" }, end: { type: "string", description: "where it ends (M:SS.s)" }, - hook_start: { type: "string", description: "the hook line's start (M:SS.s), inside the clip" }, - hook_end: { type: "string", description: "the hook line's end (M:SS.s)" }, + hook_start: { type: "string", description: `where its strongest line starts (M:SS.s), inside the clip: the ${HOOK_SECONDS} seconds from there open the video` }, why: { type: "string", description: "one sentence: why this stands on its own and makes people stay" }, }, }, @@ -171,8 +170,8 @@ export function clipsPrompt( `Pick the ${count} best moments that each work as a video of about ${seconds} seconds (between ${low} and ${high}).`, brief.trim() ? `What they are for, in the person's words: ${brief.trim().slice(0, 600)}` : "", "Each moment must make sense to someone who has seen nothing else: one complete thought with a payoff, starting at the start of a sentence and ending at the end of one.", - "The first three seconds decide whether anyone keeps watching, so give each moment a hook: its single most striking line (a surprising claim, a strong opinion, a sharp question, a number), one to four seconds long.", - "If the moment already opens on that line, the hook is its own first line. Otherwise give the line from inside the moment: it plays first, then the moment from its start.", + `The first ${HOOK_SECONDS} seconds decide whether anyone keeps watching, so give each moment a hook: where its single most striking line starts (a surprising claim, a strong opinion, a sharp question, a number). The ${HOOK_SECONDS} seconds from there play first, as a cut of their own, then the moment from its start.`, + `Pick a line whose first ${HOOK_SECONDS} seconds land on their own. If the moment already opens on its strongest line, give the moment's own start: nothing is played twice.`, "Moments must not overlap. Spread them across the video rather than taking them all from one part, and prefer fewer good moments to filler.", hints.length ? "Moments already picked out of this video as its highlights (a person kept the ones marked kept). Prefer them, widened or tightened to stand on their own, before looking elsewhere:\n" + @@ -206,15 +205,18 @@ export function readClipPlans(raw: unknown, cues: Cue[], duration: number, count if (body.end - body.start < CLIP_SECONDS.min) continue; if (out.some((p) => p.body.start < body.end && body.start < p.body.end)) continue; + // The hook is a cut of HOOK_SECONDS from where the line starts: a line + // the transcript holds as part of a longer one still opens the clip. let hook: Span | null = null; - const hs = sourceTime(c?.hook_start); - const he = sourceTime(c?.hook_end); - if (hs !== null && he !== null && he > hs) { - const h = snapToCues(hs, he, cues, duration); - const inside = h.start >= body.start - 0.1 && h.end <= body.end + 0.1; - const length = h.end - h.start; - // A hook that is the clip's own opening is not played twice. - if (inside && length >= HOOK_SECONDS.min && length <= HOOK_SECONDS.max && h.start > body.start + 1.5) hook = h; + let hs = sourceTime(c?.hook_start); + if (hs !== null) { + // On the line's own start when the model is a moment off, never mid-word. + const line = cues.find((q) => q.start <= hs! + 0.05 && q.end > hs!); + if (line && hs - line.start <= 0.6) hs = line.start; + const start = Math.round(Math.max(0, hs - 0.1) * 10) / 10; + const end = Math.min(body.end, Math.round((start + HOOK_SECONDS) * 10) / 10); + // Inside the moment, and not its own opening, which would play twice. + if (start >= body.start + 1.5 && start < body.end && end - start >= 1.5) hook = { start, end }; } const title = typeof c.title === "string" && c.title.trim() ? c.title.trim().slice(0, 80) : `Clip ${out.length + 1}`; out.push({ title, hook, body, why: typeof c.why === "string" ? c.why.trim().slice(0, 300) : "" }); diff --git a/src/server/instruct.ts b/src/server/instruct.ts index fd5cfac..20b55e4 100644 --- a/src/server/instruct.ts +++ b/src/server/instruct.ts @@ -529,6 +529,8 @@ export interface NeedsBackground { clips: number; /** Seconds of footage those clips hold (Infinity when one's length is unknown). */ seconds: number; + /** Making videos from a long one: new videos in the project, this cut unchanged. */ + makes?: true; } export async function cleanUp( @@ -987,7 +989,10 @@ export async function instructStep( const waits = calls.some((c) => c.name === "make_clips" || c.name === "reframe"); if (watching.length) { const seconds = watching.reduce((sum, c) => sum + taskSeconds(s.draft, c, videos, sourceSeconds), s.watched); - if (waits) return { done: { background: { clips: s.watchedClips + watching.length, seconds } } }; + if (waits) { + const makes = calls.some((c) => c.name === "make_clips"); + return { done: { background: { clips: s.watchedClips + watching.length, seconds, ...(makes ? { makes: true as const } : {}) } } }; + } if (seconds > ctx.watchBudget) return { done: { background: { clips: s.watchedClips + watching.length, seconds } } }; s.watched = seconds; s.watchedClips += watching.length; diff --git a/test/clips.test.ts b/test/clips.test.ts index 691892a..acbfd21 100644 --- a/test/clips.test.ts +++ b/test/clips.test.ts @@ -49,12 +49,19 @@ describe("Picking the moments", () => { expect(got.map((p) => p.title)).toEqual(["Cold open", "Hook from elsewhere", "Opens on its hook"]); // From the start of a line to the end of one. expect(got[0].body).toEqual({ start: 99.9, end: 154.8 }); - expect(got[0].hook).toEqual({ start: 129.9, end: 134.8 }); + // A 3 second cut from the start of the line the model chose. + expect(got[0].hook).toEqual({ start: 129.9, end: 132.9 }); // The clip already opens on it, and a hook from another part of the talk is not borrowed. expect(got[2].hook).toBeNull(); expect(got[1].hook).toBeNull(); }); + it("keeps a hook whose line runs across two transcript lines (Circulate, 2026-11-10: every hook was dropped)", () => { + // Lines of 4.2 s on the transcript; the model's hook spans the end of one and the start of the next. + const got = readClipPlans({ clips: [plan("1:41.0", "2:31.0", "2:13.0", "2:17.0")] }, cues, DURATION, 1, 45); + expect(got[0].hook).toEqual({ start: 132.9, end: 135.9 }); + }); + it("ends a moment that runs too long on the last line that fits, and makes no more than asked", () => { const got = readClipPlans({ clips: [plan("1:00.0", "4:00.0", "1:00", "1:03"), plan("8:00", "8:40", "8:00", "8:03"), plan("12:00", "12:40", "12:00", "12:03")] }, cues, DURATION, 2, 30); expect(got).toHaveLength(2); @@ -75,7 +82,7 @@ describe("Picking the moments", () => { expect("edl" in validateEdl(edl)).toBe(true); expect(edl.output).toMatchObject({ width: 1080, height: 1920 }); expect(edl.main.elements.map((e) => [e.trimStart, "duration" in e ? e.duration : null])).toEqual([ - [129.9, 4.9], + [129.9, 3], [99.9, 54.9], ]); expect(edl.captions).toMatchObject({ enabled: true, lang: "en" }); @@ -257,7 +264,7 @@ describe("Clips in the conversation", () => { const { deps } = fakeDeps(); const ctx: InstructContext = { cfg: { openrouterKey: "k" }, assetNames: new Map(), sourceSeconds: seconds, clips: deps, watchBudget: 180 }; const out = await instructStep(startInstruction(empty(), "5 clips from the talk", new Map(), [], [], videos), ctx); - expect(out).toEqual({ done: { background: { clips: 1, seconds: DURATION } } }); + expect(out).toEqual({ done: { background: { clips: 1, seconds: DURATION, makes: true } } }); }); it("in the background, waits between deliveries and ends listing the projects, with this cut unchanged", async () => { diff --git a/test/e2e/run.mjs b/test/e2e/run.mjs index 0ba6129..71eb775 100644 --- a/test/e2e/run.mjs +++ b/test/e2e/run.mjs @@ -1541,6 +1541,7 @@ console.log("14 ok: review comments off by default, on per link, pinned to an ex r = await call("POST", `/api/projects/${pi}/instruct`, { instruction: "We'd need 5/6 cut downs, with a strong hook" }); assert.ok(world.askBodies[0].tools.some((t) => t.function.name === "make_clips"), "make_clips is offered in a quick answer"); assert.equal(r.data.job?.status, "proposed", JSON.stringify(r.data)); + assert.match(r.data.said, /new video in this project/); await call("POST", `/api/projects/${pi}/ask/jobs/${r.data.job.id}/cancel`); world.askReplies = [pickClips, { content: "Made two clips from the interview." }]; r = await call("POST", `/api/projects/${pi}/instruct`, { instruction: "5 vertical clips with a strong hook for LinkedIn", background: true }); From a1c5017035cbc857eccf3e664e6824c56e6c344d Mon Sep 17 00:00:00 2001 From: Riccardo Pallaoro <95774148+pallaoro@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:57:34 +0000 Subject: [PATCH 2/3] Background work shows as one short line beside the Videos menu, not a strip above the player --- src/client/assistant.tsx | 129 ++++++++++++++++++--------------------- src/client/edit.tsx | 4 +- src/server/clips.ts | 2 +- test/clips.test.ts | 2 +- test/e2e/run.mjs | 2 +- 5 files changed, 65 insertions(+), 74 deletions(-) diff --git a/src/client/assistant.tsx b/src/client/assistant.tsx index 70b9921..d31a0f5 100644 --- a/src/client/assistant.tsx +++ b/src/client/assistant.tsx @@ -172,86 +172,77 @@ function MadeLinks({ made, onOpen }: { made: Made[]; onOpen: (id: string) => voi } /** - * The background work, when the conversation is the dashboard's chat: what a - * job is doing, its controls, and the clips it made. Nothing shows while - * nothing needs the person. + * The background work, when the conversation is the dashboard's chat: one + * short line beside the Videos menu. What a job is doing while it runs, its + * controls when it needs the person, and "6 new videos" for a few seconds when + * it made some. Nothing when nothing is happening: it never takes room from + * the player. */ -export function JobStrip({ ask, onOpen }: { ask: Ask; onOpen: (id: string) => void }) { - const { jobs, messages, notTaken, act } = ask; - const [acting, setActing] = useState(null); +export function JobStatus({ ask }: { ask: Ask }) { + const { jobs, notTaken, act } = ask; + const [acting, setActing] = useState(false); const [err, setErr] = useState(""); - const [dismissed, setDismissed] = useState>(new Set()); - const doAct = async (id: string, action: "run" | "cancel" | "apply") => { - setActing(`${id}:${action}`); + const [done, setDone] = useState(null); + const was = useRef(new Map()); + // A job seen running here that finished with videos: say so, briefly. + useEffect(() => { + for (const j of jobs) { + const before = was.current.get(j.id); + if (before && ACTIVE.includes(before) && !ACTIVE.includes(j.status) && j.status === "applied") { + const n = ask.messages?.filter((m) => m.job_id === j.id).reduce((k, m) => k + (m.made?.length ?? 0), 0) ?? 0; + setDone(n ? `${n} new video${n === 1 ? "" : "s"}` : "Done"); + } + was.current.set(j.id, j.status); + } + }, [jobs, ask.messages]); + useEffect(() => { + if (!done) return; + const t = setTimeout(() => setDone(null), 8000); + return () => clearTimeout(t); + }, [done]); + const run = async (id: string, action: "run" | "cancel" | "apply") => { + setActing(true); setErr(""); try { await act(id, action); } catch (e) { setErr(String((e as Error).message)); } finally { - setActing(null); + setActing(false); } }; - // The newest message of each finished job that made clips, until dismissed. - const madeBy = new Map(); - for (const m of messages ?? []) if (m.job_id && m.made?.length) madeBy.set(m.job_id, m.made); - const shown = jobs.filter( - (j) => - !dismissed.has(j.id) && - (j.status === "proposed" || ACTIVE.includes(j.status) || j.status === "stale" || notTaken.has(j.id) || (j.status === "applied" && madeBy.has(j.id))), - ); - if (!shown.length && !err) return null; - const close = (id: string) => setDismissed((d) => new Set(d).add(id)); + const job = jobs.find((j) => j.status === "proposed" || ACTIVE.includes(j.status) || j.status === "stale" || notTaken.has(j.id)); + const small = `${btnGhost} h-7 px-1.5 text-fine`; + if (err) return {err}; + if (!job) return done ? {done} : null; return ( -
- {shown.map((j) => { - const busy = acting !== null; - return ( -
-
- {ACTIVE.includes(j.status) ? : } - - {j.status === "proposed" - ? `Waiting for a yes: ${j.instruction}` - : j.status === "queued" - ? "Starting in the background…" - : j.status === "running" - ? `${j.doing ?? "Working"}…` - : j.status === "applied" - ? `Made ${madeBy.get(j.id)!.length} clips` - : "A change from the chat is waiting: the cut changed while it ran"} - - {j.status === "proposed" && ( - <> - - - - )} - {ACTIVE.includes(j.status) && ( - - )} - {(j.status === "stale" || notTaken.has(j.id)) && ( - <> - - {j.status === "stale" && ( - - )} - - )} - {j.status === "applied" && ( - - )} -
- {(j.status === "applied" ? madeBy.get(j.id) : j.made)?.length ? ( - - ) : null} -
- ); - })} - {err &&

{err}

} -
+ + {ACTIVE.includes(job.status) && } + + {job.status === "proposed" + ? "Run in the background?" + : job.status === "queued" + ? "Starting…" + : job.status === "running" + ? (job.doing ?? "Working…") + : "Kept aside: the cut changed"} + + {job.status === "proposed" && ( + <> + + + + )} + {ACTIVE.includes(job.status) && ( + + )} + {(job.status === "stale" || notTaken.has(job.id)) && ( + <> + + {job.status === "stale" && } + + )} + ); } diff --git a/src/client/edit.tsx b/src/client/edit.tsx index 71ac6d5..2f26aee 100644 --- a/src/client/edit.tsx +++ b/src/client/edit.tsx @@ -1,5 +1,5 @@ import { ReframeDialog } from "./reframe"; -import { AssistantPanel, JobStrip, canonical, useAsk } from "./assistant"; +import { AssistantPanel, JobStatus, canonical, useAsk } from "./assistant"; import { VideoSwitcher, useProjectVideos } from "./videos"; import { embedded, openChat, useChatCommands, useChatContext, useHostChanges } from "@clawnify/app/client"; // Footage edit projects — the timeline editor over EDL documents. @@ -1574,6 +1574,7 @@ export function EditEditor({ {saveState === "saved" ? "Saved" : saveState === "saving" ? "Saving…" : saveState} + {embedded && }