diff --git a/packages/cli/src/capture/assetDownloader.test.ts b/packages/cli/src/capture/assetDownloader.test.ts index fa9da1c65d..4028da90c8 100644 --- a/packages/cli/src/capture/assetDownloader.test.ts +++ b/packages/cli/src/capture/assetDownloader.test.ts @@ -1,5 +1,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { isPrivateUrl, safeFetch, toStandaloneSvg } from "./assetDownloader.js"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + downloadAndRewriteFonts, + isPrivateUrl, + safeFetch, + toStandaloneSvg, +} from "./assetDownloader.js"; describe("isPrivateUrl — SSRF denylist (security: F-003)", () => { it("blocks loopback, private, and metadata IPv4", () => { @@ -127,3 +135,52 @@ describe("toStandaloneSvg — scraped inline SVGs must survive as .svg files", ( expect(toStandaloneSvg("
not an svg
")).toBe("
not an svg
"); }); }); + +describe("downloadAndRewriteFonts — attempt caps", () => { + afterEach(() => vi.unstubAllGlobals()); + + async function expectFailedFontAttempts(css: string, expectedAttempts: number): Promise { + const dir = mkdtempSync(join(tmpdir(), "hf-font-attempts-")); + const fetchMock = vi.fn(async () => new Response("failed", { status: 503 })); + vi.stubGlobal("fetch", fetchMock); + + try { + await downloadAndRewriteFonts(css, dir); + expect(fetchMock).toHaveBeenCalledTimes(expectedAttempts); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + } + + it("counts failed requests toward the global 30-font cap", async () => { + const css = Array.from( + { length: 35 }, + (_, i) => + `@font-face { font-family: Family${i}; src: url(https://fonts${i}.example/font-${i}.woff2); }`, + ).join("\n"); + await expectFailedFontAttempts(css, 30); + }); + + it("counts failed requests toward the six-attempt per-family cap", async () => { + const css = Array.from( + { length: 10 }, + (_, i) => + `@font-face { font-family: Shared; src: url(https://fonts.example/font-${i}.woff2); }`, + ).join("\n"); + await expectFailedFontAttempts(css, 6); + }); + + it("does not start a font request after the capture budget is exhausted", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-font-budget-")); + const css = "@font-face { font-family: Budget; src: url(https://fonts.example/budget.woff2); }"; + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + try { + await downloadAndRewriteFonts(css, dir, { remainingMs: () => 0 }); + expect(fetchMock).not.toHaveBeenCalled(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/src/capture/assetDownloader.ts b/packages/cli/src/capture/assetDownloader.ts index 67b2c0f750..21a881f3b9 100644 --- a/packages/cli/src/capture/assetDownloader.ts +++ b/packages/cli/src/capture/assetDownloader.ts @@ -11,6 +11,10 @@ import { createHash } from "node:crypto"; import type { DesignTokens, DownloadedAsset } from "./types.js"; import type { CatalogedAsset } from "./assetCataloger.js"; +interface DownloadBudgetOptions { + remainingMs?: () => number; +} + // SVGs: hash-of-bytes filename so it can't drift from content; label-derived names mis-assigned brands. function svgContentHashSlug(svgSource: string | Buffer, isLogo: boolean): string { const hash = createHash("sha1").update(svgSource).digest("hex").slice(0, 8); @@ -43,11 +47,13 @@ export function toStandaloneSvg(outerHTML: string): string { return outerHTML.replace(original, tag); } +// fallow-ignore-next-line complexity export async function downloadAssets( tokens: DesignTokens, outputDir: string, catalogedAssets?: CatalogedAsset[], faviconLinks?: Array<{ rel: string; href: string }>, + options: DownloadBudgetOptions = {}, ): Promise { const assetsDir = join(outputDir, "assets"); mkdirSync(assetsDir, { recursive: true }); @@ -82,12 +88,14 @@ export async function downloadAssets( // 2. Favicon for (const icon of faviconLinks || []) { + const remainingMs = options.remainingMs?.() ?? 10_000; + if (remainingMs <= 0) break; if (!icon.href) continue; try { const ext = extname(new URL(icon.href).pathname) || ".ico"; const name = `favicon${ext}`; const localPath = `assets/${name}`; - const buffer = await fetchBuffer(icon.href); + const buffer = await fetchBuffer(icon.href, Math.min(10_000, remainingMs)); if (buffer) { writeFileSync(join(outputDir, localPath), buffer); assets.push({ url: icon.href, localPath, type: "favicon" }); @@ -149,13 +157,15 @@ export async function downloadAssets( let imgIdx = 0; const usedNames = new Set(); for (let i = 0; i < toDownload.length; i += BATCH_SIZE) { + const remainingMs = options.remainingMs?.() ?? 10_000; + if (remainingMs <= 0) break; const batch = toDownload.slice(i, i + BATCH_SIZE); const results = await Promise.allSettled( batch.map(async ({ url, isPoster, catalog }) => { const parsedUrl = new URL(url); const pathExt = extname(parsedUrl.pathname); const ext = pathExt && pathExt.length <= 5 ? pathExt : ".jpg"; - const buffer = await fetchBuffer(url); + const buffer = await fetchBuffer(url, Math.min(10_000, remainingMs)); if (!buffer) return null; const isSvg = ext === ".svg" || url.includes(".svg"); const minSize = isSvg ? 200 : 10000; @@ -198,10 +208,12 @@ export async function downloadAssets( // 4. OG image (if not already downloaded) if (tokens.ogImage && !downloadedUrls.has(normalizeUrl(tokens.ogImage))) { + const remainingMs = options.remainingMs?.() ?? 10_000; try { const ext = extname(new URL(tokens.ogImage).pathname) || ".jpg"; const localPath = `assets/og-image${ext}`; - const buffer = await fetchBuffer(tokens.ogImage); + const buffer = + remainingMs > 0 ? await fetchBuffer(tokens.ogImage, Math.min(10_000, remainingMs)) : null; if (buffer && buffer.length > 5000) { writeFileSync(join(outputDir, localPath), buffer); assets.push({ url: tokens.ogImage, localPath, type: "image" }); @@ -234,7 +246,12 @@ function normalizeUrl(u: string): string { * Download fonts referenced in CSS and rewrite URLs to local paths. * Returns the modified CSS string with local font paths. */ -export async function downloadAndRewriteFonts(css: string, outputDir: string): Promise { +// fallow-ignore-next-line complexity +export async function downloadAndRewriteFonts( + css: string, + outputDir: string, + options: DownloadBudgetOptions = {}, +): Promise { const assetsDir = join(outputDir, "assets", "fonts"); mkdirSync(assetsDir, { recursive: true }); @@ -247,8 +264,10 @@ export async function downloadAndRewriteFonts(css: string, outputDir: string): P if (fontUrls.size === 0) return css; - // Limit font downloads to avoid bloat. Google Fonts serves 20+ unicode-range - // subsets per weight — we only need a few per family for video production. + // Limit font download attempts to bound worst-case egress and latency. Google Fonts serves + // 20+ unicode-range subsets per weight, so successes alone cannot be the bound: six transient + // failures can intentionally suppress later URLs in that family. Latin-priority sorting below + // makes the limited attempts useful while keeping this failure tradeoff explicit. const MAX_FONTS_PER_FAMILY = 6; const MAX_TOTAL_FONTS = 30; const familyCounts = new Map(); @@ -275,10 +294,14 @@ export async function downloadAndRewriteFonts(css: string, outputDir: string): P let count = 0; for (const fontUrl of sortedUrls) { + const remainingMs = options.remainingMs?.() ?? 10_000; + if (remainingMs <= 0) break; if (count >= MAX_TOTAL_FONTS) break; const family = getFamilyForUrl(fontUrl); const familyCount = familyCounts.get(family) || 0; if (familyCount >= MAX_FONTS_PER_FAMILY) continue; + familyCounts.set(family, familyCount + 1); + count++; try { const urlObj = new URL(fontUrl); @@ -286,12 +309,10 @@ export async function downloadAndRewriteFonts(css: string, outputDir: string): P const localPath = join(assetsDir, filename); const relativePath = `assets/fonts/${filename}`; - const buffer = await fetchBuffer(fontUrl); + const buffer = await fetchBuffer(fontUrl, Math.min(10_000, remainingMs)); if (buffer) { writeFileSync(localPath, buffer); rewritten = rewritten.split(fontUrl).join(relativePath); - familyCounts.set(family, familyCount + 1); - count++; } } catch { /* skip */ @@ -391,10 +412,10 @@ export async function safeFetch(url: string, init?: RequestInit): Promise { +async function fetchBuffer(url: string, timeoutMs = 10_000): Promise { try { const res = await safeFetch(url, { - signal: AbortSignal.timeout(10000), + signal: AbortSignal.timeout(timeoutMs), headers: { "User-Agent": "HyperFrames/1.0" }, }); if (!res || !res.ok) return null; diff --git a/packages/cli/src/capture/contactSheet.test.ts b/packages/cli/src/capture/contactSheet.test.ts index 5b2dc74464..71432a754b 100644 --- a/packages/cli/src/capture/contactSheet.test.ts +++ b/packages/cli/src/capture/contactSheet.test.ts @@ -1,9 +1,13 @@ -import { mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import sharp from "sharp"; import { describe, expect, it } from "vitest"; -import { createContactSheet } from "./contactSheet.js"; +import { + createContactSheet, + createScrollContactSheet, + createSvgContactSheet, +} from "./contactSheet.js"; function tempDir(): string { return mkdtempSync(join(tmpdir(), "hf-contact-sheet-test-")); @@ -58,3 +62,64 @@ describe("createContactSheet", () => { } }, 60_000); }); + +describe("contact-sheet capture budget", () => { + it("does not start another Sharp page after the budget is exhausted", async () => { + const dir = tempDir(); + try { + for (let i = 0; i < 10; i++) { + await sharp({ + create: { + width: 4, + height: 4, + channels: 3, + background: { r: i, g: i, b: i }, + }, + }) + .png() + .toFile(join(dir, `scroll-${String(i).padStart(3, "0")}.png`)); + } + + let checks = 0; + const output = join(dir, "contact-sheet.jpg"); + const sheets = await createScrollContactSheet(dir, output, { + remainingMs: () => (checks++ === 0 ? 1000 : 0), + }); + + expect(sheets).toEqual([join(dir, "contact-sheet-1.jpg")]); + expect(existsSync(join(dir, "contact-sheet-2.jpg"))).toBe(false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, 60_000); + + it("stops SVG thumbnail rasterization before the next native operation", async () => { + const dir = tempDir(); + try { + const svgs = join(dir, "svgs"); + const { mkdirSync } = await import("node:fs"); + mkdirSync(svgs); + writeFileSync( + join(svgs, "a.svg"), + '', + ); + writeFileSync( + join(svgs, "b.svg"), + '', + ); + let checks = 0; + + const sheets = await createSvgContactSheet( + svgs, + join(dir, "svg-contact-sheet.jpg"), + undefined, + { remainingMs: () => (checks++ === 0 ? 1000 : 0) }, + ); + + expect(sheets).toEqual([]); + expect(checks).toBeGreaterThanOrEqual(2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, 60_000); +}); diff --git a/packages/cli/src/capture/contactSheet.ts b/packages/cli/src/capture/contactSheet.ts index 6289c84494..ffb5b37a22 100644 --- a/packages/cli/src/capture/contactSheet.ts +++ b/packages/cli/src/capture/contactSheet.ts @@ -18,6 +18,8 @@ interface ContactSheetOptions { quality?: number; /** Target width per cell in pixels (default: 600) */ cellWidth?: number; + /** Cooperative boundary checked before starting each native Sharp page. */ + remainingMs?: () => number; } /** @@ -39,6 +41,8 @@ export async function createContactSheet( cellWidth = 600, } = opts; + if ((opts.remainingMs?.() ?? 1) <= 0) return null; + const files = imagePaths.slice(0, maxImages); if (files.length === 0) return null; @@ -125,6 +129,7 @@ function escapeXml(s: string): string { * Output files: basePath → base-1.jpg, base-2.jpg, ... * Returns the list of written file paths (empty if no images). */ +// fallow-ignore-next-line complexity async function createContactSheetPages( imagePaths: string[], outputBasePath: string, @@ -133,7 +138,7 @@ async function createContactSheetPages( customLabels?: string[], ): Promise { if (imagePaths.length === 0) return []; - const { pageSize = imagePaths.length, ...sheetOpts } = opts; + const { pageSize = imagePaths.length, remainingMs, ...sheetOpts } = opts; const ext = outputBasePath.match(/\.[^.]+$/)?.[0] ?? ".jpg"; const base = outputBasePath.slice(0, -ext.length); @@ -141,6 +146,7 @@ async function createContactSheetPages( const results: string[] = []; for (let p = 0; p < pages; p++) { + if ((remainingMs?.() ?? 1) <= 0) break; const chunk = imagePaths.slice(p * pageSize, (p + 1) * pageSize); const chunkLabels = customLabels?.slice(p * pageSize, (p + 1) * pageSize); const outPath = pages === 1 ? outputBasePath : `${base}-${p + 1}${ext}`; @@ -170,6 +176,7 @@ async function createContactSheetPages( export async function createScrollContactSheet( screenshotsDir: string, outputPath: string, + budget: Pick = {}, ): Promise { if (!existsSync(screenshotsDir)) return []; @@ -189,7 +196,7 @@ export async function createScrollContactSheet( return createContactSheetPages( paths, outputPath, - { cols: 3, cellWidth: 600, pageSize: 9 }, + { cols: 3, cellWidth: 600, pageSize: 9, ...budget }, 0, labels, ); @@ -203,6 +210,7 @@ export async function createScrollContactSheet( export async function createSnapshotContactSheet( snapshotsDir: string, outputPath: string, + budget: Pick = {}, ): Promise { if (!existsSync(snapshotsDir)) return []; @@ -222,7 +230,7 @@ export async function createSnapshotContactSheet( return createContactSheetPages( paths, outputPath, - { cols: 3, cellWidth: 600, pageSize: 9 }, + { cols: 3, cellWidth: 600, pageSize: 9, ...budget }, 0, labels, ); @@ -236,6 +244,7 @@ export async function createSnapshotContactSheet( export async function createAssetContactSheet( assetsDir: string, outputPath: string, + budget: Pick = {}, ): Promise { if (!existsSync(assetsDir)) return []; @@ -254,6 +263,7 @@ export async function createAssetContactSheet( cellWidth: 480, labelMode: "filename", pageSize: 12, + ...budget, }); } @@ -271,6 +281,7 @@ export async function createSvgContactSheet( svgsDir: string, outputPath: string, assetsRootDir?: string, + budget: Pick = {}, ): Promise { const dirsToScan = [svgsDir, assetsRootDir].filter( (d): d is string => d !== undefined && existsSync(d), @@ -302,6 +313,7 @@ export async function createSvgContactSheet( const labels: string[] = []; for (let i = 0; i < svgPaths.length; i++) { + if ((budget.remainingMs?.() ?? 1) <= 0) break; const svgPath = svgPaths[i]!; const tmpPath = join(tmpDir, `.thumb-${i}.png`); try { @@ -334,6 +346,7 @@ export async function createSvgContactSheet( cols: 5, cellWidth: thumbSize, pageSize: 15, + ...budget, }, 0, labels, diff --git a/packages/cli/src/capture/contentExtractor.test.ts b/packages/cli/src/capture/contentExtractor.test.ts index 9683653848..5117f73d2e 100644 --- a/packages/cli/src/capture/contentExtractor.test.ts +++ b/packages/cli/src/capture/contentExtractor.test.ts @@ -2,19 +2,35 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { captionImagesWithGemini } from "./contentExtractor.js"; +import { + captionImagesWithGemini, + resolveVisionPhaseCompletion, + type VisionCaptionOutcome, +} from "./contentExtractor.js"; + +const { generateContentMock } = vi.hoisted(() => ({ + generateContentMock: vi.fn(), +})); + +vi.mock("@google/genai", () => ({ + GoogleGenAI: class { + models = { generateContent: generateContentMock }; + }, +})); // These tests exercise the OpenRouter provider path only — it makes a plain // `fetch` call we can stub, with no native (`sharp`) or `@google/genai` // dependency. OpenRouter wins over Gemini when OPENROUTER_API_KEY is set, so we // don't need to clear the Gemini keys for the OpenRouter cases. -function makeProjectWithImage(): string { +function makeProjectWithImages(files = ["hero.png"]): string { const dir = mkdtempSync(join(tmpdir(), "hf-caption-")); mkdirSync(join(dir, "assets"), { recursive: true }); // Contents are irrelevant to the OpenRouter path (it just base64-encodes the // bytes); only the .png extension matters for the image filter. - writeFileSync(join(dir, "assets", "hero.png"), Buffer.from([0x89, 0x50, 0x4e, 0x47])); + for (const file of files) { + writeFileSync(join(dir, "assets", file), Buffer.from([0x89, 0x50, 0x4e, 0x47])); + } return dir; } @@ -28,7 +44,7 @@ describe("captionImagesWithGemini — OpenRouter provider", () => { }); it("captions via OpenRouter when OPENROUTER_API_KEY is set", async () => { - const dir = makeProjectWithImage(); + const dir = makeProjectWithImages(); dirs.push(dir); vi.stubEnv("OPENROUTER_API_KEY", "or-test-key"); vi.stubEnv("HYPERFRAMES_OPENROUTER_MODEL", "google/gemini-3.1-flash-lite"); @@ -63,28 +79,267 @@ describe("captionImagesWithGemini — OpenRouter provider", () => { }); it("degrades gracefully (no throw, no captions) when OpenRouter returns a non-OK status", async () => { - const dir = makeProjectWithImage(); + const dir = makeProjectWithImages(); dirs.push(dir); vi.stubEnv("OPENROUTER_API_KEY", "or-bad-key"); vi.stubGlobal( "fetch", vi.fn( - async () => new Response("invalid api key", { status: 401, statusText: "Unauthorized" }), + async () => + new Response("provider detail containing or-bad-key", { + status: 401, + statusText: "Unauthorized", + }), ), ); const warnings: string[] = []; - // captionOne throws on !res.ok, but the throw is per-image inside - // Promise.allSettled, so it's filtered out as a rejected result rather than - // bubbling up — same silent degradation as the existing Gemini path. - const captions = await captionImagesWithGemini(dir, () => {}, warnings); + let outcome: VisionCaptionOutcome | undefined; + const captions = await captionImagesWithGemini(dir, () => {}, warnings, { + onOutcome: (value) => { + outcome = value; + }, + }); + + expect(captions).toEqual({}); + expect(warnings).toEqual(["OpenRouter vision failed for 1 asset(s); captions omitted."]); + expect(warnings.join(" ")).not.toContain("or-bad-key"); + expect(outcome).toEqual({ + timedOutRequests: 0, + failedRequests: 1, + budgetExhausted: false, + }); + if (!outcome) throw new Error("Expected vision caption outcome"); + expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({ + status: "degraded", + reason: "provider-error", + }); + }); + + it("terminates captioning when the vision provider never responds", async () => { + const dir = makeProjectWithImages(); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", "or-hanging-key"); + vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20"); + + vi.stubGlobal( + "fetch", + vi.fn((_url: string, init?: RequestInit) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(init.signal?.reason), { + once: true, + }); + }); + }), + ); + + const warnings: string[] = []; + const result = await Promise.race([ + captionImagesWithGemini(dir, () => {}, warnings), + new Promise<"hung">((resolve) => setTimeout(() => resolve("hung"), 250)), + ]); + + expect(result).toEqual({}); + }); + + it.each([ + { label: "success JSON", status: 200, partialBody: '{"choices":[' }, + { label: "failure text", status: 500, partialBody: "upstream failed: " }, + ])( + "terminates captioning when OpenRouter $label body never ends", + async ({ status, partialBody }) => { + const dir = makeProjectWithImages(); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", "or-stalled-body-key"); + vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20"); + + vi.stubGlobal( + "fetch", + vi.fn(async () => { + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(partialBody)); + }, + }); + return new Response(body, { + status, + headers: { "content-type": "application/json" }, + }); + }), + ); + + const warnings: string[] = []; + let outcome: VisionCaptionOutcome | undefined; + const result = await Promise.race([ + captionImagesWithGemini(dir, () => {}, warnings, { + onOutcome: (value) => { + outcome = value; + }, + }), + new Promise<"hung">((resolve) => setTimeout(() => resolve("hung"), 250)), + ]); + + expect(result).toEqual({}); + expect(warnings).toEqual(["OpenRouter vision timed out for 1 asset(s); captions omitted."]); + expect(outcome).toEqual({ + timedOutRequests: 1, + failedRequests: 0, + budgetExhausted: false, + }); + }, + ); + + it("keeps successful captions while reporting a failed sibling request", async () => { + const dir = makeProjectWithImages(["failed.png", "success.png"]); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", "or-mixed-failure-key"); + + let requestCount = 0; + vi.stubGlobal( + "fetch", + vi.fn(async () => { + requestCount++; + if (requestCount === 1) { + return new Response("upstream exploded with or-mixed-failure-key", { + status: 500, + statusText: "Internal Server Error", + }); + } + return new Response( + JSON.stringify({ choices: [{ message: { content: "Successful image." } }] }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }), + ); + + const warnings: string[] = []; + let outcome: VisionCaptionOutcome | undefined; + const captions = await captionImagesWithGemini(dir, () => {}, warnings, { + onOutcome: (value) => { + outcome = value; + }, + }); + + expect(captions).toEqual({ "success.png": "Successful image." }); + expect(warnings).toEqual(["OpenRouter vision failed for 1 asset(s); captions omitted."]); + expect(warnings.join(" ")).not.toContain("or-mixed-failure-key"); + expect(outcome).toEqual({ + timedOutRequests: 0, + failedRequests: 1, + budgetExhausted: false, + }); + }); + + it("keeps a successful sibling caption when another request times out", async () => { + const dir = makeProjectWithImages(["hang.png", "success.png"]); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", "or-mixed-key"); + vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20"); + + vi.stubGlobal( + "fetch", + vi.fn((_url: string, init?: RequestInit) => { + const body = JSON.parse(typeof init?.body === "string" ? init.body : "{}"); + const imageUrl = body.messages?.[0]?.content?.[1]?.image_url?.url; + if (typeof imageUrl === "string" && imageUrl.includes("iVBORw==")) { + // Both fixtures contain the same bytes, so use call order: the first + // request hangs while the second succeeds independently. + const fetchMock = vi.mocked(fetch); + if (fetchMock.mock.calls.length === 1) return new Promise(() => {}); + } + return Promise.resolve( + new Response( + JSON.stringify({ choices: [{ message: { content: "Successful image." } }] }), + { + status: 200, + headers: { "content-type": "application/json" }, + }, + ), + ); + }), + ); + + const warnings: string[] = []; + let outcome: VisionCaptionOutcome | undefined; + const captions = await captionImagesWithGemini(dir, () => {}, warnings, { + onOutcome: (value) => { + outcome = value; + }, + }); + + expect(captions).toEqual({ "success.png": "Successful image." }); + expect(outcome).toEqual({ + timedOutRequests: 1, + failedRequests: 0, + budgetExhausted: false, + }); + if (!outcome) throw new Error("Expected vision caption outcome"); + expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({ + status: "degraded", + reason: "request-timeout", + }); + }); + + it("does not call a configured provider when vision is skipped", async () => { + const dir = makeProjectWithImages(); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", "or-unused-key"); + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + const captions = await captionImagesWithGemini(dir, () => {}, [], { skipVision: true }); + + expect(captions).toEqual({}); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("does not start a vision batch after the capture budget is exhausted", async () => { + const dir = makeProjectWithImages(); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", "or-budget-key"); + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + const captions = await captionImagesWithGemini(dir, () => {}, [], { + remainingMs: () => 0, + }); expect(captions).toEqual({}); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("classifies non-provider pipeline failures without leaking the local path", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-caption-no-assets-")); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", "or-unused-key"); + + const warnings: string[] = []; + let outcome: VisionCaptionOutcome | undefined; + const captions = await captionImagesWithGemini(dir, () => {}, warnings, { + onOutcome: (value) => { + outcome = value; + }, + }); + + expect(captions).toEqual({}); + expect(warnings).toEqual(["OpenRouter captioning failed internally; captions omitted."]); + expect(warnings.join(" ")).not.toContain(dir); + expect(outcome).toEqual({ + timedOutRequests: 0, + failedRequests: 0, + budgetExhausted: false, + internalError: true, + }); + if (!outcome) throw new Error("Expected vision caption outcome"); + expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({ + status: "degraded", + reason: "internal-error", + }); }); it("skips captioning entirely when no provider key is present", async () => { - const dir = makeProjectWithImage(); + const dir = makeProjectWithImages(); dirs.push(dir); vi.stubEnv("OPENROUTER_API_KEY", ""); vi.stubEnv("GEMINI_API_KEY", ""); @@ -100,3 +355,62 @@ describe("captionImagesWithGemini — OpenRouter provider", () => { expect(fetchMock).not.toHaveBeenCalled(); }); }); + +describe("captionImagesWithGemini — Gemini provider", () => { + const dirs: string[] = []; + + afterEach(() => { + generateContentMock.mockReset(); + vi.unstubAllEnvs(); + for (const d of dirs) rmSync(d, { recursive: true, force: true }); + dirs.length = 0; + }); + + it("terminates captioning when Gemini never settles even if the SDK ignores abort", async () => { + const dir = makeProjectWithImages(); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", ""); + vi.stubEnv("GEMINI_API_KEY", "gemini-hanging-key"); + vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20"); + generateContentMock.mockImplementation(() => new Promise(() => {})); + + const result = await Promise.race([ + captionImagesWithGemini(dir, () => {}, []), + new Promise<"hung">((resolve) => setTimeout(() => resolve("hung"), 250)), + ]); + + expect(result).toEqual({}); + }); + + it("reports a rejected Gemini request without leaking its error detail", async () => { + const dir = makeProjectWithImages(); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", ""); + vi.stubEnv("GEMINI_API_KEY", "gemini-secret-key"); + generateContentMock.mockRejectedValue( + new Error("provider echoed gemini-secret-key in its response"), + ); + + const warnings: string[] = []; + let outcome: VisionCaptionOutcome | undefined; + const captions = await captionImagesWithGemini(dir, () => {}, warnings, { + onOutcome: (value) => { + outcome = value; + }, + }); + + expect(captions).toEqual({}); + expect(warnings).toEqual(["Gemini vision failed for 1 asset(s); captions omitted."]); + expect(warnings.join(" ")).not.toContain("gemini-secret-key"); + expect(outcome).toEqual({ + timedOutRequests: 0, + failedRequests: 1, + budgetExhausted: false, + }); + if (!outcome) throw new Error("Expected vision caption outcome"); + expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({ + status: "degraded", + reason: "provider-error", + }); + }); +}); diff --git a/packages/cli/src/capture/contentExtractor.ts b/packages/cli/src/capture/contentExtractor.ts index 2caee86e48..acea6b11ea 100644 --- a/packages/cli/src/capture/contentExtractor.ts +++ b/packages/cli/src/capture/contentExtractor.ts @@ -15,6 +15,80 @@ import type sharpType from "sharp"; import type { CatalogedAsset } from "./assetCataloger.js"; import type { DesignTokens } from "./types.js"; +const DEFAULT_VISION_REQUEST_TIMEOUT_MS = 30_000; + +export interface VisionCaptionOutcome { + timedOutRequests: number; + failedRequests: number; + budgetExhausted: boolean; + /** Failure outside provider request handling (filesystem, module, or programming error). */ + internalError?: boolean; +} + +interface VisionCaptionOptions { + skipVision?: boolean; + remainingMs?: () => number; + onOutcome?: (outcome: VisionCaptionOutcome) => void; +} + +export function resolveVisionPhaseCompletion( + outcome: VisionCaptionOutcome, + remainingMs: number, +): + | { status: "completed" } + | { + status: "degraded"; + reason: "budget-exhausted" | "request-timeout" | "provider-error" | "internal-error"; + } { + if (outcome.budgetExhausted || remainingMs <= 0) { + return { status: "degraded", reason: "budget-exhausted" }; + } + if (outcome.internalError) { + return { status: "degraded", reason: "internal-error" }; + } + if (outcome.timedOutRequests > 0) { + return { status: "degraded", reason: "request-timeout" }; + } + if (outcome.failedRequests > 0) { + return { status: "degraded", reason: "provider-error" }; + } + return { status: "completed" }; +} + +class VisionRequestTimeoutError extends Error { + constructor(timeoutMs: number) { + super(`Vision request timed out after ${timeoutMs}ms`); + this.name = "VisionRequestTimeoutError"; + } +} + +function resolveVisionRequestTimeoutMs(): number { + const configured = Number(process.env.HYPERFRAMES_VISION_TIMEOUT_MS); + return Number.isFinite(configured) && configured > 0 + ? configured + : DEFAULT_VISION_REQUEST_TIMEOUT_MS; +} + +async function runBoundedVisionRequest( + request: (signal: AbortSignal) => Promise, + timeoutMs: number, +): Promise { + const controller = new AbortController(); + let timer: ReturnType | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + reject(new VisionRequestTimeoutError(timeoutMs)); + controller.abort(); + }, timeoutMs); + }); + + try { + return await Promise.race([request(controller.signal), timeout]); + } finally { + if (timer) clearTimeout(timer); + } +} + /** * Detect JS libraries via window globals, DOM fingerprints, script URLs, * and WebGL shader analysis. @@ -165,15 +239,37 @@ export async function extractVisibleText(page: Page): Promise { * Batches requests to stay under free-tier rate limits. * Returns a map of filename -> caption string. */ +// fallow-ignore-next-line complexity export async function captionImagesWithGemini( outputDir: string, progress: (stage: string, detail?: string) => void, warnings: string[], + options: VisionCaptionOptions = {}, ): Promise> { const geminiCaptions: Record = {}; + let timedOutCount = 0; + let failedRequestCount = 0; + let budgetExhausted = false; + let internalError = false; + const reportOutcome = (): void => { + const outcome: VisionCaptionOutcome = { + timedOutRequests: timedOutCount, + failedRequests: failedRequestCount, + budgetExhausted, + }; + if (internalError) outcome.internalError = true; + options.onOutcome?.(outcome); + }; + if (options.skipVision) { + reportOutcome(); + return geminiCaptions; + } const openRouterKey = process.env.OPENROUTER_API_KEY; const geminiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY; - if (!openRouterKey && !geminiKey) return geminiCaptions; + if (!openRouterKey && !geminiKey) { + reportOutcome(); + return geminiCaptions; + } // OpenRouter takes priority when both keys are set — it's the explicit opt-in // for users without Google access. Both providers satisfy the same @@ -186,6 +282,7 @@ export async function captionImagesWithGemini( const model = useOpenRouter ? process.env.HYPERFRAMES_OPENROUTER_MODEL || "google/gemini-3.1-flash-lite" : process.env.HYPERFRAMES_GEMINI_MODEL || "gemini-3.1-flash-lite-preview"; + const requestTimeoutMs = resolveVisionRequestTimeoutMs(); progress("design", `Captioning images with ${providerName} vision...`); try { @@ -196,53 +293,71 @@ export async function captionImagesWithGemini( base64: string; prompt: string; maxTokens: number; + timeoutMs: number; }) => Promise; let captionOne: CaptionOne; if (openRouterKey) { - captionOne = async ({ mimeType, base64, prompt, maxTokens }) => { - const res = await fetch("https://openrouter.ai/api/v1/chat/completions", { - method: "POST", - headers: { - Authorization: `Bearer ${openRouterKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - model, - messages: [ - { - role: "user", - content: [ - { type: "text", text: prompt }, - { type: "image_url", image_url: { url: `data:${mimeType};base64,${base64}` } }, - ], - }, - ], - max_tokens: maxTokens, - }), - }); - if (!res.ok) { - const detail = await res.text().catch(() => ""); - throw new Error(`OpenRouter ${res.status} ${res.statusText}: ${detail.slice(0, 200)}`); - } - const data = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }>; - }; - return data.choices?.[0]?.message?.content?.trim() || ""; + captionOne = async ({ mimeType, base64, prompt, maxTokens, timeoutMs }) => { + return runBoundedVisionRequest(async (signal) => { + const res = await fetch("https://openrouter.ai/api/v1/chat/completions", { + method: "POST", + headers: { + Authorization: `Bearer ${openRouterKey}`, + "Content-Type": "application/json", + }, + signal, + body: JSON.stringify({ + model, + messages: [ + { + role: "user", + content: [ + { type: "text", text: prompt }, + { + type: "image_url", + image_url: { url: `data:${mimeType};base64,${base64}` }, + }, + ], + }, + ], + max_tokens: maxTokens, + }), + }); + if (!res.ok) { + await res.text(); + throw new Error(`OpenRouter request failed with HTTP ${res.status}`); + } + const data = (await res.json()) as { + choices?: Array<{ message?: { content?: string } }>; + }; + return data.choices?.[0]?.message?.content?.trim() || ""; + }, timeoutMs); }; } else { // Unreachable when geminiKey is unset (guarded above); re-narrow for TS. if (!geminiKey) return geminiCaptions; const { GoogleGenAI } = await import("@google/genai"); const ai = new GoogleGenAI({ apiKey: geminiKey }); - captionOne = async ({ mimeType, base64, prompt, maxTokens }) => { - const response = await ai.models.generateContent({ - model, - contents: [ - { role: "user", parts: [{ inlineData: { mimeType, data: base64 } }, { text: prompt }] }, - ], - config: { maxOutputTokens: maxTokens }, - }); + captionOne = async ({ mimeType, base64, prompt, maxTokens, timeoutMs }) => { + const response = await runBoundedVisionRequest( + (signal) => + ai.models.generateContent({ + model, + contents: [ + { + role: "user", + parts: [{ inlineData: { mimeType, data: base64 } }, { text: prompt }], + }, + ], + config: { + maxOutputTokens: maxTokens, + abortSignal: signal, + httpOptions: { timeout: timeoutMs }, + }, + }), + timeoutMs, + ); return response.text?.trim() || ""; }; } @@ -253,10 +368,35 @@ export async function captionImagesWithGemini( // Caption in parallel batches. Gemini free tier is ~5 RPM (slow but $0), // paid/OpenRouter ~2000 RPM. We batch 20 with a 2s inter-batch pause and rely - // on Promise.allSettled so a rate-limited image degrades to "" rather than - // failing the batch. + // on Promise.allSettled so one rate-limited image does not fail its siblings; + // rejected requests are aggregated into a sanitized warning below. const BATCH_SIZE = 20; + const collectCaptionResults = ( + results: PromiseSettledResult<{ file: string; caption: string }>[], + ): { timeouts: number; failures: number } => { + let timeouts = 0; + let failures = 0; + for (const result of results) { + if (result.status === "fulfilled" && result.value.caption) { + geminiCaptions[result.value.file] = result.value.caption; + } else if ( + result.status === "rejected" && + result.reason instanceof VisionRequestTimeoutError + ) { + timeouts++; + } else if (result.status === "rejected") { + failures++; + } + } + return { timeouts, failures }; + }; for (let i = 0; i < imageFiles.length; i += BATCH_SIZE) { + const remainingMs = options.remainingMs?.() ?? Number.POSITIVE_INFINITY; + if (remainingMs <= 0) { + budgetExhausted = true; + break; + } + const timeoutMs = Math.max(1, Math.min(requestTimeoutMs, remainingMs)); const batch = imageFiles.slice(i, i + BATCH_SIZE); const results = await Promise.allSettled( batch.map(async (file: string) => { @@ -273,15 +413,14 @@ export async function captionImagesWithGemini( prompt: "Describe this website image in ONE short sentence for a video storyboard. Focus on: what it shows, dominant colors, whether background is light or dark. Be factual, not creative.", maxTokens: 500, + timeoutMs, }); return { file, caption }; }), ); - for (const result of results) { - if (result.status === "fulfilled" && result.value.caption) { - geminiCaptions[result.value.file] = result.value.caption; - } - } + const counts = collectCaptionResults(results); + timedOutCount += counts.timeouts; + failedRequestCount += counts.failures; // Pace requests between batches (paid tier: 2000+ RPM, free tier: rate-limited) if (i + BATCH_SIZE < imageFiles.length) { await new Promise((r) => setTimeout(r, 2000)); // 2s pause between batches — paid tier handles 2000 RPM, free tier retries via Promise.allSettled @@ -322,6 +461,7 @@ export async function captionImagesWithGemini( `Skipped ${svgFiles.length} SVG caption(s): sharp could not load (${(err as Error).message}). ` + `Reinstall with optional dependencies enabled (e.g. \`npm i sharp\`) to caption SVG assets.`, ); + reportOutcome(); return geminiCaptions; } progress("design", `Rasterizing + captioning ${svgFiles.length} SVGs via vision API...`); @@ -329,6 +469,12 @@ export async function captionImagesWithGemini( const SVG_RENDER_SIZE = 256; // px — enough resolution for Gemini to read wordmarks, small enough to keep payload sub-MB let svgsSkipped = 0; for (let i = 0; i < svgFiles.length; i += SVG_BATCH) { + const remainingMs = options.remainingMs?.() ?? Number.POSITIVE_INFINITY; + if (remainingMs <= 0) { + budgetExhausted = true; + break; + } + const timeoutMs = Math.max(1, Math.min(requestTimeoutMs, remainingMs)); const batch = svgFiles.slice(i, i + SVG_BATCH); const results = await Promise.allSettled( batch.map(async ({ relPath }) => { @@ -373,15 +519,14 @@ export async function captionImagesWithGemini( "If you see a wordmark, READ THE LETTERS LITERALLY — do not guess a brand from context. " + "Be factual.", maxTokens: 300, + timeoutMs, }); return { file: relPath, caption }; }), ); - for (const result of results) { - if (result.status === "fulfilled" && result.value.caption) { - geminiCaptions[result.value.file] = result.value.caption; - } - } + const counts = collectCaptionResults(results); + timedOutCount += counts.timeouts; + failedRequestCount += counts.failures; if (i + SVG_BATCH < svgFiles.length) { await new Promise((r) => setTimeout(r, 2000)); } @@ -398,10 +543,22 @@ export async function captionImagesWithGemini( ); } } - } catch (err) { - warnings.push(`${providerName} captioning failed: ${err}`); + if (timedOutCount > 0) { + warnings.push( + `${providerName} vision timed out for ${timedOutCount} asset(s); captions omitted.`, + ); + } + if (failedRequestCount > 0) { + warnings.push( + `${providerName} vision failed for ${failedRequestCount} asset(s); captions omitted.`, + ); + } + } catch { + internalError = true; + warnings.push(`${providerName} captioning failed internally; captions omitted.`); } + reportOutcome(); return geminiCaptions; } diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts index fe58a972f4..b57be6e4ae 100644 --- a/packages/cli/src/capture/index.ts +++ b/packages/cli/src/capture/index.ts @@ -37,12 +37,17 @@ import { extractVisibleText, captionImagesWithGemini, generateAssetDescriptions, + resolveVisionPhaseCompletion, } from "./contentExtractor.js"; +import type { VisionCaptionOutcome } from "./contentExtractor.js"; import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js"; -import type { CaptureOptions, CaptureResult } from "./types.js"; +import { detectBlockedPage } from "./pageBlockDetection.js"; +import type { CaptureOptions, CapturePhase, CapturePhaseProgress, CaptureResult } from "./types.js"; export type { CaptureOptions, CaptureResult } from "./types.js"; +const DEFAULT_POST_NAVIGATION_BUDGET_MS = 120_000; + // fallow-ignore-next-line complexity export async function captureWebsite( opts: CaptureOptions, @@ -57,12 +62,49 @@ export async function captureWebsite( settleTime = 3000, maxScreenshots: _maxScreenshots = 24, skipAssets = false, + skipVision = false, + postNavigationBudgetMs = DEFAULT_POST_NAVIGATION_BUDGET_MS, + onPhase, } = opts; const warnings: string[] = []; const progress = (stage: string, detail?: string) => { onProgress?.(stage, detail); }; + const budgetMs = + Number.isFinite(postNavigationBudgetMs) && postNavigationBudgetMs > 0 + ? postNavigationBudgetMs + : DEFAULT_POST_NAVIGATION_BUDGET_MS; + let postNavigationDeadline: number | undefined; + const remainingMs = (): number => + postNavigationDeadline === undefined + ? budgetMs + : Math.max(0, postNavigationDeadline - Date.now()); + let lastPhase: CapturePhaseProgress = { + schema: "hyperframes.capture.phase.v1", + phase: "browser", + status: "started", + remainingMs: null, + }; + const phase = ( + name: CapturePhase, + status: CapturePhaseProgress["status"], + reason?: CapturePhaseProgress["reason"], + ): void => { + const remaining = postNavigationDeadline === undefined ? null : remainingMs(); + lastPhase = reason + ? { + schema: "hyperframes.capture.phase.v1", + phase: name, + status, + remainingMs: remaining, + reason, + } + : { schema: "hyperframes.capture.phase.v1", phase: name, status, remainingMs: remaining }; + onPhase?.(lastPhase); + }; + + phase("browser", "started"); // Load .env file from repo root if it exists (for GEMINI_API_KEY, etc.) loadEnvFile(outputDir); @@ -102,6 +144,8 @@ export async function captureWebsite( // Goal: Catalog animations + take screenshots (with JS rendering) // ═══════════════════════════════════════════════════════════════ + phase("browser", "completed"); + phase("navigation", "started"); progress("animations", "Cataloging animations (full JS)..."); const page1 = await chromeBrowser.newPage(); @@ -197,12 +241,13 @@ export async function captureWebsite( // Use networkidle2 (allows 2 ongoing connections) instead of networkidle0 — // modern SPAs often have persistent WebSocket/analytics connections that // prevent networkidle0 from ever resolving. - await page1.goto(url, { waitUntil: "networkidle2", timeout }); + const navigationResponse = await page1.goto(url, { waitUntil: "networkidle2", timeout }); + postNavigationDeadline = Date.now() + budgetMs; await new Promise((r) => setTimeout(r, settleTime)); // Check if the page loaded real content or an anti-bot challenge - // Use structural detection (DOM elements + cookies), not text regex matching — - // text matching causes false positives on sites that mention "blocked" or "verify" in copy + // Combine structural evidence with the main response status/title. Low text + // alone stays non-fatal so image-led sites are not rejected. const pageContentCheck = (await page1.evaluate(`(() => { var text = (document.body.innerText || "").trim(); var title = document.title || ""; @@ -210,19 +255,31 @@ export async function captureWebsite( var hasCfTurnstile = !!document.querySelector('.cf-turnstile, [data-sitekey], iframe[src*="challenges.cloudflare.com"], #challenge-running, #challenge-form'); // Structural: page is almost empty (challenge pages have minimal DOM) var bodyChildCount = document.body.children.length; - var isMinimalDom = bodyChildCount <= 5 && text.length < 500; - // Title-based: only check title on near-empty pages - var hasChallengeTitle = isMinimalDom && /just a moment|attention required|access denied/i.test(title); - var isChallenged = hasCfTurnstile || hasChallengeTitle; - return { textLength: text.length, title: title, isChallenged: isChallenged, bodyChildCount: bodyChildCount }; - })()`)) as { textLength: number; title: string; isChallenged: boolean; bodyChildCount: number }; - - if (pageContentCheck.isChallenged || pageContentCheck.textLength < 100) { - const reason = pageContentCheck.isChallenged - ? "Anti-bot protection detected (Cloudflare challenge or similar)" - : "Page has very little text content (" + - pageContentCheck.textLength + - " chars) — may be blocked or a client-rendered SPA that needs more time"; + return { textLength: text.length, title: title, hasChallengeElement: hasCfTurnstile, bodyChildCount: bodyChildCount }; + })()`)) as { + textLength: number; + title: string; + hasChallengeElement: boolean; + bodyChildCount: number; + }; + + const blockedReason = detectBlockedPage({ + httpStatus: navigationResponse?.status() ?? null, + ...pageContentCheck, + }); + if (blockedReason) { + phase("navigation", "degraded", "blocked"); + throw new Error(blockedReason); + } + + phase("navigation", "completed"); + phase("core-extraction", "started"); + + if (pageContentCheck.textLength < 100) { + const reason = + "Page has very little text content (" + + pageContentCheck.textLength + + " chars) — may be blocked or a client-rendered SPA that needs more time"; warnings.push(reason); progress("warn", reason); } @@ -231,29 +288,36 @@ export async function captureWebsite( // Framer and other modern sites use IntersectionObserver — images only load // when scrolled into view. We scroll the full page, then wait for all images // to finish loading before proceeding. - await page1.evaluate(`(async () => { + const lazyLoadBudgetMs = Math.min(15_000, remainingMs()); + if (lazyLoadBudgetMs > 0) { + await page1.evaluate(`(async () => { + var lazyLoadDeadline = Date.now() + ${lazyLoadBudgetMs}; var h = document.body.scrollHeight; for (var y = 0; y < h; y += window.innerHeight * 0.7) { + if (Date.now() >= lazyLoadDeadline) break; window.scrollTo(0, y); await new Promise(function(r) { setTimeout(r, 400); }); } // Scroll to very bottom to catch footer lazy-loads - window.scrollTo(0, document.body.scrollHeight); - await new Promise(function(r) { setTimeout(r, 800); }); + if (Date.now() < lazyLoadDeadline) { + window.scrollTo(0, document.body.scrollHeight); + await new Promise(function(r) { setTimeout(r, Math.min(800, Math.max(0, lazyLoadDeadline - Date.now()))); }); + } // Wait for all images to finish loading var imgs = Array.from(document.querySelectorAll('img')); var pending = imgs.filter(function(img) { return !img.complete; }); - if (pending.length > 0) { + var imageWaitMs = Math.min(5000, Math.max(0, lazyLoadDeadline - Date.now())); + if (pending.length > 0 && imageWaitMs > 0) { await Promise.race([ Promise.all(pending.map(function(img) { return new Promise(function(r) { img.onload = r; img.onerror = r; }); })), - new Promise(function(r) { setTimeout(r, 5000); }) + new Promise(function(r) { setTimeout(r, imageWaitMs); }) ]); } window.scrollTo(0, 0); - await new Promise(function(r) { setTimeout(r, 500); }); })()`); + } await page1.evaluate(`window.scrollTo(0, 0)`); await new Promise((r) => setTimeout(r, 300)); @@ -289,13 +353,14 @@ export async function captureWebsite( /* DOM scan failed — non-critical */ } - if (discoveredLotties.length > 0) { + if (discoveredLotties.length > 0 && remainingMs() > 0) { const lottieDir = join(outputDir, "assets", "lottie"); mkdirSync(lottieDir, { recursive: true }); - const savedCount = await saveLottieAnimations(discoveredLotties, lottieDir); + const lottieBudget = { remainingMs }; + const savedCount = await saveLottieAnimations(discoveredLotties, lottieDir, lottieBudget); // Generate manifest + preview thumbnails so the agent can SEE what each animation is - if (savedCount > 0) { - await renderLottiePreviews(chromeBrowser, lottieDir, outputDir); + if (savedCount > 0 && remainingMs() > 0) { + await renderLottiePreviews(chromeBrowser, lottieDir, outputDir, lottieBudget); progress("lottie", `${savedCount} Lottie animation(s) saved`); } } @@ -368,7 +433,7 @@ export async function captureWebsite( // Capture scroll-position viewport screenshots progress("screenshots", "Capturing scroll screenshots..."); const { captureScrollScreenshots } = await import("./screenshotCapture.js"); - const screenshots = await captureScrollScreenshots(page1, outputDir); + const screenshots = await captureScrollScreenshots(page1, outputDir, { remainingMs }); progress("screenshots", `${screenshots.length} scroll screenshots captured`); // Catalog all assets (must run before extractHtml which converts img src to data URLs) @@ -437,10 +502,15 @@ export async function captureWebsite( // Generate video manifest — screenshot each