diff --git a/packages/cli/src/capture/assetDownloader.test.ts b/packages/cli/src/capture/assetDownloader.test.ts index fa9da1c65d..4028da90c8 100644 --- a/packages/cli/src/capture/assetDownloader.test.ts +++ b/packages/cli/src/capture/assetDownloader.test.ts @@ -1,5 +1,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { isPrivateUrl, safeFetch, toStandaloneSvg } from "./assetDownloader.js"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + downloadAndRewriteFonts, + isPrivateUrl, + safeFetch, + toStandaloneSvg, +} from "./assetDownloader.js"; describe("isPrivateUrl — SSRF denylist (security: F-003)", () => { it("blocks loopback, private, and metadata IPv4", () => { @@ -127,3 +135,52 @@ describe("toStandaloneSvg — scraped inline SVGs must survive as .svg files", ( expect(toStandaloneSvg("
not an svg
")).toBe("
not an svg
"); }); }); + +describe("downloadAndRewriteFonts — attempt caps", () => { + afterEach(() => vi.unstubAllGlobals()); + + async function expectFailedFontAttempts(css: string, expectedAttempts: number): Promise { + const dir = mkdtempSync(join(tmpdir(), "hf-font-attempts-")); + const fetchMock = vi.fn(async () => new Response("failed", { status: 503 })); + vi.stubGlobal("fetch", fetchMock); + + try { + await downloadAndRewriteFonts(css, dir); + expect(fetchMock).toHaveBeenCalledTimes(expectedAttempts); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + } + + it("counts failed requests toward the global 30-font cap", async () => { + const css = Array.from( + { length: 35 }, + (_, i) => + `@font-face { font-family: Family${i}; src: url(https://fonts${i}.example/font-${i}.woff2); }`, + ).join("\n"); + await expectFailedFontAttempts(css, 30); + }); + + it("counts failed requests toward the six-attempt per-family cap", async () => { + const css = Array.from( + { length: 10 }, + (_, i) => + `@font-face { font-family: Shared; src: url(https://fonts.example/font-${i}.woff2); }`, + ).join("\n"); + await expectFailedFontAttempts(css, 6); + }); + + it("does not start a font request after the capture budget is exhausted", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-font-budget-")); + const css = "@font-face { font-family: Budget; src: url(https://fonts.example/budget.woff2); }"; + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + try { + await downloadAndRewriteFonts(css, dir, { remainingMs: () => 0 }); + expect(fetchMock).not.toHaveBeenCalled(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/src/capture/assetDownloader.ts b/packages/cli/src/capture/assetDownloader.ts index 67b2c0f750..9e783b9348 100644 --- a/packages/cli/src/capture/assetDownloader.ts +++ b/packages/cli/src/capture/assetDownloader.ts @@ -11,6 +11,10 @@ import { createHash } from "node:crypto"; import type { DesignTokens, DownloadedAsset } from "./types.js"; import type { CatalogedAsset } from "./assetCataloger.js"; +interface DownloadBudgetOptions { + remainingMs?: () => number; +} + // SVGs: hash-of-bytes filename so it can't drift from content; label-derived names mis-assigned brands. function svgContentHashSlug(svgSource: string | Buffer, isLogo: boolean): string { const hash = createHash("sha1").update(svgSource).digest("hex").slice(0, 8); @@ -43,11 +47,13 @@ export function toStandaloneSvg(outerHTML: string): string { return outerHTML.replace(original, tag); } +// fallow-ignore-next-line complexity export async function downloadAssets( tokens: DesignTokens, outputDir: string, catalogedAssets?: CatalogedAsset[], faviconLinks?: Array<{ rel: string; href: string }>, + options: DownloadBudgetOptions = {}, ): Promise { const assetsDir = join(outputDir, "assets"); mkdirSync(assetsDir, { recursive: true }); @@ -82,12 +88,14 @@ export async function downloadAssets( // 2. Favicon for (const icon of faviconLinks || []) { + const remainingMs = options.remainingMs?.() ?? 10_000; + if (remainingMs <= 0) break; if (!icon.href) continue; try { const ext = extname(new URL(icon.href).pathname) || ".ico"; const name = `favicon${ext}`; const localPath = `assets/${name}`; - const buffer = await fetchBuffer(icon.href); + const buffer = await fetchBuffer(icon.href, Math.min(10_000, remainingMs)); if (buffer) { writeFileSync(join(outputDir, localPath), buffer); assets.push({ url: icon.href, localPath, type: "favicon" }); @@ -149,13 +157,15 @@ export async function downloadAssets( let imgIdx = 0; const usedNames = new Set(); for (let i = 0; i < toDownload.length; i += BATCH_SIZE) { + const remainingMs = options.remainingMs?.() ?? 10_000; + if (remainingMs <= 0) break; const batch = toDownload.slice(i, i + BATCH_SIZE); const results = await Promise.allSettled( batch.map(async ({ url, isPoster, catalog }) => { const parsedUrl = new URL(url); const pathExt = extname(parsedUrl.pathname); const ext = pathExt && pathExt.length <= 5 ? pathExt : ".jpg"; - const buffer = await fetchBuffer(url); + const buffer = await fetchBuffer(url, Math.min(10_000, remainingMs)); if (!buffer) return null; const isSvg = ext === ".svg" || url.includes(".svg"); const minSize = isSvg ? 200 : 10000; @@ -198,10 +208,12 @@ export async function downloadAssets( // 4. OG image (if not already downloaded) if (tokens.ogImage && !downloadedUrls.has(normalizeUrl(tokens.ogImage))) { + const remainingMs = options.remainingMs?.() ?? 10_000; try { const ext = extname(new URL(tokens.ogImage).pathname) || ".jpg"; const localPath = `assets/og-image${ext}`; - const buffer = await fetchBuffer(tokens.ogImage); + const buffer = + remainingMs > 0 ? await fetchBuffer(tokens.ogImage, Math.min(10_000, remainingMs)) : null; if (buffer && buffer.length > 5000) { writeFileSync(join(outputDir, localPath), buffer); assets.push({ url: tokens.ogImage, localPath, type: "image" }); @@ -234,7 +246,12 @@ function normalizeUrl(u: string): string { * Download fonts referenced in CSS and rewrite URLs to local paths. * Returns the modified CSS string with local font paths. */ -export async function downloadAndRewriteFonts(css: string, outputDir: string): Promise { +// fallow-ignore-next-line complexity +export async function downloadAndRewriteFonts( + css: string, + outputDir: string, + options: DownloadBudgetOptions = {}, +): Promise { const assetsDir = join(outputDir, "assets", "fonts"); mkdirSync(assetsDir, { recursive: true }); @@ -275,10 +292,14 @@ export async function downloadAndRewriteFonts(css: string, outputDir: string): P let count = 0; for (const fontUrl of sortedUrls) { + const remainingMs = options.remainingMs?.() ?? 10_000; + if (remainingMs <= 0) break; if (count >= MAX_TOTAL_FONTS) break; const family = getFamilyForUrl(fontUrl); const familyCount = familyCounts.get(family) || 0; if (familyCount >= MAX_FONTS_PER_FAMILY) continue; + familyCounts.set(family, familyCount + 1); + count++; try { const urlObj = new URL(fontUrl); @@ -286,12 +307,10 @@ export async function downloadAndRewriteFonts(css: string, outputDir: string): P const localPath = join(assetsDir, filename); const relativePath = `assets/fonts/${filename}`; - const buffer = await fetchBuffer(fontUrl); + const buffer = await fetchBuffer(fontUrl, Math.min(10_000, remainingMs)); if (buffer) { writeFileSync(localPath, buffer); rewritten = rewritten.split(fontUrl).join(relativePath); - familyCounts.set(family, familyCount + 1); - count++; } } catch { /* skip */ @@ -391,10 +410,10 @@ export async function safeFetch(url: string, init?: RequestInit): Promise { +async function fetchBuffer(url: string, timeoutMs = 10_000): Promise { try { const res = await safeFetch(url, { - signal: AbortSignal.timeout(10000), + signal: AbortSignal.timeout(timeoutMs), headers: { "User-Agent": "HyperFrames/1.0" }, }); if (!res || !res.ok) return null; diff --git a/packages/cli/src/capture/contactSheet.ts b/packages/cli/src/capture/contactSheet.ts index 6289c84494..ffb5b37a22 100644 --- a/packages/cli/src/capture/contactSheet.ts +++ b/packages/cli/src/capture/contactSheet.ts @@ -18,6 +18,8 @@ interface ContactSheetOptions { quality?: number; /** Target width per cell in pixels (default: 600) */ cellWidth?: number; + /** Cooperative boundary checked before starting each native Sharp page. */ + remainingMs?: () => number; } /** @@ -39,6 +41,8 @@ export async function createContactSheet( cellWidth = 600, } = opts; + if ((opts.remainingMs?.() ?? 1) <= 0) return null; + const files = imagePaths.slice(0, maxImages); if (files.length === 0) return null; @@ -125,6 +129,7 @@ function escapeXml(s: string): string { * Output files: basePath → base-1.jpg, base-2.jpg, ... * Returns the list of written file paths (empty if no images). */ +// fallow-ignore-next-line complexity async function createContactSheetPages( imagePaths: string[], outputBasePath: string, @@ -133,7 +138,7 @@ async function createContactSheetPages( customLabels?: string[], ): Promise { if (imagePaths.length === 0) return []; - const { pageSize = imagePaths.length, ...sheetOpts } = opts; + const { pageSize = imagePaths.length, remainingMs, ...sheetOpts } = opts; const ext = outputBasePath.match(/\.[^.]+$/)?.[0] ?? ".jpg"; const base = outputBasePath.slice(0, -ext.length); @@ -141,6 +146,7 @@ async function createContactSheetPages( const results: string[] = []; for (let p = 0; p < pages; p++) { + if ((remainingMs?.() ?? 1) <= 0) break; const chunk = imagePaths.slice(p * pageSize, (p + 1) * pageSize); const chunkLabels = customLabels?.slice(p * pageSize, (p + 1) * pageSize); const outPath = pages === 1 ? outputBasePath : `${base}-${p + 1}${ext}`; @@ -170,6 +176,7 @@ async function createContactSheetPages( export async function createScrollContactSheet( screenshotsDir: string, outputPath: string, + budget: Pick = {}, ): Promise { if (!existsSync(screenshotsDir)) return []; @@ -189,7 +196,7 @@ export async function createScrollContactSheet( return createContactSheetPages( paths, outputPath, - { cols: 3, cellWidth: 600, pageSize: 9 }, + { cols: 3, cellWidth: 600, pageSize: 9, ...budget }, 0, labels, ); @@ -203,6 +210,7 @@ export async function createScrollContactSheet( export async function createSnapshotContactSheet( snapshotsDir: string, outputPath: string, + budget: Pick = {}, ): Promise { if (!existsSync(snapshotsDir)) return []; @@ -222,7 +230,7 @@ export async function createSnapshotContactSheet( return createContactSheetPages( paths, outputPath, - { cols: 3, cellWidth: 600, pageSize: 9 }, + { cols: 3, cellWidth: 600, pageSize: 9, ...budget }, 0, labels, ); @@ -236,6 +244,7 @@ export async function createSnapshotContactSheet( export async function createAssetContactSheet( assetsDir: string, outputPath: string, + budget: Pick = {}, ): Promise { if (!existsSync(assetsDir)) return []; @@ -254,6 +263,7 @@ export async function createAssetContactSheet( cellWidth: 480, labelMode: "filename", pageSize: 12, + ...budget, }); } @@ -271,6 +281,7 @@ export async function createSvgContactSheet( svgsDir: string, outputPath: string, assetsRootDir?: string, + budget: Pick = {}, ): Promise { const dirsToScan = [svgsDir, assetsRootDir].filter( (d): d is string => d !== undefined && existsSync(d), @@ -302,6 +313,7 @@ export async function createSvgContactSheet( const labels: string[] = []; for (let i = 0; i < svgPaths.length; i++) { + if ((budget.remainingMs?.() ?? 1) <= 0) break; const svgPath = svgPaths[i]!; const tmpPath = join(tmpDir, `.thumb-${i}.png`); try { @@ -334,6 +346,7 @@ export async function createSvgContactSheet( cols: 5, cellWidth: thumbSize, pageSize: 15, + ...budget, }, 0, labels, diff --git a/packages/cli/src/capture/contentExtractor.ts b/packages/cli/src/capture/contentExtractor.ts index 2caee86e48..824ee01788 100644 --- a/packages/cli/src/capture/contentExtractor.ts +++ b/packages/cli/src/capture/contentExtractor.ts @@ -15,6 +15,47 @@ import type sharpType from "sharp"; import type { CatalogedAsset } from "./assetCataloger.js"; import type { DesignTokens } from "./types.js"; +const DEFAULT_VISION_REQUEST_TIMEOUT_MS = 30_000; + +interface VisionCaptionOptions { + skipVision?: boolean; + remainingMs?: () => number; +} + +class VisionRequestTimeoutError extends Error { + constructor(timeoutMs: number) { + super(`Vision request timed out after ${timeoutMs}ms`); + this.name = "VisionRequestTimeoutError"; + } +} + +function resolveVisionRequestTimeoutMs(): number { + const configured = Number(process.env.HYPERFRAMES_VISION_TIMEOUT_MS); + return Number.isFinite(configured) && configured > 0 + ? configured + : DEFAULT_VISION_REQUEST_TIMEOUT_MS; +} + +async function runBoundedVisionRequest( + request: (signal: AbortSignal) => Promise, + timeoutMs: number, +): Promise { + const controller = new AbortController(); + let timer: ReturnType | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + reject(new VisionRequestTimeoutError(timeoutMs)); + controller.abort(); + }, timeoutMs); + }); + + try { + return await Promise.race([request(controller.signal), timeout]); + } finally { + if (timer) clearTimeout(timer); + } +} + /** * Detect JS libraries via window globals, DOM fingerprints, script URLs, * and WebGL shader analysis. @@ -165,12 +206,15 @@ export async function extractVisibleText(page: Page): Promise { * Batches requests to stay under free-tier rate limits. * Returns a map of filename -> caption string. */ +// fallow-ignore-next-line complexity export async function captionImagesWithGemini( outputDir: string, progress: (stage: string, detail?: string) => void, warnings: string[], + options: VisionCaptionOptions = {}, ): Promise> { const geminiCaptions: Record = {}; + if (options.skipVision) return geminiCaptions; const openRouterKey = process.env.OPENROUTER_API_KEY; const geminiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY; if (!openRouterKey && !geminiKey) return geminiCaptions; @@ -186,6 +230,7 @@ export async function captionImagesWithGemini( const model = useOpenRouter ? process.env.HYPERFRAMES_OPENROUTER_MODEL || "google/gemini-3.1-flash-lite" : process.env.HYPERFRAMES_GEMINI_MODEL || "gemini-3.1-flash-lite-preview"; + const requestTimeoutMs = resolveVisionRequestTimeoutMs(); progress("design", `Captioning images with ${providerName} vision...`); try { @@ -196,31 +241,40 @@ export async function captionImagesWithGemini( base64: string; prompt: string; maxTokens: number; + timeoutMs: number; }) => Promise; let captionOne: CaptionOne; if (openRouterKey) { - captionOne = async ({ mimeType, base64, prompt, maxTokens }) => { - const res = await fetch("https://openrouter.ai/api/v1/chat/completions", { - method: "POST", - headers: { - Authorization: `Bearer ${openRouterKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - model, - messages: [ - { - role: "user", - content: [ - { type: "text", text: prompt }, - { type: "image_url", image_url: { url: `data:${mimeType};base64,${base64}` } }, - ], + captionOne = async ({ mimeType, base64, prompt, maxTokens, timeoutMs }) => { + const res = await runBoundedVisionRequest( + (signal) => + fetch("https://openrouter.ai/api/v1/chat/completions", { + method: "POST", + headers: { + Authorization: `Bearer ${openRouterKey}`, + "Content-Type": "application/json", }, - ], - max_tokens: maxTokens, - }), - }); + signal, + body: JSON.stringify({ + model, + messages: [ + { + role: "user", + content: [ + { type: "text", text: prompt }, + { + type: "image_url", + image_url: { url: `data:${mimeType};base64,${base64}` }, + }, + ], + }, + ], + max_tokens: maxTokens, + }), + }), + timeoutMs, + ); if (!res.ok) { const detail = await res.text().catch(() => ""); throw new Error(`OpenRouter ${res.status} ${res.statusText}: ${detail.slice(0, 200)}`); @@ -235,14 +289,25 @@ export async function captionImagesWithGemini( if (!geminiKey) return geminiCaptions; const { GoogleGenAI } = await import("@google/genai"); const ai = new GoogleGenAI({ apiKey: geminiKey }); - captionOne = async ({ mimeType, base64, prompt, maxTokens }) => { - const response = await ai.models.generateContent({ - model, - contents: [ - { role: "user", parts: [{ inlineData: { mimeType, data: base64 } }, { text: prompt }] }, - ], - config: { maxOutputTokens: maxTokens }, - }); + captionOne = async ({ mimeType, base64, prompt, maxTokens, timeoutMs }) => { + const response = await runBoundedVisionRequest( + (signal) => + ai.models.generateContent({ + model, + contents: [ + { + role: "user", + parts: [{ inlineData: { mimeType, data: base64 } }, { text: prompt }], + }, + ], + config: { + maxOutputTokens: maxTokens, + abortSignal: signal, + httpOptions: { timeout: timeoutMs }, + }, + }), + timeoutMs, + ); return response.text?.trim() || ""; }; } @@ -256,7 +321,27 @@ export async function captionImagesWithGemini( // on Promise.allSettled so a rate-limited image degrades to "" rather than // failing the batch. const BATCH_SIZE = 20; + let timedOutCount = 0; + const collectCaptionResults = ( + results: PromiseSettledResult<{ file: string; caption: string }>[], + ): number => { + let timeouts = 0; + for (const result of results) { + if (result.status === "fulfilled" && result.value.caption) { + geminiCaptions[result.value.file] = result.value.caption; + } else if ( + result.status === "rejected" && + result.reason instanceof VisionRequestTimeoutError + ) { + timeouts++; + } + } + return timeouts; + }; for (let i = 0; i < imageFiles.length; i += BATCH_SIZE) { + const remainingMs = options.remainingMs?.() ?? Number.POSITIVE_INFINITY; + if (remainingMs <= 0) break; + const timeoutMs = Math.max(1, Math.min(requestTimeoutMs, remainingMs)); const batch = imageFiles.slice(i, i + BATCH_SIZE); const results = await Promise.allSettled( batch.map(async (file: string) => { @@ -273,15 +358,12 @@ export async function captionImagesWithGemini( prompt: "Describe this website image in ONE short sentence for a video storyboard. Focus on: what it shows, dominant colors, whether background is light or dark. Be factual, not creative.", maxTokens: 500, + timeoutMs, }); return { file, caption }; }), ); - for (const result of results) { - if (result.status === "fulfilled" && result.value.caption) { - geminiCaptions[result.value.file] = result.value.caption; - } - } + timedOutCount += collectCaptionResults(results); // Pace requests between batches (paid tier: 2000+ RPM, free tier: rate-limited) if (i + BATCH_SIZE < imageFiles.length) { await new Promise((r) => setTimeout(r, 2000)); // 2s pause between batches — paid tier handles 2000 RPM, free tier retries via Promise.allSettled @@ -329,6 +411,9 @@ export async function captionImagesWithGemini( const SVG_RENDER_SIZE = 256; // px — enough resolution for Gemini to read wordmarks, small enough to keep payload sub-MB let svgsSkipped = 0; for (let i = 0; i < svgFiles.length; i += SVG_BATCH) { + const remainingMs = options.remainingMs?.() ?? Number.POSITIVE_INFINITY; + if (remainingMs <= 0) break; + const timeoutMs = Math.max(1, Math.min(requestTimeoutMs, remainingMs)); const batch = svgFiles.slice(i, i + SVG_BATCH); const results = await Promise.allSettled( batch.map(async ({ relPath }) => { @@ -373,15 +458,12 @@ export async function captionImagesWithGemini( "If you see a wordmark, READ THE LETTERS LITERALLY — do not guess a brand from context. " + "Be factual.", maxTokens: 300, + timeoutMs, }); return { file: relPath, caption }; }), ); - for (const result of results) { - if (result.status === "fulfilled" && result.value.caption) { - geminiCaptions[result.value.file] = result.value.caption; - } - } + timedOutCount += collectCaptionResults(results); if (i + SVG_BATCH < svgFiles.length) { await new Promise((r) => setTimeout(r, 2000)); } @@ -398,6 +480,11 @@ export async function captionImagesWithGemini( ); } } + if (timedOutCount > 0) { + warnings.push( + `${providerName} vision timed out for ${timedOutCount} asset(s); captions omitted.`, + ); + } } catch (err) { warnings.push(`${providerName} captioning failed: ${err}`); } diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts index fe58a972f4..8b90164bd0 100644 --- a/packages/cli/src/capture/index.ts +++ b/packages/cli/src/capture/index.ts @@ -39,10 +39,12 @@ import { generateAssetDescriptions, } from "./contentExtractor.js"; import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js"; -import type { CaptureOptions, CaptureResult } from "./types.js"; +import type { CaptureOptions, CapturePhase, CapturePhaseProgress, CaptureResult } from "./types.js"; export type { CaptureOptions, CaptureResult } from "./types.js"; +const DEFAULT_POST_NAVIGATION_BUDGET_MS = 120_000; + // fallow-ignore-next-line complexity export async function captureWebsite( opts: CaptureOptions, @@ -57,12 +59,49 @@ export async function captureWebsite( settleTime = 3000, maxScreenshots: _maxScreenshots = 24, skipAssets = false, + skipVision = false, + postNavigationBudgetMs = DEFAULT_POST_NAVIGATION_BUDGET_MS, + onPhase, } = opts; const warnings: string[] = []; const progress = (stage: string, detail?: string) => { onProgress?.(stage, detail); }; + const budgetMs = + Number.isFinite(postNavigationBudgetMs) && postNavigationBudgetMs > 0 + ? postNavigationBudgetMs + : DEFAULT_POST_NAVIGATION_BUDGET_MS; + let postNavigationDeadline: number | undefined; + const remainingMs = (): number => + postNavigationDeadline === undefined + ? budgetMs + : Math.max(0, postNavigationDeadline - Date.now()); + let lastPhase: CapturePhaseProgress = { + schema: "hyperframes.capture.phase.v1", + phase: "browser", + status: "started", + remainingMs: null, + }; + const phase = ( + name: CapturePhase, + status: CapturePhaseProgress["status"], + reason?: CapturePhaseProgress["reason"], + ): void => { + const remaining = postNavigationDeadline === undefined ? null : remainingMs(); + lastPhase = reason + ? { + schema: "hyperframes.capture.phase.v1", + phase: name, + status, + remainingMs: remaining, + reason, + } + : { schema: "hyperframes.capture.phase.v1", phase: name, status, remainingMs: remaining }; + onPhase?.(lastPhase); + }; + + phase("browser", "started"); // Load .env file from repo root if it exists (for GEMINI_API_KEY, etc.) loadEnvFile(outputDir); @@ -102,6 +141,8 @@ export async function captureWebsite( // Goal: Catalog animations + take screenshots (with JS rendering) // ═══════════════════════════════════════════════════════════════ + phase("browser", "completed"); + phase("navigation", "started"); progress("animations", "Cataloging animations (full JS)..."); const page1 = await chromeBrowser.newPage(); @@ -198,7 +239,10 @@ export async function captureWebsite( // modern SPAs often have persistent WebSocket/analytics connections that // prevent networkidle0 from ever resolving. await page1.goto(url, { waitUntil: "networkidle2", timeout }); + postNavigationDeadline = Date.now() + budgetMs; await new Promise((r) => setTimeout(r, settleTime)); + phase("navigation", "completed"); + phase("core-extraction", "started"); // Check if the page loaded real content or an anti-bot challenge // Use structural detection (DOM elements + cookies), not text regex matching — @@ -231,29 +275,36 @@ export async function captureWebsite( // Framer and other modern sites use IntersectionObserver — images only load // when scrolled into view. We scroll the full page, then wait for all images // to finish loading before proceeding. - await page1.evaluate(`(async () => { + const lazyLoadBudgetMs = Math.min(15_000, remainingMs()); + if (lazyLoadBudgetMs > 0) { + await page1.evaluate(`(async () => { + var lazyLoadDeadline = Date.now() + ${lazyLoadBudgetMs}; var h = document.body.scrollHeight; for (var y = 0; y < h; y += window.innerHeight * 0.7) { + if (Date.now() >= lazyLoadDeadline) break; window.scrollTo(0, y); await new Promise(function(r) { setTimeout(r, 400); }); } // Scroll to very bottom to catch footer lazy-loads - window.scrollTo(0, document.body.scrollHeight); - await new Promise(function(r) { setTimeout(r, 800); }); + if (Date.now() < lazyLoadDeadline) { + window.scrollTo(0, document.body.scrollHeight); + await new Promise(function(r) { setTimeout(r, Math.min(800, Math.max(0, lazyLoadDeadline - Date.now()))); }); + } // Wait for all images to finish loading var imgs = Array.from(document.querySelectorAll('img')); var pending = imgs.filter(function(img) { return !img.complete; }); - if (pending.length > 0) { + var imageWaitMs = Math.min(5000, Math.max(0, lazyLoadDeadline - Date.now())); + if (pending.length > 0 && imageWaitMs > 0) { await Promise.race([ Promise.all(pending.map(function(img) { return new Promise(function(r) { img.onload = r; img.onerror = r; }); })), - new Promise(function(r) { setTimeout(r, 5000); }) + new Promise(function(r) { setTimeout(r, imageWaitMs); }) ]); } window.scrollTo(0, 0); - await new Promise(function(r) { setTimeout(r, 500); }); })()`); + } await page1.evaluate(`window.scrollTo(0, 0)`); await new Promise((r) => setTimeout(r, 300)); @@ -289,12 +340,12 @@ export async function captureWebsite( /* DOM scan failed — non-critical */ } - if (discoveredLotties.length > 0) { + if (discoveredLotties.length > 0 && remainingMs() > 0) { const lottieDir = join(outputDir, "assets", "lottie"); mkdirSync(lottieDir, { recursive: true }); const savedCount = await saveLottieAnimations(discoveredLotties, lottieDir); // Generate manifest + preview thumbnails so the agent can SEE what each animation is - if (savedCount > 0) { + if (savedCount > 0 && remainingMs() > 0) { await renderLottiePreviews(chromeBrowser, lottieDir, outputDir); progress("lottie", `${savedCount} Lottie animation(s) saved`); } @@ -368,7 +419,7 @@ export async function captureWebsite( // Capture scroll-position viewport screenshots progress("screenshots", "Capturing scroll screenshots..."); const { captureScrollScreenshots } = await import("./screenshotCapture.js"); - const screenshots = await captureScrollScreenshots(page1, outputDir); + const screenshots = await captureScrollScreenshots(page1, outputDir, { remainingMs }); progress("screenshots", `${screenshots.length} scroll screenshots captured`); // Catalog all assets (must run before extractHtml which converts img src to data URLs) @@ -437,10 +488,14 @@ export async function captureWebsite( // Generate video manifest — screenshot each