diff --git a/packages/cli/src/capture/assetDownloader.test.ts b/packages/cli/src/capture/assetDownloader.test.ts
index fa9da1c65d..4028da90c8 100644
--- a/packages/cli/src/capture/assetDownloader.test.ts
+++ b/packages/cli/src/capture/assetDownloader.test.ts
@@ -1,5 +1,13 @@
import { afterEach, describe, expect, it, vi } from "vitest";
-import { isPrivateUrl, safeFetch, toStandaloneSvg } from "./assetDownloader.js";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+ downloadAndRewriteFonts,
+ isPrivateUrl,
+ safeFetch,
+ toStandaloneSvg,
+} from "./assetDownloader.js";
describe("isPrivateUrl — SSRF denylist (security: F-003)", () => {
it("blocks loopback, private, and metadata IPv4", () => {
@@ -127,3 +135,52 @@ describe("toStandaloneSvg — scraped inline SVGs must survive as .svg files", (
expect(toStandaloneSvg("
not an svg
")).toBe("not an svg
");
});
});
+
+describe("downloadAndRewriteFonts — attempt caps", () => {
+ afterEach(() => vi.unstubAllGlobals());
+
+ async function expectFailedFontAttempts(css: string, expectedAttempts: number): Promise {
+ const dir = mkdtempSync(join(tmpdir(), "hf-font-attempts-"));
+ const fetchMock = vi.fn(async () => new Response("failed", { status: 503 }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ try {
+ await downloadAndRewriteFonts(css, dir);
+ expect(fetchMock).toHaveBeenCalledTimes(expectedAttempts);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ }
+
+ it("counts failed requests toward the global 30-font cap", async () => {
+ const css = Array.from(
+ { length: 35 },
+ (_, i) =>
+ `@font-face { font-family: Family${i}; src: url(https://fonts${i}.example/font-${i}.woff2); }`,
+ ).join("\n");
+ await expectFailedFontAttempts(css, 30);
+ });
+
+ it("counts failed requests toward the six-attempt per-family cap", async () => {
+ const css = Array.from(
+ { length: 10 },
+ (_, i) =>
+ `@font-face { font-family: Shared; src: url(https://fonts.example/font-${i}.woff2); }`,
+ ).join("\n");
+ await expectFailedFontAttempts(css, 6);
+ });
+
+ it("does not start a font request after the capture budget is exhausted", async () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-font-budget-"));
+ const css = "@font-face { font-family: Budget; src: url(https://fonts.example/budget.woff2); }";
+ const fetchMock = vi.fn();
+ vi.stubGlobal("fetch", fetchMock);
+
+ try {
+ await downloadAndRewriteFonts(css, dir, { remainingMs: () => 0 });
+ expect(fetchMock).not.toHaveBeenCalled();
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/packages/cli/src/capture/assetDownloader.ts b/packages/cli/src/capture/assetDownloader.ts
index 67b2c0f750..21a881f3b9 100644
--- a/packages/cli/src/capture/assetDownloader.ts
+++ b/packages/cli/src/capture/assetDownloader.ts
@@ -11,6 +11,10 @@ import { createHash } from "node:crypto";
import type { DesignTokens, DownloadedAsset } from "./types.js";
import type { CatalogedAsset } from "./assetCataloger.js";
+interface DownloadBudgetOptions {
+ remainingMs?: () => number;
+}
+
// SVGs: hash-of-bytes filename so it can't drift from content; label-derived names mis-assigned brands.
function svgContentHashSlug(svgSource: string | Buffer, isLogo: boolean): string {
const hash = createHash("sha1").update(svgSource).digest("hex").slice(0, 8);
@@ -43,11 +47,13 @@ export function toStandaloneSvg(outerHTML: string): string {
return outerHTML.replace(original, tag);
}
+// fallow-ignore-next-line complexity
export async function downloadAssets(
tokens: DesignTokens,
outputDir: string,
catalogedAssets?: CatalogedAsset[],
faviconLinks?: Array<{ rel: string; href: string }>,
+ options: DownloadBudgetOptions = {},
): Promise {
const assetsDir = join(outputDir, "assets");
mkdirSync(assetsDir, { recursive: true });
@@ -82,12 +88,14 @@ export async function downloadAssets(
// 2. Favicon
for (const icon of faviconLinks || []) {
+ const remainingMs = options.remainingMs?.() ?? 10_000;
+ if (remainingMs <= 0) break;
if (!icon.href) continue;
try {
const ext = extname(new URL(icon.href).pathname) || ".ico";
const name = `favicon${ext}`;
const localPath = `assets/${name}`;
- const buffer = await fetchBuffer(icon.href);
+ const buffer = await fetchBuffer(icon.href, Math.min(10_000, remainingMs));
if (buffer) {
writeFileSync(join(outputDir, localPath), buffer);
assets.push({ url: icon.href, localPath, type: "favicon" });
@@ -149,13 +157,15 @@ export async function downloadAssets(
let imgIdx = 0;
const usedNames = new Set();
for (let i = 0; i < toDownload.length; i += BATCH_SIZE) {
+ const remainingMs = options.remainingMs?.() ?? 10_000;
+ if (remainingMs <= 0) break;
const batch = toDownload.slice(i, i + BATCH_SIZE);
const results = await Promise.allSettled(
batch.map(async ({ url, isPoster, catalog }) => {
const parsedUrl = new URL(url);
const pathExt = extname(parsedUrl.pathname);
const ext = pathExt && pathExt.length <= 5 ? pathExt : ".jpg";
- const buffer = await fetchBuffer(url);
+ const buffer = await fetchBuffer(url, Math.min(10_000, remainingMs));
if (!buffer) return null;
const isSvg = ext === ".svg" || url.includes(".svg");
const minSize = isSvg ? 200 : 10000;
@@ -198,10 +208,12 @@ export async function downloadAssets(
// 4. OG image (if not already downloaded)
if (tokens.ogImage && !downloadedUrls.has(normalizeUrl(tokens.ogImage))) {
+ const remainingMs = options.remainingMs?.() ?? 10_000;
try {
const ext = extname(new URL(tokens.ogImage).pathname) || ".jpg";
const localPath = `assets/og-image${ext}`;
- const buffer = await fetchBuffer(tokens.ogImage);
+ const buffer =
+ remainingMs > 0 ? await fetchBuffer(tokens.ogImage, Math.min(10_000, remainingMs)) : null;
if (buffer && buffer.length > 5000) {
writeFileSync(join(outputDir, localPath), buffer);
assets.push({ url: tokens.ogImage, localPath, type: "image" });
@@ -234,7 +246,12 @@ function normalizeUrl(u: string): string {
* Download fonts referenced in CSS and rewrite URLs to local paths.
* Returns the modified CSS string with local font paths.
*/
-export async function downloadAndRewriteFonts(css: string, outputDir: string): Promise {
+// fallow-ignore-next-line complexity
+export async function downloadAndRewriteFonts(
+ css: string,
+ outputDir: string,
+ options: DownloadBudgetOptions = {},
+): Promise {
const assetsDir = join(outputDir, "assets", "fonts");
mkdirSync(assetsDir, { recursive: true });
@@ -247,8 +264,10 @@ export async function downloadAndRewriteFonts(css: string, outputDir: string): P
if (fontUrls.size === 0) return css;
- // Limit font downloads to avoid bloat. Google Fonts serves 20+ unicode-range
- // subsets per weight — we only need a few per family for video production.
+ // Limit font download attempts to bound worst-case egress and latency. Google Fonts serves
+ // 20+ unicode-range subsets per weight, so successes alone cannot be the bound: six transient
+ // failures can intentionally suppress later URLs in that family. Latin-priority sorting below
+ // makes the limited attempts useful while keeping this failure tradeoff explicit.
const MAX_FONTS_PER_FAMILY = 6;
const MAX_TOTAL_FONTS = 30;
const familyCounts = new Map();
@@ -275,10 +294,14 @@ export async function downloadAndRewriteFonts(css: string, outputDir: string): P
let count = 0;
for (const fontUrl of sortedUrls) {
+ const remainingMs = options.remainingMs?.() ?? 10_000;
+ if (remainingMs <= 0) break;
if (count >= MAX_TOTAL_FONTS) break;
const family = getFamilyForUrl(fontUrl);
const familyCount = familyCounts.get(family) || 0;
if (familyCount >= MAX_FONTS_PER_FAMILY) continue;
+ familyCounts.set(family, familyCount + 1);
+ count++;
try {
const urlObj = new URL(fontUrl);
@@ -286,12 +309,10 @@ export async function downloadAndRewriteFonts(css: string, outputDir: string): P
const localPath = join(assetsDir, filename);
const relativePath = `assets/fonts/${filename}`;
- const buffer = await fetchBuffer(fontUrl);
+ const buffer = await fetchBuffer(fontUrl, Math.min(10_000, remainingMs));
if (buffer) {
writeFileSync(localPath, buffer);
rewritten = rewritten.split(fontUrl).join(relativePath);
- familyCounts.set(family, familyCount + 1);
- count++;
}
} catch {
/* skip */
@@ -391,10 +412,10 @@ export async function safeFetch(url: string, init?: RequestInit): Promise {
+async function fetchBuffer(url: string, timeoutMs = 10_000): Promise {
try {
const res = await safeFetch(url, {
- signal: AbortSignal.timeout(10000),
+ signal: AbortSignal.timeout(timeoutMs),
headers: { "User-Agent": "HyperFrames/1.0" },
});
if (!res || !res.ok) return null;
diff --git a/packages/cli/src/capture/contactSheet.test.ts b/packages/cli/src/capture/contactSheet.test.ts
index 5b2dc74464..71432a754b 100644
--- a/packages/cli/src/capture/contactSheet.test.ts
+++ b/packages/cli/src/capture/contactSheet.test.ts
@@ -1,9 +1,13 @@
-import { mkdtempSync, rmSync } from "node:fs";
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import sharp from "sharp";
import { describe, expect, it } from "vitest";
-import { createContactSheet } from "./contactSheet.js";
+import {
+ createContactSheet,
+ createScrollContactSheet,
+ createSvgContactSheet,
+} from "./contactSheet.js";
function tempDir(): string {
return mkdtempSync(join(tmpdir(), "hf-contact-sheet-test-"));
@@ -58,3 +62,64 @@ describe("createContactSheet", () => {
}
}, 60_000);
});
+
+describe("contact-sheet capture budget", () => {
+ it("does not start another Sharp page after the budget is exhausted", async () => {
+ const dir = tempDir();
+ try {
+ for (let i = 0; i < 10; i++) {
+ await sharp({
+ create: {
+ width: 4,
+ height: 4,
+ channels: 3,
+ background: { r: i, g: i, b: i },
+ },
+ })
+ .png()
+ .toFile(join(dir, `scroll-${String(i).padStart(3, "0")}.png`));
+ }
+
+ let checks = 0;
+ const output = join(dir, "contact-sheet.jpg");
+ const sheets = await createScrollContactSheet(dir, output, {
+ remainingMs: () => (checks++ === 0 ? 1000 : 0),
+ });
+
+ expect(sheets).toEqual([join(dir, "contact-sheet-1.jpg")]);
+ expect(existsSync(join(dir, "contact-sheet-2.jpg"))).toBe(false);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ }, 60_000);
+
+ it("stops SVG thumbnail rasterization before the next native operation", async () => {
+ const dir = tempDir();
+ try {
+ const svgs = join(dir, "svgs");
+ const { mkdirSync } = await import("node:fs");
+ mkdirSync(svgs);
+ writeFileSync(
+ join(svgs, "a.svg"),
+ '',
+ );
+ writeFileSync(
+ join(svgs, "b.svg"),
+ '',
+ );
+ let checks = 0;
+
+ const sheets = await createSvgContactSheet(
+ svgs,
+ join(dir, "svg-contact-sheet.jpg"),
+ undefined,
+ { remainingMs: () => (checks++ === 0 ? 1000 : 0) },
+ );
+
+ expect(sheets).toEqual([]);
+ expect(checks).toBeGreaterThanOrEqual(2);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ }, 60_000);
+});
diff --git a/packages/cli/src/capture/contactSheet.ts b/packages/cli/src/capture/contactSheet.ts
index 6289c84494..ffb5b37a22 100644
--- a/packages/cli/src/capture/contactSheet.ts
+++ b/packages/cli/src/capture/contactSheet.ts
@@ -18,6 +18,8 @@ interface ContactSheetOptions {
quality?: number;
/** Target width per cell in pixels (default: 600) */
cellWidth?: number;
+ /** Cooperative boundary checked before starting each native Sharp page. */
+ remainingMs?: () => number;
}
/**
@@ -39,6 +41,8 @@ export async function createContactSheet(
cellWidth = 600,
} = opts;
+ if ((opts.remainingMs?.() ?? 1) <= 0) return null;
+
const files = imagePaths.slice(0, maxImages);
if (files.length === 0) return null;
@@ -125,6 +129,7 @@ function escapeXml(s: string): string {
* Output files: basePath → base-1.jpg, base-2.jpg, ...
* Returns the list of written file paths (empty if no images).
*/
+// fallow-ignore-next-line complexity
async function createContactSheetPages(
imagePaths: string[],
outputBasePath: string,
@@ -133,7 +138,7 @@ async function createContactSheetPages(
customLabels?: string[],
): Promise {
if (imagePaths.length === 0) return [];
- const { pageSize = imagePaths.length, ...sheetOpts } = opts;
+ const { pageSize = imagePaths.length, remainingMs, ...sheetOpts } = opts;
const ext = outputBasePath.match(/\.[^.]+$/)?.[0] ?? ".jpg";
const base = outputBasePath.slice(0, -ext.length);
@@ -141,6 +146,7 @@ async function createContactSheetPages(
const results: string[] = [];
for (let p = 0; p < pages; p++) {
+ if ((remainingMs?.() ?? 1) <= 0) break;
const chunk = imagePaths.slice(p * pageSize, (p + 1) * pageSize);
const chunkLabels = customLabels?.slice(p * pageSize, (p + 1) * pageSize);
const outPath = pages === 1 ? outputBasePath : `${base}-${p + 1}${ext}`;
@@ -170,6 +176,7 @@ async function createContactSheetPages(
export async function createScrollContactSheet(
screenshotsDir: string,
outputPath: string,
+ budget: Pick = {},
): Promise {
if (!existsSync(screenshotsDir)) return [];
@@ -189,7 +196,7 @@ export async function createScrollContactSheet(
return createContactSheetPages(
paths,
outputPath,
- { cols: 3, cellWidth: 600, pageSize: 9 },
+ { cols: 3, cellWidth: 600, pageSize: 9, ...budget },
0,
labels,
);
@@ -203,6 +210,7 @@ export async function createScrollContactSheet(
export async function createSnapshotContactSheet(
snapshotsDir: string,
outputPath: string,
+ budget: Pick = {},
): Promise {
if (!existsSync(snapshotsDir)) return [];
@@ -222,7 +230,7 @@ export async function createSnapshotContactSheet(
return createContactSheetPages(
paths,
outputPath,
- { cols: 3, cellWidth: 600, pageSize: 9 },
+ { cols: 3, cellWidth: 600, pageSize: 9, ...budget },
0,
labels,
);
@@ -236,6 +244,7 @@ export async function createSnapshotContactSheet(
export async function createAssetContactSheet(
assetsDir: string,
outputPath: string,
+ budget: Pick = {},
): Promise {
if (!existsSync(assetsDir)) return [];
@@ -254,6 +263,7 @@ export async function createAssetContactSheet(
cellWidth: 480,
labelMode: "filename",
pageSize: 12,
+ ...budget,
});
}
@@ -271,6 +281,7 @@ export async function createSvgContactSheet(
svgsDir: string,
outputPath: string,
assetsRootDir?: string,
+ budget: Pick = {},
): Promise {
const dirsToScan = [svgsDir, assetsRootDir].filter(
(d): d is string => d !== undefined && existsSync(d),
@@ -302,6 +313,7 @@ export async function createSvgContactSheet(
const labels: string[] = [];
for (let i = 0; i < svgPaths.length; i++) {
+ if ((budget.remainingMs?.() ?? 1) <= 0) break;
const svgPath = svgPaths[i]!;
const tmpPath = join(tmpDir, `.thumb-${i}.png`);
try {
@@ -334,6 +346,7 @@ export async function createSvgContactSheet(
cols: 5,
cellWidth: thumbSize,
pageSize: 15,
+ ...budget,
},
0,
labels,
diff --git a/packages/cli/src/capture/contentExtractor.test.ts b/packages/cli/src/capture/contentExtractor.test.ts
index 9683653848..d392305048 100644
--- a/packages/cli/src/capture/contentExtractor.test.ts
+++ b/packages/cli/src/capture/contentExtractor.test.ts
@@ -2,19 +2,35 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import { captionImagesWithGemini } from "./contentExtractor.js";
+import {
+ captionImagesWithGemini,
+ resolveVisionPhaseCompletion,
+ type VisionCaptionOutcome,
+} from "./contentExtractor.js";
+
+const { generateContentMock } = vi.hoisted(() => ({
+ generateContentMock: vi.fn(),
+}));
+
+vi.mock("@google/genai", () => ({
+ GoogleGenAI: class {
+ models = { generateContent: generateContentMock };
+ },
+}));
// These tests exercise the OpenRouter provider path only — it makes a plain
// `fetch` call we can stub, with no native (`sharp`) or `@google/genai`
// dependency. OpenRouter wins over Gemini when OPENROUTER_API_KEY is set, so we
// don't need to clear the Gemini keys for the OpenRouter cases.
-function makeProjectWithImage(): string {
+function makeProjectWithImages(files = ["hero.png"]): string {
const dir = mkdtempSync(join(tmpdir(), "hf-caption-"));
mkdirSync(join(dir, "assets"), { recursive: true });
// Contents are irrelevant to the OpenRouter path (it just base64-encodes the
// bytes); only the .png extension matters for the image filter.
- writeFileSync(join(dir, "assets", "hero.png"), Buffer.from([0x89, 0x50, 0x4e, 0x47]));
+ for (const file of files) {
+ writeFileSync(join(dir, "assets", file), Buffer.from([0x89, 0x50, 0x4e, 0x47]));
+ }
return dir;
}
@@ -28,7 +44,7 @@ describe("captionImagesWithGemini — OpenRouter provider", () => {
});
it("captions via OpenRouter when OPENROUTER_API_KEY is set", async () => {
- const dir = makeProjectWithImage();
+ const dir = makeProjectWithImages();
dirs.push(dir);
vi.stubEnv("OPENROUTER_API_KEY", "or-test-key");
vi.stubEnv("HYPERFRAMES_OPENROUTER_MODEL", "google/gemini-3.1-flash-lite");
@@ -63,28 +79,238 @@ describe("captionImagesWithGemini — OpenRouter provider", () => {
});
it("degrades gracefully (no throw, no captions) when OpenRouter returns a non-OK status", async () => {
- const dir = makeProjectWithImage();
+ const dir = makeProjectWithImages();
dirs.push(dir);
vi.stubEnv("OPENROUTER_API_KEY", "or-bad-key");
vi.stubGlobal(
"fetch",
vi.fn(
- async () => new Response("invalid api key", { status: 401, statusText: "Unauthorized" }),
+ async () =>
+ new Response("provider detail containing or-bad-key", {
+ status: 401,
+ statusText: "Unauthorized",
+ }),
),
);
const warnings: string[] = [];
- // captionOne throws on !res.ok, but the throw is per-image inside
- // Promise.allSettled, so it's filtered out as a rejected result rather than
- // bubbling up — same silent degradation as the existing Gemini path.
- const captions = await captionImagesWithGemini(dir, () => {}, warnings);
+ let outcome: VisionCaptionOutcome | undefined;
+ const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
+ onOutcome: (value) => {
+ outcome = value;
+ },
+ });
+
+ expect(captions).toEqual({});
+ expect(warnings).toEqual(["OpenRouter vision failed for 1 asset(s); captions omitted."]);
+ expect(warnings.join(" ")).not.toContain("or-bad-key");
+ expect(outcome).toEqual({
+ timedOutRequests: 0,
+ failedRequests: 1,
+ budgetExhausted: false,
+ });
+ if (!outcome) throw new Error("Expected vision caption outcome");
+ expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
+ status: "degraded",
+ reason: "provider-error",
+ });
+ });
+
+ it("terminates captioning when the vision provider never responds", async () => {
+ const dir = makeProjectWithImages();
+ dirs.push(dir);
+ vi.stubEnv("OPENROUTER_API_KEY", "or-hanging-key");
+ vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20");
+
+ vi.stubGlobal(
+ "fetch",
+ vi.fn((_url: string, init?: RequestInit) => {
+ return new Promise((_resolve, reject) => {
+ init?.signal?.addEventListener("abort", () => reject(init.signal?.reason), {
+ once: true,
+ });
+ });
+ }),
+ );
+
+ const warnings: string[] = [];
+ const result = await Promise.race([
+ captionImagesWithGemini(dir, () => {}, warnings),
+ new Promise<"hung">((resolve) => setTimeout(() => resolve("hung"), 250)),
+ ]);
+
+ expect(result).toEqual({});
+ });
+
+ it.each([
+ { label: "success JSON", status: 200, partialBody: '{"choices":[' },
+ { label: "failure text", status: 500, partialBody: "upstream failed: " },
+ ])(
+ "terminates captioning when OpenRouter $label body never ends",
+ async ({ status, partialBody }) => {
+ const dir = makeProjectWithImages();
+ dirs.push(dir);
+ vi.stubEnv("OPENROUTER_API_KEY", "or-stalled-body-key");
+ vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20");
+
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () => {
+ const body = new ReadableStream({
+ start(controller) {
+ controller.enqueue(new TextEncoder().encode(partialBody));
+ },
+ });
+ return new Response(body, {
+ status,
+ headers: { "content-type": "application/json" },
+ });
+ }),
+ );
+
+ const warnings: string[] = [];
+ let outcome: VisionCaptionOutcome | undefined;
+ const result = await Promise.race([
+ captionImagesWithGemini(dir, () => {}, warnings, {
+ onOutcome: (value) => {
+ outcome = value;
+ },
+ }),
+ new Promise<"hung">((resolve) => setTimeout(() => resolve("hung"), 250)),
+ ]);
+
+ expect(result).toEqual({});
+ expect(warnings).toEqual(["OpenRouter vision timed out for 1 asset(s); captions omitted."]);
+ expect(outcome).toEqual({
+ timedOutRequests: 1,
+ failedRequests: 0,
+ budgetExhausted: false,
+ });
+ },
+ );
+
+ it("keeps successful captions while reporting a failed sibling request", async () => {
+ const dir = makeProjectWithImages(["failed.png", "success.png"]);
+ dirs.push(dir);
+ vi.stubEnv("OPENROUTER_API_KEY", "or-mixed-failure-key");
+
+ let requestCount = 0;
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () => {
+ requestCount++;
+ if (requestCount === 1) {
+ return new Response("upstream exploded with or-mixed-failure-key", {
+ status: 500,
+ statusText: "Internal Server Error",
+ });
+ }
+ return new Response(
+ JSON.stringify({ choices: [{ message: { content: "Successful image." } }] }),
+ { status: 200, headers: { "content-type": "application/json" } },
+ );
+ }),
+ );
+
+ const warnings: string[] = [];
+ let outcome: VisionCaptionOutcome | undefined;
+ const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
+ onOutcome: (value) => {
+ outcome = value;
+ },
+ });
+
+ expect(captions).toEqual({ "success.png": "Successful image." });
+ expect(warnings).toEqual(["OpenRouter vision failed for 1 asset(s); captions omitted."]);
+ expect(warnings.join(" ")).not.toContain("or-mixed-failure-key");
+ expect(outcome).toEqual({
+ timedOutRequests: 0,
+ failedRequests: 1,
+ budgetExhausted: false,
+ });
+ });
+
+ it("keeps a successful sibling caption when another request times out", async () => {
+ const dir = makeProjectWithImages(["hang.png", "success.png"]);
+ dirs.push(dir);
+ vi.stubEnv("OPENROUTER_API_KEY", "or-mixed-key");
+ vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20");
+
+ vi.stubGlobal(
+ "fetch",
+ vi.fn((_url: string, init?: RequestInit) => {
+ const body = JSON.parse(typeof init?.body === "string" ? init.body : "{}");
+ const imageUrl = body.messages?.[0]?.content?.[1]?.image_url?.url;
+ if (typeof imageUrl === "string" && imageUrl.includes("iVBORw==")) {
+ // Both fixtures contain the same bytes, so use call order: the first
+ // request hangs while the second succeeds independently.
+ const fetchMock = vi.mocked(fetch);
+ if (fetchMock.mock.calls.length === 1) return new Promise(() => {});
+ }
+ return Promise.resolve(
+ new Response(
+ JSON.stringify({ choices: [{ message: { content: "Successful image." } }] }),
+ {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ },
+ ),
+ );
+ }),
+ );
+
+ const warnings: string[] = [];
+ let outcome: VisionCaptionOutcome | undefined;
+ const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
+ onOutcome: (value) => {
+ outcome = value;
+ },
+ });
+
+ expect(captions).toEqual({ "success.png": "Successful image." });
+ expect(outcome).toEqual({
+ timedOutRequests: 1,
+ failedRequests: 0,
+ budgetExhausted: false,
+ });
+ if (!outcome) throw new Error("Expected vision caption outcome");
+ expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
+ status: "degraded",
+ reason: "request-timeout",
+ });
+ });
+
+ it("does not call a configured provider when vision is skipped", async () => {
+ const dir = makeProjectWithImages();
+ dirs.push(dir);
+ vi.stubEnv("OPENROUTER_API_KEY", "or-unused-key");
+ const fetchMock = vi.fn();
+ vi.stubGlobal("fetch", fetchMock);
+
+ const captions = await captionImagesWithGemini(dir, () => {}, [], { skipVision: true });
expect(captions).toEqual({});
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+
+ it("does not start a vision batch after the capture budget is exhausted", async () => {
+ const dir = makeProjectWithImages();
+ dirs.push(dir);
+ vi.stubEnv("OPENROUTER_API_KEY", "or-budget-key");
+ const fetchMock = vi.fn();
+ vi.stubGlobal("fetch", fetchMock);
+
+ const captions = await captionImagesWithGemini(dir, () => {}, [], {
+ remainingMs: () => 0,
+ });
+
+ expect(captions).toEqual({});
+ expect(fetchMock).not.toHaveBeenCalled();
});
it("skips captioning entirely when no provider key is present", async () => {
- const dir = makeProjectWithImage();
+ const dir = makeProjectWithImages();
dirs.push(dir);
vi.stubEnv("OPENROUTER_API_KEY", "");
vi.stubEnv("GEMINI_API_KEY", "");
@@ -100,3 +326,62 @@ describe("captionImagesWithGemini — OpenRouter provider", () => {
expect(fetchMock).not.toHaveBeenCalled();
});
});
+
+describe("captionImagesWithGemini — Gemini provider", () => {
+ const dirs: string[] = [];
+
+ afterEach(() => {
+ generateContentMock.mockReset();
+ vi.unstubAllEnvs();
+ for (const d of dirs) rmSync(d, { recursive: true, force: true });
+ dirs.length = 0;
+ });
+
+ it("terminates captioning when Gemini never settles even if the SDK ignores abort", async () => {
+ const dir = makeProjectWithImages();
+ dirs.push(dir);
+ vi.stubEnv("OPENROUTER_API_KEY", "");
+ vi.stubEnv("GEMINI_API_KEY", "gemini-hanging-key");
+ vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20");
+ generateContentMock.mockImplementation(() => new Promise(() => {}));
+
+ const result = await Promise.race([
+ captionImagesWithGemini(dir, () => {}, []),
+ new Promise<"hung">((resolve) => setTimeout(() => resolve("hung"), 250)),
+ ]);
+
+ expect(result).toEqual({});
+ });
+
+ it("reports a rejected Gemini request without leaking its error detail", async () => {
+ const dir = makeProjectWithImages();
+ dirs.push(dir);
+ vi.stubEnv("OPENROUTER_API_KEY", "");
+ vi.stubEnv("GEMINI_API_KEY", "gemini-secret-key");
+ generateContentMock.mockRejectedValue(
+ new Error("provider echoed gemini-secret-key in its response"),
+ );
+
+ const warnings: string[] = [];
+ let outcome: VisionCaptionOutcome | undefined;
+ const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
+ onOutcome: (value) => {
+ outcome = value;
+ },
+ });
+
+ expect(captions).toEqual({});
+ expect(warnings).toEqual(["Gemini vision failed for 1 asset(s); captions omitted."]);
+ expect(warnings.join(" ")).not.toContain("gemini-secret-key");
+ expect(outcome).toEqual({
+ timedOutRequests: 0,
+ failedRequests: 1,
+ budgetExhausted: false,
+ });
+ if (!outcome) throw new Error("Expected vision caption outcome");
+ expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
+ status: "degraded",
+ reason: "provider-error",
+ });
+ });
+});
diff --git a/packages/cli/src/capture/contentExtractor.ts b/packages/cli/src/capture/contentExtractor.ts
index 2caee86e48..7b9a909a61 100644
--- a/packages/cli/src/capture/contentExtractor.ts
+++ b/packages/cli/src/capture/contentExtractor.ts
@@ -15,6 +15,75 @@ import type sharpType from "sharp";
import type { CatalogedAsset } from "./assetCataloger.js";
import type { DesignTokens } from "./types.js";
+const DEFAULT_VISION_REQUEST_TIMEOUT_MS = 30_000;
+
+export interface VisionCaptionOutcome {
+ timedOutRequests: number;
+ failedRequests: number;
+ budgetExhausted: boolean;
+}
+
+interface VisionCaptionOptions {
+ skipVision?: boolean;
+ remainingMs?: () => number;
+ onOutcome?: (outcome: VisionCaptionOutcome) => void;
+}
+
+export function resolveVisionPhaseCompletion(
+ outcome: VisionCaptionOutcome,
+ remainingMs: number,
+):
+ | { status: "completed" }
+ | {
+ status: "degraded";
+ reason: "budget-exhausted" | "request-timeout" | "provider-error";
+ } {
+ if (outcome.budgetExhausted || remainingMs <= 0) {
+ return { status: "degraded", reason: "budget-exhausted" };
+ }
+ if (outcome.timedOutRequests > 0) {
+ return { status: "degraded", reason: "request-timeout" };
+ }
+ if (outcome.failedRequests > 0) {
+ return { status: "degraded", reason: "provider-error" };
+ }
+ return { status: "completed" };
+}
+
+class VisionRequestTimeoutError extends Error {
+ constructor(timeoutMs: number) {
+ super(`Vision request timed out after ${timeoutMs}ms`);
+ this.name = "VisionRequestTimeoutError";
+ }
+}
+
+function resolveVisionRequestTimeoutMs(): number {
+ const configured = Number(process.env.HYPERFRAMES_VISION_TIMEOUT_MS);
+ return Number.isFinite(configured) && configured > 0
+ ? configured
+ : DEFAULT_VISION_REQUEST_TIMEOUT_MS;
+}
+
+async function runBoundedVisionRequest(
+ request: (signal: AbortSignal) => Promise,
+ timeoutMs: number,
+): Promise {
+ const controller = new AbortController();
+ let timer: ReturnType | undefined;
+ const timeout = new Promise((_resolve, reject) => {
+ timer = setTimeout(() => {
+ reject(new VisionRequestTimeoutError(timeoutMs));
+ controller.abort();
+ }, timeoutMs);
+ });
+
+ try {
+ return await Promise.race([request(controller.signal), timeout]);
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+}
+
/**
* Detect JS libraries via window globals, DOM fingerprints, script URLs,
* and WebGL shader analysis.
@@ -165,15 +234,34 @@ export async function extractVisibleText(page: Page): Promise {
* Batches requests to stay under free-tier rate limits.
* Returns a map of filename -> caption string.
*/
+// fallow-ignore-next-line complexity
export async function captionImagesWithGemini(
outputDir: string,
progress: (stage: string, detail?: string) => void,
warnings: string[],
+ options: VisionCaptionOptions = {},
): Promise> {
const geminiCaptions: Record = {};
+ let timedOutCount = 0;
+ let failedRequestCount = 0;
+ let budgetExhausted = false;
+ const reportOutcome = (): void => {
+ options.onOutcome?.({
+ timedOutRequests: timedOutCount,
+ failedRequests: failedRequestCount,
+ budgetExhausted,
+ });
+ };
+ if (options.skipVision) {
+ reportOutcome();
+ return geminiCaptions;
+ }
const openRouterKey = process.env.OPENROUTER_API_KEY;
const geminiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
- if (!openRouterKey && !geminiKey) return geminiCaptions;
+ if (!openRouterKey && !geminiKey) {
+ reportOutcome();
+ return geminiCaptions;
+ }
// OpenRouter takes priority when both keys are set — it's the explicit opt-in
// for users without Google access. Both providers satisfy the same
@@ -186,6 +274,7 @@ export async function captionImagesWithGemini(
const model = useOpenRouter
? process.env.HYPERFRAMES_OPENROUTER_MODEL || "google/gemini-3.1-flash-lite"
: process.env.HYPERFRAMES_GEMINI_MODEL || "gemini-3.1-flash-lite-preview";
+ const requestTimeoutMs = resolveVisionRequestTimeoutMs();
progress("design", `Captioning images with ${providerName} vision...`);
try {
@@ -196,53 +285,71 @@ export async function captionImagesWithGemini(
base64: string;
prompt: string;
maxTokens: number;
+ timeoutMs: number;
}) => Promise;
let captionOne: CaptionOne;
if (openRouterKey) {
- captionOne = async ({ mimeType, base64, prompt, maxTokens }) => {
- const res = await fetch("https://openrouter.ai/api/v1/chat/completions", {
- method: "POST",
- headers: {
- Authorization: `Bearer ${openRouterKey}`,
- "Content-Type": "application/json",
- },
- body: JSON.stringify({
- model,
- messages: [
- {
- role: "user",
- content: [
- { type: "text", text: prompt },
- { type: "image_url", image_url: { url: `data:${mimeType};base64,${base64}` } },
- ],
- },
- ],
- max_tokens: maxTokens,
- }),
- });
- if (!res.ok) {
- const detail = await res.text().catch(() => "");
- throw new Error(`OpenRouter ${res.status} ${res.statusText}: ${detail.slice(0, 200)}`);
- }
- const data = (await res.json()) as {
- choices?: Array<{ message?: { content?: string } }>;
- };
- return data.choices?.[0]?.message?.content?.trim() || "";
+ captionOne = async ({ mimeType, base64, prompt, maxTokens, timeoutMs }) => {
+ return runBoundedVisionRequest(async (signal) => {
+ const res = await fetch("https://openrouter.ai/api/v1/chat/completions", {
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${openRouterKey}`,
+ "Content-Type": "application/json",
+ },
+ signal,
+ body: JSON.stringify({
+ model,
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: prompt },
+ {
+ type: "image_url",
+ image_url: { url: `data:${mimeType};base64,${base64}` },
+ },
+ ],
+ },
+ ],
+ max_tokens: maxTokens,
+ }),
+ });
+ if (!res.ok) {
+ await res.text();
+ throw new Error(`OpenRouter request failed with HTTP ${res.status}`);
+ }
+ const data = (await res.json()) as {
+ choices?: Array<{ message?: { content?: string } }>;
+ };
+ return data.choices?.[0]?.message?.content?.trim() || "";
+ }, timeoutMs);
};
} else {
// Unreachable when geminiKey is unset (guarded above); re-narrow for TS.
if (!geminiKey) return geminiCaptions;
const { GoogleGenAI } = await import("@google/genai");
const ai = new GoogleGenAI({ apiKey: geminiKey });
- captionOne = async ({ mimeType, base64, prompt, maxTokens }) => {
- const response = await ai.models.generateContent({
- model,
- contents: [
- { role: "user", parts: [{ inlineData: { mimeType, data: base64 } }, { text: prompt }] },
- ],
- config: { maxOutputTokens: maxTokens },
- });
+ captionOne = async ({ mimeType, base64, prompt, maxTokens, timeoutMs }) => {
+ const response = await runBoundedVisionRequest(
+ (signal) =>
+ ai.models.generateContent({
+ model,
+ contents: [
+ {
+ role: "user",
+ parts: [{ inlineData: { mimeType, data: base64 } }, { text: prompt }],
+ },
+ ],
+ config: {
+ maxOutputTokens: maxTokens,
+ abortSignal: signal,
+ httpOptions: { timeout: timeoutMs },
+ },
+ }),
+ timeoutMs,
+ );
return response.text?.trim() || "";
};
}
@@ -253,10 +360,35 @@ export async function captionImagesWithGemini(
// Caption in parallel batches. Gemini free tier is ~5 RPM (slow but $0),
// paid/OpenRouter ~2000 RPM. We batch 20 with a 2s inter-batch pause and rely
- // on Promise.allSettled so a rate-limited image degrades to "" rather than
- // failing the batch.
+ // on Promise.allSettled so one rate-limited image does not fail its siblings;
+ // rejected requests are aggregated into a sanitized warning below.
const BATCH_SIZE = 20;
+ const collectCaptionResults = (
+ results: PromiseSettledResult<{ file: string; caption: string }>[],
+ ): { timeouts: number; failures: number } => {
+ let timeouts = 0;
+ let failures = 0;
+ for (const result of results) {
+ if (result.status === "fulfilled" && result.value.caption) {
+ geminiCaptions[result.value.file] = result.value.caption;
+ } else if (
+ result.status === "rejected" &&
+ result.reason instanceof VisionRequestTimeoutError
+ ) {
+ timeouts++;
+ } else if (result.status === "rejected") {
+ failures++;
+ }
+ }
+ return { timeouts, failures };
+ };
for (let i = 0; i < imageFiles.length; i += BATCH_SIZE) {
+ const remainingMs = options.remainingMs?.() ?? Number.POSITIVE_INFINITY;
+ if (remainingMs <= 0) {
+ budgetExhausted = true;
+ break;
+ }
+ const timeoutMs = Math.max(1, Math.min(requestTimeoutMs, remainingMs));
const batch = imageFiles.slice(i, i + BATCH_SIZE);
const results = await Promise.allSettled(
batch.map(async (file: string) => {
@@ -273,15 +405,14 @@ export async function captionImagesWithGemini(
prompt:
"Describe this website image in ONE short sentence for a video storyboard. Focus on: what it shows, dominant colors, whether background is light or dark. Be factual, not creative.",
maxTokens: 500,
+ timeoutMs,
});
return { file, caption };
}),
);
- for (const result of results) {
- if (result.status === "fulfilled" && result.value.caption) {
- geminiCaptions[result.value.file] = result.value.caption;
- }
- }
+ const counts = collectCaptionResults(results);
+ timedOutCount += counts.timeouts;
+ failedRequestCount += counts.failures;
// Pace requests between batches (paid tier: 2000+ RPM, free tier: rate-limited)
if (i + BATCH_SIZE < imageFiles.length) {
await new Promise((r) => setTimeout(r, 2000)); // 2s pause between batches — paid tier handles 2000 RPM, free tier retries via Promise.allSettled
@@ -322,6 +453,7 @@ export async function captionImagesWithGemini(
`Skipped ${svgFiles.length} SVG caption(s): sharp could not load (${(err as Error).message}). ` +
`Reinstall with optional dependencies enabled (e.g. \`npm i sharp\`) to caption SVG assets.`,
);
+ reportOutcome();
return geminiCaptions;
}
progress("design", `Rasterizing + captioning ${svgFiles.length} SVGs via vision API...`);
@@ -329,6 +461,12 @@ export async function captionImagesWithGemini(
const SVG_RENDER_SIZE = 256; // px — enough resolution for Gemini to read wordmarks, small enough to keep payload sub-MB
let svgsSkipped = 0;
for (let i = 0; i < svgFiles.length; i += SVG_BATCH) {
+ const remainingMs = options.remainingMs?.() ?? Number.POSITIVE_INFINITY;
+ if (remainingMs <= 0) {
+ budgetExhausted = true;
+ break;
+ }
+ const timeoutMs = Math.max(1, Math.min(requestTimeoutMs, remainingMs));
const batch = svgFiles.slice(i, i + SVG_BATCH);
const results = await Promise.allSettled(
batch.map(async ({ relPath }) => {
@@ -373,15 +511,14 @@ export async function captionImagesWithGemini(
"If you see a wordmark, READ THE LETTERS LITERALLY — do not guess a brand from context. " +
"Be factual.",
maxTokens: 300,
+ timeoutMs,
});
return { file: relPath, caption };
}),
);
- for (const result of results) {
- if (result.status === "fulfilled" && result.value.caption) {
- geminiCaptions[result.value.file] = result.value.caption;
- }
- }
+ const counts = collectCaptionResults(results);
+ timedOutCount += counts.timeouts;
+ failedRequestCount += counts.failures;
if (i + SVG_BATCH < svgFiles.length) {
await new Promise((r) => setTimeout(r, 2000));
}
@@ -398,10 +535,22 @@ export async function captionImagesWithGemini(
);
}
}
- } catch (err) {
- warnings.push(`${providerName} captioning failed: ${err}`);
+ if (timedOutCount > 0) {
+ warnings.push(
+ `${providerName} vision timed out for ${timedOutCount} asset(s); captions omitted.`,
+ );
+ }
+ if (failedRequestCount > 0) {
+ warnings.push(
+ `${providerName} vision failed for ${failedRequestCount} asset(s); captions omitted.`,
+ );
+ }
+ } catch {
+ failedRequestCount = Math.max(1, failedRequestCount);
+ warnings.push(`${providerName} captioning failed; captions omitted.`);
}
+ reportOutcome();
return geminiCaptions;
}
diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts
index fe58a972f4..15d8e3d4ae 100644
--- a/packages/cli/src/capture/index.ts
+++ b/packages/cli/src/capture/index.ts
@@ -37,12 +37,17 @@ import {
extractVisibleText,
captionImagesWithGemini,
generateAssetDescriptions,
+ resolveVisionPhaseCompletion,
} from "./contentExtractor.js";
+import type { VisionCaptionOutcome } from "./contentExtractor.js";
import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js";
-import type { CaptureOptions, CaptureResult } from "./types.js";
+import { detectBlockedPage } from "./pageBlockDetection.js";
+import type { CaptureOptions, CapturePhase, CapturePhaseProgress, CaptureResult } from "./types.js";
export type { CaptureOptions, CaptureResult } from "./types.js";
+const DEFAULT_POST_NAVIGATION_BUDGET_MS = 120_000;
+
// fallow-ignore-next-line complexity
export async function captureWebsite(
opts: CaptureOptions,
@@ -57,12 +62,49 @@ export async function captureWebsite(
settleTime = 3000,
maxScreenshots: _maxScreenshots = 24,
skipAssets = false,
+ skipVision = false,
+ postNavigationBudgetMs = DEFAULT_POST_NAVIGATION_BUDGET_MS,
+ onPhase,
} = opts;
const warnings: string[] = [];
const progress = (stage: string, detail?: string) => {
onProgress?.(stage, detail);
};
+ const budgetMs =
+ Number.isFinite(postNavigationBudgetMs) && postNavigationBudgetMs > 0
+ ? postNavigationBudgetMs
+ : DEFAULT_POST_NAVIGATION_BUDGET_MS;
+ let postNavigationDeadline: number | undefined;
+ const remainingMs = (): number =>
+ postNavigationDeadline === undefined
+ ? budgetMs
+ : Math.max(0, postNavigationDeadline - Date.now());
+ let lastPhase: CapturePhaseProgress = {
+ schema: "hyperframes.capture.phase.v1",
+ phase: "browser",
+ status: "started",
+ remainingMs: null,
+ };
+ const phase = (
+ name: CapturePhase,
+ status: CapturePhaseProgress["status"],
+ reason?: CapturePhaseProgress["reason"],
+ ): void => {
+ const remaining = postNavigationDeadline === undefined ? null : remainingMs();
+ lastPhase = reason
+ ? {
+ schema: "hyperframes.capture.phase.v1",
+ phase: name,
+ status,
+ remainingMs: remaining,
+ reason,
+ }
+ : { schema: "hyperframes.capture.phase.v1", phase: name, status, remainingMs: remaining };
+ onPhase?.(lastPhase);
+ };
+
+ phase("browser", "started");
// Load .env file from repo root if it exists (for GEMINI_API_KEY, etc.)
loadEnvFile(outputDir);
@@ -102,6 +144,8 @@ export async function captureWebsite(
// Goal: Catalog animations + take screenshots (with JS rendering)
// ═══════════════════════════════════════════════════════════════
+ phase("browser", "completed");
+ phase("navigation", "started");
progress("animations", "Cataloging animations (full JS)...");
const page1 = await chromeBrowser.newPage();
@@ -197,12 +241,13 @@ export async function captureWebsite(
// Use networkidle2 (allows 2 ongoing connections) instead of networkidle0 —
// modern SPAs often have persistent WebSocket/analytics connections that
// prevent networkidle0 from ever resolving.
- await page1.goto(url, { waitUntil: "networkidle2", timeout });
+ const navigationResponse = await page1.goto(url, { waitUntil: "networkidle2", timeout });
+ postNavigationDeadline = Date.now() + budgetMs;
await new Promise((r) => setTimeout(r, settleTime));
// Check if the page loaded real content or an anti-bot challenge
- // Use structural detection (DOM elements + cookies), not text regex matching —
- // text matching causes false positives on sites that mention "blocked" or "verify" in copy
+ // Combine structural evidence with the main response status/title. Low text
+ // alone stays non-fatal so image-led sites are not rejected.
const pageContentCheck = (await page1.evaluate(`(() => {
var text = (document.body.innerText || "").trim();
var title = document.title || "";
@@ -210,19 +255,28 @@ export async function captureWebsite(
var hasCfTurnstile = !!document.querySelector('.cf-turnstile, [data-sitekey], iframe[src*="challenges.cloudflare.com"], #challenge-running, #challenge-form');
// Structural: page is almost empty (challenge pages have minimal DOM)
var bodyChildCount = document.body.children.length;
- var isMinimalDom = bodyChildCount <= 5 && text.length < 500;
- // Title-based: only check title on near-empty pages
- var hasChallengeTitle = isMinimalDom && /just a moment|attention required|access denied/i.test(title);
- var isChallenged = hasCfTurnstile || hasChallengeTitle;
- return { textLength: text.length, title: title, isChallenged: isChallenged, bodyChildCount: bodyChildCount };
- })()`)) as { textLength: number; title: string; isChallenged: boolean; bodyChildCount: number };
-
- if (pageContentCheck.isChallenged || pageContentCheck.textLength < 100) {
- const reason = pageContentCheck.isChallenged
- ? "Anti-bot protection detected (Cloudflare challenge or similar)"
- : "Page has very little text content (" +
- pageContentCheck.textLength +
- " chars) — may be blocked or a client-rendered SPA that needs more time";
+ return { textLength: text.length, title: title, hasChallengeElement: hasCfTurnstile, bodyChildCount: bodyChildCount };
+ })()`)) as {
+ textLength: number;
+ title: string;
+ hasChallengeElement: boolean;
+ bodyChildCount: number;
+ };
+
+ const blockedReason = detectBlockedPage({
+ httpStatus: navigationResponse?.status() ?? null,
+ ...pageContentCheck,
+ });
+ if (blockedReason) throw new Error(blockedReason);
+
+ phase("navigation", "completed");
+ phase("core-extraction", "started");
+
+ if (pageContentCheck.textLength < 100) {
+ const reason =
+ "Page has very little text content (" +
+ pageContentCheck.textLength +
+ " chars) — may be blocked or a client-rendered SPA that needs more time";
warnings.push(reason);
progress("warn", reason);
}
@@ -231,29 +285,36 @@ export async function captureWebsite(
// Framer and other modern sites use IntersectionObserver — images only load
// when scrolled into view. We scroll the full page, then wait for all images
// to finish loading before proceeding.
- await page1.evaluate(`(async () => {
+ const lazyLoadBudgetMs = Math.min(15_000, remainingMs());
+ if (lazyLoadBudgetMs > 0) {
+ await page1.evaluate(`(async () => {
+ var lazyLoadDeadline = Date.now() + ${lazyLoadBudgetMs};
var h = document.body.scrollHeight;
for (var y = 0; y < h; y += window.innerHeight * 0.7) {
+ if (Date.now() >= lazyLoadDeadline) break;
window.scrollTo(0, y);
await new Promise(function(r) { setTimeout(r, 400); });
}
// Scroll to very bottom to catch footer lazy-loads
- window.scrollTo(0, document.body.scrollHeight);
- await new Promise(function(r) { setTimeout(r, 800); });
+ if (Date.now() < lazyLoadDeadline) {
+ window.scrollTo(0, document.body.scrollHeight);
+ await new Promise(function(r) { setTimeout(r, Math.min(800, Math.max(0, lazyLoadDeadline - Date.now()))); });
+ }
// Wait for all images to finish loading
var imgs = Array.from(document.querySelectorAll('img'));
var pending = imgs.filter(function(img) { return !img.complete; });
- if (pending.length > 0) {
+ var imageWaitMs = Math.min(5000, Math.max(0, lazyLoadDeadline - Date.now()));
+ if (pending.length > 0 && imageWaitMs > 0) {
await Promise.race([
Promise.all(pending.map(function(img) {
return new Promise(function(r) { img.onload = r; img.onerror = r; });
})),
- new Promise(function(r) { setTimeout(r, 5000); })
+ new Promise(function(r) { setTimeout(r, imageWaitMs); })
]);
}
window.scrollTo(0, 0);
- await new Promise(function(r) { setTimeout(r, 500); });
})()`);
+ }
await page1.evaluate(`window.scrollTo(0, 0)`);
await new Promise((r) => setTimeout(r, 300));
@@ -289,13 +350,14 @@ export async function captureWebsite(
/* DOM scan failed — non-critical */
}
- if (discoveredLotties.length > 0) {
+ if (discoveredLotties.length > 0 && remainingMs() > 0) {
const lottieDir = join(outputDir, "assets", "lottie");
mkdirSync(lottieDir, { recursive: true });
- const savedCount = await saveLottieAnimations(discoveredLotties, lottieDir);
+ const lottieBudget = { remainingMs };
+ const savedCount = await saveLottieAnimations(discoveredLotties, lottieDir, lottieBudget);
// Generate manifest + preview thumbnails so the agent can SEE what each animation is
- if (savedCount > 0) {
- await renderLottiePreviews(chromeBrowser, lottieDir, outputDir);
+ if (savedCount > 0 && remainingMs() > 0) {
+ await renderLottiePreviews(chromeBrowser, lottieDir, outputDir, lottieBudget);
progress("lottie", `${savedCount} Lottie animation(s) saved`);
}
}
@@ -368,7 +430,7 @@ export async function captureWebsite(
// Capture scroll-position viewport screenshots
progress("screenshots", "Capturing scroll screenshots...");
const { captureScrollScreenshots } = await import("./screenshotCapture.js");
- const screenshots = await captureScrollScreenshots(page1, outputDir);
+ const screenshots = await captureScrollScreenshots(page1, outputDir, { remainingMs });
progress("screenshots", `${screenshots.length} scroll screenshots captured`);
// Catalog all assets (must run before extractHtml which converts img src to data URLs)
@@ -437,10 +499,15 @@ export async function captureWebsite(
// Generate video manifest — screenshot each