Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions packages/cli/src/capture/assetDownloader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -264,8 +264,10 @@ export async function downloadAndRewriteFonts(

if (fontUrls.size === 0) return css;

// Limit font downloads to avoid bloat. Google Fonts serves 20+ unicode-range
// subsets per weight — we only need a few per family for video production.
// Limit font download attempts to bound worst-case egress and latency. Google Fonts serves
// 20+ unicode-range subsets per weight, so successes alone cannot be the bound: six transient
// failures can intentionally suppress later URLs in that family. Latin-priority sorting below
// makes the limited attempts useful while keeping this failure tradeoff explicit.
const MAX_FONTS_PER_FAMILY = 6;
const MAX_TOTAL_FONTS = 30;
const familyCounts = new Map<string, number>();
Expand Down
19 changes: 17 additions & 2 deletions packages/cli/src/capture/contentExtractor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { captionImagesWithGemini } from "./contentExtractor.js";
import {
captionImagesWithGemini,
resolveVisionPhaseCompletion,
type VisionCaptionOutcome,
} from "./contentExtractor.js";

const { generateContentMock } = vi.hoisted(() => ({
generateContentMock: vi.fn(),
Expand Down Expand Up @@ -151,9 +155,20 @@ describe("captionImagesWithGemini — OpenRouter provider", () => {
);

const warnings: string[] = [];
const captions = await captionImagesWithGemini(dir, () => {}, warnings);
let outcome: VisionCaptionOutcome | undefined;
const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
onOutcome: (value) => {
outcome = value;
},
});

expect(captions).toEqual({ "success.png": "Successful image." });
expect(outcome).toEqual({ timedOutRequests: 1, budgetExhausted: false });
if (!outcome) throw new Error("Expected vision caption outcome");
expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
status: "degraded",
reason: "request-timeout",
});
});

it("does not call a configured provider when vision is skipped", async () => {
Expand Down
49 changes: 44 additions & 5 deletions packages/cli/src/capture/contentExtractor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,30 @@ import type { DesignTokens } from "./types.js";

const DEFAULT_VISION_REQUEST_TIMEOUT_MS = 30_000;

export interface VisionCaptionOutcome {
timedOutRequests: number;
budgetExhausted: boolean;
}

interface VisionCaptionOptions {
skipVision?: boolean;
remainingMs?: () => number;
onOutcome?: (outcome: VisionCaptionOutcome) => void;
}

export function resolveVisionPhaseCompletion(
outcome: VisionCaptionOutcome,
remainingMs: number,
):
| { status: "completed" }
| { status: "degraded"; reason: "budget-exhausted" | "request-timeout" } {
if (outcome.budgetExhausted || remainingMs <= 0) {
return { status: "degraded", reason: "budget-exhausted" };
}
if (outcome.timedOutRequests > 0) {
return { status: "degraded", reason: "request-timeout" };
}
return { status: "completed" };
}

class VisionRequestTimeoutError extends Error {
Expand Down Expand Up @@ -214,10 +235,21 @@ export async function captionImagesWithGemini(
options: VisionCaptionOptions = {},
): Promise<Record<string, string>> {
const geminiCaptions: Record<string, string> = {};
if (options.skipVision) return geminiCaptions;
let timedOutCount = 0;
let budgetExhausted = false;
const reportOutcome = (): void => {
options.onOutcome?.({ timedOutRequests: timedOutCount, budgetExhausted });
};
if (options.skipVision) {
reportOutcome();
return geminiCaptions;
}
const openRouterKey = process.env.OPENROUTER_API_KEY;
const geminiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
if (!openRouterKey && !geminiKey) return geminiCaptions;
if (!openRouterKey && !geminiKey) {
reportOutcome();
return geminiCaptions;
}

// OpenRouter takes priority when both keys are set — it's the explicit opt-in
// for users without Google access. Both providers satisfy the same
Expand Down Expand Up @@ -321,7 +353,6 @@ export async function captionImagesWithGemini(
// on Promise.allSettled so a rate-limited image degrades to "" rather than
// failing the batch.
const BATCH_SIZE = 20;
let timedOutCount = 0;
const collectCaptionResults = (
results: PromiseSettledResult<{ file: string; caption: string }>[],
): number => {
Expand All @@ -340,7 +371,10 @@ export async function captionImagesWithGemini(
};
for (let i = 0; i < imageFiles.length; i += BATCH_SIZE) {
const remainingMs = options.remainingMs?.() ?? Number.POSITIVE_INFINITY;
if (remainingMs <= 0) break;
if (remainingMs <= 0) {
budgetExhausted = true;
break;
}
const timeoutMs = Math.max(1, Math.min(requestTimeoutMs, remainingMs));
const batch = imageFiles.slice(i, i + BATCH_SIZE);
const results = await Promise.allSettled(
Expand Down Expand Up @@ -404,6 +438,7 @@ export async function captionImagesWithGemini(
`Skipped ${svgFiles.length} SVG caption(s): sharp could not load (${(err as Error).message}). ` +
`Reinstall with optional dependencies enabled (e.g. \`npm i sharp\`) to caption SVG assets.`,
);
reportOutcome();
return geminiCaptions;
}
progress("design", `Rasterizing + captioning ${svgFiles.length} SVGs via vision API...`);
Expand All @@ -412,7 +447,10 @@ export async function captionImagesWithGemini(
let svgsSkipped = 0;
for (let i = 0; i < svgFiles.length; i += SVG_BATCH) {
const remainingMs = options.remainingMs?.() ?? Number.POSITIVE_INFINITY;
if (remainingMs <= 0) break;
if (remainingMs <= 0) {
budgetExhausted = true;
break;
}
const timeoutMs = Math.max(1, Math.min(requestTimeoutMs, remainingMs));
const batch = svgFiles.slice(i, i + SVG_BATCH);
const results = await Promise.allSettled(
Expand Down Expand Up @@ -489,6 +527,7 @@ export async function captionImagesWithGemini(
warnings.push(`${providerName} captioning failed: ${err}`);
}

reportOutcome();
return geminiCaptions;
}

Expand Down
20 changes: 16 additions & 4 deletions packages/cli/src/capture/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ import {
extractVisibleText,
captionImagesWithGemini,
generateAssetDescriptions,
resolveVisionPhaseCompletion,
} from "./contentExtractor.js";
import type { VisionCaptionOutcome } from "./contentExtractor.js";
import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js";
import type { CaptureOptions, CapturePhase, CapturePhaseProgress, CaptureResult } from "./types.js";

Expand Down Expand Up @@ -343,10 +345,11 @@ export async function captureWebsite(
if (discoveredLotties.length > 0 && remainingMs() > 0) {
const lottieDir = join(outputDir, "assets", "lottie");
mkdirSync(lottieDir, { recursive: true });
const savedCount = await saveLottieAnimations(discoveredLotties, lottieDir);
const lottieBudget = { remainingMs };
const savedCount = await saveLottieAnimations(discoveredLotties, lottieDir, lottieBudget);
// Generate manifest + preview thumbnails so the agent can SEE what each animation is
if (savedCount > 0 && remainingMs() > 0) {
await renderLottiePreviews(chromeBrowser, lottieDir, outputDir);
await renderLottiePreviews(chromeBrowser, lottieDir, outputDir, lottieBudget);
progress("lottie", `${savedCount} Lottie animation(s) saved`);
}
}
Expand Down Expand Up @@ -494,6 +497,7 @@ export async function captureWebsite(
networkVideoUrls: discoveredVideoUrls, // Layer 1 (live Set, read after sampling)
sampleMs: Math.min(12000, videoBudgetMs), // Layer 2: poll DOM within the shared budget
downloadBudgetMs: videoBudgetMs,
remainingMs,
});
}
} catch {
Expand Down Expand Up @@ -670,13 +674,21 @@ export async function captureWebsite(
phase("vision", "degraded", "budget-exhausted");
} else {
phase("vision", "started");
let visionOutcome: VisionCaptionOutcome = {
timedOutRequests: 0,
budgetExhausted: false,
};
geminiCaptions = await captionImagesWithGemini(outputDir, progress, warnings, {
remainingMs,
onOutcome: (outcome) => {
visionOutcome = outcome;
},
});
const completion = resolveVisionPhaseCompletion(visionOutcome, remainingMs());
phase(
"vision",
remainingMs() > 0 ? "completed" : "degraded",
remainingMs() > 0 ? undefined : "budget-exhausted",
completion.status,
completion.status === "degraded" ? completion.reason : undefined,
);
}

Expand Down
12 changes: 0 additions & 12 deletions packages/cli/src/capture/indexBudget.test.ts

This file was deleted.

133 changes: 123 additions & 10 deletions packages/cli/src/capture/mediaCapture.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,127 @@
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
import { remainingVideoDownloadTimeoutMs } from "./mediaCapture.js";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Browser, Page } from "puppeteer-core";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
captureVideoManifest,
remainingVideoDownloadTimeoutMs,
renderLottiePreviews,
saveLottieAnimations,
} from "./mediaCapture.js";

const source = readFileSync(new URL("./mediaCapture.ts", import.meta.url), "utf-8");
const tempDirs: string[] = [];

function tempDir(): string {
const dir = mkdtempSync(join(tmpdir(), "hf-media-budget-"));
tempDirs.push(dir);
return dir;
}

afterEach(() => {
vi.useRealTimers();
vi.unstubAllGlobals();
for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true });
tempDirs.length = 0;
});

describe("Lottie capture budget", () => {
it("does not start another Lottie fetch after the live budget expires", async () => {
const dir = tempDir();
let remainingMs = 10_000;
const fetchMock = vi.fn(async () => {
remainingMs = 0;
return new Response(JSON.stringify({ w: 100, h: 100, layers: [] }), { status: 200 });
});
vi.stubGlobal("fetch", fetchMock);

const saved = await saveLottieAnimations(
[{ url: "https://one.example/a.json" }, { url: "https://two.example/b.json" }],
dir,
{ remainingMs: () => remainingMs },
);

expect(saved).toBe(1);
expect(fetchMock).toHaveBeenCalledTimes(1);
});

it("does not open another Lottie preview page after the live budget expires", async () => {
const dir = tempDir();
const lottieDir = join(dir, "assets", "lottie");
mkdirSync(join(dir, "extracted"), { recursive: true });
mkdirSync(lottieDir, { recursive: true });
const lottie = JSON.stringify({ w: 100, h: 100, fr: 30, ip: 0, op: 30, layers: [] });
writeFileSync(join(lottieDir, "a.json"), lottie);
writeFileSync(join(lottieDir, "b.json"), lottie);

let remainingMs = 10_000;
const previewPage = {
setViewport: vi.fn(async () => undefined),
setContent: vi.fn(async () => undefined),
evaluate: vi.fn(async () => undefined),
waitForFunction: vi.fn(async () => undefined),
screenshot: vi.fn(async () => undefined),
close: vi.fn(async () => undefined),
};
const newPage = vi.fn(async () => {
remainingMs = 0;
return previewPage;
});
const browser = { newPage } as unknown as Browser;

await renderLottiePreviews(browser, lottieDir, dir, { remainingMs: () => remainingMs });

expect(newPage).toHaveBeenCalledTimes(1);
expect(previewPage.setViewport).not.toHaveBeenCalled();
expect(previewPage.screenshot).not.toHaveBeenCalled();
});
});

describe("video capture live budget", () => {
it("does not preview or download when the budget expires during DOM sampling", async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
const dir = tempDir();
mkdirSync(join(dir, "extracted"), { recursive: true });
const descriptor = {
src: "https://video.example/hero.mp4",
filename: "hero.mp4",
width: 640,
height: 360,
sourceWidth: 640,
sourceHeight: 360,
top: 0,
left: 0,
heading: "Hero",
caption: "Demo",
ariaLabel: "",
};
const screenshot = vi.fn(async () => Buffer.from("preview"));
const evaluate = vi.fn(async (expression: unknown) =>
typeof expression === "function" ? { x: 0, y: 0, width: 640, height: 360 } : [descriptor],
);
const page = { evaluate, screenshot } as unknown as Page;
const fetchMock = vi.fn(
async () =>
new Response(Buffer.alloc(2048), {
status: 200,
headers: { "content-type": "video/mp4" },
}),
);
vi.stubGlobal("fetch", fetchMock);

const capture = captureVideoManifest(page, dir, () => {}, {
sampleMs: 10_000,
downloadBudgetMs: 10_000,
remainingMs: () => Math.max(0, 1_000 - Date.now()),
});
await vi.runAllTimersAsync();
await capture;

expect(screenshot).not.toHaveBeenCalled();
expect(fetchMock).not.toHaveBeenCalled();
});
});

describe("remainingVideoDownloadTimeoutMs", () => {
it("caps a video request to the remaining capture budget", () => {
Expand All @@ -16,10 +135,4 @@ describe("remainingVideoDownloadTimeoutMs", () => {
it("retains the existing per-request ceiling when more budget remains", () => {
expect(remainingVideoDownloadTimeoutMs(1_000, 300_000, 2_000)).toBe(120_000);
});

it("checks the aggregate budget before starting each preview/download iteration", () => {
expect(source).toContain(
"if (remainingVideoDownloadTimeoutMs(dlStart, downloadBudgetMs) <= 0) break;",
);
});
});
Loading
Loading