From 22e3cca966e4f3c5dfe0f8b716dec80259206300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 31 Jul 2026 18:46:24 +0000 Subject: [PATCH 1/5] fix(cli): reject blocked website captures --- packages/cli/src/capture/index.ts | 44 ++++++----- .../src/capture/pageBlockDetection.test.ts | 76 +++++++++++++++++++ .../cli/src/capture/pageBlockDetection.ts | 34 +++++++++ 3 files changed, 136 insertions(+), 18 deletions(-) create mode 100644 packages/cli/src/capture/pageBlockDetection.test.ts create mode 100644 packages/cli/src/capture/pageBlockDetection.ts diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts index b9955f1214..fa5627de83 100644 --- a/packages/cli/src/capture/index.ts +++ b/packages/cli/src/capture/index.ts @@ -41,6 +41,7 @@ import { } from "./contentExtractor.js"; import type { VisionCaptionOutcome } from "./contentExtractor.js"; import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js"; +import { detectBlockedPage } from "./pageBlockDetection.js"; import type { CaptureOptions, CapturePhase, CapturePhaseProgress, CaptureResult } from "./types.js"; export type { CaptureOptions, CaptureResult } from "./types.js"; @@ -240,15 +241,13 @@ export async function captureWebsite( // Use networkidle2 (allows 2 ongoing connections) instead of networkidle0 — // modern SPAs often have persistent WebSocket/analytics connections that // prevent networkidle0 from ever resolving. - await page1.goto(url, { waitUntil: "networkidle2", timeout }); + const navigationResponse = await page1.goto(url, { waitUntil: "networkidle2", timeout }); postNavigationDeadline = Date.now() + budgetMs; await new Promise((r) => setTimeout(r, settleTime)); - phase("navigation", "completed"); - phase("core-extraction", "started"); // Check if the page loaded real content or an anti-bot challenge - // Use structural detection (DOM elements + cookies), not text regex matching — - // text matching causes false positives on sites that mention "blocked" or "verify" in copy + // Combine structural evidence with the main response status/title. Low text + // alone stays non-fatal so image-led sites are not rejected. const pageContentCheck = (await page1.evaluate(`(() => { var text = (document.body.innerText || "").trim(); var title = document.title || ""; @@ -256,19 +255,28 @@ export async function captureWebsite( var hasCfTurnstile = !!document.querySelector('.cf-turnstile, [data-sitekey], iframe[src*="challenges.cloudflare.com"], #challenge-running, #challenge-form'); // Structural: page is almost empty (challenge pages have minimal DOM) var bodyChildCount = document.body.children.length; - var isMinimalDom = bodyChildCount <= 5 && text.length < 500; - // Title-based: only check title on near-empty pages - var hasChallengeTitle = isMinimalDom && /just a moment|attention required|access denied/i.test(title); - var isChallenged = hasCfTurnstile || hasChallengeTitle; - return { textLength: text.length, title: title, isChallenged: isChallenged, bodyChildCount: bodyChildCount }; - })()`)) as { textLength: number; title: string; isChallenged: boolean; bodyChildCount: number }; - - if (pageContentCheck.isChallenged || pageContentCheck.textLength < 100) { - const reason = pageContentCheck.isChallenged - ? "Anti-bot protection detected (Cloudflare challenge or similar)" - : "Page has very little text content (" + - pageContentCheck.textLength + - " chars) — may be blocked or a client-rendered SPA that needs more time"; + return { textLength: text.length, title: title, hasChallengeElement: hasCfTurnstile, bodyChildCount: bodyChildCount }; + })()`)) as { + textLength: number; + title: string; + hasChallengeElement: boolean; + bodyChildCount: number; + }; + + const blockedReason = detectBlockedPage({ + httpStatus: navigationResponse?.status() ?? null, + ...pageContentCheck, + }); + if (blockedReason) throw new Error(blockedReason); + + phase("navigation", "completed"); + phase("core-extraction", "started"); + + if (pageContentCheck.textLength < 100) { + const reason = + "Page has very little text content (" + + pageContentCheck.textLength + + " chars) — may be blocked or a client-rendered SPA that needs more time"; warnings.push(reason); progress("warn", reason); } diff --git a/packages/cli/src/capture/pageBlockDetection.test.ts b/packages/cli/src/capture/pageBlockDetection.test.ts new file mode 100644 index 0000000000..b180e55fb1 --- /dev/null +++ b/packages/cli/src/capture/pageBlockDetection.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { detectBlockedPage } from "./pageBlockDetection.js"; + +describe("detectBlockedPage", () => { + it.each([ + { + label: "local Fuji response", + evidence: { + httpStatus: 403, + title: "403 - Forbidden", + textLength: 50, + bodyChildCount: 1, + hasChallengeElement: false, + }, + }, + { + label: "EC2-style soft-block response", + evidence: { + httpStatus: 200, + title: "Access denied", + textLength: 18, + bodyChildCount: 2, + hasChallengeElement: false, + }, + }, + ])("rejects a minimal protection page: $label", ({ evidence }) => { + expect(detectBlockedPage(evidence)).toMatch(/capture blocked/i); + }); + + it("rejects a structural challenge on a minimal page", () => { + expect( + detectBlockedPage({ + httpStatus: 200, + title: "Please verify you are human", + textLength: 300, + bodyChildCount: 3, + hasChallengeElement: true, + }), + ).toMatch(/capture blocked/i); + }); + + it.each([ + { + label: "an image-led page with little text", + evidence: { + httpStatus: 200, + title: "Photography portfolio", + textLength: 18, + bodyChildCount: 4, + hasChallengeElement: false, + }, + }, + { + label: "a real page discussing access denial", + evidence: { + httpStatus: 200, + title: "Why websites say Access Denied", + textLength: 2_000, + bodyChildCount: 20, + hasChallengeElement: false, + }, + }, + { + label: "a full page with an embedded CAPTCHA", + evidence: { + httpStatus: 200, + title: "Contact us", + textLength: 2_000, + bodyChildCount: 20, + hasChallengeElement: true, + }, + }, + ])("allows $label", ({ evidence }) => { + expect(detectBlockedPage(evidence)).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/capture/pageBlockDetection.ts b/packages/cli/src/capture/pageBlockDetection.ts new file mode 100644 index 0000000000..a103c081b5 --- /dev/null +++ b/packages/cli/src/capture/pageBlockDetection.ts @@ -0,0 +1,34 @@ +export interface PageLoadEvidence { + httpStatus: number | null; + title: string; + textLength: number; + bodyChildCount: number; + hasChallengeElement: boolean; +} + +/** + * Returns an actionable failure only for strong protection-page signals. + * Low text by itself is intentionally not fatal because image-led sites and + * client-rendered applications can legitimately have very little body copy. + */ +export function detectBlockedPage(evidence: PageLoadEvidence): string | undefined { + const isMinimalDom = evidence.bodyChildCount <= 5 && evidence.textLength < 500; + const hasBlockedStatus = + evidence.httpStatus === 401 || evidence.httpStatus === 403 || evidence.httpStatus === 429; + const hasBlockedTitle = + /^(?:error\s*)?(?:401|403|429)(?:\s*[-:—]\s*|\s+)|forbidden|access denied|attention required|just a moment/i.test( + evidence.title, + ); + + if (!isMinimalDom || (!evidence.hasChallengeElement && !hasBlockedStatus && !hasBlockedTitle)) { + return undefined; + } + + const statusDetail = + evidence.httpStatus === null ? "no HTTP status" : `HTTP ${evidence.httpStatus}`; + return ( + `Website capture blocked: the loaded page matched an access-protection response ` + + `(${statusDetail}, title ${JSON.stringify(evidence.title)}, ${evidence.textLength} text chars). ` + + "The site may reject automated or data-center traffic; retry from an allowed network or provide source assets directly." + ); +} From dfc60797a2adeddef120f4c800f06b30cc155117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 31 Jul 2026 18:56:06 +0000 Subject: [PATCH 2/5] fix(cli): report bounded vision failures --- .../cli/src/capture/contentExtractor.test.ts | 154 +++++++++++++++++- packages/cli/src/capture/contentExtractor.ts | 113 ++++++++----- packages/cli/src/capture/index.ts | 1 + packages/cli/src/capture/types.ts | 2 +- 4 files changed, 218 insertions(+), 52 deletions(-) diff --git a/packages/cli/src/capture/contentExtractor.test.ts b/packages/cli/src/capture/contentExtractor.test.ts index fa1e077611..d392305048 100644 --- a/packages/cli/src/capture/contentExtractor.test.ts +++ b/packages/cli/src/capture/contentExtractor.test.ts @@ -86,17 +86,35 @@ describe("captionImagesWithGemini — OpenRouter provider", () => { vi.stubGlobal( "fetch", vi.fn( - async () => new Response("invalid api key", { status: 401, statusText: "Unauthorized" }), + async () => + new Response("provider detail containing or-bad-key", { + status: 401, + statusText: "Unauthorized", + }), ), ); const warnings: string[] = []; - // captionOne throws on !res.ok, but the throw is per-image inside - // Promise.allSettled, so it's filtered out as a rejected result rather than - // bubbling up — same silent degradation as the existing Gemini path. - const captions = await captionImagesWithGemini(dir, () => {}, warnings); + let outcome: VisionCaptionOutcome | undefined; + const captions = await captionImagesWithGemini(dir, () => {}, warnings, { + onOutcome: (value) => { + outcome = value; + }, + }); expect(captions).toEqual({}); + expect(warnings).toEqual(["OpenRouter vision failed for 1 asset(s); captions omitted."]); + expect(warnings.join(" ")).not.toContain("or-bad-key"); + expect(outcome).toEqual({ + timedOutRequests: 0, + failedRequests: 1, + budgetExhausted: false, + }); + if (!outcome) throw new Error("Expected vision caption outcome"); + expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({ + status: "degraded", + reason: "provider-error", + }); }); it("terminates captioning when the vision provider never responds", async () => { @@ -125,6 +143,94 @@ describe("captionImagesWithGemini — OpenRouter provider", () => { expect(result).toEqual({}); }); + it.each([ + { label: "success JSON", status: 200, partialBody: '{"choices":[' }, + { label: "failure text", status: 500, partialBody: "upstream failed: " }, + ])( + "terminates captioning when OpenRouter $label body never ends", + async ({ status, partialBody }) => { + const dir = makeProjectWithImages(); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", "or-stalled-body-key"); + vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20"); + + vi.stubGlobal( + "fetch", + vi.fn(async () => { + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(partialBody)); + }, + }); + return new Response(body, { + status, + headers: { "content-type": "application/json" }, + }); + }), + ); + + const warnings: string[] = []; + let outcome: VisionCaptionOutcome | undefined; + const result = await Promise.race([ + captionImagesWithGemini(dir, () => {}, warnings, { + onOutcome: (value) => { + outcome = value; + }, + }), + new Promise<"hung">((resolve) => setTimeout(() => resolve("hung"), 250)), + ]); + + expect(result).toEqual({}); + expect(warnings).toEqual(["OpenRouter vision timed out for 1 asset(s); captions omitted."]); + expect(outcome).toEqual({ + timedOutRequests: 1, + failedRequests: 0, + budgetExhausted: false, + }); + }, + ); + + it("keeps successful captions while reporting a failed sibling request", async () => { + const dir = makeProjectWithImages(["failed.png", "success.png"]); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", "or-mixed-failure-key"); + + let requestCount = 0; + vi.stubGlobal( + "fetch", + vi.fn(async () => { + requestCount++; + if (requestCount === 1) { + return new Response("upstream exploded with or-mixed-failure-key", { + status: 500, + statusText: "Internal Server Error", + }); + } + return new Response( + JSON.stringify({ choices: [{ message: { content: "Successful image." } }] }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }), + ); + + const warnings: string[] = []; + let outcome: VisionCaptionOutcome | undefined; + const captions = await captionImagesWithGemini(dir, () => {}, warnings, { + onOutcome: (value) => { + outcome = value; + }, + }); + + expect(captions).toEqual({ "success.png": "Successful image." }); + expect(warnings).toEqual(["OpenRouter vision failed for 1 asset(s); captions omitted."]); + expect(warnings.join(" ")).not.toContain("or-mixed-failure-key"); + expect(outcome).toEqual({ + timedOutRequests: 0, + failedRequests: 1, + budgetExhausted: false, + }); + }); + it("keeps a successful sibling caption when another request times out", async () => { const dir = makeProjectWithImages(["hang.png", "success.png"]); dirs.push(dir); @@ -163,7 +269,11 @@ describe("captionImagesWithGemini — OpenRouter provider", () => { }); expect(captions).toEqual({ "success.png": "Successful image." }); - expect(outcome).toEqual({ timedOutRequests: 1, budgetExhausted: false }); + expect(outcome).toEqual({ + timedOutRequests: 1, + failedRequests: 0, + budgetExhausted: false, + }); if (!outcome) throw new Error("Expected vision caption outcome"); expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({ status: "degraded", @@ -242,4 +352,36 @@ describe("captionImagesWithGemini — Gemini provider", () => { expect(result).toEqual({}); }); + + it("reports a rejected Gemini request without leaking its error detail", async () => { + const dir = makeProjectWithImages(); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", ""); + vi.stubEnv("GEMINI_API_KEY", "gemini-secret-key"); + generateContentMock.mockRejectedValue( + new Error("provider echoed gemini-secret-key in its response"), + ); + + const warnings: string[] = []; + let outcome: VisionCaptionOutcome | undefined; + const captions = await captionImagesWithGemini(dir, () => {}, warnings, { + onOutcome: (value) => { + outcome = value; + }, + }); + + expect(captions).toEqual({}); + expect(warnings).toEqual(["Gemini vision failed for 1 asset(s); captions omitted."]); + expect(warnings.join(" ")).not.toContain("gemini-secret-key"); + expect(outcome).toEqual({ + timedOutRequests: 0, + failedRequests: 1, + budgetExhausted: false, + }); + if (!outcome) throw new Error("Expected vision caption outcome"); + expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({ + status: "degraded", + reason: "provider-error", + }); + }); }); diff --git a/packages/cli/src/capture/contentExtractor.ts b/packages/cli/src/capture/contentExtractor.ts index c5a3f2eb61..7b9a909a61 100644 --- a/packages/cli/src/capture/contentExtractor.ts +++ b/packages/cli/src/capture/contentExtractor.ts @@ -19,6 +19,7 @@ const DEFAULT_VISION_REQUEST_TIMEOUT_MS = 30_000; export interface VisionCaptionOutcome { timedOutRequests: number; + failedRequests: number; budgetExhausted: boolean; } @@ -33,13 +34,19 @@ export function resolveVisionPhaseCompletion( remainingMs: number, ): | { status: "completed" } - | { status: "degraded"; reason: "budget-exhausted" | "request-timeout" } { + | { + status: "degraded"; + reason: "budget-exhausted" | "request-timeout" | "provider-error"; + } { if (outcome.budgetExhausted || remainingMs <= 0) { return { status: "degraded", reason: "budget-exhausted" }; } if (outcome.timedOutRequests > 0) { return { status: "degraded", reason: "request-timeout" }; } + if (outcome.failedRequests > 0) { + return { status: "degraded", reason: "provider-error" }; + } return { status: "completed" }; } @@ -236,9 +243,14 @@ export async function captionImagesWithGemini( ): Promise> { const geminiCaptions: Record = {}; let timedOutCount = 0; + let failedRequestCount = 0; let budgetExhausted = false; const reportOutcome = (): void => { - options.onOutcome?.({ timedOutRequests: timedOutCount, budgetExhausted }); + options.onOutcome?.({ + timedOutRequests: timedOutCount, + failedRequests: failedRequestCount, + budgetExhausted, + }); }; if (options.skipVision) { reportOutcome(); @@ -279,42 +291,40 @@ export async function captionImagesWithGemini( let captionOne: CaptionOne; if (openRouterKey) { captionOne = async ({ mimeType, base64, prompt, maxTokens, timeoutMs }) => { - const res = await runBoundedVisionRequest( - (signal) => - fetch("https://openrouter.ai/api/v1/chat/completions", { - method: "POST", - headers: { - Authorization: `Bearer ${openRouterKey}`, - "Content-Type": "application/json", - }, - signal, - body: JSON.stringify({ - model, - messages: [ - { - role: "user", - content: [ - { type: "text", text: prompt }, - { - type: "image_url", - image_url: { url: `data:${mimeType};base64,${base64}` }, - }, - ], - }, - ], - max_tokens: maxTokens, - }), + return runBoundedVisionRequest(async (signal) => { + const res = await fetch("https://openrouter.ai/api/v1/chat/completions", { + method: "POST", + headers: { + Authorization: `Bearer ${openRouterKey}`, + "Content-Type": "application/json", + }, + signal, + body: JSON.stringify({ + model, + messages: [ + { + role: "user", + content: [ + { type: "text", text: prompt }, + { + type: "image_url", + image_url: { url: `data:${mimeType};base64,${base64}` }, + }, + ], + }, + ], + max_tokens: maxTokens, }), - timeoutMs, - ); - if (!res.ok) { - const detail = await res.text().catch(() => ""); - throw new Error(`OpenRouter ${res.status} ${res.statusText}: ${detail.slice(0, 200)}`); - } - const data = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }>; - }; - return data.choices?.[0]?.message?.content?.trim() || ""; + }); + if (!res.ok) { + await res.text(); + throw new Error(`OpenRouter request failed with HTTP ${res.status}`); + } + const data = (await res.json()) as { + choices?: Array<{ message?: { content?: string } }>; + }; + return data.choices?.[0]?.message?.content?.trim() || ""; + }, timeoutMs); }; } else { // Unreachable when geminiKey is unset (guarded above); re-narrow for TS. @@ -350,13 +360,14 @@ export async function captionImagesWithGemini( // Caption in parallel batches. Gemini free tier is ~5 RPM (slow but $0), // paid/OpenRouter ~2000 RPM. We batch 20 with a 2s inter-batch pause and rely - // on Promise.allSettled so a rate-limited image degrades to "" rather than - // failing the batch. + // on Promise.allSettled so one rate-limited image does not fail its siblings; + // rejected requests are aggregated into a sanitized warning below. const BATCH_SIZE = 20; const collectCaptionResults = ( results: PromiseSettledResult<{ file: string; caption: string }>[], - ): number => { + ): { timeouts: number; failures: number } => { let timeouts = 0; + let failures = 0; for (const result of results) { if (result.status === "fulfilled" && result.value.caption) { geminiCaptions[result.value.file] = result.value.caption; @@ -365,9 +376,11 @@ export async function captionImagesWithGemini( result.reason instanceof VisionRequestTimeoutError ) { timeouts++; + } else if (result.status === "rejected") { + failures++; } } - return timeouts; + return { timeouts, failures }; }; for (let i = 0; i < imageFiles.length; i += BATCH_SIZE) { const remainingMs = options.remainingMs?.() ?? Number.POSITIVE_INFINITY; @@ -397,7 +410,9 @@ export async function captionImagesWithGemini( return { file, caption }; }), ); - timedOutCount += collectCaptionResults(results); + const counts = collectCaptionResults(results); + timedOutCount += counts.timeouts; + failedRequestCount += counts.failures; // Pace requests between batches (paid tier: 2000+ RPM, free tier: rate-limited) if (i + BATCH_SIZE < imageFiles.length) { await new Promise((r) => setTimeout(r, 2000)); // 2s pause between batches — paid tier handles 2000 RPM, free tier retries via Promise.allSettled @@ -501,7 +516,9 @@ export async function captionImagesWithGemini( return { file: relPath, caption }; }), ); - timedOutCount += collectCaptionResults(results); + const counts = collectCaptionResults(results); + timedOutCount += counts.timeouts; + failedRequestCount += counts.failures; if (i + SVG_BATCH < svgFiles.length) { await new Promise((r) => setTimeout(r, 2000)); } @@ -523,8 +540,14 @@ export async function captionImagesWithGemini( `${providerName} vision timed out for ${timedOutCount} asset(s); captions omitted.`, ); } - } catch (err) { - warnings.push(`${providerName} captioning failed: ${err}`); + if (failedRequestCount > 0) { + warnings.push( + `${providerName} vision failed for ${failedRequestCount} asset(s); captions omitted.`, + ); + } + } catch { + failedRequestCount = Math.max(1, failedRequestCount); + warnings.push(`${providerName} captioning failed; captions omitted.`); } reportOutcome(); diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts index fa5627de83..15d8e3d4ae 100644 --- a/packages/cli/src/capture/index.ts +++ b/packages/cli/src/capture/index.ts @@ -684,6 +684,7 @@ export async function captureWebsite( phase("vision", "started"); let visionOutcome: VisionCaptionOutcome = { timedOutRequests: 0, + failedRequests: 0, budgetExhausted: false, }; geminiCaptions = await captionImagesWithGemini(outputDir, progress, warnings, { diff --git a/packages/cli/src/capture/types.ts b/packages/cli/src/capture/types.ts index 2ecacf390b..9efda669d6 100644 --- a/packages/cli/src/capture/types.ts +++ b/packages/cli/src/capture/types.ts @@ -26,7 +26,7 @@ export interface CapturePhaseProgress { status: "started" | "completed" | "degraded"; /** Null before the post-navigation budget begins. */ remainingMs: number | null; - reason?: "budget-exhausted" | "disabled" | "request-timeout"; + reason?: "budget-exhausted" | "disabled" | "request-timeout" | "provider-error"; } export interface CaptureOptions { From b38e90740407ac50fd83e124b8211d3efc7282be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 31 Jul 2026 18:56:17 +0000 Subject: [PATCH 3/5] fix(cli): narrow blocked page titles --- .../src/capture/pageBlockDetection.test.ts | 20 +++++++++++++++++++ .../cli/src/capture/pageBlockDetection.ts | 4 ++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/capture/pageBlockDetection.test.ts b/packages/cli/src/capture/pageBlockDetection.test.ts index b180e55fb1..0c2cd9f0d6 100644 --- a/packages/cli/src/capture/pageBlockDetection.test.ts +++ b/packages/cli/src/capture/pageBlockDetection.test.ts @@ -70,6 +70,26 @@ describe("detectBlockedPage", () => { hasChallengeElement: true, }, }, + { + label: "a minimal photography page named Forbidden Fruit Photography", + evidence: { + httpStatus: 200, + title: "Forbidden Fruit Photography", + textLength: 40, + bodyChildCount: 4, + hasChallengeElement: false, + }, + }, + { + label: "a minimal photo essay named Attention Required", + evidence: { + httpStatus: 200, + title: "Attention Required: A Photo Essay", + textLength: 60, + bodyChildCount: 5, + hasChallengeElement: false, + }, + }, ])("allows $label", ({ evidence }) => { expect(detectBlockedPage(evidence)).toBeUndefined(); }); diff --git a/packages/cli/src/capture/pageBlockDetection.ts b/packages/cli/src/capture/pageBlockDetection.ts index a103c081b5..435a9c5d3d 100644 --- a/packages/cli/src/capture/pageBlockDetection.ts +++ b/packages/cli/src/capture/pageBlockDetection.ts @@ -16,8 +16,8 @@ export function detectBlockedPage(evidence: PageLoadEvidence): string | undefine const hasBlockedStatus = evidence.httpStatus === 401 || evidence.httpStatus === 403 || evidence.httpStatus === 429; const hasBlockedTitle = - /^(?:error\s*)?(?:401|403|429)(?:\s*[-:—]\s*|\s+)|forbidden|access denied|attention required|just a moment/i.test( - evidence.title, + /^(?:(?:error\s*)?(?:401|403|429)(?:\s*(?:[-:—]\s*)?(?:forbidden|unauthorized|access denied|too many requests))?|forbidden|access denied|attention required|just a moment(?:\.{3})?)(?:\s*[|—-]\s*(?:cloudflare|sucuri website firewall))?$/i.test( + evidence.title.trim(), ); if (!isMinimalDom || (!evidence.hasChallengeElement && !hasBlockedStatus && !hasBlockedTitle)) { From ac9458888c7a6e05aa6b4d5d74063b5859782d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 31 Jul 2026 19:03:28 +0000 Subject: [PATCH 4/5] fix(cli): omit skipped Lottie previews --- packages/cli/src/capture/mediaCapture.test.ts | 51 ++++++++++++++++++- packages/cli/src/capture/mediaCapture.ts | 8 +-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/capture/mediaCapture.test.ts b/packages/cli/src/capture/mediaCapture.test.ts index 10aabdf121..f87a379ee1 100644 --- a/packages/cli/src/capture/mediaCapture.test.ts +++ b/packages/cli/src/capture/mediaCapture.test.ts @@ -1,4 +1,4 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Browser, Page } from "puppeteer-core"; @@ -75,6 +75,55 @@ describe("Lottie capture budget", () => { expect(previewPage.setViewport).not.toHaveBeenCalled(); expect(previewPage.screenshot).not.toHaveBeenCalled(); }); + + it("omits a preview path when the budget expires after Lottie readiness", async () => { + const dir = tempDir(); + const lottieDir = join(dir, "assets", "lottie"); + mkdirSync(join(dir, "extracted"), { recursive: true }); + mkdirSync(lottieDir, { recursive: true }); + writeFileSync( + join(lottieDir, "logo.json"), + JSON.stringify({ + nm: "Logo", + w: 100, + h: 100, + fr: 30, + ip: 0, + op: 30, + layers: [], + }), + ); + + const screenshot = vi.fn(async () => undefined); + const previewPage = { + setViewport: vi.fn(async () => undefined), + setContent: vi.fn(async () => undefined), + evaluate: vi.fn(async () => undefined), + waitForFunction: vi.fn(async () => undefined), + screenshot, + close: vi.fn(async () => undefined), + }; + const browser = { newPage: vi.fn(async () => previewPage) } as unknown as Browser; + let budgetChecks = 0; + + await renderLottiePreviews(browser, lottieDir, dir, { + remainingMs: () => (++budgetChecks < 4 ? 10_000 : 0), + }); + + const manifest = JSON.parse( + readFileSync(join(dir, "extracted", "lottie-manifest.json"), "utf-8"), + ); + expect(screenshot).not.toHaveBeenCalled(); + expect(manifest).toHaveLength(1); + expect(manifest[0]).toMatchObject({ + file: "assets/lottie/logo.json", + name: "Logo", + width: 100, + height: 100, + }); + expect(manifest[0]).not.toHaveProperty("preview"); + expect(existsSync(join(lottieDir, "previews", "logo-preview.png"))).toBe(false); + }); }); describe("video capture live budget", () => { diff --git a/packages/cli/src/capture/mediaCapture.ts b/packages/cli/src/capture/mediaCapture.ts index e5add76747..8e149999d1 100644 --- a/packages/cli/src/capture/mediaCapture.ts +++ b/packages/cli/src/capture/mediaCapture.ts @@ -122,7 +122,7 @@ export async function saveLottieAnimations( * * Opens each Lottie JSON in a headless Chrome page via lottie-web, * seeks to ~30% through the animation, and takes a transparent screenshot. - * Writes a lottie-manifest.json with metadata + preview paths. + * Writes a lottie-manifest.json with metadata and successfully rendered preview paths. */ // fallow-ignore-next-line complexity export async function renderLottiePreviews( @@ -133,7 +133,7 @@ export async function renderLottiePreviews( ): Promise { const manifest: Array<{ file: string; - preview: string; + preview?: string; name: string; width: number; height: number; @@ -152,6 +152,7 @@ export async function renderLottiePreviews( const fr = raw.fr || 30; const dur = ((raw.op || 0) - (raw.ip || 0)) / fr; const previewName = file.replace(".json", "-preview.png"); + let preview: string | undefined; // Render a mid-frame thumbnail using Puppeteer + lottie-web // Skip huge Lottie files for preview (CDP has a ~256MB message limit) @@ -202,6 +203,7 @@ export async function renderLottiePreviews( type: "png", omitBackground: true, }); + preview = `assets/lottie/previews/${previewName}`; } } catch { /* preview rendering failed — non-critical */ @@ -211,7 +213,7 @@ export async function renderLottiePreviews( manifest.push({ file: `assets/lottie/${file}`, - preview: `assets/lottie/previews/${previewName}`, + ...(preview ? { preview } : {}), name: raw.nm || file, width: raw.w || 0, height: raw.h || 0, From 9ae00072614f35785e4a1d7bad48787451462a91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 31 Jul 2026 20:25:02 +0000 Subject: [PATCH 5/5] fix(cli): preserve capture failure diagnostics --- .../cli/src/capture/contentExtractor.test.ts | 29 +++++++++++++++++++ packages/cli/src/capture/contentExtractor.ts | 18 ++++++++---- packages/cli/src/capture/index.ts | 5 +++- .../src/capture/pageBlockDetection.test.ts | 10 +++++++ .../cli/src/capture/pageBlockDetection.ts | 2 +- packages/cli/src/capture/types.ts | 8 ++++- 6 files changed, 64 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/capture/contentExtractor.test.ts b/packages/cli/src/capture/contentExtractor.test.ts index d392305048..5117f73d2e 100644 --- a/packages/cli/src/capture/contentExtractor.test.ts +++ b/packages/cli/src/capture/contentExtractor.test.ts @@ -309,6 +309,35 @@ describe("captionImagesWithGemini — OpenRouter provider", () => { expect(fetchMock).not.toHaveBeenCalled(); }); + it("classifies non-provider pipeline failures without leaking the local path", async () => { + const dir = mkdtempSync(join(tmpdir(), "hf-caption-no-assets-")); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", "or-unused-key"); + + const warnings: string[] = []; + let outcome: VisionCaptionOutcome | undefined; + const captions = await captionImagesWithGemini(dir, () => {}, warnings, { + onOutcome: (value) => { + outcome = value; + }, + }); + + expect(captions).toEqual({}); + expect(warnings).toEqual(["OpenRouter captioning failed internally; captions omitted."]); + expect(warnings.join(" ")).not.toContain(dir); + expect(outcome).toEqual({ + timedOutRequests: 0, + failedRequests: 0, + budgetExhausted: false, + internalError: true, + }); + if (!outcome) throw new Error("Expected vision caption outcome"); + expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({ + status: "degraded", + reason: "internal-error", + }); + }); + it("skips captioning entirely when no provider key is present", async () => { const dir = makeProjectWithImages(); dirs.push(dir); diff --git a/packages/cli/src/capture/contentExtractor.ts b/packages/cli/src/capture/contentExtractor.ts index 7b9a909a61..acea6b11ea 100644 --- a/packages/cli/src/capture/contentExtractor.ts +++ b/packages/cli/src/capture/contentExtractor.ts @@ -21,6 +21,8 @@ export interface VisionCaptionOutcome { timedOutRequests: number; failedRequests: number; budgetExhausted: boolean; + /** Failure outside provider request handling (filesystem, module, or programming error). */ + internalError?: boolean; } interface VisionCaptionOptions { @@ -36,11 +38,14 @@ export function resolveVisionPhaseCompletion( | { status: "completed" } | { status: "degraded"; - reason: "budget-exhausted" | "request-timeout" | "provider-error"; + reason: "budget-exhausted" | "request-timeout" | "provider-error" | "internal-error"; } { if (outcome.budgetExhausted || remainingMs <= 0) { return { status: "degraded", reason: "budget-exhausted" }; } + if (outcome.internalError) { + return { status: "degraded", reason: "internal-error" }; + } if (outcome.timedOutRequests > 0) { return { status: "degraded", reason: "request-timeout" }; } @@ -245,12 +250,15 @@ export async function captionImagesWithGemini( let timedOutCount = 0; let failedRequestCount = 0; let budgetExhausted = false; + let internalError = false; const reportOutcome = (): void => { - options.onOutcome?.({ + const outcome: VisionCaptionOutcome = { timedOutRequests: timedOutCount, failedRequests: failedRequestCount, budgetExhausted, - }); + }; + if (internalError) outcome.internalError = true; + options.onOutcome?.(outcome); }; if (options.skipVision) { reportOutcome(); @@ -546,8 +554,8 @@ export async function captionImagesWithGemini( ); } } catch { - failedRequestCount = Math.max(1, failedRequestCount); - warnings.push(`${providerName} captioning failed; captions omitted.`); + internalError = true; + warnings.push(`${providerName} captioning failed internally; captions omitted.`); } reportOutcome(); diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts index 15d8e3d4ae..b57be6e4ae 100644 --- a/packages/cli/src/capture/index.ts +++ b/packages/cli/src/capture/index.ts @@ -267,7 +267,10 @@ export async function captureWebsite( httpStatus: navigationResponse?.status() ?? null, ...pageContentCheck, }); - if (blockedReason) throw new Error(blockedReason); + if (blockedReason) { + phase("navigation", "degraded", "blocked"); + throw new Error(blockedReason); + } phase("navigation", "completed"); phase("core-extraction", "started"); diff --git a/packages/cli/src/capture/pageBlockDetection.test.ts b/packages/cli/src/capture/pageBlockDetection.test.ts index 0c2cd9f0d6..bf4eaef9a6 100644 --- a/packages/cli/src/capture/pageBlockDetection.test.ts +++ b/packages/cli/src/capture/pageBlockDetection.test.ts @@ -23,6 +23,16 @@ describe("detectBlockedPage", () => { hasChallengeElement: false, }, }, + { + label: "classic Cloudflare interstitial", + evidence: { + httpStatus: 503, + title: "Attention Required! | Cloudflare", + textLength: 120, + bodyChildCount: 3, + hasChallengeElement: false, + }, + }, ])("rejects a minimal protection page: $label", ({ evidence }) => { expect(detectBlockedPage(evidence)).toMatch(/capture blocked/i); }); diff --git a/packages/cli/src/capture/pageBlockDetection.ts b/packages/cli/src/capture/pageBlockDetection.ts index 435a9c5d3d..fc18797f23 100644 --- a/packages/cli/src/capture/pageBlockDetection.ts +++ b/packages/cli/src/capture/pageBlockDetection.ts @@ -16,7 +16,7 @@ export function detectBlockedPage(evidence: PageLoadEvidence): string | undefine const hasBlockedStatus = evidence.httpStatus === 401 || evidence.httpStatus === 403 || evidence.httpStatus === 429; const hasBlockedTitle = - /^(?:(?:error\s*)?(?:401|403|429)(?:\s*(?:[-:—]\s*)?(?:forbidden|unauthorized|access denied|too many requests))?|forbidden|access denied|attention required|just a moment(?:\.{3})?)(?:\s*[|—-]\s*(?:cloudflare|sucuri website firewall))?$/i.test( + /^(?:(?:error\s*)?(?:401|403|429)(?:\s*(?:[-:—]\s*)?(?:forbidden|unauthorized|access denied|too many requests))?|forbidden|access denied|attention required!?|just a moment(?:\.{3})?)(?:\s*[|—-]\s*(?:cloudflare|sucuri website firewall))?$/i.test( evidence.title.trim(), ); diff --git a/packages/cli/src/capture/types.ts b/packages/cli/src/capture/types.ts index 9efda669d6..f64b5f318d 100644 --- a/packages/cli/src/capture/types.ts +++ b/packages/cli/src/capture/types.ts @@ -26,7 +26,13 @@ export interface CapturePhaseProgress { status: "started" | "completed" | "degraded"; /** Null before the post-navigation budget begins. */ remainingMs: number | null; - reason?: "budget-exhausted" | "disabled" | "request-timeout" | "provider-error"; + reason?: + | "budget-exhausted" + | "disabled" + | "request-timeout" + | "provider-error" + | "internal-error" + | "blocked"; } export interface CaptureOptions {